diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/Makefile | 10 | ||||
| -rw-r--r-- | tools/buildman/README | 35 | ||||
| -rw-r--r-- | tools/buildman/builder.py | 51 | ||||
| -rw-r--r-- | tools/buildman/builderthread.py | 34 | ||||
| -rw-r--r-- | tools/buildman/cmdline.py | 8 | ||||
| -rw-r--r-- | tools/buildman/control.py | 52 | ||||
| -rw-r--r-- | tools/buildman/func_test.py | 46 | ||||
| -rw-r--r-- | tools/buildman/test.py | 20 | ||||
| -rw-r--r-- | tools/buildman/toolchain.py | 2 |
9 files changed, 177 insertions, 81 deletions
diff --git a/tools/Makefile b/tools/Makefile index 99be724b82..c2b2634004 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -58,7 +58,7 @@ hostprogs-$(CONFIG_FIT_SIGNATURE) += fit_info fit_check_sign hostprogs-$(CONFIG_CMD_BOOTEFI_SELFTEST) += file2include FIT_OBJS-$(CONFIG_FIT) := fit_common.o fit_image.o image-host.o common/image-fit.o -FIT_SIG_OBJS-$(CONFIG_FIT_SIGNATURE) := common/image-sig.o +FIT_SIG_OBJS-$(CONFIG_FIT_SIGNATURE) := common/image-sig.o common/image-fit-sig.o FIT_CIPHER_OBJS-$(CONFIG_FIT_CIPHER) := common/image-cipher.o # The following files are synced with upstream DTC. @@ -161,7 +161,7 @@ endif ifneq ($(CONFIG_MX23)$(CONFIG_MX28)$(CONFIG_ARMADA_38X)$(CONFIG_ARMADA_39X)$(CONFIG_FIT_SIGNATURE),) HOSTCFLAGS_kwbimage.o += \ $(shell pkg-config --cflags libssl libcrypto 2> /dev/null || echo "") -HOSTLOADLIBES_mkimage += \ +HOSTLDLIBS_mkimage += \ $(shell pkg-config --libs libssl libcrypto 2> /dev/null || echo "-lssl -lcrypto") # OS X deprecate openssl in favour of CommonCrypto, supress deprecation @@ -175,9 +175,9 @@ endif HOSTCFLAGS_fit_image.o += -DMKIMAGE_DTC=\"$(CONFIG_MKIMAGE_DTC_PATH)\" -HOSTLOADLIBES_dumpimage := $(HOSTLOADLIBES_mkimage) -HOSTLOADLIBES_fit_info := $(HOSTLOADLIBES_mkimage) -HOSTLOADLIBES_fit_check_sign := $(HOSTLOADLIBES_mkimage) +HOSTLDLIBS_dumpimage := $(HOSTLDLIBS_mkimage) +HOSTLDLIBS_fit_info := $(HOSTLDLIBS_mkimage) +HOSTLDLIBS_fit_check_sign := $(HOSTLDLIBS_mkimage) hostprogs-$(CONFIG_EXYNOS5250) += mkexynosspl hostprogs-$(CONFIG_EXYNOS5420) += mkexynosspl diff --git a/tools/buildman/README b/tools/buildman/README index c1ac0d0f58..4cf0114157 100644 --- a/tools/buildman/README +++ b/tools/buildman/README @@ -1056,19 +1056,46 @@ toolchain. For example: buildman -O clang-7 --board sandbox +Doing a simple build +==================== + +In some cases you just want to build a single board and get the full output, use +the -w option, for example: + + buildman -o /tmp/build --board sandbox -w + +This will write the full build into /tmp/build including object files. + + Other options ============= -Buildman has various other command line options. Try --help to see them. +Buildman has various other command-line options. Try --help to see them. + +To find out what toolchain prefix buildman will use for a build, use the -A +option. + +To request that compiler warnings be promoted to errors, use -E. This passes the +-Werror flag to the compiler. Note that the build can still produce warnings +with -E, e.g. the migration warnings: -To find out what architecture or toolchain prefix buildman will use for a build, -see the -a and -A options. + ===================== WARNING ====================== + This board does not use CONFIG_DM_MMC. Please update + ... + ==================================================== When doing builds, Buildman's return code will reflect the overall result: 0 (success) No errors or warnings found 128 Errors found - 129 Warnings found + 129 Warnings found (only if no -W) + +You can use -W to tell Buildman to return 0 (success) instead of 129 when +warnings are found. Note that it can be useful to combine -E and -W. This means +that all compiler warnings will produce failures (code 128) and all other +warnings will produce success (since 129 is changed to 0). + +If there are both warnings and errors, errors win, so buildman returns 128. How to change from MAKEALL diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 3fd4fac695..70c55c588a 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -174,6 +174,8 @@ class Builder: in_tree: Build U-Boot in-tree instead of specifying an output directory separate from the source code. This option is really only useful for testing in-tree builds. + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. Private members: _base_board_dict: Last-summarised Dict of boards @@ -224,7 +226,7 @@ class Builder: no_subdirs=False, full_path=False, verbose_build=False, incremental=False, per_board_out_dir=False, config_only=False, squash_config_y=False, - warnings_as_errors=False): + warnings_as_errors=False, work_in_output=False): """Create a new Builder object Args: @@ -250,10 +252,15 @@ class Builder: config_only: Only configure each build, don't build it squash_config_y: Convert CONFIG options with the value 'y' to '1' warnings_as_errors: Treat all compiler warnings as errors + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. """ self.toolchains = toolchains self.base_dir = base_dir - self._working_dir = os.path.join(base_dir, '.bm-work') + if work_in_output: + self._working_dir = base_dir + else: + self._working_dir = os.path.join(base_dir, '.bm-work') self.threads = [] self.do_make = self.Make self.gnu_make = gnu_make @@ -280,6 +287,7 @@ class Builder: self.config_only = config_only self.squash_config_y = squash_config_y self.config_filenames = BASE_CONFIG_FILENAMES + self.work_in_output = work_in_output if not self.squash_config_y: self.config_filenames += EXTRA_CONFIG_FILENAMES @@ -329,7 +337,7 @@ class Builder: show_errors: True to show summarised error/warning info show_sizes: Show size deltas - show_detail: Show detail for each board + show_detail: Show size delta detail for each board if show_sizes show_bloat: Show detail for each function list_error_boards: Show the boards which caused each error/warning show_config: Show config deltas @@ -477,6 +485,7 @@ class Builder: if self.commits: commit = self.commits[commit_upto] subject = commit.subject.translate(trans_valid_chars) + # See _GetOutputSpaceRemovals() which parses this name commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, self.commit_count, commit.hash, subject[:20])) elif not self.no_subdirs: @@ -992,7 +1001,7 @@ class Builder: board.target board_dict: Dict containing boards for which we built this commit, keyed by board.target. The value is an Outcome object. - show_detail: Show detail for each board + show_detail: Show size delta detail for each board show_bloat: Show detail for each function """ arch_list = {} @@ -1109,7 +1118,7 @@ class Builder: environment: Dictionary keyed by environment variable, Each value is the value of environment variable. show_sizes: Show image size deltas - show_detail: Show detail for each board + show_detail: Show size delta detail for each board if show_sizes show_bloat: Show detail for each function show_config: Show config changes show_environment: Show environment changes @@ -1474,6 +1483,8 @@ class Builder: Args: thread_num: Number of thread to check. """ + if self.work_in_output: + return self._working_dir return os.path.join(self._working_dir, '%02d' % thread_num) def _PrepareThread(self, thread_num, setup_git): @@ -1515,12 +1526,15 @@ class Builder: for thread in range(max_threads): self._PrepareThread(thread, setup_git) - def _PrepareOutputSpace(self): + def _GetOutputSpaceRemovals(self): """Get the output directories ready to receive files. - We delete any output directories which look like ones we need to - create. Having left over directories is confusing when the user wants - to check the output manually. + Figure out what needs to be deleted in the output directory before it + can be used. We only delete old buildman directories which have the + expected name pattern. See _GetOutputDir(). + + Returns: + List of full paths of directories to remove """ if not self.commits: return @@ -1531,12 +1545,26 @@ class Builder: to_remove = [] for dirname in glob.glob(os.path.join(self.base_dir, '*')): if dirname not in dir_list: - to_remove.append(dirname) + leaf = dirname[len(self.base_dir) + 1:] + m = re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf) + if m: + to_remove.append(dirname) + return to_remove + + def _PrepareOutputSpace(self): + """Get the output directories ready to receive files. + + We delete any output directories which look like ones we need to + create. Having left over directories is confusing when the user wants + to check the output manually. + """ + to_remove = self._GetOutputSpaceRemovals() if to_remove: - Print('Removing %d old build directories' % len(to_remove), + Print('Removing %d old build directories...' % len(to_remove), newline=False) for dirname in to_remove: shutil.rmtree(dirname) + Print('done') def BuildBoards(self, commits, board_selected, keep_outputs, verbose): """Build all commits for a list of boards @@ -1571,6 +1599,7 @@ class Builder: job.board = brd job.commits = commits job.keep_outputs = keep_outputs + job.work_in_output = self.work_in_output job.step = self._step self.queue.put(job) diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 570c1f6595..7561f39942 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -39,11 +39,18 @@ class BuilderJob: Members: board: Board object to build - commits: List of commit options to build. + commits: List of Commit objects to build + keep_outputs: True to save build output files + step: 1 to process every commit, n to process every nth commit + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. """ def __init__(self): self.board = None self.commits = [] + self.keep_outputs = False + self.step = 1 + self.work_in_output = False class ResultThread(threading.Thread): @@ -114,7 +121,7 @@ class BuilderThread(threading.Thread): **kwargs) def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only, - force_build, force_build_failures): + force_build, force_build_failures, work_in_output): """Build a particular commit. If the build is already done, and we are not forcing a build, we skip @@ -129,6 +136,8 @@ class BuilderThread(threading.Thread): force_build: Force a build even if one was previously done force_build_failures: Force a bulid if the previous result showed failure + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. Returns: tuple containing: @@ -139,7 +148,7 @@ class BuilderThread(threading.Thread): # self.Make() below, in the event that we do a build. result = command.CommandResult() result.return_code = 0 - if self.builder.in_tree: + if work_in_output or self.builder.in_tree: out_dir = work_dir else: if self.per_board_out_dir: @@ -261,14 +270,18 @@ class BuilderThread(threading.Thread): result.out_dir = out_dir return result, do_config - def _WriteResult(self, result, keep_outputs): + def _WriteResult(self, result, keep_outputs, work_in_output): """Write a built result to the output directory. Args: result: CommandResult object containing result to write keep_outputs: True to store the output binaries, False to delete them + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. """ + if work_in_output: + return # Fatal error if result.return_code < 0: return @@ -430,7 +443,8 @@ class BuilderThread(threading.Thread): result, request_config = self.RunCommit(commit_upto, brd, work_dir, do_config, self.builder.config_only, force_build or self.builder.force_build, - self.builder.force_build_failures) + self.builder.force_build_failures, + work_in_output=job.work_in_output) failed = result.return_code or result.stderr did_config = do_config if failed and not do_config: @@ -438,7 +452,8 @@ class BuilderThread(threading.Thread): # with a reconfig. if self.builder.force_config_on_failure: result, request_config = self.RunCommit(commit_upto, - brd, work_dir, True, False, True, False) + brd, work_dir, True, False, True, False, + work_in_output=job.work_in_output) did_config = True if not self.builder.force_reconfig: do_config = request_config @@ -477,15 +492,16 @@ class BuilderThread(threading.Thread): raise ValueError('Interrupt') # We have the build results, so output the result - self._WriteResult(result, job.keep_outputs) + self._WriteResult(result, job.keep_outputs, job.work_in_output) self.builder.out_queue.put(result) else: # Just build the currently checked-out build result, request_config = self.RunCommit(None, brd, work_dir, True, self.builder.config_only, True, - self.builder.force_build_failures) + self.builder.force_build_failures, + work_in_output=job.work_in_output) result.commit_upto = 0 - self._WriteResult(result, job.keep_outputs) + self._WriteResult(result, job.keep_outputs, job.work_in_output) self.builder.out_queue.put(result) def run(self): diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index b41209373d..17ea015a95 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -13,8 +13,6 @@ def ParseArgs(): args: command lin arguments """ parser = OptionParser() - parser.add_option('-a', '--print-arch', action='store_true', - help='Print the architecture for a board (ARCH=)') parser.add_option('-A', '--print-prefix', action='store_true', help='Print the tool-chain prefix for a board (CROSS_COMPILE=)') parser.add_option('-b', '--branch', type='string', @@ -31,7 +29,7 @@ def ParseArgs(): help='Reconfigure for every commit (disable incremental build)') parser.add_option('-d', '--detail', dest='show_detail', action='store_true', default=False, - help='Show detailed information for each board in summary') + help='Show detailed size delta for each board in the -S summary') parser.add_option('-D', '--config-only', action='store_true', default=False, help="Don't build, just configure each commit") parser.add_option('-e', '--show_errors', action='store_true', @@ -106,6 +104,10 @@ def ParseArgs(): default=False, help='Show build results while the build progresses') parser.add_option('-V', '--verbose-build', action='store_true', default=False, help='Run make with V=1, logging all output') + parser.add_option('-w', '--work-in-output', action='store_true', + default=False, help='Use the output directory as the work directory') + parser.add_option('-W', '--ignore-warnings', action='store_true', + default=False, help='Return success even if there are warnings') parser.add_option('-x', '--exclude', dest='exclude', type='string', action='append', help='Specify a list of boards to exclude, separated by comma') diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 969d866547..5ddc598c95 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -85,38 +85,15 @@ def ShowActions(series, why_selected, boards_selected, builder, options, for warning in board_warnings: print(col.Color(col.YELLOW, warning)) -def CheckOutputDir(output_dir): - """Make sure that the output directory is not within the current directory - - If we try to use an output directory which is within the current directory - (which is assumed to hold the U-Boot source) we may end up deleting the - U-Boot source code. Detect this and print an error in this case. - - Args: - output_dir: Output directory path to check - """ - path = os.path.realpath(output_dir) - cwd_path = os.path.realpath('.') - while True: - if os.path.realpath(path) == cwd_path: - Print("Cannot use output directory '%s' since it is within the current directory '%s'" % - (path, cwd_path)) - sys.exit(1) - parent = os.path.dirname(path) - if parent == path: - break - path = parent - -def ShowToolchainInfo(boards, toolchains, print_arch, print_prefix): +def ShowToolchainPrefix(boards, toolchains): """Show information about a the tool chain used by one or more boards - The function checks that all boards use the same toolchain. + The function checks that all boards use the same toolchain, then prints + the correct value for CROSS_COMPILE. Args: boards: Boards object containing selected boards toolchains: Toolchains object containing available toolchains - print_arch: True to print ARCH value - print_prefix: True to print CROSS_COMPILE value Return: None on success, string error message otherwise @@ -129,10 +106,7 @@ def ShowToolchainInfo(boards, toolchains, print_arch, print_prefix): return 'Supplied boards must share one toolchain' return False tc = tc_set.pop() - if print_arch: - print(tc.GetEnvArgs(toolchain.VAR_ARCH)) - if print_prefix: - print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) + print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) return None def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, @@ -228,9 +202,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, if not len(selected): sys.exit(col.Color(col.RED, 'No matching boards found')) - if options.print_arch or options.print_prefix: - err = ShowToolchainInfo(boards, toolchains, options.print_arch, - options.print_prefix) + if options.print_prefix: + err = ShowToolchainInfo(boards, toolchains) if err: sys.exit(col.Color(col.RED, err)) return 0 @@ -263,6 +236,13 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, str = ("No commits found to process in branch '%s': " "set branch's upstream or use -c flag" % options.branch) sys.exit(col.Color(col.RED, str)) + if options.work_in_output: + if len(selected) != 1: + sys.exit(col.Color(col.RED, + '-w can only be used with a single board')) + if count != 1: + sys.exit(col.Color(col.RED, + '-w can only be used with a single commit')) # Read the metadata from the commits. First look at the upstream commit, # then the ones in the branch. We would like to do something like @@ -324,7 +304,6 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, output_dir = os.path.join(options.output_dir, dirname) if clean_dir and os.path.exists(output_dir): shutil.rmtree(output_dir) - CheckOutputDir(output_dir) builder = Builder(toolchains, output_dir, options.git_dir, options.threads, options.jobs, gnu_make=gnu_make, checkout=True, show_unknown=options.show_unknown, step=options.step, @@ -334,7 +313,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, per_board_out_dir=options.per_board_out_dir, config_only=options.config_only, squash_config_y=not options.preserve_config_y, - warnings_as_errors=options.warnings_as_errors) + warnings_as_errors=options.warnings_as_errors, + work_in_output=options.work_in_output) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func @@ -378,6 +358,6 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, options.keep_outputs, options.verbose) if fail: return 128 - elif warned: + elif warned and not options.ignore_warnings: return 129 return 0 diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 4c3d497294..2a256a9263 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -16,6 +16,7 @@ import control import gitutil import terminal import toolchain +import tools settings_data = ''' # Buildman settings file @@ -208,7 +209,7 @@ class TestFunctional(unittest.TestCase): def tearDown(self): shutil.rmtree(self._base_dir) - shutil.rmtree(self._output_dir) + #shutil.rmtree(self._output_dir) def setupToolchains(self): self._toolchains = toolchain.Toolchains() @@ -218,12 +219,12 @@ class TestFunctional(unittest.TestCase): return command.RunPipe([[self._buildman_pathname] + list(args)], capture=True, capture_stderr=True) - def _RunControl(self, *args, **kwargs): + def _RunControl(self, *args, clean_dir=False, boards=None): sys.argv = [sys.argv[0]] + list(args) options, args = cmdline.ParseArgs() result = control.DoBuildman(options, args, toolchains=self._toolchains, - make_func=self._HandleMake, boards=self._boards, - clean_dir=kwargs.get('clean_dir', True)) + make_func=self._HandleMake, boards=boards or self._boards, + clean_dir=clean_dir) self._builder = control.builder return result @@ -397,6 +398,12 @@ class TestFunctional(unittest.TestCase): combined='Test configuration complete') elif stage == 'build': stderr = '' + out_dir = '' + for arg in args: + if arg.startswith('O='): + out_dir = arg[2:] + fname = os.path.join(cwd or '', out_dir, 'u-boot') + tools.WriteFile(fname, b'U-Boot') if type(commit) is not str: stderr = self._error.get((brd.target, commit.sequence)) if stderr: @@ -527,11 +534,26 @@ class TestFunctional(unittest.TestCase): self.assertEqual(self._builder.count, self._total_builds) self.assertEqual(self._builder.fail, 0) - def testBadOutputDir(self): - """Test building with an output dir the same as out current dir""" - self._test_branch = '/__dev/__testbranch' - with self.assertRaises(SystemExit): - self._RunControl('-b', self._test_branch, '-o', os.getcwd()) - with self.assertRaises(SystemExit): - self._RunControl('-b', self._test_branch, '-o', - os.path.join(os.getcwd(), 'test')) + def testWorkInOutput(self): + """Test the -w option which should write directly to the output dir""" + board_list = board.Boards() + board_list.AddBoard(board.Board(*boards[0])) + self._RunControl('-o', self._output_dir, '-w', clean_dir=False, + boards=board_list) + self.assertTrue( + os.path.exists(os.path.join(self._output_dir, 'u-boot'))) + + def testWorkInOutputFail(self): + """Test the -w option failures""" + with self.assertRaises(SystemExit) as e: + self._RunControl('-o', self._output_dir, '-w', clean_dir=False) + self.assertIn("single board", str(e.exception)) + self.assertFalse( + os.path.exists(os.path.join(self._output_dir, 'u-boot'))) + + board_list = board.Boards() + board_list.AddBoard(board.Board(*boards[0])) + with self.assertRaises(SystemExit) as e: + self._RunControl('-b', self._test_branch, '-o', self._output_dir, + '-w', clean_dir=False, boards=board_list) + self.assertIn("single commit", str(e.exception)) diff --git a/tools/buildman/test.py b/tools/buildman/test.py index acd862b3b0..2aaedf44ac 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -22,6 +22,7 @@ import commit import terminal import test_util import toolchain +import tools use_network = True @@ -469,6 +470,25 @@ class TestBuild(unittest.TestCase): self.assertEqual('HOSTCC=clang CC=clang', tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS)) + def testPrepareOutputSpace(self): + def _Touch(fname): + tools.WriteFile(os.path.join(base_dir, fname), b'') + + base_dir = tempfile.mkdtemp() + + # Add various files that we want removed and left alone + to_remove = ['01_of_22_g0982734987_title', '102_of_222_g92bf_title', + '01_of_22_g2938abd8_title'] + to_leave = ['something_else', '01-something.patch', '01_of_22_another'] + for name in to_remove + to_leave: + _Touch(name) + + build = builder.Builder(self.toolchains, base_dir, None, 1, 2) + build.commits = self.commits + build.commit_count = len(commits) + result = set(build._GetOutputSpaceRemovals()) + expected = set([os.path.join(base_dir, f) for f in to_remove]) + self.assertEqual(expected, result) if __name__ == "__main__": unittest.main() diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index 89c54d688a..4456a805c7 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -490,7 +490,7 @@ class Toolchains: if arch == 'aarch64': arch = 'arm64' base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' - versions = ['7.3.0', '6.4.0', '4.9.4'] + versions = ['9.2.0', '7.3.0', '6.4.0', '4.9.4'] links = [] for version in versions: url = '%s/%s/%s/' % (base, arch, version) |