diff options
Diffstat (limited to 'tools/patman')
| -rw-r--r-- | tools/patman/checkpatch.py | 6 | ||||
| -rw-r--r-- | tools/patman/commit.py | 14 | ||||
| -rw-r--r-- | tools/patman/control.py | 178 | ||||
| -rw-r--r-- | tools/patman/func_test.py | 169 | ||||
| -rw-r--r-- | tools/patman/gitutil.py | 48 | ||||
| -rwxr-xr-x | tools/patman/main.py | 196 | ||||
| -rw-r--r-- | tools/patman/patchstream.py | 36 | ||||
| -rw-r--r-- | tools/patman/series.py | 2 | ||||
| -rw-r--r-- | tools/patman/settings.py | 10 | ||||
| -rw-r--r-- | tools/patman/terminal.py | 4 | ||||
| -rw-r--r-- | tools/patman/test_util.py | 21 | ||||
| -rw-r--r-- | tools/patman/tools.py | 12 | ||||
| -rw-r--r-- | tools/patman/tout.py | 6 |
13 files changed, 550 insertions, 152 deletions
diff --git a/tools/patman/checkpatch.py b/tools/patman/checkpatch.py index 07c3e2739a..263bac3fc9 100644 --- a/tools/patman/checkpatch.py +++ b/tools/patman/checkpatch.py @@ -41,6 +41,12 @@ def FindCheckPatch(): def CheckPatch(fname, verbose=False, show_types=False): """Run checkpatch.pl on a file. + Args: + fname: Filename to check + verbose: True to print out every line of the checkpatch output as it is + parsed + show_types: Tell checkpatch to show the type (number) of each message + Returns: namedtuple containing: ok: False=failure, True=ok diff --git a/tools/patman/commit.py b/tools/patman/commit.py index 48d0529c53..8d583c4ed3 100644 --- a/tools/patman/commit.py +++ b/tools/patman/commit.py @@ -2,6 +2,7 @@ # Copyright (c) 2011 The Chromium OS Authors. # +import collections import re # Separates a tag: at the beginning of the subject from the rest of it @@ -23,6 +24,9 @@ class Commit: notes: List of lines in the commit (not series) notes change_id: the Change-Id: tag that was stripped from this commit and can be used to generate the Message-Id. + rtags: Response tags (e.g. Reviewed-by) collected by the commit, dict: + key: rtag type (e.g. 'Reviewed-by') + value: Set of people who gave that rtag, each a name/email string """ def __init__(self, hash): self.hash = hash @@ -33,6 +37,7 @@ class Commit: self.signoff_set = set() self.notes = [] self.change_id = None + self.rtags = collections.defaultdict(set) def AddChange(self, version, info): """Add a new change line to the change list for a version. @@ -88,3 +93,12 @@ class Commit: return False self.signoff_set.add(signoff) return True + + def AddRtag(self, rtag_type, who): + """Add a response tag to a commit + + Args: + key: rtag type (e.g. 'Reviewed-by') + who: Person who gave that rtag, e.g. 'Fred Bloggs <fred@bloggs.org>' + """ + self.rtags[rtag_type].add(who) diff --git a/tools/patman/control.py b/tools/patman/control.py new file mode 100644 index 0000000000..67e8f397ef --- /dev/null +++ b/tools/patman/control.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright 2020 Google LLC +# +"""Handles the main control logic of patman + +This module provides various functions called by the main program to implement +the features of patman. +""" + +import os +import sys + +from patman import checkpatch +from patman import gitutil +from patman import patchstream +from patman import terminal + +def setup(): + """Do required setup before doing anything""" + gitutil.Setup() + +def prepare_patches(col, branch, count, start, end, ignore_binary): + """Figure out what patches to generate, then generate them + + The patch files are written to the current directory, e.g. 0001_xxx.patch + 0002_yyy.patch + + Args: + col (terminal.Color): Colour output object + branch (str): Branch to create patches from (None = current) + count (int): Number of patches to produce, or -1 to produce patches for + the current branch back to the upstream commit + start (int): Start partch to use (0=first / top of branch) + end (int): End patch to use (0=last one in series, 1=one before that, + etc.) + ignore_binary (bool): Don't generate patches for binary files + + Returns: + Tuple: + Series object for this series (set of patches) + Filename of the cover letter as a string (None if none) + patch_files: List of patch filenames, each a string, e.g. + ['0001_xxx.patch', '0002_yyy.patch'] + """ + if count == -1: + # Work out how many patches to send if we can + count = (gitutil.CountCommitsToBranch(branch) - start) + + if not count: + str = 'No commits found to process - please use -c flag, or run:\n' \ + ' git branch --set-upstream-to remote/branch' + sys.exit(col.Color(col.RED, str)) + + # Read the metadata from the commits + to_do = count - end + series = patchstream.GetMetaData(branch, start, to_do) + cover_fname, patch_files = gitutil.CreatePatches( + branch, start, to_do, ignore_binary, series) + + # Fix up the patch files to our liking, and insert the cover letter + patchstream.FixPatches(series, patch_files) + if cover_fname and series.get('cover'): + patchstream.InsertCoverLetter(cover_fname, series, to_do) + return series, cover_fname, patch_files + +def check_patches(series, patch_files, run_checkpatch, verbose): + """Run some checks on a set of patches + + This santiy-checks the patman tags like Series-version and runs the patches + through checkpatch + + Args: + series (Series): Series object for this series (set of patches) + patch_files (list): List of patch filenames, each a string, e.g. + ['0001_xxx.patch', '0002_yyy.patch'] + run_checkpatch (bool): True to run checkpatch.pl + verbose (bool): True to print out every line of the checkpatch output as + it is parsed + + Returns: + bool: True if the patches had no errors, False if they did + """ + # Do a few checks on the series + series.DoChecks() + + # Check the patches, and run them through 'git am' just to be sure + if run_checkpatch: + ok = checkpatch.CheckPatches(verbose, patch_files) + else: + ok = True + return ok + + +def email_patches(col, series, cover_fname, patch_files, process_tags, its_a_go, + ignore_bad_tags, add_maintainers, limit, dry_run, in_reply_to, + thread, smtp_server): + """Email patches to the recipients + + This emails out the patches and cover letter using 'git send-email'. Each + patch is copied to recipients identified by the patch tag and output from + the get_maintainer.pl script. The cover letter is copied to all recipients + of any patch. + + To make this work a CC file is created holding the recipients for each patch + and the cover letter. See the main program 'cc_cmd' for this logic. + + Args: + col (terminal.Color): Colour output object + series (Series): Series object for this series (set of patches) + cover_fname (str): Filename of the cover letter as a string (None if + none) + patch_files (list): List of patch filenames, each a string, e.g. + ['0001_xxx.patch', '0002_yyy.patch'] + process_tags (bool): True to process subject tags in each patch, e.g. + for 'dm: spi: Add SPI support' this would be 'dm' and 'spi'. The + tags are looked up in the configured sendemail.aliasesfile and also + in ~/.patman (see README) + its_a_go (bool): True if we are going to actually send the patches, + False if the patches have errors and will not be sent unless + @ignore_errors + ignore_bad_tags (bool): True to just print a warning for unknown tags, + False to halt with an error + add_maintainers (bool): Run the get_maintainer.pl script for each patch + limit (int): Limit on the number of people that can be cc'd on a single + patch or the cover letter (None if no limit) + dry_run (bool): Don't actually email the patches, just print out what + would be sent + in_reply_to (str): If not None we'll pass this to git as --in-reply-to. + Should be a message ID that this is in reply to. + thread (bool): True to add --thread to git send-email (make all patches + reply to cover-letter or first patch in series) + smtp_server (str): SMTP server to use to send patches (None for default) + """ + cc_file = series.MakeCcFile(process_tags, cover_fname, not ignore_bad_tags, + add_maintainers, limit) + + # Email the patches out (giving the user time to check / cancel) + cmd = '' + if its_a_go: + cmd = gitutil.EmailPatches( + series, cover_fname, patch_files, dry_run, not ignore_bad_tags, + cc_file, in_reply_to=in_reply_to, thread=thread, + smtp_server=smtp_server) + else: + print(col.Color(col.RED, "Not sending emails due to errors/warnings")) + + # For a dry run, just show our actions as a sanity check + if dry_run: + series.ShowActions(patch_files, cmd, process_tags) + if not its_a_go: + print(col.Color(col.RED, "Email would not be sent")) + + os.remove(cc_file) + +def send(args): + """Create, check and send patches by email + + Args: + args (argparse.Namespace): Arguments to patman + """ + setup() + col = terminal.Color() + series, cover_fname, patch_files = prepare_patches( + col, args.branch, args.count, args.start, args.end, + args.ignore_binary) + ok = check_patches(series, patch_files, args.check_patch, + args.verbose) + + ok = ok and gitutil.CheckSuppressCCConfig() + + its_a_go = ok or args.ignore_errors + if its_a_go: + email_patches( + col, series, cover_fname, patch_files, args.process_tags, + its_a_go, args.ignore_bad_tags, args.add_maintainers, + args.limit, args.dry_run, args.in_reply_to, args.thread, + args.smtp_server) diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index dc30078cce..810af9c604 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -14,15 +14,23 @@ import unittest from io import StringIO +from patman import control from patman import gitutil from patman import patchstream from patman import settings +from patman import terminal from patman import tools +from patman.test_util import capture_sys_output + +try: + import pygit2 + HAVE_PYGIT2= True +except ModuleNotFoundError: + HAVE_PYGIT2 = False @contextlib.contextmanager def capture(): - import sys oldout,olderr = sys.stdout, sys.stderr try: out=[StringIO(), StringIO()] @@ -37,6 +45,8 @@ def capture(): class TestFunctional(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp(prefix='patman.') + self.gitdir = os.path.join(self.tmpdir, 'git') + self.repo = None def tearDown(self): shutil.rmtree(self.tmpdir) @@ -286,3 +296,160 @@ Changes in v2: if expected: expected = expected.splitlines() self.assertEqual(expected, lines[start:(start+len(expected))]) + + def make_commit_with_file(self, subject, body, fname, text): + """Create a file and add it to the git repo with a new commit + + Args: + subject (str): Subject for the commit + body (str): Body text of the commit + fname (str): Filename of file to create + text (str): Text to put into the file + """ + path = os.path.join(self.gitdir, fname) + tools.WriteFile(path, text, binary=False) + index = self.repo.index + index.add(fname) + author = pygit2.Signature('Test user', 'test@email.com') + committer = author + tree = index.write_tree() + message = subject + '\n' + body + self.repo.create_commit('HEAD', author, committer, message, tree, + [self.repo.head.target]) + + def make_git_tree(self): + """Make a simple git tree suitable for testing + + It has three branches: + 'base' has two commits: PCI, main + 'first' has base as upstream and two more commits: I2C, SPI + 'second' has base as upstream and three more: video, serial, bootm + + Returns: + pygit2 repository + """ + repo = pygit2.init_repository(self.gitdir) + self.repo = repo + new_tree = repo.TreeBuilder().write() + + author = pygit2.Signature('Test user', 'test@email.com') + committer = author + commit = repo.create_commit('HEAD', author, committer, + 'Created master', new_tree, []) + + self.make_commit_with_file('Initial commit', ''' +Add a README + +''', 'README', '''This is the README file +describing this project +in very little detail''') + + self.make_commit_with_file('pci: PCI implementation', ''' +Here is a basic PCI implementation + +''', 'pci.c', '''This is a file +it has some contents +and some more things''') + self.make_commit_with_file('main: Main program', ''' +Hello here is the second commit. +''', 'main.c', '''This is the main file +there is very little here +but we can always add more later +if we want to + +Series-to: u-boot +Series-cc: Barry Crump <bcrump@whataroa.nz> +''') + base_target = repo.revparse_single('HEAD') + self.make_commit_with_file('i2c: I2C things', ''' +This has some stuff to do with I2C +''', 'i2c.c', '''And this is the file contents +with some I2C-related things in it''') + self.make_commit_with_file('spi: SPI fixes', ''' +SPI needs some fixes +and here they are +''', 'spi.c', '''Some fixes for SPI in this +file to make SPI work +better than before''') + first_target = repo.revparse_single('HEAD') + + target = repo.revparse_single('HEAD~2') + repo.reset(target.oid, pygit2.GIT_CHECKOUT_FORCE) + self.make_commit_with_file('video: Some video improvements', ''' +Fix up the video so that +it looks more purple. Purple is +a very nice colour. +''', 'video.c', '''More purple here +Purple and purple +Even more purple +Could not be any more purple''') + self.make_commit_with_file('serial: Add a serial driver', ''' +Here is the serial driver +for my chip. + +Cover-letter: +Series for my board +This series implements support +for my glorious board. +END +''', 'serial.c', '''The code for the +serial driver is here''') + self.make_commit_with_file('bootm: Make it boot', ''' +This makes my board boot +with a fix to the bootm +command +''', 'bootm.c', '''Fix up the bootm +command to make the code as +complicated as possible''') + second_target = repo.revparse_single('HEAD') + + repo.branches.local.create('first', first_target) + repo.config.set_multivar('branch.first.remote', '', '.') + repo.config.set_multivar('branch.first.merge', '', 'refs/heads/base') + + repo.branches.local.create('second', second_target) + repo.config.set_multivar('branch.second.remote', '', '.') + repo.config.set_multivar('branch.second.merge', '', 'refs/heads/base') + + repo.branches.local.create('base', base_target) + return repo + + @unittest.skipIf(not HAVE_PYGIT2, 'Missing python3-pygit2') + def testBranch(self): + """Test creating patches from a branch""" + repo = self.make_git_tree() + target = repo.lookup_reference('refs/heads/first') + self.repo.checkout(target, strategy=pygit2.GIT_CHECKOUT_FORCE) + control.setup() + try: + orig_dir = os.getcwd() + os.chdir(self.gitdir) + + # Check that it can detect the current branch + self.assertEqual(2, gitutil.CountCommitsToBranch(None)) + col = terminal.Color() + with capture_sys_output() as _: + _, cover_fname, patch_files = control.prepare_patches( + col, branch=None, count=-1, start=0, end=0, + ignore_binary=False) + self.assertIsNone(cover_fname) + self.assertEqual(2, len(patch_files)) + + # Check that it can detect a different branch + self.assertEqual(3, gitutil.CountCommitsToBranch('second')) + with capture_sys_output() as _: + _, cover_fname, patch_files = control.prepare_patches( + col, branch='second', count=-1, start=0, end=0, + ignore_binary=False) + self.assertIsNotNone(cover_fname) + self.assertEqual(3, len(patch_files)) + + # Check that it can skip patches at the end + with capture_sys_output() as _: + _, cover_fname, patch_files = control.prepare_patches( + col, branch='second', count=-1, start=0, end=1, + ignore_binary=False) + self.assertIsNotNone(cover_fname) + self.assertEqual(2, len(patch_files)) + finally: + os.chdir(orig_dir) diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index 5189840eab..192d8e69b3 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -49,17 +49,24 @@ def LogCmd(commit_range, git_dir=None, oneline=False, reverse=False, cmd.append('--') return cmd -def CountCommitsToBranch(): +def CountCommitsToBranch(branch): """Returns number of commits between HEAD and the tracking branch. This looks back to the tracking branch and works out the number of commits since then. + Args: + branch: Branch to count from (None for current branch) + Return: Number of patches that exist on top of the branch """ - pipe = [LogCmd('@{upstream}..', oneline=True), - ['wc', '-l']] + if branch: + us, msg = GetUpstream('.git', branch) + rev_range = '%s..%s' % (us, branch) + else: + rev_range = '@{upstream}..' + pipe = [LogCmd(rev_range, oneline=True), ['wc', '-l']] stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout patch_count = int(stdout) return patch_count @@ -252,17 +259,20 @@ def Fetch(git_dir=None, work_tree=None): if result.return_code != 0: raise OSError('git fetch: %s' % result.stderr) -def CreatePatches(start, count, ignore_binary, series): +def CreatePatches(branch, start, count, ignore_binary, series): """Create a series of patches from the top of the current branch. The patch files are written to the current directory using git format-patch. Args: + branch: Branch to create patches from (None for current branch) start: Commit to start from: 0=HEAD, 1=next one, etc. count: number of commits to include + ignore_binary: Don't generate patches for binary files + series: Series object for this series (set of patches) Return: - Filename of cover letter + Filename of cover letter (None if none) List of filenames of patch files """ if series.get('version'): @@ -275,7 +285,8 @@ def CreatePatches(start, count, ignore_binary, series): prefix = series.GetPatchPrefix() if prefix: cmd += ['--subject-prefix=%s' % prefix] - cmd += ['HEAD~%d..HEAD~%d' % (start + count, start)] + brname = branch or 'HEAD' + cmd += ['%s~%d..%s~%d' % (brname, start + count, brname, start)] stdout = command.RunList(cmd) files = stdout.splitlines() @@ -333,6 +344,31 @@ def BuildEmailList(in_list, tag=None, alias=None, raise_on_error=True): return ['%s %s%s%s' % (tag, quote, email, quote) for email in result] return result +def CheckSuppressCCConfig(): + """Check if sendemail.suppresscc is configured correctly. + + Returns: + True if the option is configured correctly, False otherwise. + """ + suppresscc = command.OutputOneLine('git', 'config', 'sendemail.suppresscc', + raise_on_error=False) + + # Other settings should be fine. + if suppresscc == 'all' or suppresscc == 'cccmd': + col = terminal.Color() + + print((col.Color(col.RED, "error") + + ": git config sendemail.suppresscc set to %s\n" % (suppresscc)) + + " patman needs --cc-cmd to be run to set the cc list.\n" + + " Please run:\n" + + " git config --unset sendemail.suppresscc\n" + + " Or read the man page:\n" + + " git send-email --help\n" + + " and set an option that runs --cc-cmd\n") + return False + + return True + def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname, self_only=False, alias=None, in_reply_to=None, thread=False, smtp_server=None): diff --git a/tools/patman/main.py b/tools/patman/main.py index 28a9a26087..b96000807e 100755 --- a/tools/patman/main.py +++ b/tools/patman/main.py @@ -6,10 +6,11 @@ """See README for more information""" -from optparse import OptionParser +from argparse import ArgumentParser import os import re import sys +import traceback import unittest if __name__ == "__main__": @@ -18,76 +19,93 @@ if __name__ == "__main__": sys.path.append(os.path.join(our_path, '..')) # Our modules -from patman import checkpatch from patman import command +from patman import control from patman import gitutil -from patman import patchstream from patman import project from patman import settings from patman import terminal +from patman import test_util from patman import test_checkpatch - -parser = OptionParser() -parser.add_option('-H', '--full-help', action='store_true', dest='full_help', +def AddCommonArgs(parser): + parser.add_argument('-b', '--branch', type=str, + help="Branch to process (by default, the current branch)") + parser.add_argument('-c', '--count', dest='count', type=int, + default=-1, help='Automatically create patches from top n commits') + parser.add_argument('-e', '--end', type=int, default=0, + help='Commits to skip at end of patch list') + parser.add_argument('-D', '--debug', action='store_true', + help='Enabling debugging (provides a full traceback on error)') + parser.add_argument('-s', '--start', dest='start', type=int, + default=0, help='Commit to start creating patches from (0 = HEAD)') + +epilog = '''Create patches from commits in a branch, check them and email them +as specified by tags you place in the commits. Use -n to do a dry run first.''' + +parser = ArgumentParser(epilog=epilog) +subparsers = parser.add_subparsers(dest='cmd') +send = subparsers.add_parser('send') +send.add_argument('-H', '--full-help', action='store_true', dest='full_help', default=False, help='Display the README file') -parser.add_option('-c', '--count', dest='count', type='int', - default=-1, help='Automatically create patches from top n commits') -parser.add_option('-i', '--ignore-errors', action='store_true', +send.add_argument('-i', '--ignore-errors', action='store_true', dest='ignore_errors', default=False, help='Send patches email even if patch errors are found') -parser.add_option('-l', '--limit-cc', dest='limit', type='int', - default=None, help='Limit the cc list to LIMIT entries [default: %default]') -parser.add_option('-m', '--no-maintainers', action='store_false', +send.add_argument('-l', '--limit-cc', dest='limit', type=int, default=None, + help='Limit the cc list to LIMIT entries [default: %(default)s]') +send.add_argument('-m', '--no-maintainers', action='store_false', dest='add_maintainers', default=True, help="Don't cc the file maintainers automatically") -parser.add_option('-n', '--dry-run', action='store_true', dest='dry_run', +send.add_argument('-n', '--dry-run', action='store_true', dest='dry_run', default=False, help="Do a dry run (create but don't email patches)") -parser.add_option('-p', '--project', default=project.DetectProject(), +send.add_argument('-p', '--project', default=project.DetectProject(), help="Project name; affects default option values and " - "aliases [default: %default]") -parser.add_option('-r', '--in-reply-to', type='string', action='store', + "aliases [default: %(default)s]") +send.add_argument('-r', '--in-reply-to', type=str, action='store', help="Message ID that this series is in reply to") -parser.add_option('-s', '--start', dest='start', type='int', - default=0, help='Commit to start creating patches from (0 = HEAD)') -parser.add_option('-t', '--ignore-bad-tags', action='store_true', +send.add_argument('-t', '--ignore-bad-tags', action='store_true', default=False, help='Ignore bad tags / aliases') -parser.add_option('-v', '--verbose', action='store_true', dest='verbose', +send.add_argument('-v', '--verbose', action='store_true', dest='verbose', default=False, help='Verbose output of errors and warnings') -parser.add_option('-T', '--thread', action='store_true', dest='thread', +send.add_argument('-T', '--thread', action='store_true', dest='thread', default=False, help='Create patches as a single thread') -parser.add_option('--cc-cmd', dest='cc_cmd', type='string', action='store', +send.add_argument('--cc-cmd', dest='cc_cmd', type=str, action='store', default=None, help='Output cc list for patch file (used by git)') -parser.add_option('--no-binary', action='store_true', dest='ignore_binary', +send.add_argument('--no-binary', action='store_true', dest='ignore_binary', default=False, help="Do not output contents of changes in binary files") -parser.add_option('--no-check', action='store_false', dest='check_patch', +send.add_argument('--no-check', action='store_false', dest='check_patch', default=True, help="Don't check for patch compliance") -parser.add_option('--no-tags', action='store_false', dest='process_tags', +send.add_argument('--no-tags', action='store_false', dest='process_tags', default=True, help="Don't process subject tags as aliases") -parser.add_option('--smtp-server', type='str', +send.add_argument('--smtp-server', type=str, help="Specify the SMTP server to 'git send-email'") -parser.add_option('--test', action='store_true', dest='test', - default=False, help='run tests') - -parser.usage += """ +AddCommonArgs(send) -Create patches from commits in a branch, check them and email them as -specified by tags you place in the commits. Use -n to do a dry run first.""" +send.add_argument('patchfiles', nargs='*') +test_parser = subparsers.add_parser('test', help='Run tests') +AddCommonArgs(test_parser) # Parse options twice: first to get the project and second to handle # defaults properly (which depends on project). -(options, args) = parser.parse_args() -settings.Setup(gitutil, parser, options.project, '') -(options, args) = parser.parse_args() +argv = sys.argv[1:] +if len(argv) < 1 or argv[0].startswith('-'): + argv = ['send'] + argv +args = parser.parse_args(argv) +if hasattr(args, 'project'): + settings.Setup(gitutil, send, args.project, '') + args = parser.parse_args(argv) if __name__ != "__main__": pass +if not args.debug: + sys.tracebacklimit = 0 + # Run our meagre tests -elif options.test: +if args.cmd == 'test': import doctest from patman import func_test @@ -101,87 +119,31 @@ elif options.test: suite = doctest.DocTestSuite(module) suite.run(result) - # TODO: Surely we can just 'print' result? - print(result) - for test, err in result.errors: - print(err) - for test, err in result.failures: - print(err) - -# Called from git with a patch filename as argument -# Printout a list of additional CC recipients for this patch -elif options.cc_cmd: - fd = open(options.cc_cmd, 'r') - re_line = re.compile('(\S*) (.*)') - for line in fd.readlines(): - match = re_line.match(line) - if match and match.group(1) == args[0]: - for cc in match.group(2).split('\0'): - cc = cc.strip() - if cc: - print(cc) - fd.close() - -elif options.full_help: - pager = os.getenv('PAGER') - if not pager: - pager = 'more' - fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), - 'README') - command.Run(pager, fname) + sys.exit(test_util.ReportResult('patman', None, result)) # Process commits, produce patches files, check them, email them -else: - gitutil.Setup() - - if options.count == -1: - # Work out how many patches to send if we can - options.count = gitutil.CountCommitsToBranch() - options.start - - col = terminal.Color() - if not options.count: - str = 'No commits found to process - please use -c flag' - sys.exit(col.Color(col.RED, str)) - - # Read the metadata from the commits - if options.count: - series = patchstream.GetMetaData(options.start, options.count) - cover_fname, args = gitutil.CreatePatches(options.start, options.count, - options.ignore_binary, series) - - # Fix up the patch files to our liking, and insert the cover letter - patchstream.FixPatches(series, args) - if cover_fname and series.get('cover'): - patchstream.InsertCoverLetter(cover_fname, series, options.count) - - # Do a few checks on the series - series.DoChecks() - - # Check the patches, and run them through 'git am' just to be sure - if options.check_patch: - ok = checkpatch.CheckPatches(options.verbose, args) - else: - ok = True - - cc_file = series.MakeCcFile(options.process_tags, cover_fname, - not options.ignore_bad_tags, - options.add_maintainers, options.limit) - - # Email the patches out (giving the user time to check / cancel) - cmd = '' - its_a_go = ok or options.ignore_errors - if its_a_go: - cmd = gitutil.EmailPatches(series, cover_fname, args, - options.dry_run, not options.ignore_bad_tags, cc_file, - in_reply_to=options.in_reply_to, thread=options.thread, - smtp_server=options.smtp_server) - else: - print(col.Color(col.RED, "Not sending emails due to errors/warnings")) - - # For a dry run, just show our actions as a sanity check - if options.dry_run: - series.ShowActions(args, cmd, options.process_tags) - if not its_a_go: - print(col.Color(col.RED, "Email would not be sent")) +elif args.cmd == 'send': + # Called from git with a patch filename as argument + # Printout a list of additional CC recipients for this patch + if args.cc_cmd: + fd = open(args.cc_cmd, 'r') + re_line = re.compile('(\S*) (.*)') + for line in fd.readlines(): + match = re_line.match(line) + if match and match.group(1) == args.patchfiles[0]: + for cc in match.group(2).split('\0'): + cc = cc.strip() + if cc: + print(cc) + fd.close() + + elif args.full_help: + pager = os.getenv('PAGER') + if not pager: + pager = 'more' + fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), + 'README') + command.Run(pager, fname) - os.remove(cc_file) + else: + control.send(args) diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py index 4fe465e9ab..ba0a13f632 100644 --- a/tools/patman/patchstream.py +++ b/tools/patman/patchstream.py @@ -37,7 +37,7 @@ re_change_id = re.compile('^Change-Id: *(.*)') re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)') # Commit tags that we want to collect and keep -re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)') +re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc|Fixes): (.*)') # The start of a new commit in the git log re_commit = re.compile('^commit ([0-9a-f]*)$') @@ -112,6 +112,15 @@ class PatchStream: self.in_section = 'commit-' + name self.skip_blank = False + def AddCommitRtag(self, rtag_type, who): + """Add a response tag to the current commit + + Args: + key: rtag type (e.g. 'Reviewed-by') + who: Person who gave that rtag, e.g. 'Fred Bloggs <fred@bloggs.org>' + """ + self.commit.AddRtag(rtag_type, who) + def CloseCommit(self): """Save the current commit into our commit list, and reset our state""" if self.commit and self.is_log: @@ -260,6 +269,10 @@ class PatchStream: else: self.section.append(line) + # If we are not in a section, it is an unexpected END + elif line == 'END': + raise ValueError("'END' wihout section") + # Detect the commit subject elif not is_blank and self.state == STATE_PATCH_SUBJECT: self.commit.subject = line @@ -338,6 +351,9 @@ class PatchStream: elif name == 'changes': self.in_change = 'Commit' self.change_version = self.ParseVersion(value, line) + else: + self.warn.append('Line %d: Ignoring Commit-%s' % + (self.linenum, name)) # Detect the start of a new commit elif commit_match: @@ -346,12 +362,14 @@ class PatchStream: # Detect tags in the commit message elif tag_match: + rtag_type, who = tag_match.groups() + self.AddCommitRtag(rtag_type, who) # Remove Tested-by self, since few will take much notice - if (tag_match.group(1) == 'Tested-by' and - tag_match.group(2).find(os.getenv('USER') + '@') != -1): + if (rtag_type == 'Tested-by' and + who.find(os.getenv('USER') + '@') != -1): self.warn.append("Ignoring %s" % line) - elif tag_match.group(1) == 'Patch-cc': - self.commit.AddCc(tag_match.group(2).split(',')) + elif rtag_type == 'Patch-cc': + self.commit.AddCc(who.split(',')) else: out = [line] @@ -512,17 +530,19 @@ def GetMetaDataForList(commit_range, git_dir=None, count=None, ps.Finalize() return series -def GetMetaData(start, count): +def GetMetaData(branch, start, count): """Reads out patch series metadata from the commits This does a 'git log' on the relevant commits and pulls out the tags we are interested in. Args: - start: Commit to start from: 0=HEAD, 1=next one, etc. + branch: Branch to use (None for current branch) + start: Commit to start from: 0=branch HEAD, 1=next one, etc. count: Number of commits to list """ - return GetMetaDataForList('HEAD~%d' % start, None, count) + return GetMetaDataForList('%s~%d' % (branch if branch else 'HEAD', start), + None, count) def GetMetaDataForTest(text): """Process metadata from a file containing a git log. Used for tests diff --git a/tools/patman/series.py b/tools/patman/series.py index b7eef37d03..9f885c8987 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -244,7 +244,7 @@ class Series(dict): add_maintainers: Either: True/False to call the get_maintainers to CC maintainers List of maintainers to include (for testing) - limit: Limit the length of the Cc list + limit: Limit the length of the Cc list (None if no limit) Return: Filename of temp file created """ diff --git a/tools/patman/settings.py b/tools/patman/settings.py index 635561ac05..732bd40106 100644 --- a/tools/patman/settings.py +++ b/tools/patman/settings.py @@ -233,17 +233,19 @@ def _UpdateDefaults(parser, config): config: An instance of _ProjectConfigParser that we will query for settings. """ - defaults = parser.get_default_values() + defaults = parser.parse_known_args()[0] + defaults = vars(defaults) for name, val in config.items('settings'): - if hasattr(defaults, name): - default_val = getattr(defaults, name) + if name in defaults: + default_val = defaults[name] if isinstance(default_val, bool): val = config.getboolean('settings', name) elif isinstance(default_val, int): val = config.getint('settings', name) - parser.set_default(name, val) + defaults[name] = val else: print("WARNING: Unknown setting %s" % name) + parser.set_defaults(**defaults) def _ReadAliasFile(fname): """Read in the U-Boot git alias file if it exists. diff --git a/tools/patman/terminal.py b/tools/patman/terminal.py index c709438bdc..60dbce3ce1 100644 --- a/tools/patman/terminal.py +++ b/tools/patman/terminal.py @@ -122,7 +122,7 @@ def TrimAsciiLen(text, size): return out -def Print(text='', newline=True, colour=None, limit_to_line=False): +def Print(text='', newline=True, colour=None, limit_to_line=False, bright=True): """Handle a line of output to the terminal. In test mode this is recorded in a list. Otherwise it is output to the @@ -140,7 +140,7 @@ def Print(text='', newline=True, colour=None, limit_to_line=False): else: if colour: col = Color() - text = col.Color(colour, text) + text = col.Color(colour, text, bright=bright) if newline: print(text) last_print_len = None diff --git a/tools/patman/test_util.py b/tools/patman/test_util.py index aac58fb72f..4e261755dc 100644 --- a/tools/patman/test_util.py +++ b/tools/patman/test_util.py @@ -16,12 +16,14 @@ from io import StringIO use_concurrent = True try: - from concurrencytest import ConcurrentTestSuite, fork_for_tests + from concurrencytest.concurrencytest import ConcurrentTestSuite + from concurrencytest.concurrencytest import fork_for_tests except: use_concurrent = False -def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): +def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None, + extra_args=None): """Run tests and check that we get 100% coverage Args: @@ -34,6 +36,8 @@ def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): calculation build_dir: Build directory, used to locate libfdt.py required: List of modules which must be in the coverage report + extra_args (str): Extra arguments to pass to the tool before the -t/test + arg Raises: ValueError if the code coverage is not 100% @@ -47,13 +51,14 @@ def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): glob_list = [] glob_list += exclude_list glob_list += ['*libfdt.py', '*site-packages*', '*dist-packages*'] - test_cmd = 'test' if 'binman' in prog else '-t' + glob_list += ['*concurrencytest*'] + test_cmd = 'test' if 'binman' in prog or 'patman' in prog else '-t' prefix = '' if build_dir: prefix = 'PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools ' % build_dir cmd = ('%spython3-coverage run ' - '--omit "%s" %s %s -P1' % (prefix, ','.join(glob_list), - prog, test_cmd)) + '--omit "%s" %s %s %s -P1' % (prefix, ','.join(glob_list), + prog, extra_args or '', test_cmd)) os.system(cmd) stdout = command.Output('python3-coverage', 'report') lines = stdout.splitlines() @@ -123,12 +128,12 @@ def ReportResult(toolname:str, test_name: str, result: unittest.TestResult): for test, err in result.failures: print(err, result.failures) if result.skipped: - print('%d binman test%s SKIPPED:' % - (len(result.skipped), 's' if len(result.skipped) > 1 else '')) + print('%d %s test%s SKIPPED:' % (len(result.skipped), toolname, + 's' if len(result.skipped) > 1 else '')) for skip_info in result.skipped: print('%s: %s' % (skip_info[0], skip_info[1])) if result.errors or result.failures: - print('binman tests FAILED') + print('%s tests FAILED' % toolname) return 1 return 0 diff --git a/tools/patman/tools.py b/tools/patman/tools.py index b50370dfe8..d41115a22c 100644 --- a/tools/patman/tools.py +++ b/tools/patman/tools.py @@ -114,14 +114,16 @@ def SetInputDirs(dirname): indir = dirname tout.Debug("Using input directories %s" % indir) -def GetInputFilename(fname): +def GetInputFilename(fname, allow_missing=False): """Return a filename for use as input. Args: fname: Filename to use for new file + allow_missing: True if the filename can be missing Returns: - The full path of the filename, within the input directory + The full path of the filename, within the input directory, or + None on error """ if not indir or fname[:1] == '/': return fname @@ -130,6 +132,8 @@ def GetInputFilename(fname): if os.path.exists(pathname): return pathname + if allow_missing: + return None raise ValueError("Filename '%s' not found in input path (%s) (cwd='%s')" % (fname, ','.join(indir), os.getcwd())) @@ -270,7 +274,7 @@ def ReadFile(fname, binary=True): #(fname, len(data), len(data))) return data -def WriteFile(fname, data): +def WriteFile(fname, data, binary=True): """Write data into a file. Args: @@ -279,7 +283,7 @@ def WriteFile(fname, data): """ #self._out.Info("Write file '%s' size %d (%#0x)" % #(fname, len(data), len(data))) - with open(Filename(fname), 'wb') as fd: + with open(Filename(fname), binary and 'wb' or 'w') as fd: fd.write(data) def GetBytes(byte, size): diff --git a/tools/patman/tout.py b/tools/patman/tout.py index c7e3272096..33305263d8 100644 --- a/tools/patman/tout.py +++ b/tools/patman/tout.py @@ -83,7 +83,10 @@ def _Output(level, msg, color=None): ClearProgress() if color: msg = _color.Color(color, msg) - print(msg) + if level < NOTICE: + print(msg, file=sys.stderr) + else: + print(msg) def DoOutput(level, msg): """Output a message to the terminal. @@ -168,6 +171,7 @@ def Init(_verbose=WARNING, stdout=sys.stdout): # TODO(sjg): Move this into Chromite libraries when we have them stdout_is_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() + stderr_is_tty = hasattr(sys.stderr, 'isatty') and sys.stderr.isatty() def Uninit(): ClearProgress() |