diff options
Diffstat (limited to 'tools')
38 files changed, 711 insertions, 356 deletions
diff --git a/tools/binman/README b/tools/binman/README index 927fa856ac..ac193f16cf 100644 --- a/tools/binman/README +++ b/tools/binman/README @@ -702,6 +702,20 @@ To enable Python test coverage on Debian-type distributions (e.g. Ubuntu): $ sudo apt-get install python-coverage python-pytest +Concurrent tests +---------------- + +Binman tries to run tests concurrently. This means that the tests make use of +all available CPUs to run. + + To enable this: + + $ sudo apt-get install python-subunit python3-subunit + +Use '-P 1' to disable this. It is automatically disabled when code coverage is +being used (-T) since they are incompatible. + + Advanced Features / Technical docs ---------------------------------- diff --git a/tools/binman/README.entries b/tools/binman/README.entries index 9fc2f83280..357946d630 100644 --- a/tools/binman/README.entries +++ b/tools/binman/README.entries @@ -224,6 +224,20 @@ See README.x86 for information about x86 binary blobs. +Entry: intel-refcode: Entry containing an Intel Reference Code file +------------------------------------------------------------------- + +Properties / Entry arguments: + - filename: Filename of file to read into entry + +This file contains code for setting up the platform on some Intel systems. +This is executed by U-Boot when needed early during startup. A typical +filename is 'refcode.bin'. + +See README.x86 for information about x86 binary blobs. + + + Entry: intel-vbt: Entry containing an Intel Video BIOS Table (VBT) file ----------------------------------------------------------------------- @@ -627,6 +641,7 @@ Entry: vblock: An entry which contains a Chromium OS verified boot block ------------------------------------------------------------------------ Properties / Entry arguments: + - content: List of phandles to entries to sign - keydir: Directory containing the public keys to use - keyblock: Name of the key file to use (inside keydir) - signprivate: Name of provide key file to use (inside keydir) diff --git a/tools/binman/binman.py b/tools/binman/binman.py index 439908e665..aad2e9c8bc 100755 --- a/tools/binman/binman.py +++ b/tools/binman/binman.py @@ -9,6 +9,8 @@ """See README for more information""" +from __future__ import print_function + import glob import multiprocessing import os @@ -85,13 +87,25 @@ def RunTests(debug, processes, args): else: suite.run(result) - print result + # Remove errors which just indicate a missing test. Since Python v3.5 If an + # ImportError or AttributeError occurs while traversing name then a + # synthetic test that raises that error when run will be returned. These + # errors are included in the errors accumulated by result.errors. + if test_name: + errors = [] + for test, err in result.errors: + if ("has no attribute '%s'" % test_name) not in err: + errors.append((test, err)) + result.testsRun -= 1 + result.errors = errors + + print(result) for test, err in result.errors: - print test.id(), err + print(test.id(), err) for test, err in result.failures: - print err, result.failures + print(err, result.failures) if result.errors or result.failures: - print 'binman tests FAILED' + print('binman tests FAILED') return 1 return 0 @@ -143,9 +157,9 @@ def RunBinman(options, args): try: ret_code = control.Binman(options, args) except Exception as e: - print 'binman: %s' % e + print('binman: %s' % e) if options.debug: - print + print() traceback.print_exc() ret_code = 1 return ret_code diff --git a/tools/binman/bsection.py b/tools/binman/bsection.py index 0ba542ee98..03dfa2f805 100644 --- a/tools/binman/bsection.py +++ b/tools/binman/bsection.py @@ -8,7 +8,6 @@ from __future__ import print_function from collections import OrderedDict -from sets import Set import sys import fdt_util @@ -109,7 +108,7 @@ class Section(object): def GetFdtSet(self): """Get the set of device tree files used by this image""" - fdt_set = Set() + fdt_set = set() for entry in self._entries.values(): fdt_set.update(entry.GetFdtSet()) return fdt_set @@ -254,7 +253,7 @@ class Section(object): """ for entry in self._entries.values(): offset_dict = entry.GetOffsets() - for name, info in offset_dict.iteritems(): + for name, info in offset_dict.items(): self._SetEntryOffsetSize(name, *info) def PackEntries(self): @@ -333,7 +332,7 @@ class Section(object): def GetData(self): """Get the contents of the section""" - section_data = chr(self._pad_byte) * self._size + section_data = tools.GetBytes(self._pad_byte, self._size) for entry in self._entries.values(): data = entry.GetData() diff --git a/tools/binman/control.py b/tools/binman/control.py index b32e4e1996..20186ee198 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -5,6 +5,8 @@ # Creates binary images from input files controlled by a description # +from __future__ import print_function + from collections import OrderedDict import os import sys @@ -129,12 +131,15 @@ def Binman(options, args): if options.image: skip = [] - for name, image in images.iteritems(): - if name not in options.image: - del images[name] + new_images = OrderedDict() + for name, image in images.items(): + if name in options.image: + new_images[name] = image + else: skip.append(name) + images = new_images if skip and options.verbosity >= 2: - print 'Skipping images: %s' % ', '.join(skip) + print('Skipping images: %s' % ', '.join(skip)) state.Prepare(images, dtb) @@ -170,7 +175,7 @@ def Binman(options, args): except Exception as e: if options.map: fname = image.WriteMap() - print "Wrote map file '%s' to show errors" % fname + print("Wrote map file '%s' to show errors" % fname) raise image.SetImagePos() if options.update_fdt: diff --git a/tools/binman/elf.py b/tools/binman/elf.py index 97df8e32c5..828681d76d 100644 --- a/tools/binman/elf.py +++ b/tools/binman/elf.py @@ -59,7 +59,7 @@ def GetSymbols(fname, patterns): flags[1] == 'w') # Sort dict by address - return OrderedDict(sorted(syms.iteritems(), key=lambda x: x[1].address)) + return OrderedDict(sorted(syms.items(), key=lambda x: x[1].address)) def GetSymbolAddress(fname, sym_name): """Get a value of a symbol from an ELF file @@ -98,7 +98,7 @@ def LookupAndWriteSymbols(elf_fname, entry, section): base = syms.get('__image_copy_start') if not base: return - for name, sym in syms.iteritems(): + for name, sym in syms.items(): if name.startswith('_binman'): msg = ("Section '%s': Symbol '%s'\n in entry '%s'" % (section.GetPath(), name, entry.GetPath())) diff --git a/tools/binman/elf_test.py b/tools/binman/elf_test.py index b68530c19b..42d94cbbbe 100644 --- a/tools/binman/elf_test.py +++ b/tools/binman/elf_test.py @@ -22,7 +22,7 @@ class FakeEntry: """ def __init__(self, contents_size): self.contents_size = contents_size - self.data = 'a' * contents_size + self.data = tools.GetBytes(ord('a'), contents_size) def GetPath(self): return 'entry_path' @@ -122,7 +122,8 @@ class TestElf(unittest.TestCase): section = FakeSection(sym_value=None) elf_fname = os.path.join(binman_dir, 'test', 'u_boot_binman_syms') syms = elf.LookupAndWriteSymbols(elf_fname, entry, section) - self.assertEqual(chr(255) * 16 + 'a' * 4, entry.data) + self.assertEqual(tools.GetBytes(255, 16) + tools.GetBytes(ord('a'), 4), + entry.data) def testDebug(self): """Check that enabling debug in the elf module produced debug output""" diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 648cfd241f..d842d89dd6 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -18,7 +18,6 @@ except: have_importlib = False import os -from sets import Set import sys import fdt_util @@ -178,8 +177,8 @@ class Entry(object): # It would be better to use isinstance(self, Entry_blob_dtb) here but # we cannot access Entry_blob_dtb if fname and fname.endswith('.dtb'): - return Set([fname]) - return Set() + return set([fname]) + return set() def ExpandEntries(self): pass diff --git a/tools/binman/entry_test.py b/tools/binman/entry_test.py index 1f7ff5b4e4..b30a7beecc 100644 --- a/tools/binman/entry_test.py +++ b/tools/binman/entry_test.py @@ -41,7 +41,11 @@ class TestEntry(unittest.TestCase): del sys.modules['importlib'] global entry if entry: - reload(entry) + if sys.version_info[0] >= 3: + import importlib + importlib.reload(entry) + else: + reload(entry) else: import entry entry.Entry.Create(None, self.GetNode(), 'u-boot-spl') diff --git a/tools/binman/etype/_testing.py b/tools/binman/etype/_testing.py index 3e345bd952..ac62d2e200 100644 --- a/tools/binman/etype/_testing.py +++ b/tools/binman/etype/_testing.py @@ -75,7 +75,7 @@ class Entry__testing(Entry): def ObtainContents(self): if self.return_unknown_contents or not self.return_contents: return False - self.data = 'a' + self.data = b'a' self.contents_size = len(self.data) if self.return_contents_once: self.return_contents = False diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py index ae80bbee53..f56a1f8768 100644 --- a/tools/binman/etype/blob.py +++ b/tools/binman/etype/blob.py @@ -60,7 +60,7 @@ class Entry_blob(Entry): except AttributeError: data = lz4.compress(data) ''' - data = tools.Run('lz4', '-c', self._pathname) + data = tools.Run('lz4', '-c', self._pathname, binary=True) self.SetContents(data) return True diff --git a/tools/binman/etype/fill.py b/tools/binman/etype/fill.py index dcfe978a5b..68efe42ec0 100644 --- a/tools/binman/etype/fill.py +++ b/tools/binman/etype/fill.py @@ -5,7 +5,7 @@ from entry import Entry import fdt_util - +import tools class Entry_fill(Entry): """An entry which is filled to a particular byte value @@ -28,5 +28,5 @@ class Entry_fill(Entry): self.fill_value = fdt_util.GetByte(self._node, 'fill-byte', 0) def ObtainContents(self): - self.SetContents(chr(self.fill_value) * self.size) + self.SetContents(tools.GetBytes(self.fill_value, self.size)) return True diff --git a/tools/binman/etype/fmap.py b/tools/binman/etype/fmap.py index bf35a5bbf4..e6b5c5c74c 100644 --- a/tools/binman/etype/fmap.py +++ b/tools/binman/etype/fmap.py @@ -7,6 +7,7 @@ from entry import Entry import fmap_util +import tools class Entry_fmap(Entry): @@ -46,7 +47,7 @@ class Entry_fmap(Entry): if pos is not None: pos -= entry.section.GetRootSkipAtStart() areas.append(fmap_util.FmapArea(pos or 0, entry.size or 0, - entry.name, 0)) + tools.FromUnicode(entry.name), 0)) entries = self.section._image.GetEntries() areas = [] diff --git a/tools/binman/etype/gbb.py b/tools/binman/etype/gbb.py index 8fe10f4713..a94c0fca9d 100644 --- a/tools/binman/etype/gbb.py +++ b/tools/binman/etype/gbb.py @@ -64,7 +64,7 @@ class Entry_gbb(Entry): self.gbb_flags = 0 flags_node = node.FindNode('flags') if flags_node: - for flag, value in gbb_flag_properties.iteritems(): + for flag, value in gbb_flag_properties.items(): if fdt_util.GetBool(flags_node, flag): self.gbb_flags |= value diff --git a/tools/binman/etype/text.py b/tools/binman/etype/text.py index c4aa510a87..9ee04d7c9d 100644 --- a/tools/binman/etype/text.py +++ b/tools/binman/etype/text.py @@ -7,6 +7,7 @@ from collections import OrderedDict from entry import Entry, EntryArg import fdt_util +import tools class Entry_text(Entry): @@ -48,9 +49,11 @@ class Entry_text(Entry): """ def __init__(self, section, etype, node): Entry.__init__(self, section, etype, node) - self.text_label, = self.GetEntryArgsOrProps( - [EntryArg('text-label', str)]) - self.value, = self.GetEntryArgsOrProps([EntryArg(self.text_label, str)]) + label, = self.GetEntryArgsOrProps([EntryArg('text-label', str)]) + self.text_label = tools.ToStr(label) if type(label) != str else label + value, = self.GetEntryArgsOrProps([EntryArg(self.text_label, str)]) + value = tools.ToBytes(value) if value is not None else value + self.value = value def ObtainContents(self): if not self.value: diff --git a/tools/binman/etype/u_boot_dtb_with_ucode.py b/tools/binman/etype/u_boot_dtb_with_ucode.py index 444c51b8b7..188888e022 100644 --- a/tools/binman/etype/u_boot_dtb_with_ucode.py +++ b/tools/binman/etype/u_boot_dtb_with_ucode.py @@ -26,7 +26,7 @@ class Entry_u_boot_dtb_with_ucode(Entry_blob_dtb): """ def __init__(self, section, etype, node): Entry_blob_dtb.__init__(self, section, etype, node) - self.ucode_data = '' + self.ucode_data = b'' self.collate = False self.ucode_offset = None self.ucode_size = None @@ -65,7 +65,7 @@ class Entry_u_boot_dtb_with_ucode(Entry_blob_dtb): for node in self.ucode.subnodes: data_prop = node.props.get('data') if data_prop: - self.ucode_data += ''.join(data_prop.bytes) + self.ucode_data += data_prop.bytes if self.collate: node.DeleteProp('data') return True diff --git a/tools/binman/etype/u_boot_spl_bss_pad.py b/tools/binman/etype/u_boot_spl_bss_pad.py index 00b7ac5004..66a296a6f8 100644 --- a/tools/binman/etype/u_boot_spl_bss_pad.py +++ b/tools/binman/etype/u_boot_spl_bss_pad.py @@ -38,5 +38,5 @@ class Entry_u_boot_spl_bss_pad(Entry_blob): bss_size = elf.GetSymbolAddress(fname, '__bss_size') if not bss_size: self.Raise('Expected __bss_size symbol in spl/u-boot-spl') - self.SetContents(chr(0) * bss_size) + self.SetContents(tools.GetBytes(0, bss_size)) return True diff --git a/tools/binman/etype/u_boot_ucode.py b/tools/binman/etype/u_boot_ucode.py index a00e530295..dee8848db7 100644 --- a/tools/binman/etype/u_boot_ucode.py +++ b/tools/binman/etype/u_boot_ucode.py @@ -69,7 +69,7 @@ class Entry_u_boot_ucode(Entry_blob): if entry and entry.target_offset: found = True if not found: - self.data = '' + self.data = b'' return True # Get the microcode from the device tree entry. If it is not available # yet, return False so we will be called later. If the section simply @@ -87,7 +87,7 @@ class Entry_u_boot_ucode(Entry_blob): if not fdt_entry.collate: # This binary can be empty - self.data = '' + self.data = b'' return True # Write it out to a file diff --git a/tools/binman/etype/vblock.py b/tools/binman/etype/vblock.py index 334ff9f966..91fa2f7808 100644 --- a/tools/binman/etype/vblock.py +++ b/tools/binman/etype/vblock.py @@ -51,7 +51,7 @@ class Entry_vblock(Entry): def ObtainContents(self): # Join up the data files to be signed - input_data = '' + input_data = b'' for entry_phandle in self.content: data = self.section.GetContentsByPhandle(entry_phandle, self) if data is None: diff --git a/tools/binman/fmap_util.py b/tools/binman/fmap_util.py index be3cbee87b..d0f956b622 100644 --- a/tools/binman/fmap_util.py +++ b/tools/binman/fmap_util.py @@ -8,9 +8,12 @@ import collections import struct +import sys + +import tools # constants imported from lib/fmap.h -FMAP_SIGNATURE = '__FMAP__' +FMAP_SIGNATURE = b'__FMAP__' FMAP_VER_MAJOR = 1 FMAP_VER_MINOR = 0 FMAP_STRLEN = 32 @@ -50,6 +53,8 @@ FmapArea = collections.namedtuple('FmapArea', FMAP_AREA_NAMES) def NameToFmap(name): + if type(name) == bytes and sys.version_info[0] >= 3: + name = name.decode('utf-8') # pragma: no cover (for Python 2) return name.replace('\0', '').replace('-', '_').upper() def ConvertName(field_names, fields): @@ -65,7 +70,7 @@ def ConvertName(field_names, fields): value: value of that field (string for the ones we support) """ name_index = field_names.index('name') - fields[name_index] = NameToFmap(fields[name_index]) + fields[name_index] = tools.ToBytes(NameToFmap(fields[name_index])) def DecodeFmap(data): """Decode a flashmap into a header and list of areas @@ -106,7 +111,8 @@ def EncodeFmap(image_size, name, areas): ConvertName(names, params) return struct.pack(fmt, *params) - values = FmapHeader(FMAP_SIGNATURE, 1, 0, 0, image_size, name, len(areas)) + values = FmapHeader(FMAP_SIGNATURE, 1, 0, 0, image_size, + tools.FromUnicode(name), len(areas)) blob = _FormatBlob(FMAP_HEADER_FORMAT, FMAP_HEADER_NAMES, values) for area in areas: blob += _FormatBlob(FMAP_AREA_FORMAT, FMAP_AREA_NAMES, area) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index daea1ea138..cc57ef3e04 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -29,38 +29,38 @@ import tools import tout # Contents of test files, corresponding to different entry types -U_BOOT_DATA = '1234' -U_BOOT_IMG_DATA = 'img' -U_BOOT_SPL_DATA = '56780123456789abcde' -U_BOOT_TPL_DATA = 'tpl' -BLOB_DATA = '89' -ME_DATA = '0abcd' -VGA_DATA = 'vga' -U_BOOT_DTB_DATA = 'udtb' -U_BOOT_SPL_DTB_DATA = 'spldtb' -U_BOOT_TPL_DTB_DATA = 'tpldtb' -X86_START16_DATA = 'start16' -X86_START16_SPL_DATA = 'start16spl' -X86_START16_TPL_DATA = 'start16tpl' -PPC_MPC85XX_BR_DATA = 'ppcmpc85xxbr' -U_BOOT_NODTB_DATA = 'nodtb with microcode pointer somewhere in here' -U_BOOT_SPL_NODTB_DATA = 'splnodtb with microcode pointer somewhere in here' -U_BOOT_TPL_NODTB_DATA = 'tplnodtb with microcode pointer somewhere in here' -FSP_DATA = 'fsp' -CMC_DATA = 'cmc' -VBT_DATA = 'vbt' -MRC_DATA = 'mrc' +U_BOOT_DATA = b'1234' +U_BOOT_IMG_DATA = b'img' +U_BOOT_SPL_DATA = b'56780123456789abcde' +U_BOOT_TPL_DATA = b'tpl' +BLOB_DATA = b'89' +ME_DATA = b'0abcd' +VGA_DATA = b'vga' +U_BOOT_DTB_DATA = b'udtb' +U_BOOT_SPL_DTB_DATA = b'spldtb' +U_BOOT_TPL_DTB_DATA = b'tpldtb' +X86_START16_DATA = b'start16' +X86_START16_SPL_DATA = b'start16spl' +X86_START16_TPL_DATA = b'start16tpl' +PPC_MPC85XX_BR_DATA = b'ppcmpc85xxbr' +U_BOOT_NODTB_DATA = b'nodtb with microcode pointer somewhere in here' +U_BOOT_SPL_NODTB_DATA = b'splnodtb with microcode pointer somewhere in here' +U_BOOT_TPL_NODTB_DATA = b'tplnodtb with microcode pointer somewhere in here' +FSP_DATA = b'fsp' +CMC_DATA = b'cmc' +VBT_DATA = b'vbt' +MRC_DATA = b'mrc' TEXT_DATA = 'text' TEXT_DATA2 = 'text2' TEXT_DATA3 = 'text3' -CROS_EC_RW_DATA = 'ecrw' -GBB_DATA = 'gbbd' -BMPBLK_DATA = 'bmp' -VBLOCK_DATA = 'vblk' -FILES_DATA = ("sorry I'm late\nOh, don't bother apologising, I'm " + - "sorry you're alive\n") -COMPRESS_DATA = 'data to compress' -REFCODE_DATA = 'refcode' +CROS_EC_RW_DATA = b'ecrw' +GBB_DATA = b'gbbd' +BMPBLK_DATA = b'bmp' +VBLOCK_DATA = b'vblk' +FILES_DATA = (b"sorry I'm late\nOh, don't bother apologising, I'm " + + b"sorry you're alive\n") +COMPRESS_DATA = b'data to compress' +REFCODE_DATA = b'refcode' class TestFunctional(unittest.TestCase): @@ -119,11 +119,11 @@ class TestFunctional(unittest.TestCase): TestFunctional._MakeInputFile('refcode.bin', REFCODE_DATA) # ELF file with a '_dt_ucode_base_size' symbol - with open(self.TestFile('u_boot_ucode_ptr')) as fd: + with open(self.TestFile('u_boot_ucode_ptr'), 'rb') as fd: TestFunctional._MakeInputFile('u-boot', fd.read()) # Intel flash descriptor file - with open(self.TestFile('descriptor.bin')) as fd: + with open(self.TestFile('descriptor.bin'), 'rb') as fd: TestFunctional._MakeInputFile('descriptor.bin', fd.read()) shutil.copytree(self.TestFile('files'), @@ -214,7 +214,7 @@ class TestFunctional(unittest.TestCase): if verbosity is not None: args.append('-v%d' % verbosity) if entry_args: - for arg, value in entry_args.iteritems(): + for arg, value in entry_args.items(): args.append('-a%s=%s' % (arg, value)) if images: for image in images: @@ -236,7 +236,7 @@ class TestFunctional(unittest.TestCase): """ tools.PrepareOutputDir(None) dtb = fdt_util.EnsureCompiled(self.TestFile(fname)) - with open(dtb) as fd: + with open(dtb, 'rb') as fd: data = fd.read() TestFunctional._MakeInputFile(outfile, data) tools.FinaliseOutputDir() @@ -291,7 +291,6 @@ class TestFunctional(unittest.TestCase): # Use the compiled test file as the u-boot-dtb input if use_real_dtb: dtb_data = self._SetupDtb(fname) - infile = os.path.join(self._indir, 'u-boot.dtb') # For testing purposes, make a copy of the DT for SPL and TPL. Add # a node indicating which it is, so aid verification. @@ -317,7 +316,7 @@ class TestFunctional(unittest.TestCase): map_data = fd.read() else: map_data = None - with open(image_fname) as fd: + with open(image_fname, 'rb') as fd: return fd.read(), dtb_data, map_data, out_dtb_fname finally: # Put the test file back @@ -379,7 +378,7 @@ class TestFunctional(unittest.TestCase): Args: Filename of ELF file to use as SPL """ - with open(self.TestFile(src_fname)) as fd: + with open(self.TestFile(src_fname), 'rb') as fd: TestFunctional._MakeInputFile('spl/u-boot-spl', fd.read()) @classmethod @@ -396,7 +395,7 @@ class TestFunctional(unittest.TestCase): for grep in grep_list: if grep in target: return - self.fail("Error: '%' not found in '%s'" % (grep_list, target)) + self.fail("Error: '%s' not found in '%s'" % (grep_list, target)) def CheckNoGaps(self, entries): """Check that all entries fit together without gaps @@ -541,7 +540,7 @@ class TestFunctional(unittest.TestCase): self.assertEqual(len(U_BOOT_DATA), image._size) fname = tools.GetOutputFilename('image1.bin') self.assertTrue(os.path.exists(fname)) - with open(fname) as fd: + with open(fname, 'rb') as fd: data = fd.read() self.assertEqual(U_BOOT_DATA, data) @@ -549,11 +548,11 @@ class TestFunctional(unittest.TestCase): self.assertEqual(3 + len(U_BOOT_DATA) + 5, image._size) fname = tools.GetOutputFilename('image2.bin') self.assertTrue(os.path.exists(fname)) - with open(fname) as fd: + with open(fname, 'rb') as fd: data = fd.read() self.assertEqual(U_BOOT_DATA, data[3:7]) - self.assertEqual(chr(0) * 3, data[:3]) - self.assertEqual(chr(0) * 5, data[7:]) + self.assertEqual(tools.GetBytes(0, 3), data[:3]) + self.assertEqual(tools.GetBytes(0, 5), data[7:]) def testBadAlign(self): """Test that an invalid alignment value is detected""" @@ -732,7 +731,8 @@ class TestFunctional(unittest.TestCase): """Test that the image pad byte can be specified""" self._SetupSplElf() data = self._DoReadFile('021_image_pad.dts') - self.assertEqual(U_BOOT_SPL_DATA + (chr(0xff) * 1) + U_BOOT_DATA, data) + self.assertEqual(U_BOOT_SPL_DATA + tools.GetBytes(0xff, 1) + + U_BOOT_DATA, data) def testImageName(self): """Test that image files can be named""" @@ -755,8 +755,8 @@ class TestFunctional(unittest.TestCase): """Test that entries can be sorted""" self._SetupSplElf() data = self._DoReadFile('024_sorted.dts') - self.assertEqual(chr(0) * 1 + U_BOOT_SPL_DATA + chr(0) * 2 + - U_BOOT_DATA, data) + self.assertEqual(tools.GetBytes(0, 1) + U_BOOT_SPL_DATA + + tools.GetBytes(0, 2) + U_BOOT_DATA, data) def testPackZeroOffset(self): """Test that an entry at offset 0 is not given a new offset""" @@ -798,12 +798,12 @@ class TestFunctional(unittest.TestCase): """Test that a basic x86 ROM can be created""" self._SetupSplElf() data = self._DoReadFile('029_x86-rom.dts') - self.assertEqual(U_BOOT_DATA + chr(0) * 7 + U_BOOT_SPL_DATA + - chr(0) * 2, data) + self.assertEqual(U_BOOT_DATA + tools.GetBytes(0, 7) + U_BOOT_SPL_DATA + + tools.GetBytes(0, 2), data) def testPackX86RomMeNoDesc(self): """Test that an invalid Intel descriptor entry is detected""" - TestFunctional._MakeInputFile('descriptor.bin', '') + TestFunctional._MakeInputFile('descriptor.bin', b'') with self.assertRaises(ValueError) as e: self._DoTestFile('031_x86-rom-me.dts') self.assertIn("Node '/binman/intel-descriptor': Cannot find FD " @@ -900,8 +900,8 @@ class TestFunctional(unittest.TestCase): """ first, pos_and_size = self._RunMicrocodeTest('034_x86_ucode.dts', U_BOOT_NODTB_DATA) - self.assertEqual('nodtb with microcode' + pos_and_size + - ' somewhere in here', first) + self.assertEqual(b'nodtb with microcode' + pos_and_size + + b' somewhere in here', first) def _RunPackUbootSingleMicrocode(self): """Test that x86 microcode can be handled correctly @@ -932,8 +932,8 @@ class TestFunctional(unittest.TestCase): pos_and_size = struct.pack('<2L', 0xfffffe00 + ucode_pos, len(ucode_data)) first = data[:len(U_BOOT_NODTB_DATA)] - self.assertEqual('nodtb with microcode' + pos_and_size + - ' somewhere in here', first) + self.assertEqual(b'nodtb with microcode' + pos_and_size + + b' somewhere in here', first) def testPackUbootSingleMicrocode(self): """Test that x86 microcode can be handled correctly with fdt_normal. @@ -970,7 +970,7 @@ class TestFunctional(unittest.TestCase): """Test that a U-Boot binary without the microcode symbol is detected""" # ELF file without a '_dt_ucode_base_size' symbol try: - with open(self.TestFile('u_boot_no_ucode_ptr')) as fd: + with open(self.TestFile('u_boot_no_ucode_ptr'), 'rb') as fd: TestFunctional._MakeInputFile('u-boot', fd.read()) with self.assertRaises(ValueError) as e: @@ -980,7 +980,7 @@ class TestFunctional(unittest.TestCase): finally: # Put the original file back - with open(self.TestFile('u_boot_ucode_ptr')) as fd: + with open(self.TestFile('u_boot_ucode_ptr'), 'rb') as fd: TestFunctional._MakeInputFile('u-boot', fd.read()) def testMicrocodeNotInImage(self): @@ -993,7 +993,7 @@ class TestFunctional(unittest.TestCase): def testWithoutMicrocode(self): """Test that we can cope with an image without microcode (e.g. qemu)""" - with open(self.TestFile('u_boot_no_ucode_ptr')) as fd: + with open(self.TestFile('u_boot_no_ucode_ptr'), 'rb') as fd: TestFunctional._MakeInputFile('u-boot', fd.read()) data, dtb, _, _ = self._DoReadFileDtb('044_x86_optional_ucode.dts', True) @@ -1006,7 +1006,7 @@ class TestFunctional(unittest.TestCase): used_len = len(U_BOOT_NODTB_DATA) + fdt_len third = data[used_len:] - self.assertEqual(chr(0) * (0x200 - used_len), third) + self.assertEqual(tools.GetBytes(0, 0x200 - used_len), third) def testUnknownPosSize(self): """Test that microcode must be placed within the image""" @@ -1035,7 +1035,8 @@ class TestFunctional(unittest.TestCase): # ELF file with a '__bss_size' symbol self._SetupSplElf() data = self._DoReadFile('047_spl_bss_pad.dts') - self.assertEqual(U_BOOT_SPL_DATA + (chr(0) * 10) + U_BOOT_DATA, data) + self.assertEqual(U_BOOT_SPL_DATA + tools.GetBytes(0, 10) + U_BOOT_DATA, + data) def testSplBssPadMissing(self): """Test that a missing symbol is detected""" @@ -1067,8 +1068,8 @@ class TestFunctional(unittest.TestCase): self._SetupSplElf('u_boot_ucode_ptr') first, pos_and_size = self._RunMicrocodeTest(dts, U_BOOT_SPL_NODTB_DATA, ucode_second=ucode_second) - self.assertEqual('splnodtb with microc' + pos_and_size + - 'ter somewhere in here', first) + self.assertEqual(b'splnodtb with microc' + pos_and_size + + b'ter somewhere in here', first) def testPackUbootSplMicrocode(self): """Test that x86 microcode can be handled correctly in SPL""" @@ -1109,9 +1110,9 @@ class TestFunctional(unittest.TestCase): self._SetupSplElf('u_boot_binman_syms') data = self._DoReadFile('053_symbols.dts') sym_values = struct.pack('<LQL', 0x24 + 0, 0x24 + 24, 0x24 + 20) - expected = (sym_values + U_BOOT_SPL_DATA[16:] + chr(0xff) + - U_BOOT_DATA + - sym_values + U_BOOT_SPL_DATA[16:]) + expected = (sym_values + U_BOOT_SPL_DATA[16:] + + tools.GetBytes(0xff, 1) + U_BOOT_DATA + sym_values + + U_BOOT_SPL_DATA[16:]) self.assertEqual(expected, data) def testPackUnitAddress(self): @@ -1122,8 +1123,9 @@ class TestFunctional(unittest.TestCase): def testSections(self): """Basic test of sections""" data = self._DoReadFile('055_sections.dts') - expected = (U_BOOT_DATA + '!' * 12 + U_BOOT_DATA + 'a' * 12 + - U_BOOT_DATA + '&' * 4) + expected = (U_BOOT_DATA + tools.GetBytes(ord('!'), 12) + + U_BOOT_DATA + tools.GetBytes(ord('a'), 12) + + U_BOOT_DATA + tools.GetBytes(ord('&'), 4)) self.assertEqual(expected, data) def testMap(self): @@ -1281,8 +1283,10 @@ class TestFunctional(unittest.TestCase): } data, _, _, _ = self._DoReadFileDtb('066_text.dts', entry_args=entry_args) - expected = (TEXT_DATA + chr(0) * (8 - len(TEXT_DATA)) + TEXT_DATA2 + - TEXT_DATA3 + 'some text') + expected = (tools.ToBytes(TEXT_DATA) + + tools.GetBytes(0, 8 - len(TEXT_DATA)) + + tools.ToBytes(TEXT_DATA2) + tools.ToBytes(TEXT_DATA3) + + b'some text') self.assertEqual(expected, data) def testEntryDocs(self): @@ -1303,32 +1307,33 @@ class TestFunctional(unittest.TestCase): """Basic test of generation of a flashrom fmap""" data = self._DoReadFile('067_fmap.dts') fhdr, fentries = fmap_util.DecodeFmap(data[32:]) - expected = U_BOOT_DATA + '!' * 12 + U_BOOT_DATA + 'a' * 12 + expected = (U_BOOT_DATA + tools.GetBytes(ord('!'), 12) + + U_BOOT_DATA + tools.GetBytes(ord('a'), 12)) self.assertEqual(expected, data[:32]) - self.assertEqual('__FMAP__', fhdr.signature) + self.assertEqual(b'__FMAP__', fhdr.signature) self.assertEqual(1, fhdr.ver_major) self.assertEqual(0, fhdr.ver_minor) self.assertEqual(0, fhdr.base) self.assertEqual(16 + 16 + fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 3, fhdr.image_size) - self.assertEqual('FMAP', fhdr.name) + self.assertEqual(b'FMAP', fhdr.name) self.assertEqual(3, fhdr.nareas) for fentry in fentries: self.assertEqual(0, fentry.flags) self.assertEqual(0, fentries[0].offset) self.assertEqual(4, fentries[0].size) - self.assertEqual('RO_U_BOOT', fentries[0].name) + self.assertEqual(b'RO_U_BOOT', fentries[0].name) self.assertEqual(16, fentries[1].offset) self.assertEqual(4, fentries[1].size) - self.assertEqual('RW_U_BOOT', fentries[1].name) + self.assertEqual(b'RW_U_BOOT', fentries[1].name) self.assertEqual(32, fentries[2].offset) self.assertEqual(fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 3, fentries[2].size) - self.assertEqual('FMAP', fentries[2].name) + self.assertEqual(b'FMAP', fentries[2].name) def testBlobNamedByArg(self): """Test we can add a blob with the filename coming from an entry arg""" @@ -1341,7 +1346,7 @@ class TestFunctional(unittest.TestCase): def testFill(self): """Test for an fill entry type""" data = self._DoReadFile('069_fill.dts') - expected = 8 * chr(0xff) + 8 * chr(0) + expected = tools.GetBytes(0xff, 8) + tools.GetBytes(0, 8) self.assertEqual(expected, data) def testFillNoSize(self): @@ -1357,7 +1362,7 @@ class TestFunctional(unittest.TestCase): fname = pipe_list[0][-1] # Append our GBB data to the file, which will happen every time the # futility command is called. - with open(fname, 'a') as fd: + with open(fname, 'ab') as fd: fd.write(GBB_DATA) return command.CommandResult() @@ -1371,7 +1376,8 @@ class TestFunctional(unittest.TestCase): data, _, _, _ = self._DoReadFileDtb('071_gbb.dts', entry_args=entry_args) # Since futility - expected = GBB_DATA + GBB_DATA + 8 * chr(0) + (0x2180 - 16) * chr(0) + expected = (GBB_DATA + GBB_DATA + tools.GetBytes(0, 8) + + tools.GetBytes(0, 0x2180 - 16)) self.assertEqual(expected, data) def testGbbTooSmall(self): @@ -1431,7 +1437,7 @@ class TestFunctional(unittest.TestCase): def testTpl(self): """Test that an image with TPL and ots device tree can be created""" # ELF file with a '__bss_size' symbol - with open(self.TestFile('bss_data')) as fd: + with open(self.TestFile('bss_data'), 'rb') as fd: TestFunctional._MakeInputFile('tpl/u-boot-tpl', fd.read()) data = self._DoReadFile('078_u_boot_tpl.dts') self.assertEqual(U_BOOT_TPL_DATA + U_BOOT_TPL_DTB_DATA, data) @@ -1446,7 +1452,7 @@ class TestFunctional(unittest.TestCase): def testFillZero(self): """Test for an fill entry type with a size of 0""" data = self._DoReadFile('080_fill_empty.dts') - self.assertEqual(chr(0) * 16, data) + self.assertEqual(tools.GetBytes(0, 16), data) def testTextMissing(self): """Test for a text entry type where there is no text""" @@ -1557,7 +1563,7 @@ class TestFunctional(unittest.TestCase): out = os.path.join(self._indir, 'lz4.tmp') with open(out, 'wb') as fd: fd.write(data) - return tools.Run('lz4', '-dc', out) + return tools.Run('lz4', '-dc', out, binary=True) ''' try: orig = lz4.frame.decompress(data) @@ -1595,7 +1601,7 @@ class TestFunctional(unittest.TestCase): files = entries['files'] entries = files._section._entries - orig = '' + orig = b'' for i in range(1, 3): key = '%d.dat' % i start = entries[key].image_pos @@ -1623,10 +1629,10 @@ class TestFunctional(unittest.TestCase): """Test an expanding entry""" data, _, map_data, _ = self._DoReadFileDtb('088_expand_size.dts', map=True) - expect = ('a' * 8 + U_BOOT_DATA + - MRC_DATA + 'b' * 1 + U_BOOT_DATA + - 'c' * 8 + U_BOOT_DATA + - 'd' * 8) + expect = (tools.GetBytes(ord('a'), 8) + U_BOOT_DATA + + MRC_DATA + tools.GetBytes(ord('b'), 1) + U_BOOT_DATA + + tools.GetBytes(ord('c'), 8) + U_BOOT_DATA + + tools.GetBytes(ord('d'), 8)) self.assertEqual(expect, data) self.assertEqual('''ImagePos Offset Size Name 00000000 00000000 00000028 main-section @@ -1658,7 +1664,7 @@ class TestFunctional(unittest.TestCase): hash_node = dtb.GetNode('/binman/u-boot/hash').props['value'] m = hashlib.sha256() m.update(U_BOOT_DATA) - self.assertEqual(m.digest(), ''.join(hash_node.value)) + self.assertEqual(m.digest(), b''.join(hash_node.value)) def testHashNoAlgo(self): with self.assertRaises(ValueError) as e: @@ -1681,8 +1687,8 @@ class TestFunctional(unittest.TestCase): hash_node = dtb.GetNode('/binman/section/hash').props['value'] m = hashlib.sha256() m.update(U_BOOT_DATA) - m.update(16 * 'a') - self.assertEqual(m.digest(), ''.join(hash_node.value)) + m.update(tools.GetBytes(ord('a'), 16)) + self.assertEqual(m.digest(), b''.join(hash_node.value)) def testPackUBootTplMicrocode(self): """Test that x86 microcode can be handled correctly in TPL @@ -1693,18 +1699,18 @@ class TestFunctional(unittest.TestCase): u-boot-tpl.dtb with the microcode removed the microcode """ - with open(self.TestFile('u_boot_ucode_ptr')) as fd: + with open(self.TestFile('u_boot_ucode_ptr'), 'rb') as fd: TestFunctional._MakeInputFile('tpl/u-boot-tpl', fd.read()) first, pos_and_size = self._RunMicrocodeTest('093_x86_tpl_ucode.dts', U_BOOT_TPL_NODTB_DATA) - self.assertEqual('tplnodtb with microc' + pos_and_size + - 'ter somewhere in here', first) + self.assertEqual(b'tplnodtb with microc' + pos_and_size + + b'ter somewhere in here', first) def testFmapX86(self): """Basic test of generation of a flashrom fmap""" data = self._DoReadFile('094_fmap_x86.dts') fhdr, fentries = fmap_util.DecodeFmap(data[32:]) - expected = U_BOOT_DATA + MRC_DATA + 'a' * (32 - 7) + expected = U_BOOT_DATA + MRC_DATA + tools.GetBytes(ord('a'), 32 - 7) self.assertEqual(expected, data[:32]) fhdr, fentries = fmap_util.DecodeFmap(data[32:]) @@ -1712,21 +1718,21 @@ class TestFunctional(unittest.TestCase): self.assertEqual(0, fentries[0].offset) self.assertEqual(4, fentries[0].size) - self.assertEqual('U_BOOT', fentries[0].name) + self.assertEqual(b'U_BOOT', fentries[0].name) self.assertEqual(4, fentries[1].offset) self.assertEqual(3, fentries[1].size) - self.assertEqual('INTEL_MRC', fentries[1].name) + self.assertEqual(b'INTEL_MRC', fentries[1].name) self.assertEqual(32, fentries[2].offset) self.assertEqual(fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 3, fentries[2].size) - self.assertEqual('FMAP', fentries[2].name) + self.assertEqual(b'FMAP', fentries[2].name) def testFmapX86Section(self): """Basic test of generation of a flashrom fmap""" data = self._DoReadFile('095_fmap_x86_section.dts') - expected = U_BOOT_DATA + MRC_DATA + 'b' * (32 - 7) + expected = U_BOOT_DATA + MRC_DATA + tools.GetBytes(ord('b'), 32 - 7) self.assertEqual(expected, data[:32]) fhdr, fentries = fmap_util.DecodeFmap(data[36:]) @@ -1734,28 +1740,28 @@ class TestFunctional(unittest.TestCase): self.assertEqual(0, fentries[0].offset) self.assertEqual(4, fentries[0].size) - self.assertEqual('U_BOOT', fentries[0].name) + self.assertEqual(b'U_BOOT', fentries[0].name) self.assertEqual(4, fentries[1].offset) self.assertEqual(3, fentries[1].size) - self.assertEqual('INTEL_MRC', fentries[1].name) + self.assertEqual(b'INTEL_MRC', fentries[1].name) self.assertEqual(36, fentries[2].offset) self.assertEqual(fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 3, fentries[2].size) - self.assertEqual('FMAP', fentries[2].name) + self.assertEqual(b'FMAP', fentries[2].name) def testElf(self): """Basic test of ELF entries""" self._SetupSplElf() - with open(self.TestFile('bss_data')) as fd: + with open(self.TestFile('bss_data'), 'rb') as fd: TestFunctional._MakeInputFile('-boot', fd.read()) data = self._DoReadFile('096_elf.dts') def testElfStripg(self): """Basic test of ELF entries""" self._SetupSplElf() - with open(self.TestFile('bss_data')) as fd: + with open(self.TestFile('bss_data'), 'rb') as fd: TestFunctional._MakeInputFile('-boot', fd.read()) data = self._DoReadFile('097_elf_strip.dts') @@ -1771,7 +1777,7 @@ class TestFunctional(unittest.TestCase): # We should not get an inmage, but there should be a map file self.assertFalse(os.path.exists(tools.GetOutputFilename('image.bin'))) self.assertTrue(os.path.exists(map_fname)) - map_data = tools.ReadFile(map_fname) + map_data = tools.ReadFile(map_fname, binary=False) self.assertEqual('''ImagePos Offset Size Name <none> 00000000 00000007 main-section <none> 00000000 00000004 u-boot @@ -1797,9 +1803,12 @@ class TestFunctional(unittest.TestCase): 0000002c 00000000 00000004 u-boot ''', map_data) self.assertEqual(data, - 4 * chr(0x26) + U_BOOT_DATA + 12 * chr(0x21) + - 4 * chr(0x26) + U_BOOT_DATA + 12 * chr(0x61) + - 4 * chr(0x26) + U_BOOT_DATA + 8 * chr(0x26)) + tools.GetBytes(0x26, 4) + U_BOOT_DATA + + tools.GetBytes(0x21, 12) + + tools.GetBytes(0x26, 4) + U_BOOT_DATA + + tools.GetBytes(0x61, 12) + + tools.GetBytes(0x26, 4) + U_BOOT_DATA + + tools.GetBytes(0x26, 8)) if __name__ == "__main__": diff --git a/tools/binman/state.py b/tools/binman/state.py index d945e4bf65..af9678649c 100644 --- a/tools/binman/state.py +++ b/tools/binman/state.py @@ -7,7 +7,6 @@ import hashlib import re -from sets import Set import os import tools @@ -24,10 +23,10 @@ entry_args = {} use_fake_dtb = False # Set of all device tree files references by images -fdt_set = Set() +fdt_set = set() # Same as above, but excluding the main one -fdt_subset = Set() +fdt_subset = set() # The DTB which contains the full image information main_dtb = None @@ -136,7 +135,7 @@ def Prepare(images, dtb): main_dtb = dtb fdt_files.clear() fdt_files['u-boot.dtb'] = dtb - fdt_subset = Set() + fdt_subset = set() if not use_fake_dtb: for image in images.values(): fdt_subset.update(image.GetFdtSet()) diff --git a/tools/dtoc/dtb_platdata.py b/tools/dtoc/dtb_platdata.py index 17a3dccb11..037e82c8bb 100644 --- a/tools/dtoc/dtb_platdata.py +++ b/tools/dtoc/dtb_platdata.py @@ -17,6 +17,7 @@ import sys import fdt import fdt_util +import tools # When we see these properties we ignore them - i.e. do not create a structure member PROP_IGNORE_LIST = [ @@ -99,7 +100,7 @@ def get_value(ftype, value): if ftype == fdt.TYPE_INT: return '%#x' % fdt_util.fdt32_to_cpu(value) elif ftype == fdt.TYPE_BYTE: - return '%#x' % ord(value[0]) + return '%#x' % tools.ToByte(value[0]) elif ftype == fdt.TYPE_STRING: return '"%s"' % value elif ftype == fdt.TYPE_BOOL: @@ -449,7 +450,7 @@ class DtbPlatdata(object): self.out(';\n') self.out('};\n') - for alias, struct_name in self._aliases.iteritems(): + for alias, struct_name in self._aliases.items(): if alias not in sorted(structs): self.out('#define %s%s %s%s\n'% (STRUCT_PREFIX, alias, STRUCT_PREFIX, struct_name)) @@ -464,7 +465,8 @@ class DtbPlatdata(object): var_name = conv_name_to_c(node.name) self.buf('static const struct %s%s %s%s = {\n' % (STRUCT_PREFIX, struct_name, VAL_PREFIX, var_name)) - for pname, prop in node.props.items(): + for pname in sorted(node.props): + prop = node.props[pname] if pname in PROP_IGNORE_LIST or pname[0] == '#': continue member_name = conv_name_to_c(prop.name) @@ -498,7 +500,7 @@ class DtbPlatdata(object): vals.append(get_value(prop.type, val)) # Put 8 values per line to avoid very long lines. - for i in xrange(0, len(vals), 8): + for i in range(0, len(vals), 8): if i: self.buf(',\n\t\t') self.buf(', '.join(vals[i:i + 8])) diff --git a/tools/dtoc/dtoc.py b/tools/dtoc/dtoc.py index 2277af9bf7..c1a1d3534d 100755 --- a/tools/dtoc/dtoc.py +++ b/tools/dtoc/dtoc.py @@ -25,6 +25,8 @@ options. For more information about the use of this options and tool please see doc/driver-model/of-plat.txt """ +from __future__ import print_function + from optparse import OptionParser import os import sys @@ -64,11 +66,11 @@ def run_tests(args): suite = unittest.TestLoader().loadTestsFromTestCase(module) suite.run(result) - print result + print(result) for _, err in result.errors: - print err + print(err) for _, err in result.failures: - print err + print(err) def RunTestCoverage(): """Run the tests and check that we get 100% coverage""" diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 9ad72f89ec..d9471c4381 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -11,6 +11,7 @@ import sys import fdt_util import libfdt from libfdt import QUIET_NOTFOUND +import tools # This deals with a device tree, presenting it as an assortment of Node and # Prop objects, representing nodes and properties, respectively. This file @@ -28,6 +29,66 @@ def CheckErr(errnum, msg): raise ValueError('Error %d: %s: %s' % (errnum, libfdt.fdt_strerror(errnum), msg)) + +def BytesToValue(data): + """Converts a string of bytes into a type and value + + Args: + A bytes value (which on Python 2 is an alias for str) + + Return: + A tuple: + Type of data + Data, either a single element or a list of elements. Each element + is one of: + TYPE_STRING: str/bytes value from the property + TYPE_INT: a byte-swapped integer stored as a 4-byte str/bytes + TYPE_BYTE: a byte stored as a single-byte str/bytes + """ + data = bytes(data) + size = len(data) + strings = data.split(b'\0') + is_string = True + count = len(strings) - 1 + if count > 0 and not len(strings[-1]): + for string in strings[:-1]: + if not string: + is_string = False + break + for ch in string: + # Handle Python 2 treating bytes as str + if type(ch) == str: + ch = ord(ch) + if ch < 32 or ch > 127: + is_string = False + break + else: + is_string = False + if is_string: + if count == 1: + if sys.version_info[0] >= 3: # pragma: no cover + return TYPE_STRING, strings[0].decode() + else: + return TYPE_STRING, strings[0] + else: + if sys.version_info[0] >= 3: # pragma: no cover + return TYPE_STRING, [s.decode() for s in strings[:-1]] + else: + return TYPE_STRING, strings[:-1] + if size % 4: + if size == 1: + return TYPE_BYTE, tools.ToChar(data[0]) + else: + return TYPE_BYTE, [tools.ToChar(ch) for ch in list(data)] + val = [] + for i in range(0, size, 4): + val.append(data[i:i + 4]) + if size == 4: + return TYPE_INT, val[0] + else: + return TYPE_INT, val + + class Prop: """A device tree property @@ -37,18 +98,18 @@ class Prop: bytes type: Value type """ - def __init__(self, node, offset, name, bytes): + def __init__(self, node, offset, name, data): self._node = node self._offset = offset self.name = name self.value = None - self.bytes = str(bytes) + self.bytes = bytes(data) self.dirty = False - if not bytes: + if not data: self.type = TYPE_BOOL self.value = True return - self.type, self.value = self.BytesToValue(bytes) + self.type, self.value = BytesToValue(bytes(data)) def RefreshOffset(self, poffset): self._offset = poffset @@ -87,55 +148,6 @@ class Prop: while len(self.value) < len(newprop.value): self.value.append(val) - def BytesToValue(self, bytes): - """Converts a string of bytes into a type and value - - Args: - A string containing bytes - - Return: - A tuple: - Type of data - Data, either a single element or a list of elements. Each element - is one of: - TYPE_STRING: string value from the property - TYPE_INT: a byte-swapped integer stored as a 4-byte string - TYPE_BYTE: a byte stored as a single-byte string - """ - bytes = str(bytes) - size = len(bytes) - strings = bytes.split('\0') - is_string = True - count = len(strings) - 1 - if count > 0 and not strings[-1]: - for string in strings[:-1]: - if not string: - is_string = False - break - for ch in string: - if ch < ' ' or ch > '~': - is_string = False - break - else: - is_string = False - if is_string: - if count == 1: - return TYPE_STRING, strings[0] - else: - return TYPE_STRING, strings[:-1] - if size % 4: - if size == 1: - return TYPE_BYTE, bytes[0] - else: - return TYPE_BYTE, list(bytes) - val = [] - for i in range(0, size, 4): - val.append(bytes[i:i + 4]) - if size == 4: - return TYPE_INT, val[0] - else: - return TYPE_INT, val - @classmethod def GetEmpty(self, type): """Get an empty / zero value of the given type @@ -181,8 +193,8 @@ class Prop: Args: bytes: New property value to set """ - self.bytes = str(bytes) - self.type, self.value = self.BytesToValue(bytes) + self.bytes = bytes + self.type, self.value = BytesToValue(bytes) self.dirty = True def Sync(self, auto_resize=False): @@ -334,7 +346,8 @@ class Node: Args: prop_name: Name of property """ - self.props[prop_name] = Prop(self, None, prop_name, '\0' * 4) + self.props[prop_name] = Prop(self, None, prop_name, + tools.GetBytes(0, 4)) def AddEmptyProp(self, prop_name, len): """Add a property with a fixed data size, for filling in later @@ -346,7 +359,7 @@ class Node: prop_name: Name of property len: Length of data in property """ - value = chr(0) * len + value = tools.GetBytes(0, len) self.props[prop_name] = Prop(self, None, prop_name, value) def SetInt(self, prop_name, val): @@ -385,7 +398,9 @@ class Node: prop_name: Name of property to set val: String value to set (will be \0-terminated in DT) """ - self.props[prop_name].SetData(val + chr(0)) + if sys.version_info[0] >= 3: # pragma: no cover + val = bytes(val, 'utf-8') + self.props[prop_name].SetData(val + b'\0') def AddString(self, prop_name, val): """Add a new string property to a node @@ -397,7 +412,9 @@ class Node: prop_name: Name of property to add val: String value of property """ - self.props[prop_name] = Prop(self, None, prop_name, val + chr(0)) + if sys.version_info[0] >= 3: # pragma: no cover + val = bytes(val, 'utf-8') + self.props[prop_name] = Prop(self, None, prop_name, val + b'\0') def AddSubnode(self, name): """Add a new subnode to the node @@ -448,8 +465,11 @@ class Node: # Sync properties now, whose offsets should not have been disturbed. # We do this after subnodes, since this disturbs the offsets of these - # properties. - prop_list = sorted(self.props.values(), key=lambda prop: prop._offset, + # properties. Note that new properties will have an offset of None here, + # which Python 3 cannot sort against int. So use a large value instead + # to ensure that the new properties are added first. + prop_list = sorted(self.props.values(), + key=lambda prop: prop._offset or 1 << 31, reverse=True) for prop in prop_list: prop.Sync(auto_resize) @@ -469,7 +489,7 @@ class Fdt: if self._fname: self._fname = fdt_util.EnsureCompiled(self._fname) - with open(self._fname) as fd: + with open(self._fname, 'rb') as fd: self._fdt_obj = libfdt.Fdt(fd.read()) @staticmethod @@ -483,7 +503,7 @@ class Fdt: Fdt object containing the data """ fdt = Fdt(None) - fdt._fdt_obj = libfdt.Fdt(bytearray(data)) + fdt._fdt_obj = libfdt.Fdt(bytes(data)) return fdt def LookupPhandle(self, phandle): @@ -573,7 +593,7 @@ class Fdt: Returns: The FDT contents as a string of bytes """ - return self._fdt_obj.as_bytearray() + return bytes(self._fdt_obj.as_bytearray()) def GetFdtObj(self): """Get the contents of the FDT diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index 5fbfc8877b..f47879ac00 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -16,14 +16,6 @@ import tempfile import command import tools -VERSION3 = sys.version_info > (3, 0) - -def get_plain_bytes(val): - """Handle Python 3 strings""" - if isinstance(val, bytes): - val = val.decode('utf-8') - return val.encode('raw_unicode_escape') - def fdt32_to_cpu(val): """Convert a device tree cell to an integer @@ -33,9 +25,6 @@ def fdt32_to_cpu(val): Return: A native-endian integer value """ - if VERSION3: - # This code is not reached in Python 2 - val = get_plain_bytes(val) # pragma: no cover return struct.unpack('>I', val)[0] def fdt_cells_to_cpu(val, cells): @@ -45,11 +34,11 @@ def fdt_cells_to_cpu(val, cells): Value to convert (array of one or more 4-character strings) Return: - A native-endian long value + A native-endian integer value """ if not cells: return 0 - out = long(fdt32_to_cpu(val[0])) + out = int(fdt32_to_cpu(val[0])) if cells == 2: out = out << 32 | fdt32_to_cpu(val[1]) return out diff --git a/tools/dtoc/test_dtoc.py b/tools/dtoc/test_dtoc.py index cb6d6e7baf..b915b27856 100644 --- a/tools/dtoc/test_dtoc.py +++ b/tools/dtoc/test_dtoc.py @@ -8,6 +8,8 @@ This includes unit tests for some functions and functional tests for the dtoc tool. """ +from __future__ import print_function + import collections import os import struct @@ -97,7 +99,7 @@ class TestDtoc(unittest.TestCase): if expected != actual: self._WritePythonString('/tmp/binman.expected', expected) self._WritePythonString('/tmp/binman.actual', actual) - print 'Failures written to /tmp/binman.{expected,actual}' + print('Failures written to /tmp/binman.{expected,actual}') self.assertEquals(expected, actual) def test_name(self): @@ -197,16 +199,16 @@ struct dtd_sandbox_spl_test_2 { data = infile.read() self._CheckStrings(C_HEADER + ''' static const struct dtd_sandbox_spl_test dtv_spl_test = { +\t.boolval\t\t= true, \t.bytearray\t\t= {0x6, 0x0, 0x0}, \t.byteval\t\t= 0x5, +\t.intarray\t\t= {0x2, 0x3, 0x4, 0x0}, \t.intval\t\t\t= 0x1, -\t.notstring\t\t= {0x20, 0x21, 0x22, 0x10, 0x0}, \t.longbytearray\t\t= {0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x10, \t\t0x11}, -\t.stringval\t\t= "message", -\t.boolval\t\t= true, -\t.intarray\t\t= {0x2, 0x3, 0x4, 0x0}, +\t.notstring\t\t= {0x20, 0x21, 0x22, 0x10, 0x0}, \t.stringarray\t\t= {"multi-word", "message", ""}, +\t.stringval\t\t= "message", }; U_BOOT_DEVICE(spl_test) = { \t.name\t\t= "sandbox_spl_test", @@ -217,12 +219,12 @@ U_BOOT_DEVICE(spl_test) = { static const struct dtd_sandbox_spl_test dtv_spl_test2 = { \t.bytearray\t\t= {0x1, 0x23, 0x34}, \t.byteval\t\t= 0x8, +\t.intarray\t\t= {0x5, 0x0, 0x0, 0x0}, \t.intval\t\t\t= 0x3, \t.longbytearray\t\t= {0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, \t\t0x0}, -\t.stringval\t\t= "message2", -\t.intarray\t\t= {0x5, 0x0, 0x0, 0x0}, \t.stringarray\t\t= {"another", "multi-word", "message"}, +\t.stringval\t\t= "message2", }; U_BOOT_DEVICE(spl_test2) = { \t.name\t\t= "sandbox_spl_test", diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index 8d70dd2a29..bf469dbd54 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -4,6 +4,8 @@ # Written by Simon Glass <sjg@chromium.org> # +from __future__ import print_function + from optparse import OptionParser import glob import os @@ -17,7 +19,7 @@ for dirname in ['../patman', '..']: import command import fdt -from fdt import TYPE_BYTE, TYPE_INT, TYPE_STRING, TYPE_BOOL +from fdt import TYPE_BYTE, TYPE_INT, TYPE_STRING, TYPE_BOOL, BytesToValue import fdt_util from fdt_util import fdt32_to_cpu import libfdt @@ -45,7 +47,7 @@ def _GetPropertyValue(dtb, node, prop_name): # Add 12, which is sizeof(struct fdt_property), to get to start of data offset = prop.GetOffset() + 12 data = dtb.GetContents()[offset:offset + len(prop.value)] - return prop, [chr(x) for x in data] + return prop, [tools.ToChar(x) for x in data] class TestFdt(unittest.TestCase): @@ -83,13 +85,13 @@ class TestFdt(unittest.TestCase): def testFlush(self): """Check that we can flush the device tree out to its file""" fname = self.dtb._fname - with open(fname) as fd: + with open(fname, 'rb') as fd: data = fd.read() os.remove(fname) with self.assertRaises(IOError): - open(fname) + open(fname, 'rb') self.dtb.Flush() - with open(fname) as fd: + with open(fname, 'rb') as fd: data = fd.read() def testPack(self): @@ -119,6 +121,10 @@ class TestFdt(unittest.TestCase): node = self.dtb.GetNode('/spl-test') self.assertEqual(self.dtb, node.GetFdt()) + def testBytesToValue(self): + self.assertEqual(BytesToValue(b'this\0is\0'), + (TYPE_STRING, ['this', 'is'])) + class TestNode(unittest.TestCase): """Test operation of the Node class""" @@ -277,7 +283,7 @@ class TestProp(unittest.TestCase): """Tests the GetEmpty() function for the various supported types""" self.assertEqual(True, fdt.Prop.GetEmpty(fdt.TYPE_BOOL)) self.assertEqual(chr(0), fdt.Prop.GetEmpty(fdt.TYPE_BYTE)) - self.assertEqual(chr(0) * 4, fdt.Prop.GetEmpty(fdt.TYPE_INT)) + self.assertEqual(tools.GetBytes(0, 4), fdt.Prop.GetEmpty(fdt.TYPE_INT)) self.assertEqual('', fdt.Prop.GetEmpty(fdt.TYPE_STRING)) def testGetOffset(self): @@ -381,7 +387,7 @@ class TestProp(unittest.TestCase): self.node.AddString('string', val) self.dtb.Sync(auto_resize=True) data = self.fdt.getprop(self.node.Offset(), 'string') - self.assertEqual(val + '\0', data) + self.assertEqual(tools.ToBytes(val) + b'\0', data) self.fdt.pack() self.node.SetString('string', val + 'x') @@ -391,21 +397,21 @@ class TestProp(unittest.TestCase): self.node.SetString('string', val[:-1]) prop = self.node.props['string'] - prop.SetData(val) + prop.SetData(tools.ToBytes(val)) self.dtb.Sync(auto_resize=False) data = self.fdt.getprop(self.node.Offset(), 'string') - self.assertEqual(val, data) + self.assertEqual(tools.ToBytes(val), data) self.node.AddEmptyProp('empty', 5) self.dtb.Sync(auto_resize=True) prop = self.node.props['empty'] - prop.SetData(val) + prop.SetData(tools.ToBytes(val)) self.dtb.Sync(auto_resize=False) data = self.fdt.getprop(self.node.Offset(), 'empty') - self.assertEqual(val, data) + self.assertEqual(tools.ToBytes(val), data) - self.node.SetData('empty', '123') - self.assertEqual('123', prop.bytes) + self.node.SetData('empty', b'123') + self.assertEqual(b'123', prop.bytes) def testFromData(self): dtb2 = fdt.Fdt.FromData(self.dtb.GetContents()) @@ -496,18 +502,22 @@ class TestFdtUtil(unittest.TestCase): self.assertEqual(2, fdt_util.fdt_cells_to_cpu(val, 1)) dtb2 = fdt.FdtScan('tools/dtoc/dtoc_test_addr64.dts') - node2 = dtb2.GetNode('/test1') - val = node2.props['reg'].value + node1 = dtb2.GetNode('/test1') + val = node1.props['reg'].value self.assertEqual(0x1234, fdt_util.fdt_cells_to_cpu(val, 2)) + node2 = dtb2.GetNode('/test2') + val = node2.props['reg'].value + self.assertEqual(0x1234567890123456, fdt_util.fdt_cells_to_cpu(val, 2)) + self.assertEqual(0x9876543210987654, fdt_util.fdt_cells_to_cpu(val[2:], + 2)) + self.assertEqual(0x12345678, fdt_util.fdt_cells_to_cpu(val, 1)) + def testEnsureCompiled(self): """Test a degenerate case of this function""" dtb = fdt_util.EnsureCompiled('tools/dtoc/dtoc_test_simple.dts') self.assertEqual(dtb, fdt_util.EnsureCompiled(dtb)) - def testGetPlainBytes(self): - self.assertEqual('fred', fdt_util.get_plain_bytes('fred')) - def RunTestCoverage(): """Run the tests and check that we get 100% coverage""" @@ -535,11 +545,11 @@ def RunTests(args): suite = unittest.TestLoader().loadTestsFromTestCase(module) suite.run(result) - print result + print(result) for _, err in result.errors: - print err + print(err) for _, err in result.failures: - print err + print(err) if __name__ != '__main__': sys.exit(1) diff --git a/tools/patman/cros_subprocess.py b/tools/patman/cros_subprocess.py index ebd4300dfd..06be64cc2c 100644 --- a/tools/patman/cros_subprocess.py +++ b/tools/patman/cros_subprocess.py @@ -54,7 +54,7 @@ class Popen(subprocess.Popen): """ def __init__(self, args, stdin=None, stdout=PIPE_PTY, stderr=PIPE_PTY, - shell=False, cwd=None, env=None, **kwargs): + shell=False, cwd=None, env=None, binary=False, **kwargs): """Cut-down constructor Args: @@ -72,6 +72,7 @@ class Popen(subprocess.Popen): """ stdout_pty = None stderr_pty = None + self.binary = binary if stdout == PIPE_PTY: stdout_pty = pty.openpty() @@ -100,6 +101,19 @@ class Popen(subprocess.Popen): if kwargs: raise ValueError("Unit tests do not test extra args - please add tests") + def ConvertData(self, data): + """Convert stdout/stderr data to the correct format for output + + Args: + data: Data to convert, or None for '' + + Returns: + Converted data, as bytes + """ + if data is None: + return b'' + return data + def CommunicateFilter(self, output): """Interact with process: Read data from stdout and stderr. @@ -156,11 +170,11 @@ class Popen(subprocess.Popen): self.stdin.close() if self.stdout: read_set.append(self.stdout) - stdout = [] + stdout = b'' if self.stderr and self.stderr != self.stdout: read_set.append(self.stderr) - stderr = [] - combined = [] + stderr = b'' + combined = b'' input_offset = 0 while read_set or write_set: @@ -186,46 +200,40 @@ class Popen(subprocess.Popen): write_set.remove(self.stdin) if self.stdout in rlist: - data = "" + data = b'' # We will get an error on read if the pty is closed try: data = os.read(self.stdout.fileno(), 1024) except OSError: pass - if data == "": + if not len(data): self.stdout.close() read_set.remove(self.stdout) else: - stdout.append(data) - combined.append(data) + stdout += data + combined += data if output: output(sys.stdout, data) if self.stderr in rlist: - data = "" + data = b'' # We will get an error on read if the pty is closed try: data = os.read(self.stderr.fileno(), 1024) except OSError: pass - if data == "": + if not len(data): self.stderr.close() read_set.remove(self.stderr) else: - stderr.append(data) - combined.append(data) + stderr += data + combined += data if output: output(sys.stderr, data) # All data exchanged. Translate lists into strings. - if stdout is not None: - stdout = ''.join(stdout) - else: - stdout = '' - if stderr is not None: - stderr = ''.join(stderr) - else: - stderr = '' - combined = ''.join(combined) + stdout = self.ConvertData(stdout) + stderr = self.ConvertData(stderr) + combined = self.ConvertData(combined) # Translate newlines, if requested. We cannot let the file # object do the translation: It is based on stdio, which is diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index d79e716074..50a2741439 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -12,15 +12,20 @@ import sys import tempfile import unittest +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + import gitutil import patchstream import settings +import tools @contextlib.contextmanager def capture(): import sys - from cStringIO import StringIO oldout,olderr = sys.stdout, sys.stderr try: out=[StringIO(), StringIO()] @@ -124,10 +129,10 @@ class TestFunctional(unittest.TestCase): """ process_tags = True ignore_bad_tags = True - stefan = u'Stefan Brüns <stefan.bruens@rwth-aachen.de>' + stefan = b'Stefan Br\xc3\xbcns <stefan.bruens@rwth-aachen.de>'.decode('utf-8') rick = 'Richard III <richard@palace.gov>' - mel = u'Lord Mëlchett <clergy@palace.gov>' - ed = u'Lond Edmund Blackaddër <weasel@blackadder.org' + mel = b'Lord M\xc3\xablchett <clergy@palace.gov>'.decode('utf-8') + ed = b'Lond Edmund Blackadd\xc3\xabr <weasel@blackadder.org'.decode('utf-8') fred = 'Fred Bloggs <f.bloggs@napier.net>' add_maintainers = [stefan, rick] dry_run = True @@ -159,7 +164,6 @@ class TestFunctional(unittest.TestCase): os.remove(cc_file) lines = out[0].splitlines() - #print '\n'.join(lines) self.assertEqual('Cleaned %s patches' % len(series.commits), lines[0]) self.assertEqual('Change log missing for v2', lines[1]) self.assertEqual('Change log missing for v3', lines[2]) @@ -174,27 +178,30 @@ class TestFunctional(unittest.TestCase): while 'Cc:' in lines[line]: line += 1 self.assertEqual('To: u-boot@lists.denx.de', lines[line]) - self.assertEqual('Cc: %s' % stefan.encode('utf-8'), lines[line + 1]) + self.assertEqual('Cc: %s' % tools.FromUnicode(stefan), + lines[line + 1]) self.assertEqual('Version: 3', lines[line + 2]) self.assertEqual('Prefix:\t RFC', lines[line + 3]) self.assertEqual('Cover: 4 lines', lines[line + 4]) line += 5 - self.assertEqual(' Cc: %s' % mel.encode('utf-8'), lines[line + 0]) - self.assertEqual(' Cc: %s' % rick, lines[line + 1]) - self.assertEqual(' Cc: %s' % fred, lines[line + 2]) - self.assertEqual(' Cc: %s' % ed.encode('utf-8'), lines[line + 3]) + self.assertEqual(' Cc: %s' % fred, lines[line + 0]) + self.assertEqual(' Cc: %s' % tools.FromUnicode(ed), + lines[line + 1]) + self.assertEqual(' Cc: %s' % tools.FromUnicode(mel), + lines[line + 2]) + self.assertEqual(' Cc: %s' % rick, lines[line + 3]) expected = ('Git command: git send-email --annotate ' '--in-reply-to="%s" --to "u-boot@lists.denx.de" ' '--cc "%s" --cc-cmd "%s --cc-cmd %s" %s %s' % (in_reply_to, stefan, sys.argv[0], cc_file, cover_fname, - ' '.join(args))).encode('utf-8') + ' '.join(args))) line += 4 - self.assertEqual(expected, lines[line]) + self.assertEqual(expected, tools.ToUnicode(lines[line])) - self.assertEqual(('%s %s, %s' % (args[0], rick, stefan)) - .encode('utf-8'), cc_lines[0]) - self.assertEqual(('%s %s, %s, %s, %s' % (args[1], fred, rick, stefan, - ed)).encode('utf-8'), cc_lines[1]) + self.assertEqual(('%s %s, %s' % (args[0], rick, stefan)), + tools.ToUnicode(cc_lines[0])) + self.assertEqual(('%s %s, %s, %s, %s' % (args[1], fred, ed, rick, + stefan)), tools.ToUnicode(cc_lines[1])) expected = ''' This is a test of how the cover @@ -223,7 +230,6 @@ Simon Glass (2): ''' lines = open(cover_fname).read().splitlines() - #print '\n'.join(lines) self.assertEqual( 'Subject: [RFC PATCH v3 0/2] test: A test patch series', lines[3]) @@ -231,7 +237,6 @@ Simon Glass (2): for i, fname in enumerate(args): lines = open(fname).read().splitlines() - #print '\n'.join(lines) subject = [line for line in lines if line.startswith('Subject')] self.assertEqual('Subject: [RFC %d/%d]' % (i + 1, count), subject[0][:18]) diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index 9905bb0bbd..dce7fa25b6 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -12,6 +12,7 @@ import terminal import checkpatch import settings +import tools # True to use --no-decorate - we check this in Setup() use_no_decorate = True @@ -325,6 +326,7 @@ def BuildEmailList(in_list, tag=None, alias=None, raise_on_error=True): raw += LookupEmail(item, alias, raise_on_error=raise_on_error) result = [] for item in raw: + item = tools.FromUnicode(item) if not item in result: result.append(item) if tag: @@ -395,11 +397,11 @@ def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname, git_config_to = command.Output('git', 'config', 'sendemail.to', raise_on_error=False) if not git_config_to: - print ("No recipient.\n" - "Please add something like this to a commit\n" - "Series-to: Fred Bloggs <f.blogs@napier.co.nz>\n" - "Or do something like this\n" - "git config sendemail.to u-boot@lists.denx.de") + print("No recipient.\n" + "Please add something like this to a commit\n" + "Series-to: Fred Bloggs <f.blogs@napier.co.nz>\n" + "Or do something like this\n" + "git config sendemail.to u-boot@lists.denx.de") return cc = BuildEmailList(list(set(series.get('cc')) - set(series.get('to'))), '--cc', alias, raise_on_error) @@ -410,9 +412,7 @@ def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname, if smtp_server: cmd.append('--smtp-server=%s' % smtp_server) if in_reply_to: - if type(in_reply_to) != str: - in_reply_to = in_reply_to.encode('utf-8') - cmd.append('--in-reply-to="%s"' % in_reply_to) + cmd.append('--in-reply-to="%s"' % tools.FromUnicode(in_reply_to)) if thread: cmd.append('--thread') diff --git a/tools/patman/patman.py b/tools/patman/patman.py index 27a2febf70..9605a36eff 100755 --- a/tools/patman/patman.py +++ b/tools/patman/patman.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # SPDX-License-Identifier: GPL-2.0+ # # Copyright (c) 2011 The Chromium OS Authors. diff --git a/tools/patman/series.py b/tools/patman/series.py index 2735afaf88..67103f03e6 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -11,6 +11,7 @@ import get_maintainer import gitutil import settings import terminal +import tools # Series-xxx tags that we understand valid_series = ['to', 'cc', 'version', 'changes', 'prefix', 'notes', 'name', @@ -114,16 +115,16 @@ class Series(dict): commit = self.commits[upto] print(col.Color(col.GREEN, ' %s' % args[upto])) cc_list = list(self._generated_cc[commit.patch]) - for email in set(cc_list) - to_set - cc_set: + for email in sorted(set(cc_list) - to_set - cc_set): if email == None: email = col.Color(col.YELLOW, "<alias '%s' not found>" % tag) if email: print(' Cc: ', email) print - for item in to_set: + for item in sorted(to_set): print('To:\t ', item) - for item in cc_set - to_set: + for item in sorted(cc_set - to_set): print('Cc:\t ', item) print('Version: ', self.get('version')) print('Prefix:\t ', self.get('prefix')) @@ -131,7 +132,7 @@ class Series(dict): print('Cover: %d lines' % len(self.cover)) cover_cc = gitutil.BuildEmailList(self.get('cover_cc', '')) all_ccs = itertools.chain(cover_cc, *self._generated_cc.values()) - for email in set(all_ccs) - to_set - cc_set: + for email in sorted(set(all_ccs) - to_set - cc_set): print(' Cc: ', email) if cmd: print('Git command: %s' % cmd) @@ -238,19 +239,18 @@ class Series(dict): for x in set(cc) & set(settings.bounces): print(col.Color(col.YELLOW, 'Skipping "%s"' % x)) cc = set(cc) - set(settings.bounces) - cc = [m.encode('utf-8') if type(m) != str else m for m in cc] + cc = [tools.FromUnicode(m) for m in cc] if limit is not None: cc = cc[:limit] all_ccs += cc - print(commit.patch, ', '.join(set(cc)), file=fd) + print(commit.patch, ', '.join(sorted(set(cc))), file=fd) self._generated_cc[commit.patch] = cc if cover_fname: cover_cc = gitutil.BuildEmailList(self.get('cover_cc', '')) - cover_cc = [m.encode('utf-8') if type(m) != str else m - for m in cover_cc] - cc_list = ', '.join([x.decode('utf-8') - for x in set(cover_cc + all_ccs)]) + cover_cc = [tools.FromUnicode(m) for m in cover_cc] + cc_list = ', '.join([tools.ToUnicode(x) + for x in sorted(set(cover_cc + all_ccs))]) print(cover_fname, cc_list.encode('utf-8'), file=fd) fd.close() diff --git a/tools/patman/settings.py b/tools/patman/settings.py index ea2bc74f75..c98911d522 100644 --- a/tools/patman/settings.py +++ b/tools/patman/settings.py @@ -14,6 +14,7 @@ import re import command import gitutil +import tools """Default settings per-project. @@ -57,26 +58,26 @@ class _ProjectConfigParser(ConfigParser.SafeConfigParser): # Check to make sure that bogus project gets general alias. >>> config = _ProjectConfigParser("zzz") >>> config.readfp(StringIO(sample_config)) - >>> config.get("alias", "enemies") - u'Evil <evil@example.com>' + >>> str(config.get("alias", "enemies")) + 'Evil <evil@example.com>' # Check to make sure that alias gets overridden by project. >>> config = _ProjectConfigParser("sm") >>> config.readfp(StringIO(sample_config)) - >>> config.get("alias", "enemies") - u'Green G. <ugly@example.com>' + >>> str(config.get("alias", "enemies")) + 'Green G. <ugly@example.com>' # Check to make sure that settings get merged with project. >>> config = _ProjectConfigParser("linux") >>> config.readfp(StringIO(sample_config)) - >>> sorted(config.items("settings")) - [(u'am_hero', u'True'), (u'process_tags', u'False')] + >>> sorted((str(a), str(b)) for (a, b) in config.items("settings")) + [('am_hero', 'True'), ('process_tags', 'False')] # Check to make sure that settings works with unknown project. >>> config = _ProjectConfigParser("unknown") >>> config.readfp(StringIO(sample_config)) - >>> sorted(config.items("settings")) - [(u'am_hero', u'True')] + >>> sorted((str(a), str(b)) for (a, b) in config.items("settings")) + [('am_hero', 'True')] """ def __init__(self, project_name): """Construct _ProjectConfigParser. @@ -99,17 +100,6 @@ class _ProjectConfigParser(ConfigParser.SafeConfigParser): for setting_name, setting_value in project_defaults.items(): self.set(project_settings, setting_name, setting_value) - def _to_unicode(self, val): - """Make sure a value is of type 'unicode' - - Args: - val: string or unicode object - - Returns: - unicode version of val - """ - return val if isinstance(val, unicode) else val.decode('utf-8') - def get(self, section, option, *args, **kwargs): """Extend SafeConfigParser to try project_section before section. @@ -127,7 +117,7 @@ class _ProjectConfigParser(ConfigParser.SafeConfigParser): val = ConfigParser.SafeConfigParser.get( self, section, option, *args, **kwargs ) - return self._to_unicode(val) + return tools.ToUnicode(val) def items(self, section, *args, **kwargs): """Extend SafeConfigParser to add project_section to section. @@ -162,8 +152,8 @@ class _ProjectConfigParser(ConfigParser.SafeConfigParser): item_dict = dict(top_items) item_dict.update(project_items) - return {(self._to_unicode(item), self._to_unicode(val)) - for item, val in item_dict.iteritems()} + return {(tools.ToUnicode(item), tools.ToUnicode(val)) + for item, val in item_dict.items()} def ReadGitAliases(fname): """Read a git alias file. This is in the form used by git: diff --git a/tools/patman/test_util.py b/tools/patman/test_util.py index 687d40704a..ea36cd1633 100644 --- a/tools/patman/test_util.py +++ b/tools/patman/test_util.py @@ -3,6 +3,8 @@ # Copyright (c) 2016 Google, Inc # +from __future__ import print_function + from contextlib import contextmanager import glob import os @@ -15,6 +17,8 @@ try: except ImportError: from io import StringIO +PYTHON = 'python%d' % sys.version_info[0] + def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): """Run tests and check that we get 100% coverage @@ -41,11 +45,12 @@ def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): else: glob_list = [] glob_list += exclude_list - glob_list += ['*libfdt.py', '*site-packages*'] - cmd = ('PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools python-coverage run ' - '--omit "%s" %s -P1 -t' % (build_dir, ','.join(glob_list), prog)) + glob_list += ['*libfdt.py', '*site-packages*', '*dist-packages*'] + cmd = ('PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools %s-coverage run ' + '--omit "%s" %s -P1 -t' % (build_dir, PYTHON, ','.join(glob_list), + prog)) os.system(cmd) - stdout = command.Output('python-coverage', 'report') + stdout = command.Output('%s-coverage' % PYTHON, 'report') lines = stdout.splitlines() if required: # Convert '/path/to/name.py' just the module name 'name' @@ -54,18 +59,18 @@ def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): missing_list = required missing_list.difference_update(test_set) if missing_list: - print 'Missing tests for %s' % (', '.join(missing_list)) - print stdout + print('Missing tests for %s' % (', '.join(missing_list))) + print(stdout) ok = False coverage = lines[-1].split(' ')[-1] ok = True - print coverage + print(coverage) if coverage != '100%': - print stdout - print ("Type 'python-coverage html' to get a report in " - 'htmlcov/index.html') - print 'Coverage error: %s, but should be 100%%' % coverage + print(stdout) + print("Type '%s-coverage html' to get a report in " + 'htmlcov/index.html' % PYTHON) + print('Coverage error: %s, but should be 100%%' % coverage) ok = False if not ok: raise ValueError('Test coverage failure') diff --git a/tools/patman/tools.py b/tools/patman/tools.py index bf099798e6..8e9f22afe8 100644 --- a/tools/patman/tools.py +++ b/tools/patman/tools.py @@ -7,6 +7,7 @@ import command import glob import os import shutil +import sys import tempfile import tout @@ -167,9 +168,9 @@ def PathHasFile(fname): return True return False -def Run(name, *args): +def Run(name, *args, **kwargs): try: - return command.Run(name, *args, cwd=outdir, capture=True) + return command.Run(name, *args, cwd=outdir, capture=True, **kwargs) except: if not PathHasFile(name): msg = "Plesae install tool '%s'" % name @@ -213,7 +214,7 @@ def Filename(fname): # If not found, just return the standard, unchanged path return fname -def ReadFile(fname): +def ReadFile(fname, binary=True): """Read and return the contents of a file. Args: @@ -222,7 +223,7 @@ def ReadFile(fname): Returns: data read from file, as a string. """ - with open(Filename(fname), 'rb') as fd: + with open(Filename(fname), binary and 'rb' or 'r') as fd: data = fd.read() #self._out.Info("Read file '%s' size %d (%#0x)" % #(fname, len(data), len(data))) @@ -239,3 +240,105 @@ def WriteFile(fname, data): #(fname, len(data), len(data))) with open(Filename(fname), 'wb') as fd: fd.write(data) + +def GetBytes(byte, size): + """Get a string of bytes of a given size + + This handles the unfortunate different between Python 2 and Python 2. + + Args: + byte: Numeric byte value to use + size: Size of bytes/string to return + + Returns: + A bytes type with 'byte' repeated 'size' times + """ + if sys.version_info[0] >= 3: + data = bytes([byte]) * size + else: + data = chr(byte) * size + return data + +def ToUnicode(val): + """Make sure a value is a unicode string + + This allows some amount of compatibility between Python 2 and Python3. For + the former, it returns a unicode object. + + Args: + val: string or unicode object + + Returns: + unicode version of val + """ + if sys.version_info[0] >= 3: + return val + return val if isinstance(val, unicode) else val.decode('utf-8') + +def FromUnicode(val): + """Make sure a value is a non-unicode string + + This allows some amount of compatibility between Python 2 and Python3. For + the former, it converts a unicode object to a string. + + Args: + val: string or unicode object + + Returns: + non-unicode version of val + """ + if sys.version_info[0] >= 3: + return val + return val if isinstance(val, str) else val.encode('utf-8') + +def ToByte(ch): + """Convert a character to an ASCII value + + This is useful because in Python 2 bytes is an alias for str, but in + Python 3 they are separate types. This function converts the argument to + an ASCII value in either case. + + Args: + ch: A string (Python 2) or byte (Python 3) value + + Returns: + integer ASCII value for ch + """ + return ord(ch) if type(ch) == str else ch + +def ToChar(byte): + """Convert a byte to a character + + This is useful because in Python 2 bytes is an alias for str, but in + Python 3 they are separate types. This function converts an ASCII value to + a value with the appropriate type in either case. + + Args: + byte: A byte or str value + """ + return chr(byte) if type(byte) != str else byte + +def ToChars(byte_list): + """Convert a list of bytes to a str/bytes type + + Args: + byte_list: List of ASCII values representing the string + + Returns: + string made by concatenating all the ASCII values + """ + return ''.join([chr(byte) for byte in byte_list]) + +def ToBytes(string): + """Convert a str type into a bytes type + + Args: + string: string to convert value + + Returns: + Python 3: A bytes type + Python 2: A string type + """ + if sys.version_info[0] >= 3: + return string.encode('utf-8') + return string diff --git a/tools/proftool.c b/tools/proftool.c index c1803fa78a..fecb9d6e99 100644 --- a/tools/proftool.c +++ b/tools/proftool.c @@ -205,12 +205,12 @@ static struct func_info *find_caller_by_offset(uint32_t offset) return low >= 0 ? &func_list[low] : NULL; } -static int read_calls(FILE *fin, int count) +static int read_calls(FILE *fin, size_t count) { struct trace_call *call_data; int i; - notice("call count: %d\n", count); + notice("call count: %zu\n", count); call_list = (struct trace_call *)calloc(count, sizeof(*call_data)); if (!call_list) { error("Cannot allocate call_list\n"); diff --git a/tools/rmboard.py b/tools/rmboard.py new file mode 100755 index 0000000000..df4f04b01c --- /dev/null +++ b/tools/rmboard.py @@ -0,0 +1,150 @@ +#! /usr/bin/python +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2019 Google LLC +# + +""" +Script to remove boards + +Usage: + rmboard.py <board_name>... + +A single commit is created for each board removed. + +Some boards may depend on files provided by another and this will cause +problems, generally the removal of files which should not be removed. + +This script works by: + - Looking through the MAINTAINERS files which mention a board to find out + what files the board uses + - Looking through the Kconfig files which mention a board to find one that + needs to have material removed + +Search for ## to update the commit message manually. +""" + +from __future__ import print_function + +import glob +import os +import re +import sys + +# Bring in the patman libraries +our_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(our_path, '../tools/patman')) + +import command + +def rm_kconfig_include(path): + """Remove a path from Kconfig files + + This function finds the given path in a 'source' statement in a Kconfig + file and removes that line from the file. This is needed because the path + is going to be removed, so any reference to it will cause a problem with + Kconfig parsing. + + The changes are made locally and then added to the git staging area. + + Args: + path: Path to search for and remove + """ + cmd = ['git', 'grep', path] + stdout = command.RunPipe([cmd], capture=True, raise_on_error=False).stdout + if not stdout: + return + fname = stdout.split(':')[0] + + print("Fixing up '%s' to remove reference to '%s'" % (fname, path)) + cmd = ['sed', '-i', '\|%s|d' % path, fname] + stdout = command.RunPipe([cmd], capture=True).stdout + + cmd = ['git', 'add', fname] + stdout = command.RunPipe([cmd], capture=True).stdout + +def rm_board(board): + """Create a commit which removes a single board + + This looks up the MAINTAINERS file to file files that need to be removed, + then removes pieces from the Kconfig files that mention the board. + + + Args: + board: Board name to remove + """ + + # Find all MAINTAINERS and Kconfig files which mention the board + cmd = ['git', 'grep', '-l', board] + stdout = command.RunPipe([cmd], capture=True).stdout + maintain = [] + kconfig = [] + for line in stdout.splitlines(): + line = line.strip() + if 'MAINTAINERS' in line: + if line not in maintain: + maintain.append(line) + elif 'Kconfig' in line: + kconfig.append(line) + paths = [] + cc = [] + + # Look through the MAINTAINERS file to find things to remove + for fname in maintain: + with open(fname) as fd: + for line in fd: + line = line.strip() + fields = re.split('[ \t]', line, 1) + if len(fields) == 2: + if fields[0] == 'M:': + cc.append(fields[1]) + elif fields[0] == 'F:': + paths.append(fields[1].strip()) + + # Expand any wildcards in the MAINTAINERS file + real = [] + for path in paths: + if path[-1] == '/': + path = path[:-1] + if '*' in path: + globbed = glob.glob(path) + print("Expanded '%s' to '%s'" % (path, globbed)) + real += globbed + else: + real.append(path) + + # Search for Kconfig files in the resulting list. Remove any 'source' lines + # which reference Kconfig files we want to remove + for path in real: + cmd = ['find', path] + stdout = (command.RunPipe([cmd], capture=True, raise_on_error=False). + stdout) + for fname in stdout.splitlines(): + if fname.endswith('Kconfig'): + rm_kconfig_include(fname) + + # Remove unwanted files + cmd = ['git', 'rm', '-r'] + real + stdout = command.RunPipe([cmd], capture=True).stdout + + ## Change the messages as needed + msg = '''arm: Remove %s board + +This board has not been converted to CONFIG_DM_MMC by the deadline. +Remove it. + +''' % board + for name in cc: + msg += 'Patch-cc: %s\n' % name + + # Create the commit + cmd = ['git', 'commit', '-s', '-m', msg] + stdout = command.RunPipe([cmd], capture=True).stdout + + # Check if the board is mentioned anywhere else. The user will need to deal + # with this + cmd = ['git', 'grep', '-il', board] + print(command.RunPipe([cmd], capture=True, raise_on_error=False).stdout) + print(' '.join(cmd)) + +for board in sys.argv[1:]: + rm_board(board) |