From 0dc706fe544dc1121c15008ec50809cd6d72ff53 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 9 Jul 2020 18:39:31 -0600 Subject: binman: Add support for calling mkimage As a first step to integrating mkimage into binman, add a new entry type that feeds data into mkimage for processing and incorporates that output into the image. Signed-off-by: Simon Glass --- tools/binman/ftest.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'tools/binman/ftest.py') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 5e24920088..39e67b9042 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -3357,6 +3357,13 @@ class TestFunctional(unittest.TestCase): data = self._DoReadFile('154_intel_fsp_t.dts') self.assertEqual(FSP_T_DATA, data[:len(FSP_T_DATA)]) + def testMkimage(self): + """Test using mkimage to build an image""" + data = self._DoReadFile('156_mkimage.dts') + + # Just check that the data appears in the file somewhere + self.assertIn(U_BOOT_SPL_DATA, data) + if __name__ == "__main__": unittest.main() -- cgit From ce867ad7c8eaae2b14699aee7235695d605c1dda Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 9 Jul 2020 18:39:36 -0600 Subject: binman: Add an etype for external binary blobs It is useful to be able to distinguish between ordinary blobs such as u-boot.bin and external blobs that cannot be build by the U-Boot build system. If the external blobs are not available for some reason, then we know that a value image cannot be built. Introduce a new 'blob-ext' entry type for that. Signed-off-by: Simon Glass Reviewed-by: Bin Meng --- tools/binman/ftest.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'tools/binman/ftest.py') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 39e67b9042..f8d5191672 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -3364,6 +3364,18 @@ class TestFunctional(unittest.TestCase): # Just check that the data appears in the file somewhere self.assertIn(U_BOOT_SPL_DATA, data) + def testExtblob(self): + """Test an image with an external blob""" + data = self._DoReadFile('157_blob_ext.dts') + self.assertEqual(REFCODE_DATA, data) + + def testExtblobMissing(self): + """Test an image with a missing external blob""" + with self.assertRaises(ValueError) as e: + self._DoReadFile('158_blob_ext_missing.dts') + self.assertIn("Filename 'missing-file' not found in input path", + str(e.exception)) + if __name__ == "__main__": unittest.main() -- cgit From 4f9f1056ecf2d9e54803b89fd0784240a8d89fd8 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 9 Jul 2020 18:39:38 -0600 Subject: binman: Allow external binaries to be missing Sometimes it is useful to build an image even though external binaries are not present. This allows the build system to continue to function without these files, albeit not producing valid images. U-Boot does with with ATF (ARM Trusted Firmware) today. Add a new flag to binman to request this behaviour. Signed-off-by: Simon Glass Reviewed-by: Bin Meng --- tools/binman/ftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'tools/binman/ftest.py') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index f8d5191672..7c8b3eb3a0 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -285,7 +285,7 @@ class TestFunctional(unittest.TestCase): def _DoTestFile(self, fname, debug=False, map=False, update_dtb=False, entry_args=None, images=None, use_real_dtb=False, - verbosity=None): + verbosity=None, allow_missing=False): """Run binman with a given test file Args: @@ -319,6 +319,8 @@ class TestFunctional(unittest.TestCase): if entry_args: for arg, value in entry_args.items(): args.append('-a%s=%s' % (arg, value)) + if allow_missing: + args.append('-M') if images: for image in images: args += ['-i', image] @@ -3376,6 +3378,10 @@ class TestFunctional(unittest.TestCase): self.assertIn("Filename 'missing-file' not found in input path", str(e.exception)) + def testExtblobMissingOk(self): + """Test an image with an missing external blob that is allowed""" + self._DoTestFile('158_blob_ext_missing.dts', allow_missing=True) + if __name__ == "__main__": unittest.main() -- cgit From 38fdb4cb35e9260a6aa78ffcfa68d39bfc3523de Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 9 Jul 2020 18:39:39 -0600 Subject: patman: Update errors and warnings to use stderr When warnings and errors are produced by tools they should be written to stderr. Update the tout implementation to handle this. Signed-off-by: Simon Glass Reviewed-by: Bin Meng --- tools/binman/ftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/binman/ftest.py') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 7c8b3eb3a0..928d3608a3 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -3232,7 +3232,7 @@ class TestFunctional(unittest.TestCase): with test_util.capture_sys_output() as (stdout, stderr): control.ReplaceEntries(updated_fname, None, outdir, []) self.assertIn("Skipping entry '/u-boot' from missing file", - stdout.getvalue()) + stderr.getvalue()) def testReplaceCmdMap(self): """Test replacing a file fron an image on the command line""" -- cgit From b1cca9552c6c930d094806a64b2096411783d13f Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 9 Jul 2020 18:39:40 -0600 Subject: binman: Detect when valid images are not produced When external blobs are missing, show a message indicating that the images are not functional. Signed-off-by: Simon Glass --- tools/binman/ftest.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'tools/binman/ftest.py') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 928d3608a3..cc551c9f17 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -3380,7 +3380,19 @@ class TestFunctional(unittest.TestCase): def testExtblobMissingOk(self): """Test an image with an missing external blob that is allowed""" - self._DoTestFile('158_blob_ext_missing.dts', allow_missing=True) + with test_util.capture_sys_output() as (stdout, stderr): + self._DoTestFile('158_blob_ext_missing.dts', allow_missing=True) + err = stderr.getvalue() + self.assertRegex(err, "Image 'main-section'.*missing.*: blob-ext") + + def testExtblobMissingOkSect(self): + """Test an image with an missing external blob that is allowed""" + with test_util.capture_sys_output() as (stdout, stderr): + self._DoTestFile('159_blob_ext_missing_sect.dts', + allow_missing=True) + err = stderr.getvalue() + self.assertRegex(err, "Image 'main-section'.*missing.*: " + "blob-ext blob-ext2") if __name__ == "__main__": -- cgit From 0ba4b3dfee57a4e5803abd881e9654fb0414a3c1 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 9 Jul 2020 18:39:41 -0600 Subject: binman: Allow missing Intel blobs Update the Intel blob entries to support missing binaries. Signed-off-by: Simon Glass Reviewed-by: Bin Meng --- tools/binman/ftest.py | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) (limited to 'tools/binman/ftest.py') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index cc551c9f17..146d4c51d3 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -160,8 +160,7 @@ class TestFunctional(unittest.TestCase): tools.ReadFile(cls.ElfTestFile('u_boot_ucode_ptr'))) # Intel flash descriptor file - with open(cls.TestFile('descriptor.bin'), 'rb') as fd: - TestFunctional._MakeInputFile('descriptor.bin', fd.read()) + cls._SetupDescriptor() shutil.copytree(cls.TestFile('files'), os.path.join(cls._indir, 'files')) @@ -507,6 +506,11 @@ class TestFunctional(unittest.TestCase): TestFunctional._MakeInputFile('tpl/u-boot-tpl', tools.ReadFile(cls.ElfTestFile(src_fname))) + @classmethod + def _SetupDescriptor(cls): + with open(cls.TestFile('descriptor.bin'), 'rb') as fd: + TestFunctional._MakeInputFile('descriptor.bin', fd.read()) + @classmethod def TestFile(cls, fname): return os.path.join(cls._binman_dir, 'test', fname) @@ -933,11 +937,14 @@ class TestFunctional(unittest.TestCase): def testPackX86RomMeNoDesc(self): """Test that an invalid Intel descriptor entry is detected""" - TestFunctional._MakeInputFile('descriptor.bin', b'') - with self.assertRaises(ValueError) as e: - self._DoTestFile('031_x86_rom_me.dts') - self.assertIn("Node '/binman/intel-descriptor': Cannot find Intel Flash Descriptor (FD) signature", - str(e.exception)) + try: + TestFunctional._MakeInputFile('descriptor.bin', b'') + with self.assertRaises(ValueError) as e: + self._DoTestFile('031_x86_rom_me.dts') + self.assertIn("Node '/binman/intel-descriptor': Cannot find Intel Flash Descriptor (FD) signature", + str(e.exception)) + finally: + self._SetupDescriptor() def testPackX86RomBadDesc(self): """Test that the Intel requires a descriptor entry""" @@ -3394,6 +3401,26 @@ class TestFunctional(unittest.TestCase): self.assertRegex(err, "Image 'main-section'.*missing.*: " "blob-ext blob-ext2") + def testPackX86RomMeMissingDesc(self): + """Test that an missing Intel descriptor entry is allowed""" + pathname = os.path.join(self._indir, 'descriptor.bin') + os.remove(pathname) + with test_util.capture_sys_output() as (stdout, stderr): + self._DoTestFile('031_x86_rom_me.dts', allow_missing=True) + err = stderr.getvalue() + self.assertRegex(err, + "Image 'main-section'.*missing.*: intel-descriptor") + + def testPackX86RomMissingIfwi(self): + """Test that an x86 ROM with Integrated Firmware Image can be created""" + self._SetupIfwi('fitimage.bin') + pathname = os.path.join(self._indir, 'fitimage.bin') + os.remove(pathname) + with test_util.capture_sys_output() as (stdout, stderr): + self._DoTestFile('111_x86_rom_ifwi.dts', allow_missing=True) + err = stderr.getvalue() + self.assertRegex(err, "Image 'main-section'.*missing.*: intel-ifwi") + if __name__ == "__main__": unittest.main() -- cgit From b3295fd4e3d61a7ceabbad10a293db2dcd38b38b Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 9 Jul 2020 18:39:42 -0600 Subject: binman: Allow zero-length entries to overlap Some binary blobs unfortunately obtain their position in the image from other binary blobs, such as Intel's 'descriptor'. In this case we cannot rely on packing to work. It is not possible to produce a valid image in any case, due to the missing blobs. Allow zero-length overlaps so that this does not cause any problems. Signed-off-by: Simon Glass Reviewed-by: Bin Meng --- tools/binman/ftest.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'tools/binman/ftest.py') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 146d4c51d3..614ac4ed39 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -3421,6 +3421,10 @@ class TestFunctional(unittest.TestCase): err = stderr.getvalue() self.assertRegex(err, "Image 'main-section'.*missing.*: intel-ifwi") + def testPackOverlap(self): + """Test that zero-size overlapping regions are ignored""" + self._DoTestFile('160_pack_overlap_zero.dts') + if __name__ == "__main__": unittest.main() -- cgit From fdc34368ddcd1d2eb46f9a1829f080f8c1760dee Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 9 Jul 2020 18:39:45 -0600 Subject: binman: Add support for generating a FIT FIT (Flat Image Tree) is the main image format used by U-Boot. In some cases scripts are used to create FITs within the U-Boot build system. This is not ideal for various reasons: - Each architecture has its own slightly different script - There are no tests - Some are written in shell, some in Python To help address this, add support for FIT generation to binman. This works by putting the FIT source directly in the binman definition, with the ability to adjust parameters, etc. The contents of each FIT image come from sub-entries of the image, as is normal with binman. Signed-off-by: Simon Glass --- tools/binman/ftest.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) (limited to 'tools/binman/ftest.py') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 614ac4ed39..ea72eff8c5 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -6,10 +6,12 @@ # # python -m unittest func_test.TestFunctional.testHelp +import collections import gzip import hashlib from optparse import OptionParser import os +import re import shutil import struct import sys @@ -3425,6 +3427,58 @@ class TestFunctional(unittest.TestCase): """Test that zero-size overlapping regions are ignored""" self._DoTestFile('160_pack_overlap_zero.dts') + def testSimpleFit(self): + """Test an image with a FIT inside""" + data = self._DoReadFile('161_fit.dts') + self.assertEqual(U_BOOT_DATA, data[:len(U_BOOT_DATA)]) + self.assertEqual(U_BOOT_NODTB_DATA, data[-len(U_BOOT_NODTB_DATA):]) + fit_data = data[len(U_BOOT_DATA):-len(U_BOOT_NODTB_DATA)] + + # The data should be inside the FIT + dtb = fdt.Fdt.FromData(fit_data) + dtb.Scan() + fnode = dtb.GetNode('/images/kernel') + self.assertIn('data', fnode.props) + + fname = os.path.join(self._indir, 'fit_data.fit') + tools.WriteFile(fname, fit_data) + out = tools.Run('dumpimage', '-l', fname) + + # Check a few features to make sure the plumbing works. We don't need + # to test the operation of mkimage or dumpimage here. First convert the + # output into a dict where the keys are the fields printed by dumpimage + # and the values are a list of values for each field + lines = out.splitlines() + + # Converts "Compression: gzip compressed" into two groups: + # 'Compression' and 'gzip compressed' + re_line = re.compile(r'^ *([^:]*)(?:: *(.*))?$') + vals = collections.defaultdict(list) + for line in lines: + mat = re_line.match(line) + vals[mat.group(1)].append(mat.group(2)) + + self.assertEquals('FIT description: test-desc', lines[0]) + self.assertIn('Created:', lines[1]) + self.assertIn('Image 0 (kernel)', vals) + self.assertIn('Hash value', vals) + data_sizes = vals.get('Data Size') + self.assertIsNotNone(data_sizes) + self.assertEqual(2, len(data_sizes)) + # Format is "4 Bytes = 0.00 KiB = 0.00 MiB" so take the first word + self.assertEqual(len(U_BOOT_DATA), int(data_sizes[0].split()[0])) + self.assertEqual(len(U_BOOT_SPL_DTB_DATA), int(data_sizes[1].split()[0])) + + def testFitExternal(self): + """Test an image with an FIT""" + data = self._DoReadFile('162_fit_external.dts') + fit_data = data[len(U_BOOT_DATA):-2] # _testing is 2 bytes + + # The data should be outside the FIT + dtb = fdt.Fdt.FromData(fit_data) + dtb.Scan() + fnode = dtb.GetNode('/images/kernel') + self.assertNotIn('data', fnode.props) if __name__ == "__main__": unittest.main() -- cgit From 52b10dd7def5fa77366ee89bffde597b2e03d1a1 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 25 Jul 2020 15:11:19 -0600 Subject: binman: Don't change the descriptor in tests At present testPackX86RomMeNoDesc removes the contents of the descriptor.bin file and testPackX86RomMeMissingDesc removes the file completely. If a test that relies on this file happens to run after it is removed, it will not work. Since we have no control over the selecting of tests that run in parallel and series, we must avoid changing the files. Update this tests to use separate files instead. Signed-off-by: Simon Glass --- tools/binman/ftest.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'tools/binman/ftest.py') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index ea72eff8c5..bf7f59fb84 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -940,9 +940,9 @@ class TestFunctional(unittest.TestCase): def testPackX86RomMeNoDesc(self): """Test that an invalid Intel descriptor entry is detected""" try: - TestFunctional._MakeInputFile('descriptor.bin', b'') + TestFunctional._MakeInputFile('descriptor-empty.bin', b'') with self.assertRaises(ValueError) as e: - self._DoTestFile('031_x86_rom_me.dts') + self._DoTestFile('163_x86_rom_me_empty.dts') self.assertIn("Node '/binman/intel-descriptor': Cannot find Intel Flash Descriptor (FD) signature", str(e.exception)) finally: @@ -3405,10 +3405,8 @@ class TestFunctional(unittest.TestCase): def testPackX86RomMeMissingDesc(self): """Test that an missing Intel descriptor entry is allowed""" - pathname = os.path.join(self._indir, 'descriptor.bin') - os.remove(pathname) with test_util.capture_sys_output() as (stdout, stderr): - self._DoTestFile('031_x86_rom_me.dts', allow_missing=True) + self._DoTestFile('164_x86_rom_me_missing.dts', allow_missing=True) err = stderr.getvalue() self.assertRegex(err, "Image 'main-section'.*missing.*: intel-descriptor") -- cgit