binman: Add support for generating a FIT

FIT (Flat Image Tree) is the main image format used by U-Boot. In some
cases scripts are used to create FITs within the U-Boot build system. This
is not ideal for various reasons:

- Each architecture has its own slightly different script
- There are no tests
- Some are written in shell, some in Python

To help address this, add support for FIT generation to binman. This works
by putting the FIT source directly in the binman definition, with the
ability to adjust parameters, etc. The contents of each FIT image come
from sub-entries of the image, as is normal with binman.

Signed-off-by: Simon Glass <sjg@chromium.org>

5 files changed, 384 insertions, 0 deletions

diff --git a/tools/binman/README.entries b/tools/binman/README.entries
index f45f51428a..bf8edce02b 100644
--- a/tools/binman/README.entries
+++ b/tools/binman/README.entries
@@ -311,6 +311,46 @@ byte value of a region.
 
 
 
+Entry: fit: Entry containing a FIT
+----------------------------------
+
+This calls mkimage to create a FIT (U-Boot Flat Image Tree) based on the
+input provided.
+
+Nodes for the FIT should be written out in the binman configuration just as
+they would be in a file passed to mkimage.
+
+For example, this creates an image containing a FIT with U-Boot SPL:
+
+    binman {
+        fit {
+            description = "Test FIT";
+
+            images {
+                kernel@1 {
+                    description = "SPL";
+                    os = "u-boot";
+                    type = "rkspi";
+                    arch = "arm";
+                    compression = "none";
+                    load = <0>;
+                    entry = <0>;
+
+                    u-boot-spl {
+                    };
+                };
+            };
+        };
+    };
+
+Properties:
+    fit,external-offset: Indicates that the contents of the FIT are external
+        and provides the external offset. This is passsed to mkimage via
+        the -E and -p flags.
+
+
+
+
 Entry: fmap: An entry which contains an Fmap section
 ----------------------------------------------------
 
diff --git a/tools/binman/etype/fit.py b/tools/binman/etype/fit.py
new file mode 100644
index 0000000000..75712f4409
--- /dev/null
+++ b/tools/binman/etype/fit.py
@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2016 Google, Inc
+# Written by Simon Glass <sjg@chromium.org>
+#
+# Entry-type module for producing a FIT
+#
+
+from collections import defaultdict, OrderedDict
+import libfdt
+
+from binman.entry import Entry
+from dtoc import fdt_util
+from dtoc.fdt import Fdt
+from patman import tools
+
+class Entry_fit(Entry):
+    """Entry containing a FIT
+
+    This calls mkimage to create a FIT (U-Boot Flat Image Tree) based on the
+    input provided.
+
+    Nodes for the FIT should be written out in the binman configuration just as
+    they would be in a file passed to mkimage.
+
+    For example, this creates an image containing a FIT with U-Boot SPL:
+
+        binman {
+            fit {
+                description = "Test FIT";
+
+                images {
+                    kernel@1 {
+                        description = "SPL";
+                        os = "u-boot";
+                        type = "rkspi";
+                        arch = "arm";
+                        compression = "none";
+                        load = <0>;
+                        entry = <0>;
+
+                        u-boot-spl {
+                        };
+                    };
+                };
+            };
+        };
+
+    Properties:
+        fit,external-offset: Indicates that the contents of the FIT are external
+            and provides the external offset. This is passsed to mkimage via
+            the -E and -p flags.
+
+    """
+    def __init__(self, section, etype, node):
+        """
+        Members:
+            _fit: FIT file being built
+            _fit_content: dict:
+                key: relative path to entry Node (from the base of the FIT)
+                value: List of Entry objects comprising the contents of this
+                    node
+        """
+        super().__init__(section, etype, node)
+        self._fit = None
+        self._fit_content = defaultdict(list)
+        self._fit_props = {}
+
+    def ReadNode(self):
+        self._ReadSubnodes()
+        super().ReadNode()
+
+    def _ReadSubnodes(self):
+        def _AddNode(base_node, depth, node):
+            """Add a node to the FIT
+
+            Args:
+                base_node: Base Node of the FIT (with 'description' property)
+                depth: Current node depth (0 is the base node)
+                node: Current node to process
+
+            There are two cases to deal with:
+                - hash and signature nodes which become part of the FIT
+                - binman entries which are used to define the 'data' for each
+                  image
+            """
+            for pname, prop in node.props.items():
+                if pname.startswith('fit,'):
+                    self._fit_props[pname] = prop
+                else:
+                    fsw.property(pname, prop.bytes)
+
+            rel_path = node.path[len(base_node.path):]
+            has_images = depth == 2 and rel_path.startswith('/images/')
+            for subnode in node.subnodes:
+                if has_images and not (subnode.name.startswith('hash') or
+                                       subnode.name.startswith('signature')):
+                    # This is a content node. We collect all of these together
+                    # and put them in the 'data' property. They do not appear
+                    # in the FIT.
+                    entry = Entry.Create(self.section, subnode)
+                    entry.ReadNode()
+                    self._fit_content[rel_path].append(entry)
+                else:
+                    with fsw.add_node(subnode.name):
+                        _AddNode(base_node, depth + 1, subnode)
+
+        # Build a new tree with all nodes and properties starting from the
+        # entry node
+        fsw = libfdt.FdtSw()
+        fsw.finish_reservemap()
+        with fsw.add_node(''):
+            _AddNode(self._node, 0, self._node)
+        fdt = fsw.as_fdt()
+
+        # Pack this new FDT and scan it so we can add the data later
+        fdt.pack()
+        self._fdt = Fdt.FromData(fdt.as_bytearray())
+        self._fdt.Scan()
+
+    def ObtainContents(self):
+        """Obtain the contents of the FIT
+
+        This adds the 'data' properties to the input ITB (Image-tree Binary)
+        then runs mkimage to process it.
+        """
+        data = self._BuildInput(self._fdt)
+        if data == False:
+            return False
+        uniq = self.GetUniqueName()
+        input_fname = tools.GetOutputFilename('%s.itb' % uniq)
+        output_fname = tools.GetOutputFilename('%s.fit' % uniq)
+        tools.WriteFile(input_fname, data)
+        tools.WriteFile(output_fname, data)
+
+        args = []
+        ext_offset = self._fit_props.get('fit,external-offset')
+        if ext_offset is not None:
+            args += ['-E', '-p', '%x' % fdt_util.fdt32_to_cpu(ext_offset.value)]
+        tools.Run('mkimage', '-t', '-F', output_fname, *args)
+
+        self.SetContents(tools.ReadFile(output_fname))
+        return True
+
+    def _BuildInput(self, fdt):
+        """Finish the FIT by adding the 'data' properties to it
+
+        Arguments:
+            fdt: FIT to update
+
+        Returns:
+            New fdt contents (bytes)
+        """
+        for path, entries in self._fit_content.items():
+            node = fdt.GetNode(path)
+            data = b''
+            for entry in entries:
+                if not entry.ObtainContents():
+                    return False
+                data += entry.GetData()
+            node.AddData('data', data)
+
+        fdt.Sync(auto_resize=True)
+        data = fdt.GetContents()
+        return data
diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py
index 614ac4ed39..ea72eff8c5 100644
--- a/tools/binman/ftest.py
+++ b/tools/binman/ftest.py
@@ -6,10 +6,12 @@
 #
 #    python -m unittest func_test.TestFunctional.testHelp
 
+import collections
 import gzip
 import hashlib
 from optparse import OptionParser
 import os
+import re
 import shutil
 import struct
 import sys
@@ -3425,6 +3427,58 @@ class TestFunctional(unittest.TestCase):
         """Test that zero-size overlapping regions are ignored"""
         self._DoTestFile('160_pack_overlap_zero.dts')
 
+    def testSimpleFit(self):
+        """Test an image with a FIT inside"""
+        data = self._DoReadFile('161_fit.dts')
+        self.assertEqual(U_BOOT_DATA, data[:len(U_BOOT_DATA)])
+        self.assertEqual(U_BOOT_NODTB_DATA, data[-len(U_BOOT_NODTB_DATA):])
+        fit_data = data[len(U_BOOT_DATA):-len(U_BOOT_NODTB_DATA)]
+
+        # The data should be inside the FIT
+        dtb = fdt.Fdt.FromData(fit_data)
+        dtb.Scan()
+        fnode = dtb.GetNode('/images/kernel')
+        self.assertIn('data', fnode.props)
+
+        fname = os.path.join(self._indir, 'fit_data.fit')
+        tools.WriteFile(fname, fit_data)
+        out = tools.Run('dumpimage', '-l', fname)
+
+        # Check a few features to make sure the plumbing works. We don't need
+        # to test the operation of mkimage or dumpimage here. First convert the
+        # output into a dict where the keys are the fields printed by dumpimage
+        # and the values are a list of values for each field
+        lines = out.splitlines()
+
+        # Converts "Compression:  gzip compressed" into two groups:
+        # 'Compression' and 'gzip compressed'
+        re_line = re.compile(r'^ *([^:]*)(?:: *(.*))?$')
+        vals = collections.defaultdict(list)
+        for line in lines:
+            mat = re_line.match(line)
+            vals[mat.group(1)].append(mat.group(2))
+
+        self.assertEquals('FIT description: test-desc', lines[0])
+        self.assertIn('Created:', lines[1])
+        self.assertIn('Image 0 (kernel)', vals)
+        self.assertIn('Hash value', vals)
+        data_sizes = vals.get('Data Size')
+        self.assertIsNotNone(data_sizes)
+        self.assertEqual(2, len(data_sizes))
+        # Format is "4 Bytes = 0.00 KiB = 0.00 MiB" so take the first word
+        self.assertEqual(len(U_BOOT_DATA), int(data_sizes[0].split()[0]))
+        self.assertEqual(len(U_BOOT_SPL_DTB_DATA), int(data_sizes[1].split()[0]))
+
+    def testFitExternal(self):
+        """Test an image with an FIT"""
+        data = self._DoReadFile('162_fit_external.dts')
+        fit_data = data[len(U_BOOT_DATA):-2]  # _testing is 2 bytes
+
+        # The data should be outside the FIT
+        dtb = fdt.Fdt.FromData(fit_data)
+        dtb.Scan()
+        fnode = dtb.GetNode('/images/kernel')
+        self.assertNotIn('data', fnode.props)
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tools/binman/test/161_fit.dts b/tools/binman/test/161_fit.dts
new file mode 100644
index 0000000000..c52d760b73
--- /dev/null
+++ b/tools/binman/test/161_fit.dts
@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		u-boot {
+		};
+		fit {
+			description = "test-desc";
+			#address-cells = <1>;
+
+			images {
+				kernel {
+					description = "Vanilla Linux kernel";
+					type = "kernel";
+					arch = "ppc";
+					os = "linux";
+					compression = "gzip";
+					load = <00000000>;
+					entry = <00000000>;
+					hash-1 {
+						algo = "crc32";
+					};
+					hash-2 {
+						algo = "sha1";
+					};
+					u-boot {
+					};
+				};
+				fdt-1 {
+					description = "Flattened Device Tree blob";
+					type = "flat_dt";
+					arch = "ppc";
+					compression = "none";
+					hash-1 {
+						algo = "crc32";
+					};
+					hash-2 {
+						algo = "sha1";
+					};
+					u-boot-spl-dtb {
+					};
+				};
+			};
+
+			configurations {
+				default = "conf-1";
+				conf-1 {
+					description = "Boot Linux kernel with FDT blob";
+					kernel = "kernel";
+					fdt = "fdt-1";
+				};
+			};
+		};
+		u-boot-nodtb {
+		};
+	};
+};
diff --git a/tools/binman/test/162_fit_external.dts b/tools/binman/test/162_fit_external.dts
new file mode 100644
index 0000000000..19518e05a5
--- /dev/null
+++ b/tools/binman/test/162_fit_external.dts
@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		u-boot {
+		};
+		fit {
+			fit,external-offset = <0>;
+			description = "test-desc";
+			#address-cells = <1>;
+
+			images {
+				kernel {
+					description = "Vanilla Linux kernel";
+					type = "kernel";
+					arch = "ppc";
+					os = "linux";
+					compression = "gzip";
+					load = <00000000>;
+					entry = <00000000>;
+					hash-1 {
+						algo = "crc32";
+					};
+					hash-2 {
+						algo = "sha1";
+					};
+					u-boot {
+					};
+				};
+				fdt-1 {
+					description = "Flattened Device Tree blob";
+					type = "flat_dt";
+					arch = "ppc";
+					compression = "none";
+					hash-1 {
+						algo = "crc32";
+					};
+					hash-2 {
+						algo = "sha1";
+					};
+					_testing {
+						return-contents-later;
+					};
+				};
+			};
+
+			configurations {
+				default = "conf-1";
+				conf-1 {
+					description = "Boot Linux kernel with FDT blob";
+					kernel = "kernel";
+					fdt = "fdt-1";
+				};
+			};
+		};
+		u-boot-nodtb {
+		};
+	};
+};