diff options
36 files changed, 952 insertions, 310 deletions
diff --git a/arch/arm/mach-imx/spl.c b/arch/arm/mach-imx/spl.c index 87dbdf3011..49bb3b928d 100644 --- a/arch/arm/mach-imx/spl.c +++ b/arch/arm/mach-imx/spl.c @@ -189,7 +189,7 @@ int g_dnl_bind_fixup(struct usb_device_descriptor *dev, const char *name) #if defined(CONFIG_SPL_MMC_SUPPORT) /* called from spl_mmc to see type of boot mode for storage (RAW or FAT) */ -u32 spl_boot_mode(const u32 boot_device) +u32 spl_mmc_boot_mode(const u32 boot_device) { #if defined(CONFIG_MX7) || defined(CONFIG_IMX8M) || defined(CONFIG_IMX8) switch (get_boot_device()) { diff --git a/arch/arm/mach-k3/am6_init.c b/arch/arm/mach-k3/am6_init.c index 3768bccafa..b692806352 100644 --- a/arch/arm/mach-k3/am6_init.c +++ b/arch/arm/mach-k3/am6_init.c @@ -199,7 +199,7 @@ void board_init_f(ulong dummy) #endif } -u32 spl_boot_mode(const u32 boot_device) +u32 spl_mmc_boot_mode(const u32 boot_device) { #if defined(CONFIG_SUPPORT_EMMC_BOOT) u32 devstat = readl(CTRLMMR_MAIN_DEVSTAT); diff --git a/arch/arm/mach-k3/j721e_init.c b/arch/arm/mach-k3/j721e_init.c index f34090f9cc..71fc20c30b 100644 --- a/arch/arm/mach-k3/j721e_init.c +++ b/arch/arm/mach-k3/j721e_init.c @@ -223,7 +223,7 @@ void board_init_f(ulong dummy) #endif } -u32 spl_boot_mode(const u32 boot_device) +u32 spl_mmc_boot_mode(const u32 boot_device) { switch (boot_device) { case BOOT_DEVICE_MMC1: diff --git a/arch/arm/mach-omap2/boot-common.c b/arch/arm/mach-omap2/boot-common.c index 734fa9d9e6..7538523724 100644 --- a/arch/arm/mach-omap2/boot-common.c +++ b/arch/arm/mach-omap2/boot-common.c @@ -187,7 +187,7 @@ u32 spl_boot_device(void) return gd->arch.omap_boot_device; } -u32 spl_boot_mode(const u32 boot_device) +u32 spl_mmc_boot_mode(const u32 boot_device) { return gd->arch.omap_boot_mode; } diff --git a/arch/arm/mach-rockchip/spl.c b/arch/arm/mach-rockchip/spl.c index 48ab0e60c6..0b76af6080 100644 --- a/arch/arm/mach-rockchip/spl.c +++ b/arch/arm/mach-rockchip/spl.c @@ -58,7 +58,7 @@ u32 spl_boot_device(void) return boot_device; } -u32 spl_boot_mode(const u32 boot_device) +u32 spl_mmc_boot_mode(const u32 boot_device) { return MMCSD_MODE_RAW; } diff --git a/arch/arm/mach-socfpga/spl_a10.c b/arch/arm/mach-socfpga/spl_a10.c index b10be33268..d2f52f2f2c 100644 --- a/arch/arm/mach-socfpga/spl_a10.c +++ b/arch/arm/mach-socfpga/spl_a10.c @@ -92,7 +92,7 @@ u32 spl_boot_device(void) } #ifdef CONFIG_SPL_MMC_SUPPORT -u32 spl_boot_mode(const u32 boot_device) +u32 spl_mmc_boot_mode(const u32 boot_device) { #if defined(CONFIG_SPL_FS_FAT) || defined(CONFIG_SPL_FS_EXT4) return MMCSD_MODE_FS; diff --git a/arch/arm/mach-socfpga/spl_agilex.c b/arch/arm/mach-socfpga/spl_agilex.c index ecc1a35c49..aa9f3e646c 100644 --- a/arch/arm/mach-socfpga/spl_agilex.c +++ b/arch/arm/mach-socfpga/spl_agilex.c @@ -28,7 +28,7 @@ u32 spl_boot_device(void) } #ifdef CONFIG_SPL_MMC_SUPPORT -u32 spl_boot_mode(const u32 boot_device) +u32 spl_mmc_boot_mode(const u32 boot_device) { #if defined(CONFIG_SPL_FS_FAT) || defined(CONFIG_SPL_FS_EXT4) return MMCSD_MODE_FS; diff --git a/arch/arm/mach-socfpga/spl_gen5.c b/arch/arm/mach-socfpga/spl_gen5.c index a01e2a5cb9..e9967ac450 100644 --- a/arch/arm/mach-socfpga/spl_gen5.c +++ b/arch/arm/mach-socfpga/spl_gen5.c @@ -49,7 +49,7 @@ u32 spl_boot_device(void) } #ifdef CONFIG_SPL_MMC_SUPPORT -u32 spl_boot_mode(const u32 boot_device) +u32 spl_mmc_boot_mode(const u32 boot_device) { #if defined(CONFIG_SPL_FS_FAT) || defined(CONFIG_SPL_FS_EXT4) return MMCSD_MODE_FS; diff --git a/arch/arm/mach-socfpga/spl_s10.c b/arch/arm/mach-socfpga/spl_s10.c index d89151d902..08427dd83e 100644 --- a/arch/arm/mach-socfpga/spl_s10.c +++ b/arch/arm/mach-socfpga/spl_s10.c @@ -30,7 +30,7 @@ u32 spl_boot_device(void) } #ifdef CONFIG_SPL_MMC_SUPPORT -u32 spl_boot_mode(const u32 boot_device) +u32 spl_mmc_boot_mode(const u32 boot_device) { #if defined(CONFIG_SPL_FS_FAT) || defined(CONFIG_SPL_FS_EXT4) return MMCSD_MODE_FS; diff --git a/arch/arm/mach-stm32mp/spl.c b/arch/arm/mach-stm32mp/spl.c index ca4231cd0d..f85391c6af 100644 --- a/arch/arm/mach-stm32mp/spl.c +++ b/arch/arm/mach-stm32mp/spl.c @@ -44,12 +44,12 @@ u32 spl_boot_device(void) return BOOT_DEVICE_MMC1; } -u32 spl_boot_mode(const u32 boot_device) +u32 spl_mmc_boot_mode(const u32 boot_device) { return MMCSD_MODE_RAW; } -int spl_boot_partition(const u32 boot_device) +int spl_mmc_boot_partition(const u32 boot_device) { switch (boot_device) { case BOOT_DEVICE_MMC1: diff --git a/arch/arm/mach-uniphier/mmc-boot-mode.c b/arch/arm/mach-uniphier/mmc-boot-mode.c index 19b4560494..b48495365c 100644 --- a/arch/arm/mach-uniphier/mmc-boot-mode.c +++ b/arch/arm/mach-uniphier/mmc-boot-mode.c @@ -8,7 +8,7 @@ #include <mmc.h> #include <spl.h> -u32 spl_boot_mode(const u32 boot_device) +u32 spl_mmc_boot_mode(const u32 boot_device) { struct mmc *mmc; diff --git a/cmd/Kconfig b/cmd/Kconfig index 95a67e9d02..6ce9e5521c 100644 --- a/cmd/Kconfig +++ b/cmd/Kconfig @@ -1060,21 +1060,34 @@ config CMD_MMC help MMC memory mapped support. +if CMD_MMC + +config CMD_BKOPS_ENABLE + bool "mmc bkops enable" + depends on CMD_MMC + default n + help + Enable command for setting manual background operations handshake + on a eMMC device. The feature is optionally available on eMMC devices + conforming to standard >= 4.41. + config CMD_MMC_RPMB bool "Enable support for RPMB in the mmc command" - depends on CMD_MMC + depends on SUPPORT_EMMC_RPMB help Enable the commands for reading, writing and programming the key for the Replay Protection Memory Block partition in eMMC. config CMD_MMC_SWRITE bool "mmc swrite" - depends on CMD_MMC && MMC_WRITE + depends on MMC_WRITE select IMAGE_SPARSE help Enable support for the "mmc swrite" command to write Android sparse images to eMMC. +endif + config CMD_MTD bool "mtd" depends on MTD @@ -1607,15 +1620,6 @@ config CMD_BSP option provides a way to control this. The commands that are enabled vary depending on the board. -config CMD_BKOPS_ENABLE - bool "mmc bkops enable" - depends on CMD_MMC - default n - help - Enable command for setting manual background operations handshake - on a eMMC device. The feature is optionally available on eMMC devices - conforming to standard >= 4.41. - config CMD_BLOCK_CACHE bool "blkcache - control and stats for block cache" depends on BLOCK_CACHE @@ -54,6 +54,8 @@ static void print_mmcinfo(struct mmc *mmc) if (!IS_SD(mmc) && mmc->version >= MMC_VERSION_4_41) { bool has_enh = (mmc->part_support & ENHNCD_SUPPORT) != 0; bool usr_enh = has_enh && (mmc->part_attr & EXT_CSD_ENH_USR); + u8 wp, ext_csd[MMC_MAX_BLOCK_LEN]; + int ret; #if CONFIG_IS_ENABLED(MMC_HW_PARTITIONING) puts("HC WP Group Size: "); @@ -90,6 +92,28 @@ static void print_mmcinfo(struct mmc *mmc) putc('\n'); } } + ret = mmc_send_ext_csd(mmc, ext_csd); + if (ret) + return; + wp = ext_csd[EXT_CSD_BOOT_WP_STATUS]; + for (i = 0; i < 2; ++i) { + printf("Boot area %d is ", i); + switch (wp & 3) { + case 0: + printf("not write protected\n"); + break; + case 1: + printf("power on protected\n"); + break; + case 2: + printf("permanently protected\n"); + break; + default: + printf("in reserved protection state\n"); + break; + } + wp >>= 2; + } } } static struct mmc *init_mmc_device(int dev, bool force_init) @@ -872,9 +896,30 @@ static int do_mmc_bkops_enable(cmd_tbl_t *cmdtp, int flag, } #endif +static int do_mmc_boot_wp(cmd_tbl_t *cmdtp, int flag, + int argc, char * const argv[]) +{ + int err; + struct mmc *mmc; + + mmc = init_mmc_device(curr_device, false); + if (!mmc) + return CMD_RET_FAILURE; + if (IS_SD(mmc)) { + printf("It is not an eMMC device\n"); + return CMD_RET_FAILURE; + } + err = mmc_boot_wp(mmc); + if (err) + return CMD_RET_FAILURE; + printf("boot areas protected\n"); + return CMD_RET_SUCCESS; +} + static cmd_tbl_t cmd_mmc[] = { U_BOOT_CMD_MKENT(info, 1, 0, do_mmcinfo, "", ""), U_BOOT_CMD_MKENT(read, 4, 1, do_mmc_read, "", ""), + U_BOOT_CMD_MKENT(wp, 1, 0, do_mmc_boot_wp, "", ""), #if CONFIG_IS_ENABLED(MMC_WRITE) U_BOOT_CMD_MKENT(write, 4, 0, do_mmc_write, "", ""), U_BOOT_CMD_MKENT(erase, 3, 0, do_mmc_erase, "", ""), @@ -944,6 +989,7 @@ U_BOOT_CMD( "mmc part - lists available partition on current mmc device\n" "mmc dev [dev] [part] - show or set current mmc device [partition]\n" "mmc list - lists available devices\n" + "mmc wp - power on write protect booot partitions\n" #if CONFIG_IS_ENABLED(MMC_HW_PARTITIONING) "mmc hwpartition [args...] - does hardware partitioning\n" " arguments (sizes in 512-byte blocks):\n" diff --git a/common/bouncebuf.c b/common/bouncebuf.c index 614eb36c78..0ace152b98 100644 --- a/common/bouncebuf.c +++ b/common/bouncebuf.c @@ -31,17 +31,19 @@ static int addr_aligned(struct bounce_buffer *state) return 1; } -int bounce_buffer_start(struct bounce_buffer *state, void *data, - size_t len, unsigned int flags) +int bounce_buffer_start_extalign(struct bounce_buffer *state, void *data, + size_t len, unsigned int flags, + size_t alignment, + int (*addr_is_aligned)(struct bounce_buffer *state)) { state->user_buffer = data; state->bounce_buffer = data; state->len = len; - state->len_aligned = roundup(len, ARCH_DMA_MINALIGN); + state->len_aligned = roundup(len, alignment); state->flags = flags; - if (!addr_aligned(state)) { - state->bounce_buffer = memalign(ARCH_DMA_MINALIGN, + if (!addr_is_aligned(state)) { + state->bounce_buffer = memalign(alignment, state->len_aligned); if (!state->bounce_buffer) return -ENOMEM; @@ -62,6 +64,14 @@ int bounce_buffer_start(struct bounce_buffer *state, void *data, return 0; } +int bounce_buffer_start(struct bounce_buffer *state, void *data, + size_t len, unsigned int flags) +{ + return bounce_buffer_start_extalign(state, data, len, flags, + ARCH_DMA_MINALIGN, + addr_aligned); +} + int bounce_buffer_stop(struct bounce_buffer *state) { if (state->flags & GEN_BB_WRITE) { diff --git a/common/spl/spl_mmc.c b/common/spl/spl_mmc.c index a2ea363e96..a68cdec8dc 100644 --- a/common/spl/spl_mmc.c +++ b/common/spl/spl_mmc.c @@ -298,7 +298,7 @@ static int spl_mmc_do_fs_boot(struct spl_image_info *spl_image, struct mmc *mmc, } #endif -u32 __weak spl_boot_mode(const u32 boot_device) +u32 __weak spl_mmc_boot_mode(const u32 boot_device) { #if defined(CONFIG_SPL_FS_FAT) || defined(CONFIG_SPL_FS_EXT4) return MMCSD_MODE_FS; @@ -310,8 +310,7 @@ u32 __weak spl_boot_mode(const u32 boot_device) } #ifdef CONFIG_SYS_MMCSD_RAW_MODE_U_BOOT_USE_PARTITION -__weak -int spl_boot_partition(const u32 boot_device) +int __weak spl_mmc_boot_partition(const u32 boot_device) { return CONFIG_SYS_MMCSD_RAW_MODE_U_BOOT_PARTITION; } @@ -350,7 +349,7 @@ int spl_mmc_load(struct spl_image_info *spl_image, } } - boot_mode = spl_boot_mode(bootdev->boot_device); + boot_mode = spl_mmc_boot_mode(bootdev->boot_device); err = -EINVAL; switch (boot_mode) { case MMCSD_MODE_EMMCBOOT: @@ -431,7 +430,7 @@ int spl_mmc_load_image(struct spl_image_info *spl_image, NULL, #endif #ifdef CONFIG_SYS_MMCSD_RAW_MODE_U_BOOT_PARTITION - spl_boot_partition(bootdev->boot_device), + spl_mmc_boot_partition(bootdev->boot_device), #else 0, #endif diff --git a/drivers/mmc/Kconfig b/drivers/mmc/Kconfig index bb38787eca..8f0df568b9 100644 --- a/drivers/mmc/Kconfig +++ b/drivers/mmc/Kconfig @@ -358,6 +358,7 @@ config RENESAS_SDHI depends on ARCH_RMOBILE depends on BLK && DM_MMC depends on OF_CONTROL + select BOUNCE_BUFFER help This selects support for the Matsushita SD/MMC Host Controller on Renesas R-Car SoCs. diff --git a/drivers/mmc/Makefile b/drivers/mmc/Makefile index 615b724bf0..e84c792999 100644 --- a/drivers/mmc/Makefile +++ b/drivers/mmc/Makefile @@ -38,7 +38,7 @@ obj-$(CONFIG_MMC_MXC) += mxcmmc.o obj-$(CONFIG_MMC_MXS) += mxsmmc.o obj-$(CONFIG_MMC_PCI) += pci_mmc.o obj-$(CONFIG_PXA_MMC_GENERIC) += pxa_mmc_gen.o -obj-$(CONFIG_SUPPORT_EMMC_RPMB) += rpmb.o +obj-$(CONFIG_$(SPL_TPL_)SUPPORT_EMMC_RPMB) += rpmb.o obj-$(CONFIG_MMC_SANDBOX) += sandbox_mmc.o obj-$(CONFIG_SH_MMCIF) += sh_mmcif.o obj-$(CONFIG_SH_SDHI) += sh_sdhi.o diff --git a/drivers/mmc/iproc_sdhci.c b/drivers/mmc/iproc_sdhci.c index 831dd32eb7..c2319b4134 100644 --- a/drivers/mmc/iproc_sdhci.c +++ b/drivers/mmc/iproc_sdhci.c @@ -136,7 +136,7 @@ static void sdhci_iproc_writeb(struct sdhci_host *host, u8 val, int reg) } #endif -static void sdhci_iproc_set_ios_post(struct sdhci_host *host) +static int sdhci_iproc_set_ios_post(struct sdhci_host *host) { u32 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2); @@ -147,6 +147,8 @@ static void sdhci_iproc_set_ios_post(struct sdhci_host *host) ctrl |= UHS_DDR50_BUS_SPEED; sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2); + + return 0; } static struct sdhci_ops sdhci_platform_ops = { @@ -176,8 +178,7 @@ static int iproc_sdhci_probe(struct udevice *dev) u32 f_min_max[2]; int ret; - iproc_host = (struct sdhci_iproc_host *) - malloc(sizeof(struct sdhci_iproc_host)); + iproc_host = malloc(sizeof(struct sdhci_iproc_host)); if (!iproc_host) { printf("%s: sdhci host malloc fail!\n", __func__); return -ENOMEM; @@ -189,7 +190,7 @@ static int iproc_sdhci_probe(struct udevice *dev) host->ioaddr = (void *)devfdt_get_addr(dev); host->voltages = MMC_VDD_165_195 | MMC_VDD_32_33 | MMC_VDD_33_34; - host->quirks = SDHCI_QUIRK_BROKEN_VOLTAGE; + host->quirks = SDHCI_QUIRK_BROKEN_VOLTAGE | SDHCI_QUIRK_BROKEN_R1B; host->host_caps = MMC_MODE_DDR_52MHz; host->index = fdtdec_get_uint(gd->fdt_blob, node, "index", 0); host->ops = &sdhci_platform_ops; @@ -198,6 +199,7 @@ static int iproc_sdhci_probe(struct udevice *dev) "clock-freq-min-max", f_min_max, 2); if (ret) { printf("sdhci: clock-freq-min-max not found\n"); + free(iproc_host); return ret; } host->max_clk = f_min_max[1]; @@ -210,16 +212,18 @@ static int iproc_sdhci_probe(struct udevice *dev) memcpy(&iproc_host->host, host, sizeof(struct sdhci_host)); - ret = sdhci_setup_cfg(&plat->cfg, &iproc_host->host, - f_min_max[1], f_min_max[0]); - if (ret) - return ret; - iproc_host->host.mmc = &plat->mmc; iproc_host->host.mmc->dev = dev; iproc_host->host.mmc->priv = &iproc_host->host; upriv->mmc = iproc_host->host.mmc; + ret = sdhci_setup_cfg(&plat->cfg, &iproc_host->host, + f_min_max[1], f_min_max[0]); + if (ret) { + free(iproc_host); + return ret; + } + return sdhci_probe(dev); } diff --git a/drivers/mmc/mmc-uclass.c b/drivers/mmc/mmc-uclass.c index c75892a72c..cb26d841be 100644 --- a/drivers/mmc/mmc-uclass.c +++ b/drivers/mmc/mmc-uclass.c @@ -13,6 +13,22 @@ #include <linux/compat.h> #include "mmc_private.h" +int dm_mmc_get_b_max(struct udevice *dev, void *dst, lbaint_t blkcnt) +{ + struct dm_mmc_ops *ops = mmc_get_ops(dev); + struct mmc *mmc = mmc_get_mmc_dev(dev); + + if (ops->get_b_max) + return ops->get_b_max(dev, dst, blkcnt); + else + return mmc->cfg->b_max; +} + +int mmc_get_b_max(struct mmc *mmc, void *dst, lbaint_t blkcnt) +{ + return dm_mmc_get_b_max(mmc->dev, dst, blkcnt); +} + int dm_mmc_send_cmd(struct udevice *dev, struct mmc_cmd *cmd, struct mmc_data *data) { diff --git a/drivers/mmc/mmc.c b/drivers/mmc/mmc.c index 3e36566693..523c055967 100644 --- a/drivers/mmc/mmc.c +++ b/drivers/mmc/mmc.c @@ -409,6 +409,16 @@ static int mmc_read_blocks(struct mmc *mmc, void *dst, lbaint_t start, return blkcnt; } +#if !CONFIG_IS_ENABLED(DM_MMC) +static int mmc_get_b_max(struct mmc *mmc, void *dst, lbaint_t blkcnt) +{ + if (mmc->cfg->ops->get_b_max) + return mmc->cfg->ops->get_b_max(mmc, dst, blkcnt); + else + return mmc->cfg->b_max; +} +#endif + #if CONFIG_IS_ENABLED(BLK) ulong mmc_bread(struct udevice *dev, lbaint_t start, lbaint_t blkcnt, void *dst) #else @@ -422,6 +432,7 @@ ulong mmc_bread(struct blk_desc *block_dev, lbaint_t start, lbaint_t blkcnt, int dev_num = block_dev->devnum; int err; lbaint_t cur, blocks_todo = blkcnt; + uint b_max; if (blkcnt == 0) return 0; @@ -451,9 +462,10 @@ ulong mmc_bread(struct blk_desc *block_dev, lbaint_t start, lbaint_t blkcnt, return 0; } + b_max = mmc_get_b_max(mmc, dst, blkcnt); + do { - cur = (blocks_todo > mmc->cfg->b_max) ? - mmc->cfg->b_max : blocks_todo; + cur = (blocks_todo > b_max) ? b_max : blocks_todo; if (mmc_read_blocks(mmc, dst, start, cur) != cur) { pr_debug("%s: Failed to read blocks\n", __func__); return 0; @@ -718,7 +730,7 @@ static int mmc_complete_op_cond(struct mmc *mmc) } -static int mmc_send_ext_csd(struct mmc *mmc, u8 *ext_csd) +int mmc_send_ext_csd(struct mmc *mmc, u8 *ext_csd) { struct mmc_cmd cmd; struct mmc_data data; @@ -810,6 +822,11 @@ int mmc_switch(struct mmc *mmc, u8 set, u8 index, u8 value) return __mmc_switch(mmc, set, index, value, true); } +int mmc_boot_wp(struct mmc *mmc) +{ + return mmc_switch(mmc, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_BOOT_WP, 1); +} + #if !CONFIG_IS_ENABLED(MMC_TINY) static int mmc_set_card_speed(struct mmc *mmc, enum bus_mode mode, bool hsdowngrade) diff --git a/drivers/mmc/renesas-sdhi.c b/drivers/mmc/renesas-sdhi.c index c3b13136f8..88a7160b0a 100644 --- a/drivers/mmc/renesas-sdhi.c +++ b/drivers/mmc/renesas-sdhi.c @@ -4,6 +4,7 @@ */ #include <common.h> +#include <bouncebuf.h> #include <clk.h> #include <fdtdec.h> #include <malloc.h> @@ -689,12 +690,94 @@ static int renesas_sdhi_wait_dat0(struct udevice *dev, int state, } #endif +#define RENESAS_SDHI_DMA_ALIGNMENT 128 + +static int renesas_sdhi_addr_aligned_gen(uintptr_t ubuf, + size_t len, size_t len_aligned) +{ + /* Check if start is aligned */ + if (!IS_ALIGNED(ubuf, RENESAS_SDHI_DMA_ALIGNMENT)) { + debug("Unaligned buffer address %lx\n", ubuf); + return 0; + } + + /* Check if length is aligned */ + if (len != len_aligned) { + debug("Unaligned buffer length %zu\n", len); + return 0; + } + +#ifdef CONFIG_PHYS_64BIT + /* Check if below 32bit boundary */ + if ((ubuf >> 32) || (ubuf + len_aligned) >> 32) { + debug("Buffer above 32bit boundary %lx-%lx\n", + ubuf, ubuf + len_aligned); + return 0; + } +#endif + + /* Aligned */ + return 1; +} + +static int renesas_sdhi_addr_aligned(struct bounce_buffer *state) +{ + uintptr_t ubuf = (uintptr_t)state->user_buffer; + + return renesas_sdhi_addr_aligned_gen(ubuf, state->len, + state->len_aligned); +} + static int renesas_sdhi_send_cmd(struct udevice *dev, struct mmc_cmd *cmd, struct mmc_data *data) { + struct bounce_buffer bbstate; + unsigned int bbflags; + bool bbok = false; + size_t len; + void *buf; int ret; + if (data) { + if (data->flags & MMC_DATA_READ) { + buf = data->dest; + bbflags = GEN_BB_WRITE; + } else { + buf = (void *)data->src; + bbflags = GEN_BB_READ; + } + len = data->blocks * data->blocksize; + + ret = bounce_buffer_start_extalign(&bbstate, buf, len, bbflags, + RENESAS_SDHI_DMA_ALIGNMENT, + renesas_sdhi_addr_aligned); + /* + * If the amount of data to transfer is too large, we can get + * -ENOMEM when starting the bounce buffer. If that happens, + * fall back to PIO as it was before, otherwise use the BB. + */ + if (!ret) { + bbok = true; + if (data->flags & MMC_DATA_READ) + data->dest = bbstate.bounce_buffer; + else + data->src = bbstate.bounce_buffer; + } + } + ret = tmio_sd_send_cmd(dev, cmd, data); + + if (data && bbok) { + buf = bbstate.user_buffer; + + bounce_buffer_stop(&bbstate); + + if (data->flags & MMC_DATA_READ) + data->dest = buf; + else + data->src = buf; + } + if (ret) return ret; @@ -712,6 +795,24 @@ static int renesas_sdhi_send_cmd(struct udevice *dev, struct mmc_cmd *cmd, return 0; } +int renesas_sdhi_get_b_max(struct udevice *dev, void *dst, lbaint_t blkcnt) +{ + struct tmio_sd_priv *priv = dev_get_priv(dev); + struct mmc_uclass_priv *upriv = dev_get_uclass_priv(dev); + struct mmc *mmc = upriv->mmc; + size_t len = blkcnt * mmc->read_bl_len; + size_t len_align = roundup(len, RENESAS_SDHI_DMA_ALIGNMENT); + + if (renesas_sdhi_addr_aligned_gen((uintptr_t)dst, len, len_align)) { + if (priv->quirks & TMIO_SD_CAP_16BIT) + return U16_MAX; + else + return U32_MAX; + } else { + return (CONFIG_SYS_MALLOC_LEN / 4) / mmc->read_bl_len; + } +} + static const struct dm_mmc_ops renesas_sdhi_ops = { .send_cmd = renesas_sdhi_send_cmd, .set_ios = renesas_sdhi_set_ios, @@ -724,6 +825,7 @@ static const struct dm_mmc_ops renesas_sdhi_ops = { #if CONFIG_IS_ENABLED(MMC_UHS_SUPPORT) .wait_dat0 = renesas_sdhi_wait_dat0, #endif + .get_b_max = renesas_sdhi_get_b_max, }; #define RENESAS_GEN2_QUIRKS TMIO_SD_CAP_RCAR_GEN2 @@ -889,6 +991,7 @@ static int renesas_sdhi_probe(struct udevice *dev) return ret; } + priv->quirks = quirks; ret = tmio_sd_probe(dev, quirks); renesas_sdhi_filter_caps(dev); diff --git a/drivers/mmc/tmio-common.c b/drivers/mmc/tmio-common.c index faf18191b3..1dc13db9ea 100644 --- a/drivers/mmc/tmio-common.c +++ b/drivers/mmc/tmio-common.c @@ -358,14 +358,16 @@ static int tmio_sd_dma_xfer(struct udevice *dev, struct mmc_data *data) } /* check if the address is DMA'able */ -static bool tmio_sd_addr_is_dmaable(const char *src) +static bool tmio_sd_addr_is_dmaable(struct mmc_data *data) { - uintptr_t addr = (uintptr_t)src; + uintptr_t addr = (uintptr_t)data->src; if (!IS_ALIGNED(addr, TMIO_SD_DMA_MINALIGN)) return false; #if defined(CONFIG_RCAR_GEN3) + if (!(data->flags & MMC_DATA_READ) && !IS_ALIGNED(addr, 128)) + return false; /* Gen3 DMA has 32bit limit */ if (addr >> 32) return false; @@ -480,7 +482,7 @@ int tmio_sd_send_cmd(struct udevice *dev, struct mmc_cmd *cmd, if (data) { /* use DMA if the HW supports it and the buffer is aligned */ if (priv->caps & TMIO_SD_CAP_DMA_INTERNAL && - tmio_sd_addr_is_dmaable(data->src)) + tmio_sd_addr_is_dmaable(data)) ret = tmio_sd_dma_xfer(dev, data); else ret = tmio_sd_pio_xfer(dev, cmd, data); diff --git a/drivers/mmc/tmio-common.h b/drivers/mmc/tmio-common.h index 047458849b..2f671df4bc 100644 --- a/drivers/mmc/tmio-common.h +++ b/drivers/mmc/tmio-common.h @@ -147,6 +147,7 @@ struct tmio_sd_priv { u8 adjust_hs400_calibrate; u8 hs400_bad_tap; const u8 *adjust_hs400_calib_table; + u32 quirks; #endif ulong (*clk_get_rate)(struct tmio_sd_priv *); }; diff --git a/include/bouncebuf.h b/include/bouncebuf.h index fd9b0f3b28..7427bd12e2 100644 --- a/include/bouncebuf.h +++ b/include/bouncebuf.h @@ -62,6 +62,21 @@ struct bounce_buffer { */ int bounce_buffer_start(struct bounce_buffer *state, void *data, size_t len, unsigned int flags); + +/** + * bounce_buffer_start() -- Start the bounce buffer session with external align check function + * state: stores state passed between bounce_buffer_{start,stop} + * data: pointer to buffer to be aligned + * len: length of the buffer + * flags: flags describing the transaction, see above. + * alignment: alignment of the newly allocated bounce buffer + * addr_is_aligned: function for checking the alignment instead of the default one + */ +int bounce_buffer_start_extalign(struct bounce_buffer *state, void *data, + size_t len, unsigned int flags, + size_t alignment, + int (*addr_is_aligned)(struct bounce_buffer *state)); + /** * bounce_buffer_stop() -- Finish the bounce buffer session * state: stores state passed between bounce_buffer_{start,stop} diff --git a/include/configs/rcar-gen3-common.h b/include/configs/rcar-gen3-common.h index 6528f1fa62..8f400ba05a 100644 --- a/include/configs/rcar-gen3-common.h +++ b/include/configs/rcar-gen3-common.h @@ -47,7 +47,7 @@ #define CONFIG_SYS_MONITOR_BASE 0x00000000 #define CONFIG_SYS_MONITOR_LEN (1 * 1024 * 1024) -#define CONFIG_SYS_MALLOC_LEN (1 * 1024 * 1024) +#define CONFIG_SYS_MALLOC_LEN (64 * 1024 * 1024) #define CONFIG_SYS_BOOTM_LEN (64 << 20) /* The HF/QSPI layout permits up to 1 MiB large bootloader blob */ diff --git a/include/mmc.h b/include/mmc.h index e83c22423b..5e9d15cb41 100644 --- a/include/mmc.h +++ b/include/mmc.h @@ -223,6 +223,9 @@ static inline bool mmc_is_tuning_cmd(uint cmdidx) #define EXT_CSD_WR_REL_PARAM 166 /* R */ #define EXT_CSD_WR_REL_SET 167 /* R/W */ #define EXT_CSD_RPMB_MULT 168 /* RO */ +#define EXT_CSD_USER_WP 171 /* R/W & R/W/C_P & R/W/E_P */ +#define EXT_CSD_BOOT_WP 173 /* R/W & R/W/C_P */ +#define EXT_CSD_BOOT_WP_STATUS 174 /* R */ #define EXT_CSD_ERASE_GROUP_DEF 175 /* R/W */ #define EXT_CSD_BOOT_BUS_WIDTH 177 #define EXT_CSD_PART_CONF 179 /* R/W */ @@ -488,6 +491,19 @@ struct dm_mmc_ops { * @return 0 if not present, 1 if present, -ve on error */ int (*host_power_cycle)(struct udevice *dev); + + /** + * get_b_max - get maximum length of single transfer + * Called before reading blocks from the card, + * useful for system which have e.g. DMA limits + * on various memory ranges. + * + * @dev: Device to check + * @dst: Destination buffer in memory + * @blkcnt: Total number of blocks in this transfer + * @return maximum number of blocks for this transfer + */ + int (*get_b_max)(struct udevice *dev, void *dst, lbaint_t blkcnt); }; #define mmc_get_ops(dev) ((struct dm_mmc_ops *)(dev)->driver->ops) @@ -501,6 +517,7 @@ int dm_mmc_execute_tuning(struct udevice *dev, uint opcode); int dm_mmc_wait_dat0(struct udevice *dev, int state, int timeout_us); int dm_mmc_host_power_cycle(struct udevice *dev); int dm_mmc_deferred_probe(struct udevice *dev); +int dm_mmc_get_b_max(struct udevice *dev, void *dst, lbaint_t blkcnt); /* Transition functions for compatibility */ int mmc_set_ios(struct mmc *mmc); @@ -511,6 +528,7 @@ int mmc_wait_dat0(struct mmc *mmc, int state, int timeout_us); int mmc_set_enhanced_strobe(struct mmc *mmc); int mmc_host_power_cycle(struct mmc *mmc); int mmc_deferred_probe(struct mmc *mmc); +int mmc_get_b_max(struct mmc *mmc, void *dst, lbaint_t blkcnt); #else struct mmc_ops { @@ -521,6 +539,7 @@ struct mmc_ops { int (*getcd)(struct mmc *mmc); int (*getwp)(struct mmc *mmc); int (*host_power_cycle)(struct mmc *mmc); + int (*get_b_max)(struct mmc *mmc, void *dst, lbaint_t blkcnt); }; #endif @@ -893,6 +912,26 @@ int mmc_get_env_dev(void); */ struct blk_desc *mmc_get_blk_desc(struct mmc *mmc); +/** + * mmc_send_ext_csd() - read the extended CSD register + * + * @mmc: MMC device + * @ext_csd a cache aligned buffer of length MMC_MAX_BLOCK_LEN allocated by + * the caller, e.g. using + * ALLOC_CACHE_ALIGN_BUFFER(u8, ext_csd, MMC_MAX_BLOCK_LEN) + * Return: 0 for success + */ +int mmc_send_ext_csd(struct mmc *mmc, u8 *ext_csd); + +/** + * mmc_boot_wp() - power on write protect boot partitions + * + * The boot partitions are write protected until the next power cycle. + * + * Return: 0 for success + */ +int mmc_boot_wp(struct mmc *mmc); + static inline enum dma_data_direction mmc_get_dma_dir(struct mmc_data *data) { return data->flags & MMC_DATA_WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE; diff --git a/include/spl.h b/include/spl.h index 5d8d14dbf5..8b15cd4914 100644 --- a/include/spl.h +++ b/include/spl.h @@ -238,8 +238,36 @@ int spl_load_imx_container(struct spl_image_info *spl_image, /* SPL common functions */ void preloader_console_init(void); u32 spl_boot_device(void); -u32 spl_boot_mode(const u32 boot_device); -int spl_boot_partition(const u32 boot_device); + +/** + * spl_mmc_boot_mode() - Lookup function for the mode of an MMC boot source. + * @boot_device: ID of the device which the MMC driver wants to read + * from. Common values are e.g. BOOT_DEVICE_MMC1, + * BOOT_DEVICE_MMC2, BOOT_DEVICE_MMC2_2. + * + * This function should return one of MMCSD_MODE_FS, MMCSD_MODE_EMMCBOOT, or + * MMCSD_MODE_RAW for each MMC boot source which is defined for the target. The + * boot_device parameter tells which device the MMC driver is interested in. + * + * If not overridden, it is weakly defined in common/spl/spl_mmc.c. + * + * Note: It is important to use the boot_device parameter instead of e.g. + * spl_boot_device() as U-Boot is not always loaded from the same device as SPL. + */ +u32 spl_mmc_boot_mode(const u32 boot_device); + +/** + * spl_mmc_boot_partition() - MMC partition to load U-Boot from. + * @boot_device: ID of the device which the MMC driver wants to load + * U-Boot from. + * + * This function should return the partition number which the SPL + * should load U-Boot from (on the given boot_device) when + * CONFIG_SYS_MMCSD_RAW_MODE_U_BOOT_USE_PARTITION is set. + * + * If not overridden, it is weakly defined in common/spl/spl_mmc.c. + */ +int spl_mmc_boot_partition(const u32 boot_device); void spl_set_bd(void); /** diff --git a/tools/buildman/README b/tools/buildman/README index 4cf0114157..f3a0dc7288 100644 --- a/tools/buildman/README +++ b/tools/buildman/README @@ -51,23 +51,25 @@ Theory of Operation Buildman is a builder. It is not make, although it runs make. It does not produce any useful output on the terminal while building, except for -progress information (except with -v, see below). All the output (errors, -warnings and binaries if you ask for them) is stored in output -directories, which you can look at while the build is progressing, or when -it is finished. +progress information (but see -v below). All the output (errors, warnings and +binaries if you ask for them) is stored in output directories, which you can +look at from a separate 'buildman -s' instance while the build is progressing, +or when it is finished. Buildman is designed to build entire git branches, i.e. muliple commits. It -can be run repeatedly on the same branch. In this case it will automatically -rebuild commits which have changed (and remove its old results for that -commit). It is possible to build a branch for one board, then later build it -for another board. If you want buildman to re-build a commit it has already -built (e.g. because of a toolchain update), use the -f flag. +can be run repeatedly on the same branch after making changes to commits on +that branch. In this case it will automatically rebuild commits which have +changed (and remove its old results for that commit). It is possible to build +a branch for one board, then later build it for another board. This adds to +the output, so now you have results for two boards. If you want buildman to +re-build a commit it has already built (e.g. because of a toolchain update), +use the -f flag. Buildman produces a concise summary of which boards succeeded and failed. It shows which commit introduced which board failure using a simple -red/green colour coding. Full error information can be requested, in which -case it is de-duped and displayed against the commit that introduced the -error. An example workflow is below. +red/green colour coding (with yellow/cyan for warnings). Full error +information can be requested, in which case it is de-duped and displayed +against the commit that introduced the error. An example workflow is below. Buildman stores image size information and can report changes in image size from commit to commit. An example of this is below. @@ -75,16 +77,20 @@ from commit to commit. An example of this is below. Buildman starts multiple threads, and each thread builds for one board at a time. A thread starts at the first commit, configures the source for your board and builds it. Then it checks out the next commit and does an -incremental build. Eventually the thread reaches the last commit and stops. -If errors or warnings are found along the way, the thread will reconfigure -after every commit, and your build will be very slow. This is because a -file that produces just a warning would not normally be rebuilt in an -incremental build. +incremental build (i.e. not using 'make xxx_defconfig' unless you use -C). +Eventually the thread reaches the last commit and stops. If a commit causes +an error or warning, buildman will try it again after reconfiguring (but see +-Q). Thus some commits may be built twice, with the first result silently +discarded. Lots of errors and warnings will causes lots of reconfigures and your +build will be very slow. This is because a file that produces just a warning +would not normally be rebuilt in an incremental build. Once a thread finishes +building all the commits for a board, it starts on the commits for another +board. Buildman works in an entirely separate place from your U-Boot repository. It creates a separate working directory for each thread, and puts the output files in the working directory, organised by commit name and board -name, in a two-level hierarchy. +name, in a two-level hierarchy (but see -P). Buildman is invoked in your U-Boot directory, the one with the .git directory. It clones this repository into a copy for each thread, and the @@ -92,20 +98,23 @@ threads do not affect the state of your git repository. Any checkouts done by the thread affect only the working directory for that thread. Buildman automatically selects the correct tool chain for each board. You -must supply suitable tool chains, but buildman takes care of selecting the -right one. +must supply suitable tool chains (see --fetch-arch), but buildman takes care +of selecting the right one. Buildman generally builds a branch (with the -b flag), and in this case -builds the upstream commit as well, for comparison. It cannot build -individual commits at present, unless (maybe) you point it at an empty -branch. Put all your commits in a branch, set the branch's upstream to a -valid value, and all will be well. Otherwise buildman will perform random -actions. Use -n to check what the random actions might be. +builds the upstream commit as well, for comparison. So even if you have one +commit in your branch, two commits will be built. Put all your commits in a +branch, set the branch's upstream to a valid value, and all will be well. +Otherwise buildman will perform random actions. Use -n to check what the +random actions might be. + +Buildman effectively has two modes: without -s it builds, with -s it +summarises the results of previous (or active) builds. -If you just want to build the current source tree, leave off the -b flag -and add -e. This will display results and errors as they happen. You can -still look at them later using -se. Note that buildman will assume that the -source has changed, and will build all specified boards in this case. +If you just want to build the current source tree, leave off the -b flag. +This will display results and errors as they happen. You can still look at +them later using -se. Note that buildman will assume that the source has +changed, and will build all specified boards in this case. Buildman is optimised for building many commits at once, for many boards. On multi-core machines, Buildman is fast because it uses most of the @@ -142,9 +151,9 @@ You can also use -x to specifically exclude some boards. For example: means to build all arm boards except nvidia, freescale and anything ending with 'ball'. -For building specific boards you can use the --boards option, which takes a -comma-separated list of board target names and be used multiple times on -the command line: +For building specific boards you can use the --boards (or --bo) option, which +takes a comma-separated list of board target names and be used multiple times +on the command line: buildman --boards sandbox,snow --boards @@ -492,6 +501,8 @@ If it can't detect the upstream branch, try checking out the branch, and doing something like 'git branch --set-upstream-to upstream/master' or something similar. Buildman will try to guess a suitable upstream branch if it can't find one (you will see a message like" Guessing upstream as ...). +You can also use the -c option to manually specify the number of commits to +build. As an example: @@ -542,12 +553,13 @@ Buildman will set up some working directories, and get started. After a minute or so it will settle down to a steady pace, with a display like this: Building 18 commits for 1059 boards (4 threads, 1 job per thread) - 528 36 124 /19062 1:13:30 : SIMPC8313_SP + 528 36 124 /19062 -18374 1:13:30 : SIMPC8313_SP This means that it is building 19062 board/commit combinations. So far it has managed to successfully build 528. Another 36 have built with warnings, -and 124 more didn't build at all. Buildman expects to complete the process -in around an hour and a quarter. Use this time to buy a faster computer. +and 124 more didn't build at all. It has 18374 builds left to complete. +Buildman expects to complete the process in around an hour and a quarter. +Use this time to buy a faster computer. To find out how the build went, ask for a summary with -s. You can do this @@ -579,32 +591,32 @@ $ ./tools/buildman/buildman -b lcd9b -s This shows which commits have succeeded and which have failed. In this case the build is still in progress so many boards are not built yet (use -u to -see which ones). But still we can see a few failures. The galaxy5200_LOWBOOT +see which ones). But already we can see a few failures. The galaxy5200_LOWBOOT never builds correctly. This could be a problem with our toolchain, or it could be a bug in the upstream. The good news is that we probably don't need to blame our commits. The bad news is that our commits are not tested on that board. -Commit 12 broke lubbock. That's what the '+ lubbock' means. The failure -is never fixed by a later commit, or you would see lubbock again, in green, -without the +. +Commit 12 broke lubbock. That's what the '+ lubbock', in red, means. The +failure is never fixed by a later commit, or you would see lubbock again, in +green, without the +. To see the actual error: -$ ./tools/buildman/buildman -b <branch> -se lubbock +$ ./tools/buildman/buildman -b <branch> -se ... 12: lcd: Add support for flushing LCD fb from dcache after update arm: + lubbock +common/libcommon.o: In function `lcd_sync': -+/u-boot/lcd9b/.bm-work/00/common/lcd.c:120: undefined reference to `flush_dcache_range' ++common/lcd.c:120: undefined reference to `flush_dcache_range' +arm-none-linux-gnueabi-ld: BFD (Sourcery G++ Lite 2010q1-202) 2.19.51.20090709 assertion fail /scratch/julian/2010q1-release-linux-lite/obj/binutils-src-2010q1-202-arm-none-linux-gnueabi-i686-pc-linux-gnu/bfd/elf32-arm.c:12572 -+make: *** [/u-boot/lcd9b/.bm-work/00/build/u-boot] Error 139 ++make: *** [build/u-boot] Error 139 13: tegra: Align LCD frame buffer to section boundary 14: tegra: Support control of cache settings for LCD 15: tegra: fdt: Add LCD definitions for Seaboard 16: lcd: Add CONFIG_CONSOLE_SCROLL_LINES option to speed console --/u-boot/lcd9b/.bm-work/00/common/lcd.c:120: undefined reference to `flush_dcache_range' -+/u-boot/lcd9b/.bm-work/00/common/lcd.c:125: undefined reference to `flush_dcache_range' +-common/lcd.c:120: undefined reference to `flush_dcache_range' ++common/lcd.c:125: undefined reference to `flush_dcache_range' 17: tegra: Enable display/lcd support on Seaboard 18: wip @@ -612,6 +624,21 @@ So the problem is in lcd.c, due to missing cache operations. This information should be enough to work out what that commit is doing to break these boards. (In this case pxa did not have cache operations defined). +Note that if there were other boards with errors, the above command would +show their errors also. Each line is shown only once. So if lubbock and snow +produce the same error, we just see: + +12: lcd: Add support for flushing LCD fb from dcache after update + arm: + lubbock snow ++common/libcommon.o: In function `lcd_sync': ++common/lcd.c:120: undefined reference to `flush_dcache_range' ++arm-none-linux-gnueabi-ld: BFD (Sourcery G++ Lite 2010q1-202) 2.19.51.20090709 assertion fail /scratch/julian/2010q1-release-linux-lite/obj/binutils-src-2010q1-202-arm-none-linux-gnueabi-i686-pc-linux-gnu/bfd/elf32-arm.c:12572 ++make: *** [build/u-boot] Error 139 + +But if you did want to see just the errors for lubbock, use: + +$ ./tools/buildman/buildman -b <branch> -se lubbock + If you see error lines marked with '-', that means that the errors were fixed by that commit. Sometimes commits can be in the wrong order, so that a breakage is introduced for a few commits and fixed by later commits. This @@ -622,13 +649,14 @@ At commit 16, the error moves: you can see that the old error at line 120 is fixed, but there is a new one at line 126. This is probably only because we added some code and moved the broken line further down the file. -If many boards have the same error, then -e will display the error only -once. This makes the output as concise as possible. To see which boards have -each error, use -l. So it is safe to omit the board name - you will not get -lots of repeated output for every board. +As mentioned, if many boards have the same error, then -e will display the +error only once. This makes the output as concise as possible. To see which +boards have each error, use -l. So it is safe to omit the board name - you +will not get lots of repeated output for every board. Buildman tries to distinguish warnings from errors, and shows warning lines -separately with a 'w' prefix. +separately with a 'w' prefix. Warnings introduced show as yellow. Warnings +fixed show as cyan. The full build output in this case is available in: @@ -930,12 +958,11 @@ will build commits in us-buildman that are not in upstream/master. Building Faster =============== -By default, buildman executes 'make mrproper' prior to building the first -commit for each board. This causes everything to be built from scratch. If you -trust the build system's incremental build capabilities, you can pass the -I -flag to skip the 'make mproper' invocation, which will reduce the amount of -work 'make' does, and hence speed up the build. This flag will speed up any -buildman invocation, since it reduces the amount of work done on any build. +By default, buildman doesn't execute 'make mrproper' prior to building the +first commit for each board. This reduces the amount of work 'make' does, and +hence speeds up the build. To force use of 'make mrproper', use -the -m flag. +This flag will slow down any buildman invocation, since it increases the amount +of work done on any build. One possible application of buildman is as part of a continual edit, build, edit, build, ... cycle; repeatedly applying buildman to the same change or @@ -966,7 +993,7 @@ Combining all of these options together yields the command-line shown below. This will provide the quickest possible feedback regarding the current content of the source tree, thus allowing rapid tested evolution of the code. - SOURCE_DATE_EPOCH=0 ./tools/buildman/buildman -I -P tegra + SOURCE_DATE_EPOCH=0 ./tools/buildman/buildman -P tegra Checking configuration @@ -1087,15 +1114,18 @@ with -E, e.g. the migration warnings: When doing builds, Buildman's return code will reflect the overall result: 0 (success) No errors or warnings found - 128 Errors found - 129 Warnings found (only if no -W) + 100 Errors found + 101 Warnings found (only if no -W) -You can use -W to tell Buildman to return 0 (success) instead of 129 when +You can use -W to tell Buildman to return 0 (success) instead of 101 when warnings are found. Note that it can be useful to combine -E and -W. This means -that all compiler warnings will produce failures (code 128) and all other -warnings will produce success (since 129 is changed to 0). +that all compiler warnings will produce failures (code 100) and all other +warnings will produce success (since 101 is changed to 0). + +If there are both warnings and errors, errors win, so buildman returns 100. -If there are both warnings and errors, errors win, so buildman returns 128. +The -y option is provided (for use with -s) to ignore the bountiful device-tree +warnings. Similarly, -Y tells buildman to ignore the migration warnings. How to change from MAKEALL @@ -1202,12 +1232,16 @@ Some options you might like are: TODO ==== -This has mostly be written in my spare time as a response to my difficulties -in testing large series of patches. Apart from tidying up there is quite a -bit of scope for improvement. Things like better error diffs and easier -access to log files. Also it would be nice if buildman could 'hunt' for -problems, perhaps by building a few boards for each arch, or checking -commits for changed files and building only boards which use those files. +Many improvements have been made over the years. There is still quite a bit of +scope for more though, e.g.: + +- easier access to log files +- 'hunting' for problems, perhaps by building a few boards for each arch, or + checking commits for changed files and building only boards which use those + files +- using the same git repo for all threads instead of cloning it. Currently + it uses about 500MB per thread, so on a 64-thread machine this is 32GB for + the build. Credits @@ -1223,3 +1257,4 @@ sjg@chromium.org Halloween 2012 Updated 12-12-12 Updated 23-02-13 +Updated 09-04-20 diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 70c55c588a..30ebe1d820 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -24,7 +24,6 @@ import terminal from terminal import Print import toolchain - """ Theory of Operation @@ -91,6 +90,15 @@ u-boot/ source directory .git/ repository """ +"""Holds information about a particular error line we are outputing + + char: Character representation: '+': error, '-': fixed error, 'w+': warning, + 'w-' = fixed warning + boards: List of Board objects which have line in the error/warning output + errline: The text of the error line +""" +ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline') + # Possible build outcomes OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4)) @@ -154,8 +162,6 @@ class Builder: force_build_failures: If a previously-built build (i.e. built on a previous run of buildman) is marked as failed, rebuild it. git_dir: Git directory containing source repository - last_line_len: Length of the last line we printed (used for erasing - it with new progress information) num_jobs: Number of jobs to run at once (passed to make as -j) num_threads: Number of builder threads to run out_queue: Queue of results to process @@ -186,6 +192,7 @@ class Builder: _next_delay_update: Next time we plan to display a progress update (datatime) _show_unknown: Show unknown boards (those not built) in summary + _start_time: Start time for the build _timestamps: List of timestamps for the completion of the last last _timestamp_count builds. Each is a datetime object. _timestamp_count: Number of timestamps to keep in our list. @@ -224,7 +231,7 @@ class Builder: def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, gnu_make='make', checkout=True, show_unknown=True, step=1, no_subdirs=False, full_path=False, verbose_build=False, - incremental=False, per_board_out_dir=False, + mrproper=False, per_board_out_dir=False, config_only=False, squash_config_y=False, warnings_as_errors=False, work_in_output=False): """Create a new Builder object @@ -245,8 +252,7 @@ class Builder: full_path: Return the full path in CROSS_COMPILE and don't set PATH verbose_build: Run build with V=1 and don't use 'make -s' - incremental: Always perform incremental builds; don't run make - mrproper when configuring + mrproper: Always run 'make mrproper' when configuring per_board_out_dir: Build in a separate persistent directory per board rather than a thread-specific directory config_only: Only configure each build, don't build it @@ -275,6 +281,7 @@ class Builder: self._build_period_us = None self._complete_delay = None self._next_delay_update = datetime.now() + self._start_time = datetime.now() self.force_config_on_failure = True self.force_build_failures = False self.force_reconfig = False @@ -299,17 +306,18 @@ class Builder: self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') self._re_dtb_warning = re.compile('(.*): Warning .*') self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') + self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n', + re.MULTILINE | re.DOTALL) self.queue = queue.Queue() self.out_queue = queue.Queue() for i in range(self.num_threads): - t = builderthread.BuilderThread(self, i, incremental, + t = builderthread.BuilderThread(self, i, mrproper, per_board_out_dir) t.setDaemon(True) t.start() self.threads.append(t) - self.last_line_len = 0 t = builderthread.ResultThread(self) t.setDaemon(True) t.start() @@ -332,16 +340,22 @@ class Builder: def SetDisplayOptions(self, show_errors=False, show_sizes=False, show_detail=False, show_bloat=False, list_error_boards=False, show_config=False, - show_environment=False): + show_environment=False, filter_dtb_warnings=False, + filter_migration_warnings=False): """Setup display options for the builder. - show_errors: True to show summarised error/warning info - show_sizes: Show size deltas - show_detail: Show size delta detail for each board if show_sizes - show_bloat: Show detail for each function - list_error_boards: Show the boards which caused each error/warning - show_config: Show config deltas - show_environment: Show environment deltas + Args: + show_errors: True to show summarised error/warning info + show_sizes: Show size deltas + show_detail: Show size delta detail for each board if show_sizes + show_bloat: Show detail for each function + list_error_boards: Show the boards which caused each error/warning + show_config: Show config deltas + show_environment: Show environment deltas + filter_dtb_warnings: Filter out any warnings from the device-tree + compiler + filter_migration_warnings: Filter out any warnings about migrating + a board to driver model """ self._show_errors = show_errors self._show_sizes = show_sizes @@ -350,6 +364,8 @@ class Builder: self._list_error_boards = list_error_boards self._show_config = show_config self._show_environment = show_environment + self._filter_dtb_warnings = filter_dtb_warnings + self._filter_migration_warnings = filter_migration_warnings def _AddTimestamp(self): """Add a new timestamp to the list and record the build period. @@ -380,22 +396,6 @@ class Builder: self._timestamps.popleft() count -= 1 - def ClearLine(self, length): - """Clear any characters on the current line - - Make way for a new line of length 'length', by outputting enough - spaces to clear out the old line. Then remember the new length for - next time. - - Args: - length: Length of new line, in characters - """ - if length < self.last_line_len: - Print(' ' * (self.last_line_len - length), newline=False) - Print('\r', newline=False) - self.last_line_len = length - sys.stdout.flush() - def SelectCommit(self, commit, checkout=True): """Checkout the selected commit for this build """ @@ -441,8 +441,7 @@ class Builder: if result.already_done: self.already_done += 1 if self._verbose: - Print('\r', newline=False) - self.ClearLine(0) + terminal.PrintClear() boards_selected = {target : result.brd} self.ResetResultSummary(boards_selected) self.ProduceResultSummary(result.commit_upto, self.commits, @@ -456,22 +455,21 @@ class Builder: line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) line += self.col.Color(self.col.RED, '%5d' % self.fail) - name = ' /%-5d ' % self.count + line += ' /%-5d ' % self.count + remaining = self.count - self.upto + if remaining: + line += self.col.Color(self.col.MAGENTA, ' -%-5d ' % remaining) + else: + line += ' ' * 8 # Add our current completion time estimate self._AddTimestamp() if self._complete_delay: - name += '%s : ' % self._complete_delay - # When building all boards for a commit, we can print a commit - # progress message. - if result and result.commit_upto is None: - name += 'commit %2d/%-3d' % (self.commit_upto + 1, - self.commit_count) - - name += target - Print(line + name, newline=False) - length = 16 + len(name) - self.ClearLine(length) + line += '%s : ' % self._complete_delay + + line += target + terminal.PrintClear() + Print(line, newline=False, limit_to_line=True) def _GetOutputDir(self, commit_upto): """Get the name of the output directory for a commit number @@ -567,9 +565,16 @@ class Builder: New list with only interesting lines included """ out_lines = [] + if self._filter_migration_warnings: + text = '\n'.join(lines) + text = self._re_migration_warning.sub('', text) + lines = text.splitlines() for line in lines: - if not self.re_make_err.search(line): - out_lines.append(line) + if self.re_make_err.search(line): + continue + if self._filter_dtb_warnings and self._re_dtb_warning.search(line): + continue + out_lines.append(line) return out_lines def ReadFuncSizes(self, fname, fd): @@ -1128,32 +1133,52 @@ class Builder: Args: line: Error line to search for + line_boards: boards to search, each a Board Return: - String containing a list of boards with that error line, or - '' if the user has not requested such a list + List of boards with that error line, or [] if the user has not + requested such a list """ + boards = [] + board_set = set() if self._list_error_boards: - names = [] for board in line_boards[line]: - if not board.target in names: - names.append(board.target) - names_str = '(%s) ' % ','.join(names) - else: - names_str = '' - return names_str + if not board in board_set: + boards.append(board) + board_set.add(board) + return boards def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, char): + """Calculate the required output based on changes in errors + + Args: + base_lines: List of errors/warnings for previous commit + base_line_boards: Dict keyed by error line, containing a list + of the Board objects with that error in the previous commit + lines: List of errors/warning for this commit, each a str + line_boards: Dict keyed by error line, containing a list + of the Board objects with that error in this commit + char: Character representing error ('') or warning ('w'). The + broken ('+') or fixed ('-') characters are added in this + function + + Returns: + Tuple + List of ErrLine objects for 'better' lines + List of ErrLine objects for 'worse' lines + """ better_lines = [] worse_lines = [] for line in lines: if line not in base_lines: - worse_lines.append(char + '+' + - _BoardList(line, line_boards) + line) + errline = ErrLine(char + '+', _BoardList(line, line_boards), + line) + worse_lines.append(errline) for line in base_lines: if line not in lines: - better_lines.append(char + '-' + - _BoardList(line, base_line_boards) + line) + errline = ErrLine(char + '-', + _BoardList(line, base_line_boards), line) + better_lines.append(errline) return better_lines, worse_lines def _CalcConfig(delta, name, config): @@ -1209,6 +1234,34 @@ class Builder: col = self.col.YELLOW Print(' ' + line, newline=True, colour=col) + def _OutputErrLines(err_lines, colour): + """Output the line of error/warning lines, if not empty + + Also increments self._error_lines if err_lines not empty + + Args: + err_lines: List of ErrLine objects, each an error or warning + line, possibly including a list of boards with that + error/warning + colour: Colour to use for output + """ + if err_lines: + out_list = [] + for line in err_lines: + boards = '' + names = [board.target for board in line.boards] + board_str = ' '.join(names) if names else '' + if board_str: + out = self.col.Color(colour, line.char + '(') + out += self.col.Color(self.col.MAGENTA, board_str, + bright=False) + out += self.col.Color(colour, ') %s' % line.errline) + else: + out = self.col.Color(colour, line.char + line.errline) + out_list.append(out) + Print('\n'.join(out_list)) + self._error_lines += 1 + ok_boards = [] # List of boards fixed since last commit warn_boards = [] # List of boards with warnings since last commit @@ -1239,7 +1292,7 @@ class Builder: else: new_boards.append(target) - # Get a list of errors that have appeared, and disappeared + # Get a list of errors and warnings that have appeared, and disappeared better_err, worse_err = _CalcErrorDelta(self._base_err_lines, self._base_err_line_boards, err_lines, err_line_boards, '') better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, @@ -1262,18 +1315,10 @@ class Builder: for arch, target_list in arch_list.items(): Print('%10s: %s' % (arch, target_list)) self._error_lines += 1 - if better_err: - Print('\n'.join(better_err), colour=self.col.GREEN) - self._error_lines += 1 - if worse_err: - Print('\n'.join(worse_err), colour=self.col.RED) - self._error_lines += 1 - if better_warn: - Print('\n'.join(better_warn), colour=self.col.CYAN) - self._error_lines += 1 - if worse_warn: - Print('\n'.join(worse_warn), colour=self.col.MAGENTA) - self._error_lines += 1 + _OutputErrLines(better_err, colour=self.col.GREEN) + _OutputErrLines(worse_err, colour=self.col.RED) + _OutputErrLines(better_warn, colour=self.col.CYAN) + _OutputErrLines(worse_warn, colour=self.col.YELLOW) if show_sizes: self.PrintSizeSummary(board_selected, board_dict, show_detail, @@ -1506,12 +1551,15 @@ class Builder: if setup_git and self.git_dir: src_dir = os.path.abspath(self.git_dir) if os.path.exists(git_dir): + Print('\rFetching repo for thread %d' % thread_num, + newline=False) gitutil.Fetch(git_dir, thread_dir) + terminal.PrintClear() else: Print('\rCloning repo for thread %d' % thread_num, newline=False) gitutil.Clone(src_dir, thread_dir) - Print('\r%s\r' % (' ' * 30), newline=False) + terminal.PrintClear() def _PrepareWorkingSpace(self, max_threads, setup_git): """Prepare the working directory for use. @@ -1564,7 +1612,7 @@ class Builder: newline=False) for dirname in to_remove: shutil.rmtree(dirname) - Print('done') + terminal.PrintClear() def BuildBoards(self, commits, board_selected, keep_outputs, verbose): """Build all commits for a list of boards @@ -1612,5 +1660,19 @@ class Builder: # Wait until we have processed all output self.out_queue.join() Print() - self.ClearLine(0) + + msg = 'Completed: %d total built' % self.count + if self.already_done: + msg += ' (%d previously' % self.already_done + if self.already_done != self.count: + msg += ', %d newly' % (self.count - self.already_done) + msg += ')' + duration = datetime.now() - self._start_time + if duration > timedelta(microseconds=1000000): + if duration.microseconds >= 500000: + duration = duration + timedelta(seconds=1) + duration = duration - timedelta(microseconds=duration.microseconds) + msg += ', duration %s' % duration + Print(msg) + return (self.fail, self.warned) diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 7561f39942..fc6e1ab25d 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -90,12 +90,12 @@ class BuilderThread(threading.Thread): thread_num: Our thread number (0-n-1), used to decide on a temporary directory """ - def __init__(self, builder, thread_num, incremental, per_board_out_dir): + def __init__(self, builder, thread_num, mrproper, per_board_out_dir): """Set up a new builder thread""" threading.Thread.__init__(self) self.builder = builder self.thread_num = thread_num - self.incremental = incremental + self.mrproper = mrproper self.per_board_out_dir = per_board_out_dir def Make(self, commit, brd, stage, cwd, *args, **kwargs): @@ -243,7 +243,7 @@ class BuilderThread(threading.Thread): # If we need to reconfigure, do that now if do_config: config_out = '' - if not self.incremental: + if self.mrproper: result = self.Make(commit, brd, 'mrproper', cwd, 'mrproper', *args, env=env) config_out += result.combined diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index 17ea015a95..1377b9d2be 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -55,8 +55,9 @@ def ParseArgs(): parser.add_option('-i', '--in-tree', dest='in_tree', action='store_true', default=False, help='Build in the source tree instead of a separate directory') + # -I will be removed after April 2021 parser.add_option('-I', '--incremental', action='store_true', - default=False, help='Do not run make mrproper (when reconfiguring)') + default=False, help='Deprecated, does nothing. See -m') parser.add_option('-j', '--jobs', dest='jobs', type='int', default=None, help='Number of jobs to run at once (passed to make)') parser.add_option('-k', '--keep-outputs', action='store_true', @@ -69,6 +70,8 @@ def ParseArgs(): default=False, help='Show a list of boards next to each error/warning') parser.add_option('--list-tool-chains', action='store_true', default=False, help='List available tool chains (use -v to see probing detail)') + parser.add_option('-m', '--mrproper', action='store_true', + default=False, help="Run 'make mrproper before reconfiguring") parser.add_option('-n', '--dry-run', action='store_true', dest='dry_run', default=False, help="Do a dry run (describe actions, but do nothing)") parser.add_option('-N', '--no-subdirs', action='store_true', dest='no_subdirs', @@ -111,6 +114,12 @@ def ParseArgs(): parser.add_option('-x', '--exclude', dest='exclude', type='string', action='append', help='Specify a list of boards to exclude, separated by comma') + parser.add_option('-y', '--filter-dtb-warnings', action='store_true', + default=False, + help='Filter out device-tree-compiler warnings from output') + parser.add_option('-Y', '--filter-migration-warnings', action='store_true', + default=False, + help='Filter out migration warnings from output') parser.usage += """ [list of target/arch/cpu/board/vendor/soc to build] diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 5ddc598c95..30c030fd16 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -172,6 +172,10 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, print() return 0 + if options.incremental: + print(col.Color(col.RED, + 'Warning: -I has been removed. See documentation')) + # Work out what subset of the boards we are building if not boards: if not os.path.exists(options.output_dir): @@ -309,7 +313,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, show_unknown=options.show_unknown, step=options.step, no_subdirs=options.no_subdirs, full_path=options.full_path, verbose_build=options.verbose_build, - incremental=options.incremental, + mrproper=options.mrproper, per_board_out_dir=options.per_board_out_dir, config_only=options.config_only, squash_config_y=not options.preserve_config_y, @@ -341,23 +345,23 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, commits = None Print(GetActionSummary(options.summary, commits, board_selected, - options)) + options)) # We can't show function sizes without board details at present if options.show_bloat: options.show_detail = True - builder.SetDisplayOptions(options.show_errors, options.show_sizes, - options.show_detail, options.show_bloat, - options.list_error_boards, - options.show_config, - options.show_environment) + builder.SetDisplayOptions( + options.show_errors, options.show_sizes, options.show_detail, + options.show_bloat, options.list_error_boards, options.show_config, + options.show_environment, options.filter_dtb_warnings, + options.filter_migration_warnings) if options.summary: builder.ShowSummary(commits, board_selected) else: fail, warned = builder.BuildBoards(commits, board_selected, options.keep_outputs, options.verbose) if fail: - return 128 + return 100 elif warned and not options.ignore_warnings: - return 129 + return 101 return 0 diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 2a256a9263..1fbc6f6b00 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -454,7 +454,7 @@ class TestFunctional(unittest.TestCase): # Only sandbox should succeed, the others don't have toolchains self.assertEqual(self._builder.fail, self._total_builds - self._commits) - self.assertEqual(ret_code, 128) + self.assertEqual(ret_code, 100) for commit in range(self._commits): for board in self._boards.GetList(): @@ -476,15 +476,15 @@ class TestFunctional(unittest.TestCase): self._RunControl('-b', TEST_BRANCH, '-c2', '-o', self._output_dir) self.assertEqual(self._builder.count, 2 * len(boards)) self.assertEqual(self._builder.fail, 0) - # Each board has a mrproper, config, and then one make per commit - self.assertEqual(self._make_calls, len(boards) * (2 + 2)) + # Each board has a config, and then one make per commit + self.assertEqual(self._make_calls, len(boards) * (1 + 2)) def testIncremental(self): """Test building a branch twice - the second time should do nothing""" self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) # Each board has a mrproper, config, and then one make per commit - self.assertEqual(self._make_calls, len(boards) * (self._commits + 2)) + self.assertEqual(self._make_calls, len(boards) * (self._commits + 1)) self._make_calls = 0 self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False) self.assertEqual(self._make_calls, 0) @@ -496,14 +496,26 @@ class TestFunctional(unittest.TestCase): self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) self._make_calls = 0 self._RunControl('-b', TEST_BRANCH, '-f', '-o', self._output_dir, clean_dir=False) - # Each board has a mrproper, config, and then one make per commit - self.assertEqual(self._make_calls, len(boards) * (self._commits + 2)) + # Each board has a config and one make per commit + self.assertEqual(self._make_calls, len(boards) * (self._commits + 1)) def testForceReconfigure(self): """The -f flag should force a rebuild""" self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir) - # Each commit has a mrproper, config and make - self.assertEqual(self._make_calls, len(boards) * self._commits * 3) + # Each commit has a config and make + self.assertEqual(self._make_calls, len(boards) * self._commits * 2) + + def testForceReconfigure(self): + """The -f flag should force a rebuild""" + self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir) + # Each commit has a config and make + self.assertEqual(self._make_calls, len(boards) * self._commits * 2) + + def testMrproper(self): + """The -f flag should force a rebuild""" + self._RunControl('-b', TEST_BRANCH, '-m', '-o', self._output_dir) + # Each board has a mkproper, config and then one make per commit + self.assertEqual(self._make_calls, len(boards) * (self._commits + 2)) def testErrors(self): """Test handling of build errors""" @@ -525,7 +537,7 @@ class TestFunctional(unittest.TestCase): self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, '-F', clean_dir=False) self.assertEqual(self._builder.count, self._total_builds) self.assertEqual(self._builder.fail, 0) - self.assertEqual(self._make_calls, 3) + self.assertEqual(self._make_calls, 2) def testBranchWithSlash(self): """Test building a branch with a '/' in the name""" diff --git a/tools/buildman/test.py b/tools/buildman/test.py index 2aaedf44ac..d32b22653f 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -36,6 +36,14 @@ main: /usr/sbin x86: i386 x86_64 ''' +migration = '''===================== WARNING ====================== +This board does not use CONFIG_DM. CONFIG_DM will be +compulsory starting with the v2020.01 release. +Failure to update may result in board removal. +See doc/driver-model/migration.rst for more info. +==================================================== +''' + errors = [ '''main.c: In function 'main_loop': main.c:260:6: warning: unused variable 'joe' [-Wunused-variable] @@ -79,13 +87,14 @@ make: *** [sub-make] Error 2 # hash, subject, return code, list of errors/warnings commits = [ - ['1234', 'upstream/master, ok', 0, []], + ['1234', 'upstream/master, migration warning', 0, []], ['5678', 'Second commit, a warning', 0, errors[0:1]], ['9012', 'Third commit, error', 1, errors[0:2]], ['3456', 'Fourth commit, warning', 0, [errors[0], errors[2]]], ['7890', 'Fifth commit, link errors', 1, [errors[0], errors[3]]], ['abcd', 'Sixth commit, fixes all errors', 0, []], - ['ef01', 'Seventh commit, check directory suppression', 1, [errors[4]]], + ['ef01', 'Seventh commit, fix migration, check directory suppression', 1, + [errors[4]]], ] boards = [ @@ -118,6 +127,8 @@ class TestBuild(unittest.TestCase): comm.subject = commit_info[1] comm.return_code = commit_info[2] comm.error_list = commit_info[3] + if sequence < 6: + comm.error_list += [migration] comm.sequence = sequence sequence += 1 self.commits.append(comm) @@ -143,9 +154,14 @@ class TestBuild(unittest.TestCase): terminal.SetPrintTestMode() self._col = terminal.Color() - def Make(self, commit, brd, stage, *args, **kwargs): - global base_dir + self.base_dir = tempfile.mkdtemp() + if not os.path.isdir(self.base_dir): + os.mkdir(self.base_dir) + def tearDown(self): + shutil.rmtree(self.base_dir) + + def Make(self, commit, brd, stage, *args, **kwargs): result = command.CommandResult() boardnum = int(brd.target[-1]) result.return_code = 0 @@ -156,7 +172,9 @@ class TestBuild(unittest.TestCase): boardnum == 4 and commit.sequence == 6): result.return_code = commit.return_code result.stderr = (''.join(commit.error_list) - % {'basedir' : base_dir + '/.bm-work/00/'}) + % {'basedir' : self.base_dir + '/.bm-work/00/'}) + elif commit.sequence < 6: + result.stderr = migration result.combined = result.stdout + result.stderr return result @@ -173,17 +191,19 @@ class TestBuild(unittest.TestCase): expect += col.Color(expected_colour, ' %s' % board) self.assertEqual(text, expect) - def testOutput(self): - """Test basic builder operation and output + def _SetupTest(self, echo_lines=False, **kwdisplay_args): + """Set up the test by running a build and summary - This does a line-by-line verification of the summary output. - """ - global base_dir + Args: + echo_lines: True to echo lines to the terminal to aid test + development + kwdisplay_args: Dict of arguemnts to pass to + Builder.SetDisplayOptions() - base_dir = tempfile.mkdtemp() - if not os.path.isdir(base_dir): - os.mkdir(base_dir) - build = builder.Builder(self.toolchains, base_dir, None, 1, 2, + Returns: + Iterator containing the output lines, each a PrintLine() object + """ + build = builder.Builder(self.toolchains, self.base_dir, None, 1, 2, checkout=False, show_unknown=False) build.do_make = self.Make board_selected = self.boards.GetSelectedDict() @@ -198,124 +218,238 @@ class TestBuild(unittest.TestCase): if line.text.strip(): count += 1 - # We should get two starting messages, then an update for every commit - # built. - self.assertEqual(count, len(commits) * len(boards) + 2) - build.SetDisplayOptions(show_errors=True); + # We should get two starting messages, an update for every commit built + # and a summary message + self.assertEqual(count, len(commits) * len(boards) + 3) + build.SetDisplayOptions(**kwdisplay_args); build.ShowSummary(self.commits, board_selected) - #terminal.EchoPrintTestLines() - lines = terminal.GetPrintTestLines() + if echo_lines: + terminal.EchoPrintTestLines() + return iter(terminal.GetPrintTestLines()) + + def _CheckOutput(self, lines, list_error_boards=False, + filter_dtb_warnings=False, + filter_migration_warnings=False): + """Check for expected output from the build summary + + Args: + lines: Iterator containing the lines returned from the summary + list_error_boards: Adjust the check for output produced with the + --list-error-boards flag + filter_dtb_warnings: Adjust the check for output produced with the + --filter-dtb-warnings flag + """ + def add_line_prefix(prefix, boards, error_str, colour): + """Add a prefix to each line of a string + + The training \n in error_str is removed before processing + + Args: + prefix: String prefix to add + error_str: Error string containing the lines + colour: Expected colour for the line. Note that the board list, + if present, always appears in magenta + + Returns: + New string where each line has the prefix added + """ + lines = error_str.strip().splitlines() + new_lines = [] + for line in lines: + if boards: + expect = self._col.Color(colour, prefix + '(') + expect += self._col.Color(self._col.MAGENTA, boards, + bright=False) + expect += self._col.Color(colour, ') %s' % line) + else: + expect = self._col.Color(colour, prefix + line) + new_lines.append(expect) + return '\n'.join(new_lines) - # Upstream commit: no errors - self.assertEqual(lines[0].text, '01: %s' % commits[0][1]) + col = terminal.Color() + boards01234 = ('board0 board1 board2 board3 board4' + if list_error_boards else '') + boards1234 = 'board1 board2 board3 board4' if list_error_boards else '' + boards234 = 'board2 board3 board4' if list_error_boards else '' + boards34 = 'board3 board4' if list_error_boards else '' + boards4 = 'board4' if list_error_boards else '' + + # Upstream commit: migration warnings only + self.assertEqual(next(lines).text, '01: %s' % commits[0][1]) + + if not filter_migration_warnings: + self.assertSummary(next(lines).text, 'arm', 'w+', + ['board0', 'board1'], outcome=OUTCOME_WARN) + self.assertSummary(next(lines).text, 'powerpc', 'w+', + ['board2', 'board3'], outcome=OUTCOME_WARN) + self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], + outcome=OUTCOME_WARN) + + self.assertEqual(next(lines).text, + add_line_prefix('+', boards01234, migration, col.RED)) # Second commit: all archs should fail with warnings - self.assertEqual(lines[1].text, '02: %s' % commits[1][1]) + self.assertEqual(next(lines).text, '02: %s' % commits[1][1]) - col = terminal.Color() - self.assertSummary(lines[2].text, 'arm', 'w+', ['board1'], - outcome=OUTCOME_WARN) - self.assertSummary(lines[3].text, 'powerpc', 'w+', ['board2', 'board3'], - outcome=OUTCOME_WARN) - self.assertSummary(lines[4].text, 'sandbox', 'w+', ['board4'], - outcome=OUTCOME_WARN) + if filter_migration_warnings: + self.assertSummary(next(lines).text, 'arm', 'w+', + ['board1'], outcome=OUTCOME_WARN) + self.assertSummary(next(lines).text, 'powerpc', 'w+', + ['board2', 'board3'], outcome=OUTCOME_WARN) + self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], + outcome=OUTCOME_WARN) # Second commit: The warnings should be listed - self.assertEqual(lines[5].text, 'w+%s' % - errors[0].rstrip().replace('\n', '\nw+')) - self.assertEqual(lines[5].colour, col.MAGENTA) + self.assertEqual(next(lines).text, + add_line_prefix('w+', boards1234, errors[0], col.YELLOW)) # Third commit: Still fails - self.assertEqual(lines[6].text, '03: %s' % commits[2][1]) - self.assertSummary(lines[7].text, 'arm', '', ['board1'], - outcome=OUTCOME_OK) - self.assertSummary(lines[8].text, 'powerpc', '+', ['board2', 'board3']) - self.assertSummary(lines[9].text, 'sandbox', '+', ['board4']) + self.assertEqual(next(lines).text, '03: %s' % commits[2][1]) + if filter_migration_warnings: + self.assertSummary(next(lines).text, 'arm', '', + ['board1'], outcome=OUTCOME_OK) + self.assertSummary(next(lines).text, 'powerpc', '+', + ['board2', 'board3']) + self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) # Expect a compiler error - self.assertEqual(lines[10].text, '+%s' % - errors[1].rstrip().replace('\n', '\n+')) + self.assertEqual(next(lines).text, + add_line_prefix('+', boards234, errors[1], col.RED)) # Fourth commit: Compile errors are fixed, just have warning for board3 - self.assertEqual(lines[11].text, '04: %s' % commits[3][1]) - expect = '%10s: ' % 'powerpc' - expect += ' ' + col.Color(col.GREEN, '') - expect += ' ' - expect += col.Color(col.GREEN, ' %s' % 'board2') - expect += ' ' + col.Color(col.YELLOW, 'w+') - expect += ' ' - expect += col.Color(col.YELLOW, ' %s' % 'board3') - self.assertEqual(lines[12].text, expect) - self.assertSummary(lines[13].text, 'sandbox', 'w+', ['board4'], - outcome=OUTCOME_WARN) + self.assertEqual(next(lines).text, '04: %s' % commits[3][1]) + if filter_migration_warnings: + expect = '%10s: ' % 'powerpc' + expect += ' ' + col.Color(col.GREEN, '') + expect += ' ' + expect += col.Color(col.GREEN, ' %s' % 'board2') + expect += ' ' + col.Color(col.YELLOW, 'w+') + expect += ' ' + expect += col.Color(col.YELLOW, ' %s' % 'board3') + self.assertEqual(next(lines).text, expect) + else: + self.assertSummary(next(lines).text, 'powerpc', 'w+', + ['board2', 'board3'], outcome=OUTCOME_WARN) + self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], + outcome=OUTCOME_WARN) # Compile error fixed - self.assertEqual(lines[14].text, '-%s' % - errors[1].rstrip().replace('\n', '\n-')) - self.assertEqual(lines[14].colour, col.GREEN) + self.assertEqual(next(lines).text, + add_line_prefix('-', boards234, errors[1], col.GREEN)) - self.assertEqual(lines[15].text, 'w+%s' % - errors[2].rstrip().replace('\n', '\nw+')) - self.assertEqual(lines[15].colour, col.MAGENTA) + if not filter_dtb_warnings: + self.assertEqual( + next(lines).text, + add_line_prefix('w+', boards34, errors[2], col.YELLOW)) # Fifth commit - self.assertEqual(lines[16].text, '05: %s' % commits[4][1]) - self.assertSummary(lines[17].text, 'powerpc', '', ['board3'], - outcome=OUTCOME_OK) - self.assertSummary(lines[18].text, 'sandbox', '+', ['board4']) + self.assertEqual(next(lines).text, '05: %s' % commits[4][1]) + if filter_migration_warnings: + self.assertSummary(next(lines).text, 'powerpc', '', ['board3'], + outcome=OUTCOME_OK) + self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) # The second line of errors[3] is a duplicate, so buildman will drop it expect = errors[3].rstrip().split('\n') expect = [expect[0]] + expect[2:] - self.assertEqual(lines[19].text, '+%s' % - '\n'.join(expect).replace('\n', '\n+')) + expect = '\n'.join(expect) + self.assertEqual(next(lines).text, + add_line_prefix('+', boards4, expect, col.RED)) - self.assertEqual(lines[20].text, 'w-%s' % - errors[2].rstrip().replace('\n', '\nw-')) + if not filter_dtb_warnings: + self.assertEqual( + next(lines).text, + add_line_prefix('w-', boards34, errors[2], col.CYAN)) # Sixth commit - self.assertEqual(lines[21].text, '06: %s' % commits[5][1]) - self.assertSummary(lines[22].text, 'sandbox', '', ['board4'], - outcome=OUTCOME_OK) + self.assertEqual(next(lines).text, '06: %s' % commits[5][1]) + if filter_migration_warnings: + self.assertSummary(next(lines).text, 'sandbox', '', ['board4'], + outcome=OUTCOME_OK) + else: + self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], + outcome=OUTCOME_WARN) # The second line of errors[3] is a duplicate, so buildman will drop it expect = errors[3].rstrip().split('\n') expect = [expect[0]] + expect[2:] - self.assertEqual(lines[23].text, '-%s' % - '\n'.join(expect).replace('\n', '\n-')) - - self.assertEqual(lines[24].text, 'w-%s' % - errors[0].rstrip().replace('\n', '\nw-')) + expect = '\n'.join(expect) + self.assertEqual(next(lines).text, + add_line_prefix('-', boards4, expect, col.GREEN)) + self.assertEqual(next(lines).text, + add_line_prefix('w-', boards4, errors[0], col.CYAN)) # Seventh commit - self.assertEqual(lines[25].text, '07: %s' % commits[6][1]) - self.assertSummary(lines[26].text, 'sandbox', '+', ['board4']) + self.assertEqual(next(lines).text, '07: %s' % commits[6][1]) + if filter_migration_warnings: + self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) + else: + self.assertSummary(next(lines).text, 'arm', '', ['board0', 'board1'], + outcome=OUTCOME_OK) + self.assertSummary(next(lines).text, 'powerpc', '', + ['board2', 'board3'], outcome=OUTCOME_OK) + self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) # Pick out the correct error lines expect_str = errors[4].rstrip().replace('%(basedir)s', '').split('\n') expect = expect_str[3:8] + [expect_str[-1]] - self.assertEqual(lines[27].text, '+%s' % - '\n'.join(expect).replace('\n', '\n+')) + expect = '\n'.join(expect) + if not filter_migration_warnings: + self.assertEqual( + next(lines).text, + add_line_prefix('-', boards01234, migration, col.GREEN)) + + self.assertEqual(next(lines).text, + add_line_prefix('+', boards4, expect, col.RED)) # Now the warnings lines expect = [expect_str[0]] + expect_str[10:12] + [expect_str[9]] - self.assertEqual(lines[28].text, 'w+%s' % - '\n'.join(expect).replace('\n', '\nw+')) + expect = '\n'.join(expect) + self.assertEqual(next(lines).text, + add_line_prefix('w+', boards4, expect, col.YELLOW)) - self.assertEqual(len(lines), 29) - shutil.rmtree(base_dir) + def testOutput(self): + """Test basic builder operation and output + + This does a line-by-line verification of the summary output. + """ + lines = self._SetupTest(show_errors=True) + self._CheckOutput(lines, list_error_boards=False, + filter_dtb_warnings=False) + + def testErrorBoards(self): + """Test output with --list-error-boards + + This does a line-by-line verification of the summary output. + """ + lines = self._SetupTest(show_errors=True, list_error_boards=True) + self._CheckOutput(lines, list_error_boards=True) + + def testFilterDtb(self): + """Test output with --filter-dtb-warnings + + This does a line-by-line verification of the summary output. + """ + lines = self._SetupTest(show_errors=True, filter_dtb_warnings=True) + self._CheckOutput(lines, filter_dtb_warnings=True) + + def testFilterMigration(self): + """Test output with --filter-migration-warnings + + This does a line-by-line verification of the summary output. + """ + lines = self._SetupTest(show_errors=True, + filter_migration_warnings=True) + self._CheckOutput(lines, filter_migration_warnings=True) def _testGit(self): """Test basic builder operation by building a branch""" - base_dir = tempfile.mkdtemp() - if not os.path.isdir(base_dir): - os.mkdir(base_dir) options = Options() options.git = os.getcwd() options.summary = False options.jobs = None options.dry_run = False - #options.git = os.path.join(base_dir, 'repo') + #options.git = os.path.join(self.base_dir, 'repo') options.branch = 'test-buildman' options.force_build = False options.list_tool_chains = False @@ -328,7 +462,6 @@ class TestBuild(unittest.TestCase): options.keep_outputs = False args = ['tegra20'] control.DoBuildman(options, args) - shutil.rmtree(base_dir) def testBoardSingle(self): """Test single board selection""" diff --git a/tools/patman/patman.py b/tools/patman/patman.py index cf53e532dd..7f4ac9aef4 100755 --- a/tools/patman/patman.py +++ b/tools/patman/patman.py @@ -93,7 +93,7 @@ elif options.test: suite = unittest.TestLoader().loadTestsFromTestCase(module) suite.run(result) - for module in ['gitutil', 'settings']: + for module in ['gitutil', 'settings', 'terminal']: suite = doctest.DocTestSuite(module) suite.run(result) diff --git a/tools/patman/terminal.py b/tools/patman/terminal.py index 7a3b658b00..5c9e3eea20 100644 --- a/tools/patman/terminal.py +++ b/tools/patman/terminal.py @@ -10,6 +10,8 @@ This module handles terminal interaction including ANSI color codes. from __future__ import print_function import os +import re +import shutil import sys # Selection of when we want our output to be colored @@ -19,6 +21,13 @@ COLOR_IF_TERMINAL, COLOR_ALWAYS, COLOR_NEVER = range(3) print_test_mode = False print_test_list = [] +# The length of the last line printed without a newline +last_print_len = None + +# credit: +# stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python +ansi_escape = re.compile(r'\x1b(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') + class PrintLine: """A line of text output @@ -36,7 +45,86 @@ class PrintLine: return 'newline=%s, colour=%s, text=%s' % (self.newline, self.colour, self.text) -def Print(text='', newline=True, colour=None): +def CalcAsciiLen(text): + """Calculate the length of a string, ignoring any ANSI sequences + + When displayed on a terminal, ANSI sequences don't take any space, so we + need to ignore them when calculating the length of a string. + + Args: + text: Text to check + + Returns: + Length of text, after skipping ANSI sequences + + >>> col = Color(COLOR_ALWAYS) + >>> text = col.Color(Color.RED, 'abc') + >>> len(text) + 14 + >>> CalcAsciiLen(text) + 3 + >>> + >>> text += 'def' + >>> CalcAsciiLen(text) + 6 + >>> text += col.Color(Color.RED, 'abc') + >>> CalcAsciiLen(text) + 9 + """ + result = ansi_escape.sub('', text) + return len(result) + +def TrimAsciiLen(text, size): + """Trim a string containing ANSI sequences to the given ASCII length + + The string is trimmed with ANSI sequences being ignored for the length + calculation. + + >>> col = Color(COLOR_ALWAYS) + >>> text = col.Color(Color.RED, 'abc') + >>> len(text) + 14 + >>> CalcAsciiLen(TrimAsciiLen(text, 4)) + 3 + >>> CalcAsciiLen(TrimAsciiLen(text, 2)) + 2 + >>> text += 'def' + >>> CalcAsciiLen(TrimAsciiLen(text, 4)) + 4 + >>> text += col.Color(Color.RED, 'ghi') + >>> CalcAsciiLen(TrimAsciiLen(text, 7)) + 7 + """ + if CalcAsciiLen(text) < size: + return text + pos = 0 + out = '' + left = size + + # Work through each ANSI sequence in turn + for m in ansi_escape.finditer(text): + # Find the text before the sequence and add it to our string, making + # sure it doesn't overflow + before = text[pos:m.start()] + toadd = before[:left] + out += toadd + + # Figure out how much non-ANSI space we have left + left -= len(toadd) + + # Add the ANSI sequence and move to the position immediately after it + out += m.group() + pos = m.start() + len(m.group()) + + # Deal with text after the last ANSI sequence + after = text[pos:] + toadd = after[:left] + out += toadd + + return out + + +def Print(text='', newline=True, colour=None, limit_to_line=False): """Handle a line of output to the terminal. In test mode this is recorded in a list. Otherwise it is output to the @@ -47,17 +135,31 @@ def Print(text='', newline=True, colour=None): newline: True to add a new line at the end of the text colour: Colour to use for the text """ + global last_print_len + if print_test_mode: print_test_list.append(PrintLine(text, newline, colour)) else: if colour: col = Color() text = col.Color(colour, text) - print(text, end='') if newline: - print() + print(text) + last_print_len = None else: - sys.stdout.flush() + if limit_to_line: + cols = shutil.get_terminal_size().columns + text = TrimAsciiLen(text, cols) + print(text, end='', flush=True) + last_print_len = CalcAsciiLen(text) + +def PrintClear(): + """Clear a previously line that was printed with no newline""" + global last_print_len + + if last_print_len: + print('\r%s\r' % (' '* last_print_len), end='', flush=True) + last_print_len = None def SetPrintTestMode(): """Go into test mode, where all printing is recorded""" |