U-Boot SPL to Enable OPTEE on Colibri iMX8QXP

Since I posted a patch that didn’t work, I figured I’d circle back and post an initial patch that does work to enable the SPL boot.

The main thing that I was missing when I tried this a few weeks ago is that the SPL just uses a subset of the device tree (in order to keep small), and the nodes relevant to what the SPL needs to do must be tagged with the ‘u-boot,dm-spl’ (or ‘u-boot,dm-pre-reloc’) property in order to be included in this smaller device tree. This is made clear in the u-boot doc/README.SPL, so once I realized that, the rest of the effort went pretty smoothly.

Once I patch u-boot-toradex in order to support SPL on the colibri-imx8x, I added the following to my machine conf file:

UBOOT_MAKE_TARGET = "u-boot.bin"
ATF_PLATFORM = "imx8qx"
IMXBOOT_TARGETS = "flash_spl"
MACHINE_FEATURES += "optee"
SPL_BINARY = "spl/u-boot-spl.bin"
IMX_BOOT_SOC_TARGET = "iMX8QX"

With these modification, OPTEE works as I hoped!

Anyway, here is my initial version of the u-boot-toradex patch that works to enable SPL:

diff --git a/arch/arm/dts/fsl-imx8qxp-colibri-u-boot.dtsi b/arch/arm/dts/fsl-imx8qxp-colibri-u-boot.dtsi
index 46b996f3b3..d5a4e458e5 100644
--- a/arch/arm/dts/fsl-imx8qxp-colibri-u-boot.dtsi
+++ b/arch/arm/dts/fsl-imx8qxp-colibri-u-boot.dtsi
@@ -22,26 +22,26 @@
                dr_mode = "peripheral";
                chipidea,usb = <&usbotg1>;
                status = "okay";
-               u-boot,dm-pre-proper;
+               u-boot,dm-spl;
        };
 
 };
 
 &{/imx8qx-pm} {
 
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &mu {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &clk {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &iomuxc {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &{/regulators} {
@@ -49,71 +49,75 @@
 };
 
 &{/mu@5d1c0000/iomuxc/colibri-imx8x} {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pinctrl_usdhc1 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pinctrl_usdhc2_gpio {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pinctrl_usdhc2 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
+};
+
+&pinctrl_lpuart3 {
+       u-boot,dm-spl;
 };
 
 &pd_lsio {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_lsio_gpio0 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_lsio_gpio1 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_lsio_gpio2 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_lsio_gpio3 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_lsio_gpio4 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_lsio_gpio5 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_lsio_gpio6 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_lsio_gpio7 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_dma {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_dma_lpuart0 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_dma_lpuart3 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_conn {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_conn_usbotg0 {
@@ -133,87 +137,87 @@
 };
 
 &pd_conn_sdch0 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_conn_sdch1 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &pd_conn_sdch2 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &gpio0 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &gpio1 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &gpio2 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &gpio3 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &gpio4 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &gpio5 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &gpio6 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &gpio7 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &lpuart3 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &usbmisc1 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &usbphy1 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &usbotg1 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &usbotg3 {
        phys = <&usbphynop1>;
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &usbphynop1 {
        compatible = "cdns,usb3-phy";
        reg = <0x0 0x5B160000 0x0 0x40000>;
        #phy-cells = <0>;
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
 
 &usdhc1 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
        /delete-property/ assigned-clock-parents;
 };
 
 &usdhc2 {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
        /delete-property/ assigned-clock-parents;
 };
 
 &wu {
-       u-boot,dm-pre-proper;
+       u-boot,dm-spl;
 };
diff --git a/board/toradex/colibri-imx8x/Makefile b/board/toradex/colibri-imx8x/Makefile
index 783d4fb5ba..bc474713b1 100644
--- a/board/toradex/colibri-imx8x/Makefile
+++ b/board/toradex/colibri-imx8x/Makefile
@@ -4,3 +4,4 @@
 #
 
 obj-y += colibri-imx8x.o
+obj-$(CONFIG_SPL_BUILD) += spl.o
diff --git a/board/toradex/colibri-imx8x/colibri-imx8x-imximage.cfg b/board/toradex/colibri-imx8x/colibri-imx8x-imximage.cfg
index 1dcd13271d..766f840043 100644
--- a/board/toradex/colibri-imx8x/colibri-imx8x-imximage.cfg
+++ b/board/toradex/colibri-imx8x/colibri-imx8x-imximage.cfg
@@ -19,6 +19,4 @@ CONTAINER
 /* Add scfw image with exec attribute */
 IMAGE SCU mx8qx-colibri-scfw-tcm.bin
 /* Add ATF image with exec attribute */
-IMAGE A35 bl31.bin 0x80000000
-/* Add U-Boot image with load attribute */
-DATA A35 u-boot-dtb.bin 0x80020000
+IMAGE A35 spl/u-boot-spl.bin 0x00100000
diff --git a/board/toradex/colibri-imx8x/colibri-imx8x.c b/board/toradex/colibri-imx8x/colibri-imx8x.c
index 5bc4fba5b9..d2726f57e3 100644
--- a/board/toradex/colibri-imx8x/colibri-imx8x.c
+++ b/board/toradex/colibri-imx8x/colibri-imx8x.c
@@ -2,6 +2,7 @@
 /*
  * Copyright 2018-2021 Toradex
  */
+
 #include <common.h>
 #include <cpu_func.h>
 #include <env.h>
@@ -102,6 +103,15 @@ int board_early_init_f(void)
 
        setup_iomux_uart();
 
+/* Dual bootloader feature will require CAAM access, but JR0 and JR1 will be
+ * assigned to seco for imx8, use JR3 instead.
+ */
+#if defined(CONFIG_SPL_BUILD) && defined(CONFIG_DUAL_BOOTLOADER)
+       sc_pm_set_resource_power_mode(-1, SC_R_CAAM_JR3, SC_PM_PW_MODE_ON);
+       sc_pm_set_resource_power_mode(-1, SC_R_CAAM_JR3_OUT, SC_PM_PW_MODE_ON);
+#endif
+
+
        return 0;
 }
 
@@ -186,12 +196,12 @@ int board_init(void)
        return 0;
 }
 
-/* todo: With that function in ther is no console output in linux, drop for now */
-#if 0
 void board_quiesce_devices(void)
 {
        const char *power_on_devices[] = {
-               "dma_lpuart3",
+               /* UART3 - Debug console */
+               "PD_UART3_RX",
+               "PD_UART3_TX",
 
                /* HIFI DSP boot */
                "audio_sai0",
@@ -200,7 +210,6 @@ void board_quiesce_devices(void)
 
        power_off_pd_devices(power_on_devices, ARRAY_SIZE(power_on_devices));
 }
-#endif
 
 void detail_board_ddr_info(void)
 {
diff --git a/board/toradex/colibri-imx8x/spl.c b/board/toradex/colibri-imx8x/spl.c
new file mode 100644
index 0000000000..ffd19bcb93
--- /dev/null
+++ b/board/toradex/colibri-imx8x/spl.c
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2018 NXP
+ *
+ * SPDX-License-Identifier:    GPL-2.0+
+ */
+
+#include <common.h>
+#include <dm.h>
+#include <spl.h>
+#include <dm/uclass.h>
+#include <dm/device.h>
+#include <dm/uclass-internal.h>
+#include <dm/device-internal.h>
+#include <dm/lists.h>
+#include <bootm.h>
+
+DECLARE_GLOBAL_DATA_PTR;
+
+void spl_board_init(void)
+{
+       struct udevice *dev;
+
+       uclass_find_first_device(UCLASS_MISC, &dev);
+
+       for (; dev; uclass_find_next_device(&dev)) {
+               if (device_probe(dev))
+                       continue;
+       }
+
+       board_early_init_f();
+
+       timer_init();
+
+#ifdef CONFIG_SPL_SERIAL_SUPPORT
+       preloader_console_init();
+
+       puts("Normal Boot\n");
+#endif
+}
+
+void spl_board_prepare_for_boot(void)
+{
+       board_quiesce_devices();
+}
+
+#ifdef CONFIG_SPL_LOAD_FIT
+int board_fit_config_name_match(const char *name)
+{
+       /* Just empty function now - can't decide what to choose */
+       debug("%s: %s\n", __func__, name);
+
+       return 0;
+}
+#endif
+
+void board_init_f(ulong dummy)
+{
+       /* Clear the BSS. */
+       memset(__bss_start, 0, __bss_end - __bss_start);
+
+       arch_cpu_init();
+
+       board_init_r(NULL, 0);
+}
diff --git a/board/toradex/colibri-imx8x/uboot-container.cfg b/board/toradex/colibri-imx8x/uboot-container.cfg
new file mode 100644
index 0000000000..dd9593962f
--- /dev/null
+++ b/board/toradex/colibri-imx8x/uboot-container.cfg
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright 2019 NXP
+ */
+
+#define __ASSEMBLY__
+
+/* This file is to create a container image could be loaded by SPL */
+/* BOOT_FROM SD 0x400 */
+BOOT_FROM EMMC_FASTBOOT 0x400
+SOC_TYPE IMX8QX
+CONTAINER
+IMAGE A35 bl31.bin     0x80000000
+IMAGE A35 u-boot.bin   CONFIG_SYS_TEXT_BASE
diff --git a/configs/colibri-imx8x_defconfig b/configs/colibri-imx8x_defconfig
index 61c17a0b0a..221043e060 100644
--- a/configs/colibri-imx8x_defconfig
+++ b/configs/colibri-imx8x_defconfig
@@ -1,15 +1,32 @@
 CONFIG_ARM=y
+CONFIG_SPL_SYS_ICACHE_OFF=y
+CONFIG_SPL_SYS_DCACHE_OFF=y
 CONFIG_ARCH_IMX8=y
 CONFIG_SYS_TEXT_BASE=0x80020000
+CONFIG_SPL_GPIO_SUPPORT=y
+CONFIG_SPL_LIBCOMMON_SUPPORT=y
+CONFIG_SPL_LIBGENERIC_SUPPORT=y
 CONFIG_SYS_MALLOC_F_LEN=0x8000
 CONFIG_ENV_SIZE=0x2000
 CONFIG_ENV_OFFSET=0xFFFFDE00
 CONFIG_DM_GPIO=y
+CONFIG_SPL_LOAD_IMX_CONTAINER=y
+CONFIG_IMX_CONTAINER_CFG="board/toradex/colibri-imx8x/uboot-container.cfg"
+CONFIG_SPL_MMC_SUPPORT=y
+CONFIG_SPL_EFI_PARTITION=n
+CONFIG_SPL_DOS_PARTITION=n
+CONFIG_SPL_DM_SEQ_ALIAS=y
+CONFIG_SPL_SERIAL_SUPPORT=y
+CONFIG_SPL_DRIVERS_MISC_SUPPORT=y
+CONFIG_SPL_BANNER_PRINT=y
+CONFIG_USE_TINY_PRINTF=y
 CONFIG_BOOTAUX_RESERVED_MEM_BASE=0x88000000
 CONFIG_BOOTAUX_RESERVED_MEM_SIZE=0x08000000
 CONFIG_TARGET_COLIBRI_IMX8X=y
 CONFIG_SNVS_SEC_SC=y
 CONFIG_NR_DRAM_BANKS=3
+CONFIG_SPL=y
+CONFIG_SPL_TEXT_BASE=0x100000
 CONFIG_IMX_BOOTAUX=y
 CONFIG_IMX_VSERVICE_SHARED_BUFFER=0x90400000
 CONFIG_DISTRO_DEFAULTS=y
@@ -19,6 +36,11 @@ CONFIG_OF_SYSTEM_SETUP=y
 CONFIG_SYS_EXTRA_OPTIONS="IMX_CONFIG=board/toradex/colibri-imx8x/colibri-imx8x-imximage.cfg"
 CONFIG_BOOTDELAY=1
 CONFIG_LOG=y
+CONFIG_SPL_BOARD_INIT=y
+CONFIG_SPL_SEPARATE_BSS=y
+CONFIG_SPL_POWER_SUPPORT=y
+CONFIG_SPL_POWER_DOMAIN=y
+CONFIG_SPL_WATCHDOG_SUPPORT=y
 # CONFIG_DISPLAY_BOARDINFO is not set
 CONFIG_DISPLAY_BOARDINFO_LATE=y
 CONFIG_BOARD_EARLY_INIT_F=y
@@ -50,10 +72,17 @@ CONFIG_BOOTCOUNT_LIMIT=y
 CONFIG_BOOTCOUNT_ENV=y
 CONFIG_CLK_IMX8=y
 CONFIG_CPU=y
+CONFIG_SPL_OF_CONTROL=y
+CONFIG_SPL_DM=y
+CONFIG_SPL_CLK=y
+CONFIG_FASTBOOT=y
+CONFIG_CMD_FASTBOOT=y
 CONFIG_USB_FUNCTION_FASTBOOT=y
+CONFIG_FASTBOOT_UUU_SUPPORT=y
 CONFIG_FASTBOOT_BUF_ADDR=0x82800000
 CONFIG_FASTBOOT_BUF_SIZE=0x08000000
 CONFIG_FASTBOOT_FLASH=y
+CONFIG_FASTBOOT_USB_DEV=1
 CONFIG_MXC_GPIO=y
 CONFIG_DM_PCA953X=y
 CONFIG_DM_I2C=y
@@ -82,15 +111,19 @@ CONFIG_DM_PCI=y
 CONFIG_PHY=y
 CONFIG_CDNS3_USB_PHY=y
 CONFIG_PINCTRL=y
+CONFIG_SPL_PINCTRL=y
 CONFIG_PINCTRL_IMX8=y
 CONFIG_POWER_DOMAIN=y
 CONFIG_IMX8_POWER_DOMAIN=y
 CONFIG_DM_REGULATOR=y
+CONFIG_SPL_DM_REGULATOR=y
 CONFIG_DM_REGULATOR_FIXED=y
 CONFIG_DM_REGULATOR_GPIO=y
+CONFIG_SPL_DM_REGULATOR_FIXED=y
 # CONFIG_REQUIRE_SERIAL_CONSOLE is not set
 CONFIG_DM_SERIAL=y
 CONFIG_FSL_LPUART=y
+CONFIG_SPL_TINY_MEMSET=y
 CONFIG_DM_THERMAL=y
 CONFIG_IMX_SCU_THERMAL=y
 CONFIG_USB=y
@@ -99,6 +132,11 @@ CONFIG_USB_XHCI_HCD=y
 CONFIG_USB_CDNS3=y
 CONFIG_USB_CDNS3_GADGET=y
 CONFIG_USB_GADGET=y
+CONFIG_SPL_DM_USB_GADGET=y
+CONFIG_SPL_PHY=y
+CONFIG_SPL_USB_GADGET=y
+CONFIG_SPL_USB_SDP_SUPPORT=y
+CONFIG_SPL_SDP_USB_DEV=1
 CONFIG_USB_GADGET_MANUFACTURER="FSL"
 CONFIG_USB_GADGET_VENDOR_NUM=0x0525
 CONFIG_USB_GADGET_PRODUCT_NUM=0xa4a5
@@ -107,3 +145,4 @@ CONFIG_SDP_LOADADDR=0x80400000
 CONFIG_PANIC_HANG=y
 CONFIG_OF_LIBFDT_OVERLAY=y
 # CONFIG_EFI_LOADER is not set
+CONFIG_AHAB_BOOT=y
diff --git a/include/configs/colibri-imx8x.h b/include/configs/colibri-imx8x.h
index f9eece3646..b34660dc5d 100644
--- a/include/configs/colibri-imx8x.h
+++ b/include/configs/colibri-imx8x.h
@@ -9,8 +9,38 @@
 #include <linux/sizes.h>
 #include <asm/arch/imx-regs.h>
 
+#ifdef CONFIG_SPL_BUILD
+#define CONFIG_SPL_MAX_SIZE                            (192 * 1024)
+#define CONFIG_SYS_MONITOR_LEN                         (1024 * 1024)
+#define CONFIG_SYS_MMCSD_RAW_MODE_U_BOOT_USE_SECTOR
+#define CONFIG_SYS_MMCSD_RAW_MODE_U_BOOT_SECTOR                0x1040 /* (32K + 2Mb)/sector_size */
+
+/*
+ * 0x08081000 - 0x08180FFF is for m4_0 xip image,
+  * So 3rd container image may start from 0x8181000
+ */
+#define CONFIG_SYS_UBOOT_BASE 0x08181000
+#define CONFIG_SYS_MMCSD_FS_BOOT_PARTITION             0
+
+#define CONFIG_SPL_LDSCRIPT            "arch/arm/cpu/armv8/u-boot-spl.lds"
+#define CONFIG_SPL_STACK               0x013fff0
+#define CONFIG_SPL_BSS_START_ADDR      0x00130000
+#define CONFIG_SPL_BSS_MAX_SIZE                0x1000  /* 4 KB */
+#define CONFIG_SYS_SPL_MALLOC_START    0x82200000
+#define CONFIG_SYS_SPL_MALLOC_SIZE     0x80000 /* 512 KB */
+#define CONFIG_SERIAL_LPUART_BASE      0x5a090000
+#define CONFIG_MALLOC_F_ADDR           0x00138000
+
+#define CONFIG_SPL_RAW_IMAGE_ARM_TRUSTED_FIRMWARE
+
+#define CONFIG_SPL_ABORT_ON_RAW_IMAGE
+
+#endif
+
 #define CONFIG_REMAKE_ELF
 
+#define CONFIG_BOARD_EARLY_INIT_F
+
 #define CONFIG_SYS_FSL_ESDHC_ADDR       0
 #define USDHC1_BASE_ADDR                0x5B010000
 #define USDHC2_BASE_ADDR                0x5B020000