Elfloader: NVIDIA Jetson Orin support

elfloader-tool/include/arch-arm/64/mode/assembler.h

@@ -67,7 +67,15 @@
 #define MT_NORMAL_WT      5
 #define MAIR(_attr, _mt)  ((_attr) << ((_mt) * 8))
 
+.macro disable_mmu sctlr tmp
+    __disable_mmu \sctlr, \tmp
+    ic      ialluis
+    dsb     sy
+    isb
+.endm
+
 .macro enable_mmu sctlr tmp
+    dsb     sy
     mrs     \tmp, \sctlr
     orr     \tmp, \tmp, #(1 << 0)
     orr     \tmp, \tmp, #(1 << 2)
@@ -76,17 +84,11 @@
     isb
 .endm
 
-.macro disable_mmu sctlr tmp
-    mrs     \tmp, \sctlr
-    bic     \tmp, \tmp, #(1 << 0)
-    bic     \tmp, \tmp, #(1 << 2)
-    bic     \tmp, \tmp, #(1 << 12)
-    msr     \sctlr, \tmp
+.macro __disable_mmu sctlr tmp
+    dsb     sy
     isb
-.endm
-
-.macro disable_id_cache sctlr tmp
     mrs     \tmp, \sctlr
+    bic     \tmp, \tmp, #(1 << 0)
     bic     \tmp, \tmp, #(1 << 2)
     bic     \tmp, \tmp, #(1 << 12)
     msr     \sctlr, \tmp

elfloader-tool/src/arch-arm/armv/armv8-a/64/mmu-hyp.S

@@ -21,31 +21,42 @@
 .extern invalidate_icache
 .extern _boot_pgd_down
 
-BEGIN_FUNC(disable_caches_hyp)
-    stp     x29, x30, [sp, #-16]!
-    mov     x29, sp
-    bl      flush_dcache
-    disable_id_cache sctlr_el2, x9
-    ldp     x29, x30, [sp], #16
+BEGIN_FUNC(disable_mmu_caches_hyp)
+    /* Assume D-cache already cleaned to PoC */
+    disable_mmu sctlr_el2, x9
+    ret
+END_FUNC(disable_mmu_caches_hyp)
+
+BEGIN_FUNC(clean_dcache_by_range)
+    /* Ordering needed for strongly-ordered mem, not needed for NORMAL mem.
+     * See ARM DDI 0487I.a, page D7-5063.
+     */
+    dmb     sy
+
+    /* Extract minimum DCache CL size into x3 and CL mask into x4 */
+    mrs x2, ctr_el0
+    ubfx x4, x2, #16, #4
+    mov x3, #4
+    lsl x3, x3, x4
+    sub x4, x3, #1
+
+    /* Apply mask to start address before entering the loop */
+    bic x4, x0, x4
+clean_dcache_by_range_loop:
+    dc cvac, x4
+    add x4, x4, x3
+    cmp x4, x1
+    b.lt clean_dcache_by_range_loop
+    dsb sy
+    isb
     ret
-END_FUNC(disable_caches_hyp)
+END_FUNC(clean_dcache_by_range)
 
 BEGIN_FUNC(leave_hyp)
     /* We call nested functions, follow the ABI. */
     stp     x29, x30, [sp, #-16]!
     mov     x29, sp
 
-    bl      flush_dcache
-
-    /* Ensure I-cache, D-cache and mmu are disabled for EL2/Stage1 */
-    disable_mmu sctlr_el2, x9
-
-    /*
-     * Invalidate the local I-cache so that any instructions fetched
-     * speculatively are discarded.
-     */
-    bl      invalidate_icache
-
     /* Ensure I-cache, D-cache and mmu are disabled for EL1/Stage2 */
     mov     x9, #(1 << 31)
     msr     hcr_el2, x9
@@ -71,15 +82,6 @@ BEGIN_FUNC(leave_hyp)
 END_FUNC(leave_hyp)
 
 BEGIN_FUNC(arm_enable_hyp_mmu)
-    stp     x29, x30, [sp, #-16]!
-    mov     x29, sp
-
-    bl      flush_dcache
-
-    disable_mmu sctlr_el2, x8
-
-    bl      invalidate_icache
-
     /*
      *   DEVICE_nGnRnE      000     00000000
      *   DEVICE_nGnRE       001     00000100
@@ -107,13 +109,13 @@ BEGIN_FUNC(arm_enable_hyp_mmu)
     dsb     ish
     isb
 
-    enable_mmu  sctlr_el2, x8
+    /* Invalidate icache */
     ic  ialluis
-    dsb ish
+    dsb sy
     isb
-    tlbi    alle2is
-    dsb     ish
-    isb
-    ldp     x29, x30, [sp], #16
+
+    enable_mmu  sctlr_el2, x8
+    /* NOTE: enable_mmu already contains an isb after enabling. */
+
     ret
 END_FUNC(arm_enable_hyp_mmu)

elfloader-tool/src/arch-arm/armv/armv8-a/64/mmu.S

@@ -43,17 +43,6 @@ BEGIN_FUNC(arm_enable_mmu)
     stp     x29, x30, [sp, #-16]!
     mov     x29, sp
 
-    bl      flush_dcache
-
-    /* Ensure I-cache, D-cache and mmu are disabled for EL1/Stage1 */
-    disable_mmu sctlr_el1 , x8
-
-    /*
-     * Invalidate the local I-cache so that any instructions fetched
-     * speculatively are discarded.
-     */
-    bl      invalidate_icache
-
     /*
      *   DEVICE_nGnRnE      000     00000000
      *   DEVICE_nGnRE       001     00000100

elfloader-tool/src/arch-arm/smp_boot.c

@@ -45,12 +45,19 @@ void non_boot_main(void)
         abort();
     }
 
-#ifndef CONFIG_ARM_HYPERVISOR_SUPPORT
     if (is_hyp_mode()) {
         extern void leave_hyp(void);
+        extern void disable_mmu_caches_hyp(void);
+#ifdef CONFIG_ARCH_AARCH64
+        /* Disable the MMU and cacheability unconditionally on ARM64.
+         * The 32 bit ARM platforms do not expect the MMU to be turned
+         * off, so we leave them alone. */
+        disable_mmu_caches_hyp();
+#endif
+#ifndef CONFIG_ARM_HYPERVISOR_SUPPORT
         leave_hyp();
-    }
 #endif
+    }
     /* Enable the MMU, and enter the kernel. */
     if (is_hyp_mode()) {
         arm_enable_hyp_mmu();

elfloader-tool/src/arch-arm/sys_boot.c

@@ -168,6 +168,92 @@ void main(UNUSED void *arg)
     abort();
 }
 
+/* ARMv8 64-bit specific implementation of continue_boot() */
+#if defined(CONFIG_ARCH_AARCH64)
+void continue_boot(int was_relocated)
+{
+    if (was_relocated) {
+        printf("ELF loader relocated, continuing boot...\n");
+    }
+
+    /*
+     * If we were relocated, we need to re-initialise the
+     * driver model so all its pointers are set up properly.
+     */
+    if (was_relocated) {
+        initialise_devices();
+    }
+
+    /* If in EL2, disable MMU and I/D cacheability unconditionally */
+    if (is_hyp_mode()) {
+        extern void disable_mmu_caches_hyp(void);
+        extern void clean_dcache_by_range(paddr_t start, paddr_t end);
+
+        paddr_t start = kernel_info.phys_region_start;
+        paddr_t end = kernel_info.phys_region_end;
+        clean_dcache_by_range(start, end);
+        start = (paddr_t)user_info.phys_region_start;
+        end = (paddr_t)user_info.phys_region_end;
+        clean_dcache_by_range(start, end);
+        start = (paddr_t)_text;
+        end = (paddr_t)_end;
+        clean_dcache_by_range(start, end);
+        if (dtb) {
+            start = (paddr_t)dtb;
+            end = start + dtb_size;
+            clean_dcache_by_range(start, end);
+        }
+
+#if defined(CONFIG_ARCH_AARCH64)
+        /* Disable the MMU and cacheability unconditionally on ARM64.
+         * The 32 bit ARM platforms do not expect the MMU to be turned
+         * off, so we leave them alone. */
+        disable_mmu_caches_hyp();
+#endif
+
+#if (defined(CONFIG_ARCH_ARM_V7A) || defined(CONFIG_ARCH_ARM_V8A)) && !defined(CONFIG_ARM_HYPERVISOR_SUPPORT)
+        extern void leave_hyp(void);
+        /* Switch to EL1, assume EL2 MMU already disabled for ARMv8. */
+        leave_hyp();
+#endif
+    /* Setup MMU. */
+    if (is_hyp_mode()) {
+        init_hyp_boot_vspace(&kernel_info);
+    } else {
+        /* If we are not in HYP mode, we enable the SV MMU and paging
+         * just in case the kernel does not support hyp mode. */
+        init_boot_vspace(&kernel_info);
+    }
+
+#if CONFIG_MAX_NUM_NODES > 1
+    smp_boot();
+#endif /* CONFIG_MAX_NUM_NODES */
+
+    if (is_hyp_mode()) {
+        printf("Enabling hypervisor MMU and paging\n");
+        arm_enable_hyp_mmu();
+    } else {
+        printf("Enabling MMU and paging\n");
+        arm_enable_mmu();
+    }
+
+    /* Enter kernel. The UART may no longer be accessible here. */
+    if ((uintptr_t)uart_get_mmio() < kernel_info.virt_region_start) {
+        printf("Jumping to kernel-image entry point...\n\n");
+    }
+
+    ((init_arm_kernel_t)kernel_info.virt_entry)(user_info.phys_region_start,
+                                                user_info.phys_region_end,
+                                                user_info.phys_virt_offset,
+                                                user_info.virt_entry,
+                                                (word_t)dtb,
+                                                dtb_size);
+
+    /* We should never get here. */
+    printf("ERROR: Kernel returned back to the ELF Loader\n");
+    abort();
+}
+#else
 void continue_boot(int was_relocated)
 {
     if (was_relocated) {
@@ -232,3 +318,4 @@ void continue_boot(int was_relocated)
     printf("ERROR: Kernel returned back to the ELF Loader\n");
     abort();
 }
+#endif

elfloader-tool/src/binaries/efi/gnuefi/elf_aarch64_efi.lds

@@ -61,6 +61,7 @@ SECTIONS
   .dynstr   : { *(.dynstr) }
   . = ALIGN(4096);
   .note.gnu.build-id : { *(.note.gnu.build-id) }
+  _end = .;
   /DISCARD/ :
   {
     *(.rel.reloc)