Introduce "ondemand" linker sections

Kconfig.zephyr

@@ -260,6 +260,20 @@ config LINKER_USE_PINNED_SECTION
 	  Requires that pinned sections exist in the architecture, SoC,
 	  board or custom linker script.
 
+config LINKER_USE_ONDEMAND_SECTION
+	bool "Use Evictable Linker Section"
+	depends on DEMAND_MAPPING
+	depends on !LINKER_USE_PINNED_SECTION
+	depends on !ARCH_MAPS_ALL_RAM
+	help
+	  If enabled, the symbols which may be evicted from memory
+	  will be put into a linker section reserved for on-demand symbols.
+	  During boot, the corresponding memory will be mapped as paged out.
+	  This is conceptually the opposite of CONFIG_LINKER_USE_PINNED_SECTION.
+
+	  Requires that on-demand sections exist in the architecture, SoC,
+	  board or custom linker script.
+
 config LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT
 	bool "Generic sections are present at boot" if DEMAND_PAGING && LINKER_USE_PINNED_SECTION
 	default y

doc/kernel/memory_management/demand_paging.rst

@@ -179,6 +179,11 @@ which must be implemented:
   free a backing store location (the ``location`` token) which can
   then be used for subsequent page out operation.
 
+* :c:func:`k_mem_paging_backing_store_location_query()` is called to obtain
+  the ``location`` token corresponding to storage content to be virtually
+  mapped and paged-in on demand. Most useful with
+  :kconfig:option:`CONFIG_DEMAND_MAPPING`.
+
 * :c:func:`k_mem_paging_backing_store_page_in()` copies a data page
   from the backing store location associated with the provided
   ``location`` token to the page pointed by ``K_MEM_SCRATCH_PAGE``.

include/zephyr/arch/arm64/scripts/linker.ld

@@ -70,6 +70,9 @@ MEMORY
     LINKER_DT_REGIONS()
     /* Used by and documented in include/linker/intlist.ld */
     IDT_LIST  (wx) : ORIGIN = 0xFFFF8000, LENGTH = 32K
+#ifdef CONFIG_LINKER_USE_ONDEMAND_SECTION
+    ONDEMAND (rx) : ORIGIN = (CONFIG_KERNEL_VM_BASE), LENGTH = (CONFIG_KERNEL_VM_SIZE)
+#endif
 }
 
 ENTRY(CONFIG_KERNEL_ENTRY)
@@ -315,6 +318,35 @@ SECTIONS
 
     GROUP_END(RAMABLE_REGION)
 
+#ifdef CONFIG_LINKER_USE_ONDEMAND_SECTION
+    . = z_mapped_end;
+    MMU_ALIGN;
+    lnkr_ondemand_start = .;
+
+    ONDEMAND_TEXT_SECTION_NAME (.) : AT(ADDR(_BSS_SECTION_NAME))
+    {
+        lnkr_ondemand_text_start = .;
+        *(.ondemand_text)
+        *(.ondemand_text.*)
+        MMU_ALIGN;
+        lnkr_ondemand_text_end = .;
+    } > ONDEMAND
+    lnkr_ondemand_text_size = SIZEOF(ONDEMAND_TEXT_SECTION_NAME);
+
+    ONDEMAND_RODATA_SECTION_NAME :
+    {
+        lnkr_ondemand_rodata_start = .;
+        *(.ondemand_rodata)
+        *(.ondemand_rodata.*)
+        MMU_ALIGN;
+        lnkr_ondemand_rodata_end = .;
+    } > ONDEMAND
+    lnkr_ondemand_rodata_size = SIZEOF(ONDEMAND_RODATA_SECTION_NAME);
+
+    lnkr_ondemand_end = .;
+    lnkr_ondemand_load_start = LOADADDR(ONDEMAND_TEXT_SECTION_NAME);
+#endif
+
 #include <zephyr/linker/debug-sections.ld>
 
     SECTION_PROLOGUE(.ARM.attributes, 0,)

include/zephyr/kernel/mm/demand_paging.h

@@ -349,6 +349,22 @@ int k_mem_paging_backing_store_location_get(struct k_mem_page_frame *pf,
  */
 void k_mem_paging_backing_store_location_free(uintptr_t location);
 
+/**
+ * Obtain persistent location token for on-demand content
+ *
+ * Unlike k_mem_paging_backing_store_location_get() this does not allocate
+ * any backing store space. Instead, it returns a location token corresponding
+ * to some fixed storage content to be paged in on demand. This is expected
+ * to be used in conjonction with CONFIG_LINKER_USE_ONDEMAND_SECTION and the
+ * K_MEM_MAP_UNPAGED flag to create demand mappings at boot time. This may
+ * also be used e.g. to implement file-based mmap().
+ *
+ * @param addr Virtual address to obtain a location token for
+ * @param [out] location storage location token
+ * @return 0 for success or negative error code
+ */
+int k_mem_paging_backing_store_location_query(void *addr, uintptr_t *location);
+
 /**
  * Copy a data page from K_MEM_SCRATCH_PAGE to the specified location
  *

include/zephyr/linker/linker-defs.h

@@ -337,6 +337,24 @@ static inline bool lnkr_is_region_pinned(uint8_t *addr, size_t sz)
 
 #endif /* CONFIG_LINKER_USE_PINNED_SECTION */
 
+#ifdef CONFIG_LINKER_USE_ONDEMAND_SECTION
+/* lnkr_ondemand_start[] and lnkr_ondemand_end[] must encapsulate
+ * all the on-demand sections as these are used by
+ * the MMU code to mark the virtual pages with the appropriate backing store
+ * location token to have them be paged in on demand.
+ */
+extern char lnkr_ondemand_start[];
+extern char lnkr_ondemand_end[];
+extern char lnkr_ondemand_load_start[];
+
+extern char lnkr_ondemand_text_start[];
+extern char lnkr_ondemand_text_end[];
+extern char lnkr_ondemand_text_size[];
+extern char lnkr_ondemand_rodata_start[];
+extern char lnkr_ondemand_rodata_end[];
+extern char lnkr_ondemand_rodata_size[];
+
+#endif /* CONFIG_LINKER_USE_ONDEMAND_SECTION */
 #endif /* ! _ASMLANGUAGE */
 
 #endif /* ZEPHYR_INCLUDE_LINKER_LINKER_DEFS_H_ */

include/zephyr/linker/section_tags.h

@@ -100,6 +100,14 @@
 #define __pinned_noinit	__noinit
 #endif /* CONFIG_LINKER_USE_PINNED_SECTION */
 
+#if defined(CONFIG_LINKER_USE_ONDEMAND_SECTION)
+#define __ondemand_func	Z_GENERIC_DOT_SECTION(ONDEMAND_TEXT_SECTION_NAME)
+#define __ondemand_rodata	Z_GENERIC_DOT_SECTION(ONDEMAND_RODATA_SECTION_NAME)
+#else
+#define __ondemand_func
+#define __ondemand_rodata
+#endif /* CONFIG_LINKER_USE_ONDEMAND_SECTION */
+
 #if defined(CONFIG_LINKER_USE_PINNED_SECTION)
 #define __isr		__pinned_func
 #else

include/zephyr/linker/sections.h

@@ -101,6 +101,11 @@
 #define PINNED_NOINIT_SECTION_NAME	pinned_noinit
 #endif
 
+#if defined(CONFIG_LINKER_USE_ONDEMAND_SECTION)
+#define ONDEMAND_TEXT_SECTION_NAME	ondemand_text
+#define ONDEMAND_RODATA_SECTION_NAME	ondemand_rodata
+#endif
+
 /* Short section references for use in ASM files */
 #if defined(_ASMLANGUAGE)
 /* Various text section names */
@@ -140,6 +145,14 @@
 #define PINNED_NOINIT			NOINIT
 #endif /* CONFIG_LINKER_USE_PINNED_SECTION */
 
+#if defined(CONFIG_LINKER_USE_ONDEMAND_SECTION)
+#define ONDEMAND_TEXT			ONDEMAND_TEXT_SECTION_NAME
+#define ONDEMAND_RODATA		ONDEMAND_RODATA_SECTION_NAME
+#else
+#define ONDEMAND_TEXT			TEXT
+#define ONDEMAND_RODATA		RODATA
+#endif /* CONFIG_LINKER_USE_ONDEMAND_SECTION */
+
 #endif /* _ASMLANGUAGE */
 
 #include <zephyr/linker/section_tags.h>

kernel/mmu.c

@@ -1019,6 +1019,34 @@
 }
 #endif /* CONFIG_LINKER_USE_BOOT_SECTION) || CONFIG_LINKER_USE_PINNED_SECTION */
 
+#ifdef CONFIG_LINKER_USE_ONDEMAND_SECTION
+static void z_paging_ondemand_section_map(void)
+{
+	uint8_t *addr;
+	size_t size;
+	uintptr_t location;
+	uint32_t flags;
+
+	size = (uintptr_t)lnkr_ondemand_text_size;
+	flags = K_MEM_MAP_UNPAGED | K_MEM_PERM_EXEC | K_MEM_CACHE_WB;
+	VIRT_FOREACH(lnkr_ondemand_text_start, size, addr) {
+		k_mem_paging_backing_store_location_query(addr, &location);
+		arch_mem_map(addr, location, CONFIG_MMU_PAGE_SIZE, flags);
+		sys_bitarray_set_region(&virt_region_bitmap, 1,
+					virt_to_bitmap_offset(addr, CONFIG_MMU_PAGE_SIZE));
+	}
+
+	size = (uintptr_t)lnkr_ondemand_rodata_size;
+	flags = K_MEM_MAP_UNPAGED | K_MEM_CACHE_WB;
+	VIRT_FOREACH(lnkr_ondemand_rodata_start, size, addr) {
+		k_mem_paging_backing_store_location_query(addr, &location);
+		arch_mem_map(addr, location, CONFIG_MMU_PAGE_SIZE, flags);
+		sys_bitarray_set_region(&virt_region_bitmap, 1,
+					virt_to_bitmap_offset(addr, CONFIG_MMU_PAGE_SIZE));
+	}
+}
+#endif /* CONFIG_LINKER_USE_ONDEMAND_SECTION */
+
 void z_mem_manage_init(void)
 {
 	uintptr_t phys;
@@ -1095,6 +1123,11 @@
 		}
 	}
 #endif /* CONFIG_DEMAND_PAGING */
+
+#ifdef CONFIG_LINKER_USE_ONDEMAND_SECTION
+	z_paging_ondemand_section_map();
+#endif
+
 #if __ASSERT_ON
 	page_frames_initialized = true;
 #endif