x86: add brk allocation for very, very early allocations

Impact: new interface

Add a brk()-like allocator which effectively extends the bss in order
to allow very early code to do dynamic allocations.  This is better than
using statically allocated arrays for data in subsystems which may never
get used.

The space for brk allocations is in the bss ELF segment, so that the
space is mapped properly by the code which maps the kernel, and so
that bootloaders keep the space free rather than putting a ramdisk or
something into it.

The bss itself, delimited by __bss_stop, ends before the brk area
(__brk_base to __brk_limit).  The kernel text, data and bss is reserved
up to __bss_stop.

Any brk-allocated data is reserved separately just before the kernel
pagetable is built, as that code allocates from unreserved spaces
in the e820 map, potentially allocating from any unused brk memory.
Ultimately any unused memory in the brk area is used in the general
kernel memory pool.

Initially the brk space is set to 1MB, which is probably much larger
than any user needs (the largest current user is i386 head_32.S's code
to build the pagetables to map the kernel, which can get fairly large
with a big kernel image and no PSE support).  So long as the system
has sufficient memory for the bootloader to reserve the kernel+1MB brk,
there are no bad effects resulting from an over-large brk.

Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com>
Signed-off-by: H. Peter Anvin <hpa@zytor.com>

diff --git a/arch/x86/include/asm/sections.h b/arch/x86/include/asm/sections.h
index 2b8c5160388fb4863f05c33c15e42e22c53fc7a2..1b7ee5d673c23552514e7f6d6466bbcaa10ebeda 100644 (file)
--- a/arch/x86/include/asm/sections.h
+++ b/arch/x86/include/asm/sections.h
@@ -1 +1,8 @@
+#ifndef _ASM_X86_SECTIONS_H
+#define _ASM_X86_SECTIONS_H
+
 #include <asm-generic/sections.h>
+
+extern char __brk_base[], __brk_limit[];
+
+#endif /* _ASM_X86_SECTIONS_H */

diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index 05c6f6b11fd5d25354ddcd7d3d07cf7cf7bf3fe5..45454f3fa121d7341a3a263f87061b9e8afe8d9e 100644 (file)
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -100,6 +100,10 @@ extern struct boot_params boot_params;
  */
 #define LOWMEMSIZE()   (0x9f000)
 
+/* exceedingly early brk-like allocator */
+extern unsigned long _brk_end;
+void *extend_brk(size_t size, size_t align);
+
 #ifdef __i386__
 
 void __init i386_start_kernel(void);

diff --git a/arch/x86/kernel/head32.c b/arch/x86/kernel/head32.c
index ac108d1fe182a44abcd0da75f18aae5fde32e1c0..29f1095b08499399358a3ddcc7946a7eb52de5d4 100644 (file)
--- a/arch/x86/kernel/head32.c
+++ b/arch/x86/kernel/head32.c
@@ -18,7 +18,7 @@ void __init i386_start_kernel(void)
 {
        reserve_trampoline_memory();
 
-       reserve_early(__pa_symbol(&_text), __pa_symbol(&_end), "TEXT DATA BSS");
+       reserve_early(__pa_symbol(&_text), __pa_symbol(&__bss_stop), "TEXT DATA BSS");
 
 #ifdef CONFIG_BLK_DEV_INITRD
        /* Reserve INITRD */

diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index f5b2722476907bcd3fe7ace735238330c604b9e9..70eaa852c732a2db3ae5c809f56f27b9c0f42dc1 100644 (file)
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -100,7 +100,7 @@ void __init x86_64_start_reservations(char *real_mode_data)
 
        reserve_trampoline_memory();
 
-       reserve_early(__pa_symbol(&_text), __pa_symbol(&_end), "TEXT DATA BSS");
+       reserve_early(__pa_symbol(&_text), __pa_symbol(&__bss_stop), "TEXT DATA BSS");
 
 #ifdef CONFIG_BLK_DEV_INITRD
        /* Reserve INITRD */

diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index f28c56e6bf94a4cc30617f6b39d0349f13f9ebff..e6b742d2a3f503e0d83707c477811cf430868b4a 100644 (file)
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -114,6 +114,9 @@
 
 unsigned int boot_cpu_id __read_mostly;
 
+static __initdata unsigned long _brk_start = (unsigned long)__brk_base;
+unsigned long _brk_end = (unsigned long)__brk_base;
+
 #ifdef CONFIG_X86_64
 int default_cpu_present_to_apicid(int mps_cpu)
 {
@@ -337,6 +340,34 @@ static void __init relocate_initrd(void)
 }
 #endif
 
+void * __init extend_brk(size_t size, size_t align)
+{
+       size_t mask = align - 1;
+       void *ret;
+
+       BUG_ON(_brk_start == 0);
+       BUG_ON(align & mask);
+
+       _brk_end = (_brk_end + mask) & ~mask;
+       BUG_ON((char *)(_brk_end + size) > __brk_limit);
+
+       ret = (void *)_brk_end;
+       _brk_end += size;
+
+       memset(ret, 0, size);
+
+       return ret;
+}
+
+static void __init reserve_brk(void)
+{
+       if (_brk_end > _brk_start)
+               reserve_early(__pa(_brk_start), __pa(_brk_end), "BRK");
+
+       /* Mark brk area as locked down and no longer taking any new allocations */
+       _brk_start = 0;
+}
+
 static void __init reserve_initrd(void)
 {
        u64 ramdisk_image = boot_params.hdr.ramdisk_image;
@@ -717,11 +748,7 @@ void __init setup_arch(char **cmdline_p)
        init_mm.start_code = (unsigned long) _text;
        init_mm.end_code = (unsigned long) _etext;
        init_mm.end_data = (unsigned long) _edata;
-#ifdef CONFIG_X86_32
-       init_mm.brk = init_pg_tables_end + PAGE_OFFSET;
-#else
-       init_mm.brk = (unsigned long) &_end;
-#endif
+       init_mm.brk = _brk_end;
 
        code_resource.start = virt_to_phys(_text);
        code_resource.end = virt_to_phys(_etext)-1;
@@ -842,6 +869,8 @@ void __init setup_arch(char **cmdline_p)
        setup_bios_corruption_check();
 #endif
 
+       reserve_brk();
+
        /* max_pfn_mapped is updated here */
        max_low_pfn_mapped = init_memory_mapping(0, max_low_pfn<<PAGE_SHIFT);
        max_pfn_mapped = max_low_pfn_mapped;

diff --git a/arch/x86/kernel/vmlinux_32.lds.S b/arch/x86/kernel/vmlinux_32.lds.S
index 0d860963f268f5a79c1198a6dffba0fee22f897d..27e44aa215853da1b983a203ad44891c3f3f66d2 100644 (file)
--- a/arch/x86/kernel/vmlinux_32.lds.S
+++ b/arch/x86/kernel/vmlinux_32.lds.S
@@ -189,7 +189,14 @@ SECTIONS
        *(.bss)
        . = ALIGN(4);
        __bss_stop = .;
+
+       . = ALIGN(PAGE_SIZE);
+       __brk_base = . ;
+       . += 1024 * 1024 ;
+       __brk_limit = . ;
+
        _end = . ;
+
        /* This is where the kernel creates the early boot page tables */
        . = ALIGN(PAGE_SIZE);
        pg0 = . ;

diff --git a/arch/x86/kernel/vmlinux_64.lds.S b/arch/x86/kernel/vmlinux_64.lds.S
index fe5d21ce72417f4d11b1005d9e957eae4bb11758..ff373423138cd6a0ddb32eac33f0bd1b98d8325a 100644 (file)
--- a/arch/x86/kernel/vmlinux_64.lds.S
+++ b/arch/x86/kernel/vmlinux_64.lds.S
@@ -247,6 +247,11 @@ SECTIONS
        *(.bss.page_aligned)
        *(.bss)
        __bss_stop = .;
+
+       . = ALIGN(PAGE_SIZE);
+       __brk_base = . ;
+       . += 1024 * 1024 ;
+       __brk_limit = . ;
   }
 
   _end = . ;

diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 9c4294986af779ed62ab088af0dae0f0048eb0c9..1280565670e4c75736c823688f92e67015f5d8f4 100644 (file)
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -16,6 +16,7 @@
 #include <asm/processor.h>
 #include <asm/tlbflush.h>
 #include <asm/sections.h>
+#include <asm/setup.h>
 #include <asm/uaccess.h>
 #include <asm/pgalloc.h>
 #include <asm/proto.h>
@@ -95,7 +96,7 @@ static inline unsigned long highmap_start_pfn(void)
 
 static inline unsigned long highmap_end_pfn(void)
 {
-       return __pa(roundup((unsigned long)_end, PMD_SIZE)) >> PAGE_SHIFT;
+       return __pa(roundup(_brk_end, PMD_SIZE)) >> PAGE_SHIFT;
 }
 
 #endif
@@ -711,7 +712,7 @@ static int cpa_process_alias(struct cpa_data *cpa)
         * No need to redo, when the primary call touched the high
         * mapping already:
         */
-       if (within(vaddr, (unsigned long) _text, (unsigned long) _end))
+       if (within(vaddr, (unsigned long) _text, _brk_end))
                return 0;
 
        /*

diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index cb6afa4ec95c524ce8e95fa20b47b88a2f65dac9..72f6a76dbfb9b7d8bd7e216c9c500555a3570b25 100644 (file)
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1723,9 +1723,9 @@ __init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd,
 {
        pmd_t *kernel_pmd;
 
-       init_pg_tables_start = __pa(pgd);
-       init_pg_tables_end = __pa(pgd) + xen_start_info->nr_pt_frames*PAGE_SIZE;
-       max_pfn_mapped = PFN_DOWN(init_pg_tables_end + 512*1024);
+       max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->pt_base) +
+                                 xen_start_info->nr_pt_frames * PAGE_SIZE +
+                                 512*1024);
 
        kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd);
        memcpy(level2_kernel_pgt, kernel_pmd, sizeof(pmd_t) * PTRS_PER_PMD);