2 * ACPI 3.0 based NUMA setup
3 * Copyright 2004 Andi Kleen, SuSE Labs.
5 * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
7 * Called from acpi_numa_init while reading the SRAT and SLIT tables.
8 * Assumes all memory regions belonging to a single proximity domain
9 * are in one chunk. Holes between them will be included in the node.
12 #include <linux/kernel.h>
13 #include <linux/acpi.h>
14 #include <linux/mmzone.h>
15 #include <linux/bitmap.h>
16 #include <linux/module.h>
17 #include <linux/topology.h>
18 #include <linux/bootmem.h>
20 #include <asm/proto.h>
23 #include <asm/genapic.h>
25 int acpi_numa __initdata;
27 static struct acpi_table_slit *acpi_slit;
29 static nodemask_t nodes_parsed __initdata;
30 static struct bootnode nodes[MAX_NUMNODES] __initdata;
31 static struct bootnode nodes_add[MAX_NUMNODES];
32 static int found_add_area __initdata;
33 int hotadd_percent __initdata = 0;
35 static int num_node_memblks __initdata;
36 static struct bootnode node_memblk_range[NR_NODE_MEMBLKS] __initdata;
37 static int memblk_nodeid[NR_NODE_MEMBLKS] __initdata;
39 /* Too small nodes confuse the VM badly. Usually they result
41 #define NODE_MIN_SIZE (4*1024*1024)
43 static __init int setup_node(int pxm)
45 return acpi_map_pxm_to_node(pxm);
48 static __init int conflicting_memblks(unsigned long start, unsigned long end)
51 for (i = 0; i < num_node_memblks; i++) {
52 struct bootnode *nd = &node_memblk_range[i];
53 if (nd->start == nd->end)
55 if (nd->end > start && nd->start < end)
56 return memblk_nodeid[i];
57 if (nd->end == end && nd->start == start)
58 return memblk_nodeid[i];
63 static __init void cutoff_node(int i, unsigned long start, unsigned long end)
65 struct bootnode *nd = &nodes[i];
70 if (nd->start < start) {
72 if (nd->end < nd->start)
77 if (nd->start > nd->end)
82 static __init void bad_srat(void)
85 printk(KERN_ERR "SRAT: SRAT not used.\n");
88 for (i = 0; i < MAX_LOCAL_APIC; i++)
89 apicid_to_node[i] = NUMA_NO_NODE;
90 for (i = 0; i < MAX_NUMNODES; i++)
91 nodes_add[i].start = nodes[i].end = 0;
92 remove_all_active_ranges();
95 static __init inline int srat_disabled(void)
97 return numa_off || acpi_numa < 0;
100 /* Callback for SLIT parsing */
101 void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
106 length = slit->header.length;
107 phys = find_e820_area(0, max_pfn_mapped<<PAGE_SHIFT, length,
111 panic(" Can not save slit!\n");
113 acpi_slit = __va(phys);
114 memcpy(acpi_slit, slit, length);
115 reserve_early(phys, phys + length, "ACPI SLIT");
118 /* Callback for Proximity Domain -> x2APIC mapping */
120 acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
127 if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
131 if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
133 pxm = pa->proximity_domain;
134 node = setup_node(pxm);
136 printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
141 apic_id = pa->apic_id;
142 apicid_to_node[apic_id] = node;
144 printk(KERN_INFO "SRAT: PXM %u -> APIC %u -> Node %u\n",
148 /* Callback for Proximity Domain -> LAPIC mapping */
150 acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
157 if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
161 if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
163 pxm = pa->proximity_domain_lo;
164 node = setup_node(pxm);
166 printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
171 if (get_uv_system_type() >= UV_X2APIC)
172 apic_id = (pa->apic_id << 8) | pa->local_sapic_eid;
174 apic_id = pa->apic_id;
175 apicid_to_node[apic_id] = node;
177 printk(KERN_INFO "SRAT: PXM %u -> APIC %u -> Node %u\n",
181 static int update_end_of_memory(unsigned long end) {return -1;}
182 static int hotadd_enough_memory(struct bootnode *nd) {return 1;}
183 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
184 static inline int save_add_info(void) {return 1;}
186 static inline int save_add_info(void) {return 0;}
189 * Update nodes_add and decide if to include add are in the zone.
190 * Both SPARSE and RESERVE need nodes_add information.
191 * This code supports one contiguous hot add area per node.
194 reserve_hotadd(int node, unsigned long start, unsigned long end)
196 unsigned long s_pfn = start >> PAGE_SHIFT;
197 unsigned long e_pfn = end >> PAGE_SHIFT;
198 int ret = 0, changed = 0;
199 struct bootnode *nd = &nodes_add[node];
201 /* I had some trouble with strange memory hotadd regions breaking
202 the boot. Be very strict here and reject anything unexpected.
203 If you want working memory hotadd write correct SRATs.
205 The node size check is a basic sanity check to guard against
207 if ((signed long)(end - start) < NODE_MIN_SIZE) {
208 printk(KERN_ERR "SRAT: Hotplug area too small\n");
212 /* This check might be a bit too strict, but I'm keeping it for now. */
213 if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
215 "SRAT: Hotplug area %lu -> %lu has existing memory\n",
220 if (!hotadd_enough_memory(&nodes_add[node])) {
221 printk(KERN_ERR "SRAT: Hotplug area too large\n");
227 if (nd->start == nd->end) {
232 if (nd->start == end) {
236 if (nd->end == start) {
241 printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
244 ret = update_end_of_memory(nd->end);
247 printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n", nd->start, nd->end);
251 /* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
253 acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
255 struct bootnode *nd, oldnode;
256 unsigned long start, end;
262 if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
266 if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
269 if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
271 start = ma->base_address;
272 end = start + ma->length;
273 pxm = ma->proximity_domain;
274 node = setup_node(pxm);
276 printk(KERN_ERR "SRAT: Too many proximity domains.\n");
280 i = conflicting_memblks(start, end);
283 "SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n",
284 pxm, start, end, nodes[i].start, nodes[i].end);
287 "SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n",
288 pxm, start, end, node_to_pxm(i),
289 nodes[i].start, nodes[i].end);
295 if (!node_test_and_set(node, nodes_parsed)) {
299 if (start < nd->start)
305 printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm,
307 e820_register_active_regions(node, start >> PAGE_SHIFT,
309 push_node_boundaries(node, nd->start >> PAGE_SHIFT,
310 nd->end >> PAGE_SHIFT);
312 if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) &&
313 (reserve_hotadd(node, start, end) < 0)) {
314 /* Ignore hotadd region. Undo damage */
315 printk(KERN_NOTICE "SRAT: Hotplug region ignored\n");
317 if ((nd->start | nd->end) == 0)
318 node_clear(node, nodes_parsed);
321 node_memblk_range[num_node_memblks].start = start;
322 node_memblk_range[num_node_memblks].end = end;
323 memblk_nodeid[num_node_memblks] = node;
327 /* Sanity check to catch more bad SRATs (they are amazingly common).
328 Make sure the PXMs cover all memory. */
329 static int __init nodes_cover_memory(const struct bootnode *nodes)
332 unsigned long pxmram, e820ram;
335 for_each_node_mask(i, nodes_parsed) {
336 unsigned long s = nodes[i].start >> PAGE_SHIFT;
337 unsigned long e = nodes[i].end >> PAGE_SHIFT;
339 pxmram -= absent_pages_in_range(s, e);
340 if ((long)pxmram < 0)
344 e820ram = max_pfn - absent_pages_in_range(0, max_pfn);
345 /* We seem to lose 3 pages somewhere. Allow a bit of slack. */
346 if ((long)(e820ram - pxmram) >= 1*1024*1024) {
348 "SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n",
349 (pxmram << PAGE_SHIFT) >> 20,
350 (e820ram << PAGE_SHIFT) >> 20);
356 static void __init unparse_node(int node)
359 node_clear(node, nodes_parsed);
360 for (i = 0; i < MAX_LOCAL_APIC; i++) {
361 if (apicid_to_node[i] == node)
362 apicid_to_node[i] = NUMA_NO_NODE;
366 void __init acpi_numa_arch_fixup(void) {}
368 /* Use the information discovered above to actually set up the nodes. */
369 int __init acpi_scan_nodes(unsigned long start, unsigned long end)
376 /* First clean up the node list */
377 for (i = 0; i < MAX_NUMNODES; i++) {
378 cutoff_node(i, start, end);
380 * don't confuse VM with a node that doesn't have the
384 (nodes[i].end - nodes[i].start) < NODE_MIN_SIZE) {
390 if (!nodes_cover_memory(nodes)) {
395 memnode_shift = compute_hash_shift(node_memblk_range, num_node_memblks,
397 if (memnode_shift < 0) {
399 "SRAT: No NUMA node hash function found. Contact maintainer\n");
404 node_possible_map = nodes_parsed;
406 /* Finally register nodes */
407 for_each_node_mask(i, node_possible_map)
408 setup_node_bootmem(i, nodes[i].start, nodes[i].end);
409 /* Try again in case setup_node_bootmem missed one due
410 to missing bootmem */
411 for_each_node_mask(i, node_possible_map)
413 setup_node_bootmem(i, nodes[i].start, nodes[i].end);
415 for (i = 0; i < nr_cpu_ids; i++) {
416 int node = early_cpu_to_node(i);
418 if (node == NUMA_NO_NODE)
420 if (!node_isset(node, node_possible_map))
427 #ifdef CONFIG_NUMA_EMU
428 static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
429 [0 ... MAX_NUMNODES-1] = PXM_INVAL
431 static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
432 [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
434 static int __init find_node_by_addr(unsigned long addr)
436 int ret = NUMA_NO_NODE;
439 for_each_node_mask(i, nodes_parsed) {
441 * Find the real node that this emulated node appears on. For
442 * the sake of simplicity, we only use a real node's starting
443 * address to determine which emulated node it appears on.
445 if (addr >= nodes[i].start && addr < nodes[i].end) {
454 * In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
455 * mappings that respect the real ACPI topology but reflect our emulated
456 * environment. For each emulated node, we find which real node it appears on
457 * and create PXM to NID mappings for those fake nodes which mirror that
458 * locality. SLIT will now represent the correct distances between emulated
459 * nodes as a result of the real topology.
461 void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes)
465 printk(KERN_INFO "Faking PXM affinity for fake nodes on real "
467 for (i = 0; i < num_nodes; i++) {
470 nid = find_node_by_addr(fake_nodes[i].start);
471 if (nid == NUMA_NO_NODE)
473 pxm = node_to_pxm(nid);
474 if (pxm == PXM_INVAL)
476 fake_node_to_pxm_map[i] = pxm;
478 * For each apicid_to_node mapping that exists for this real
479 * node, it must now point to the fake node ID.
481 for (j = 0; j < MAX_LOCAL_APIC; j++)
482 if (apicid_to_node[j] == nid)
483 fake_apicid_to_node[j] = i;
485 for (i = 0; i < num_nodes; i++)
486 __acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i);
487 memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node));
489 nodes_clear(nodes_parsed);
490 for (i = 0; i < num_nodes; i++)
491 if (fake_nodes[i].start != fake_nodes[i].end)
492 node_set(i, nodes_parsed);
493 WARN_ON(!nodes_cover_memory(fake_nodes));
496 static int null_slit_node_compare(int a, int b)
498 return node_to_pxm(a) == node_to_pxm(b);
501 static int null_slit_node_compare(int a, int b)
505 #endif /* CONFIG_NUMA_EMU */
507 void __init srat_reserve_add_area(int nodeid)
509 if (found_add_area && nodes_add[nodeid].end) {
512 printk(KERN_INFO "SRAT: Reserving hot-add memory space "
513 "for node %d at %Lx-%Lx\n",
514 nodeid, nodes_add[nodeid].start, nodes_add[nodeid].end);
515 total_mb = (nodes_add[nodeid].end - nodes_add[nodeid].start)
517 total_mb *= sizeof(struct page);
519 printk(KERN_INFO "SRAT: This will cost you %Lu MB of "
520 "pre-allocated memory.\n", (unsigned long long)total_mb);
521 reserve_bootmem_node(NODE_DATA(nodeid), nodes_add[nodeid].start,
522 nodes_add[nodeid].end - nodes_add[nodeid].start,
527 int __node_distance(int a, int b)
532 return null_slit_node_compare(a, b) ? LOCAL_DISTANCE :
534 index = acpi_slit->locality_count * node_to_pxm(a);
535 return acpi_slit->entry[index + node_to_pxm(b)];
538 EXPORT_SYMBOL(__node_distance);
540 #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || defined(CONFIG_ACPI_HOTPLUG_MEMORY)
541 int memory_add_physaddr_to_nid(u64 start)
546 if (nodes_add[i].start <= start && nodes_add[i].end > start)
551 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);