4 * Manages VM statistics
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
8 * Copyright (C) 2006 Silicon Graphics, Inc.,
9 * Christoph Lameter <christoph@lameter.com>
12 #include <linux/config.h>
14 #include <linux/module.h>
17 * Accumulate the page_state information across all CPUs.
18 * The result is unavoidably approximate - it can change
19 * during and after execution of this function.
21 DEFINE_PER_CPU(struct page_state, page_states) = {0};
23 static void __get_page_state(struct page_state *ret, int nr, cpumask_t *cpumask)
27 memset(ret, 0, nr * sizeof(unsigned long));
28 cpus_and(*cpumask, *cpumask, cpu_online_map);
30 for_each_cpu_mask(cpu, *cpumask) {
36 in = (unsigned long *)&per_cpu(page_states, cpu);
38 next_cpu = next_cpu(cpu, *cpumask);
39 if (likely(next_cpu < NR_CPUS))
40 prefetch(&per_cpu(page_states, next_cpu));
42 out = (unsigned long *)ret;
43 for (off = 0; off < nr; off++)
48 void get_full_page_state(struct page_state *ret)
50 cpumask_t mask = CPU_MASK_ALL;
52 __get_page_state(ret, sizeof(*ret) / sizeof(unsigned long), &mask);
55 unsigned long read_page_state_offset(unsigned long offset)
57 unsigned long ret = 0;
60 for_each_online_cpu(cpu) {
63 in = (unsigned long)&per_cpu(page_states, cpu) + offset;
64 ret += *((unsigned long *)in);
69 void __mod_page_state_offset(unsigned long offset, unsigned long delta)
73 ptr = &__get_cpu_var(page_states);
74 *(unsigned long *)(ptr + offset) += delta;
76 EXPORT_SYMBOL(__mod_page_state_offset);
78 void mod_page_state_offset(unsigned long offset, unsigned long delta)
83 local_irq_save(flags);
84 ptr = &__get_cpu_var(page_states);
85 *(unsigned long *)(ptr + offset) += delta;
86 local_irq_restore(flags);
88 EXPORT_SYMBOL(mod_page_state_offset);
90 void __get_zone_counts(unsigned long *active, unsigned long *inactive,
91 unsigned long *free, struct pglist_data *pgdat)
93 struct zone *zones = pgdat->node_zones;
99 for (i = 0; i < MAX_NR_ZONES; i++) {
100 *active += zones[i].nr_active;
101 *inactive += zones[i].nr_inactive;
102 *free += zones[i].free_pages;
106 void get_zone_counts(unsigned long *active,
107 unsigned long *inactive, unsigned long *free)
109 struct pglist_data *pgdat;
114 for_each_online_pgdat(pgdat) {
115 unsigned long l, m, n;
116 __get_zone_counts(&l, &m, &n, pgdat);
124 * Manage combined zone based / global counters
126 * vm_stat contains the global counters
128 atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
129 EXPORT_SYMBOL(vm_stat);
133 #define STAT_THRESHOLD 32
136 * Determine pointer to currently valid differential byte given a zone and
139 * Preemption must be off
141 static inline s8 *diff_pointer(struct zone *zone, enum zone_stat_item item)
143 return &zone_pcp(zone, smp_processor_id())->vm_stat_diff[item];
147 * For use when we know that interrupts are disabled.
149 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
155 p = diff_pointer(zone, item);
158 if (unlikely(x > STAT_THRESHOLD || x < -STAT_THRESHOLD)) {
159 zone_page_state_add(x, zone, item);
165 EXPORT_SYMBOL(__mod_zone_page_state);
168 * For an unknown interrupt state
170 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
175 local_irq_save(flags);
176 __mod_zone_page_state(zone, item, delta);
177 local_irq_restore(flags);
179 EXPORT_SYMBOL(mod_zone_page_state);
182 * Optimized increment and decrement functions.
184 * These are only for a single page and therefore can take a struct page *
185 * argument instead of struct zone *. This allows the inclusion of the code
186 * generated for page_zone(page) into the optimized functions.
188 * No overflow check is necessary and therefore the differential can be
189 * incremented or decremented in place which may allow the compilers to
190 * generate better code.
192 * The increment or decrement is known and therefore one boundary check can
195 * Some processors have inc/dec instructions that are atomic vs an interrupt.
196 * However, the code must first determine the differential location in a zone
197 * based on the processor number and then inc/dec the counter. There is no
198 * guarantee without disabling preemption that the processor will not change
199 * in between and therefore the atomicity vs. interrupt cannot be exploited
200 * in a useful way here.
202 void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
204 struct zone *zone = page_zone(page);
205 s8 *p = diff_pointer(zone, item);
209 if (unlikely(*p > STAT_THRESHOLD)) {
210 zone_page_state_add(*p, zone, item);
214 EXPORT_SYMBOL(__inc_zone_page_state);
216 void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
218 struct zone *zone = page_zone(page);
219 s8 *p = diff_pointer(zone, item);
223 if (unlikely(*p < -STAT_THRESHOLD)) {
224 zone_page_state_add(*p, zone, item);
228 EXPORT_SYMBOL(__dec_zone_page_state);
230 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
236 zone = page_zone(page);
237 local_irq_save(flags);
238 p = diff_pointer(zone, item);
242 if (unlikely(*p > STAT_THRESHOLD)) {
243 zone_page_state_add(*p, zone, item);
246 local_irq_restore(flags);
248 EXPORT_SYMBOL(inc_zone_page_state);
250 void dec_zone_page_state(struct page *page, enum zone_stat_item item)
256 zone = page_zone(page);
257 local_irq_save(flags);
258 p = diff_pointer(zone, item);
262 if (unlikely(*p < -STAT_THRESHOLD)) {
263 zone_page_state_add(*p, zone, item);
266 local_irq_restore(flags);
268 EXPORT_SYMBOL(dec_zone_page_state);
271 * Update the zone counters for one cpu.
273 void refresh_cpu_vm_stats(int cpu)
279 for_each_zone(zone) {
280 struct per_cpu_pageset *pcp;
282 pcp = zone_pcp(zone, cpu);
284 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
285 if (pcp->vm_stat_diff[i]) {
286 local_irq_save(flags);
287 zone_page_state_add(pcp->vm_stat_diff[i],
289 pcp->vm_stat_diff[i] = 0;
290 local_irq_restore(flags);
295 static void __refresh_cpu_vm_stats(void *dummy)
297 refresh_cpu_vm_stats(smp_processor_id());
301 * Consolidate all counters.
303 * Note that the result is less inaccurate but still inaccurate
304 * if concurrent processes are allowed to run.
306 void refresh_vm_stats(void)
308 on_each_cpu(__refresh_cpu_vm_stats, NULL, 0, 1);
310 EXPORT_SYMBOL(refresh_vm_stats);
314 #ifdef CONFIG_PROC_FS
316 #include <linux/seq_file.h>
318 static void *frag_start(struct seq_file *m, loff_t *pos)
322 for (pgdat = first_online_pgdat();
324 pgdat = next_online_pgdat(pgdat))
330 static void *frag_next(struct seq_file *m, void *arg, loff_t *pos)
332 pg_data_t *pgdat = (pg_data_t *)arg;
335 return next_online_pgdat(pgdat);
338 static void frag_stop(struct seq_file *m, void *arg)
343 * This walks the free areas for each zone.
345 static int frag_show(struct seq_file *m, void *arg)
347 pg_data_t *pgdat = (pg_data_t *)arg;
349 struct zone *node_zones = pgdat->node_zones;
353 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
354 if (!populated_zone(zone))
357 spin_lock_irqsave(&zone->lock, flags);
358 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
359 for (order = 0; order < MAX_ORDER; ++order)
360 seq_printf(m, "%6lu ", zone->free_area[order].nr_free);
361 spin_unlock_irqrestore(&zone->lock, flags);
367 struct seq_operations fragmentation_op = {
374 static char *vmstat_text[] = {
375 /* Zoned VM counters */
380 "nr_page_table_pages",
414 "pgscan_kswapd_high",
415 "pgscan_kswapd_normal",
416 "pgscan_kswapd_dma32",
419 "pgscan_direct_high",
420 "pgscan_direct_normal",
421 "pgscan_direct_dma32",
435 * Output information about zones in @pgdat.
437 static int zoneinfo_show(struct seq_file *m, void *arg)
439 pg_data_t *pgdat = arg;
441 struct zone *node_zones = pgdat->node_zones;
444 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; zone++) {
447 if (!populated_zone(zone))
450 spin_lock_irqsave(&zone->lock, flags);
451 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);
459 "\n scanned %lu (a: %lu i: %lu)"
469 zone->nr_scan_active, zone->nr_scan_inactive,
471 zone->present_pages);
473 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
474 seq_printf(m, "\n %-12s %lu", vmstat_text[i],
475 zone_page_state(zone, i));
478 "\n protection: (%lu",
479 zone->lowmem_reserve[0]);
480 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)
481 seq_printf(m, ", %lu", zone->lowmem_reserve[i]);
485 for_each_online_cpu(i) {
486 struct per_cpu_pageset *pageset;
489 pageset = zone_pcp(zone, i);
490 for (j = 0; j < ARRAY_SIZE(pageset->pcp); j++) {
491 if (pageset->pcp[j].count)
494 if (j == ARRAY_SIZE(pageset->pcp))
496 for (j = 0; j < ARRAY_SIZE(pageset->pcp); j++) {
503 pageset->pcp[j].count,
504 pageset->pcp[j].high,
505 pageset->pcp[j].batch);
511 "\n numa_foreign: %lu"
512 "\n interleave_hit: %lu"
514 "\n other_node: %lu",
517 pageset->numa_foreign,
518 pageset->interleave_hit,
520 pageset->other_node);
524 "\n all_unreclaimable: %u"
525 "\n prev_priority: %i"
526 "\n temp_priority: %i"
528 zone->all_unreclaimable,
531 zone->zone_start_pfn);
532 spin_unlock_irqrestore(&zone->lock, flags);
538 struct seq_operations zoneinfo_op = {
539 .start = frag_start, /* iterate over all zones. The same as in
543 .show = zoneinfo_show,
546 static void *vmstat_start(struct seq_file *m, loff_t *pos)
549 struct page_state *ps;
552 if (*pos >= ARRAY_SIZE(vmstat_text))
555 v = kmalloc(NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long)
556 + sizeof(*ps), GFP_KERNEL);
559 return ERR_PTR(-ENOMEM);
560 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
561 v[i] = global_page_state(i);
562 ps = (struct page_state *)(v + NR_VM_ZONE_STAT_ITEMS);
563 get_full_page_state(ps);
564 ps->pgpgin /= 2; /* sectors -> kbytes */
569 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos)
572 if (*pos >= ARRAY_SIZE(vmstat_text))
574 return (unsigned long *)m->private + *pos;
577 static int vmstat_show(struct seq_file *m, void *arg)
579 unsigned long *l = arg;
580 unsigned long off = l - (unsigned long *)m->private;
582 seq_printf(m, "%s %lu\n", vmstat_text[off], *l);
586 static void vmstat_stop(struct seq_file *m, void *arg)
592 struct seq_operations vmstat_op = {
593 .start = vmstat_start,
599 #endif /* CONFIG_PROC_FS */