Merge git://oss.sgi.com:8090/oss/git/xfs-2.6

[linux-2.6-omap-h63xx.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 6b92a945ae6bbb8ee9d3e2d04b43136734185cff..8c960b469593a9a6818fa4dd9c08a4b591916c86 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -137,9 +137,9 @@ static inline int bad_range(struct zone *zone, struct page *page)
 static void bad_page(struct page *page)
 {
        printk(KERN_EMERG "Bad page state in process '%s'\n"
-               "page:%p flags:0x%0*lx mapping:%p mapcount:%d count:%d\n"
-               "Trying to fix it up, but a reboot is needed\n"
-               "Backtrace:\n",
+               KERN_EMERG "page:%p flags:0x%0*lx mapping:%p mapcount:%d count:%d\n"
+               KERN_EMERG "Trying to fix it up, but a reboot is needed\n"
+               KERN_EMERG "Backtrace:\n",
                current->comm, page, (int)(2*sizeof(unsigned long)),
                (unsigned long)page->flags, page->mapping,
                page_mapcount(page), page_count(page));
@@ -308,7 +308,7 @@ static inline int page_is_buddy(struct page *page, int order)
  * -- wli
  */
 
-static inline void __free_pages_bulk (struct page *page,
+static inline void __free_one_page(struct page *page,
                struct zone *zone, unsigned int order)
 {
        unsigned long page_idx;
@@ -383,40 +383,45 @@ static inline int free_pages_check(struct page *page)
  * And clear the zone's pages_scanned counter, to hold off the "all pages are
  * pinned" detection logic.
  */
-static int
-free_pages_bulk(struct zone *zone, int count,
-               struct list_head *list, unsigned int order)
+static void free_pages_bulk(struct zone *zone, int count,
+                                       struct list_head *list, int order)
 {
-       struct page *page = NULL;
-       int ret = 0;
-
        spin_lock(&zone->lock);
        zone->all_unreclaimable = 0;
        zone->pages_scanned = 0;
-       while (!list_empty(list) && count--) {
+       while (count--) {
+               struct page *page;
+
+               BUG_ON(list_empty(list));
                page = list_entry(list->prev, struct page, lru);
-               /* have to delete it as __free_pages_bulk list manipulates */
+               /* have to delete it as __free_one_page list manipulates */
                list_del(&page->lru);
-               __free_pages_bulk(page, zone, order);
-               ret++;
+               __free_one_page(page, zone, order);
        }
        spin_unlock(&zone->lock);
-       return ret;
 }
 
-void __free_pages_ok(struct page *page, unsigned int order)
+static void free_one_page(struct zone *zone, struct page *page, int order)
 {
-       unsigned long flags;
        LIST_HEAD(list);
+       list_add(&page->lru, &list);
+       free_pages_bulk(zone, 1, &list, order);
+}
+
+static void __free_pages_ok(struct page *page, unsigned int order)
+{
+       unsigned long flags;
        int i;
        int reserved = 0;
 
        arch_free_page(page, order);
+       if (!PageHighMem(page))
+               mutex_debug_check_no_locks_freed(page_address(page),
+                                                PAGE_SIZE<<order);
 
 #ifndef CONFIG_MMU
-       if (order > 0)
-               for (i = 1 ; i < (1 << order) ; ++i)
-                       __put_page(page + i);
+       for (i = 1 ; i < (1 << order) ; ++i)
+               __put_page(page + i);
 #endif
 
        for (i = 0 ; i < (1 << order) ; ++i)
@@ -424,11 +429,10 @@ void __free_pages_ok(struct page *page, unsigned int order)
        if (reserved)
                return;
 
-       list_add(&page->lru, &list);
-       kernel_map_pages(page, 1<<order, 0);
+       kernel_map_pages(page, 1 << order, 0);
        local_irq_save(flags);
        __mod_page_state(pgfree, 1 << order);
-       free_pages_bulk(page_zone(page), 1, &list, order);
+       free_one_page(page_zone(page), page, order);
        local_irq_restore(flags);
 }
 
@@ -602,9 +606,8 @@ void drain_remote_pages(void)
                        struct per_cpu_pages *pcp;
 
                        pcp = &pset->pcp[i];
-                       if (pcp->count)
-                               pcp->count -= free_pages_bulk(zone, pcp->count,
-                                               &pcp->list, 0);
+                       free_pages_bulk(zone, pcp->count, &pcp->list, 0);
+                       pcp->count = 0;
                }
        }
        local_irq_restore(flags);
@@ -627,8 +630,8 @@ static void __drain_pages(unsigned int cpu)
 
                        pcp = &pset->pcp[i];
                        local_irq_save(flags);
-                       pcp->count -= free_pages_bulk(zone, pcp->count,
-                                               &pcp->list, 0);
+                       free_pages_bulk(zone, pcp->count, &pcp->list, 0);
+                       pcp->count = 0;
                        local_irq_restore(flags);
                }
        }
@@ -719,8 +722,10 @@ static void fastcall free_hot_cold_page(struct page *page, int cold)
        __inc_page_state(pgfree);
        list_add(&page->lru, &pcp->list);
        pcp->count++;
-       if (pcp->count >= pcp->high)
-               pcp->count -= free_pages_bulk(zone, pcp->batch, &pcp->list, 0);
+       if (pcp->count >= pcp->high) {
+               free_pages_bulk(zone, pcp->batch, &pcp->list, 0);
+               pcp->count -= pcp->batch;
+       }
        local_irq_restore(flags);
        put_cpu();
 }
@@ -759,7 +764,7 @@ static struct page *buffered_rmqueue(struct zonelist *zonelist,
 
 again:
        cpu  = get_cpu();
-       if (order == 0) {
+       if (likely(order == 0)) {
                struct per_cpu_pages *pcp;
 
                pcp = &zone_pcp(zone, cpu)->pcp[cold];
@@ -926,7 +931,8 @@ restart:
         *
         * The caller may dip into page reserves a bit more if the caller
         * cannot run direct reclaim, or if the caller has realtime scheduling
-        * policy.
+        * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
+        * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
         */
        alloc_flags = ALLOC_WMARK_MIN;
        if ((unlikely(rt_task(p)) && !in_interrupt()) || !wait)
@@ -974,6 +980,7 @@ rebalance:
        cond_resched();
 
        /* We now go into synchronous reclaim */
+       cpuset_memory_pressure_bump();
        p->flags |= PF_MEMALLOC;
        reclaim_state.reclaimed_slab = 0;
        p->reclaim_state = &reclaim_state;
@@ -1735,7 +1742,7 @@ void __devinit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
        unsigned long end_pfn = start_pfn + size;
        unsigned long pfn;
 
-       for (pfn = start_pfn; pfn < end_pfn; pfn++, page++) {
+       for (pfn = start_pfn; pfn < end_pfn; pfn++) {
                if (!early_pfn_valid(pfn))
                        continue;
                page = pfn_to_page(pfn);