]> pilppa.org Git - linux-2.6-omap-h63xx.git/blob - fs/buffer.c
[PATCH] __bread oops fix
[linux-2.6-omap-h63xx.git] / fs / buffer.c
1 /*
2  *  linux/fs/buffer.c
3  *
4  *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
5  */
6
7 /*
8  * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9  *
10  * Removed a lot of unnecessary code and simplified things now that
11  * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12  *
13  * Speed up hash, lru, and free list operations.  Use gfp() for allocating
14  * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
15  *
16  * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17  *
18  * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19  */
20
21 #include <linux/config.h>
22 #include <linux/kernel.h>
23 #include <linux/syscalls.h>
24 #include <linux/fs.h>
25 #include <linux/mm.h>
26 #include <linux/percpu.h>
27 #include <linux/slab.h>
28 #include <linux/smp_lock.h>
29 #include <linux/blkdev.h>
30 #include <linux/file.h>
31 #include <linux/quotaops.h>
32 #include <linux/highmem.h>
33 #include <linux/module.h>
34 #include <linux/writeback.h>
35 #include <linux/hash.h>
36 #include <linux/suspend.h>
37 #include <linux/buffer_head.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
43 #include <linux/bit_spinlock.h>
44
45 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
46 static void invalidate_bh_lrus(void);
47
48 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
49
50 inline void
51 init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
52 {
53         bh->b_end_io = handler;
54         bh->b_private = private;
55 }
56
57 static int sync_buffer(void *word)
58 {
59         struct block_device *bd;
60         struct buffer_head *bh
61                 = container_of(word, struct buffer_head, b_state);
62
63         smp_mb();
64         bd = bh->b_bdev;
65         if (bd)
66                 blk_run_address_space(bd->bd_inode->i_mapping);
67         io_schedule();
68         return 0;
69 }
70
71 void fastcall __lock_buffer(struct buffer_head *bh)
72 {
73         wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
74                                                         TASK_UNINTERRUPTIBLE);
75 }
76 EXPORT_SYMBOL(__lock_buffer);
77
78 void fastcall unlock_buffer(struct buffer_head *bh)
79 {
80         clear_buffer_locked(bh);
81         smp_mb__after_clear_bit();
82         wake_up_bit(&bh->b_state, BH_Lock);
83 }
84
85 /*
86  * Block until a buffer comes unlocked.  This doesn't stop it
87  * from becoming locked again - you have to lock it yourself
88  * if you want to preserve its state.
89  */
90 void __wait_on_buffer(struct buffer_head * bh)
91 {
92         wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
93 }
94
95 static void
96 __clear_page_buffers(struct page *page)
97 {
98         ClearPagePrivate(page);
99         set_page_private(page, 0);
100         page_cache_release(page);
101 }
102
103 static void buffer_io_error(struct buffer_head *bh)
104 {
105         char b[BDEVNAME_SIZE];
106
107         printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
108                         bdevname(bh->b_bdev, b),
109                         (unsigned long long)bh->b_blocknr);
110 }
111
112 /*
113  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
114  * unlock the buffer. This is what ll_rw_block uses too.
115  */
116 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
117 {
118         if (uptodate) {
119                 set_buffer_uptodate(bh);
120         } else {
121                 /* This happens, due to failed READA attempts. */
122                 clear_buffer_uptodate(bh);
123         }
124         unlock_buffer(bh);
125         put_bh(bh);
126 }
127
128 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
129 {
130         char b[BDEVNAME_SIZE];
131
132         if (uptodate) {
133                 set_buffer_uptodate(bh);
134         } else {
135                 if (!buffer_eopnotsupp(bh) && printk_ratelimit()) {
136                         buffer_io_error(bh);
137                         printk(KERN_WARNING "lost page write due to "
138                                         "I/O error on %s\n",
139                                        bdevname(bh->b_bdev, b));
140                 }
141                 set_buffer_write_io_error(bh);
142                 clear_buffer_uptodate(bh);
143         }
144         unlock_buffer(bh);
145         put_bh(bh);
146 }
147
148 /*
149  * Write out and wait upon all the dirty data associated with a block
150  * device via its mapping.  Does not take the superblock lock.
151  */
152 int sync_blockdev(struct block_device *bdev)
153 {
154         int ret = 0;
155
156         if (bdev) {
157                 int err;
158
159                 ret = filemap_fdatawrite(bdev->bd_inode->i_mapping);
160                 err = filemap_fdatawait(bdev->bd_inode->i_mapping);
161                 if (!ret)
162                         ret = err;
163         }
164         return ret;
165 }
166 EXPORT_SYMBOL(sync_blockdev);
167
168 /*
169  * Write out and wait upon all dirty data associated with this
170  * superblock.  Filesystem data as well as the underlying block
171  * device.  Takes the superblock lock.
172  */
173 int fsync_super(struct super_block *sb)
174 {
175         sync_inodes_sb(sb, 0);
176         DQUOT_SYNC(sb);
177         lock_super(sb);
178         if (sb->s_dirt && sb->s_op->write_super)
179                 sb->s_op->write_super(sb);
180         unlock_super(sb);
181         if (sb->s_op->sync_fs)
182                 sb->s_op->sync_fs(sb, 1);
183         sync_blockdev(sb->s_bdev);
184         sync_inodes_sb(sb, 1);
185
186         return sync_blockdev(sb->s_bdev);
187 }
188
189 /*
190  * Write out and wait upon all dirty data associated with this
191  * device.   Filesystem data as well as the underlying block
192  * device.  Takes the superblock lock.
193  */
194 int fsync_bdev(struct block_device *bdev)
195 {
196         struct super_block *sb = get_super(bdev);
197         if (sb) {
198                 int res = fsync_super(sb);
199                 drop_super(sb);
200                 return res;
201         }
202         return sync_blockdev(bdev);
203 }
204
205 /**
206  * freeze_bdev  --  lock a filesystem and force it into a consistent state
207  * @bdev:       blockdevice to lock
208  *
209  * This takes the block device bd_mount_sem to make sure no new mounts
210  * happen on bdev until thaw_bdev() is called.
211  * If a superblock is found on this device, we take the s_umount semaphore
212  * on it to make sure nobody unmounts until the snapshot creation is done.
213  */
214 struct super_block *freeze_bdev(struct block_device *bdev)
215 {
216         struct super_block *sb;
217
218         down(&bdev->bd_mount_sem);
219         sb = get_super(bdev);
220         if (sb && !(sb->s_flags & MS_RDONLY)) {
221                 sb->s_frozen = SB_FREEZE_WRITE;
222                 smp_wmb();
223
224                 sync_inodes_sb(sb, 0);
225                 DQUOT_SYNC(sb);
226
227                 lock_super(sb);
228                 if (sb->s_dirt && sb->s_op->write_super)
229                         sb->s_op->write_super(sb);
230                 unlock_super(sb);
231
232                 if (sb->s_op->sync_fs)
233                         sb->s_op->sync_fs(sb, 1);
234
235                 sync_blockdev(sb->s_bdev);
236                 sync_inodes_sb(sb, 1);
237
238                 sb->s_frozen = SB_FREEZE_TRANS;
239                 smp_wmb();
240
241                 sync_blockdev(sb->s_bdev);
242
243                 if (sb->s_op->write_super_lockfs)
244                         sb->s_op->write_super_lockfs(sb);
245         }
246
247         sync_blockdev(bdev);
248         return sb;      /* thaw_bdev releases s->s_umount and bd_mount_sem */
249 }
250 EXPORT_SYMBOL(freeze_bdev);
251
252 /**
253  * thaw_bdev  -- unlock filesystem
254  * @bdev:       blockdevice to unlock
255  * @sb:         associated superblock
256  *
257  * Unlocks the filesystem and marks it writeable again after freeze_bdev().
258  */
259 void thaw_bdev(struct block_device *bdev, struct super_block *sb)
260 {
261         if (sb) {
262                 BUG_ON(sb->s_bdev != bdev);
263
264                 if (sb->s_op->unlockfs)
265                         sb->s_op->unlockfs(sb);
266                 sb->s_frozen = SB_UNFROZEN;
267                 smp_wmb();
268                 wake_up(&sb->s_wait_unfrozen);
269                 drop_super(sb);
270         }
271
272         up(&bdev->bd_mount_sem);
273 }
274 EXPORT_SYMBOL(thaw_bdev);
275
276 /*
277  * sync everything.  Start out by waking pdflush, because that writes back
278  * all queues in parallel.
279  */
280 static void do_sync(unsigned long wait)
281 {
282         wakeup_pdflush(0);
283         sync_inodes(0);         /* All mappings, inodes and their blockdevs */
284         DQUOT_SYNC(NULL);
285         sync_supers();          /* Write the superblocks */
286         sync_filesystems(0);    /* Start syncing the filesystems */
287         sync_filesystems(wait); /* Waitingly sync the filesystems */
288         sync_inodes(wait);      /* Mappings, inodes and blockdevs, again. */
289         if (!wait)
290                 printk("Emergency Sync complete\n");
291         if (unlikely(laptop_mode))
292                 laptop_sync_completion();
293 }
294
295 asmlinkage long sys_sync(void)
296 {
297         do_sync(1);
298         return 0;
299 }
300
301 void emergency_sync(void)
302 {
303         pdflush_operation(do_sync, 0);
304 }
305
306 /*
307  * Generic function to fsync a file.
308  *
309  * filp may be NULL if called via the msync of a vma.
310  */
311  
312 int file_fsync(struct file *filp, struct dentry *dentry, int datasync)
313 {
314         struct inode * inode = dentry->d_inode;
315         struct super_block * sb;
316         int ret, err;
317
318         /* sync the inode to buffers */
319         ret = write_inode_now(inode, 0);
320
321         /* sync the superblock to buffers */
322         sb = inode->i_sb;
323         lock_super(sb);
324         if (sb->s_op->write_super)
325                 sb->s_op->write_super(sb);
326         unlock_super(sb);
327
328         /* .. finally sync the buffers to disk */
329         err = sync_blockdev(sb->s_bdev);
330         if (!ret)
331                 ret = err;
332         return ret;
333 }
334
335 static long do_fsync(unsigned int fd, int datasync)
336 {
337         struct file * file;
338         struct address_space *mapping;
339         int ret, err;
340
341         ret = -EBADF;
342         file = fget(fd);
343         if (!file)
344                 goto out;
345
346         ret = -EINVAL;
347         if (!file->f_op || !file->f_op->fsync) {
348                 /* Why?  We can still call filemap_fdatawrite */
349                 goto out_putf;
350         }
351
352         mapping = file->f_mapping;
353
354         current->flags |= PF_SYNCWRITE;
355         ret = filemap_fdatawrite(mapping);
356
357         /*
358          * We need to protect against concurrent writers,
359          * which could cause livelocks in fsync_buffers_list
360          */
361         down(&mapping->host->i_sem);
362         err = file->f_op->fsync(file, file->f_dentry, datasync);
363         if (!ret)
364                 ret = err;
365         up(&mapping->host->i_sem);
366         err = filemap_fdatawait(mapping);
367         if (!ret)
368                 ret = err;
369         current->flags &= ~PF_SYNCWRITE;
370
371 out_putf:
372         fput(file);
373 out:
374         return ret;
375 }
376
377 asmlinkage long sys_fsync(unsigned int fd)
378 {
379         return do_fsync(fd, 0);
380 }
381
382 asmlinkage long sys_fdatasync(unsigned int fd)
383 {
384         return do_fsync(fd, 1);
385 }
386
387 /*
388  * Various filesystems appear to want __find_get_block to be non-blocking.
389  * But it's the page lock which protects the buffers.  To get around this,
390  * we get exclusion from try_to_free_buffers with the blockdev mapping's
391  * private_lock.
392  *
393  * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
394  * may be quite high.  This code could TryLock the page, and if that
395  * succeeds, there is no need to take private_lock. (But if
396  * private_lock is contended then so is mapping->tree_lock).
397  */
398 static struct buffer_head *
399 __find_get_block_slow(struct block_device *bdev, sector_t block, int unused)
400 {
401         struct inode *bd_inode = bdev->bd_inode;
402         struct address_space *bd_mapping = bd_inode->i_mapping;
403         struct buffer_head *ret = NULL;
404         pgoff_t index;
405         struct buffer_head *bh;
406         struct buffer_head *head;
407         struct page *page;
408         int all_mapped = 1;
409
410         index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
411         page = find_get_page(bd_mapping, index);
412         if (!page)
413                 goto out;
414
415         spin_lock(&bd_mapping->private_lock);
416         if (!page_has_buffers(page))
417                 goto out_unlock;
418         head = page_buffers(page);
419         bh = head;
420         do {
421                 if (bh->b_blocknr == block) {
422                         ret = bh;
423                         get_bh(bh);
424                         goto out_unlock;
425                 }
426                 if (!buffer_mapped(bh))
427                         all_mapped = 0;
428                 bh = bh->b_this_page;
429         } while (bh != head);
430
431         /* we might be here because some of the buffers on this page are
432          * not mapped.  This is due to various races between
433          * file io on the block device and getblk.  It gets dealt with
434          * elsewhere, don't buffer_error if we had some unmapped buffers
435          */
436         if (all_mapped) {
437                 printk("__find_get_block_slow() failed. "
438                         "block=%llu, b_blocknr=%llu\n",
439                         (unsigned long long)block, (unsigned long long)bh->b_blocknr);
440                 printk("b_state=0x%08lx, b_size=%u\n", bh->b_state, bh->b_size);
441                 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
442         }
443 out_unlock:
444         spin_unlock(&bd_mapping->private_lock);
445         page_cache_release(page);
446 out:
447         return ret;
448 }
449
450 /* If invalidate_buffers() will trash dirty buffers, it means some kind
451    of fs corruption is going on. Trashing dirty data always imply losing
452    information that was supposed to be just stored on the physical layer
453    by the user.
454
455    Thus invalidate_buffers in general usage is not allwowed to trash
456    dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
457    be preserved.  These buffers are simply skipped.
458   
459    We also skip buffers which are still in use.  For example this can
460    happen if a userspace program is reading the block device.
461
462    NOTE: In the case where the user removed a removable-media-disk even if
463    there's still dirty data not synced on disk (due a bug in the device driver
464    or due an error of the user), by not destroying the dirty buffers we could
465    generate corruption also on the next media inserted, thus a parameter is
466    necessary to handle this case in the most safe way possible (trying
467    to not corrupt also the new disk inserted with the data belonging to
468    the old now corrupted disk). Also for the ramdisk the natural thing
469    to do in order to release the ramdisk memory is to destroy dirty buffers.
470
471    These are two special cases. Normal usage imply the device driver
472    to issue a sync on the device (without waiting I/O completion) and
473    then an invalidate_buffers call that doesn't trash dirty buffers.
474
475    For handling cache coherency with the blkdev pagecache the 'update' case
476    is been introduced. It is needed to re-read from disk any pinned
477    buffer. NOTE: re-reading from disk is destructive so we can do it only
478    when we assume nobody is changing the buffercache under our I/O and when
479    we think the disk contains more recent information than the buffercache.
480    The update == 1 pass marks the buffers we need to update, the update == 2
481    pass does the actual I/O. */
482 void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers)
483 {
484         invalidate_bh_lrus();
485         /*
486          * FIXME: what about destroy_dirty_buffers?
487          * We really want to use invalidate_inode_pages2() for
488          * that, but not until that's cleaned up.
489          */
490         invalidate_inode_pages(bdev->bd_inode->i_mapping);
491 }
492
493 /*
494  * Kick pdflush then try to free up some ZONE_NORMAL memory.
495  */
496 static void free_more_memory(void)
497 {
498         struct zone **zones;
499         pg_data_t *pgdat;
500
501         wakeup_pdflush(1024);
502         yield();
503
504         for_each_pgdat(pgdat) {
505                 zones = pgdat->node_zonelists[gfp_zone(GFP_NOFS)].zones;
506                 if (*zones)
507                         try_to_free_pages(zones, GFP_NOFS);
508         }
509 }
510
511 /*
512  * I/O completion handler for block_read_full_page() - pages
513  * which come unlocked at the end of I/O.
514  */
515 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
516 {
517         unsigned long flags;
518         struct buffer_head *first;
519         struct buffer_head *tmp;
520         struct page *page;
521         int page_uptodate = 1;
522
523         BUG_ON(!buffer_async_read(bh));
524
525         page = bh->b_page;
526         if (uptodate) {
527                 set_buffer_uptodate(bh);
528         } else {
529                 clear_buffer_uptodate(bh);
530                 if (printk_ratelimit())
531                         buffer_io_error(bh);
532                 SetPageError(page);
533         }
534
535         /*
536          * Be _very_ careful from here on. Bad things can happen if
537          * two buffer heads end IO at almost the same time and both
538          * decide that the page is now completely done.
539          */
540         first = page_buffers(page);
541         local_irq_save(flags);
542         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
543         clear_buffer_async_read(bh);
544         unlock_buffer(bh);
545         tmp = bh;
546         do {
547                 if (!buffer_uptodate(tmp))
548                         page_uptodate = 0;
549                 if (buffer_async_read(tmp)) {
550                         BUG_ON(!buffer_locked(tmp));
551                         goto still_busy;
552                 }
553                 tmp = tmp->b_this_page;
554         } while (tmp != bh);
555         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
556         local_irq_restore(flags);
557
558         /*
559          * If none of the buffers had errors and they are all
560          * uptodate then we can set the page uptodate.
561          */
562         if (page_uptodate && !PageError(page))
563                 SetPageUptodate(page);
564         unlock_page(page);
565         return;
566
567 still_busy:
568         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
569         local_irq_restore(flags);
570         return;
571 }
572
573 /*
574  * Completion handler for block_write_full_page() - pages which are unlocked
575  * during I/O, and which have PageWriteback cleared upon I/O completion.
576  */
577 void end_buffer_async_write(struct buffer_head *bh, int uptodate)
578 {
579         char b[BDEVNAME_SIZE];
580         unsigned long flags;
581         struct buffer_head *first;
582         struct buffer_head *tmp;
583         struct page *page;
584
585         BUG_ON(!buffer_async_write(bh));
586
587         page = bh->b_page;
588         if (uptodate) {
589                 set_buffer_uptodate(bh);
590         } else {
591                 if (printk_ratelimit()) {
592                         buffer_io_error(bh);
593                         printk(KERN_WARNING "lost page write due to "
594                                         "I/O error on %s\n",
595                                bdevname(bh->b_bdev, b));
596                 }
597                 set_bit(AS_EIO, &page->mapping->flags);
598                 clear_buffer_uptodate(bh);
599                 SetPageError(page);
600         }
601
602         first = page_buffers(page);
603         local_irq_save(flags);
604         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
605
606         clear_buffer_async_write(bh);
607         unlock_buffer(bh);
608         tmp = bh->b_this_page;
609         while (tmp != bh) {
610                 if (buffer_async_write(tmp)) {
611                         BUG_ON(!buffer_locked(tmp));
612                         goto still_busy;
613                 }
614                 tmp = tmp->b_this_page;
615         }
616         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
617         local_irq_restore(flags);
618         end_page_writeback(page);
619         return;
620
621 still_busy:
622         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
623         local_irq_restore(flags);
624         return;
625 }
626
627 /*
628  * If a page's buffers are under async readin (end_buffer_async_read
629  * completion) then there is a possibility that another thread of
630  * control could lock one of the buffers after it has completed
631  * but while some of the other buffers have not completed.  This
632  * locked buffer would confuse end_buffer_async_read() into not unlocking
633  * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
634  * that this buffer is not under async I/O.
635  *
636  * The page comes unlocked when it has no locked buffer_async buffers
637  * left.
638  *
639  * PageLocked prevents anyone starting new async I/O reads any of
640  * the buffers.
641  *
642  * PageWriteback is used to prevent simultaneous writeout of the same
643  * page.
644  *
645  * PageLocked prevents anyone from starting writeback of a page which is
646  * under read I/O (PageWriteback is only ever set against a locked page).
647  */
648 static void mark_buffer_async_read(struct buffer_head *bh)
649 {
650         bh->b_end_io = end_buffer_async_read;
651         set_buffer_async_read(bh);
652 }
653
654 void mark_buffer_async_write(struct buffer_head *bh)
655 {
656         bh->b_end_io = end_buffer_async_write;
657         set_buffer_async_write(bh);
658 }
659 EXPORT_SYMBOL(mark_buffer_async_write);
660
661
662 /*
663  * fs/buffer.c contains helper functions for buffer-backed address space's
664  * fsync functions.  A common requirement for buffer-based filesystems is
665  * that certain data from the backing blockdev needs to be written out for
666  * a successful fsync().  For example, ext2 indirect blocks need to be
667  * written back and waited upon before fsync() returns.
668  *
669  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
670  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
671  * management of a list of dependent buffers at ->i_mapping->private_list.
672  *
673  * Locking is a little subtle: try_to_free_buffers() will remove buffers
674  * from their controlling inode's queue when they are being freed.  But
675  * try_to_free_buffers() will be operating against the *blockdev* mapping
676  * at the time, not against the S_ISREG file which depends on those buffers.
677  * So the locking for private_list is via the private_lock in the address_space
678  * which backs the buffers.  Which is different from the address_space 
679  * against which the buffers are listed.  So for a particular address_space,
680  * mapping->private_lock does *not* protect mapping->private_list!  In fact,
681  * mapping->private_list will always be protected by the backing blockdev's
682  * ->private_lock.
683  *
684  * Which introduces a requirement: all buffers on an address_space's
685  * ->private_list must be from the same address_space: the blockdev's.
686  *
687  * address_spaces which do not place buffers at ->private_list via these
688  * utility functions are free to use private_lock and private_list for
689  * whatever they want.  The only requirement is that list_empty(private_list)
690  * be true at clear_inode() time.
691  *
692  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
693  * filesystems should do that.  invalidate_inode_buffers() should just go
694  * BUG_ON(!list_empty).
695  *
696  * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
697  * take an address_space, not an inode.  And it should be called
698  * mark_buffer_dirty_fsync() to clearly define why those buffers are being
699  * queued up.
700  *
701  * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
702  * list if it is already on a list.  Because if the buffer is on a list,
703  * it *must* already be on the right one.  If not, the filesystem is being
704  * silly.  This will save a ton of locking.  But first we have to ensure
705  * that buffers are taken *off* the old inode's list when they are freed
706  * (presumably in truncate).  That requires careful auditing of all
707  * filesystems (do it inside bforget()).  It could also be done by bringing
708  * b_inode back.
709  */
710
711 /*
712  * The buffer's backing address_space's private_lock must be held
713  */
714 static inline void __remove_assoc_queue(struct buffer_head *bh)
715 {
716         list_del_init(&bh->b_assoc_buffers);
717 }
718
719 int inode_has_buffers(struct inode *inode)
720 {
721         return !list_empty(&inode->i_data.private_list);
722 }
723
724 /*
725  * osync is designed to support O_SYNC io.  It waits synchronously for
726  * all already-submitted IO to complete, but does not queue any new
727  * writes to the disk.
728  *
729  * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
730  * you dirty the buffers, and then use osync_inode_buffers to wait for
731  * completion.  Any other dirty buffers which are not yet queued for
732  * write will not be flushed to disk by the osync.
733  */
734 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
735 {
736         struct buffer_head *bh;
737         struct list_head *p;
738         int err = 0;
739
740         spin_lock(lock);
741 repeat:
742         list_for_each_prev(p, list) {
743                 bh = BH_ENTRY(p);
744                 if (buffer_locked(bh)) {
745                         get_bh(bh);
746                         spin_unlock(lock);
747                         wait_on_buffer(bh);
748                         if (!buffer_uptodate(bh))
749                                 err = -EIO;
750                         brelse(bh);
751                         spin_lock(lock);
752                         goto repeat;
753                 }
754         }
755         spin_unlock(lock);
756         return err;
757 }
758
759 /**
760  * sync_mapping_buffers - write out and wait upon a mapping's "associated"
761  *                        buffers
762  * @mapping: the mapping which wants those buffers written
763  *
764  * Starts I/O against the buffers at mapping->private_list, and waits upon
765  * that I/O.
766  *
767  * Basically, this is a convenience function for fsync().
768  * @mapping is a file or directory which needs those buffers to be written for
769  * a successful fsync().
770  */
771 int sync_mapping_buffers(struct address_space *mapping)
772 {
773         struct address_space *buffer_mapping = mapping->assoc_mapping;
774
775         if (buffer_mapping == NULL || list_empty(&mapping->private_list))
776                 return 0;
777
778         return fsync_buffers_list(&buffer_mapping->private_lock,
779                                         &mapping->private_list);
780 }
781 EXPORT_SYMBOL(sync_mapping_buffers);
782
783 /*
784  * Called when we've recently written block `bblock', and it is known that
785  * `bblock' was for a buffer_boundary() buffer.  This means that the block at
786  * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
787  * dirty, schedule it for IO.  So that indirects merge nicely with their data.
788  */
789 void write_boundary_block(struct block_device *bdev,
790                         sector_t bblock, unsigned blocksize)
791 {
792         struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
793         if (bh) {
794                 if (buffer_dirty(bh))
795                         ll_rw_block(WRITE, 1, &bh);
796                 put_bh(bh);
797         }
798 }
799
800 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
801 {
802         struct address_space *mapping = inode->i_mapping;
803         struct address_space *buffer_mapping = bh->b_page->mapping;
804
805         mark_buffer_dirty(bh);
806         if (!mapping->assoc_mapping) {
807                 mapping->assoc_mapping = buffer_mapping;
808         } else {
809                 if (mapping->assoc_mapping != buffer_mapping)
810                         BUG();
811         }
812         if (list_empty(&bh->b_assoc_buffers)) {
813                 spin_lock(&buffer_mapping->private_lock);
814                 list_move_tail(&bh->b_assoc_buffers,
815                                 &mapping->private_list);
816                 spin_unlock(&buffer_mapping->private_lock);
817         }
818 }
819 EXPORT_SYMBOL(mark_buffer_dirty_inode);
820
821 /*
822  * Add a page to the dirty page list.
823  *
824  * It is a sad fact of life that this function is called from several places
825  * deeply under spinlocking.  It may not sleep.
826  *
827  * If the page has buffers, the uptodate buffers are set dirty, to preserve
828  * dirty-state coherency between the page and the buffers.  It the page does
829  * not have buffers then when they are later attached they will all be set
830  * dirty.
831  *
832  * The buffers are dirtied before the page is dirtied.  There's a small race
833  * window in which a writepage caller may see the page cleanness but not the
834  * buffer dirtiness.  That's fine.  If this code were to set the page dirty
835  * before the buffers, a concurrent writepage caller could clear the page dirty
836  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
837  * page on the dirty page list.
838  *
839  * We use private_lock to lock against try_to_free_buffers while using the
840  * page's buffer list.  Also use this to protect against clean buffers being
841  * added to the page after it was set dirty.
842  *
843  * FIXME: may need to call ->reservepage here as well.  That's rather up to the
844  * address_space though.
845  */
846 int __set_page_dirty_buffers(struct page *page)
847 {
848         struct address_space * const mapping = page->mapping;
849
850         spin_lock(&mapping->private_lock);
851         if (page_has_buffers(page)) {
852                 struct buffer_head *head = page_buffers(page);
853                 struct buffer_head *bh = head;
854
855                 do {
856                         set_buffer_dirty(bh);
857                         bh = bh->b_this_page;
858                 } while (bh != head);
859         }
860         spin_unlock(&mapping->private_lock);
861
862         if (!TestSetPageDirty(page)) {
863                 write_lock_irq(&mapping->tree_lock);
864                 if (page->mapping) {    /* Race with truncate? */
865                         if (mapping_cap_account_dirty(mapping))
866                                 inc_page_state(nr_dirty);
867                         radix_tree_tag_set(&mapping->page_tree,
868                                                 page_index(page),
869                                                 PAGECACHE_TAG_DIRTY);
870                 }
871                 write_unlock_irq(&mapping->tree_lock);
872                 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
873         }
874         
875         return 0;
876 }
877 EXPORT_SYMBOL(__set_page_dirty_buffers);
878
879 /*
880  * Write out and wait upon a list of buffers.
881  *
882  * We have conflicting pressures: we want to make sure that all
883  * initially dirty buffers get waited on, but that any subsequently
884  * dirtied buffers don't.  After all, we don't want fsync to last
885  * forever if somebody is actively writing to the file.
886  *
887  * Do this in two main stages: first we copy dirty buffers to a
888  * temporary inode list, queueing the writes as we go.  Then we clean
889  * up, waiting for those writes to complete.
890  * 
891  * During this second stage, any subsequent updates to the file may end
892  * up refiling the buffer on the original inode's dirty list again, so
893  * there is a chance we will end up with a buffer queued for write but
894  * not yet completed on that list.  So, as a final cleanup we go through
895  * the osync code to catch these locked, dirty buffers without requeuing
896  * any newly dirty buffers for write.
897  */
898 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
899 {
900         struct buffer_head *bh;
901         struct list_head tmp;
902         int err = 0, err2;
903
904         INIT_LIST_HEAD(&tmp);
905
906         spin_lock(lock);
907         while (!list_empty(list)) {
908                 bh = BH_ENTRY(list->next);
909                 list_del_init(&bh->b_assoc_buffers);
910                 if (buffer_dirty(bh) || buffer_locked(bh)) {
911                         list_add(&bh->b_assoc_buffers, &tmp);
912                         if (buffer_dirty(bh)) {
913                                 get_bh(bh);
914                                 spin_unlock(lock);
915                                 /*
916                                  * Ensure any pending I/O completes so that
917                                  * ll_rw_block() actually writes the current
918                                  * contents - it is a noop if I/O is still in
919                                  * flight on potentially older contents.
920                                  */
921                                 ll_rw_block(SWRITE, 1, &bh);
922                                 brelse(bh);
923                                 spin_lock(lock);
924                         }
925                 }
926         }
927
928         while (!list_empty(&tmp)) {
929                 bh = BH_ENTRY(tmp.prev);
930                 __remove_assoc_queue(bh);
931                 get_bh(bh);
932                 spin_unlock(lock);
933                 wait_on_buffer(bh);
934                 if (!buffer_uptodate(bh))
935                         err = -EIO;
936                 brelse(bh);
937                 spin_lock(lock);
938         }
939         
940         spin_unlock(lock);
941         err2 = osync_buffers_list(lock, list);
942         if (err)
943                 return err;
944         else
945                 return err2;
946 }
947
948 /*
949  * Invalidate any and all dirty buffers on a given inode.  We are
950  * probably unmounting the fs, but that doesn't mean we have already
951  * done a sync().  Just drop the buffers from the inode list.
952  *
953  * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
954  * assumes that all the buffers are against the blockdev.  Not true
955  * for reiserfs.
956  */
957 void invalidate_inode_buffers(struct inode *inode)
958 {
959         if (inode_has_buffers(inode)) {
960                 struct address_space *mapping = &inode->i_data;
961                 struct list_head *list = &mapping->private_list;
962                 struct address_space *buffer_mapping = mapping->assoc_mapping;
963
964                 spin_lock(&buffer_mapping->private_lock);
965                 while (!list_empty(list))
966                         __remove_assoc_queue(BH_ENTRY(list->next));
967                 spin_unlock(&buffer_mapping->private_lock);
968         }
969 }
970
971 /*
972  * Remove any clean buffers from the inode's buffer list.  This is called
973  * when we're trying to free the inode itself.  Those buffers can pin it.
974  *
975  * Returns true if all buffers were removed.
976  */
977 int remove_inode_buffers(struct inode *inode)
978 {
979         int ret = 1;
980
981         if (inode_has_buffers(inode)) {
982                 struct address_space *mapping = &inode->i_data;
983                 struct list_head *list = &mapping->private_list;
984                 struct address_space *buffer_mapping = mapping->assoc_mapping;
985
986                 spin_lock(&buffer_mapping->private_lock);
987                 while (!list_empty(list)) {
988                         struct buffer_head *bh = BH_ENTRY(list->next);
989                         if (buffer_dirty(bh)) {
990                                 ret = 0;
991                                 break;
992                         }
993                         __remove_assoc_queue(bh);
994                 }
995                 spin_unlock(&buffer_mapping->private_lock);
996         }
997         return ret;
998 }
999
1000 /*
1001  * Create the appropriate buffers when given a page for data area and
1002  * the size of each buffer.. Use the bh->b_this_page linked list to
1003  * follow the buffers created.  Return NULL if unable to create more
1004  * buffers.
1005  *
1006  * The retry flag is used to differentiate async IO (paging, swapping)
1007  * which may not fail from ordinary buffer allocations.
1008  */
1009 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
1010                 int retry)
1011 {
1012         struct buffer_head *bh, *head;
1013         long offset;
1014
1015 try_again:
1016         head = NULL;
1017         offset = PAGE_SIZE;
1018         while ((offset -= size) >= 0) {
1019                 bh = alloc_buffer_head(GFP_NOFS);
1020                 if (!bh)
1021                         goto no_grow;
1022
1023                 bh->b_bdev = NULL;
1024                 bh->b_this_page = head;
1025                 bh->b_blocknr = -1;
1026                 head = bh;
1027
1028                 bh->b_state = 0;
1029                 atomic_set(&bh->b_count, 0);
1030                 bh->b_size = size;
1031
1032                 /* Link the buffer to its page */
1033                 set_bh_page(bh, page, offset);
1034
1035                 bh->b_end_io = NULL;
1036         }
1037         return head;
1038 /*
1039  * In case anything failed, we just free everything we got.
1040  */
1041 no_grow:
1042         if (head) {
1043                 do {
1044                         bh = head;
1045                         head = head->b_this_page;
1046                         free_buffer_head(bh);
1047                 } while (head);
1048         }
1049
1050         /*
1051          * Return failure for non-async IO requests.  Async IO requests
1052          * are not allowed to fail, so we have to wait until buffer heads
1053          * become available.  But we don't want tasks sleeping with 
1054          * partially complete buffers, so all were released above.
1055          */
1056         if (!retry)
1057                 return NULL;
1058
1059         /* We're _really_ low on memory. Now we just
1060          * wait for old buffer heads to become free due to
1061          * finishing IO.  Since this is an async request and
1062          * the reserve list is empty, we're sure there are 
1063          * async buffer heads in use.
1064          */
1065         free_more_memory();
1066         goto try_again;
1067 }
1068 EXPORT_SYMBOL_GPL(alloc_page_buffers);
1069
1070 static inline void
1071 link_dev_buffers(struct page *page, struct buffer_head *head)
1072 {
1073         struct buffer_head *bh, *tail;
1074
1075         bh = head;
1076         do {
1077                 tail = bh;
1078                 bh = bh->b_this_page;
1079         } while (bh);
1080         tail->b_this_page = head;
1081         attach_page_buffers(page, head);
1082 }
1083
1084 /*
1085  * Initialise the state of a blockdev page's buffers.
1086  */ 
1087 static void
1088 init_page_buffers(struct page *page, struct block_device *bdev,
1089                         sector_t block, int size)
1090 {
1091         struct buffer_head *head = page_buffers(page);
1092         struct buffer_head *bh = head;
1093         int uptodate = PageUptodate(page);
1094
1095         do {
1096                 if (!buffer_mapped(bh)) {
1097                         init_buffer(bh, NULL, NULL);
1098                         bh->b_bdev = bdev;
1099                         bh->b_blocknr = block;
1100                         if (uptodate)
1101                                 set_buffer_uptodate(bh);
1102                         set_buffer_mapped(bh);
1103                 }
1104                 block++;
1105                 bh = bh->b_this_page;
1106         } while (bh != head);
1107 }
1108
1109 /*
1110  * Create the page-cache page that contains the requested block.
1111  *
1112  * This is user purely for blockdev mappings.
1113  */
1114 static struct page *
1115 grow_dev_page(struct block_device *bdev, sector_t block,
1116                 pgoff_t index, int size)
1117 {
1118         struct inode *inode = bdev->bd_inode;
1119         struct page *page;
1120         struct buffer_head *bh;
1121
1122         page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
1123         if (!page)
1124                 return NULL;
1125
1126         if (!PageLocked(page))
1127                 BUG();
1128
1129         if (page_has_buffers(page)) {
1130                 bh = page_buffers(page);
1131                 if (bh->b_size == size) {
1132                         init_page_buffers(page, bdev, block, size);
1133                         return page;
1134                 }
1135                 if (!try_to_free_buffers(page))
1136                         goto failed;
1137         }
1138
1139         /*
1140          * Allocate some buffers for this page
1141          */
1142         bh = alloc_page_buffers(page, size, 0);
1143         if (!bh)
1144                 goto failed;
1145
1146         /*
1147          * Link the page to the buffers and initialise them.  Take the
1148          * lock to be atomic wrt __find_get_block(), which does not
1149          * run under the page lock.
1150          */
1151         spin_lock(&inode->i_mapping->private_lock);
1152         link_dev_buffers(page, bh);
1153         init_page_buffers(page, bdev, block, size);
1154         spin_unlock(&inode->i_mapping->private_lock);
1155         return page;
1156
1157 failed:
1158         BUG();
1159         unlock_page(page);
1160         page_cache_release(page);
1161         return NULL;
1162 }
1163
1164 /*
1165  * Create buffers for the specified block device block's page.  If
1166  * that page was dirty, the buffers are set dirty also.
1167  *
1168  * Except that's a bug.  Attaching dirty buffers to a dirty
1169  * blockdev's page can result in filesystem corruption, because
1170  * some of those buffers may be aliases of filesystem data.
1171  * grow_dev_page() will go BUG() if this happens.
1172  */
1173 static inline int
1174 grow_buffers(struct block_device *bdev, sector_t block, int size)
1175 {
1176         struct page *page;
1177         pgoff_t index;
1178         int sizebits;
1179
1180         sizebits = -1;
1181         do {
1182                 sizebits++;
1183         } while ((size << sizebits) < PAGE_SIZE);
1184
1185         index = block >> sizebits;
1186         block = index << sizebits;
1187
1188         /* Create a page with the proper size buffers.. */
1189         page = grow_dev_page(bdev, block, index, size);
1190         if (!page)
1191                 return 0;
1192         unlock_page(page);
1193         page_cache_release(page);
1194         return 1;
1195 }
1196
1197 static struct buffer_head *
1198 __getblk_slow(struct block_device *bdev, sector_t block, int size)
1199 {
1200         /* Size must be multiple of hard sectorsize */
1201         if (unlikely(size & (bdev_hardsect_size(bdev)-1) ||
1202                         (size < 512 || size > PAGE_SIZE))) {
1203                 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1204                                         size);
1205                 printk(KERN_ERR "hardsect size: %d\n",
1206                                         bdev_hardsect_size(bdev));
1207
1208                 dump_stack();
1209                 return NULL;
1210         }
1211
1212         for (;;) {
1213                 struct buffer_head * bh;
1214
1215                 bh = __find_get_block(bdev, block, size);
1216                 if (bh)
1217                         return bh;
1218
1219                 if (!grow_buffers(bdev, block, size))
1220                         free_more_memory();
1221         }
1222 }
1223
1224 /*
1225  * The relationship between dirty buffers and dirty pages:
1226  *
1227  * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1228  * the page is tagged dirty in its radix tree.
1229  *
1230  * At all times, the dirtiness of the buffers represents the dirtiness of
1231  * subsections of the page.  If the page has buffers, the page dirty bit is
1232  * merely a hint about the true dirty state.
1233  *
1234  * When a page is set dirty in its entirety, all its buffers are marked dirty
1235  * (if the page has buffers).
1236  *
1237  * When a buffer is marked dirty, its page is dirtied, but the page's other
1238  * buffers are not.
1239  *
1240  * Also.  When blockdev buffers are explicitly read with bread(), they
1241  * individually become uptodate.  But their backing page remains not
1242  * uptodate - even if all of its buffers are uptodate.  A subsequent
1243  * block_read_full_page() against that page will discover all the uptodate
1244  * buffers, will set the page uptodate and will perform no I/O.
1245  */
1246
1247 /**
1248  * mark_buffer_dirty - mark a buffer_head as needing writeout
1249  * @bh: the buffer_head to mark dirty
1250  *
1251  * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1252  * backing page dirty, then tag the page as dirty in its address_space's radix
1253  * tree and then attach the address_space's inode to its superblock's dirty
1254  * inode list.
1255  *
1256  * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1257  * mapping->tree_lock and the global inode_lock.
1258  */
1259 void fastcall mark_buffer_dirty(struct buffer_head *bh)
1260 {
1261         if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh))
1262                 __set_page_dirty_nobuffers(bh->b_page);
1263 }
1264
1265 /*
1266  * Decrement a buffer_head's reference count.  If all buffers against a page
1267  * have zero reference count, are clean and unlocked, and if the page is clean
1268  * and unlocked then try_to_free_buffers() may strip the buffers from the page
1269  * in preparation for freeing it (sometimes, rarely, buffers are removed from
1270  * a page but it ends up not being freed, and buffers may later be reattached).
1271  */
1272 void __brelse(struct buffer_head * buf)
1273 {
1274         if (atomic_read(&buf->b_count)) {
1275                 put_bh(buf);
1276                 return;
1277         }
1278         printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1279         WARN_ON(1);
1280 }
1281
1282 /*
1283  * bforget() is like brelse(), except it discards any
1284  * potentially dirty data.
1285  */
1286 void __bforget(struct buffer_head *bh)
1287 {
1288         clear_buffer_dirty(bh);
1289         if (!list_empty(&bh->b_assoc_buffers)) {
1290                 struct address_space *buffer_mapping = bh->b_page->mapping;
1291
1292                 spin_lock(&buffer_mapping->private_lock);
1293                 list_del_init(&bh->b_assoc_buffers);
1294                 spin_unlock(&buffer_mapping->private_lock);
1295         }
1296         __brelse(bh);
1297 }
1298
1299 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1300 {
1301         lock_buffer(bh);
1302         if (buffer_uptodate(bh)) {
1303                 unlock_buffer(bh);
1304                 return bh;
1305         } else {
1306                 get_bh(bh);
1307                 bh->b_end_io = end_buffer_read_sync;
1308                 submit_bh(READ, bh);
1309                 wait_on_buffer(bh);
1310                 if (buffer_uptodate(bh))
1311                         return bh;
1312         }
1313         brelse(bh);
1314         return NULL;
1315 }
1316
1317 /*
1318  * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
1319  * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
1320  * refcount elevated by one when they're in an LRU.  A buffer can only appear
1321  * once in a particular CPU's LRU.  A single buffer can be present in multiple
1322  * CPU's LRUs at the same time.
1323  *
1324  * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1325  * sb_find_get_block().
1326  *
1327  * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
1328  * a local interrupt disable for that.
1329  */
1330
1331 #define BH_LRU_SIZE     8
1332
1333 struct bh_lru {
1334         struct buffer_head *bhs[BH_LRU_SIZE];
1335 };
1336
1337 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1338
1339 #ifdef CONFIG_SMP
1340 #define bh_lru_lock()   local_irq_disable()
1341 #define bh_lru_unlock() local_irq_enable()
1342 #else
1343 #define bh_lru_lock()   preempt_disable()
1344 #define bh_lru_unlock() preempt_enable()
1345 #endif
1346
1347 static inline void check_irqs_on(void)
1348 {
1349 #ifdef irqs_disabled
1350         BUG_ON(irqs_disabled());
1351 #endif
1352 }
1353
1354 /*
1355  * The LRU management algorithm is dopey-but-simple.  Sorry.
1356  */
1357 static void bh_lru_install(struct buffer_head *bh)
1358 {
1359         struct buffer_head *evictee = NULL;
1360         struct bh_lru *lru;
1361
1362         check_irqs_on();
1363         bh_lru_lock();
1364         lru = &__get_cpu_var(bh_lrus);
1365         if (lru->bhs[0] != bh) {
1366                 struct buffer_head *bhs[BH_LRU_SIZE];
1367                 int in;
1368                 int out = 0;
1369
1370                 get_bh(bh);
1371                 bhs[out++] = bh;
1372                 for (in = 0; in < BH_LRU_SIZE; in++) {
1373                         struct buffer_head *bh2 = lru->bhs[in];
1374
1375                         if (bh2 == bh) {
1376                                 __brelse(bh2);
1377                         } else {
1378                                 if (out >= BH_LRU_SIZE) {
1379                                         BUG_ON(evictee != NULL);
1380                                         evictee = bh2;
1381                                 } else {
1382                                         bhs[out++] = bh2;
1383                                 }
1384                         }
1385                 }
1386                 while (out < BH_LRU_SIZE)
1387                         bhs[out++] = NULL;
1388                 memcpy(lru->bhs, bhs, sizeof(bhs));
1389         }
1390         bh_lru_unlock();
1391
1392         if (evictee)
1393                 __brelse(evictee);
1394 }
1395
1396 /*
1397  * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
1398  */
1399 static inline struct buffer_head *
1400 lookup_bh_lru(struct block_device *bdev, sector_t block, int size)
1401 {
1402         struct buffer_head *ret = NULL;
1403         struct bh_lru *lru;
1404         int i;
1405
1406         check_irqs_on();
1407         bh_lru_lock();
1408         lru = &__get_cpu_var(bh_lrus);
1409         for (i = 0; i < BH_LRU_SIZE; i++) {
1410                 struct buffer_head *bh = lru->bhs[i];
1411
1412                 if (bh && bh->b_bdev == bdev &&
1413                                 bh->b_blocknr == block && bh->b_size == size) {
1414                         if (i) {
1415                                 while (i) {
1416                                         lru->bhs[i] = lru->bhs[i - 1];
1417                                         i--;
1418                                 }
1419                                 lru->bhs[0] = bh;
1420                         }
1421                         get_bh(bh);
1422                         ret = bh;
1423                         break;
1424                 }
1425         }
1426         bh_lru_unlock();
1427         return ret;
1428 }
1429
1430 /*
1431  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
1432  * it in the LRU and mark it as accessed.  If it is not present then return
1433  * NULL
1434  */
1435 struct buffer_head *
1436 __find_get_block(struct block_device *bdev, sector_t block, int size)
1437 {
1438         struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1439
1440         if (bh == NULL) {
1441                 bh = __find_get_block_slow(bdev, block, size);
1442                 if (bh)
1443                         bh_lru_install(bh);
1444         }
1445         if (bh)
1446                 touch_buffer(bh);
1447         return bh;
1448 }
1449 EXPORT_SYMBOL(__find_get_block);
1450
1451 /*
1452  * __getblk will locate (and, if necessary, create) the buffer_head
1453  * which corresponds to the passed block_device, block and size. The
1454  * returned buffer has its reference count incremented.
1455  *
1456  * __getblk() cannot fail - it just keeps trying.  If you pass it an
1457  * illegal block number, __getblk() will happily return a buffer_head
1458  * which represents the non-existent block.  Very weird.
1459  *
1460  * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1461  * attempt is failing.  FIXME, perhaps?
1462  */
1463 struct buffer_head *
1464 __getblk(struct block_device *bdev, sector_t block, int size)
1465 {
1466         struct buffer_head *bh = __find_get_block(bdev, block, size);
1467
1468         might_sleep();
1469         if (bh == NULL)
1470                 bh = __getblk_slow(bdev, block, size);
1471         return bh;
1472 }
1473 EXPORT_SYMBOL(__getblk);
1474
1475 /*
1476  * Do async read-ahead on a buffer..
1477  */
1478 void __breadahead(struct block_device *bdev, sector_t block, int size)
1479 {
1480         struct buffer_head *bh = __getblk(bdev, block, size);
1481         if (likely(bh)) {
1482                 ll_rw_block(READA, 1, &bh);
1483                 brelse(bh);
1484         }
1485 }
1486 EXPORT_SYMBOL(__breadahead);
1487
1488 /**
1489  *  __bread() - reads a specified block and returns the bh
1490  *  @bdev: the block_device to read from
1491  *  @block: number of block
1492  *  @size: size (in bytes) to read
1493  * 
1494  *  Reads a specified block, and returns buffer head that contains it.
1495  *  It returns NULL if the block was unreadable.
1496  */
1497 struct buffer_head *
1498 __bread(struct block_device *bdev, sector_t block, int size)
1499 {
1500         struct buffer_head *bh = __getblk(bdev, block, size);
1501
1502         if (likely(bh) && !buffer_uptodate(bh))
1503                 bh = __bread_slow(bh);
1504         return bh;
1505 }
1506 EXPORT_SYMBOL(__bread);
1507
1508 /*
1509  * invalidate_bh_lrus() is called rarely - but not only at unmount.
1510  * This doesn't race because it runs in each cpu either in irq
1511  * or with preempt disabled.
1512  */
1513 static void invalidate_bh_lru(void *arg)
1514 {
1515         struct bh_lru *b = &get_cpu_var(bh_lrus);
1516         int i;
1517
1518         for (i = 0; i < BH_LRU_SIZE; i++) {
1519                 brelse(b->bhs[i]);
1520                 b->bhs[i] = NULL;
1521         }
1522         put_cpu_var(bh_lrus);
1523 }
1524         
1525 static void invalidate_bh_lrus(void)
1526 {
1527         on_each_cpu(invalidate_bh_lru, NULL, 1, 1);
1528 }
1529
1530 void set_bh_page(struct buffer_head *bh,
1531                 struct page *page, unsigned long offset)
1532 {
1533         bh->b_page = page;
1534         if (offset >= PAGE_SIZE)
1535                 BUG();
1536         if (PageHighMem(page))
1537                 /*
1538                  * This catches illegal uses and preserves the offset:
1539                  */
1540                 bh->b_data = (char *)(0 + offset);
1541         else
1542                 bh->b_data = page_address(page) + offset;
1543 }
1544 EXPORT_SYMBOL(set_bh_page);
1545
1546 /*
1547  * Called when truncating a buffer on a page completely.
1548  */
1549 static inline void discard_buffer(struct buffer_head * bh)
1550 {
1551         lock_buffer(bh);
1552         clear_buffer_dirty(bh);
1553         bh->b_bdev = NULL;
1554         clear_buffer_mapped(bh);
1555         clear_buffer_req(bh);
1556         clear_buffer_new(bh);
1557         clear_buffer_delay(bh);
1558         unlock_buffer(bh);
1559 }
1560
1561 /**
1562  * try_to_release_page() - release old fs-specific metadata on a page
1563  *
1564  * @page: the page which the kernel is trying to free
1565  * @gfp_mask: memory allocation flags (and I/O mode)
1566  *
1567  * The address_space is to try to release any data against the page
1568  * (presumably at page->private).  If the release was successful, return `1'.
1569  * Otherwise return zero.
1570  *
1571  * The @gfp_mask argument specifies whether I/O may be performed to release
1572  * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
1573  *
1574  * NOTE: @gfp_mask may go away, and this function may become non-blocking.
1575  */
1576 int try_to_release_page(struct page *page, gfp_t gfp_mask)
1577 {
1578         struct address_space * const mapping = page->mapping;
1579
1580         BUG_ON(!PageLocked(page));
1581         if (PageWriteback(page))
1582                 return 0;
1583         
1584         if (mapping && mapping->a_ops->releasepage)
1585                 return mapping->a_ops->releasepage(page, gfp_mask);
1586         return try_to_free_buffers(page);
1587 }
1588 EXPORT_SYMBOL(try_to_release_page);
1589
1590 /**
1591  * block_invalidatepage - invalidate part of all of a buffer-backed page
1592  *
1593  * @page: the page which is affected
1594  * @offset: the index of the truncation point
1595  *
1596  * block_invalidatepage() is called when all or part of the page has become
1597  * invalidatedby a truncate operation.
1598  *
1599  * block_invalidatepage() does not have to release all buffers, but it must
1600  * ensure that no dirty buffer is left outside @offset and that no I/O
1601  * is underway against any of the blocks which are outside the truncation
1602  * point.  Because the caller is about to free (and possibly reuse) those
1603  * blocks on-disk.
1604  */
1605 int block_invalidatepage(struct page *page, unsigned long offset)
1606 {
1607         struct buffer_head *head, *bh, *next;
1608         unsigned int curr_off = 0;
1609         int ret = 1;
1610
1611         BUG_ON(!PageLocked(page));
1612         if (!page_has_buffers(page))
1613                 goto out;
1614
1615         head = page_buffers(page);
1616         bh = head;
1617         do {
1618                 unsigned int next_off = curr_off + bh->b_size;
1619                 next = bh->b_this_page;
1620
1621                 /*
1622                  * is this block fully invalidated?
1623                  */
1624                 if (offset <= curr_off)
1625                         discard_buffer(bh);
1626                 curr_off = next_off;
1627                 bh = next;
1628         } while (bh != head);
1629
1630         /*
1631          * We release buffers only if the entire page is being invalidated.
1632          * The get_block cached value has been unconditionally invalidated,
1633          * so real IO is not possible anymore.
1634          */
1635         if (offset == 0)
1636                 ret = try_to_release_page(page, 0);
1637 out:
1638         return ret;
1639 }
1640 EXPORT_SYMBOL(block_invalidatepage);
1641
1642 int do_invalidatepage(struct page *page, unsigned long offset)
1643 {
1644         int (*invalidatepage)(struct page *, unsigned long);
1645         invalidatepage = page->mapping->a_ops->invalidatepage;
1646         if (invalidatepage == NULL)
1647                 invalidatepage = block_invalidatepage;
1648         return (*invalidatepage)(page, offset);
1649 }
1650
1651 /*
1652  * We attach and possibly dirty the buffers atomically wrt
1653  * __set_page_dirty_buffers() via private_lock.  try_to_free_buffers
1654  * is already excluded via the page lock.
1655  */
1656 void create_empty_buffers(struct page *page,
1657                         unsigned long blocksize, unsigned long b_state)
1658 {
1659         struct buffer_head *bh, *head, *tail;
1660
1661         head = alloc_page_buffers(page, blocksize, 1);
1662         bh = head;
1663         do {
1664                 bh->b_state |= b_state;
1665                 tail = bh;
1666                 bh = bh->b_this_page;
1667         } while (bh);
1668         tail->b_this_page = head;
1669
1670         spin_lock(&page->mapping->private_lock);
1671         if (PageUptodate(page) || PageDirty(page)) {
1672                 bh = head;
1673                 do {
1674                         if (PageDirty(page))
1675                                 set_buffer_dirty(bh);
1676                         if (PageUptodate(page))
1677                                 set_buffer_uptodate(bh);
1678                         bh = bh->b_this_page;
1679                 } while (bh != head);
1680         }
1681         attach_page_buffers(page, head);
1682         spin_unlock(&page->mapping->private_lock);
1683 }
1684 EXPORT_SYMBOL(create_empty_buffers);
1685
1686 /*
1687  * We are taking a block for data and we don't want any output from any
1688  * buffer-cache aliases starting from return from that function and
1689  * until the moment when something will explicitly mark the buffer
1690  * dirty (hopefully that will not happen until we will free that block ;-)
1691  * We don't even need to mark it not-uptodate - nobody can expect
1692  * anything from a newly allocated buffer anyway. We used to used
1693  * unmap_buffer() for such invalidation, but that was wrong. We definitely
1694  * don't want to mark the alias unmapped, for example - it would confuse
1695  * anyone who might pick it with bread() afterwards...
1696  *
1697  * Also..  Note that bforget() doesn't lock the buffer.  So there can
1698  * be writeout I/O going on against recently-freed buffers.  We don't
1699  * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1700  * only if we really need to.  That happens here.
1701  */
1702 void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1703 {
1704         struct buffer_head *old_bh;
1705
1706         might_sleep();
1707
1708         old_bh = __find_get_block_slow(bdev, block, 0);
1709         if (old_bh) {
1710                 clear_buffer_dirty(old_bh);
1711                 wait_on_buffer(old_bh);
1712                 clear_buffer_req(old_bh);
1713                 __brelse(old_bh);
1714         }
1715 }
1716 EXPORT_SYMBOL(unmap_underlying_metadata);
1717
1718 /*
1719  * NOTE! All mapped/uptodate combinations are valid:
1720  *
1721  *      Mapped  Uptodate        Meaning
1722  *
1723  *      No      No              "unknown" - must do get_block()
1724  *      No      Yes             "hole" - zero-filled
1725  *      Yes     No              "allocated" - allocated on disk, not read in
1726  *      Yes     Yes             "valid" - allocated and up-to-date in memory.
1727  *
1728  * "Dirty" is valid only with the last case (mapped+uptodate).
1729  */
1730
1731 /*
1732  * While block_write_full_page is writing back the dirty buffers under
1733  * the page lock, whoever dirtied the buffers may decide to clean them
1734  * again at any time.  We handle that by only looking at the buffer
1735  * state inside lock_buffer().
1736  *
1737  * If block_write_full_page() is called for regular writeback
1738  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1739  * locked buffer.   This only can happen if someone has written the buffer
1740  * directly, with submit_bh().  At the address_space level PageWriteback
1741  * prevents this contention from occurring.
1742  */
1743 static int __block_write_full_page(struct inode *inode, struct page *page,
1744                         get_block_t *get_block, struct writeback_control *wbc)
1745 {
1746         int err;
1747         sector_t block;
1748         sector_t last_block;
1749         struct buffer_head *bh, *head;
1750         int nr_underway = 0;
1751
1752         BUG_ON(!PageLocked(page));
1753
1754         last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1755
1756         if (!page_has_buffers(page)) {
1757                 create_empty_buffers(page, 1 << inode->i_blkbits,
1758                                         (1 << BH_Dirty)|(1 << BH_Uptodate));
1759         }
1760
1761         /*
1762          * Be very careful.  We have no exclusion from __set_page_dirty_buffers
1763          * here, and the (potentially unmapped) buffers may become dirty at
1764          * any time.  If a buffer becomes dirty here after we've inspected it
1765          * then we just miss that fact, and the page stays dirty.
1766          *
1767          * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1768          * handle that here by just cleaning them.
1769          */
1770
1771         block = page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1772         head = page_buffers(page);
1773         bh = head;
1774
1775         /*
1776          * Get all the dirty buffers mapped to disk addresses and
1777          * handle any aliases from the underlying blockdev's mapping.
1778          */
1779         do {
1780                 if (block > last_block) {
1781                         /*
1782                          * mapped buffers outside i_size will occur, because
1783                          * this page can be outside i_size when there is a
1784                          * truncate in progress.
1785                          */
1786                         /*
1787                          * The buffer was zeroed by block_write_full_page()
1788                          */
1789                         clear_buffer_dirty(bh);
1790                         set_buffer_uptodate(bh);
1791                 } else if (!buffer_mapped(bh) && buffer_dirty(bh)) {
1792                         err = get_block(inode, block, bh, 1);
1793                         if (err)
1794                                 goto recover;
1795                         if (buffer_new(bh)) {
1796                                 /* blockdev mappings never come here */
1797                                 clear_buffer_new(bh);
1798                                 unmap_underlying_metadata(bh->b_bdev,
1799                                                         bh->b_blocknr);
1800                         }
1801                 }
1802                 bh = bh->b_this_page;
1803                 block++;
1804         } while (bh != head);
1805
1806         do {
1807                 if (!buffer_mapped(bh))
1808                         continue;
1809                 /*
1810                  * If it's a fully non-blocking write attempt and we cannot
1811                  * lock the buffer then redirty the page.  Note that this can
1812                  * potentially cause a busy-wait loop from pdflush and kswapd
1813                  * activity, but those code paths have their own higher-level
1814                  * throttling.
1815                  */
1816                 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1817                         lock_buffer(bh);
1818                 } else if (test_set_buffer_locked(bh)) {
1819                         redirty_page_for_writepage(wbc, page);
1820                         continue;
1821                 }
1822                 if (test_clear_buffer_dirty(bh)) {
1823                         mark_buffer_async_write(bh);
1824                 } else {
1825                         unlock_buffer(bh);
1826                 }
1827         } while ((bh = bh->b_this_page) != head);
1828
1829         /*
1830          * The page and its buffers are protected by PageWriteback(), so we can
1831          * drop the bh refcounts early.
1832          */
1833         BUG_ON(PageWriteback(page));
1834         set_page_writeback(page);
1835
1836         do {
1837                 struct buffer_head *next = bh->b_this_page;
1838                 if (buffer_async_write(bh)) {
1839                         submit_bh(WRITE, bh);
1840                         nr_underway++;
1841                 }
1842                 bh = next;
1843         } while (bh != head);
1844         unlock_page(page);
1845
1846         err = 0;
1847 done:
1848         if (nr_underway == 0) {
1849                 /*
1850                  * The page was marked dirty, but the buffers were
1851                  * clean.  Someone wrote them back by hand with
1852                  * ll_rw_block/submit_bh.  A rare case.
1853                  */
1854                 int uptodate = 1;
1855                 do {
1856                         if (!buffer_uptodate(bh)) {
1857                                 uptodate = 0;
1858                                 break;
1859                         }
1860                         bh = bh->b_this_page;
1861                 } while (bh != head);
1862                 if (uptodate)
1863                         SetPageUptodate(page);
1864                 end_page_writeback(page);
1865                 /*
1866                  * The page and buffer_heads can be released at any time from
1867                  * here on.
1868                  */
1869                 wbc->pages_skipped++;   /* We didn't write this page */
1870         }
1871         return err;
1872
1873 recover:
1874         /*
1875          * ENOSPC, or some other error.  We may already have added some
1876          * blocks to the file, so we need to write these out to avoid
1877          * exposing stale data.
1878          * The page is currently locked and not marked for writeback
1879          */
1880         bh = head;
1881         /* Recovery: lock and submit the mapped buffers */
1882         do {
1883                 if (buffer_mapped(bh) && buffer_dirty(bh)) {
1884                         lock_buffer(bh);
1885                         mark_buffer_async_write(bh);
1886                 } else {
1887                         /*
1888                          * The buffer may have been set dirty during
1889                          * attachment to a dirty page.
1890                          */
1891                         clear_buffer_dirty(bh);
1892                 }
1893         } while ((bh = bh->b_this_page) != head);
1894         SetPageError(page);
1895         BUG_ON(PageWriteback(page));
1896         set_page_writeback(page);
1897         unlock_page(page);
1898         do {
1899                 struct buffer_head *next = bh->b_this_page;
1900                 if (buffer_async_write(bh)) {
1901                         clear_buffer_dirty(bh);
1902                         submit_bh(WRITE, bh);
1903                         nr_underway++;
1904                 }
1905                 bh = next;
1906         } while (bh != head);
1907         goto done;
1908 }
1909
1910 static int __block_prepare_write(struct inode *inode, struct page *page,
1911                 unsigned from, unsigned to, get_block_t *get_block)
1912 {
1913         unsigned block_start, block_end;
1914         sector_t block;
1915         int err = 0;
1916         unsigned blocksize, bbits;
1917         struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1918
1919         BUG_ON(!PageLocked(page));
1920         BUG_ON(from > PAGE_CACHE_SIZE);
1921         BUG_ON(to > PAGE_CACHE_SIZE);
1922         BUG_ON(from > to);
1923
1924         blocksize = 1 << inode->i_blkbits;
1925         if (!page_has_buffers(page))
1926                 create_empty_buffers(page, blocksize, 0);
1927         head = page_buffers(page);
1928
1929         bbits = inode->i_blkbits;
1930         block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1931
1932         for(bh = head, block_start = 0; bh != head || !block_start;
1933             block++, block_start=block_end, bh = bh->b_this_page) {
1934                 block_end = block_start + blocksize;
1935                 if (block_end <= from || block_start >= to) {
1936                         if (PageUptodate(page)) {
1937                                 if (!buffer_uptodate(bh))
1938                                         set_buffer_uptodate(bh);
1939                         }
1940                         continue;
1941                 }
1942                 if (buffer_new(bh))
1943                         clear_buffer_new(bh);
1944                 if (!buffer_mapped(bh)) {
1945                         err = get_block(inode, block, bh, 1);
1946                         if (err)
1947                                 break;
1948                         if (buffer_new(bh)) {
1949                                 unmap_underlying_metadata(bh->b_bdev,
1950                                                         bh->b_blocknr);
1951                                 if (PageUptodate(page)) {
1952                                         set_buffer_uptodate(bh);
1953                                         continue;
1954                                 }
1955                                 if (block_end > to || block_start < from) {
1956                                         void *kaddr;
1957
1958                                         kaddr = kmap_atomic(page, KM_USER0);
1959                                         if (block_end > to)
1960                                                 memset(kaddr+to, 0,
1961                                                         block_end-to);
1962                                         if (block_start < from)
1963                                                 memset(kaddr+block_start,
1964                                                         0, from-block_start);
1965                                         flush_dcache_page(page);
1966                                         kunmap_atomic(kaddr, KM_USER0);
1967                                 }
1968                                 continue;
1969                         }
1970                 }
1971                 if (PageUptodate(page)) {
1972                         if (!buffer_uptodate(bh))
1973                                 set_buffer_uptodate(bh);
1974                         continue; 
1975                 }
1976                 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1977                      (block_start < from || block_end > to)) {
1978                         ll_rw_block(READ, 1, &bh);
1979                         *wait_bh++=bh;
1980                 }
1981         }
1982         /*
1983          * If we issued read requests - let them complete.
1984          */
1985         while(wait_bh > wait) {
1986                 wait_on_buffer(*--wait_bh);
1987                 if (!buffer_uptodate(*wait_bh))
1988                         err = -EIO;
1989         }
1990         if (!err) {
1991                 bh = head;
1992                 do {
1993                         if (buffer_new(bh))
1994                                 clear_buffer_new(bh);
1995                 } while ((bh = bh->b_this_page) != head);
1996                 return 0;
1997         }
1998         /* Error case: */
1999         /*
2000          * Zero out any newly allocated blocks to avoid exposing stale
2001          * data.  If BH_New is set, we know that the block was newly
2002          * allocated in the above loop.
2003          */
2004         bh = head;
2005         block_start = 0;
2006         do {
2007                 block_end = block_start+blocksize;
2008                 if (block_end <= from)
2009                         goto next_bh;
2010                 if (block_start >= to)
2011                         break;
2012                 if (buffer_new(bh)) {
2013                         void *kaddr;
2014
2015                         clear_buffer_new(bh);
2016                         kaddr = kmap_atomic(page, KM_USER0);
2017                         memset(kaddr+block_start, 0, bh->b_size);
2018                         kunmap_atomic(kaddr, KM_USER0);
2019                         set_buffer_uptodate(bh);
2020                         mark_buffer_dirty(bh);
2021                 }
2022 next_bh:
2023                 block_start = block_end;
2024                 bh = bh->b_this_page;
2025         } while (bh != head);
2026         return err;
2027 }
2028
2029 static int __block_commit_write(struct inode *inode, struct page *page,
2030                 unsigned from, unsigned to)
2031 {
2032         unsigned block_start, block_end;
2033         int partial = 0;
2034         unsigned blocksize;
2035         struct buffer_head *bh, *head;
2036
2037         blocksize = 1 << inode->i_blkbits;
2038
2039         for(bh = head = page_buffers(page), block_start = 0;
2040             bh != head || !block_start;
2041             block_start=block_end, bh = bh->b_this_page) {
2042                 block_end = block_start + blocksize;
2043                 if (block_end <= from || block_start >= to) {
2044                         if (!buffer_uptodate(bh))
2045                                 partial = 1;
2046                 } else {
2047                         set_buffer_uptodate(bh);
2048                         mark_buffer_dirty(bh);
2049                 }
2050         }
2051
2052         /*
2053          * If this is a partial write which happened to make all buffers
2054          * uptodate then we can optimize away a bogus readpage() for
2055          * the next read(). Here we 'discover' whether the page went
2056          * uptodate as a result of this (potentially partial) write.
2057          */
2058         if (!partial)
2059                 SetPageUptodate(page);
2060         return 0;
2061 }
2062
2063 /*
2064  * Generic "read page" function for block devices that have the normal
2065  * get_block functionality. This is most of the block device filesystems.
2066  * Reads the page asynchronously --- the unlock_buffer() and
2067  * set/clear_buffer_uptodate() functions propagate buffer state into the
2068  * page struct once IO has completed.
2069  */
2070 int block_read_full_page(struct page *page, get_block_t *get_block)
2071 {
2072         struct inode *inode = page->mapping->host;
2073         sector_t iblock, lblock;
2074         struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2075         unsigned int blocksize;
2076         int nr, i;
2077         int fully_mapped = 1;
2078
2079         BUG_ON(!PageLocked(page));
2080         blocksize = 1 << inode->i_blkbits;
2081         if (!page_has_buffers(page))
2082                 create_empty_buffers(page, blocksize, 0);
2083         head = page_buffers(page);
2084
2085         iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2086         lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2087         bh = head;
2088         nr = 0;
2089         i = 0;
2090
2091         do {
2092                 if (buffer_uptodate(bh))
2093                         continue;
2094
2095                 if (!buffer_mapped(bh)) {
2096                         int err = 0;
2097
2098                         fully_mapped = 0;
2099                         if (iblock < lblock) {
2100                                 err = get_block(inode, iblock, bh, 0);
2101                                 if (err)
2102                                         SetPageError(page);
2103                         }
2104                         if (!buffer_mapped(bh)) {
2105                                 void *kaddr = kmap_atomic(page, KM_USER0);
2106                                 memset(kaddr + i * blocksize, 0, blocksize);
2107                                 flush_dcache_page(page);
2108                                 kunmap_atomic(kaddr, KM_USER0);
2109                                 if (!err)
2110                                         set_buffer_uptodate(bh);
2111                                 continue;
2112                         }
2113                         /*
2114                          * get_block() might have updated the buffer
2115                          * synchronously
2116                          */
2117                         if (buffer_uptodate(bh))
2118                                 continue;
2119                 }
2120                 arr[nr++] = bh;
2121         } while (i++, iblock++, (bh = bh->b_this_page) != head);
2122
2123         if (fully_mapped)
2124                 SetPageMappedToDisk(page);
2125
2126         if (!nr) {
2127                 /*
2128                  * All buffers are uptodate - we can set the page uptodate
2129                  * as well. But not if get_block() returned an error.
2130                  */
2131                 if (!PageError(page))
2132                         SetPageUptodate(page);
2133                 unlock_page(page);
2134                 return 0;
2135         }
2136
2137         /* Stage two: lock the buffers */
2138         for (i = 0; i < nr; i++) {
2139                 bh = arr[i];
2140                 lock_buffer(bh);
2141                 mark_buffer_async_read(bh);
2142         }
2143
2144         /*
2145          * Stage 3: start the IO.  Check for uptodateness
2146          * inside the buffer lock in case another process reading
2147          * the underlying blockdev brought it uptodate (the sct fix).
2148          */
2149         for (i = 0; i < nr; i++) {
2150                 bh = arr[i];
2151                 if (buffer_uptodate(bh))
2152                         end_buffer_async_read(bh, 1);
2153                 else
2154                         submit_bh(READ, bh);
2155         }
2156         return 0;
2157 }
2158
2159 /* utility function for filesystems that need to do work on expanding
2160  * truncates.  Uses prepare/commit_write to allow the filesystem to
2161  * deal with the hole.  
2162  */
2163 int generic_cont_expand(struct inode *inode, loff_t size)
2164 {
2165         struct address_space *mapping = inode->i_mapping;
2166         struct page *page;
2167         unsigned long index, offset, limit;
2168         int err;
2169
2170         err = -EFBIG;
2171         limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
2172         if (limit != RLIM_INFINITY && size > (loff_t)limit) {
2173                 send_sig(SIGXFSZ, current, 0);
2174                 goto out;
2175         }
2176         if (size > inode->i_sb->s_maxbytes)
2177                 goto out;
2178
2179         offset = (size & (PAGE_CACHE_SIZE-1)); /* Within page */
2180
2181         /* ugh.  in prepare/commit_write, if from==to==start of block, we 
2182         ** skip the prepare.  make sure we never send an offset for the start
2183         ** of a block
2184         */
2185         if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
2186                 offset++;
2187         }
2188         index = size >> PAGE_CACHE_SHIFT;
2189         err = -ENOMEM;
2190         page = grab_cache_page(mapping, index);
2191         if (!page)
2192                 goto out;
2193         err = mapping->a_ops->prepare_write(NULL, page, offset, offset);
2194         if (!err) {
2195                 err = mapping->a_ops->commit_write(NULL, page, offset, offset);
2196         }
2197         unlock_page(page);
2198         page_cache_release(page);
2199         if (err > 0)
2200                 err = 0;
2201 out:
2202         return err;
2203 }
2204
2205 /*
2206  * For moronic filesystems that do not allow holes in file.
2207  * We may have to extend the file.
2208  */
2209
2210 int cont_prepare_write(struct page *page, unsigned offset,
2211                 unsigned to, get_block_t *get_block, loff_t *bytes)
2212 {
2213         struct address_space *mapping = page->mapping;
2214         struct inode *inode = mapping->host;
2215         struct page *new_page;
2216         pgoff_t pgpos;
2217         long status;
2218         unsigned zerofrom;
2219         unsigned blocksize = 1 << inode->i_blkbits;
2220         void *kaddr;
2221
2222         while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) {
2223                 status = -ENOMEM;
2224                 new_page = grab_cache_page(mapping, pgpos);
2225                 if (!new_page)
2226                         goto out;
2227                 /* we might sleep */
2228                 if (*bytes>>PAGE_CACHE_SHIFT != pgpos) {
2229                         unlock_page(new_page);
2230                         page_cache_release(new_page);
2231                         continue;
2232                 }
2233                 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2234                 if (zerofrom & (blocksize-1)) {
2235                         *bytes |= (blocksize-1);
2236                         (*bytes)++;
2237                 }
2238                 status = __block_prepare_write(inode, new_page, zerofrom,
2239                                                 PAGE_CACHE_SIZE, get_block);
2240                 if (status)
2241                         goto out_unmap;
2242                 kaddr = kmap_atomic(new_page, KM_USER0);
2243                 memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom);
2244                 flush_dcache_page(new_page);
2245                 kunmap_atomic(kaddr, KM_USER0);
2246                 generic_commit_write(NULL, new_page, zerofrom, PAGE_CACHE_SIZE);
2247                 unlock_page(new_page);
2248                 page_cache_release(new_page);
2249         }
2250
2251         if (page->index < pgpos) {
2252                 /* completely inside the area */
2253                 zerofrom = offset;
2254         } else {
2255                 /* page covers the boundary, find the boundary offset */
2256                 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2257
2258                 /* if we will expand the thing last block will be filled */
2259                 if (to > zerofrom && (zerofrom & (blocksize-1))) {
2260                         *bytes |= (blocksize-1);
2261                         (*bytes)++;
2262                 }
2263
2264                 /* starting below the boundary? Nothing to zero out */
2265                 if (offset <= zerofrom)
2266                         zerofrom = offset;
2267         }
2268         status = __block_prepare_write(inode, page, zerofrom, to, get_block);
2269         if (status)
2270                 goto out1;
2271         if (zerofrom < offset) {
2272                 kaddr = kmap_atomic(page, KM_USER0);
2273                 memset(kaddr+zerofrom, 0, offset-zerofrom);
2274                 flush_dcache_page(page);
2275                 kunmap_atomic(kaddr, KM_USER0);
2276                 __block_commit_write(inode, page, zerofrom, offset);
2277         }
2278         return 0;
2279 out1:
2280         ClearPageUptodate(page);
2281         return status;
2282
2283 out_unmap:
2284         ClearPageUptodate(new_page);
2285         unlock_page(new_page);
2286         page_cache_release(new_page);
2287 out:
2288         return status;
2289 }
2290
2291 int block_prepare_write(struct page *page, unsigned from, unsigned to,
2292                         get_block_t *get_block)
2293 {
2294         struct inode *inode = page->mapping->host;
2295         int err = __block_prepare_write(inode, page, from, to, get_block);
2296         if (err)
2297                 ClearPageUptodate(page);
2298         return err;
2299 }
2300
2301 int block_commit_write(struct page *page, unsigned from, unsigned to)
2302 {
2303         struct inode *inode = page->mapping->host;
2304         __block_commit_write(inode,page,from,to);
2305         return 0;
2306 }
2307
2308 int generic_commit_write(struct file *file, struct page *page,
2309                 unsigned from, unsigned to)
2310 {
2311         struct inode *inode = page->mapping->host;
2312         loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2313         __block_commit_write(inode,page,from,to);
2314         /*
2315          * No need to use i_size_read() here, the i_size
2316          * cannot change under us because we hold i_sem.
2317          */
2318         if (pos > inode->i_size) {
2319                 i_size_write(inode, pos);
2320                 mark_inode_dirty(inode);
2321         }
2322         return 0;
2323 }
2324
2325
2326 /*
2327  * nobh_prepare_write()'s prereads are special: the buffer_heads are freed
2328  * immediately, while under the page lock.  So it needs a special end_io
2329  * handler which does not touch the bh after unlocking it.
2330  *
2331  * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
2332  * a race there is benign: unlock_buffer() only use the bh's address for
2333  * hashing after unlocking the buffer, so it doesn't actually touch the bh
2334  * itself.
2335  */
2336 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2337 {
2338         if (uptodate) {
2339                 set_buffer_uptodate(bh);
2340         } else {
2341                 /* This happens, due to failed READA attempts. */
2342                 clear_buffer_uptodate(bh);
2343         }
2344         unlock_buffer(bh);
2345 }
2346
2347 /*
2348  * On entry, the page is fully not uptodate.
2349  * On exit the page is fully uptodate in the areas outside (from,to)
2350  */
2351 int nobh_prepare_write(struct page *page, unsigned from, unsigned to,
2352                         get_block_t *get_block)
2353 {
2354         struct inode *inode = page->mapping->host;
2355         const unsigned blkbits = inode->i_blkbits;
2356         const unsigned blocksize = 1 << blkbits;
2357         struct buffer_head map_bh;
2358         struct buffer_head *read_bh[MAX_BUF_PER_PAGE];
2359         unsigned block_in_page;
2360         unsigned block_start;
2361         sector_t block_in_file;
2362         char *kaddr;
2363         int nr_reads = 0;
2364         int i;
2365         int ret = 0;
2366         int is_mapped_to_disk = 1;
2367         int dirtied_it = 0;
2368
2369         if (PageMappedToDisk(page))
2370                 return 0;
2371
2372         block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2373         map_bh.b_page = page;
2374
2375         /*
2376          * We loop across all blocks in the page, whether or not they are
2377          * part of the affected region.  This is so we can discover if the
2378          * page is fully mapped-to-disk.
2379          */
2380         for (block_start = 0, block_in_page = 0;
2381                   block_start < PAGE_CACHE_SIZE;
2382                   block_in_page++, block_start += blocksize) {
2383                 unsigned block_end = block_start + blocksize;
2384                 int create;
2385
2386                 map_bh.b_state = 0;
2387                 create = 1;
2388                 if (block_start >= to)
2389                         create = 0;
2390                 ret = get_block(inode, block_in_file + block_in_page,
2391                                         &map_bh, create);
2392                 if (ret)
2393                         goto failed;
2394                 if (!buffer_mapped(&map_bh))
2395                         is_mapped_to_disk = 0;
2396                 if (buffer_new(&map_bh))
2397                         unmap_underlying_metadata(map_bh.b_bdev,
2398                                                         map_bh.b_blocknr);
2399                 if (PageUptodate(page))
2400                         continue;
2401                 if (buffer_new(&map_bh) || !buffer_mapped(&map_bh)) {
2402                         kaddr = kmap_atomic(page, KM_USER0);
2403                         if (block_start < from) {
2404                                 memset(kaddr+block_start, 0, from-block_start);
2405                                 dirtied_it = 1;
2406                         }
2407                         if (block_end > to) {
2408                                 memset(kaddr + to, 0, block_end - to);
2409                                 dirtied_it = 1;
2410                         }
2411                         flush_dcache_page(page);
2412                         kunmap_atomic(kaddr, KM_USER0);
2413                         continue;
2414                 }
2415                 if (buffer_uptodate(&map_bh))
2416                         continue;       /* reiserfs does this */
2417                 if (block_start < from || block_end > to) {
2418                         struct buffer_head *bh = alloc_buffer_head(GFP_NOFS);
2419
2420                         if (!bh) {
2421                                 ret = -ENOMEM;
2422                                 goto failed;
2423                         }
2424                         bh->b_state = map_bh.b_state;
2425                         atomic_set(&bh->b_count, 0);
2426                         bh->b_this_page = NULL;
2427                         bh->b_page = page;
2428                         bh->b_blocknr = map_bh.b_blocknr;
2429                         bh->b_size = blocksize;
2430                         bh->b_data = (char *)(long)block_start;
2431                         bh->b_bdev = map_bh.b_bdev;
2432                         bh->b_private = NULL;
2433                         read_bh[nr_reads++] = bh;
2434                 }
2435         }
2436
2437         if (nr_reads) {
2438                 struct buffer_head *bh;
2439
2440                 /*
2441                  * The page is locked, so these buffers are protected from
2442                  * any VM or truncate activity.  Hence we don't need to care
2443                  * for the buffer_head refcounts.
2444                  */
2445                 for (i = 0; i < nr_reads; i++) {
2446                         bh = read_bh[i];
2447                         lock_buffer(bh);
2448                         bh->b_end_io = end_buffer_read_nobh;
2449                         submit_bh(READ, bh);
2450                 }
2451                 for (i = 0; i < nr_reads; i++) {
2452                         bh = read_bh[i];
2453                         wait_on_buffer(bh);
2454                         if (!buffer_uptodate(bh))
2455                                 ret = -EIO;
2456                         free_buffer_head(bh);
2457                         read_bh[i] = NULL;
2458                 }
2459                 if (ret)
2460                         goto failed;
2461         }
2462
2463         if (is_mapped_to_disk)
2464                 SetPageMappedToDisk(page);
2465         SetPageUptodate(page);
2466
2467         /*
2468          * Setting the page dirty here isn't necessary for the prepare_write
2469          * function - commit_write will do that.  But if/when this function is
2470          * used within the pagefault handler to ensure that all mmapped pages
2471          * have backing space in the filesystem, we will need to dirty the page
2472          * if its contents were altered.
2473          */
2474         if (dirtied_it)
2475                 set_page_dirty(page);
2476
2477         return 0;
2478
2479 failed:
2480         for (i = 0; i < nr_reads; i++) {
2481                 if (read_bh[i])
2482                         free_buffer_head(read_bh[i]);
2483         }
2484
2485         /*
2486          * Error recovery is pretty slack.  Clear the page and mark it dirty
2487          * so we'll later zero out any blocks which _were_ allocated.
2488          */
2489         kaddr = kmap_atomic(page, KM_USER0);
2490         memset(kaddr, 0, PAGE_CACHE_SIZE);
2491         kunmap_atomic(kaddr, KM_USER0);
2492         SetPageUptodate(page);
2493         set_page_dirty(page);
2494         return ret;
2495 }
2496 EXPORT_SYMBOL(nobh_prepare_write);
2497
2498 int nobh_commit_write(struct file *file, struct page *page,
2499                 unsigned from, unsigned to)
2500 {
2501         struct inode *inode = page->mapping->host;
2502         loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2503
2504         set_page_dirty(page);
2505         if (pos > inode->i_size) {
2506                 i_size_write(inode, pos);
2507                 mark_inode_dirty(inode);
2508         }
2509         return 0;
2510 }
2511 EXPORT_SYMBOL(nobh_commit_write);
2512
2513 /*
2514  * nobh_writepage() - based on block_full_write_page() except
2515  * that it tries to operate without attaching bufferheads to
2516  * the page.
2517  */
2518 int nobh_writepage(struct page *page, get_block_t *get_block,
2519                         struct writeback_control *wbc)
2520 {
2521         struct inode * const inode = page->mapping->host;
2522         loff_t i_size = i_size_read(inode);
2523         const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2524         unsigned offset;
2525         void *kaddr;
2526         int ret;
2527
2528         /* Is the page fully inside i_size? */
2529         if (page->index < end_index)
2530                 goto out;
2531
2532         /* Is the page fully outside i_size? (truncate in progress) */
2533         offset = i_size & (PAGE_CACHE_SIZE-1);
2534         if (page->index >= end_index+1 || !offset) {
2535                 /*
2536                  * The page may have dirty, unmapped buffers.  For example,
2537                  * they may have been added in ext3_writepage().  Make them
2538                  * freeable here, so the page does not leak.
2539                  */
2540 #if 0
2541                 /* Not really sure about this  - do we need this ? */
2542                 if (page->mapping->a_ops->invalidatepage)
2543                         page->mapping->a_ops->invalidatepage(page, offset);
2544 #endif
2545                 unlock_page(page);
2546                 return 0; /* don't care */
2547         }
2548
2549         /*
2550          * The page straddles i_size.  It must be zeroed out on each and every
2551          * writepage invocation because it may be mmapped.  "A file is mapped
2552          * in multiples of the page size.  For a file that is not a multiple of
2553          * the  page size, the remaining memory is zeroed when mapped, and
2554          * writes to that region are not written out to the file."
2555          */
2556         kaddr = kmap_atomic(page, KM_USER0);
2557         memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2558         flush_dcache_page(page);
2559         kunmap_atomic(kaddr, KM_USER0);
2560 out:
2561         ret = mpage_writepage(page, get_block, wbc);
2562         if (ret == -EAGAIN)
2563                 ret = __block_write_full_page(inode, page, get_block, wbc);
2564         return ret;
2565 }
2566 EXPORT_SYMBOL(nobh_writepage);
2567
2568 /*
2569  * This function assumes that ->prepare_write() uses nobh_prepare_write().
2570  */
2571 int nobh_truncate_page(struct address_space *mapping, loff_t from)
2572 {
2573         struct inode *inode = mapping->host;
2574         unsigned blocksize = 1 << inode->i_blkbits;
2575         pgoff_t index = from >> PAGE_CACHE_SHIFT;
2576         unsigned offset = from & (PAGE_CACHE_SIZE-1);
2577         unsigned to;
2578         struct page *page;
2579         struct address_space_operations *a_ops = mapping->a_ops;
2580         char *kaddr;
2581         int ret = 0;
2582
2583         if ((offset & (blocksize - 1)) == 0)
2584                 goto out;
2585
2586         ret = -ENOMEM;
2587         page = grab_cache_page(mapping, index);
2588         if (!page)
2589                 goto out;
2590
2591         to = (offset + blocksize) & ~(blocksize - 1);
2592         ret = a_ops->prepare_write(NULL, page, offset, to);
2593         if (ret == 0) {
2594                 kaddr = kmap_atomic(page, KM_USER0);
2595                 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2596                 flush_dcache_page(page);
2597                 kunmap_atomic(kaddr, KM_USER0);
2598                 set_page_dirty(page);
2599         }
2600         unlock_page(page);
2601         page_cache_release(page);
2602 out:
2603         return ret;
2604 }
2605 EXPORT_SYMBOL(nobh_truncate_page);
2606
2607 int block_truncate_page(struct address_space *mapping,
2608                         loff_t from, get_block_t *get_block)
2609 {
2610         pgoff_t index = from >> PAGE_CACHE_SHIFT;
2611         unsigned offset = from & (PAGE_CACHE_SIZE-1);
2612         unsigned blocksize;
2613         pgoff_t iblock;
2614         unsigned length, pos;
2615         struct inode *inode = mapping->host;
2616         struct page *page;
2617         struct buffer_head *bh;
2618         void *kaddr;
2619         int err;
2620
2621         blocksize = 1 << inode->i_blkbits;
2622         length = offset & (blocksize - 1);
2623
2624         /* Block boundary? Nothing to do */
2625         if (!length)
2626                 return 0;
2627
2628         length = blocksize - length;
2629         iblock = index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2630         
2631         page = grab_cache_page(mapping, index);
2632         err = -ENOMEM;
2633         if (!page)
2634                 goto out;
2635
2636         if (!page_has_buffers(page))
2637                 create_empty_buffers(page, blocksize, 0);
2638
2639         /* Find the buffer that contains "offset" */
2640         bh = page_buffers(page);
2641         pos = blocksize;
2642         while (offset >= pos) {
2643                 bh = bh->b_this_page;
2644                 iblock++;
2645                 pos += blocksize;
2646         }
2647
2648         err = 0;
2649         if (!buffer_mapped(bh)) {
2650                 err = get_block(inode, iblock, bh, 0);
2651                 if (err)
2652                         goto unlock;
2653                 /* unmapped? It's a hole - nothing to do */
2654                 if (!buffer_mapped(bh))
2655                         goto unlock;
2656         }
2657
2658         /* Ok, it's mapped. Make sure it's up-to-date */
2659         if (PageUptodate(page))
2660                 set_buffer_uptodate(bh);
2661
2662         if (!buffer_uptodate(bh) && !buffer_delay(bh)) {
2663                 err = -EIO;
2664                 ll_rw_block(READ, 1, &bh);
2665                 wait_on_buffer(bh);
2666                 /* Uhhuh. Read error. Complain and punt. */
2667                 if (!buffer_uptodate(bh))
2668                         goto unlock;
2669         }
2670
2671         kaddr = kmap_atomic(page, KM_USER0);
2672         memset(kaddr + offset, 0, length);
2673         flush_dcache_page(page);
2674         kunmap_atomic(kaddr, KM_USER0);
2675
2676         mark_buffer_dirty(bh);
2677         err = 0;
2678
2679 unlock:
2680         unlock_page(page);
2681         page_cache_release(page);
2682 out:
2683         return err;
2684 }
2685
2686 /*
2687  * The generic ->writepage function for buffer-backed address_spaces
2688  */
2689 int block_write_full_page(struct page *page, get_block_t *get_block,
2690                         struct writeback_control *wbc)
2691 {
2692         struct inode * const inode = page->mapping->host;
2693         loff_t i_size = i_size_read(inode);
2694         const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2695         unsigned offset;
2696         void *kaddr;
2697
2698         /* Is the page fully inside i_size? */
2699         if (page->index < end_index)
2700                 return __block_write_full_page(inode, page, get_block, wbc);
2701
2702         /* Is the page fully outside i_size? (truncate in progress) */
2703         offset = i_size & (PAGE_CACHE_SIZE-1);
2704         if (page->index >= end_index+1 || !offset) {
2705                 /*
2706                  * The page may have dirty, unmapped buffers.  For example,
2707                  * they may have been added in ext3_writepage().  Make them
2708                  * freeable here, so the page does not leak.
2709                  */
2710                 do_invalidatepage(page, 0);
2711                 unlock_page(page);
2712                 return 0; /* don't care */
2713         }
2714
2715         /*
2716          * The page straddles i_size.  It must be zeroed out on each and every
2717          * writepage invokation because it may be mmapped.  "A file is mapped
2718          * in multiples of the page size.  For a file that is not a multiple of
2719          * the  page size, the remaining memory is zeroed when mapped, and
2720          * writes to that region are not written out to the file."
2721          */
2722         kaddr = kmap_atomic(page, KM_USER0);
2723         memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2724         flush_dcache_page(page);
2725         kunmap_atomic(kaddr, KM_USER0);
2726         return __block_write_full_page(inode, page, get_block, wbc);
2727 }
2728
2729 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2730                             get_block_t *get_block)
2731 {
2732         struct buffer_head tmp;
2733         struct inode *inode = mapping->host;
2734         tmp.b_state = 0;
2735         tmp.b_blocknr = 0;
2736         get_block(inode, block, &tmp, 0);
2737         return tmp.b_blocknr;
2738 }
2739
2740 static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err)
2741 {
2742         struct buffer_head *bh = bio->bi_private;
2743
2744         if (bio->bi_size)
2745                 return 1;
2746
2747         if (err == -EOPNOTSUPP) {
2748                 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2749                 set_bit(BH_Eopnotsupp, &bh->b_state);
2750         }
2751
2752         bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2753         bio_put(bio);
2754         return 0;
2755 }
2756
2757 int submit_bh(int rw, struct buffer_head * bh)
2758 {
2759         struct bio *bio;
2760         int ret = 0;
2761
2762         BUG_ON(!buffer_locked(bh));
2763         BUG_ON(!buffer_mapped(bh));
2764         BUG_ON(!bh->b_end_io);
2765
2766         if (buffer_ordered(bh) && (rw == WRITE))
2767                 rw = WRITE_BARRIER;
2768
2769         /*
2770          * Only clear out a write error when rewriting, should this
2771          * include WRITE_SYNC as well?
2772          */
2773         if (test_set_buffer_req(bh) && (rw == WRITE || rw == WRITE_BARRIER))
2774                 clear_buffer_write_io_error(bh);
2775
2776         /*
2777          * from here on down, it's all bio -- do the initial mapping,
2778          * submit_bio -> generic_make_request may further map this bio around
2779          */
2780         bio = bio_alloc(GFP_NOIO, 1);
2781
2782         bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2783         bio->bi_bdev = bh->b_bdev;
2784         bio->bi_io_vec[0].bv_page = bh->b_page;
2785         bio->bi_io_vec[0].bv_len = bh->b_size;
2786         bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2787
2788         bio->bi_vcnt = 1;
2789         bio->bi_idx = 0;
2790         bio->bi_size = bh->b_size;
2791
2792         bio->bi_end_io = end_bio_bh_io_sync;
2793         bio->bi_private = bh;
2794
2795         bio_get(bio);
2796         submit_bio(rw, bio);
2797
2798         if (bio_flagged(bio, BIO_EOPNOTSUPP))
2799                 ret = -EOPNOTSUPP;
2800
2801         bio_put(bio);
2802         return ret;
2803 }
2804
2805 /**
2806  * ll_rw_block: low-level access to block devices (DEPRECATED)
2807  * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
2808  * @nr: number of &struct buffer_heads in the array
2809  * @bhs: array of pointers to &struct buffer_head
2810  *
2811  * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
2812  * requests an I/O operation on them, either a %READ or a %WRITE.  The third
2813  * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
2814  * are sent to disk. The fourth %READA option is described in the documentation
2815  * for generic_make_request() which ll_rw_block() calls.
2816  *
2817  * This function drops any buffer that it cannot get a lock on (with the
2818  * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
2819  * clean when doing a write request, and any buffer that appears to be
2820  * up-to-date when doing read request.  Further it marks as clean buffers that
2821  * are processed for writing (the buffer cache won't assume that they are
2822  * actually clean until the buffer gets unlocked).
2823  *
2824  * ll_rw_block sets b_end_io to simple completion handler that marks
2825  * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2826  * any waiters. 
2827  *
2828  * All of the buffers must be for the same device, and must also be a
2829  * multiple of the current approved size for the device.
2830  */
2831 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
2832 {
2833         int i;
2834
2835         for (i = 0; i < nr; i++) {
2836                 struct buffer_head *bh = bhs[i];
2837
2838                 if (rw == SWRITE)
2839                         lock_buffer(bh);
2840                 else if (test_set_buffer_locked(bh))
2841                         continue;
2842
2843                 get_bh(bh);
2844                 if (rw == WRITE || rw == SWRITE) {
2845                         if (test_clear_buffer_dirty(bh)) {
2846                                 bh->b_end_io = end_buffer_write_sync;
2847                                 submit_bh(WRITE, bh);
2848                                 continue;
2849                         }
2850                 } else {
2851                         if (!buffer_uptodate(bh)) {
2852                                 bh->b_end_io = end_buffer_read_sync;
2853                                 submit_bh(rw, bh);
2854                                 continue;
2855                         }
2856                 }
2857                 unlock_buffer(bh);
2858                 put_bh(bh);
2859         }
2860 }
2861
2862 /*
2863  * For a data-integrity writeout, we need to wait upon any in-progress I/O
2864  * and then start new I/O and then wait upon it.  The caller must have a ref on
2865  * the buffer_head.
2866  */
2867 int sync_dirty_buffer(struct buffer_head *bh)
2868 {
2869         int ret = 0;
2870
2871         WARN_ON(atomic_read(&bh->b_count) < 1);
2872         lock_buffer(bh);
2873         if (test_clear_buffer_dirty(bh)) {
2874                 get_bh(bh);
2875                 bh->b_end_io = end_buffer_write_sync;
2876                 ret = submit_bh(WRITE, bh);
2877                 wait_on_buffer(bh);
2878                 if (buffer_eopnotsupp(bh)) {
2879                         clear_buffer_eopnotsupp(bh);
2880                         ret = -EOPNOTSUPP;
2881                 }
2882                 if (!ret && !buffer_uptodate(bh))
2883                         ret = -EIO;
2884         } else {
2885                 unlock_buffer(bh);
2886         }
2887         return ret;
2888 }
2889
2890 /*
2891  * try_to_free_buffers() checks if all the buffers on this particular page
2892  * are unused, and releases them if so.
2893  *
2894  * Exclusion against try_to_free_buffers may be obtained by either
2895  * locking the page or by holding its mapping's private_lock.
2896  *
2897  * If the page is dirty but all the buffers are clean then we need to
2898  * be sure to mark the page clean as well.  This is because the page
2899  * may be against a block device, and a later reattachment of buffers
2900  * to a dirty page will set *all* buffers dirty.  Which would corrupt
2901  * filesystem data on the same device.
2902  *
2903  * The same applies to regular filesystem pages: if all the buffers are
2904  * clean then we set the page clean and proceed.  To do that, we require
2905  * total exclusion from __set_page_dirty_buffers().  That is obtained with
2906  * private_lock.
2907  *
2908  * try_to_free_buffers() is non-blocking.
2909  */
2910 static inline int buffer_busy(struct buffer_head *bh)
2911 {
2912         return atomic_read(&bh->b_count) |
2913                 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
2914 }
2915
2916 static int
2917 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
2918 {
2919         struct buffer_head *head = page_buffers(page);
2920         struct buffer_head *bh;
2921
2922         bh = head;
2923         do {
2924                 if (buffer_write_io_error(bh) && page->mapping)
2925                         set_bit(AS_EIO, &page->mapping->flags);
2926                 if (buffer_busy(bh))
2927                         goto failed;
2928                 bh = bh->b_this_page;
2929         } while (bh != head);
2930
2931         do {
2932                 struct buffer_head *next = bh->b_this_page;
2933
2934                 if (!list_empty(&bh->b_assoc_buffers))
2935                         __remove_assoc_queue(bh);
2936                 bh = next;
2937         } while (bh != head);
2938         *buffers_to_free = head;
2939         __clear_page_buffers(page);
2940         return 1;
2941 failed:
2942         return 0;
2943 }
2944
2945 int try_to_free_buffers(struct page *page)
2946 {
2947         struct address_space * const mapping = page->mapping;
2948         struct buffer_head *buffers_to_free = NULL;
2949         int ret = 0;
2950
2951         BUG_ON(!PageLocked(page));
2952         if (PageWriteback(page))
2953                 return 0;
2954
2955         if (mapping == NULL) {          /* can this still happen? */
2956                 ret = drop_buffers(page, &buffers_to_free);
2957                 goto out;
2958         }
2959
2960         spin_lock(&mapping->private_lock);
2961         ret = drop_buffers(page, &buffers_to_free);
2962         if (ret) {
2963                 /*
2964                  * If the filesystem writes its buffers by hand (eg ext3)
2965                  * then we can have clean buffers against a dirty page.  We
2966                  * clean the page here; otherwise later reattachment of buffers
2967                  * could encounter a non-uptodate page, which is unresolvable.
2968                  * This only applies in the rare case where try_to_free_buffers
2969                  * succeeds but the page is not freed.
2970                  */
2971                 clear_page_dirty(page);
2972         }
2973         spin_unlock(&mapping->private_lock);
2974 out:
2975         if (buffers_to_free) {
2976                 struct buffer_head *bh = buffers_to_free;
2977
2978                 do {
2979                         struct buffer_head *next = bh->b_this_page;
2980                         free_buffer_head(bh);
2981                         bh = next;
2982                 } while (bh != buffers_to_free);
2983         }
2984         return ret;
2985 }
2986 EXPORT_SYMBOL(try_to_free_buffers);
2987
2988 int block_sync_page(struct page *page)
2989 {
2990         struct address_space *mapping;
2991
2992         smp_mb();
2993         mapping = page_mapping(page);
2994         if (mapping)
2995                 blk_run_backing_dev(mapping->backing_dev_info, page);
2996         return 0;
2997 }
2998
2999 /*
3000  * There are no bdflush tunables left.  But distributions are
3001  * still running obsolete flush daemons, so we terminate them here.
3002  *
3003  * Use of bdflush() is deprecated and will be removed in a future kernel.
3004  * The `pdflush' kernel threads fully replace bdflush daemons and this call.
3005  */
3006 asmlinkage long sys_bdflush(int func, long data)
3007 {
3008         static int msg_count;
3009
3010         if (!capable(CAP_SYS_ADMIN))
3011                 return -EPERM;
3012
3013         if (msg_count < 5) {
3014                 msg_count++;
3015                 printk(KERN_INFO
3016                         "warning: process `%s' used the obsolete bdflush"
3017                         " system call\n", current->comm);
3018                 printk(KERN_INFO "Fix your initscripts?\n");
3019         }
3020
3021         if (func == 1)
3022                 do_exit(0);
3023         return 0;
3024 }
3025
3026 /*
3027  * Buffer-head allocation
3028  */
3029 static kmem_cache_t *bh_cachep;
3030
3031 /*
3032  * Once the number of bh's in the machine exceeds this level, we start
3033  * stripping them in writeback.
3034  */
3035 static int max_buffer_heads;
3036
3037 int buffer_heads_over_limit;
3038
3039 struct bh_accounting {
3040         int nr;                 /* Number of live bh's */
3041         int ratelimit;          /* Limit cacheline bouncing */
3042 };
3043
3044 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3045
3046 static void recalc_bh_state(void)
3047 {
3048         int i;
3049         int tot = 0;
3050
3051         if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3052                 return;
3053         __get_cpu_var(bh_accounting).ratelimit = 0;
3054         for_each_cpu(i)
3055                 tot += per_cpu(bh_accounting, i).nr;
3056         buffer_heads_over_limit = (tot > max_buffer_heads);
3057 }
3058         
3059 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3060 {
3061         struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
3062         if (ret) {
3063                 get_cpu_var(bh_accounting).nr++;
3064                 recalc_bh_state();
3065                 put_cpu_var(bh_accounting);
3066         }
3067         return ret;
3068 }
3069 EXPORT_SYMBOL(alloc_buffer_head);
3070
3071 void free_buffer_head(struct buffer_head *bh)
3072 {
3073         BUG_ON(!list_empty(&bh->b_assoc_buffers));
3074         kmem_cache_free(bh_cachep, bh);
3075         get_cpu_var(bh_accounting).nr--;
3076         recalc_bh_state();
3077         put_cpu_var(bh_accounting);
3078 }
3079 EXPORT_SYMBOL(free_buffer_head);
3080
3081 static void
3082 init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags)
3083 {
3084         if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
3085                             SLAB_CTOR_CONSTRUCTOR) {
3086                 struct buffer_head * bh = (struct buffer_head *)data;
3087
3088                 memset(bh, 0, sizeof(*bh));
3089                 INIT_LIST_HEAD(&bh->b_assoc_buffers);
3090         }
3091 }
3092
3093 #ifdef CONFIG_HOTPLUG_CPU
3094 static void buffer_exit_cpu(int cpu)
3095 {
3096         int i;
3097         struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3098
3099         for (i = 0; i < BH_LRU_SIZE; i++) {
3100                 brelse(b->bhs[i]);
3101                 b->bhs[i] = NULL;
3102         }
3103 }
3104
3105 static int buffer_cpu_notify(struct notifier_block *self,
3106                               unsigned long action, void *hcpu)
3107 {
3108         if (action == CPU_DEAD)
3109                 buffer_exit_cpu((unsigned long)hcpu);
3110         return NOTIFY_OK;
3111 }
3112 #endif /* CONFIG_HOTPLUG_CPU */
3113
3114 void __init buffer_init(void)
3115 {
3116         int nrpages;
3117
3118         bh_cachep = kmem_cache_create("buffer_head",
3119                         sizeof(struct buffer_head), 0,
3120                         SLAB_RECLAIM_ACCOUNT|SLAB_PANIC, init_buffer_head, NULL);
3121
3122         /*
3123          * Limit the bh occupancy to 10% of ZONE_NORMAL
3124          */
3125         nrpages = (nr_free_buffer_pages() * 10) / 100;
3126         max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3127         hotcpu_notifier(buffer_cpu_notify, 0);
3128 }
3129
3130 EXPORT_SYMBOL(__bforget);
3131 EXPORT_SYMBOL(__brelse);
3132 EXPORT_SYMBOL(__wait_on_buffer);
3133 EXPORT_SYMBOL(block_commit_write);
3134 EXPORT_SYMBOL(block_prepare_write);
3135 EXPORT_SYMBOL(block_read_full_page);
3136 EXPORT_SYMBOL(block_sync_page);
3137 EXPORT_SYMBOL(block_truncate_page);
3138 EXPORT_SYMBOL(block_write_full_page);
3139 EXPORT_SYMBOL(cont_prepare_write);
3140 EXPORT_SYMBOL(end_buffer_async_write);
3141 EXPORT_SYMBOL(end_buffer_read_sync);
3142 EXPORT_SYMBOL(end_buffer_write_sync);
3143 EXPORT_SYMBOL(file_fsync);
3144 EXPORT_SYMBOL(fsync_bdev);
3145 EXPORT_SYMBOL(generic_block_bmap);
3146 EXPORT_SYMBOL(generic_commit_write);
3147 EXPORT_SYMBOL(generic_cont_expand);
3148 EXPORT_SYMBOL(init_buffer);
3149 EXPORT_SYMBOL(invalidate_bdev);
3150 EXPORT_SYMBOL(ll_rw_block);
3151 EXPORT_SYMBOL(mark_buffer_dirty);
3152 EXPORT_SYMBOL(submit_bh);
3153 EXPORT_SYMBOL(sync_dirty_buffer);
3154 EXPORT_SYMBOL(unlock_buffer);