ia64/linux-2.6.18-xen.hg

view fs/buffer.c @ 524:7f8b544237bf

netfront: Allow netfront in domain 0.

This is useful if your physical network device is in a utility domain.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Tue Apr 15 15:18:58 2008 +0100 (2008-04-15)
parents 3e8752eb6d9c
children cad6f60f0506
line source
1 /*
2 * linux/fs/buffer.c
3 *
4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
5 */
7 /*
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9 *
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12 *
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
15 *
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17 *
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19 */
21 #include <linux/kernel.h>
22 #include <linux/syscalls.h>
23 #include <linux/fs.h>
24 #include <linux/mm.h>
25 #include <linux/percpu.h>
26 #include <linux/slab.h>
27 #include <linux/smp_lock.h>
28 #include <linux/capability.h>
29 #include <linux/blkdev.h>
30 #include <linux/file.h>
31 #include <linux/quotaops.h>
32 #include <linux/highmem.h>
33 #include <linux/module.h>
34 #include <linux/writeback.h>
35 #include <linux/hash.h>
36 #include <linux/suspend.h>
37 #include <linux/buffer_head.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
43 #include <linux/bit_spinlock.h>
45 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
46 static void invalidate_bh_lrus(void);
48 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
50 inline void
51 init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
52 {
53 bh->b_end_io = handler;
54 bh->b_private = private;
55 }
57 static int sync_buffer(void *word)
58 {
59 struct block_device *bd;
60 struct buffer_head *bh
61 = container_of(word, struct buffer_head, b_state);
63 smp_mb();
64 bd = bh->b_bdev;
65 if (bd)
66 blk_run_address_space(bd->bd_inode->i_mapping);
67 io_schedule();
68 return 0;
69 }
71 void fastcall __lock_buffer(struct buffer_head *bh)
72 {
73 wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
74 TASK_UNINTERRUPTIBLE);
75 }
76 EXPORT_SYMBOL(__lock_buffer);
78 void fastcall unlock_buffer(struct buffer_head *bh)
79 {
80 clear_buffer_locked(bh);
81 smp_mb__after_clear_bit();
82 wake_up_bit(&bh->b_state, BH_Lock);
83 }
85 /*
86 * Block until a buffer comes unlocked. This doesn't stop it
87 * from becoming locked again - you have to lock it yourself
88 * if you want to preserve its state.
89 */
90 void __wait_on_buffer(struct buffer_head * bh)
91 {
92 wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
93 }
95 static void
96 __clear_page_buffers(struct page *page)
97 {
98 ClearPagePrivate(page);
99 set_page_private(page, 0);
100 page_cache_release(page);
101 }
103 static void buffer_io_error(struct buffer_head *bh)
104 {
105 char b[BDEVNAME_SIZE];
107 printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
108 bdevname(bh->b_bdev, b),
109 (unsigned long long)bh->b_blocknr);
110 }
112 /*
113 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
114 * unlock the buffer. This is what ll_rw_block uses too.
115 */
116 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
117 {
118 if (uptodate) {
119 set_buffer_uptodate(bh);
120 } else {
121 /* This happens, due to failed READA attempts. */
122 clear_buffer_uptodate(bh);
123 }
124 unlock_buffer(bh);
125 put_bh(bh);
126 }
128 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
129 {
130 char b[BDEVNAME_SIZE];
132 if (uptodate) {
133 set_buffer_uptodate(bh);
134 } else {
135 if (!buffer_eopnotsupp(bh) && printk_ratelimit()) {
136 buffer_io_error(bh);
137 printk(KERN_WARNING "lost page write due to "
138 "I/O error on %s\n",
139 bdevname(bh->b_bdev, b));
140 }
141 set_buffer_write_io_error(bh);
142 clear_buffer_uptodate(bh);
143 }
144 unlock_buffer(bh);
145 put_bh(bh);
146 }
148 /*
149 * Write out and wait upon all the dirty data associated with a block
150 * device via its mapping. Does not take the superblock lock.
151 */
152 int sync_blockdev(struct block_device *bdev)
153 {
154 int ret = 0;
156 if (bdev)
157 ret = filemap_write_and_wait(bdev->bd_inode->i_mapping);
158 return ret;
159 }
160 EXPORT_SYMBOL(sync_blockdev);
162 static void __fsync_super(struct super_block *sb)
163 {
164 sync_inodes_sb(sb, 0);
165 DQUOT_SYNC(sb);
166 lock_super(sb);
167 if (sb->s_dirt && sb->s_op->write_super)
168 sb->s_op->write_super(sb);
169 unlock_super(sb);
170 if (sb->s_op->sync_fs)
171 sb->s_op->sync_fs(sb, 1);
172 sync_blockdev(sb->s_bdev);
173 sync_inodes_sb(sb, 1);
174 }
176 /*
177 * Write out and wait upon all dirty data associated with this
178 * superblock. Filesystem data as well as the underlying block
179 * device. Takes the superblock lock.
180 */
181 int fsync_super(struct super_block *sb)
182 {
183 __fsync_super(sb);
184 return sync_blockdev(sb->s_bdev);
185 }
187 /*
188 * Write out and wait upon all dirty data associated with this
189 * device. Filesystem data as well as the underlying block
190 * device. Takes the superblock lock.
191 */
192 int fsync_bdev(struct block_device *bdev)
193 {
194 struct super_block *sb = get_super(bdev);
195 if (sb) {
196 int res = fsync_super(sb);
197 drop_super(sb);
198 return res;
199 }
200 return sync_blockdev(bdev);
201 }
203 /**
204 * freeze_bdev -- lock a filesystem and force it into a consistent state
205 * @bdev: blockdevice to lock
206 *
207 * This takes the block device bd_mount_mutex to make sure no new mounts
208 * happen on bdev until thaw_bdev() is called.
209 * If a superblock is found on this device, we take the s_umount semaphore
210 * on it to make sure nobody unmounts until the snapshot creation is done.
211 */
212 struct super_block *freeze_bdev(struct block_device *bdev)
213 {
214 struct super_block *sb;
216 mutex_lock(&bdev->bd_mount_mutex);
217 sb = get_super(bdev);
218 if (sb && !(sb->s_flags & MS_RDONLY)) {
219 sb->s_frozen = SB_FREEZE_WRITE;
220 smp_wmb();
222 __fsync_super(sb);
224 sb->s_frozen = SB_FREEZE_TRANS;
225 smp_wmb();
227 sync_blockdev(sb->s_bdev);
229 if (sb->s_op->write_super_lockfs)
230 sb->s_op->write_super_lockfs(sb);
231 }
233 sync_blockdev(bdev);
234 return sb; /* thaw_bdev releases s->s_umount and bd_mount_sem */
235 }
236 EXPORT_SYMBOL(freeze_bdev);
238 /**
239 * thaw_bdev -- unlock filesystem
240 * @bdev: blockdevice to unlock
241 * @sb: associated superblock
242 *
243 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
244 */
245 void thaw_bdev(struct block_device *bdev, struct super_block *sb)
246 {
247 if (sb) {
248 BUG_ON(sb->s_bdev != bdev);
250 if (sb->s_op->unlockfs)
251 sb->s_op->unlockfs(sb);
252 sb->s_frozen = SB_UNFROZEN;
253 smp_wmb();
254 wake_up(&sb->s_wait_unfrozen);
255 drop_super(sb);
256 }
258 mutex_unlock(&bdev->bd_mount_mutex);
259 }
260 EXPORT_SYMBOL(thaw_bdev);
262 /*
263 * sync everything. Start out by waking pdflush, because that writes back
264 * all queues in parallel.
265 */
266 static void do_sync(unsigned long wait)
267 {
268 wakeup_pdflush(0);
269 sync_inodes(0); /* All mappings, inodes and their blockdevs */
270 DQUOT_SYNC(NULL);
271 sync_supers(); /* Write the superblocks */
272 sync_filesystems(0); /* Start syncing the filesystems */
273 sync_filesystems(wait); /* Waitingly sync the filesystems */
274 sync_inodes(wait); /* Mappings, inodes and blockdevs, again. */
275 if (!wait)
276 printk("Emergency Sync complete\n");
277 if (unlikely(laptop_mode))
278 laptop_sync_completion();
279 }
281 asmlinkage long sys_sync(void)
282 {
283 do_sync(1);
284 return 0;
285 }
287 void emergency_sync(void)
288 {
289 pdflush_operation(do_sync, 0);
290 }
292 /*
293 * Generic function to fsync a file.
294 *
295 * filp may be NULL if called via the msync of a vma.
296 */
298 int file_fsync(struct file *filp, struct dentry *dentry, int datasync)
299 {
300 struct inode * inode = dentry->d_inode;
301 struct super_block * sb;
302 int ret, err;
304 /* sync the inode to buffers */
305 ret = write_inode_now(inode, 0);
307 /* sync the superblock to buffers */
308 sb = inode->i_sb;
309 lock_super(sb);
310 if (sb->s_op->write_super)
311 sb->s_op->write_super(sb);
312 unlock_super(sb);
314 /* .. finally sync the buffers to disk */
315 err = sync_blockdev(sb->s_bdev);
316 if (!ret)
317 ret = err;
318 return ret;
319 }
321 long do_fsync(struct file *file, int datasync)
322 {
323 int ret;
324 int err;
325 struct address_space *mapping = file->f_mapping;
327 if (!file->f_op || !file->f_op->fsync) {
328 /* Why? We can still call filemap_fdatawrite */
329 ret = -EINVAL;
330 goto out;
331 }
333 ret = filemap_fdatawrite(mapping);
335 /*
336 * We need to protect against concurrent writers, which could cause
337 * livelocks in fsync_buffers_list().
338 */
339 mutex_lock(&mapping->host->i_mutex);
340 err = file->f_op->fsync(file, file->f_dentry, datasync);
341 if (!ret)
342 ret = err;
343 mutex_unlock(&mapping->host->i_mutex);
344 err = filemap_fdatawait(mapping);
345 if (!ret)
346 ret = err;
347 out:
348 return ret;
349 }
351 static long __do_fsync(unsigned int fd, int datasync)
352 {
353 struct file *file;
354 int ret = -EBADF;
356 file = fget(fd);
357 if (file) {
358 ret = do_fsync(file, datasync);
359 fput(file);
360 }
361 return ret;
362 }
364 asmlinkage long sys_fsync(unsigned int fd)
365 {
366 return __do_fsync(fd, 0);
367 }
369 asmlinkage long sys_fdatasync(unsigned int fd)
370 {
371 return __do_fsync(fd, 1);
372 }
374 /*
375 * Various filesystems appear to want __find_get_block to be non-blocking.
376 * But it's the page lock which protects the buffers. To get around this,
377 * we get exclusion from try_to_free_buffers with the blockdev mapping's
378 * private_lock.
379 *
380 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
381 * may be quite high. This code could TryLock the page, and if that
382 * succeeds, there is no need to take private_lock. (But if
383 * private_lock is contended then so is mapping->tree_lock).
384 */
385 static struct buffer_head *
386 __find_get_block_slow(struct block_device *bdev, sector_t block)
387 {
388 struct inode *bd_inode = bdev->bd_inode;
389 struct address_space *bd_mapping = bd_inode->i_mapping;
390 struct buffer_head *ret = NULL;
391 pgoff_t index;
392 struct buffer_head *bh;
393 struct buffer_head *head;
394 struct page *page;
395 int all_mapped = 1;
397 index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
398 page = find_get_page(bd_mapping, index);
399 if (!page)
400 goto out;
402 spin_lock(&bd_mapping->private_lock);
403 if (!page_has_buffers(page))
404 goto out_unlock;
405 head = page_buffers(page);
406 bh = head;
407 do {
408 if (bh->b_blocknr == block) {
409 ret = bh;
410 get_bh(bh);
411 goto out_unlock;
412 }
413 if (!buffer_mapped(bh))
414 all_mapped = 0;
415 bh = bh->b_this_page;
416 } while (bh != head);
418 /* we might be here because some of the buffers on this page are
419 * not mapped. This is due to various races between
420 * file io on the block device and getblk. It gets dealt with
421 * elsewhere, don't buffer_error if we had some unmapped buffers
422 */
423 if (all_mapped) {
424 printk("__find_get_block_slow() failed. "
425 "block=%llu, b_blocknr=%llu\n",
426 (unsigned long long)block,
427 (unsigned long long)bh->b_blocknr);
428 printk("b_state=0x%08lx, b_size=%zu\n",
429 bh->b_state, bh->b_size);
430 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
431 }
432 out_unlock:
433 spin_unlock(&bd_mapping->private_lock);
434 page_cache_release(page);
435 out:
436 return ret;
437 }
439 /* If invalidate_buffers() will trash dirty buffers, it means some kind
440 of fs corruption is going on. Trashing dirty data always imply losing
441 information that was supposed to be just stored on the physical layer
442 by the user.
444 Thus invalidate_buffers in general usage is not allwowed to trash
445 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
446 be preserved. These buffers are simply skipped.
448 We also skip buffers which are still in use. For example this can
449 happen if a userspace program is reading the block device.
451 NOTE: In the case where the user removed a removable-media-disk even if
452 there's still dirty data not synced on disk (due a bug in the device driver
453 or due an error of the user), by not destroying the dirty buffers we could
454 generate corruption also on the next media inserted, thus a parameter is
455 necessary to handle this case in the most safe way possible (trying
456 to not corrupt also the new disk inserted with the data belonging to
457 the old now corrupted disk). Also for the ramdisk the natural thing
458 to do in order to release the ramdisk memory is to destroy dirty buffers.
460 These are two special cases. Normal usage imply the device driver
461 to issue a sync on the device (without waiting I/O completion) and
462 then an invalidate_buffers call that doesn't trash dirty buffers.
464 For handling cache coherency with the blkdev pagecache the 'update' case
465 is been introduced. It is needed to re-read from disk any pinned
466 buffer. NOTE: re-reading from disk is destructive so we can do it only
467 when we assume nobody is changing the buffercache under our I/O and when
468 we think the disk contains more recent information than the buffercache.
469 The update == 1 pass marks the buffers we need to update, the update == 2
470 pass does the actual I/O. */
471 void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers)
472 {
473 struct address_space *mapping = bdev->bd_inode->i_mapping;
475 if (mapping->nrpages == 0)
476 return;
478 invalidate_bh_lrus();
479 /*
480 * FIXME: what about destroy_dirty_buffers?
481 * We really want to use invalidate_inode_pages2() for
482 * that, but not until that's cleaned up.
483 */
484 invalidate_inode_pages(mapping);
485 }
487 /*
488 * Kick pdflush then try to free up some ZONE_NORMAL memory.
489 */
490 static void free_more_memory(void)
491 {
492 struct zone **zones;
493 pg_data_t *pgdat;
495 wakeup_pdflush(1024);
496 yield();
498 for_each_online_pgdat(pgdat) {
499 zones = pgdat->node_zonelists[gfp_zone(GFP_NOFS)].zones;
500 if (*zones)
501 try_to_free_pages(zones, GFP_NOFS);
502 }
503 }
505 /*
506 * I/O completion handler for block_read_full_page() - pages
507 * which come unlocked at the end of I/O.
508 */
509 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
510 {
511 unsigned long flags;
512 struct buffer_head *first;
513 struct buffer_head *tmp;
514 struct page *page;
515 int page_uptodate = 1;
517 BUG_ON(!buffer_async_read(bh));
519 page = bh->b_page;
520 if (uptodate) {
521 set_buffer_uptodate(bh);
522 } else {
523 clear_buffer_uptodate(bh);
524 if (printk_ratelimit())
525 buffer_io_error(bh);
526 SetPageError(page);
527 }
529 /*
530 * Be _very_ careful from here on. Bad things can happen if
531 * two buffer heads end IO at almost the same time and both
532 * decide that the page is now completely done.
533 */
534 first = page_buffers(page);
535 local_irq_save(flags);
536 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
537 clear_buffer_async_read(bh);
538 unlock_buffer(bh);
539 tmp = bh;
540 do {
541 if (!buffer_uptodate(tmp))
542 page_uptodate = 0;
543 if (buffer_async_read(tmp)) {
544 BUG_ON(!buffer_locked(tmp));
545 goto still_busy;
546 }
547 tmp = tmp->b_this_page;
548 } while (tmp != bh);
549 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
550 local_irq_restore(flags);
552 /*
553 * If none of the buffers had errors and they are all
554 * uptodate then we can set the page uptodate.
555 */
556 if (page_uptodate && !PageError(page))
557 SetPageUptodate(page);
558 unlock_page(page);
559 return;
561 still_busy:
562 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
563 local_irq_restore(flags);
564 return;
565 }
567 /*
568 * Completion handler for block_write_full_page() - pages which are unlocked
569 * during I/O, and which have PageWriteback cleared upon I/O completion.
570 */
571 static void end_buffer_async_write(struct buffer_head *bh, int uptodate)
572 {
573 char b[BDEVNAME_SIZE];
574 unsigned long flags;
575 struct buffer_head *first;
576 struct buffer_head *tmp;
577 struct page *page;
579 BUG_ON(!buffer_async_write(bh));
581 page = bh->b_page;
582 if (uptodate) {
583 set_buffer_uptodate(bh);
584 } else {
585 if (printk_ratelimit()) {
586 buffer_io_error(bh);
587 printk(KERN_WARNING "lost page write due to "
588 "I/O error on %s\n",
589 bdevname(bh->b_bdev, b));
590 }
591 set_bit(AS_EIO, &page->mapping->flags);
592 clear_buffer_uptodate(bh);
593 SetPageError(page);
594 }
596 first = page_buffers(page);
597 local_irq_save(flags);
598 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
600 clear_buffer_async_write(bh);
601 unlock_buffer(bh);
602 tmp = bh->b_this_page;
603 while (tmp != bh) {
604 if (buffer_async_write(tmp)) {
605 BUG_ON(!buffer_locked(tmp));
606 goto still_busy;
607 }
608 tmp = tmp->b_this_page;
609 }
610 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
611 local_irq_restore(flags);
612 end_page_writeback(page);
613 return;
615 still_busy:
616 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
617 local_irq_restore(flags);
618 return;
619 }
621 /*
622 * If a page's buffers are under async readin (end_buffer_async_read
623 * completion) then there is a possibility that another thread of
624 * control could lock one of the buffers after it has completed
625 * but while some of the other buffers have not completed. This
626 * locked buffer would confuse end_buffer_async_read() into not unlocking
627 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
628 * that this buffer is not under async I/O.
629 *
630 * The page comes unlocked when it has no locked buffer_async buffers
631 * left.
632 *
633 * PageLocked prevents anyone starting new async I/O reads any of
634 * the buffers.
635 *
636 * PageWriteback is used to prevent simultaneous writeout of the same
637 * page.
638 *
639 * PageLocked prevents anyone from starting writeback of a page which is
640 * under read I/O (PageWriteback is only ever set against a locked page).
641 */
642 static void mark_buffer_async_read(struct buffer_head *bh)
643 {
644 bh->b_end_io = end_buffer_async_read;
645 set_buffer_async_read(bh);
646 }
648 void mark_buffer_async_write(struct buffer_head *bh)
649 {
650 bh->b_end_io = end_buffer_async_write;
651 set_buffer_async_write(bh);
652 }
653 EXPORT_SYMBOL(mark_buffer_async_write);
656 /*
657 * fs/buffer.c contains helper functions for buffer-backed address space's
658 * fsync functions. A common requirement for buffer-based filesystems is
659 * that certain data from the backing blockdev needs to be written out for
660 * a successful fsync(). For example, ext2 indirect blocks need to be
661 * written back and waited upon before fsync() returns.
662 *
663 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
664 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
665 * management of a list of dependent buffers at ->i_mapping->private_list.
666 *
667 * Locking is a little subtle: try_to_free_buffers() will remove buffers
668 * from their controlling inode's queue when they are being freed. But
669 * try_to_free_buffers() will be operating against the *blockdev* mapping
670 * at the time, not against the S_ISREG file which depends on those buffers.
671 * So the locking for private_list is via the private_lock in the address_space
672 * which backs the buffers. Which is different from the address_space
673 * against which the buffers are listed. So for a particular address_space,
674 * mapping->private_lock does *not* protect mapping->private_list! In fact,
675 * mapping->private_list will always be protected by the backing blockdev's
676 * ->private_lock.
677 *
678 * Which introduces a requirement: all buffers on an address_space's
679 * ->private_list must be from the same address_space: the blockdev's.
680 *
681 * address_spaces which do not place buffers at ->private_list via these
682 * utility functions are free to use private_lock and private_list for
683 * whatever they want. The only requirement is that list_empty(private_list)
684 * be true at clear_inode() time.
685 *
686 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
687 * filesystems should do that. invalidate_inode_buffers() should just go
688 * BUG_ON(!list_empty).
689 *
690 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
691 * take an address_space, not an inode. And it should be called
692 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
693 * queued up.
694 *
695 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
696 * list if it is already on a list. Because if the buffer is on a list,
697 * it *must* already be on the right one. If not, the filesystem is being
698 * silly. This will save a ton of locking. But first we have to ensure
699 * that buffers are taken *off* the old inode's list when they are freed
700 * (presumably in truncate). That requires careful auditing of all
701 * filesystems (do it inside bforget()). It could also be done by bringing
702 * b_inode back.
703 */
705 /*
706 * The buffer's backing address_space's private_lock must be held
707 */
708 static inline void __remove_assoc_queue(struct buffer_head *bh)
709 {
710 list_del_init(&bh->b_assoc_buffers);
711 }
713 int inode_has_buffers(struct inode *inode)
714 {
715 return !list_empty(&inode->i_data.private_list);
716 }
718 /*
719 * osync is designed to support O_SYNC io. It waits synchronously for
720 * all already-submitted IO to complete, but does not queue any new
721 * writes to the disk.
722 *
723 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
724 * you dirty the buffers, and then use osync_inode_buffers to wait for
725 * completion. Any other dirty buffers which are not yet queued for
726 * write will not be flushed to disk by the osync.
727 */
728 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
729 {
730 struct buffer_head *bh;
731 struct list_head *p;
732 int err = 0;
734 spin_lock(lock);
735 repeat:
736 list_for_each_prev(p, list) {
737 bh = BH_ENTRY(p);
738 if (buffer_locked(bh)) {
739 get_bh(bh);
740 spin_unlock(lock);
741 wait_on_buffer(bh);
742 if (!buffer_uptodate(bh))
743 err = -EIO;
744 brelse(bh);
745 spin_lock(lock);
746 goto repeat;
747 }
748 }
749 spin_unlock(lock);
750 return err;
751 }
753 /**
754 * sync_mapping_buffers - write out and wait upon a mapping's "associated"
755 * buffers
756 * @mapping: the mapping which wants those buffers written
757 *
758 * Starts I/O against the buffers at mapping->private_list, and waits upon
759 * that I/O.
760 *
761 * Basically, this is a convenience function for fsync().
762 * @mapping is a file or directory which needs those buffers to be written for
763 * a successful fsync().
764 */
765 int sync_mapping_buffers(struct address_space *mapping)
766 {
767 struct address_space *buffer_mapping = mapping->assoc_mapping;
769 if (buffer_mapping == NULL || list_empty(&mapping->private_list))
770 return 0;
772 return fsync_buffers_list(&buffer_mapping->private_lock,
773 &mapping->private_list);
774 }
775 EXPORT_SYMBOL(sync_mapping_buffers);
777 /*
778 * Called when we've recently written block `bblock', and it is known that
779 * `bblock' was for a buffer_boundary() buffer. This means that the block at
780 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
781 * dirty, schedule it for IO. So that indirects merge nicely with their data.
782 */
783 void write_boundary_block(struct block_device *bdev,
784 sector_t bblock, unsigned blocksize)
785 {
786 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
787 if (bh) {
788 if (buffer_dirty(bh))
789 ll_rw_block(WRITE, 1, &bh);
790 put_bh(bh);
791 }
792 }
794 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
795 {
796 struct address_space *mapping = inode->i_mapping;
797 struct address_space *buffer_mapping = bh->b_page->mapping;
799 mark_buffer_dirty(bh);
800 if (!mapping->assoc_mapping) {
801 mapping->assoc_mapping = buffer_mapping;
802 } else {
803 BUG_ON(mapping->assoc_mapping != buffer_mapping);
804 }
805 if (list_empty(&bh->b_assoc_buffers)) {
806 spin_lock(&buffer_mapping->private_lock);
807 list_move_tail(&bh->b_assoc_buffers,
808 &mapping->private_list);
809 spin_unlock(&buffer_mapping->private_lock);
810 }
811 }
812 EXPORT_SYMBOL(mark_buffer_dirty_inode);
814 /*
815 * Add a page to the dirty page list.
816 *
817 * It is a sad fact of life that this function is called from several places
818 * deeply under spinlocking. It may not sleep.
819 *
820 * If the page has buffers, the uptodate buffers are set dirty, to preserve
821 * dirty-state coherency between the page and the buffers. It the page does
822 * not have buffers then when they are later attached they will all be set
823 * dirty.
824 *
825 * The buffers are dirtied before the page is dirtied. There's a small race
826 * window in which a writepage caller may see the page cleanness but not the
827 * buffer dirtiness. That's fine. If this code were to set the page dirty
828 * before the buffers, a concurrent writepage caller could clear the page dirty
829 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
830 * page on the dirty page list.
831 *
832 * We use private_lock to lock against try_to_free_buffers while using the
833 * page's buffer list. Also use this to protect against clean buffers being
834 * added to the page after it was set dirty.
835 *
836 * FIXME: may need to call ->reservepage here as well. That's rather up to the
837 * address_space though.
838 */
839 int __set_page_dirty_buffers(struct page *page)
840 {
841 struct address_space * const mapping = page_mapping(page);
843 if (unlikely(!mapping))
844 return !TestSetPageDirty(page);
846 spin_lock(&mapping->private_lock);
847 if (page_has_buffers(page)) {
848 struct buffer_head *head = page_buffers(page);
849 struct buffer_head *bh = head;
851 do {
852 set_buffer_dirty(bh);
853 bh = bh->b_this_page;
854 } while (bh != head);
855 }
856 spin_unlock(&mapping->private_lock);
858 if (!TestSetPageDirty(page)) {
859 write_lock_irq(&mapping->tree_lock);
860 if (page->mapping) { /* Race with truncate? */
861 if (mapping_cap_account_dirty(mapping))
862 __inc_zone_page_state(page, NR_FILE_DIRTY);
863 radix_tree_tag_set(&mapping->page_tree,
864 page_index(page),
865 PAGECACHE_TAG_DIRTY);
866 }
867 write_unlock_irq(&mapping->tree_lock);
868 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
869 return 1;
870 }
871 return 0;
872 }
873 EXPORT_SYMBOL(__set_page_dirty_buffers);
875 /*
876 * Write out and wait upon a list of buffers.
877 *
878 * We have conflicting pressures: we want to make sure that all
879 * initially dirty buffers get waited on, but that any subsequently
880 * dirtied buffers don't. After all, we don't want fsync to last
881 * forever if somebody is actively writing to the file.
882 *
883 * Do this in two main stages: first we copy dirty buffers to a
884 * temporary inode list, queueing the writes as we go. Then we clean
885 * up, waiting for those writes to complete.
886 *
887 * During this second stage, any subsequent updates to the file may end
888 * up refiling the buffer on the original inode's dirty list again, so
889 * there is a chance we will end up with a buffer queued for write but
890 * not yet completed on that list. So, as a final cleanup we go through
891 * the osync code to catch these locked, dirty buffers without requeuing
892 * any newly dirty buffers for write.
893 */
894 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
895 {
896 struct buffer_head *bh;
897 struct list_head tmp;
898 int err = 0, err2;
900 INIT_LIST_HEAD(&tmp);
902 spin_lock(lock);
903 while (!list_empty(list)) {
904 bh = BH_ENTRY(list->next);
905 list_del_init(&bh->b_assoc_buffers);
906 if (buffer_dirty(bh) || buffer_locked(bh)) {
907 list_add(&bh->b_assoc_buffers, &tmp);
908 if (buffer_dirty(bh)) {
909 get_bh(bh);
910 spin_unlock(lock);
911 /*
912 * Ensure any pending I/O completes so that
913 * ll_rw_block() actually writes the current
914 * contents - it is a noop if I/O is still in
915 * flight on potentially older contents.
916 */
917 ll_rw_block(SWRITE, 1, &bh);
918 brelse(bh);
919 spin_lock(lock);
920 }
921 }
922 }
924 while (!list_empty(&tmp)) {
925 bh = BH_ENTRY(tmp.prev);
926 __remove_assoc_queue(bh);
927 get_bh(bh);
928 spin_unlock(lock);
929 wait_on_buffer(bh);
930 if (!buffer_uptodate(bh))
931 err = -EIO;
932 brelse(bh);
933 spin_lock(lock);
934 }
936 spin_unlock(lock);
937 err2 = osync_buffers_list(lock, list);
938 if (err)
939 return err;
940 else
941 return err2;
942 }
944 /*
945 * Invalidate any and all dirty buffers on a given inode. We are
946 * probably unmounting the fs, but that doesn't mean we have already
947 * done a sync(). Just drop the buffers from the inode list.
948 *
949 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
950 * assumes that all the buffers are against the blockdev. Not true
951 * for reiserfs.
952 */
953 void invalidate_inode_buffers(struct inode *inode)
954 {
955 if (inode_has_buffers(inode)) {
956 struct address_space *mapping = &inode->i_data;
957 struct list_head *list = &mapping->private_list;
958 struct address_space *buffer_mapping = mapping->assoc_mapping;
960 spin_lock(&buffer_mapping->private_lock);
961 while (!list_empty(list))
962 __remove_assoc_queue(BH_ENTRY(list->next));
963 spin_unlock(&buffer_mapping->private_lock);
964 }
965 }
967 /*
968 * Remove any clean buffers from the inode's buffer list. This is called
969 * when we're trying to free the inode itself. Those buffers can pin it.
970 *
971 * Returns true if all buffers were removed.
972 */
973 int remove_inode_buffers(struct inode *inode)
974 {
975 int ret = 1;
977 if (inode_has_buffers(inode)) {
978 struct address_space *mapping = &inode->i_data;
979 struct list_head *list = &mapping->private_list;
980 struct address_space *buffer_mapping = mapping->assoc_mapping;
982 spin_lock(&buffer_mapping->private_lock);
983 while (!list_empty(list)) {
984 struct buffer_head *bh = BH_ENTRY(list->next);
985 if (buffer_dirty(bh)) {
986 ret = 0;
987 break;
988 }
989 __remove_assoc_queue(bh);
990 }
991 spin_unlock(&buffer_mapping->private_lock);
992 }
993 return ret;
994 }
996 /*
997 * Create the appropriate buffers when given a page for data area and
998 * the size of each buffer.. Use the bh->b_this_page linked list to
999 * follow the buffers created. Return NULL if unable to create more
1000 * buffers.
1002 * The retry flag is used to differentiate async IO (paging, swapping)
1003 * which may not fail from ordinary buffer allocations.
1004 */
1005 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
1006 int retry)
1008 struct buffer_head *bh, *head;
1009 long offset;
1011 try_again:
1012 head = NULL;
1013 offset = PAGE_SIZE;
1014 while ((offset -= size) >= 0) {
1015 bh = alloc_buffer_head(GFP_NOFS);
1016 if (!bh)
1017 goto no_grow;
1019 bh->b_bdev = NULL;
1020 bh->b_this_page = head;
1021 bh->b_blocknr = -1;
1022 head = bh;
1024 bh->b_state = 0;
1025 atomic_set(&bh->b_count, 0);
1026 bh->b_private = NULL;
1027 bh->b_size = size;
1029 /* Link the buffer to its page */
1030 set_bh_page(bh, page, offset);
1032 init_buffer(bh, NULL, NULL);
1034 return head;
1035 /*
1036 * In case anything failed, we just free everything we got.
1037 */
1038 no_grow:
1039 if (head) {
1040 do {
1041 bh = head;
1042 head = head->b_this_page;
1043 free_buffer_head(bh);
1044 } while (head);
1047 /*
1048 * Return failure for non-async IO requests. Async IO requests
1049 * are not allowed to fail, so we have to wait until buffer heads
1050 * become available. But we don't want tasks sleeping with
1051 * partially complete buffers, so all were released above.
1052 */
1053 if (!retry)
1054 return NULL;
1056 /* We're _really_ low on memory. Now we just
1057 * wait for old buffer heads to become free due to
1058 * finishing IO. Since this is an async request and
1059 * the reserve list is empty, we're sure there are
1060 * async buffer heads in use.
1061 */
1062 free_more_memory();
1063 goto try_again;
1065 EXPORT_SYMBOL_GPL(alloc_page_buffers);
1067 static inline void
1068 link_dev_buffers(struct page *page, struct buffer_head *head)
1070 struct buffer_head *bh, *tail;
1072 bh = head;
1073 do {
1074 tail = bh;
1075 bh = bh->b_this_page;
1076 } while (bh);
1077 tail->b_this_page = head;
1078 attach_page_buffers(page, head);
1081 /*
1082 * Initialise the state of a blockdev page's buffers.
1083 */
1084 static void
1085 init_page_buffers(struct page *page, struct block_device *bdev,
1086 sector_t block, int size)
1088 struct buffer_head *head = page_buffers(page);
1089 struct buffer_head *bh = head;
1090 int uptodate = PageUptodate(page);
1092 do {
1093 if (!buffer_mapped(bh)) {
1094 init_buffer(bh, NULL, NULL);
1095 bh->b_bdev = bdev;
1096 bh->b_blocknr = block;
1097 if (uptodate)
1098 set_buffer_uptodate(bh);
1099 set_buffer_mapped(bh);
1101 block++;
1102 bh = bh->b_this_page;
1103 } while (bh != head);
1106 /*
1107 * Create the page-cache page that contains the requested block.
1109 * This is user purely for blockdev mappings.
1110 */
1111 static struct page *
1112 grow_dev_page(struct block_device *bdev, sector_t block,
1113 pgoff_t index, int size)
1115 struct inode *inode = bdev->bd_inode;
1116 struct page *page;
1117 struct buffer_head *bh;
1119 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
1120 if (!page)
1121 return NULL;
1123 BUG_ON(!PageLocked(page));
1125 if (page_has_buffers(page)) {
1126 bh = page_buffers(page);
1127 if (bh->b_size == size) {
1128 init_page_buffers(page, bdev, block, size);
1129 return page;
1131 if (!try_to_free_buffers(page))
1132 goto failed;
1135 /*
1136 * Allocate some buffers for this page
1137 */
1138 bh = alloc_page_buffers(page, size, 0);
1139 if (!bh)
1140 goto failed;
1142 /*
1143 * Link the page to the buffers and initialise them. Take the
1144 * lock to be atomic wrt __find_get_block(), which does not
1145 * run under the page lock.
1146 */
1147 spin_lock(&inode->i_mapping->private_lock);
1148 link_dev_buffers(page, bh);
1149 init_page_buffers(page, bdev, block, size);
1150 spin_unlock(&inode->i_mapping->private_lock);
1151 return page;
1153 failed:
1154 BUG();
1155 unlock_page(page);
1156 page_cache_release(page);
1157 return NULL;
1160 /*
1161 * Create buffers for the specified block device block's page. If
1162 * that page was dirty, the buffers are set dirty also.
1164 * Except that's a bug. Attaching dirty buffers to a dirty
1165 * blockdev's page can result in filesystem corruption, because
1166 * some of those buffers may be aliases of filesystem data.
1167 * grow_dev_page() will go BUG() if this happens.
1168 */
1169 static int
1170 grow_buffers(struct block_device *bdev, sector_t block, int size)
1172 struct page *page;
1173 pgoff_t index;
1174 int sizebits;
1176 sizebits = -1;
1177 do {
1178 sizebits++;
1179 } while ((size << sizebits) < PAGE_SIZE);
1181 index = block >> sizebits;
1183 /*
1184 * Check for a block which wants to lie outside our maximum possible
1185 * pagecache index. (this comparison is done using sector_t types).
1186 */
1187 if (unlikely(index != block >> sizebits)) {
1188 char b[BDEVNAME_SIZE];
1190 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1191 "device %s\n",
1192 __FUNCTION__, (unsigned long long)block,
1193 bdevname(bdev, b));
1194 return -EIO;
1196 block = index << sizebits;
1197 /* Create a page with the proper size buffers.. */
1198 page = grow_dev_page(bdev, block, index, size);
1199 if (!page)
1200 return 0;
1201 unlock_page(page);
1202 page_cache_release(page);
1203 return 1;
1206 static struct buffer_head *
1207 __getblk_slow(struct block_device *bdev, sector_t block, int size)
1209 /* Size must be multiple of hard sectorsize */
1210 if (unlikely(size & (bdev_hardsect_size(bdev)-1) ||
1211 (size < 512 || size > PAGE_SIZE))) {
1212 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1213 size);
1214 printk(KERN_ERR "hardsect size: %d\n",
1215 bdev_hardsect_size(bdev));
1217 dump_stack();
1218 return NULL;
1221 for (;;) {
1222 struct buffer_head * bh;
1223 int ret;
1225 bh = __find_get_block(bdev, block, size);
1226 if (bh)
1227 return bh;
1229 ret = grow_buffers(bdev, block, size);
1230 if (ret < 0)
1231 return NULL;
1232 if (ret == 0)
1233 free_more_memory();
1237 /*
1238 * The relationship between dirty buffers and dirty pages:
1240 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1241 * the page is tagged dirty in its radix tree.
1243 * At all times, the dirtiness of the buffers represents the dirtiness of
1244 * subsections of the page. If the page has buffers, the page dirty bit is
1245 * merely a hint about the true dirty state.
1247 * When a page is set dirty in its entirety, all its buffers are marked dirty
1248 * (if the page has buffers).
1250 * When a buffer is marked dirty, its page is dirtied, but the page's other
1251 * buffers are not.
1253 * Also. When blockdev buffers are explicitly read with bread(), they
1254 * individually become uptodate. But their backing page remains not
1255 * uptodate - even if all of its buffers are uptodate. A subsequent
1256 * block_read_full_page() against that page will discover all the uptodate
1257 * buffers, will set the page uptodate and will perform no I/O.
1258 */
1260 /**
1261 * mark_buffer_dirty - mark a buffer_head as needing writeout
1262 * @bh: the buffer_head to mark dirty
1264 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1265 * backing page dirty, then tag the page as dirty in its address_space's radix
1266 * tree and then attach the address_space's inode to its superblock's dirty
1267 * inode list.
1269 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1270 * mapping->tree_lock and the global inode_lock.
1271 */
1272 void fastcall mark_buffer_dirty(struct buffer_head *bh)
1274 if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh))
1275 __set_page_dirty_nobuffers(bh->b_page);
1278 /*
1279 * Decrement a buffer_head's reference count. If all buffers against a page
1280 * have zero reference count, are clean and unlocked, and if the page is clean
1281 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1282 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1283 * a page but it ends up not being freed, and buffers may later be reattached).
1284 */
1285 void __brelse(struct buffer_head * buf)
1287 if (atomic_read(&buf->b_count)) {
1288 put_bh(buf);
1289 return;
1291 printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1292 WARN_ON(1);
1295 /*
1296 * bforget() is like brelse(), except it discards any
1297 * potentially dirty data.
1298 */
1299 void __bforget(struct buffer_head *bh)
1301 clear_buffer_dirty(bh);
1302 if (!list_empty(&bh->b_assoc_buffers)) {
1303 struct address_space *buffer_mapping = bh->b_page->mapping;
1305 spin_lock(&buffer_mapping->private_lock);
1306 list_del_init(&bh->b_assoc_buffers);
1307 spin_unlock(&buffer_mapping->private_lock);
1309 __brelse(bh);
1312 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1314 lock_buffer(bh);
1315 if (buffer_uptodate(bh)) {
1316 unlock_buffer(bh);
1317 return bh;
1318 } else {
1319 get_bh(bh);
1320 bh->b_end_io = end_buffer_read_sync;
1321 submit_bh(READ, bh);
1322 wait_on_buffer(bh);
1323 if (buffer_uptodate(bh))
1324 return bh;
1326 brelse(bh);
1327 return NULL;
1330 /*
1331 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1332 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1333 * refcount elevated by one when they're in an LRU. A buffer can only appear
1334 * once in a particular CPU's LRU. A single buffer can be present in multiple
1335 * CPU's LRUs at the same time.
1337 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1338 * sb_find_get_block().
1340 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1341 * a local interrupt disable for that.
1342 */
1344 #define BH_LRU_SIZE 8
1346 struct bh_lru {
1347 struct buffer_head *bhs[BH_LRU_SIZE];
1348 };
1350 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1352 #ifdef CONFIG_SMP
1353 #define bh_lru_lock() local_irq_disable()
1354 #define bh_lru_unlock() local_irq_enable()
1355 #else
1356 #define bh_lru_lock() preempt_disable()
1357 #define bh_lru_unlock() preempt_enable()
1358 #endif
1360 static inline void check_irqs_on(void)
1362 #ifdef irqs_disabled
1363 BUG_ON(irqs_disabled());
1364 #endif
1367 /*
1368 * The LRU management algorithm is dopey-but-simple. Sorry.
1369 */
1370 static void bh_lru_install(struct buffer_head *bh)
1372 struct buffer_head *evictee = NULL;
1373 struct bh_lru *lru;
1375 check_irqs_on();
1376 bh_lru_lock();
1377 lru = &__get_cpu_var(bh_lrus);
1378 if (lru->bhs[0] != bh) {
1379 struct buffer_head *bhs[BH_LRU_SIZE];
1380 int in;
1381 int out = 0;
1383 get_bh(bh);
1384 bhs[out++] = bh;
1385 for (in = 0; in < BH_LRU_SIZE; in++) {
1386 struct buffer_head *bh2 = lru->bhs[in];
1388 if (bh2 == bh) {
1389 __brelse(bh2);
1390 } else {
1391 if (out >= BH_LRU_SIZE) {
1392 BUG_ON(evictee != NULL);
1393 evictee = bh2;
1394 } else {
1395 bhs[out++] = bh2;
1399 while (out < BH_LRU_SIZE)
1400 bhs[out++] = NULL;
1401 memcpy(lru->bhs, bhs, sizeof(bhs));
1403 bh_lru_unlock();
1405 if (evictee)
1406 __brelse(evictee);
1409 /*
1410 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1411 */
1412 static struct buffer_head *
1413 lookup_bh_lru(struct block_device *bdev, sector_t block, int size)
1415 struct buffer_head *ret = NULL;
1416 struct bh_lru *lru;
1417 int i;
1419 check_irqs_on();
1420 bh_lru_lock();
1421 lru = &__get_cpu_var(bh_lrus);
1422 for (i = 0; i < BH_LRU_SIZE; i++) {
1423 struct buffer_head *bh = lru->bhs[i];
1425 if (bh && bh->b_bdev == bdev &&
1426 bh->b_blocknr == block && bh->b_size == size) {
1427 if (i) {
1428 while (i) {
1429 lru->bhs[i] = lru->bhs[i - 1];
1430 i--;
1432 lru->bhs[0] = bh;
1434 get_bh(bh);
1435 ret = bh;
1436 break;
1439 bh_lru_unlock();
1440 return ret;
1443 /*
1444 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1445 * it in the LRU and mark it as accessed. If it is not present then return
1446 * NULL
1447 */
1448 struct buffer_head *
1449 __find_get_block(struct block_device *bdev, sector_t block, int size)
1451 struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1453 if (bh == NULL) {
1454 bh = __find_get_block_slow(bdev, block);
1455 if (bh)
1456 bh_lru_install(bh);
1458 if (bh)
1459 touch_buffer(bh);
1460 return bh;
1462 EXPORT_SYMBOL(__find_get_block);
1464 /*
1465 * __getblk will locate (and, if necessary, create) the buffer_head
1466 * which corresponds to the passed block_device, block and size. The
1467 * returned buffer has its reference count incremented.
1469 * __getblk() cannot fail - it just keeps trying. If you pass it an
1470 * illegal block number, __getblk() will happily return a buffer_head
1471 * which represents the non-existent block. Very weird.
1473 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1474 * attempt is failing. FIXME, perhaps?
1475 */
1476 struct buffer_head *
1477 __getblk(struct block_device *bdev, sector_t block, int size)
1479 struct buffer_head *bh = __find_get_block(bdev, block, size);
1481 might_sleep();
1482 if (bh == NULL)
1483 bh = __getblk_slow(bdev, block, size);
1484 return bh;
1486 EXPORT_SYMBOL(__getblk);
1488 /*
1489 * Do async read-ahead on a buffer..
1490 */
1491 void __breadahead(struct block_device *bdev, sector_t block, int size)
1493 struct buffer_head *bh = __getblk(bdev, block, size);
1494 if (likely(bh)) {
1495 ll_rw_block(READA, 1, &bh);
1496 brelse(bh);
1499 EXPORT_SYMBOL(__breadahead);
1501 /**
1502 * __bread() - reads a specified block and returns the bh
1503 * @bdev: the block_device to read from
1504 * @block: number of block
1505 * @size: size (in bytes) to read
1507 * Reads a specified block, and returns buffer head that contains it.
1508 * It returns NULL if the block was unreadable.
1509 */
1510 struct buffer_head *
1511 __bread(struct block_device *bdev, sector_t block, int size)
1513 struct buffer_head *bh = __getblk(bdev, block, size);
1515 if (likely(bh) && !buffer_uptodate(bh))
1516 bh = __bread_slow(bh);
1517 return bh;
1519 EXPORT_SYMBOL(__bread);
1521 /*
1522 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1523 * This doesn't race because it runs in each cpu either in irq
1524 * or with preempt disabled.
1525 */
1526 static void invalidate_bh_lru(void *arg)
1528 struct bh_lru *b = &get_cpu_var(bh_lrus);
1529 int i;
1531 for (i = 0; i < BH_LRU_SIZE; i++) {
1532 brelse(b->bhs[i]);
1533 b->bhs[i] = NULL;
1535 put_cpu_var(bh_lrus);
1538 static void invalidate_bh_lrus(void)
1540 on_each_cpu(invalidate_bh_lru, NULL, 1, 1);
1543 void set_bh_page(struct buffer_head *bh,
1544 struct page *page, unsigned long offset)
1546 bh->b_page = page;
1547 BUG_ON(offset >= PAGE_SIZE);
1548 if (PageHighMem(page))
1549 /*
1550 * This catches illegal uses and preserves the offset:
1551 */
1552 bh->b_data = (char *)(0 + offset);
1553 else
1554 bh->b_data = page_address(page) + offset;
1556 EXPORT_SYMBOL(set_bh_page);
1558 /*
1559 * Called when truncating a buffer on a page completely.
1560 */
1561 static void discard_buffer(struct buffer_head * bh)
1563 lock_buffer(bh);
1564 clear_buffer_dirty(bh);
1565 bh->b_bdev = NULL;
1566 clear_buffer_mapped(bh);
1567 clear_buffer_req(bh);
1568 clear_buffer_new(bh);
1569 clear_buffer_delay(bh);
1570 unlock_buffer(bh);
1573 /**
1574 * try_to_release_page() - release old fs-specific metadata on a page
1576 * @page: the page which the kernel is trying to free
1577 * @gfp_mask: memory allocation flags (and I/O mode)
1579 * The address_space is to try to release any data against the page
1580 * (presumably at page->private). If the release was successful, return `1'.
1581 * Otherwise return zero.
1583 * The @gfp_mask argument specifies whether I/O may be performed to release
1584 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
1586 * NOTE: @gfp_mask may go away, and this function may become non-blocking.
1587 */
1588 int try_to_release_page(struct page *page, gfp_t gfp_mask)
1590 struct address_space * const mapping = page->mapping;
1592 BUG_ON(!PageLocked(page));
1593 if (PageWriteback(page))
1594 return 0;
1596 if (mapping && mapping->a_ops->releasepage)
1597 return mapping->a_ops->releasepage(page, gfp_mask);
1598 return try_to_free_buffers(page);
1600 EXPORT_SYMBOL(try_to_release_page);
1602 /**
1603 * block_invalidatepage - invalidate part of all of a buffer-backed page
1605 * @page: the page which is affected
1606 * @offset: the index of the truncation point
1608 * block_invalidatepage() is called when all or part of the page has become
1609 * invalidatedby a truncate operation.
1611 * block_invalidatepage() does not have to release all buffers, but it must
1612 * ensure that no dirty buffer is left outside @offset and that no I/O
1613 * is underway against any of the blocks which are outside the truncation
1614 * point. Because the caller is about to free (and possibly reuse) those
1615 * blocks on-disk.
1616 */
1617 void block_invalidatepage(struct page *page, unsigned long offset)
1619 struct buffer_head *head, *bh, *next;
1620 unsigned int curr_off = 0;
1622 BUG_ON(!PageLocked(page));
1623 if (!page_has_buffers(page))
1624 goto out;
1626 head = page_buffers(page);
1627 bh = head;
1628 do {
1629 unsigned int next_off = curr_off + bh->b_size;
1630 next = bh->b_this_page;
1632 /*
1633 * is this block fully invalidated?
1634 */
1635 if (offset <= curr_off)
1636 discard_buffer(bh);
1637 curr_off = next_off;
1638 bh = next;
1639 } while (bh != head);
1641 /*
1642 * We release buffers only if the entire page is being invalidated.
1643 * The get_block cached value has been unconditionally invalidated,
1644 * so real IO is not possible anymore.
1645 */
1646 if (offset == 0)
1647 try_to_release_page(page, 0);
1648 out:
1649 return;
1651 EXPORT_SYMBOL(block_invalidatepage);
1653 void do_invalidatepage(struct page *page, unsigned long offset)
1655 void (*invalidatepage)(struct page *, unsigned long);
1656 invalidatepage = page->mapping->a_ops->invalidatepage ? :
1657 block_invalidatepage;
1658 (*invalidatepage)(page, offset);
1661 /*
1662 * We attach and possibly dirty the buffers atomically wrt
1663 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1664 * is already excluded via the page lock.
1665 */
1666 void create_empty_buffers(struct page *page,
1667 unsigned long blocksize, unsigned long b_state)
1669 struct buffer_head *bh, *head, *tail;
1671 head = alloc_page_buffers(page, blocksize, 1);
1672 bh = head;
1673 do {
1674 bh->b_state |= b_state;
1675 tail = bh;
1676 bh = bh->b_this_page;
1677 } while (bh);
1678 tail->b_this_page = head;
1680 spin_lock(&page->mapping->private_lock);
1681 if (PageUptodate(page) || PageDirty(page)) {
1682 bh = head;
1683 do {
1684 if (PageDirty(page))
1685 set_buffer_dirty(bh);
1686 if (PageUptodate(page))
1687 set_buffer_uptodate(bh);
1688 bh = bh->b_this_page;
1689 } while (bh != head);
1691 attach_page_buffers(page, head);
1692 spin_unlock(&page->mapping->private_lock);
1694 EXPORT_SYMBOL(create_empty_buffers);
1696 /*
1697 * We are taking a block for data and we don't want any output from any
1698 * buffer-cache aliases starting from return from that function and
1699 * until the moment when something will explicitly mark the buffer
1700 * dirty (hopefully that will not happen until we will free that block ;-)
1701 * We don't even need to mark it not-uptodate - nobody can expect
1702 * anything from a newly allocated buffer anyway. We used to used
1703 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1704 * don't want to mark the alias unmapped, for example - it would confuse
1705 * anyone who might pick it with bread() afterwards...
1707 * Also.. Note that bforget() doesn't lock the buffer. So there can
1708 * be writeout I/O going on against recently-freed buffers. We don't
1709 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1710 * only if we really need to. That happens here.
1711 */
1712 void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1714 struct buffer_head *old_bh;
1716 might_sleep();
1718 old_bh = __find_get_block_slow(bdev, block);
1719 if (old_bh) {
1720 clear_buffer_dirty(old_bh);
1721 wait_on_buffer(old_bh);
1722 clear_buffer_req(old_bh);
1723 __brelse(old_bh);
1726 EXPORT_SYMBOL(unmap_underlying_metadata);
1728 /*
1729 * NOTE! All mapped/uptodate combinations are valid:
1731 * Mapped Uptodate Meaning
1733 * No No "unknown" - must do get_block()
1734 * No Yes "hole" - zero-filled
1735 * Yes No "allocated" - allocated on disk, not read in
1736 * Yes Yes "valid" - allocated and up-to-date in memory.
1738 * "Dirty" is valid only with the last case (mapped+uptodate).
1739 */
1741 /*
1742 * While block_write_full_page is writing back the dirty buffers under
1743 * the page lock, whoever dirtied the buffers may decide to clean them
1744 * again at any time. We handle that by only looking at the buffer
1745 * state inside lock_buffer().
1747 * If block_write_full_page() is called for regular writeback
1748 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1749 * locked buffer. This only can happen if someone has written the buffer
1750 * directly, with submit_bh(). At the address_space level PageWriteback
1751 * prevents this contention from occurring.
1752 */
1753 static int __block_write_full_page(struct inode *inode, struct page *page,
1754 get_block_t *get_block, struct writeback_control *wbc)
1756 int err;
1757 sector_t block;
1758 sector_t last_block;
1759 struct buffer_head *bh, *head;
1760 const unsigned blocksize = 1 << inode->i_blkbits;
1761 int nr_underway = 0;
1763 BUG_ON(!PageLocked(page));
1765 last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1767 if (!page_has_buffers(page)) {
1768 create_empty_buffers(page, blocksize,
1769 (1 << BH_Dirty)|(1 << BH_Uptodate));
1772 /*
1773 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1774 * here, and the (potentially unmapped) buffers may become dirty at
1775 * any time. If a buffer becomes dirty here after we've inspected it
1776 * then we just miss that fact, and the page stays dirty.
1778 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1779 * handle that here by just cleaning them.
1780 */
1782 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1783 head = page_buffers(page);
1784 bh = head;
1786 /*
1787 * Get all the dirty buffers mapped to disk addresses and
1788 * handle any aliases from the underlying blockdev's mapping.
1789 */
1790 do {
1791 if (block > last_block) {
1792 /*
1793 * mapped buffers outside i_size will occur, because
1794 * this page can be outside i_size when there is a
1795 * truncate in progress.
1796 */
1797 /*
1798 * The buffer was zeroed by block_write_full_page()
1799 */
1800 clear_buffer_dirty(bh);
1801 set_buffer_uptodate(bh);
1802 } else if (!buffer_mapped(bh) && buffer_dirty(bh)) {
1803 WARN_ON(bh->b_size != blocksize);
1804 err = get_block(inode, block, bh, 1);
1805 if (err)
1806 goto recover;
1807 if (buffer_new(bh)) {
1808 /* blockdev mappings never come here */
1809 clear_buffer_new(bh);
1810 unmap_underlying_metadata(bh->b_bdev,
1811 bh->b_blocknr);
1814 bh = bh->b_this_page;
1815 block++;
1816 } while (bh != head);
1818 do {
1819 if (!buffer_mapped(bh))
1820 continue;
1821 /*
1822 * If it's a fully non-blocking write attempt and we cannot
1823 * lock the buffer then redirty the page. Note that this can
1824 * potentially cause a busy-wait loop from pdflush and kswapd
1825 * activity, but those code paths have their own higher-level
1826 * throttling.
1827 */
1828 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1829 lock_buffer(bh);
1830 } else if (test_set_buffer_locked(bh)) {
1831 redirty_page_for_writepage(wbc, page);
1832 continue;
1834 if (test_clear_buffer_dirty(bh)) {
1835 mark_buffer_async_write(bh);
1836 } else {
1837 unlock_buffer(bh);
1839 } while ((bh = bh->b_this_page) != head);
1841 /*
1842 * The page and its buffers are protected by PageWriteback(), so we can
1843 * drop the bh refcounts early.
1844 */
1845 BUG_ON(PageWriteback(page));
1846 set_page_writeback(page);
1848 do {
1849 struct buffer_head *next = bh->b_this_page;
1850 if (buffer_async_write(bh)) {
1851 submit_bh(WRITE, bh);
1852 nr_underway++;
1854 bh = next;
1855 } while (bh != head);
1856 unlock_page(page);
1858 err = 0;
1859 done:
1860 if (nr_underway == 0) {
1861 /*
1862 * The page was marked dirty, but the buffers were
1863 * clean. Someone wrote them back by hand with
1864 * ll_rw_block/submit_bh. A rare case.
1865 */
1866 int uptodate = 1;
1867 do {
1868 if (!buffer_uptodate(bh)) {
1869 uptodate = 0;
1870 break;
1872 bh = bh->b_this_page;
1873 } while (bh != head);
1874 if (uptodate)
1875 SetPageUptodate(page);
1876 end_page_writeback(page);
1877 /*
1878 * The page and buffer_heads can be released at any time from
1879 * here on.
1880 */
1881 wbc->pages_skipped++; /* We didn't write this page */
1883 return err;
1885 recover:
1886 /*
1887 * ENOSPC, or some other error. We may already have added some
1888 * blocks to the file, so we need to write these out to avoid
1889 * exposing stale data.
1890 * The page is currently locked and not marked for writeback
1891 */
1892 bh = head;
1893 /* Recovery: lock and submit the mapped buffers */
1894 do {
1895 if (buffer_mapped(bh) && buffer_dirty(bh)) {
1896 lock_buffer(bh);
1897 mark_buffer_async_write(bh);
1898 } else {
1899 /*
1900 * The buffer may have been set dirty during
1901 * attachment to a dirty page.
1902 */
1903 clear_buffer_dirty(bh);
1905 } while ((bh = bh->b_this_page) != head);
1906 SetPageError(page);
1907 BUG_ON(PageWriteback(page));
1908 set_page_writeback(page);
1909 unlock_page(page);
1910 do {
1911 struct buffer_head *next = bh->b_this_page;
1912 if (buffer_async_write(bh)) {
1913 clear_buffer_dirty(bh);
1914 submit_bh(WRITE, bh);
1915 nr_underway++;
1917 bh = next;
1918 } while (bh != head);
1919 goto done;
1922 static int __block_prepare_write(struct inode *inode, struct page *page,
1923 unsigned from, unsigned to, get_block_t *get_block)
1925 unsigned block_start, block_end;
1926 sector_t block;
1927 int err = 0;
1928 unsigned blocksize, bbits;
1929 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1931 BUG_ON(!PageLocked(page));
1932 BUG_ON(from > PAGE_CACHE_SIZE);
1933 BUG_ON(to > PAGE_CACHE_SIZE);
1934 BUG_ON(from > to);
1936 blocksize = 1 << inode->i_blkbits;
1937 if (!page_has_buffers(page))
1938 create_empty_buffers(page, blocksize, 0);
1939 head = page_buffers(page);
1941 bbits = inode->i_blkbits;
1942 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1944 for(bh = head, block_start = 0; bh != head || !block_start;
1945 block++, block_start=block_end, bh = bh->b_this_page) {
1946 block_end = block_start + blocksize;
1947 if (block_end <= from || block_start >= to) {
1948 if (PageUptodate(page)) {
1949 if (!buffer_uptodate(bh))
1950 set_buffer_uptodate(bh);
1952 continue;
1954 if (buffer_new(bh))
1955 clear_buffer_new(bh);
1956 if (!buffer_mapped(bh)) {
1957 WARN_ON(bh->b_size != blocksize);
1958 err = get_block(inode, block, bh, 1);
1959 if (err)
1960 break;
1961 if (buffer_new(bh)) {
1962 unmap_underlying_metadata(bh->b_bdev,
1963 bh->b_blocknr);
1964 if (PageUptodate(page)) {
1965 set_buffer_uptodate(bh);
1966 continue;
1968 if (block_end > to || block_start < from) {
1969 void *kaddr;
1971 kaddr = kmap_atomic(page, KM_USER0);
1972 if (block_end > to)
1973 memset(kaddr+to, 0,
1974 block_end-to);
1975 if (block_start < from)
1976 memset(kaddr+block_start,
1977 0, from-block_start);
1978 flush_dcache_page(page);
1979 kunmap_atomic(kaddr, KM_USER0);
1981 continue;
1984 if (PageUptodate(page)) {
1985 if (!buffer_uptodate(bh))
1986 set_buffer_uptodate(bh);
1987 continue;
1989 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1990 (block_start < from || block_end > to)) {
1991 ll_rw_block(READ, 1, &bh);
1992 *wait_bh++=bh;
1995 /*
1996 * If we issued read requests - let them complete.
1997 */
1998 while(wait_bh > wait) {
1999 wait_on_buffer(*--wait_bh);
2000 if (!buffer_uptodate(*wait_bh))
2001 err = -EIO;
2003 if (!err) {
2004 bh = head;
2005 do {
2006 if (buffer_new(bh))
2007 clear_buffer_new(bh);
2008 } while ((bh = bh->b_this_page) != head);
2009 return 0;
2011 /* Error case: */
2012 /*
2013 * Zero out any newly allocated blocks to avoid exposing stale
2014 * data. If BH_New is set, we know that the block was newly
2015 * allocated in the above loop.
2016 */
2017 bh = head;
2018 block_start = 0;
2019 do {
2020 block_end = block_start+blocksize;
2021 if (block_end <= from)
2022 goto next_bh;
2023 if (block_start >= to)
2024 break;
2025 if (buffer_new(bh)) {
2026 void *kaddr;
2028 clear_buffer_new(bh);
2029 kaddr = kmap_atomic(page, KM_USER0);
2030 memset(kaddr+block_start, 0, bh->b_size);
2031 kunmap_atomic(kaddr, KM_USER0);
2032 set_buffer_uptodate(bh);
2033 mark_buffer_dirty(bh);
2035 next_bh:
2036 block_start = block_end;
2037 bh = bh->b_this_page;
2038 } while (bh != head);
2039 return err;
2042 static int __block_commit_write(struct inode *inode, struct page *page,
2043 unsigned from, unsigned to)
2045 unsigned block_start, block_end;
2046 int partial = 0;
2047 unsigned blocksize;
2048 struct buffer_head *bh, *head;
2050 blocksize = 1 << inode->i_blkbits;
2052 for(bh = head = page_buffers(page), block_start = 0;
2053 bh != head || !block_start;
2054 block_start=block_end, bh = bh->b_this_page) {
2055 block_end = block_start + blocksize;
2056 if (block_end <= from || block_start >= to) {
2057 if (!buffer_uptodate(bh))
2058 partial = 1;
2059 } else {
2060 set_buffer_uptodate(bh);
2061 mark_buffer_dirty(bh);
2065 /*
2066 * If this is a partial write which happened to make all buffers
2067 * uptodate then we can optimize away a bogus readpage() for
2068 * the next read(). Here we 'discover' whether the page went
2069 * uptodate as a result of this (potentially partial) write.
2070 */
2071 if (!partial)
2072 SetPageUptodate(page);
2073 return 0;
2076 /*
2077 * Generic "read page" function for block devices that have the normal
2078 * get_block functionality. This is most of the block device filesystems.
2079 * Reads the page asynchronously --- the unlock_buffer() and
2080 * set/clear_buffer_uptodate() functions propagate buffer state into the
2081 * page struct once IO has completed.
2082 */
2083 int block_read_full_page(struct page *page, get_block_t *get_block)
2085 struct inode *inode = page->mapping->host;
2086 sector_t iblock, lblock;
2087 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2088 unsigned int blocksize;
2089 int nr, i;
2090 int fully_mapped = 1;
2092 BUG_ON(!PageLocked(page));
2093 blocksize = 1 << inode->i_blkbits;
2094 if (!page_has_buffers(page))
2095 create_empty_buffers(page, blocksize, 0);
2096 head = page_buffers(page);
2098 iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2099 lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2100 bh = head;
2101 nr = 0;
2102 i = 0;
2104 do {
2105 if (buffer_uptodate(bh))
2106 continue;
2108 if (!buffer_mapped(bh)) {
2109 int err = 0;
2111 fully_mapped = 0;
2112 if (iblock < lblock) {
2113 WARN_ON(bh->b_size != blocksize);
2114 err = get_block(inode, iblock, bh, 0);
2115 if (err)
2116 SetPageError(page);
2118 if (!buffer_mapped(bh)) {
2119 void *kaddr = kmap_atomic(page, KM_USER0);
2120 memset(kaddr + i * blocksize, 0, blocksize);
2121 flush_dcache_page(page);
2122 kunmap_atomic(kaddr, KM_USER0);
2123 if (!err)
2124 set_buffer_uptodate(bh);
2125 continue;
2127 /*
2128 * get_block() might have updated the buffer
2129 * synchronously
2130 */
2131 if (buffer_uptodate(bh))
2132 continue;
2134 arr[nr++] = bh;
2135 } while (i++, iblock++, (bh = bh->b_this_page) != head);
2137 if (fully_mapped)
2138 SetPageMappedToDisk(page);
2140 if (!nr) {
2141 /*
2142 * All buffers are uptodate - we can set the page uptodate
2143 * as well. But not if get_block() returned an error.
2144 */
2145 if (!PageError(page))
2146 SetPageUptodate(page);
2147 unlock_page(page);
2148 return 0;
2151 /* Stage two: lock the buffers */
2152 for (i = 0; i < nr; i++) {
2153 bh = arr[i];
2154 lock_buffer(bh);
2155 mark_buffer_async_read(bh);
2158 /*
2159 * Stage 3: start the IO. Check for uptodateness
2160 * inside the buffer lock in case another process reading
2161 * the underlying blockdev brought it uptodate (the sct fix).
2162 */
2163 for (i = 0; i < nr; i++) {
2164 bh = arr[i];
2165 if (buffer_uptodate(bh))
2166 end_buffer_async_read(bh, 1);
2167 else
2168 submit_bh(READ, bh);
2170 return 0;
2173 /* utility function for filesystems that need to do work on expanding
2174 * truncates. Uses prepare/commit_write to allow the filesystem to
2175 * deal with the hole.
2176 */
2177 static int __generic_cont_expand(struct inode *inode, loff_t size,
2178 pgoff_t index, unsigned int offset)
2180 struct address_space *mapping = inode->i_mapping;
2181 struct page *page;
2182 unsigned long limit;
2183 int err;
2185 err = -EFBIG;
2186 limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
2187 if (limit != RLIM_INFINITY && size > (loff_t)limit) {
2188 send_sig(SIGXFSZ, current, 0);
2189 goto out;
2191 if (size > inode->i_sb->s_maxbytes)
2192 goto out;
2194 err = -ENOMEM;
2195 page = grab_cache_page(mapping, index);
2196 if (!page)
2197 goto out;
2198 err = mapping->a_ops->prepare_write(NULL, page, offset, offset);
2199 if (err) {
2200 /*
2201 * ->prepare_write() may have instantiated a few blocks
2202 * outside i_size. Trim these off again.
2203 */
2204 unlock_page(page);
2205 page_cache_release(page);
2206 vmtruncate(inode, inode->i_size);
2207 goto out;
2210 err = mapping->a_ops->commit_write(NULL, page, offset, offset);
2212 unlock_page(page);
2213 page_cache_release(page);
2214 if (err > 0)
2215 err = 0;
2216 out:
2217 return err;
2220 int generic_cont_expand(struct inode *inode, loff_t size)
2222 pgoff_t index;
2223 unsigned int offset;
2225 offset = (size & (PAGE_CACHE_SIZE - 1)); /* Within page */
2227 /* ugh. in prepare/commit_write, if from==to==start of block, we
2228 ** skip the prepare. make sure we never send an offset for the start
2229 ** of a block
2230 */
2231 if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
2232 /* caller must handle this extra byte. */
2233 offset++;
2235 index = size >> PAGE_CACHE_SHIFT;
2237 return __generic_cont_expand(inode, size, index, offset);
2240 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2242 loff_t pos = size - 1;
2243 pgoff_t index = pos >> PAGE_CACHE_SHIFT;
2244 unsigned int offset = (pos & (PAGE_CACHE_SIZE - 1)) + 1;
2246 /* prepare/commit_write can handle even if from==to==start of block. */
2247 return __generic_cont_expand(inode, size, index, offset);
2250 /*
2251 * For moronic filesystems that do not allow holes in file.
2252 * We may have to extend the file.
2253 */
2255 int cont_prepare_write(struct page *page, unsigned offset,
2256 unsigned to, get_block_t *get_block, loff_t *bytes)
2258 struct address_space *mapping = page->mapping;
2259 struct inode *inode = mapping->host;
2260 struct page *new_page;
2261 pgoff_t pgpos;
2262 long status;
2263 unsigned zerofrom;
2264 unsigned blocksize = 1 << inode->i_blkbits;
2265 void *kaddr;
2267 while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) {
2268 status = -ENOMEM;
2269 new_page = grab_cache_page(mapping, pgpos);
2270 if (!new_page)
2271 goto out;
2272 /* we might sleep */
2273 if (*bytes>>PAGE_CACHE_SHIFT != pgpos) {
2274 unlock_page(new_page);
2275 page_cache_release(new_page);
2276 continue;
2278 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2279 if (zerofrom & (blocksize-1)) {
2280 *bytes |= (blocksize-1);
2281 (*bytes)++;
2283 status = __block_prepare_write(inode, new_page, zerofrom,
2284 PAGE_CACHE_SIZE, get_block);
2285 if (status)
2286 goto out_unmap;
2287 kaddr = kmap_atomic(new_page, KM_USER0);
2288 memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom);
2289 flush_dcache_page(new_page);
2290 kunmap_atomic(kaddr, KM_USER0);
2291 generic_commit_write(NULL, new_page, zerofrom, PAGE_CACHE_SIZE);
2292 unlock_page(new_page);
2293 page_cache_release(new_page);
2296 if (page->index < pgpos) {
2297 /* completely inside the area */
2298 zerofrom = offset;
2299 } else {
2300 /* page covers the boundary, find the boundary offset */
2301 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2303 /* if we will expand the thing last block will be filled */
2304 if (to > zerofrom && (zerofrom & (blocksize-1))) {
2305 *bytes |= (blocksize-1);
2306 (*bytes)++;
2309 /* starting below the boundary? Nothing to zero out */
2310 if (offset <= zerofrom)
2311 zerofrom = offset;
2313 status = __block_prepare_write(inode, page, zerofrom, to, get_block);
2314 if (status)
2315 goto out1;
2316 if (zerofrom < offset) {
2317 kaddr = kmap_atomic(page, KM_USER0);
2318 memset(kaddr+zerofrom, 0, offset-zerofrom);
2319 flush_dcache_page(page);
2320 kunmap_atomic(kaddr, KM_USER0);
2321 __block_commit_write(inode, page, zerofrom, offset);
2323 return 0;
2324 out1:
2325 ClearPageUptodate(page);
2326 return status;
2328 out_unmap:
2329 ClearPageUptodate(new_page);
2330 unlock_page(new_page);
2331 page_cache_release(new_page);
2332 out:
2333 return status;
2336 int block_prepare_write(struct page *page, unsigned from, unsigned to,
2337 get_block_t *get_block)
2339 struct inode *inode = page->mapping->host;
2340 int err = __block_prepare_write(inode, page, from, to, get_block);
2341 if (err)
2342 ClearPageUptodate(page);
2343 return err;
2346 int block_commit_write(struct page *page, unsigned from, unsigned to)
2348 struct inode *inode = page->mapping->host;
2349 __block_commit_write(inode,page,from,to);
2350 return 0;
2353 int generic_commit_write(struct file *file, struct page *page,
2354 unsigned from, unsigned to)
2356 struct inode *inode = page->mapping->host;
2357 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2358 __block_commit_write(inode,page,from,to);
2359 /*
2360 * No need to use i_size_read() here, the i_size
2361 * cannot change under us because we hold i_mutex.
2362 */
2363 if (pos > inode->i_size) {
2364 i_size_write(inode, pos);
2365 mark_inode_dirty(inode);
2367 return 0;
2371 /*
2372 * nobh_prepare_write()'s prereads are special: the buffer_heads are freed
2373 * immediately, while under the page lock. So it needs a special end_io
2374 * handler which does not touch the bh after unlocking it.
2376 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
2377 * a race there is benign: unlock_buffer() only use the bh's address for
2378 * hashing after unlocking the buffer, so it doesn't actually touch the bh
2379 * itself.
2380 */
2381 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2383 if (uptodate) {
2384 set_buffer_uptodate(bh);
2385 } else {
2386 /* This happens, due to failed READA attempts. */
2387 clear_buffer_uptodate(bh);
2389 unlock_buffer(bh);
2392 /*
2393 * On entry, the page is fully not uptodate.
2394 * On exit the page is fully uptodate in the areas outside (from,to)
2395 */
2396 int nobh_prepare_write(struct page *page, unsigned from, unsigned to,
2397 get_block_t *get_block)
2399 struct inode *inode = page->mapping->host;
2400 const unsigned blkbits = inode->i_blkbits;
2401 const unsigned blocksize = 1 << blkbits;
2402 struct buffer_head map_bh;
2403 struct buffer_head *read_bh[MAX_BUF_PER_PAGE];
2404 unsigned block_in_page;
2405 unsigned block_start;
2406 sector_t block_in_file;
2407 char *kaddr;
2408 int nr_reads = 0;
2409 int i;
2410 int ret = 0;
2411 int is_mapped_to_disk = 1;
2412 int dirtied_it = 0;
2414 if (PageMappedToDisk(page))
2415 return 0;
2417 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2418 map_bh.b_page = page;
2420 /*
2421 * We loop across all blocks in the page, whether or not they are
2422 * part of the affected region. This is so we can discover if the
2423 * page is fully mapped-to-disk.
2424 */
2425 for (block_start = 0, block_in_page = 0;
2426 block_start < PAGE_CACHE_SIZE;
2427 block_in_page++, block_start += blocksize) {
2428 unsigned block_end = block_start + blocksize;
2429 int create;
2431 map_bh.b_state = 0;
2432 create = 1;
2433 if (block_start >= to)
2434 create = 0;
2435 map_bh.b_size = blocksize;
2436 ret = get_block(inode, block_in_file + block_in_page,
2437 &map_bh, create);
2438 if (ret)
2439 goto failed;
2440 if (!buffer_mapped(&map_bh))
2441 is_mapped_to_disk = 0;
2442 if (buffer_new(&map_bh))
2443 unmap_underlying_metadata(map_bh.b_bdev,
2444 map_bh.b_blocknr);
2445 if (PageUptodate(page))
2446 continue;
2447 if (buffer_new(&map_bh) || !buffer_mapped(&map_bh)) {
2448 kaddr = kmap_atomic(page, KM_USER0);
2449 if (block_start < from) {
2450 memset(kaddr+block_start, 0, from-block_start);
2451 dirtied_it = 1;
2453 if (block_end > to) {
2454 memset(kaddr + to, 0, block_end - to);
2455 dirtied_it = 1;
2457 flush_dcache_page(page);
2458 kunmap_atomic(kaddr, KM_USER0);
2459 continue;
2461 if (buffer_uptodate(&map_bh))
2462 continue; /* reiserfs does this */
2463 if (block_start < from || block_end > to) {
2464 struct buffer_head *bh = alloc_buffer_head(GFP_NOFS);
2466 if (!bh) {
2467 ret = -ENOMEM;
2468 goto failed;
2470 bh->b_state = map_bh.b_state;
2471 atomic_set(&bh->b_count, 0);
2472 bh->b_this_page = NULL;
2473 bh->b_page = page;
2474 bh->b_blocknr = map_bh.b_blocknr;
2475 bh->b_size = blocksize;
2476 bh->b_data = (char *)(long)block_start;
2477 bh->b_bdev = map_bh.b_bdev;
2478 bh->b_private = NULL;
2479 read_bh[nr_reads++] = bh;
2483 if (nr_reads) {
2484 struct buffer_head *bh;
2486 /*
2487 * The page is locked, so these buffers are protected from
2488 * any VM or truncate activity. Hence we don't need to care
2489 * for the buffer_head refcounts.
2490 */
2491 for (i = 0; i < nr_reads; i++) {
2492 bh = read_bh[i];
2493 lock_buffer(bh);
2494 bh->b_end_io = end_buffer_read_nobh;
2495 submit_bh(READ, bh);
2497 for (i = 0; i < nr_reads; i++) {
2498 bh = read_bh[i];
2499 wait_on_buffer(bh);
2500 if (!buffer_uptodate(bh))
2501 ret = -EIO;
2502 free_buffer_head(bh);
2503 read_bh[i] = NULL;
2505 if (ret)
2506 goto failed;
2509 if (is_mapped_to_disk)
2510 SetPageMappedToDisk(page);
2511 SetPageUptodate(page);
2513 /*
2514 * Setting the page dirty here isn't necessary for the prepare_write
2515 * function - commit_write will do that. But if/when this function is
2516 * used within the pagefault handler to ensure that all mmapped pages
2517 * have backing space in the filesystem, we will need to dirty the page
2518 * if its contents were altered.
2519 */
2520 if (dirtied_it)
2521 set_page_dirty(page);
2523 return 0;
2525 failed:
2526 for (i = 0; i < nr_reads; i++) {
2527 if (read_bh[i])
2528 free_buffer_head(read_bh[i]);
2531 /*
2532 * Error recovery is pretty slack. Clear the page and mark it dirty
2533 * so we'll later zero out any blocks which _were_ allocated.
2534 */
2535 kaddr = kmap_atomic(page, KM_USER0);
2536 memset(kaddr, 0, PAGE_CACHE_SIZE);
2537 kunmap_atomic(kaddr, KM_USER0);
2538 SetPageUptodate(page);
2539 set_page_dirty(page);
2540 return ret;
2542 EXPORT_SYMBOL(nobh_prepare_write);
2544 int nobh_commit_write(struct file *file, struct page *page,
2545 unsigned from, unsigned to)
2547 struct inode *inode = page->mapping->host;
2548 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2550 set_page_dirty(page);
2551 if (pos > inode->i_size) {
2552 i_size_write(inode, pos);
2553 mark_inode_dirty(inode);
2555 return 0;
2557 EXPORT_SYMBOL(nobh_commit_write);
2559 /*
2560 * nobh_writepage() - based on block_full_write_page() except
2561 * that it tries to operate without attaching bufferheads to
2562 * the page.
2563 */
2564 int nobh_writepage(struct page *page, get_block_t *get_block,
2565 struct writeback_control *wbc)
2567 struct inode * const inode = page->mapping->host;
2568 loff_t i_size = i_size_read(inode);
2569 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2570 unsigned offset;
2571 void *kaddr;
2572 int ret;
2574 /* Is the page fully inside i_size? */
2575 if (page->index < end_index)
2576 goto out;
2578 /* Is the page fully outside i_size? (truncate in progress) */
2579 offset = i_size & (PAGE_CACHE_SIZE-1);
2580 if (page->index >= end_index+1 || !offset) {
2581 /*
2582 * The page may have dirty, unmapped buffers. For example,
2583 * they may have been added in ext3_writepage(). Make them
2584 * freeable here, so the page does not leak.
2585 */
2586 #if 0
2587 /* Not really sure about this - do we need this ? */
2588 if (page->mapping->a_ops->invalidatepage)
2589 page->mapping->a_ops->invalidatepage(page, offset);
2590 #endif
2591 unlock_page(page);
2592 return 0; /* don't care */
2595 /*
2596 * The page straddles i_size. It must be zeroed out on each and every
2597 * writepage invocation because it may be mmapped. "A file is mapped
2598 * in multiples of the page size. For a file that is not a multiple of
2599 * the page size, the remaining memory is zeroed when mapped, and
2600 * writes to that region are not written out to the file."
2601 */
2602 kaddr = kmap_atomic(page, KM_USER0);
2603 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2604 flush_dcache_page(page);
2605 kunmap_atomic(kaddr, KM_USER0);
2606 out:
2607 ret = mpage_writepage(page, get_block, wbc);
2608 if (ret == -EAGAIN)
2609 ret = __block_write_full_page(inode, page, get_block, wbc);
2610 return ret;
2612 EXPORT_SYMBOL(nobh_writepage);
2614 /*
2615 * This function assumes that ->prepare_write() uses nobh_prepare_write().
2616 */
2617 int nobh_truncate_page(struct address_space *mapping, loff_t from)
2619 struct inode *inode = mapping->host;
2620 unsigned blocksize = 1 << inode->i_blkbits;
2621 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2622 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2623 unsigned to;
2624 struct page *page;
2625 const struct address_space_operations *a_ops = mapping->a_ops;
2626 char *kaddr;
2627 int ret = 0;
2629 if ((offset & (blocksize - 1)) == 0)
2630 goto out;
2632 ret = -ENOMEM;
2633 page = grab_cache_page(mapping, index);
2634 if (!page)
2635 goto out;
2637 to = (offset + blocksize) & ~(blocksize - 1);
2638 ret = a_ops->prepare_write(NULL, page, offset, to);
2639 if (ret == 0) {
2640 kaddr = kmap_atomic(page, KM_USER0);
2641 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2642 flush_dcache_page(page);
2643 kunmap_atomic(kaddr, KM_USER0);
2644 set_page_dirty(page);
2646 unlock_page(page);
2647 page_cache_release(page);
2648 out:
2649 return ret;
2651 EXPORT_SYMBOL(nobh_truncate_page);
2653 int block_truncate_page(struct address_space *mapping,
2654 loff_t from, get_block_t *get_block)
2656 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2657 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2658 unsigned blocksize;
2659 sector_t iblock;
2660 unsigned length, pos;
2661 struct inode *inode = mapping->host;
2662 struct page *page;
2663 struct buffer_head *bh;
2664 void *kaddr;
2665 int err;
2667 blocksize = 1 << inode->i_blkbits;
2668 length = offset & (blocksize - 1);
2670 /* Block boundary? Nothing to do */
2671 if (!length)
2672 return 0;
2674 length = blocksize - length;
2675 iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2677 page = grab_cache_page(mapping, index);
2678 err = -ENOMEM;
2679 if (!page)
2680 goto out;
2682 if (!page_has_buffers(page))
2683 create_empty_buffers(page, blocksize, 0);
2685 /* Find the buffer that contains "offset" */
2686 bh = page_buffers(page);
2687 pos = blocksize;
2688 while (offset >= pos) {
2689 bh = bh->b_this_page;
2690 iblock++;
2691 pos += blocksize;
2694 err = 0;
2695 if (!buffer_mapped(bh)) {
2696 WARN_ON(bh->b_size != blocksize);
2697 err = get_block(inode, iblock, bh, 0);
2698 if (err)
2699 goto unlock;
2700 /* unmapped? It's a hole - nothing to do */
2701 if (!buffer_mapped(bh))
2702 goto unlock;
2705 /* Ok, it's mapped. Make sure it's up-to-date */
2706 if (PageUptodate(page))
2707 set_buffer_uptodate(bh);
2709 if (!buffer_uptodate(bh) && !buffer_delay(bh)) {
2710 err = -EIO;
2711 ll_rw_block(READ, 1, &bh);
2712 wait_on_buffer(bh);
2713 /* Uhhuh. Read error. Complain and punt. */
2714 if (!buffer_uptodate(bh))
2715 goto unlock;
2718 kaddr = kmap_atomic(page, KM_USER0);
2719 memset(kaddr + offset, 0, length);
2720 flush_dcache_page(page);
2721 kunmap_atomic(kaddr, KM_USER0);
2723 mark_buffer_dirty(bh);
2724 err = 0;
2726 unlock:
2727 unlock_page(page);
2728 page_cache_release(page);
2729 out:
2730 return err;
2733 /*
2734 * The generic ->writepage function for buffer-backed address_spaces
2735 */
2736 int block_write_full_page(struct page *page, get_block_t *get_block,
2737 struct writeback_control *wbc)
2739 struct inode * const inode = page->mapping->host;
2740 loff_t i_size = i_size_read(inode);
2741 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2742 unsigned offset;
2743 void *kaddr;
2745 /* Is the page fully inside i_size? */
2746 if (page->index < end_index)
2747 return __block_write_full_page(inode, page, get_block, wbc);
2749 /* Is the page fully outside i_size? (truncate in progress) */
2750 offset = i_size & (PAGE_CACHE_SIZE-1);
2751 if (page->index >= end_index+1 || !offset) {
2752 /*
2753 * The page may have dirty, unmapped buffers. For example,
2754 * they may have been added in ext3_writepage(). Make them
2755 * freeable here, so the page does not leak.
2756 */
2757 do_invalidatepage(page, 0);
2758 unlock_page(page);
2759 return 0; /* don't care */
2762 /*
2763 * The page straddles i_size. It must be zeroed out on each and every
2764 * writepage invokation because it may be mmapped. "A file is mapped
2765 * in multiples of the page size. For a file that is not a multiple of
2766 * the page size, the remaining memory is zeroed when mapped, and
2767 * writes to that region are not written out to the file."
2768 */
2769 kaddr = kmap_atomic(page, KM_USER0);
2770 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2771 flush_dcache_page(page);
2772 kunmap_atomic(kaddr, KM_USER0);
2773 return __block_write_full_page(inode, page, get_block, wbc);
2776 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2777 get_block_t *get_block)
2779 struct buffer_head tmp;
2780 struct inode *inode = mapping->host;
2781 tmp.b_state = 0;
2782 tmp.b_blocknr = 0;
2783 tmp.b_size = 1 << inode->i_blkbits;
2784 get_block(inode, block, &tmp, 0);
2785 return tmp.b_blocknr;
2788 static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err)
2790 struct buffer_head *bh = bio->bi_private;
2792 if (bio->bi_size)
2793 return 1;
2795 if (err == -EOPNOTSUPP) {
2796 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2797 set_bit(BH_Eopnotsupp, &bh->b_state);
2800 bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2801 bio_put(bio);
2802 return 0;
2805 int submit_bh(int rw, struct buffer_head * bh)
2807 struct bio *bio;
2808 int ret = 0;
2810 BUG_ON(!buffer_locked(bh));
2811 BUG_ON(!buffer_mapped(bh));
2812 BUG_ON(!bh->b_end_io);
2814 if (buffer_ordered(bh) && (rw == WRITE))
2815 rw = WRITE_BARRIER;
2817 /*
2818 * Only clear out a write error when rewriting, should this
2819 * include WRITE_SYNC as well?
2820 */
2821 if (test_set_buffer_req(bh) && (rw == WRITE || rw == WRITE_BARRIER))
2822 clear_buffer_write_io_error(bh);
2824 /*
2825 * from here on down, it's all bio -- do the initial mapping,
2826 * submit_bio -> generic_make_request may further map this bio around
2827 */
2828 bio = bio_alloc(GFP_NOIO, 1);
2830 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2831 bio->bi_bdev = bh->b_bdev;
2832 bio->bi_io_vec[0].bv_page = bh->b_page;
2833 bio->bi_io_vec[0].bv_len = bh->b_size;
2834 bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2836 bio->bi_vcnt = 1;
2837 bio->bi_idx = 0;
2838 bio->bi_size = bh->b_size;
2840 bio->bi_end_io = end_bio_bh_io_sync;
2841 bio->bi_private = bh;
2843 bio_get(bio);
2844 submit_bio(rw, bio);
2846 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2847 ret = -EOPNOTSUPP;
2849 bio_put(bio);
2850 return ret;
2853 /**
2854 * ll_rw_block: low-level access to block devices (DEPRECATED)
2855 * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
2856 * @nr: number of &struct buffer_heads in the array
2857 * @bhs: array of pointers to &struct buffer_head
2859 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
2860 * requests an I/O operation on them, either a %READ or a %WRITE. The third
2861 * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
2862 * are sent to disk. The fourth %READA option is described in the documentation
2863 * for generic_make_request() which ll_rw_block() calls.
2865 * This function drops any buffer that it cannot get a lock on (with the
2866 * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
2867 * clean when doing a write request, and any buffer that appears to be
2868 * up-to-date when doing read request. Further it marks as clean buffers that
2869 * are processed for writing (the buffer cache won't assume that they are
2870 * actually clean until the buffer gets unlocked).
2872 * ll_rw_block sets b_end_io to simple completion handler that marks
2873 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2874 * any waiters.
2876 * All of the buffers must be for the same device, and must also be a
2877 * multiple of the current approved size for the device.
2878 */
2879 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
2881 int i;
2883 for (i = 0; i < nr; i++) {
2884 struct buffer_head *bh = bhs[i];
2886 if (rw == SWRITE)
2887 lock_buffer(bh);
2888 else if (test_set_buffer_locked(bh))
2889 continue;
2891 if (rw == WRITE || rw == SWRITE) {
2892 if (test_clear_buffer_dirty(bh)) {
2893 bh->b_end_io = end_buffer_write_sync;
2894 get_bh(bh);
2895 submit_bh(WRITE, bh);
2896 continue;
2898 } else {
2899 if (!buffer_uptodate(bh)) {
2900 bh->b_end_io = end_buffer_read_sync;
2901 get_bh(bh);
2902 submit_bh(rw, bh);
2903 continue;
2906 unlock_buffer(bh);
2910 /*
2911 * For a data-integrity writeout, we need to wait upon any in-progress I/O
2912 * and then start new I/O and then wait upon it. The caller must have a ref on
2913 * the buffer_head.
2914 */
2915 int sync_dirty_buffer(struct buffer_head *bh)
2917 int ret = 0;
2919 WARN_ON(atomic_read(&bh->b_count) < 1);
2920 lock_buffer(bh);
2921 if (test_clear_buffer_dirty(bh)) {
2922 get_bh(bh);
2923 bh->b_end_io = end_buffer_write_sync;
2924 ret = submit_bh(WRITE, bh);
2925 wait_on_buffer(bh);
2926 if (buffer_eopnotsupp(bh)) {
2927 clear_buffer_eopnotsupp(bh);
2928 ret = -EOPNOTSUPP;
2930 if (!ret && !buffer_uptodate(bh))
2931 ret = -EIO;
2932 } else {
2933 unlock_buffer(bh);
2935 return ret;
2938 /*
2939 * try_to_free_buffers() checks if all the buffers on this particular page
2940 * are unused, and releases them if so.
2942 * Exclusion against try_to_free_buffers may be obtained by either
2943 * locking the page or by holding its mapping's private_lock.
2945 * If the page is dirty but all the buffers are clean then we need to
2946 * be sure to mark the page clean as well. This is because the page
2947 * may be against a block device, and a later reattachment of buffers
2948 * to a dirty page will set *all* buffers dirty. Which would corrupt
2949 * filesystem data on the same device.
2951 * The same applies to regular filesystem pages: if all the buffers are
2952 * clean then we set the page clean and proceed. To do that, we require
2953 * total exclusion from __set_page_dirty_buffers(). That is obtained with
2954 * private_lock.
2956 * try_to_free_buffers() is non-blocking.
2957 */
2958 static inline int buffer_busy(struct buffer_head *bh)
2960 return atomic_read(&bh->b_count) |
2961 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
2964 static int
2965 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
2967 struct buffer_head *head = page_buffers(page);
2968 struct buffer_head *bh;
2970 bh = head;
2971 do {
2972 if (buffer_write_io_error(bh) && page->mapping)
2973 set_bit(AS_EIO, &page->mapping->flags);
2974 if (buffer_busy(bh))
2975 goto failed;
2976 bh = bh->b_this_page;
2977 } while (bh != head);
2979 do {
2980 struct buffer_head *next = bh->b_this_page;
2982 if (!list_empty(&bh->b_assoc_buffers))
2983 __remove_assoc_queue(bh);
2984 bh = next;
2985 } while (bh != head);
2986 *buffers_to_free = head;
2987 __clear_page_buffers(page);
2988 return 1;
2989 failed:
2990 return 0;
2993 int try_to_free_buffers(struct page *page)
2995 struct address_space * const mapping = page->mapping;
2996 struct buffer_head *buffers_to_free = NULL;
2997 int ret = 0;
2999 BUG_ON(!PageLocked(page));
3000 if (PageWriteback(page))
3001 return 0;
3003 if (mapping == NULL) { /* can this still happen? */
3004 ret = drop_buffers(page, &buffers_to_free);
3005 goto out;
3008 spin_lock(&mapping->private_lock);
3009 ret = drop_buffers(page, &buffers_to_free);
3010 if (ret) {
3011 /*
3012 * If the filesystem writes its buffers by hand (eg ext3)
3013 * then we can have clean buffers against a dirty page. We
3014 * clean the page here; otherwise later reattachment of buffers
3015 * could encounter a non-uptodate page, which is unresolvable.
3016 * This only applies in the rare case where try_to_free_buffers
3017 * succeeds but the page is not freed.
3018 */
3019 clear_page_dirty(page);
3021 spin_unlock(&mapping->private_lock);
3022 out:
3023 if (buffers_to_free) {
3024 struct buffer_head *bh = buffers_to_free;
3026 do {
3027 struct buffer_head *next = bh->b_this_page;
3028 free_buffer_head(bh);
3029 bh = next;
3030 } while (bh != buffers_to_free);
3032 return ret;
3034 EXPORT_SYMBOL(try_to_free_buffers);
3036 void block_sync_page(struct page *page)
3038 struct address_space *mapping;
3040 smp_mb();
3041 mapping = page_mapping(page);
3042 if (mapping)
3043 blk_run_backing_dev(mapping->backing_dev_info, page);
3046 /*
3047 * There are no bdflush tunables left. But distributions are
3048 * still running obsolete flush daemons, so we terminate them here.
3050 * Use of bdflush() is deprecated and will be removed in a future kernel.
3051 * The `pdflush' kernel threads fully replace bdflush daemons and this call.
3052 */
3053 asmlinkage long sys_bdflush(int func, long data)
3055 static int msg_count;
3057 if (!capable(CAP_SYS_ADMIN))
3058 return -EPERM;
3060 if (msg_count < 5) {
3061 msg_count++;
3062 printk(KERN_INFO
3063 "warning: process `%s' used the obsolete bdflush"
3064 " system call\n", current->comm);
3065 printk(KERN_INFO "Fix your initscripts?\n");
3068 if (func == 1)
3069 do_exit(0);
3070 return 0;
3073 /*
3074 * Buffer-head allocation
3075 */
3076 static kmem_cache_t *bh_cachep;
3078 /*
3079 * Once the number of bh's in the machine exceeds this level, we start
3080 * stripping them in writeback.
3081 */
3082 static int max_buffer_heads;
3084 int buffer_heads_over_limit;
3086 struct bh_accounting {
3087 int nr; /* Number of live bh's */
3088 int ratelimit; /* Limit cacheline bouncing */
3089 };
3091 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3093 static void recalc_bh_state(void)
3095 int i;
3096 int tot = 0;
3098 if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3099 return;
3100 __get_cpu_var(bh_accounting).ratelimit = 0;
3101 for_each_online_cpu(i)
3102 tot += per_cpu(bh_accounting, i).nr;
3103 buffer_heads_over_limit = (tot > max_buffer_heads);
3106 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3108 struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
3109 if (ret) {
3110 get_cpu_var(bh_accounting).nr++;
3111 recalc_bh_state();
3112 put_cpu_var(bh_accounting);
3114 return ret;
3116 EXPORT_SYMBOL(alloc_buffer_head);
3118 void free_buffer_head(struct buffer_head *bh)
3120 BUG_ON(!list_empty(&bh->b_assoc_buffers));
3121 kmem_cache_free(bh_cachep, bh);
3122 get_cpu_var(bh_accounting).nr--;
3123 recalc_bh_state();
3124 put_cpu_var(bh_accounting);
3126 EXPORT_SYMBOL(free_buffer_head);
3128 static void
3129 init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags)
3131 if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
3132 SLAB_CTOR_CONSTRUCTOR) {
3133 struct buffer_head * bh = (struct buffer_head *)data;
3135 memset(bh, 0, sizeof(*bh));
3136 INIT_LIST_HEAD(&bh->b_assoc_buffers);
3140 #ifdef CONFIG_HOTPLUG_CPU
3141 static void buffer_exit_cpu(int cpu)
3143 int i;
3144 struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3146 for (i = 0; i < BH_LRU_SIZE; i++) {
3147 brelse(b->bhs[i]);
3148 b->bhs[i] = NULL;
3150 get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr;
3151 per_cpu(bh_accounting, cpu).nr = 0;
3152 put_cpu_var(bh_accounting);
3155 static int buffer_cpu_notify(struct notifier_block *self,
3156 unsigned long action, void *hcpu)
3158 if (action == CPU_DEAD)
3159 buffer_exit_cpu((unsigned long)hcpu);
3160 return NOTIFY_OK;
3162 #endif /* CONFIG_HOTPLUG_CPU */
3164 void __init buffer_init(void)
3166 int nrpages;
3168 bh_cachep = kmem_cache_create("buffer_head",
3169 sizeof(struct buffer_head), 0,
3170 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3171 SLAB_MEM_SPREAD),
3172 init_buffer_head,
3173 NULL);
3175 /*
3176 * Limit the bh occupancy to 10% of ZONE_NORMAL
3177 */
3178 nrpages = (nr_free_buffer_pages() * 10) / 100;
3179 max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3180 hotcpu_notifier(buffer_cpu_notify, 0);
3183 EXPORT_SYMBOL(__bforget);
3184 EXPORT_SYMBOL(__brelse);
3185 EXPORT_SYMBOL(__wait_on_buffer);
3186 EXPORT_SYMBOL(block_commit_write);
3187 EXPORT_SYMBOL(block_prepare_write);
3188 EXPORT_SYMBOL(block_read_full_page);
3189 EXPORT_SYMBOL(block_sync_page);
3190 EXPORT_SYMBOL(block_truncate_page);
3191 EXPORT_SYMBOL(block_write_full_page);
3192 EXPORT_SYMBOL(cont_prepare_write);
3193 EXPORT_SYMBOL(end_buffer_read_sync);
3194 EXPORT_SYMBOL(end_buffer_write_sync);
3195 EXPORT_SYMBOL(file_fsync);
3196 EXPORT_SYMBOL(fsync_bdev);
3197 EXPORT_SYMBOL(generic_block_bmap);
3198 EXPORT_SYMBOL(generic_commit_write);
3199 EXPORT_SYMBOL(generic_cont_expand);
3200 EXPORT_SYMBOL(generic_cont_expand_simple);
3201 EXPORT_SYMBOL(init_buffer);
3202 EXPORT_SYMBOL(invalidate_bdev);
3203 EXPORT_SYMBOL(ll_rw_block);
3204 EXPORT_SYMBOL(mark_buffer_dirty);
3205 EXPORT_SYMBOL(submit_bh);
3206 EXPORT_SYMBOL(sync_dirty_buffer);
3207 EXPORT_SYMBOL(unlock_buffer);