Commit a39722034ae37f80a1803bf781fe3fe1b03e20bc
Committed by
Linus Torvalds
Linus Torvalds
1 parent
d6afe27bff
Exists in
master
and in
7 other branches
[PATCH] page_uptodate locking scalability
Use a bit spin lock in the first buffer of the page to synchronise asynch IO buffer completions, instead of the global page_uptodate_lock, which is showing some scalabilty problems. Signed-off-by: Nick Piggin <nickpiggin@yahoo.com.au> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Showing 2 changed files with 20 additions and 8 deletions Side-by-side Diff
fs/buffer.c
| ... | ... | @@ -513,8 +513,8 @@ |
| 513 | 513 | */ |
| 514 | 514 | static void end_buffer_async_read(struct buffer_head *bh, int uptodate) |
| 515 | 515 | { |
| 516 | - static DEFINE_SPINLOCK(page_uptodate_lock); | |
| 517 | 516 | unsigned long flags; |
| 517 | + struct buffer_head *first; | |
| 518 | 518 | struct buffer_head *tmp; |
| 519 | 519 | struct page *page; |
| 520 | 520 | int page_uptodate = 1; |
| ... | ... | @@ -536,7 +536,9 @@ |
| 536 | 536 | * two buffer heads end IO at almost the same time and both |
| 537 | 537 | * decide that the page is now completely done. |
| 538 | 538 | */ |
| 539 | - spin_lock_irqsave(&page_uptodate_lock, flags); | |
| 539 | + first = page_buffers(page); | |
| 540 | + local_irq_save(flags); | |
| 541 | + bit_spin_lock(BH_Uptodate_Lock, &first->b_state); | |
| 540 | 542 | clear_buffer_async_read(bh); |
| 541 | 543 | unlock_buffer(bh); |
| 542 | 544 | tmp = bh; |
| ... | ... | @@ -549,7 +551,8 @@ |
| 549 | 551 | } |
| 550 | 552 | tmp = tmp->b_this_page; |
| 551 | 553 | } while (tmp != bh); |
| 552 | - spin_unlock_irqrestore(&page_uptodate_lock, flags); | |
| 554 | + bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); | |
| 555 | + local_irq_restore(flags); | |
| 553 | 556 | |
| 554 | 557 | /* |
| 555 | 558 | * If none of the buffers had errors and they are all |
| ... | ... | @@ -561,7 +564,8 @@ |
| 561 | 564 | return; |
| 562 | 565 | |
| 563 | 566 | still_busy: |
| 564 | - spin_unlock_irqrestore(&page_uptodate_lock, flags); | |
| 567 | + bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); | |
| 568 | + local_irq_restore(flags); | |
| 565 | 569 | return; |
| 566 | 570 | } |
| 567 | 571 | |
| 568 | 572 | |
| ... | ... | @@ -572,8 +576,8 @@ |
| 572 | 576 | void end_buffer_async_write(struct buffer_head *bh, int uptodate) |
| 573 | 577 | { |
| 574 | 578 | char b[BDEVNAME_SIZE]; |
| 575 | - static DEFINE_SPINLOCK(page_uptodate_lock); | |
| 576 | 579 | unsigned long flags; |
| 580 | + struct buffer_head *first; | |
| 577 | 581 | struct buffer_head *tmp; |
| 578 | 582 | struct page *page; |
| 579 | 583 | |
| ... | ... | @@ -594,7 +598,10 @@ |
| 594 | 598 | SetPageError(page); |
| 595 | 599 | } |
| 596 | 600 | |
| 597 | - spin_lock_irqsave(&page_uptodate_lock, flags); | |
| 601 | + first = page_buffers(page); | |
| 602 | + local_irq_save(flags); | |
| 603 | + bit_spin_lock(BH_Uptodate_Lock, &first->b_state); | |
| 604 | + | |
| 598 | 605 | clear_buffer_async_write(bh); |
| 599 | 606 | unlock_buffer(bh); |
| 600 | 607 | tmp = bh->b_this_page; |
| 601 | 608 | |
| ... | ... | @@ -605,12 +612,14 @@ |
| 605 | 612 | } |
| 606 | 613 | tmp = tmp->b_this_page; |
| 607 | 614 | } |
| 608 | - spin_unlock_irqrestore(&page_uptodate_lock, flags); | |
| 615 | + bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); | |
| 616 | + local_irq_restore(flags); | |
| 609 | 617 | end_page_writeback(page); |
| 610 | 618 | return; |
| 611 | 619 | |
| 612 | 620 | still_busy: |
| 613 | - spin_unlock_irqrestore(&page_uptodate_lock, flags); | |
| 621 | + bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); | |
| 622 | + local_irq_restore(flags); | |
| 614 | 623 | return; |
| 615 | 624 | } |
| 616 | 625 |
include/linux/buffer_head.h
| ... | ... | @@ -19,6 +19,9 @@ |
| 19 | 19 | BH_Dirty, /* Is dirty */ |
| 20 | 20 | BH_Lock, /* Is locked */ |
| 21 | 21 | BH_Req, /* Has been submitted for I/O */ |
| 22 | + BH_Uptodate_Lock,/* Used by the first bh in a page, to serialise | |
| 23 | + * IO completion of other buffers in the page | |
| 24 | + */ | |
| 22 | 25 | |
| 23 | 26 | BH_Mapped, /* Has a disk mapping */ |
| 24 | 27 | BH_New, /* Disk mapping was newly created by get_block */ |