Convert all the bulk callers of lru_cache_del() to use the batched
pagevec_lru_del() function.
Change truncate_complete_page() to not delete the page from the LRU.
Do it in page_cache_release() instead. (This reintroduces the problem
with final-release-from-interrupt. THat gets fixed further on).
This patch changes the truncate locking somewhat. The removal from the
LRU now happens _after_ the page has been removed from the
address_space and has been unlocked. So there is now a window where
the shrink_cache code can discover the to-be-freed page via the LRU
list. But that's OK - the page is clean, its buffers (if any) are
clean. It's not attached to any mapping.
filemap.c | 79 +++++++++++++++++++++++++++++++++-----------------------------
swap.c | 2 -
2 files changed, 44 insertions(+), 37 deletions(-)
--- 2.5.31/mm/filemap.c~batched-lru-del Sun Aug 11 00:20:33 2002
+++ 2.5.31-akpm/mm/filemap.c Sun Aug 11 00:21:02 2002
@@ -118,10 +118,10 @@ void invalidate_inode_pages(struct inode
struct list_head *head, *curr;
struct page * page;
struct address_space *mapping = inode->i_mapping;
+ struct pagevec lru_pvec;
head = &mapping->clean_pages;
-
- spin_lock(&pagemap_lru_lock);
+ pagevec_init(&lru_pvec);
write_lock(&mapping->page_lock);
curr = head->next;
@@ -143,10 +143,10 @@ void invalidate_inode_pages(struct inode
if (page_count(page) != 1)
goto unlock;
- __lru_cache_del(page);
__remove_from_page_cache(page);
unlock_page(page);
- page_cache_release(page);
+ if (!pagevec_add(&lru_pvec, page))
+ __pagevec_lru_del(&lru_pvec);
continue;
unlock:
unlock_page(page);
@@ -154,7 +154,7 @@ unlock:
}
write_unlock(&mapping->page_lock);
- spin_unlock(&pagemap_lru_lock);
+ pagevec_lru_del(&lru_pvec);
}
static int do_invalidatepage(struct page *page, unsigned long offset)
@@ -174,16 +174,14 @@ static inline void truncate_partial_page
}
/*
- * If truncate can remove the fs-private metadata from the page, it
- * removes the page from the LRU immediately. This because some other thread
- * of control (eg, sendfile) may have a reference to the page. But dropping
- * the final reference to an LRU page in interrupt context is illegal - it may
- * deadlock over pagemap_lru_lock.
+ * If truncate cannot remove the fs-private metadata from the page, the page
+ * becomes anonymous. It will be left on the LRU and may even be mapped into
+ * user pagetables if we're racing with filemap_nopage().
*/
static void truncate_complete_page(struct page *page)
{
- if (!PagePrivate(page) || do_invalidatepage(page, 0))
- lru_cache_del(page);
+ if (PagePrivate(page))
+ do_invalidatepage(page, 0);
ClearPageDirty(page);
ClearPageUptodate(page);
@@ -204,8 +202,10 @@ static int truncate_list_pages(struct ad
struct list_head *curr;
struct page * page;
int unlocked = 0;
+ struct pagevec release_pvec;
- restart:
+ pagevec_init(&release_pvec);
+restart:
curr = head->next;
while (curr != head) {
unsigned long offset;
@@ -225,18 +225,17 @@ static int truncate_list_pages(struct ad
list_add_tail(head, curr);
write_unlock(&mapping->page_lock);
wait_on_page_writeback(page);
- page_cache_release(page);
+ if (!pagevec_add(&release_pvec, page))
+ __pagevec_release(&release_pvec);
unlocked = 1;
write_lock(&mapping->page_lock);
goto restart;
}
list_del(head);
- if (!failed)
- /* Restart after this page */
+ if (!failed) /* Restart after this page */
list_add(head, curr);
- else
- /* Restart on this page */
+ else /* Restart on this page */
list_add_tail(head, curr);
write_unlock(&mapping->page_lock);
@@ -246,25 +245,27 @@ static int truncate_list_pages(struct ad
if (*partial && (offset + 1) == start) {
truncate_partial_page(page, *partial);
*partial = 0;
- } else
+ } else {
truncate_complete_page(page);
-
+ }
unlock_page(page);
- } else
+ } else {
wait_on_page_locked(page);
-
- page_cache_release(page);
-
- if (need_resched()) {
- __set_current_state(TASK_RUNNING);
- schedule();
}
-
+ if (!pagevec_add(&release_pvec, page))
+ __pagevec_release(&release_pvec);
+ cond_resched();
write_lock(&mapping->page_lock);
goto restart;
}
curr = curr->next;
}
+ if (pagevec_count(&release_pvec)) {
+ write_unlock(&mapping->page_lock);
+ pagevec_release(&release_pvec);
+ write_lock(&mapping->page_lock);
+ unlocked = 1;
+ }
return unlocked;
}
@@ -362,8 +363,10 @@ static int invalidate_list_pages2(struct
struct list_head *curr;
struct page * page;
int unlocked = 0;
+ struct pagevec release_pvec;
- restart:
+ pagevec_init(&release_pvec);
+restart:
curr = head->prev;
while (curr != head) {
page = list_entry(curr, struct page, list);
@@ -380,7 +383,8 @@ static int invalidate_list_pages2(struct
goto restart;
}
- __unlocked = invalidate_this_page2(mapping, page, curr, head);
+ __unlocked = invalidate_this_page2(mapping,
+ page, curr, head);
unlock_page(page);
unlocked |= __unlocked;
if (!__unlocked) {
@@ -398,15 +402,18 @@ static int invalidate_list_pages2(struct
wait_on_page_locked(page);
}
- page_cache_release(page);
- if (need_resched()) {
- __set_current_state(TASK_RUNNING);
- schedule();
- }
-
+ if (!pagevec_add(&release_pvec, page))
+ __pagevec_release(&release_pvec);
+ cond_resched();
write_lock(&mapping->page_lock);
goto restart;
}
+ if (pagevec_count(&release_pvec)) {
+ write_unlock(&mapping->page_lock);
+ pagevec_release(&release_pvec);
+ write_lock(&mapping->page_lock);
+ unlocked = 1;
+ }
return unlocked;
}
--- 2.5.31/mm/swap.c~batched-lru-del Sun Aug 11 00:20:33 2002
+++ 2.5.31-akpm/mm/swap.c Sun Aug 11 00:21:02 2002
@@ -115,7 +115,7 @@ void __pagevec_release(struct pagevec *p
if (!put_page_testzero(page))
continue;
- if (!lock_held && PageLRU(page)) {
+ if (!lock_held) {
spin_lock(&pagemap_lru_lock);
lock_held = 1;
}
.
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
This archive was generated by hypermail 2b29 : Thu Aug 15 2002 - 22:00:23 EST