[CFT][PATCH] Latest -rmap15 stuff against 2.4.20-rc2-ac2

From: Paul P Komkoff Jr (i@stingr.net)
Date: Thu Nov 21 2002 - 17:55:07 EST


This is based on latest code riel committed to his rmap branch.
Rediffed against 2.4.20-rc2-ac2

ChangeSet@1.695.1.6, 2002-11-21 16:42:17-02:00, riel@duckman.distro.conectiva
  first stab at fine-tuning arjan's O(1) VM
  - split the active list in process working set and file cache
  - preferentially deactivate file cache pages, if there are many
    of those
  - speed up and slow down page aging as wanted

diff -Nru a/fs/buffer.c b/fs/buffer.c
--- a/fs/buffer.c Fri Nov 22 00:36:57 2002
+++ b/fs/buffer.c Fri Nov 22 00:36:58 2002
@@ -2915,6 +2915,30 @@
         }
 }
 
+
+/*
+ * Do some IO post-processing here!!!
+ */
+void do_io_postprocessing(void)
+{
+ int i;
+ struct buffer_head *bh, *next;
+
+ spin_lock(&lru_list_lock);
+ bh = lru_list[BUF_LOCKED];
+ if (bh) {
+ for (i = nr_buffers_type[BUF_LOCKED]; i-- > 0; bh = next) {
+ next = bh->b_next_free;
+
+ if (!buffer_locked(bh))
+ __refile_buffer(bh);
+ else
+ break;
+ }
+ }
+ spin_unlock(&lru_list_lock);
+}
+
 /*
  * This is the kernel update daemon. It was used to live in userspace
  * but since it's need to run safely we want it unkillable by mistake.
@@ -2966,6 +2990,7 @@
 #ifdef DEBUG
                 printk(KERN_DEBUG "kupdate() activated...\n");
 #endif
+ do_io_postprocessing();
                 sync_old_buffers();
                 run_task_queue(&tq_disk);
         }
diff -Nru a/fs/proc/proc_misc.c b/fs/proc/proc_misc.c
--- a/fs/proc/proc_misc.c Fri Nov 22 00:36:57 2002
+++ b/fs/proc/proc_misc.c Fri Nov 22 00:36:57 2002
@@ -191,7 +191,10 @@
                 "Cached: %8lu kB\n"
                 "SwapCached: %8lu kB\n"
                 "Active: %8u kB\n"
+ "ActiveAnon: %8u kB\n"
+ "ActiveCache: %8u kB\n"
                 "Inact_dirty: %8u kB\n"
+ "Inact_laundry:%8u kB\n"
                 "Inact_clean: %8u kB\n"
                 "Inact_target: %8u kB\n"
                 "HighTotal: %8lu kB\n"
@@ -207,8 +210,11 @@
                 K(i.bufferram),
                 K(pg_size - swapper_space.nrpages),
                 K(swapper_space.nrpages),
- K(nr_active_pages),
+ K(nr_active_anon_pages) + K(nr_active_cache_pages),
+ K(nr_active_anon_pages),
+ K(nr_active_cache_pages),
                 K(nr_inactive_dirty_pages),
+ K(nr_inactive_laundry_pages),
                 K(nr_inactive_clean_pages),
                 K(inactive_target()),
                 K(i.totalhigh),
diff -Nru a/include/linux/brlock.h b/include/linux/brlock.h
--- a/include/linux/brlock.h Fri Nov 22 00:36:57 2002
+++ b/include/linux/brlock.h Fri Nov 22 00:36:57 2002
@@ -37,6 +37,8 @@
         BR_GLOBALIRQ_LOCK,
         BR_NETPROTO_LOCK,
         BR_LLC_LOCK,
+ BR_LRU_LOCK,
+
         __BR_END
 };
 
diff -Nru a/include/linux/list.h b/include/linux/list.h
--- a/include/linux/list.h Fri Nov 22 00:36:57 2002
+++ b/include/linux/list.h Fri Nov 22 00:36:57 2002
@@ -137,8 +137,7 @@
         return head->next == head;
 }
 
-static inline void __list_splice(struct list_head *list,
- struct list_head *head)
+static inline void __list_splice(struct list_head *list, struct list_head *head)
 {
         struct list_head *first = list->next;
         struct list_head *last = list->prev;
diff -Nru a/include/linux/mm.h b/include/linux/mm.h
--- a/include/linux/mm.h Fri Nov 22 00:36:57 2002
+++ b/include/linux/mm.h Fri Nov 22 00:36:57 2002
@@ -1,5 +1,23 @@
 #ifndef _LINUX_MM_H
 #define _LINUX_MM_H
+/*
+ * Copyright (c) 2002. All rights reserved.
+ *
+ * This software may be freely redistributed under the terms of the
+ * GNU General Public License.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Authors:
+ * Linus Torvalds
+ * Stephen Tweedie
+ * Andrea Arcangeli
+ * Rik van Riel
+ * Arjan van de Ven
+ * and others
+ */
 
 #include <linux/sched.h>
 #include <linux/errno.h>
@@ -168,7 +186,7 @@
         unsigned long flags; /* atomic flags, some possibly
                                            updated asynchronously */
         struct list_head lru; /* Pageout list, eg. active_list;
- protected by pagemap_lru_lock !! */
+ protected by the lru lock !! */
         unsigned char age; /* Page aging counter. */
         struct pte_chain * pte_chain; /* Reverse pte mapping pointer.
                                          * protected by PG_chainlock
@@ -279,7 +297,7 @@
  *
  * Note that the referenced bit, the page->lru list_head and the
  * active, inactive_dirty and inactive_clean lists are protected by
- * the pagemap_lru_lock, and *NOT* by the usual PG_locked bit!
+ * the lru lock, and *NOT* by the usual PG_locked bit!
  *
  * PG_skip is used on sparc/sparc64 architectures to "skip" certain
  * parts of the address space.
@@ -300,18 +318,21 @@
 #define PG_referenced 2
 #define PG_uptodate 3
 #define PG_dirty 4
-#define PG_inactive_clean 5
-#define PG_active 6
+#define PG_active_anon 5
 #define PG_inactive_dirty 7
-#define PG_slab 8
-#define PG_skip 10
-#define PG_highmem 11
-#define PG_checked 12 /* kill me in 2.5.<early>. */
-#define PG_arch_1 13
-#define PG_reserved 14
-#define PG_launder 15 /* written out by VM pressure.. */
-#define PG_chainlock 16 /* lock bit for ->pte_chain */
-#define PG_lru 17
+#define PG_inactive_laundry 8
+#define PG_inactive_clean 9
+#define PG_slab 10
+#define PG_skip 11
+#define PG_highmem 12
+#define PG_checked 13 /* kill me in 2.5.<early>. */
+#define PG_arch_1 14
+#define PG_reserved 15
+#define PG_launder 16 /* written out by VM pressure.. */
+#define PG_chainlock 17 /* lock bit for ->pte_chain */
+#define PG_lru 18
+#define PG_active_cache 19
+
 /* Don't you dare to use high bits, they seem to be used for something else! */
 
 
@@ -429,11 +450,21 @@
 #define PageClearSlab(page) clear_bit(PG_slab, &(page)->flags)
 #define PageReserved(page) test_bit(PG_reserved, &(page)->flags)
 
-#define PageActive(page) test_bit(PG_active, &(page)->flags)
-#define SetPageActive(page) set_bit(PG_active, &(page)->flags)
-#define ClearPageActive(page) clear_bit(PG_active, &(page)->flags)
-#define TestandSetPageActive(page) test_and_set_bit(PG_active, &(page)->flags)
-#define TestandClearPageActive(page) test_and_clear_bit(PG_active, &(page)->flags)
+#define PageActiveAnon(page) test_bit(PG_active_anon, &(page)->flags)
+#define SetPageActiveAnon(page) set_bit(PG_active_anon, &(page)->flags)
+#define ClearPageActiveAnon(page) clear_bit(PG_active_anon, &(page)->flags)
+#define TestandSetPageActiveAnon(page) test_and_set_bit(PG_active_anon, &(page)->flags)
+#define TestandClearPageActiveAnon(page) test_and_clear_bit(PG_active_anon, &(page)->flags)
+
+#define PageActiveCache(page) test_bit(PG_active_cache, &(page)->flags)
+#define SetPageActiveCache(page) set_bit(PG_active_cache, &(page)->flags)
+#define ClearPageActiveCache(page) clear_bit(PG_active_cache, &(page)->flags)
+#define TestandSetPageActiveCache(page) test_and_set_bit(PG_active_cache, &(page)->flags)
+#define TestandClearPageActiveCache(page) test_and_clear_bit(PG_active_cache, &(page)->flags)
+
+#define PageInactiveLaundry(page) test_bit(PG_inactive_laundry, &(page)->flags)
+#define SetPageInactiveLaundry(page) set_bit(PG_inactive_laundry, &(page)->flags)
+#define ClearPageInactiveLaundry(page) clear_bit(PG_inactive_laundry, &(page)->flags)
 
 #define PageInactiveDirty(page) test_bit(PG_inactive_dirty, &(page)->flags)
 #define SetPageInactiveDirty(page) set_bit(PG_inactive_dirty, &(page)->flags)
diff -Nru a/include/linux/mm_inline.h b/include/linux/mm_inline.h
--- a/include/linux/mm_inline.h Fri Nov 22 00:36:57 2002
+++ b/include/linux/mm_inline.h Fri Nov 22 00:36:57 2002
@@ -2,23 +2,127 @@
 #define _LINUX_MM_INLINE_H
 
 #include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/brlock.h>
+
+
+/*
+ * Copyright (c) 2002. All rights reserved.
+ *
+ * This software may be freely redistributed under the terms of the
+ * GNU General Public License.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Authors:
+ * Linus Torvalds
+ * Stephen Tweedie
+ * Andrea Arcangeli
+ * Rik van Riel
+ * Arjan van de Ven
+ * and others
+ */
+
+GPL_HEADER()
+
+extern unsigned char active_age_bias;
 
 /*
  * These inline functions tend to need bits and pieces of all the
  * other VM include files, meaning they cannot be defined inside
  * one of the other VM include files.
+ *
+ */
+
+/**
+ * page_dirty - do we need to write the data out to disk
+ * @page: page to test
+ *
+ * Returns true if the page contains data which needs to
+ * be written to disk. Doesn't test the page tables (yet?).
+ */
+static inline int page_dirty(struct page *page)
+{
+ struct buffer_head *tmp, *bh;
+
+ if (PageDirty(page))
+ return 1;
+
+ if (page->mapping && !page->buffers)
+ return 0;
+
+ tmp = bh = page->buffers;
+
+ do {
+ if (tmp->b_state & ((1<<BH_Dirty) | (1<<BH_Lock)))
+ return 1;
+ tmp = tmp->b_this_page;
+ } while (tmp != bh);
+
+ return 0;
+}
+
+/**
+ * page_anon - is this page ram/swap backed ?
+ * @page - page to test
  *
- * The include file mess really needs to be cleaned up...
+ * Returns 1 if the page is backed by ram/swap, 0 if the page is
+ * backed by a file in a filesystem on permanent storage.
  */
+static inline int page_anon(struct page * page)
+{
+ /* Pages of an mmap()d file won't trigger this unless they get
+ * referenced on the inactive list and really are in the working
+ * set of the process... */
+ if (page->pte_chain)
+ return 1;
+
+ if (!page->mapping && !page->buffers)
+ return 1;
+
+ if (PageSwapCache(page))
+ return 1;
+
+ if (!page->mapping->a_ops->writepage)
+ return 1;
 
-static inline void add_page_to_active_list(struct page * page)
+ /* TODO: ramfs, tmpfs shm segments and ramdisk */
+
+ return 0;
+}
+
+
+
+static inline void add_page_to_active_anon_list(struct page * page, int age)
 {
         struct zone_struct * zone = page_zone(page);
         DEBUG_LRU_PAGE(page);
- SetPageActive(page);
- list_add(&page->lru, &zone->active_list);
- zone->active_pages++;
- nr_active_pages++;
+ SetPageActiveAnon(page);
+ list_add(&page->lru, &zone->active_anon_list[age]);
+ page->age = age + active_age_bias;
+ zone->active_anon_pages++;
+ nr_active_anon_pages++;
+}
+
+static inline void add_page_to_active_cache_list(struct page * page, int age)
+{
+ struct zone_struct * zone = page_zone(page);
+ DEBUG_LRU_PAGE(page);
+ SetPageActiveCache(page);
+ list_add(&page->lru, &zone->active_cache_list[age]);
+ page->age = age + active_age_bias;
+ zone->active_cache_pages++;
+ nr_active_cache_pages++;
+}
+
+static inline void add_page_to_active_list(struct page * page, int age)
+{
+ if (page_anon(page))
+ add_page_to_active_anon_list(page, age);
+ else
+ add_page_to_active_cache_list(page, age);
 }
 
 static inline void add_page_to_inactive_dirty_list(struct page * page)
@@ -31,6 +135,16 @@
         nr_inactive_dirty_pages++;
 }
 
+static inline void add_page_to_inactive_laundry_list(struct page * page)
+{
+ struct zone_struct * zone = page_zone(page);
+ DEBUG_LRU_PAGE(page);
+ SetPageInactiveLaundry(page);
+ list_add(&page->lru, &zone->inactive_laundry_list);
+ zone->inactive_laundry_pages++;
+ nr_inactive_laundry_pages++;
+}
+
 static inline void add_page_to_inactive_clean_list(struct page * page)
 {
         struct zone_struct * zone = page_zone(page);
@@ -41,13 +155,31 @@
         nr_inactive_clean_pages++;
 }
 
-static inline void del_page_from_active_list(struct page * page)
+static inline void del_page_from_active_anon_list(struct page * page)
+{
+ struct zone_struct * zone = page_zone(page);
+ unsigned char age;
+ list_del(&page->lru);
+ ClearPageActiveAnon(page);
+ nr_active_anon_pages--;
+ zone->active_anon_pages--;
+ age = page->age - active_age_bias;
+ if (age<=MAX_AGE)
+ zone->active_anon_count[age]--;
+ DEBUG_LRU_PAGE(page);
+}
+
+static inline void del_page_from_active_cache_list(struct page * page)
 {
         struct zone_struct * zone = page_zone(page);
+ unsigned char age;
         list_del(&page->lru);
- ClearPageActive(page);
- nr_active_pages--;
- zone->active_pages--;
+ ClearPageActiveCache(page);
+ nr_active_cache_pages--;
+ zone->active_cache_pages--;
+ age = page->age - active_age_bias;
+ if (age<=MAX_AGE)
+ zone->active_cache_count[age]--;
         DEBUG_LRU_PAGE(page);
 }
 
@@ -61,6 +193,16 @@
         DEBUG_LRU_PAGE(page);
 }
 
+static inline void del_page_from_inactive_laundry_list(struct page * page)
+{
+ struct zone_struct * zone = page_zone(page);
+ list_del(&page->lru);
+ ClearPageInactiveLaundry(page);
+ nr_inactive_laundry_pages--;
+ zone->inactive_laundry_pages--;
+ DEBUG_LRU_PAGE(page);
+}
+
 static inline void del_page_from_inactive_clean_list(struct page * page)
 {
         struct zone_struct * zone = page_zone(page);
@@ -184,7 +326,8 @@
 {
         int inactive, target, inactive_base;
 
- inactive_base = zone->active_pages + zone->inactive_dirty_pages;
+ inactive_base = zone->active_anon_pages + zone->active_cache_pages
+ + zone->inactive_dirty_pages;
         inactive_base /= INACTIVE_FACTOR;
 
         /* GCC should optimise this away completely. */
@@ -253,7 +396,13 @@
  */
 static inline int inactive_high(struct zone_struct * zone)
 {
- return inactive_limit(zone, VM_HIGH);
+ unsigned long active, inactive;
+ active = zone->active_anon_pages + zone->active_cache_pages
+ + zone->free_pages;
+ inactive = zone->inactive_dirty_pages + zone->inactive_clean_pages + zone->inactive_laundry_pages;
+ if (inactive * 5 > (active+inactive))
+ return -1;
+ return 1;
 }
 
 /*
@@ -263,12 +412,33 @@
 {
         int target;
 
- target = nr_active_pages + nr_inactive_dirty_pages
- + nr_inactive_clean_pages;
+ target = nr_active_anon_pages + nr_active_cache_pages
+ + nr_inactive_dirty_pages + nr_inactive_clean_pages
+ + nr_inactive_laundry_pages;
 
         target /= INACTIVE_FACTOR;
 
         return target;
+}
+
+static inline void lru_lock(struct zone_struct *zone)
+{
+ if (zone) {
+ br_read_lock(BR_LRU_LOCK);
+ spin_lock(&zone->lru_lock);
+ } else {
+ br_write_lock(BR_LRU_LOCK);
+ }
+}
+
+static inline void lru_unlock(struct zone_struct *zone)
+{
+ if (zone) {
+ spin_unlock(&zone->lru_lock);
+ br_read_unlock(BR_LRU_LOCK);
+ } else {
+ br_write_unlock(BR_LRU_LOCK);
+ }
 }
 
 #endif /* _LINUX_MM_INLINE_H */
diff -Nru a/include/linux/mmzone.h b/include/linux/mmzone.h
--- a/include/linux/mmzone.h Fri Nov 22 00:36:57 2002
+++ b/include/linux/mmzone.h Fri Nov 22 00:36:57 2002
@@ -13,11 +13,7 @@
  * Free memory management - zoned buddy allocator.
  */
 
-#ifndef CONFIG_FORCE_MAX_ZONEORDER
 #define MAX_ORDER 10
-#else
-#define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER
-#endif
 
 typedef struct free_area_struct {
         struct list_head free_list;
@@ -29,6 +25,9 @@
 
 #define MAX_CHUNKS_PER_NODE 8
 
+#define MAX_AGE 15
+#define INITIAL_AGE 3
+
 #define MAX_PER_CPU_PAGES 512
 typedef struct per_cpu_pages_s {
         int nr_pages, max_nr_pages;
@@ -50,19 +49,27 @@
         per_cpu_t cpu_pages[NR_CPUS];
         spinlock_t lock;
         unsigned long free_pages;
- unsigned long active_pages;
+ unsigned long active_anon_pages;
+ unsigned long active_cache_pages;
         unsigned long inactive_dirty_pages;
+ unsigned long inactive_laundry_pages;
         unsigned long inactive_clean_pages;
         unsigned long pages_min, pages_low, pages_high, pages_plenty;
         int need_balance;
+ int need_scan;
+ int active_anon_count[MAX_AGE+1];
+ int active_cache_count[MAX_AGE+1];
 
         /*
          * free areas of different sizes
          */
- struct list_head active_list;
+ struct list_head active_anon_list[MAX_AGE+1];
+ struct list_head active_cache_list[MAX_AGE+1];
         struct list_head inactive_dirty_list;
+ struct list_head inactive_laundry_list;
         struct list_head inactive_clean_list;
         free_area_t free_area[MAX_ORDER];
+ spinlock_t lru_lock;
 
         /*
          * wait_table -- the array holding the hash table
diff -Nru a/include/linux/module.h b/include/linux/module.h
--- a/include/linux/module.h Fri Nov 22 00:36:58 2002
+++ b/include/linux/module.h Fri Nov 22 00:36:58 2002
@@ -287,6 +287,9 @@
 static const char __module_license[] __attribute__((section(".modinfo"))) = \
 "license=" license
 
+#define GPL_HEADER() \
+static const char cpyright="This software may be freely redistributed under the terms of the GNU General Public License.";
+
 /* Define the module variable, and usage macros. */
 extern struct module __this_module;
 
@@ -302,7 +305,6 @@
 static const char __module_using_checksums[] __attribute__((section(".modinfo"))) =
 "using_checksums=1";
 #endif
-
 #else /* MODULE */
 
 #define MODULE_AUTHOR(name)
@@ -311,6 +313,7 @@
 #define MODULE_SUPPORTED_DEVICE(name)
 #define MODULE_PARM(var,type)
 #define MODULE_PARM_DESC(var,desc)
+#define GPL_HEADER()
 
 /* Create a dummy reference to the table to suppress gcc unused warnings. Put
  * the reference in the .data.exit section which is discarded when code is built
diff -Nru a/include/linux/pagemap.h b/include/linux/pagemap.h
--- a/include/linux/pagemap.h Fri Nov 22 00:36:57 2002
+++ b/include/linux/pagemap.h Fri Nov 22 00:36:57 2002
@@ -70,10 +70,6 @@
 
 #define page_hash(mapping,index) (page_hash_table+_page_hashfn(mapping,index))
 
-extern struct page * __find_get_page(struct address_space *mapping,
- unsigned long index, struct page **hash);
-#define find_get_page(mapping, index) \
- __find_get_page(mapping, index, page_hash(mapping, index))
 extern struct page * __find_lock_page (struct address_space * mapping,
                                 unsigned long index, struct page **hash);
 extern struct page * find_or_create_page(struct address_space *mapping,
@@ -90,6 +86,13 @@
 extern int add_to_page_cache_unique(struct page * page, struct address_space *mapping, unsigned long index, struct page **hash);
 
 extern void ___wait_on_page(struct page *);
+extern int wait_on_page_timeout(struct page *page, int timeout);
+
+
+extern struct page * __find_pagecache_page(struct address_space *mapping,
+ unsigned long index, struct page **hash);
+#define find_pagecache_page(mapping, index) \
+ __find_pagecache_page(mapping, index, page_hash(mapping, index))
 
 static inline void wait_on_page(struct page * page)
 {
diff -Nru a/include/linux/swap.h b/include/linux/swap.h
--- a/include/linux/swap.h Fri Nov 22 00:36:57 2002
+++ b/include/linux/swap.h Fri Nov 22 00:36:57 2002
@@ -85,8 +85,10 @@
 
 extern unsigned int nr_free_pages(void);
 extern unsigned int nr_free_buffer_pages(void);
-extern int nr_active_pages;
+extern int nr_active_anon_pages;
+extern int nr_active_cache_pages;
 extern int nr_inactive_dirty_pages;
+extern int nr_inactive_laundry_pages;
 extern int nr_inactive_clean_pages;
 extern atomic_t page_cache_size;
 extern atomic_t buffermem_pages;
@@ -115,6 +117,7 @@
 
 /* linux/mm/swap.c */
 extern void FASTCALL(lru_cache_add(struct page *));
+extern void FASTCALL(lru_cache_add_dirty(struct page *));
 extern void FASTCALL(__lru_cache_del(struct page *));
 extern void FASTCALL(lru_cache_del(struct page *));
 
@@ -175,8 +178,6 @@
 asmlinkage long sys_swapoff(const char *);
 asmlinkage long sys_swapon(const char *, int);
 
-extern spinlock_cacheline_t pagemap_lru_lock_cacheline;
-#define pagemap_lru_lock pagemap_lru_lock_cacheline.lock
 
 extern void FASTCALL(mark_page_accessed(struct page *));
 
@@ -191,13 +192,17 @@
 
 /*
  * List add/del helper macros. These must be called
- * with the pagemap_lru_lock held!
+ * with the lru lock held!
  */
 #define DEBUG_LRU_PAGE(page) \
 do { \
- if (PageActive(page)) \
+ if (PageActiveAnon(page)) \
+ BUG(); \
+ if (PageActiveCache(page)) \
                 BUG(); \
         if (PageInactiveDirty(page)) \
+ BUG(); \
+ if (PageInactiveLaundry(page)) \
                 BUG(); \
         if (PageInactiveClean(page)) \
                 BUG(); \
diff -Nru a/kernel/ksyms.c b/kernel/ksyms.c
--- a/kernel/ksyms.c Fri Nov 22 00:36:57 2002
+++ b/kernel/ksyms.c Fri Nov 22 00:36:57 2002
@@ -262,7 +262,6 @@
 EXPORT_SYMBOL(__pollwait);
 EXPORT_SYMBOL(poll_freewait);
 EXPORT_SYMBOL(ROOT_DEV);
-EXPORT_SYMBOL(__find_get_page);
 EXPORT_SYMBOL(__find_lock_page);
 EXPORT_SYMBOL(find_or_create_page);
 EXPORT_SYMBOL(grab_cache_page_nowait);
diff -Nru a/mm/filemap.c b/mm/filemap.c
--- a/mm/filemap.c Fri Nov 22 00:36:57 2002
+++ b/mm/filemap.c Fri Nov 22 00:36:57 2002
@@ -55,15 +55,14 @@
 
 spinlock_cacheline_t pagecache_lock_cacheline = {SPIN_LOCK_UNLOCKED};
 /*
- * NOTE: to avoid deadlocking you must never acquire the pagemap_lru_lock
+ * NOTE: to avoid deadlocking you must never acquire the lru lock
  * with the pagecache_lock held.
  *
  * Ordering:
  * swap_lock ->
- * pagemap_lru_lock ->
+ * lru lock ->
  * pagecache_lock
  */
-spinlock_cacheline_t pagemap_lru_lock_cacheline = {SPIN_LOCK_UNLOCKED};
 
 #define CLUSTER_PAGES (1 << page_cluster)
 #define CLUSTER_OFFSET(x) (((x) >> page_cluster) << page_cluster)
@@ -183,7 +182,7 @@
 
         head = &inode->i_mapping->clean_pages;
 
- spin_lock(&pagemap_lru_lock);
+ lru_lock(ALL_ZONES);
         spin_lock(&pagecache_lock);
         curr = head->next;
 
@@ -216,7 +215,7 @@
         }
 
         spin_unlock(&pagecache_lock);
- spin_unlock(&pagemap_lru_lock);
+ lru_unlock(ALL_ZONES);
 }
 
 static int do_flushpage(struct page *page, unsigned long offset)
@@ -880,6 +879,32 @@
                 wake_up_all(waitqueue);
 }
 
+
+/* like wait_on_page but with a timeout (in jiffies).
+ * returns 1 on timeout
+ */
+int wait_on_page_timeout(struct page *page, int timeout)
+{
+ wait_queue_head_t *waitqueue = page_waitqueue(page);
+ struct task_struct *tsk = current;
+ DECLARE_WAITQUEUE(wait, tsk);
+
+ if (!PageLocked(page))
+ return 0;
+
+ add_wait_queue(waitqueue, &wait);
+ do {
+ set_task_state(tsk, TASK_UNINTERRUPTIBLE);
+ if (!PageLocked(page))
+ break;
+ sync_page(page);
+ timeout = schedule_timeout(timeout);
+ } while (PageLocked(page) && timeout);
+ __set_task_state(tsk, TASK_RUNNING);
+ remove_wait_queue(waitqueue, &wait);
+ return PageLocked(page);
+}
+
 /*
  * Get a lock on the page, assuming we need to sleep
  * to get it..
@@ -914,26 +939,6 @@
                 __lock_page(page);
 }
 
-/*
- * a rather lightweight function, finding and getting a reference to a
- * hashed page atomically.
- */
-struct page * __find_get_page(struct address_space *mapping,
- unsigned long offset, struct page **hash)
-{
- struct page *page;
-
- /*
- * We scan the hash list read-only. Addition to and removal from
- * the hash-list needs a held write-lock.
- */
- spin_lock(&pagecache_lock);
- page = __find_page_nolock(mapping, offset, *hash);
- if (page)
- page_cache_get(page);
- spin_unlock(&pagecache_lock);
- return page;
-}
 
 /*
  * Same as above, but trylock it instead of incrementing the count.
@@ -1069,19 +1074,41 @@
          * been increased since the last time we were called, we
          * stop when the page isn't there.
          */
- spin_lock(&pagemap_lru_lock);
+ lru_lock(ALL_ZONES);
         while (--index >= start) {
                 struct page **hash = page_hash(mapping, index);
                 spin_lock(&pagecache_lock);
                 page = __find_page_nolock(mapping, index, *hash);
                 spin_unlock(&pagecache_lock);
- if (!page || !PageActive(page))
+ if (!page || !PageActiveCache(page))
                         break;
                 drop_page(page);
         }
- spin_unlock(&pagemap_lru_lock);
+ lru_unlock(ALL_ZONES);
+}
+
+/*
+ * Look up a page in the pagecache and return that page with
+ * a reference helt
+ */
+struct page * __find_pagecache_page(struct address_space *mapping,
+ unsigned long offset, struct page **hash)
+{
+ struct page *page;
+
+ /*
+ * We scan the hash list read-only. Addition to and removal from
+ * the hash-list needs a held write-lock.
+ */
+ spin_lock(&pagecache_lock);
+ page = __find_page_nolock(mapping, offset, *hash);
+ if (page)
+ page_cache_get(page);
+ spin_unlock(&pagecache_lock);
+ return page;
 }
 
+
 /* Same as grab_cache_page, but do not wait if the page is unavailable.
  * This is intended for speculative data generators, where the data can
  * be regenerated if the page couldn't be grabbed. This routine should
@@ -1092,7 +1119,7 @@
         struct page *page, **hash;
 
         hash = page_hash(mapping, index);
- page = __find_get_page(mapping, index, hash);
+ page = __find_pagecache_page(mapping, index, hash);
 
         if ( page ) {
                 if ( !TryLockPage(page) ) {
@@ -1378,7 +1405,7 @@
         /* Mark the page referenced, AFTER checking for previous usage.. */
         SetPageReferenced(page);
 
- if (unlikely(PageInactiveClean(page))) {
+ if (unlikely(PageInactiveClean(page) || PageInactiveLaundry(page))) {
                 struct zone_struct *zone = page_zone(page);
                 int free = zone->free_pages + zone->inactive_clean_pages;
 
@@ -2022,7 +2049,7 @@
          */
         hash = page_hash(mapping, pgoff);
 retry_find:
- page = __find_get_page(mapping, pgoff, hash);
+ page = __find_pagecache_page(mapping, pgoff, hash);
         if (!page)
                 goto no_cached_page;
 
@@ -2885,7 +2912,7 @@
         struct page *page, *cached_page = NULL;
         int err;
 repeat:
- page = __find_get_page(mapping, index, hash);
+ page = __find_pagecache_page(mapping, index, hash);
         if (!page) {
                 if (!cached_page) {
                         cached_page = page_cache_alloc(mapping);
diff -Nru a/mm/page_alloc.c b/mm/page_alloc.c
--- a/mm/page_alloc.c Fri Nov 22 00:36:57 2002
+++ b/mm/page_alloc.c Fri Nov 22 00:36:57 2002
@@ -26,8 +26,10 @@
 #include <linux/smp.h>
 
 int nr_swap_pages;
-int nr_active_pages;
+int nr_active_anon_pages;
+int nr_active_cache_pages;
 int nr_inactive_dirty_pages;
+int nr_inactive_laundry_pages;
 int nr_inactive_clean_pages;
 pg_data_t *pgdat_list;
 
@@ -109,16 +111,19 @@
                 BUG();
         if (PageLocked(page))
                 BUG();
- if (PageActive(page))
+ if (PageActiveAnon(page))
+ BUG();
+ if (PageActiveCache(page))
                 BUG();
         if (PageInactiveDirty(page))
                 BUG();
+ if (PageInactiveLaundry(page))
+ BUG();
         if (PageInactiveClean(page))
                 BUG();
         if (page->pte_chain)
                 BUG();
         page->flags &= ~((1<<PG_referenced) | (1<<PG_dirty));
- page->age = PAGE_AGE_START;
         
         zone = page_zone(page);
 
@@ -728,9 +733,10 @@
                 nr_free_pages() << (PAGE_SHIFT-10),
                 nr_free_highpages() << (PAGE_SHIFT-10));
 
- printk("( Active: %d, inactive_dirty: %d, inactive_clean: %d, free: %d )\n",
- nr_active_pages,
+ printk("( Active: %d/%d, inactive_laundry: %d, inactive_clean: %d, free: %d )\n",
+ nr_active_anon_pages + nr_active_cache_pages,
                 nr_inactive_dirty_pages,
+ nr_inactive_laundry_pages,
                 nr_inactive_clean_pages,
                 nr_free_pages());
 
@@ -941,12 +947,25 @@
                 zone->lock = SPIN_LOCK_UNLOCKED;
                 zone->zone_pgdat = pgdat;
                 zone->free_pages = 0;
+ zone->active_anon_pages = 0;
+ zone->active_cache_pages = 0;
                 zone->inactive_clean_pages = 0;
+ zone->inactive_laundry_pages = 0;
                 zone->inactive_dirty_pages = 0;
                 zone->need_balance = 0;
- INIT_LIST_HEAD(&zone->active_list);
+ zone->need_scan = 0;
+ for (k = 0; k <= MAX_AGE ; k++) {
+ INIT_LIST_HEAD(&zone->active_anon_list[k]);
+ zone->active_anon_count[k] = 0;
+ }
+ for (k = 0; k <= MAX_AGE ; k++) {
+ INIT_LIST_HEAD(&zone->active_cache_list[k]);
+ zone->active_cache_count[k] = 0;
+ }
                 INIT_LIST_HEAD(&zone->inactive_dirty_list);
+ INIT_LIST_HEAD(&zone->inactive_laundry_list);
                 INIT_LIST_HEAD(&zone->inactive_clean_list);
+ spin_lock_init(&zone->lru_lock);
 
                 if (!size)
                         continue;
diff -Nru a/mm/rmap.c b/mm/rmap.c
--- a/mm/rmap.c Fri Nov 22 00:36:57 2002
+++ b/mm/rmap.c Fri Nov 22 00:36:57 2002
@@ -14,7 +14,7 @@
 /*
  * Locking:
  * - the page->pte_chain is protected by the PG_chainlock bit,
- * which nests within the pagemap_lru_lock, then the
+ * which nests within the lru lock, then the
  * mm->page_table_lock, and then the page lock.
  * - because swapout locking is opposite to the locking order
  * in the page fault path, the swapout path uses trylocks
@@ -195,7 +195,7 @@
  * table entry mapping a page. Because locking order here is opposite
  * to the locking order used by the page fault path, we use trylocks.
  * Locking:
- * pagemap_lru_lock page_launder()
+ * lru lock page_launder()
  * page lock page_launder(), trylock
  * pte_chain_lock page_launder()
  * mm->page_table_lock try_to_unmap_one(), trylock
@@ -263,7 +263,7 @@
  * @page: the page to get unmapped
  *
  * Tries to remove all the page table entries which are mapping this
- * page, used in the pageout path. Caller must hold pagemap_lru_lock
+ * page, used in the pageout path. Caller must hold lru lock
  * and the page lock. Return values are:
  *
  * SWAP_SUCCESS - we succeeded in removing all mappings
diff -Nru a/mm/shmem.c b/mm/shmem.c
--- a/mm/shmem.c Fri Nov 22 00:36:57 2002
+++ b/mm/shmem.c Fri Nov 22 00:36:57 2002
@@ -581,7 +581,7 @@
          * cache and swap cache. We need to recheck the page cache
          * under the protection of the info->lock spinlock. */
 
- page = find_get_page(mapping, idx);
+ page = find_pagecache_page(mapping, idx);
         if (page) {
                 if (TryLockPage(page))
                         goto wait_retry;
diff -Nru a/mm/swap.c b/mm/swap.c
--- a/mm/swap.c Fri Nov 22 00:36:57 2002
+++ b/mm/swap.c Fri Nov 22 00:36:57 2002
@@ -36,7 +36,6 @@
 /**
  * (de)activate_page - move pages from/to active and inactive lists
  * @page: the page we want to move
- * @nolock - are we already holding the pagemap_lru_lock?
  *
  * Deactivate_page will move an active page to the right
  * inactive list, while activate_page will move a page back
@@ -51,18 +50,20 @@
          * (some pages aren't on any list at all)
          */
         ClearPageReferenced(page);
- page->age = 0;
- if (PageActive(page)) {
- del_page_from_active_list(page);
+ if (PageActiveAnon(page)) {
+ del_page_from_active_anon_list(page);
+ add_page_to_inactive_dirty_list(page);
+ } else if (PageActiveCache(page)) {
+ del_page_from_active_cache_list(page);
                 add_page_to_inactive_dirty_list(page);
         }
 }
 
 void deactivate_page(struct page * page)
 {
- spin_lock(&pagemap_lru_lock);
+ lru_lock(page_zone(page));
         deactivate_page_nolock(page);
- spin_unlock(&pagemap_lru_lock);
+ lru_unlock(page_zone(page));
 }
 
 /**
@@ -74,16 +75,54 @@
  * on the inactive_clean list it is placed on the inactive_dirty list
  * instead.
  *
- * Note: this function gets called with the pagemap_lru_lock held.
+ * Note: this function gets called with the lru lock held.
  */
+void drop_page_zone(struct zone_struct *zone, struct page * page)
+{
+ if (!TryLockPage(page)) {
+ if (page->mapping && page->buffers) {
+ page_cache_get(page);
+ lru_unlock(zone);
+ try_to_release_page(page, GFP_NOIO);
+ lru_lock(zone);
+ page_cache_release(page);
+ }
+ UnlockPage(page);
+ }
+
+ /* Make sure the page really is reclaimable. */
+ pte_chain_lock(page);
+ if (!page->mapping || PageDirty(page) || page->pte_chain ||
+ page->buffers || page_count(page) > 1)
+ deactivate_page_nolock(page);
+
+ else if (page_count(page) == 1) {
+ ClearPageReferenced(page);
+ if (PageActiveAnon(page)) {
+ del_page_from_active_anon_list(page);
+ add_page_to_inactive_clean_list(page);
+ } else if (PageActiveCache(page)) {
+ del_page_from_active_cache_list(page);
+ add_page_to_inactive_clean_list(page);
+ } else if (PageInactiveDirty(page)) {
+ del_page_from_inactive_dirty_list(page);
+ add_page_to_inactive_clean_list(page);
+ } else if (PageInactiveLaundry(page)) {
+ del_page_from_inactive_laundry_list(page);
+ add_page_to_inactive_clean_list(page);
+ }
+ }
+ pte_chain_unlock(page);
+}
+
 void drop_page(struct page * page)
 {
         if (!TryLockPage(page)) {
                 if (page->mapping && page->buffers) {
                         page_cache_get(page);
- spin_unlock(&pagemap_lru_lock);
+ lru_unlock(ALL_ZONES);
                         try_to_release_page(page, GFP_NOIO);
- spin_lock(&pagemap_lru_lock);
+ lru_lock(ALL_ZONES);
                         page_cache_release(page);
                 }
                 UnlockPage(page);
@@ -97,13 +136,18 @@
 
         else if (page_count(page) == 1) {
                 ClearPageReferenced(page);
- page->age = 0;
- if (PageActive(page)) {
- del_page_from_active_list(page);
+ if (PageActiveAnon(page)) {
+ del_page_from_active_anon_list(page);
+ add_page_to_inactive_clean_list(page);
+ } else if (PageActiveCache(page)) {
+ del_page_from_active_cache_list(page);
                         add_page_to_inactive_clean_list(page);
                 } else if (PageInactiveDirty(page)) {
                         del_page_from_inactive_dirty_list(page);
                         add_page_to_inactive_clean_list(page);
+ } else if (PageInactiveLaundry(page)) {
+ del_page_from_inactive_laundry_list(page);
+ add_page_to_inactive_clean_list(page);
                 }
         }
         pte_chain_unlock(page);
@@ -116,21 +160,21 @@
 {
         if (PageInactiveDirty(page)) {
                 del_page_from_inactive_dirty_list(page);
- add_page_to_active_list(page);
+ add_page_to_active_list(page, INITIAL_AGE);
+ } else if (PageInactiveLaundry(page)) {
+ del_page_from_inactive_laundry_list(page);
+ add_page_to_active_list(page, INITIAL_AGE);
         } else if (PageInactiveClean(page)) {
                 del_page_from_inactive_clean_list(page);
- add_page_to_active_list(page);
+ add_page_to_active_list(page, INITIAL_AGE);
         }
-
- /* Make sure the page gets a fair chance at staying active. */
- page->age = max((int)page->age, PAGE_AGE_START);
 }
 
 void activate_page(struct page * page)
 {
- spin_lock(&pagemap_lru_lock);
+ lru_lock(page_zone(page));
         activate_page_nolock(page);
- spin_unlock(&pagemap_lru_lock);
+ lru_unlock(page_zone(page));
 }
 
 /**
@@ -140,10 +184,10 @@
 void lru_cache_add(struct page * page)
 {
         if (!PageLRU(page)) {
- spin_lock(&pagemap_lru_lock);
+ lru_lock(page_zone(page));
                 SetPageLRU(page);
- add_page_to_active_list(page);
- spin_unlock(&pagemap_lru_lock);
+ add_page_to_active_list(page, INITIAL_AGE);
+ lru_unlock(page_zone(page));
         }
 }
 
@@ -152,14 +196,18 @@
  * @page: the page to add
  *
  * This function is for when the caller already holds
- * the pagemap_lru_lock.
+ * the lru lock.
  */
 void __lru_cache_del(struct page * page)
 {
- if (PageActive(page)) {
- del_page_from_active_list(page);
+ if (PageActiveAnon(page)) {
+ del_page_from_active_anon_list(page);
+ } else if (PageActiveCache(page)) {
+ del_page_from_active_cache_list(page);
         } else if (PageInactiveDirty(page)) {
                 del_page_from_inactive_dirty_list(page);
+ } else if (PageInactiveLaundry(page)) {
+ del_page_from_inactive_laundry_list(page);
         } else if (PageInactiveClean(page)) {
                 del_page_from_inactive_clean_list(page);
         }
@@ -172,9 +220,9 @@
  */
 void lru_cache_del(struct page * page)
 {
- spin_lock(&pagemap_lru_lock);
+ lru_lock(page_zone(page));
         __lru_cache_del(page);
- spin_unlock(&pagemap_lru_lock);
+ lru_unlock(page_zone(page));
 }
 
 /*
diff -Nru a/mm/swap_state.c b/mm/swap_state.c
--- a/mm/swap_state.c Fri Nov 22 00:36:57 2002
+++ b/mm/swap_state.c Fri Nov 22 00:36:57 2002
@@ -196,7 +196,7 @@
 {
         struct page *found;
 
- found = find_get_page(&swapper_space, entry.val);
+ found = find_pagecache_page(&swapper_space, entry.val);
         /*
          * Unsafe to assert PageSwapCache and mapping on page found:
          * if SMP nothing prevents swapoff from deleting this page from
@@ -224,10 +224,10 @@
                 /*
                  * First check the swap cache. Since this is normally
                  * called after lookup_swap_cache() failed, re-calling
- * that would confuse statistics: use find_get_page()
+ * that would confuse statistics: use find_pagecache_page()
                  * directly.
                  */
- found_page = find_get_page(&swapper_space, entry.val);
+ found_page = find_pagecache_page(&swapper_space, entry.val);
                 if (found_page)
                         break;
 
diff -Nru a/mm/vmscan.c b/mm/vmscan.c
--- a/mm/vmscan.c Fri Nov 22 00:36:57 2002
+++ b/mm/vmscan.c Fri Nov 22 00:36:57 2002
@@ -12,6 +12,7 @@
  * to bring the system back to freepages.high: 2.4.97, Rik van Riel.
  * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
  * Multiqueue VM started 5.8.00, Rik van Riel.
+ * O(1) rmap vm, Arjan van de ven <arjanv@redhat.com>
  */
 
 #include <linux/slab.h>
@@ -37,16 +38,36 @@
  */
 #define DEF_PRIORITY (6)
 
-static inline void age_page_up(struct page *page)
+static inline void age_page_up_nolock(struct page *page, int old_age)
 {
- page->age = min((int) (page->age + PAGE_AGE_ADV), PAGE_AGE_MAX);
-}
+ int new_age;
+
+ new_age = old_age+4;
+ if (new_age < 0)
+ new_age = 0;
+ if (new_age > MAX_AGE)
+ new_age = MAX_AGE;
+
+ if (PageActiveAnon(page)) {
+ del_page_from_active_anon_list(page);
+ add_page_to_active_anon_list(page, new_age);
+ } else if (PageActiveCache(page)) {
+ del_page_from_active_cache_list(page);
+ add_page_to_active_cache_list(page, new_age);
+ } else if (PageInactiveDirty(page)) {
+ del_page_from_inactive_dirty_list(page);
+ add_page_to_active_list(page, new_age);
+ } else if (PageInactiveLaundry(page)) {
+ del_page_from_inactive_laundry_list(page);
+ add_page_to_active_list(page, new_age);
+ } else if (PageInactiveClean(page)) {
+ del_page_from_inactive_clean_list(page);
+ add_page_to_active_list(page, new_age);
+ } else return;
 
-static inline void age_page_down(struct page *page)
-{
- page->age -= min(PAGE_AGE_DECL, (int)page->age);
 }
 
+
 /* Must be called with page's pte_chain_lock held. */
 static inline int page_mapping_inuse(struct page * page)
 {
@@ -84,9 +105,9 @@
 
         /*
          * We need to hold the pagecache_lock around all tests to make sure
- * reclaim_page() cannot race with find_get_page() and friends.
+ * reclaim_page() doesn't race with other pagecache users
          */
- spin_lock(&pagemap_lru_lock);
+ lru_lock(zone);
         spin_lock(&pagecache_lock);
         maxscan = zone->inactive_clean_pages;
         while (maxscan-- && !list_empty(&zone->inactive_clean_list)) {
@@ -94,12 +115,7 @@
                 page = list_entry(page_lru, struct page, lru);
 
                 /* Wrong page on list?! (list corruption, should not happen) */
- if (unlikely(!PageInactiveClean(page))) {
- printk("VM: reclaim_page, wrong page on list.\n");
- list_del(page_lru);
- page_zone(page)->inactive_clean_pages--;
- continue;
- }
+ BUG_ON(unlikely(!PageInactiveClean(page)));
 
                 /* Page is being freed */
                 if (unlikely(page_count(page)) == 0) {
@@ -144,7 +160,7 @@
                 UnlockPage(page);
         }
         spin_unlock(&pagecache_lock);
- spin_unlock(&pagemap_lru_lock);
+ lru_unlock(zone);
         return NULL;
 
 
@@ -152,11 +168,10 @@
         __lru_cache_del(page);
         pte_chain_unlock(page);
         spin_unlock(&pagecache_lock);
- spin_unlock(&pagemap_lru_lock);
+ lru_unlock(zone);
         if (entry.val)
                 swap_free(entry);
         UnlockPage(page);
- page->age = PAGE_AGE_START;
         if (page_count(page) != 1)
                 printk("VM: reclaim_page, found page with count %d!\n",
                                 page_count(page));
@@ -164,458 +179,626 @@
 }
 
 /**
- * page_dirty - do we need to write the data out to disk
- * @page: page to test
+ * need_rebalance_dirty - do we need to write inactive stuff to disk?
+ * @zone: the zone in question
  *
- * Returns true if the page contains data which needs to
- * be written to disk. Doesn't test the page tables (yet?).
+ * Returns true if the zone in question has an inbalance between inactive
+ * dirty on one side and inactive laundry + inactive clean on the other
+ * Right now set the balance at 50%; may need tuning later on
  */
-static inline int page_dirty(struct page *page)
+static inline int need_rebalance_dirty(zone_t * zone)
 {
- struct buffer_head *tmp, *bh;
-
- if (PageDirty(page))
+ if (zone->inactive_dirty_pages > zone->inactive_laundry_pages + zone->inactive_clean_pages)
                 return 1;
 
- if (page->mapping && !page->buffers)
- return 0;
-
- tmp = bh = page->buffers;
-
- do {
- if (tmp->b_state & ((1<<BH_Dirty) | (1<<BH_Lock)))
- return 1;
- tmp = tmp->b_this_page;
- } while (tmp != bh);
+ return 0;
+}
 
+/**
+ * need_rebalance_laundry - does the zone have too few inactive_clean pages?
+ * @zone: the zone in question
+ *
+ * Returns true if the zone in question has too few pages in inactive clean
+ * + free
+ */
+static inline int need_rebalance_laundry(zone_t * zone)
+{
+ if (free_low(zone) >= 0)
+ return 1;
         return 0;
 }
 
 /**
- * page_launder_zone - clean dirty inactive pages, move to inactive_clean list
+ * launder_page - clean dirty page, move to inactive_laundry list
  * @zone: zone to free pages in
  * @gfp_mask: what operations we are allowed to do
- * @full_flush: full-out page flushing, if we couldn't get enough clean pages
+ * @page: the page at hand, must be on the inactive dirty list
  *
- * This function is called when we are low on free / inactive_clean
- * pages, its purpose is to refill the free/clean list as efficiently
- * as possible.
- *
- * This means we do writes asynchronously as long as possible and will
- * only sleep on IO when we don't have another option. Since writeouts
- * cause disk seeks and make read IO slower, we skip writes alltogether
- * when the amount of dirty pages is small.
- *
- * This code is heavily inspired by the FreeBSD source code. Thanks
- * go out to Matthew Dillon.
- */
-int page_launder_zone(zone_t * zone, int gfp_mask, int full_flush)
-{
- int maxscan, cleaned_pages, target, maxlaunder, iopages, over_rsslimit;
- struct list_head * entry, * next;
-
- target = max_t(int, free_plenty(zone), zone->pages_min);
- cleaned_pages = iopages = 0;
-
- /* If we can get away with it, only flush 2 MB worth of dirty pages */
- if (full_flush)
- maxlaunder = 1000000;
- else {
- maxlaunder = min_t(int, 512, zone->inactive_dirty_pages / 4);
- maxlaunder = max(maxlaunder, free_plenty(zone) * 4);
- }
-
- /* The main launder loop. */
- spin_lock(&pagemap_lru_lock);
-rescan:
- maxscan = zone->inactive_dirty_pages;
- entry = zone->inactive_dirty_list.prev;
- next = entry->prev;
- while (maxscan-- && !list_empty(&zone->inactive_dirty_list) &&
- next != &zone->inactive_dirty_list) {
- struct page * page;
-
- /* Low latency reschedule point */
- if (current->need_resched) {
- spin_unlock(&pagemap_lru_lock);
- schedule();
- spin_lock(&pagemap_lru_lock);
- continue;
- }
-
- entry = next;
- next = entry->prev;
- page = list_entry(entry, struct page, lru);
-
- /* This page was removed while we looked the other way. */
- if (!PageInactiveDirty(page))
- goto rescan;
+ * per-zone lru lock is assumed to be held, but this function can drop
+ * it and sleep, so no other locks are allowed to be held.
+ *
+ * returns 0 for failure; 1 for success
+ */
+int launder_page(zone_t * zone, int gfp_mask, struct page *page)
+{
+ int over_rsslimit;
 
- if (cleaned_pages > target)
- break;
+ /*
+ * Page is being freed, don't worry about it, but report progress.
+ */
+ if (unlikely(page_count(page)) == 0)
+ return 1;
 
- /* Stop doing IO if we've laundered too many pages already. */
- if (maxlaunder < 0)
- gfp_mask &= ~(__GFP_IO|__GFP_FS);
+ BUG_ON(!PageInactiveDirty(page));
+ del_page_from_inactive_dirty_list(page);
+ add_page_to_inactive_laundry_list(page);
+ /* store the time we start IO */
+ page->age = (jiffies/HZ)&255;
+ /*
+ * The page is locked. IO in progress?
+ * If so, move to laundry and report progress
+ * Acquire PG_locked early in order to safely
+ * access page->mapping.
+ */
+ if (unlikely(TryLockPage(page))) {
+ return 1;
+ }
 
- /*
- * Page is being freed, don't worry about it.
- */
- if (unlikely(page_count(page)) == 0)
- continue;
+ /*
+ * The page is in active use or really unfreeable. Move to
+ * the active list and adjust the page age if needed.
+ */
+ pte_chain_lock(page);
+ if (page_referenced(page, &over_rsslimit) && !over_rsslimit &&
+ page_mapping_inuse(page)) {
+ del_page_from_inactive_laundry_list(page);
+ add_page_to_active_list(page, INITIAL_AGE);
+ pte_chain_unlock(page);
+ UnlockPage(page);
+ return 1;
+ }
 
- /*
- * The page is locked. IO in progress?
- * Acquire PG_locked early in order to safely
- * access page->mapping.
- */
- if (unlikely(TryLockPage(page))) {
- iopages++;
- continue;
+ /*
+ * Anonymous process memory without backing store. Try to
+ * allocate it some swap space here.
+ *
+ * XXX: implement swap clustering ?
+ */
+ if (page->pte_chain && !page->mapping && !page->buffers) {
+ page_cache_get(page);
+ pte_chain_unlock(page);
+ lru_unlock(zone);
+ if (!add_to_swap(page)) {
+ activate_page(page);
+ lru_lock(zone);
+ UnlockPage(page);
+ page_cache_release(page);
+ return 0;
                 }
-
- /*
- * The page is in active use or really unfreeable. Move to
- * the active list and adjust the page age if needed.
- */
- pte_chain_lock(page);
- if (page_referenced(page, &over_rsslimit) && !over_rsslimit &&
- page_mapping_inuse(page)) {
- del_page_from_inactive_dirty_list(page);
- add_page_to_active_list(page);
- page->age = max((int)page->age, PAGE_AGE_START);
- pte_chain_unlock(page);
+ lru_lock(zone);
+ page_cache_release(page);
+ /* Note: may be on another list ! */
+ if (!PageInactiveLaundry(page)) {
                         UnlockPage(page);
- continue;
+ return 1;
+ }
+ if (unlikely(page_count(page)) == 0) {
+ UnlockPage(page);
+ return 1;
                 }
+ pte_chain_lock(page);
+ }
 
- /*
- * Anonymous process memory without backing store. Try to
- * allocate it some swap space here.
- *
- * XXX: implement swap clustering ?
- */
- if (page->pte_chain && !page->mapping && !page->buffers) {
- /* Don't bother if we can't swap it out now. */
- if (maxlaunder < 0) {
+ /*
+ * The page is mapped into the page tables of one or more
+ * processes. Try to unmap it here.
+ */
+ if (page->pte_chain && page->mapping) {
+ switch (try_to_unmap(page)) {
+ case SWAP_ERROR:
+ case SWAP_FAIL:
+ goto page_active;
+ case SWAP_AGAIN:
                                 pte_chain_unlock(page);
                                 UnlockPage(page);
- list_del(entry);
- list_add(entry, &zone->inactive_dirty_list);
- continue;
- }
- page_cache_get(page);
- pte_chain_unlock(page);
- spin_unlock(&pagemap_lru_lock);
- if (!add_to_swap(page)) {
- activate_page(page);
- UnlockPage(page);
- page_cache_release(page);
- spin_lock(&pagemap_lru_lock);
- continue;
- }
- page_cache_release(page);
- spin_lock(&pagemap_lru_lock);
- pte_chain_lock(page);
+ return 0;
+ case SWAP_SUCCESS:
+ ; /* fall through, try freeing the page below */
+ /* fixme: add a SWAP_MLOCK case */
                 }
+ }
+ pte_chain_unlock(page);
 
+ if (PageDirty(page) && page->mapping) {
                 /*
- * The page is mapped into the page tables of one or more
- * processes. Try to unmap it here.
+ * The page can be dirtied after we start writing, but
+ * in that case the dirty bit will simply be set again
+ * and we'll need to write it again.
                  */
- if (page->pte_chain && page->mapping) {
- switch (try_to_unmap(page)) {
- case SWAP_ERROR:
- case SWAP_FAIL:
- goto page_active;
- case SWAP_AGAIN:
- pte_chain_unlock(page);
- UnlockPage(page);
- continue;
- case SWAP_SUCCESS:
- ; /* try to free the page below */
- }
+ int (*writepage)(struct page *);
+
+ writepage = page->mapping->a_ops->writepage;
+ if ((gfp_mask & __GFP_FS) && writepage) {
+ ClearPageDirty(page);
+ SetPageLaunder(page);
+ page_cache_get(page);
+ lru_unlock(zone);
+
+ writepage(page);
+
+ page_cache_release(page);
+ lru_lock(zone);
+ return 1;
+ } else {
+ del_page_from_inactive_laundry_list(page);
+ add_page_to_inactive_dirty_list(page);
+ /* FIXME: this is wrong for !__GFP_FS !!! */
+ UnlockPage(page);
+ return 0;
                 }
- pte_chain_unlock(page);
+ }
 
- if (PageDirty(page) && page->mapping) {
- /*
- * It is not critical here to write it only if
- * the page is unmapped beause any direct writer
- * like O_DIRECT would set the PG_dirty bitflag
- * on the physical page after having successfully
- * pinned it and after the I/O to the page is finished,
- * so the direct writes to the page cannot get lost.
- */
- int (*writepage)(struct page *);
+ /*
+ * If the page has buffers, try to free the buffer mappings
+ * associated with this page. If we succeed we try to free
+ * the page as well.
+ */
+ if (page->buffers) {
+ /* To avoid freeing our page before we're done. */
+ page_cache_get(page);
+ lru_unlock(zone);
 
- writepage = page->mapping->a_ops->writepage;
- if ((gfp_mask & __GFP_FS) && writepage) {
- ClearPageDirty(page);
- SetPageLaunder(page);
- page_cache_get(page);
- spin_unlock(&pagemap_lru_lock);
+ try_to_release_page(page, gfp_mask);
+ UnlockPage(page);
 
- writepage(page);
- maxlaunder--;
- iopages++;
- page_cache_release(page);
+ /*
+ * If the buffers were the last user of the page we free
+ * the page here. Because of that we shouldn't hold the
+ * lru lock yet.
+ */
+ page_cache_release(page);
 
- spin_lock(&pagemap_lru_lock);
- continue;
- } else {
- UnlockPage(page);
- list_del(entry);
- list_add(entry, &zone->inactive_dirty_list);
- continue;
- }
- }
+ lru_lock(zone);
+ return 1;
+ }
 
+ /*
+ * If the page is really freeable now, move it to the
+ * inactive_laundry list to keep LRU order.
+ *
+ * We re-test everything since the page could have been
+ * used by somebody else while we waited on IO above.
+ * This test is not safe from races; only the one in
+ * reclaim_page() needs to be.
+ */
+ pte_chain_lock(page);
+ if (page->mapping && !PageDirty(page) && !page->pte_chain &&
+ page_count(page) == 1) {
+ pte_chain_unlock(page);
+ UnlockPage(page);
+ return 1;
+ } else {
                 /*
- * If the page has buffers, try to free the buffer mappings
- * associated with this page. If we succeed we try to free
- * the page as well.
+ * OK, we don't know what to do with the page.
+ * It's no use keeping it here, so we move it
+ * back to the active list.
                  */
- if (page->buffers) {
- /* To avoid freeing our page before we're done. */
- page_cache_get(page);
+ page_active:
+ activate_page_nolock(page);
+ pte_chain_unlock(page);
+ UnlockPage(page);
+ }
+ return 0;
+}
 
- spin_unlock(&pagemap_lru_lock);
 
- if (try_to_release_page(page, gfp_mask)) {
- if (!page->mapping) {
- /*
- * We must not allow an anon page
- * with no buffers to be visible on
- * the LRU, so we unlock the page after
- * taking the lru lock
- */
- spin_lock(&pagemap_lru_lock);
- UnlockPage(page);
- __lru_cache_del(page);
+unsigned char active_age_bias = 0;
 
- /* effectively free the page here */
- page_cache_release(page);
+/* Ages down all pages on the active list */
+/* assumes the lru lock held */
+static inline void kachunk_anon(struct zone_struct * zone)
+{
+ int k;
+ if (!list_empty(&zone->active_anon_list[0]))
+ return;
+ if (!zone->active_anon_pages)
+ return;
 
- cleaned_pages++;
- continue;
- } else {
- /*
- * We freed the buffers but may have
- * slept; undo the stuff we did before
- * try_to_release_page and fall through
- * to the next step.
- * But only if the page is still on the inact. dirty
- * list.
- */
-
- spin_lock(&pagemap_lru_lock);
- /* Check if the page was removed from the list
- * while we looked the other way.
- */
- if (!PageInactiveDirty(page)) {
- page_cache_release(page);
- continue;
- }
- page_cache_release(page);
- }
- } else {
- /* failed to drop the buffers so stop here */
- UnlockPage(page);
- page_cache_release(page);
- maxlaunder--;
- iopages++;
+ for (k = 0; k < MAX_AGE; k++) {
+ list_splice_init(&zone->active_anon_list[k+1], &zone->active_anon_list[k]);
+ zone->active_anon_count[k] = zone->active_anon_count[k+1];
+ zone->active_anon_count[k+1] = 0;
+ }
+
+ active_age_bias++;
+ /* flag this zone as having had activity -> rescan to age up is desired */
+ zone->need_scan++;
+}
+
+static inline void kachunk_cache(struct zone_struct * zone)
+{
+ int k;
+ if (!list_empty(&zone->active_cache_list[0]))
+ return;
+ if (!zone->active_cache_pages)
+ return;
+
+ for (k = 0; k < MAX_AGE; k++) {
+ list_splice_init(&zone->active_cache_list[k+1], &zone->active_cache_list[k]);
+ zone->active_cache_count[k] = zone->active_cache_count[k+1];
+ zone->active_cache_count[k+1] = 0;
+ }
 
- spin_lock(&pagemap_lru_lock);
+ active_age_bias++;
+ /* flag this zone as having had activity -> rescan to age up is desired */
+ zone->need_scan++;
+}
+
+#define BATCH_WORK_AMOUNT 64
+
+/*
+ * returns the active cache ratio relative to the total active list
+ * times 10 (eg. 30% cache returns 3)
+ */
+static inline int cache_ratio(struct zone_struct * zone)
+{
+ if (!zone->size)
+ return 0;
+ return 10 * zone->active_cache_pages / (zone->active_cache_pages +
+ zone->active_anon_pages + 1);
+}
+
+/*
+ * If the active_cache list is more than 20% of all active pages,
+ * we do extra heavy reclaim from this list and less reclaiming of
+ * the active_anon pages.
+ * These arrays are indexed by cache_ratio(), ie 0%, 10%, 20% ... 100%
+ */
+static int active_anon_work[11] = {32, 32, 12, 4, 2, 1, 1, 1, 1, 1, 1};
+static int active_cache_work[11] = {32, 32, 52, 60, 62, 63, 63, 63, 63, 63, 63};
+
+/**
+ * refill_inactive_zone - scan the active list and find pages to deactivate
+ * @priority: how much are we allowed to scan
+ *
+ * This function will scan a portion of the active list of a zone to find
+ * unused pages, those pages will then be moved to the inactive list.
+ */
+int refill_inactive_zone(struct zone_struct * zone, int priority, int target)
+{
+ int maxscan = (zone->active_anon_pages + zone->active_cache_pages) >> priority;
+ struct list_head * page_lru;
+ struct page * page;
+ int over_rsslimit;
+ int progress = 0;
+ int ratio;
+
+ /* Take the lock while messing with the list... */
+ lru_lock(zone);
+ if (target < BATCH_WORK_AMOUNT)
+ target = BATCH_WORK_AMOUNT;
+
+ ratio = cache_ratio(zone);
+
+ while (maxscan-- && zone->active_anon_pages + zone->active_cache_pages > 0 && target > 0) {
+ int anon_work, cache_work;
+ anon_work = active_anon_work[ratio];
+ cache_work = active_cache_work[ratio];
+
+ while (anon_work-- >= 0 && zone->active_anon_pages) {
+ if (list_empty(&zone->active_anon_list[0])) {
+ kachunk_anon(zone);
                                 continue;
                         }
- }
 
+ page_lru = zone->active_anon_list[0].prev;
+ page = list_entry(page_lru, struct page, lru);
 
- /*
- * If the page is really freeable now, move it to the
- * inactive_clean list.
- *
- * We re-test everything since the page could have been
- * used by somebody else while we waited on IO above.
- * This test is not safe from races, but only the one
- * in reclaim_page() needs to be.
- */
- pte_chain_lock(page);
- if (page->mapping && !PageDirty(page) && !page->pte_chain &&
- page_count(page) == 1) {
- del_page_from_inactive_dirty_list(page);
- add_page_to_inactive_clean_list(page);
+ /* Wrong page on list?! (list corruption, should not happen) */
+ BUG_ON(unlikely(!PageActiveAnon(page)));
+
+ /* Needed to follow page->mapping */
+ if (TryLockPage(page)) {
+ /* The page is already locked. This for sure means
+ * someone is doing stuff with it which makes it
+ * active by definition ;)
+ */
+ del_page_from_active_anon_list(page);
+ add_page_to_active_anon_list(page, INITIAL_AGE);
+ continue;
+ }
+
+ /*
+ * Do aging on the pages.
+ */
+ pte_chain_lock(page);
+ if (page_referenced(page, &over_rsslimit) && !over_rsslimit) {
+ pte_chain_unlock(page);
+ age_page_up_nolock(page, 0);
+ UnlockPage(page);
+ continue;
+ }
                         pte_chain_unlock(page);
+
+ deactivate_page_nolock(page);
+ target--;
+ progress++;
                         UnlockPage(page);
- cleaned_pages++;
- } else {
+ }
+
+ while (cache_work-- >= 0 && zone->active_cache_pages) {
+ if (list_empty(&zone->active_cache_list[0])) {
+ kachunk_cache(zone);
+ continue;
+ }
+
+ page_lru = zone->active_cache_list[0].prev;
+ page = list_entry(page_lru, struct page, lru);
+
+ /* Wrong page on list?! (list corruption, should not happen) */
+ BUG_ON(unlikely(!PageActiveCache(page)));
+
+ /* Needed to follow page->mapping */
+ if (TryLockPage(page)) {
+ /* The page is already locked. This for sure means
+ * someone is doing stuff with it which makes it
+ * active by definition ;)
+ */
+ del_page_from_active_cache_list(page);
+ add_page_to_active_cache_list(page, INITIAL_AGE);
+ continue;
+ }
+
                         /*
- * OK, we don't know what to do with the page.
- * It's no use keeping it here, so we move it to
- * the active list.
+ * Do aging on the pages.
                          */
-page_active:
- del_page_from_inactive_dirty_list(page);
- add_page_to_active_list(page);
+ pte_chain_lock(page);
+ if (page_referenced(page, &over_rsslimit) && !over_rsslimit) {
+ pte_chain_unlock(page);
+ age_page_up_nolock(page, 0);
+ UnlockPage(page);
+ continue;
+ }
                         pte_chain_unlock(page);
+
+ deactivate_page_nolock(page);
+ target--;
+ progress++;
                         UnlockPage(page);
                 }
         }
- spin_unlock(&pagemap_lru_lock);
+ lru_unlock(zone);
 
- /* Return the number of pages moved to the inactive_clean list. */
- return cleaned_pages + iopages;
+ return progress;
 }
 
-/**
- * page_launder - clean dirty inactive pages, move to inactive_clean list
- * @gfp_mask: what operations we are allowed to do
- *
- * This function iterates over all zones and calls page_launder_zone(),
- * balancing still needs to be added...
- */
-int page_launder(int gfp_mask)
+static int need_active_anon_scan(struct zone_struct * zone)
 {
- struct zone_struct * zone;
- int freed = 0;
+ int low = 0, high = 0;
+ int k;
+ for (k=0; k < MAX_AGE/2; k++)
+ low += zone->active_anon_count[k];
 
- /* Global balancing while we have a global shortage. */
- if (free_high(ALL_ZONES) >= 0)
- for_each_zone(zone)
- if (free_plenty(zone) >= 0)
- freed += page_launder_zone(zone, gfp_mask, 0);
-
- /* Clean up the remaining zones with a serious shortage, if any. */
- for_each_zone(zone)
- if (free_low(zone) >= 0) {
- int fullflush = free_min(zone) > 0;
- freed += page_launder_zone(zone, gfp_mask, fullflush);
- }
+ for (k=MAX_AGE/2; k <= MAX_AGE; k++)
+ high += zone->active_anon_count[k];
+
+ if (high<low)
+ return 1;
+ return 0;
+}
+
+static int need_active_cache_scan(struct zone_struct * zone)
+{
+ int low = 0, high = 0;
+ int k;
+ for (k=0; k < MAX_AGE/2; k++)
+ low += zone->active_cache_count[k];
+
+ for (k=MAX_AGE/2; k <= MAX_AGE; k++)
+ high += zone->active_cache_count[k];
 
- return freed;
+ if (high<low)
+ return 1;
+ return 0;
 }
 
-/**
- * refill_inactive_zone - scan the active list and find pages to deactivate
- * @priority: how much are we allowed to scan
- *
- * This function will scan a portion of the active list of a zone to find
- * unused pages, those pages will then be moved to the inactive list.
+static int scan_active_list(struct zone_struct * zone, int age, int anon)
+{
+ struct list_head * list, *page_lru , *next;
+ struct page * page;
+ int over_rsslimit;
+
+ if (anon)
+ list = &zone->active_anon_list[age];
+ else
+ list = &zone->active_cache_list[age];
+
+ /* Take the lock while messing with the list... */
+ lru_lock(zone);
+ list_for_each_safe(page_lru, next, list) {
+ page = list_entry(page_lru, struct page, lru);
+ pte_chain_lock(page);
+ if (page_referenced(page, &over_rsslimit) && !over_rsslimit)
+ age_page_up_nolock(page, age);
+ pte_chain_unlock(page);
+ }
+ lru_unlock(zone);
+ return 0;
+}
+
+/*
+ * Move max_work pages to the inactive clean list as long as there is a need
+ * for this. If gfp_mask allows it, sleep for IO to finish.
  */
-int refill_inactive_zone(struct zone_struct * zone, int priority)
+int rebalance_laundry_zone(struct zone_struct * zone, int max_work, unsigned int gfp_mask)
 {
- int maxscan = zone->active_pages >> priority;
- int nr_deactivated = 0, over_rsslimit;
- int target = inactive_high(zone);
         struct list_head * page_lru;
+ int max_loop;
+ int work_done = 0;
         struct page * page;
 
+ max_loop = max_work;
+ if (max_loop < BATCH_WORK_AMOUNT)
+ max_loop = BATCH_WORK_AMOUNT;
         /* Take the lock while messing with the list... */
- spin_lock(&pagemap_lru_lock);
- while (maxscan-- && !list_empty(&zone->active_list)) {
- page_lru = zone->active_list.prev;
+ lru_lock(zone);
+ while (max_loop-- && !list_empty(&zone->inactive_laundry_list)) {
+ page_lru = zone->inactive_laundry_list.prev;
                 page = list_entry(page_lru, struct page, lru);
 
                 /* Wrong page on list?! (list corruption, should not happen) */
- if (unlikely(!PageActive(page))) {
- printk("VM: refill_inactive, wrong page on list.\n");
- list_del(page_lru);
- nr_active_pages--;
- continue;
- }
-
- /* Needed to follow page->mapping */
+ BUG_ON(unlikely(!PageInactiveLaundry(page)));
+
+ /* TryLock to see if the page IO is done */
                 if (TryLockPage(page)) {
- list_del(page_lru);
- list_add(page_lru, &zone->active_list);
- continue;
+ /*
+ * Page is locked (IO in progress?). If we can sleep,
+ * wait for it to finish, except when we've already
+ * done enough work.
+ */
+ if ((gfp_mask & __GFP_WAIT) && (work_done < max_work)) {
+ int timed_out;
+
+ page_cache_get(page);
+ lru_unlock(zone);
+ run_task_queue(&tq_disk);
+ timed_out = wait_on_page_timeout(page, 5 * HZ);
+ lru_lock(zone);
+ page_cache_release(page);
+ /*
+ * If we timed out and the page has been in
+ * flight for over 30 seconds, this might not
+ * be the best page to wait on; move it to
+ * the head of the dirty list.
+ */
+ if (timed_out & PageInactiveLaundry(page)) {
+ unsigned char now;
+ now = (jiffies/HZ)&255;
+ if (now - page->age > 30) {
+ del_page_from_inactive_laundry_list(page);
+ add_page_to_inactive_dirty_list(page);
+ }
+ continue;
+ }
+ /* We didn't make any progress for our caller,
+ * but we are actively avoiding a livelock
+ * so undo the decrement and wait on this page
+ * some more, until IO finishes or we timeout.
+ */
+ max_loop++;
+ continue;
+ } else
+ /* No dice, we can't wait for IO */
+ break;
                 }
+ UnlockPage(page);
 
                 /*
- * If the object the page is in is not in use we don't
- * bother with page aging. If the page is touched again
- * while on the inactive_clean list it'll be reactivated.
- * From here until the end of the current iteration
- * both PG_locked and the pte_chain_lock are held.
+ * If we get here either the IO on the page is done or
+ * IO never happened because it was clean. Either way
+ * move it to the inactive clean list.
                  */
- pte_chain_lock(page);
- if (!page_mapping_inuse(page)) {
- pte_chain_unlock(page);
- UnlockPage(page);
- drop_page(page);
- continue;
- }
+
+ /* FIXME: check if the page is still clean or is accessed ? */
+
+ del_page_from_inactive_laundry_list(page);
+ add_page_to_inactive_clean_list(page);
+ work_done++;
 
                 /*
- * Do aging on the pages.
+ * If we've done the minimal batch of work and there's
+ * no longer a need to rebalance, abort now.
                  */
- if (page_referenced(page, &over_rsslimit)) {
- age_page_up(page);
- } else {
- age_page_down(page);
- }
+ if ((work_done > BATCH_WORK_AMOUNT) && (!need_rebalance_laundry(zone)))
+ break;
+ }
 
- /*
- * If the page age is 'hot' and the process using the
- * page doesn't exceed its RSS limit we keep the page.
- * Otherwise we move it to the inactive_dirty list.
+ lru_unlock(zone);
+ return work_done;
+}
+
+/*
+ * Move max_work pages from the dirty list as long as there is a need.
+ * Start IO if the gfp_mask allows it.
+ */
+int rebalance_dirty_zone(struct zone_struct * zone, int max_work, unsigned int gfp_mask)
+{
+ struct list_head * page_lru;
+ int max_loop;
+ int work_done = 0;
+ struct page * page;
+
+ max_loop = max_work;
+ if (max_loop < BATCH_WORK_AMOUNT)
+ max_loop = BATCH_WORK_AMOUNT;
+ /* Take the lock while messing with the list... */
+ lru_lock(zone);
+ while (max_loop-- && !list_empty(&zone->inactive_dirty_list)) {
+ page_lru = zone->inactive_dirty_list.prev;
+ page = list_entry(page_lru, struct page, lru);
+
+ /* Wrong page on list?! (list corruption, should not happen) */
+ BUG_ON(unlikely(!PageInactiveDirty(page)));
+
+ /*
+ * Note: launder_page() sleeps so we can't safely look at
+ * the page after this point!
+ *
+ * If we fail (only happens if we can't do IO) we just try
+ * again on another page; launder_page makes sure we won't
+ * see the same page over and over again.
                  */
- if (page->age && !over_rsslimit) {
- list_del(page_lru);
- list_add(page_lru, &zone->active_list);
- } else {
- deactivate_page_nolock(page);
- if (++nr_deactivated > target) {
- pte_chain_unlock(page);
- UnlockPage(page);
- goto done;
- }
- }
- pte_chain_unlock(page);
- UnlockPage(page);
+ if (!launder_page(zone, gfp_mask, page))
+ continue;
 
- /* Low latency reschedule point */
- if (current->need_resched) {
- spin_unlock(&pagemap_lru_lock);
- schedule();
- spin_lock(&pagemap_lru_lock);
- }
+ work_done++;
+
+ /*
+ * If we've done the minimal batch of work and there's
+ * no longer any need to rebalance, abort now.
+ */
+ if ((work_done > BATCH_WORK_AMOUNT) && (!need_rebalance_dirty(zone)))
+ break;
         }
+ lru_unlock(zone);
+
+ return work_done;
+}
+
+/* goal percentage sets the goal of the laundry+clean+free of the total zone size */
+int rebalance_inactive_zone(struct zone_struct * zone, int max_work, unsigned int gfp_mask, int goal_percentage)
+{
+ int ret = 0;
+ /* first deactivate memory */
+ if (((zone->inactive_laundry_pages + zone->inactive_clean_pages + zone->free_pages)*100 < zone->size * goal_percentage) &&
+ (inactive_high(zone) > 0))
+ refill_inactive_zone(zone, 0, max_work + BATCH_WORK_AMOUNT);
+
+ if (need_rebalance_dirty(zone))
+ ret += rebalance_dirty_zone(zone, max_work, gfp_mask);
+ if (need_rebalance_laundry(zone))
+ ret += rebalance_laundry_zone(zone, max_work, gfp_mask);
 
-done:
- spin_unlock(&pagemap_lru_lock);
+ /* These pages will become freeable, let the OOM detection know */
+ ret += zone->inactive_laundry_pages;
 
- return nr_deactivated;
+ return ret;
 }
 
-/**
- * refill_inactive - checks all zones and refills the inactive list as needed
- *
- * This function tries to balance page eviction from all zones by aging
- * the pages from each zone in the same ratio until the global inactive
- * shortage is resolved. After that it does one last "clean-up" scan to
- * fix up local inactive shortages.
- */
-int refill_inactive(void)
+int rebalance_inactive(unsigned int gfp_mask, int percentage)
 {
- int maxtry = 1 << DEF_PRIORITY;
- zone_t * zone;
+ struct zone_struct * zone;
+ int max_work;
         int ret = 0;
 
- /* Global balancing while we have a global shortage. */
- while (maxtry-- && inactive_low(ALL_ZONES) >= 0) {
- for_each_zone(zone) {
- if (inactive_high(zone) >= 0)
- ret += refill_inactive_zone(zone, DEF_PRIORITY);
- }
- }
+ max_work = 4 * BATCH_WORK_AMOUNT;
+ /* If we're in deeper trouble, do more work */
+ if (percentage >= 50)
+ max_work = 8 * BATCH_WORK_AMOUNT;
 
- /* Local balancing for zones which really need it. */
- for_each_zone(zone) {
- if (inactive_min(zone) >= 0)
- ret += refill_inactive_zone(zone, 0);
- }
+ for_each_zone(zone)
+ ret += rebalance_inactive_zone(zone, max_work, gfp_mask, percentage);
+ /* 4 * BATCH_WORK_AMOUNT needs tuning */
 
         return ret;
 }
@@ -636,7 +819,9 @@
 
         for_each_zone(zone)
                 if (inactive_high(zone) > 0)
- refill_inactive_zone(zone, priority);
+ refill_inactive_zone(zone, priority, BATCH_WORK_AMOUNT);
+ for_each_zone(zone)
+ rebalance_dirty_zone(zone, BATCH_WORK_AMOUNT, GFP_KSWAPD);
 }
 
 /*
@@ -655,18 +840,13 @@
          * Eat memory from filesystem page cache, buffer cache,
          * dentry, inode and filesystem quota caches.
          */
- ret += page_launder(gfp_mask);
+ ret += rebalance_inactive(gfp_mask, 100);
         ret += shrink_dcache_memory(DEF_PRIORITY, gfp_mask);
         ret += shrink_icache_memory(1, gfp_mask);
 #ifdef CONFIG_QUOTA
         ret += shrink_dqcache_memory(DEF_PRIORITY, gfp_mask);
 #endif
 
- /*
- * Move pages from the active list to the inactive list.
- */
- refill_inactive();
-
         /*
          * Reclaim unused slab cache memory.
          */
@@ -682,12 +862,54 @@
          * Hmm.. Cache shrink failed - time to kill something?
          * Mhwahahhaha! This is the part I really like. Giggle.
          */
- if (ret < free_low(ANY_ZONE))
+ if (ret < free_low(ANY_ZONE) && (gfp_mask&__GFP_WAIT))
                 out_of_memory();
 
         return ret;
 }
 
+/*
+ * Worker function for kswapd and try_to_free_pages, we get
+ * called whenever there is a shortage of free/inactive_clean
+ * pages.
+ *
+ * This function will also move pages to the inactive list,
+ * if needed.
+ */
+static int do_try_to_free_pages_kswapd(unsigned int gfp_mask)
+{
+ int ret = 0;
+ struct zone_struct * zone;
+
+ ret += shrink_dcache_memory(DEF_PRIORITY, gfp_mask);
+ ret += shrink_icache_memory(DEF_PRIORITY, gfp_mask);
+#ifdef CONFIG_QUOTA
+ ret += shrink_dqcache_memory(DEF_PRIORITY, gfp_mask);
+#endif
+
+ /*
+ * Eat memory from filesystem page cache, buffer cache,
+ * dentry, inode and filesystem quota caches.
+ */
+ rebalance_inactive(gfp_mask, 5);
+
+ for_each_zone(zone)
+ while (need_rebalance_dirty(zone))
+ rebalance_dirty_zone(zone, 16 * BATCH_WORK_AMOUNT, gfp_mask);
+
+ for_each_zone(zone)
+ if (free_high(zone)>0)
+ rebalance_laundry_zone(zone, BATCH_WORK_AMOUNT, 0);
+
+ refill_freelist();
+
+ /* Start IO when needed. */
+ if (free_plenty(ALL_ZONES) > 0 || free_low(ANY_ZONE) > 0)
+ run_task_queue(&tq_disk);
+
+ return ret;
+}
+
 /**
  * refill_freelist - move inactive_clean pages to free list if needed
  *
@@ -764,7 +986,7 @@
                  * zone is very short on free pages.
                  */
                 if (free_high(ALL_ZONES) >= 0 || free_low(ANY_ZONE) > 0)
- do_try_to_free_pages(GFP_KSWAPD);
+ do_try_to_free_pages_kswapd(GFP_KSWAPD);
 
                 refill_freelist();
 
@@ -846,7 +1068,7 @@
         /* OK, the VM is very loaded. Sleep instead of using all CPU. */
         kswapd_overloaded = 1;
         set_current_state(TASK_UNINTERRUPTIBLE);
- schedule_timeout(HZ / 4);
+ schedule_timeout(HZ / 40);
         kswapd_overloaded = 0;
         return;
 }
@@ -888,6 +1110,7 @@
 void rss_free_pages(unsigned int gfp_mask)
 {
         long pause = 0;
+ struct zone_struct * zone;
 
         if (current->flags & PF_MEMALLOC)
                 return;
@@ -895,7 +1118,10 @@
         current->flags |= PF_MEMALLOC;
 
         do {
- page_launder(gfp_mask);
+ rebalance_inactive(gfp_mask, 100);
+ for_each_zone(zone)
+ if (free_plenty(zone) >= 0)
+ rebalance_laundry_zone(zone, BATCH_WORK_AMOUNT, 0);
 
                 set_current_state(TASK_UNINTERRUPTIBLE);
                 schedule_timeout(pause);
@@ -907,11 +1133,77 @@
         return;
 }
 
+/*
+ * The background page scanning daemon, started as a kernel thread
+ * from the init process.
+ *
+ * This is the part that background scans the active list to find
+ * pages that are referenced and increases their age score.
+ * It is important that this scan rate is not proportional to vm pressure
+ * per se otherwise cpu usage becomes unbounded. On the other hand, if there's
+ * no VM pressure at all it shouldn't age stuff either otherwise everything
+ * ends up at the maximum age.
+ */
+#define MAX_AGING_INTERVAL 5*HZ
+#define MIN_AGING_INTERVAL HZ/2
+int kscand(void *unused)
+{
+ struct task_struct *tsk = current;
+ struct zone_struct * zone;
+ unsigned long pause = MAX_AGING_INTERVAL;
+ int total_needscan = 0;
+ int age_faster = 0;
+ int num_zones = 0;
+ int age;
+
+ daemonize();
+ strcpy(tsk->comm, "kscand");
+ sigfillset(&tsk->blocked);
+
+ for (;;) {
+ set_current_state(TASK_INTERRUPTIBLE);
+ schedule_timeout(pause);
+ for_each_zone(zone) {
+ if (need_active_anon_scan(zone)) {
+ for (age = 0; age < MAX_AGE; age++) {
+ scan_active_list(zone, age, 1);
+ if (current->need_resched)
+ schedule();
+ }
+ }
+
+ if (need_active_cache_scan(zone)) {
+ for (age = 0; age < MAX_AGE; age++) {
+ scan_active_list(zone, age, 0);
+ if (current->need_resched)
+ schedule();
+ }
+ }
+
+ /* Check if we've been aging quickly enough */
+ if (zone->need_scan >= 2)
+ age_faster++;
+ total_needscan += zone->need_scan;
+ zone->need_scan = 0;
+ num_zones++;
+ }
+ if (age_faster)
+ pause = max(pause / 2, MIN_AGING_INTERVAL);
+ else if (total_needscan < num_zones)
+ pause = min(pause + pause / 2, MAX_AGING_INTERVAL);
+
+ total_needscan = 0;
+ num_zones = 0;
+ }
+}
+
+
 static int __init kswapd_init(void)
 {
         printk("Starting kswapd\n");
         swap_setup();
         kernel_thread(kswapd, NULL, CLONE_FS | CLONE_FILES | CLONE_SIGNAL);
+ kernel_thread(kscand, NULL, CLONE_FS | CLONE_FILES | CLONE_SIGNAL);
         return 0;
 }
 

-- 
Paul P 'Stingray' Komkoff 'Greatest' Jr /// (icq)23200764 /// (http)stingr.net
  When you're invisible, the only one really watching you is you (my keychain)
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/



This archive was generated by hypermail 2b29 : Sat Nov 23 2002 - 22:00:38 EST