diff -ruN linux-2.4.23/fs/inode.c linux-2.4.23a/fs/inode.c --- linux-2.4.23/fs/inode.c Sun Dec 7 12:31:04 2003 +++ linux-2.4.23a/fs/inode.c Thu Jan 1 20:13:14 2004 @@ -117,6 +117,7 @@ mapping->host = inode; mapping->gfp_mask = GFP_HIGHUSER; inode->i_mapping = mapping; + inode->i_color = get_start_color(); } return inode; } diff -ruN linux-2.4.23/fs/pipe.c linux-2.4.23a/fs/pipe.c --- linux-2.4.23/fs/pipe.c Mon Aug 25 07:44:43 2003 +++ linux-2.4.23a/fs/pipe.c Thu Jan 1 20:13:57 2004 @@ -439,7 +439,7 @@ { unsigned long page; - page = __get_free_page(GFP_USER); + page = __get_free_page_color(GFP_USER, &inode->i_color); if (!page) return NULL; diff -ruN linux-2.4.23/fs/select.c linux-2.4.23a/fs/select.c --- linux-2.4.23/fs/select.c Fri Jun 13 10:51:37 2003 +++ linux-2.4.23a/fs/select.c Thu Jan 1 20:24:59 2004 @@ -79,7 +79,8 @@ if (!table || POLL_TABLE_FULL(table)) { struct poll_table_page *new_table; - new_table = (struct poll_table_page *) __get_free_page(GFP_KERNEL); + new_table = (struct poll_table_page *) __get_free_page_color( + GFP_KERNEL, &filp->f_dentry->d_inode->i_color); if (!new_table) { p->error = -ENOMEM; __set_current_state(TASK_RUNNING); diff -ruN linux-2.4.23/include/linux/fs.h linux-2.4.23a/include/linux/fs.h --- linux-2.4.23/include/linux/fs.h Sun Dec 7 12:44:14 2003 +++ linux-2.4.23a/include/linux/fs.h Thu Jan 1 22:03:02 2004 @@ -460,6 +460,7 @@ unsigned long i_blocks; unsigned long i_version; unsigned short i_bytes; + unsigned int i_color; struct semaphore i_sem; struct rw_semaphore i_alloc_sem; struct semaphore i_zombie; diff -ruN linux-2.4.23/include/linux/mm.h linux-2.4.23a/include/linux/mm.h --- linux-2.4.23/include/linux/mm.h Sun Dec 7 12:44:14 2003 +++ linux-2.4.23a/include/linux/mm.h Thu Jan 1 22:03:02 2004 @@ -434,8 +434,8 @@ * can allocate highmem pages, the *get*page*() variants return * virtual kernel addresses to the allocated page(s). */ -extern struct page * FASTCALL(_alloc_pages(unsigned int gfp_mask, unsigned int order)); -extern struct page * FASTCALL(__alloc_pages(unsigned int gfp_mask, unsigned int order, zonelist_t *zonelist)); +extern struct page * FASTCALL(_alloc_pages_color(unsigned int gfp_mask, unsigned int order, unsigned int *pfn)); +extern struct page * FASTCALL(__alloc_pages_color(unsigned int gfp_mask, unsigned int order, zonelist_t *zonelist, unsigned int *pfn)); extern struct page * alloc_pages_node(int nid, unsigned int gfp_mask, unsigned int order); static inline struct page * alloc_pages(unsigned int gfp_mask, unsigned int order) @@ -445,24 +445,45 @@ */ if (order >= MAX_ORDER) return NULL; - return _alloc_pages(gfp_mask, order); + return _alloc_pages_color(gfp_mask, order, NULL); } -#define alloc_page(gfp_mask) alloc_pages(gfp_mask, 0) +static inline struct page * alloc_pages_color(unsigned int gfp_mask, unsigned int order, unsigned int *pfn) +{ + /* + * Gets optimized away by the compiler. + */ + if (order >= MAX_ORDER) + return NULL; + return _alloc_pages_color(gfp_mask, order, pfn); +} + +#define alloc_page(gfp_mask) alloc_pages_color(gfp_mask, 0, NULL) +#define alloc_page_color(gfp_mask, pfn) alloc_pages_color(gfp_mask, 0, pfn) + +extern unsigned long FASTCALL(__get_free_pages_color(unsigned int gfp_mask, unsigned int order, unsigned int *pfn)); +extern unsigned long FASTCALL(get_zeroed_page_color(unsigned int gfp_mask, unsigned int *pfn)); -extern unsigned long FASTCALL(__get_free_pages(unsigned int gfp_mask, unsigned int order)); -extern unsigned long FASTCALL(get_zeroed_page(unsigned int gfp_mask)); +#define __get_free_pages(gfp_mask, order) \ + __get_free_pages_color((gfp_mask),(order),NULL) #define __get_free_page(gfp_mask) \ - __get_free_pages((gfp_mask),0) + __get_free_pages_color((gfp_mask),0,NULL) + +#define __get_free_page_color(gfp_mask, pfn) \ + __get_free_pages_color((gfp_mask),0,pfn) #define __get_dma_pages(gfp_mask, order) \ - __get_free_pages((gfp_mask) | GFP_DMA,(order)) + __get_free_pages_color((gfp_mask) | GFP_DMA,(order),NULL) + +#define get_zeroed_page(gfp_mask) get_zeroed_page_color(gfp_mask,NULL) + +extern unsigned long get_start_color(void); /* * The old interface name will be removed in 2.5: */ -#define get_free_page get_zeroed_page +#define get_free_page(gfp_mask) get_zeroed_page_color(gfp_mask,NULL) /* * There is only one 'core' page-freeing function. diff -ruN linux-2.4.23/include/linux/mmzone.h linux-2.4.23a/include/linux/mmzone.h --- linux-2.4.23/include/linux/mmzone.h Sun Dec 7 12:44:14 2003 +++ linux-2.4.23a/include/linux/mmzone.h Thu Jan 1 22:03:02 2004 @@ -25,8 +25,8 @@ #define MAX_NR_ZONES 3 typedef struct free_area_struct { - struct list_head free_list; unsigned long *map; + struct list_head * qlist; } free_area_t; struct pglist_data; @@ -70,12 +70,13 @@ unsigned long nr_active_pages, nr_inactive_pages; /* protected by the pagecache_lock */ unsigned long nr_cache_pages; + unsigned int color; /* * free areas of different sizes */ - free_area_t free_area[MAX_ORDER]; + free_area_t free_area[MAX_ORDER+1]; /* * wait_table -- the array holding the hash table diff -ruN linux-2.4.23/include/linux/pagemap.h linux-2.4.23a/include/linux/pagemap.h --- linux-2.4.23/include/linux/pagemap.h Sun Dec 7 12:44:26 2003 +++ linux-2.4.23a/include/linux/pagemap.h Thu Jan 1 22:03:13 2004 @@ -31,9 +31,9 @@ #define page_cache_get(x) get_page(x) #define page_cache_release(x) __free_page(x) -static inline struct page *page_cache_alloc(struct address_space *x) +static inline struct page *page_cache_alloc(struct inode *x) { - return alloc_pages(x->gfp_mask, 0); + return alloc_pages_color(x->i_mapping->gfp_mask, 0, &x->i_color); } /* diff -ruN linux-2.4.23/include/linux/sched.h linux-2.4.23a/include/linux/sched.h --- linux-2.4.23/include/linux/sched.h Sun Dec 7 12:44:14 2003 +++ linux-2.4.23a/include/linux/sched.h Thu Jan 1 22:03:02 2004 @@ -226,6 +226,7 @@ unsigned long def_flags; unsigned long cpu_vm_mask; unsigned long swap_address; + unsigned int color; unsigned dumpable:1; diff -ruN linux-2.4.23/init/main.c linux-2.4.23a/init/main.c --- linux-2.4.23/init/main.c Sun Dec 7 12:31:05 2003 +++ linux-2.4.23a/init/main.c Fri Dec 26 13:16:46 2003 @@ -101,6 +101,7 @@ extern int init_pcmcia_ds(void); extern void free_initmem(void); +extern void page_color_init(void); #ifdef CONFIG_TC extern void tc_init(void); @@ -437,6 +438,7 @@ #if defined(CONFIG_SYSVIPC) ipc_init(); #endif + page_color_init(); rest_init(); } diff -ruN linux-2.4.23/kernel/fork.c linux-2.4.23a/kernel/fork.c --- linux-2.4.23/kernel/fork.c Sun Dec 7 12:31:05 2003 +++ linux-2.4.23a/kernel/fork.c Thu Jan 1 21:32:26 2004 @@ -235,6 +235,7 @@ mm->page_table_lock = SPIN_LOCK_UNLOCKED; mm->pgd = pgd_alloc(mm); mm->def_flags = 0; + mm->color = get_start_color(); if (mm->pgd) return mm; free_mm(mm); diff -ruN linux-2.4.23/mm/filemap.c linux-2.4.23a/mm/filemap.c --- linux-2.4.23/mm/filemap.c Sun Dec 7 12:31:05 2003 +++ linux-2.4.23a/mm/filemap.c Thu Jan 1 20:09:26 2004 @@ -722,7 +722,7 @@ if (page) return 0; - page = page_cache_alloc(mapping); + page = page_cache_alloc(mapping->host); if (!page) return -ENOMEM; @@ -1087,7 +1087,7 @@ } } - page = page_cache_alloc(mapping); + page = page_cache_alloc(mapping->host); if ( unlikely(!page) ) return NULL; /* Failed to allocate a page */ @@ -1533,7 +1533,7 @@ */ if (!cached_page) { spin_unlock(&pagecache_lock); - cached_page = page_cache_alloc(mapping); + cached_page = page_cache_alloc(inode); if (!cached_page) { desc->error = -ENOMEM; break; @@ -2901,7 +2901,7 @@ page = __find_get_page(mapping, index, hash); if (!page) { if (!cached_page) { - cached_page = page_cache_alloc(mapping); + cached_page = page_cache_alloc(mapping->host); if (!cached_page) return ERR_PTR(-ENOMEM); } @@ -2967,7 +2967,7 @@ page = __find_lock_page(mapping, index, hash); if (!page) { if (!*cached_page) { - *cached_page = page_cache_alloc(mapping); + *cached_page = page_cache_alloc(mapping->host); if (!*cached_page) return NULL; } diff -ruN linux-2.4.23/mm/memory.c linux-2.4.23a/mm/memory.c --- linux-2.4.23/mm/memory.c Sun Dec 7 12:31:05 2003 +++ linux-2.4.23a/mm/memory.c Thu Jan 1 21:16:49 2004 @@ -971,7 +971,7 @@ page_cache_get(old_page); spin_unlock(&mm->page_table_lock); - new_page = alloc_page(GFP_HIGHUSER); + new_page = alloc_page_color(GFP_HIGHUSER, &mm->color); if (!new_page) goto no_mem; copy_cow_page(old_page,new_page,address); @@ -1201,7 +1201,7 @@ /* Allocate our own private page. */ spin_unlock(&mm->page_table_lock); - page = alloc_page(GFP_HIGHUSER); + page = alloc_page_color(GFP_HIGHUSER, &mm->color); if (!page) goto no_mem; clear_user_highpage(page, addr); @@ -1263,7 +1263,7 @@ * Should we do an early C-O-W break? */ if (write_access && !(vma->vm_flags & VM_SHARED)) { - struct page * page = alloc_page(GFP_HIGHUSER); + struct page * page = alloc_page_color(GFP_HIGHUSER, &mm->color); if (!page) { page_cache_release(new_page); return -1; diff -ruN linux-2.4.23/mm/numa.c linux-2.4.23a/mm/numa.c --- linux-2.4.23/mm/numa.c Mon Sep 17 19:15:02 2001 +++ linux-2.4.23a/mm/numa.c Sun Dec 28 19:36:10 2003 @@ -34,9 +34,9 @@ struct page * alloc_pages_node(int nid, unsigned int gfp_mask, unsigned int order) { #ifdef CONFIG_NUMA - return __alloc_pages(gfp_mask, order, NODE_DATA(nid)->node_zonelists + (gfp_mask & GFP_ZONEMASK)); + return __alloc_pages_color(gfp_mask, order, NODE_DATA(nid)->node_zonelists + (gfp_mask & GFP_ZONEMASK), NULL); #else - return alloc_pages(gfp_mask, order); + return alloc_pages_color(gfp_mask, order, -1); #endif } @@ -82,17 +82,18 @@ memset(pgdat->valid_addr_bitmap, 0, size); } -static struct page * alloc_pages_pgdat(pg_data_t *pgdat, unsigned int gfp_mask, - unsigned int order) +static struct page * alloc_pages_pgdat_color(pg_data_t *pgdat, + unsigned int gfp_mask, unsigned int order, unsigned long pfn) { - return __alloc_pages(gfp_mask, order, pgdat->node_zonelists + (gfp_mask & GFP_ZONEMASK)); + return __alloc_pages_color(gfp_mask, order, pgdat->node_zonelists + (gfp_mask & GFP_ZONEMASK), pfn); } /* * This can be refined. Currently, tries to do round robin, instead * should do concentratic circle search, starting from current node. */ -struct page * _alloc_pages(unsigned int gfp_mask, unsigned int order) +struct page * _alloc_pages_color(unsigned int gfp_mask, + unsigned int order, unsigned long pfn) { struct page *ret = 0; pg_data_t *start, *temp; @@ -114,13 +115,13 @@ #endif start = temp; while (temp) { - if ((ret = alloc_pages_pgdat(temp, gfp_mask, order))) + if ((ret = alloc_pages_pgdat_color(temp, gfp_mask, order, pfn))) return(ret); temp = temp->node_next; } temp = pgdat_list; while (temp != start) { - if ((ret = alloc_pages_pgdat(temp, gfp_mask, order))) + if ((ret = alloc_pages_pgdat_color(temp, gfp_mask, order, pfn))) return(ret); temp = temp->node_next; } diff -ruN linux-2.4.23/mm/page_alloc.c linux-2.4.23a/mm/page_alloc.c --- linux-2.4.23/mm/page_alloc.c Sun Dec 7 12:31:05 2003 +++ linux-2.4.23a/mm/page_alloc.c Thu Jan 1 22:49:40 2004 @@ -46,15 +46,123 @@ int vm_gfp_debug = 0; -/* - * Temporary debugging check. +#include + +/* Page coloring variables and auxiliary functions */ + +unsigned int num_colors; +unsigned int num_queues; +unsigned int page_miss_count; +unsigned int page_hit_count; +unsigned int page_alloc_count; +unsigned int page_anon_alloc_count; +unsigned int page_alloc_order[MAX_ORDER]; +extern char saved_command_line[]; + +#ifdef CONFIG_PROC_FS +int page_color_getinfo(char *buf, char **start, off_t fpos, int length) +{ + int i, count; + struct list_head *queue, *curr; + char *p = buf; + zone_t *zone; + + p += sprintf(p, "colors: %d\n", num_colors); + p += sprintf(p, "hits: %d\n", page_hit_count); + p += sprintf(p, "misses: %d\n", page_miss_count); + p += sprintf(p, "pages allocated: %d\n", page_alloc_count); + p += sprintf(p, "misc pages allocated: %d\n", page_anon_alloc_count); + p += sprintf(p, "allocs by order: "); + + for(i=0; isize) + continue; + + queue = zone->free_area[0].qlist; + for(i=0; ifree_area[i+1].qlist) { + count = 0; + list_for_each(curr, queue) + count++; + p += sprintf(p, "%d ", count); + queue++; + } + p += sprintf(p, "\n"); + } + } + + return p - buf; +} +#endif /* CONFIG_PROC_FS */ + +#define COLOR(x) ((x) % num_colors) + +/* Determine the number of colors to use. This quantity + * is needed very early in the boot process; so early, in + * fact, that the machinery to parse the boot command line + * is not ready and we're still in arch-specific setup. Thus + * this code is called from free_area_init_core() below. */ -#define BAD_RANGE(zone, page) \ -( \ - (((page) - mem_map) >= ((zone)->zone_start_mapnr+(zone)->size)) \ - || (((page) - mem_map) < (zone)->zone_start_mapnr) \ - || ((zone) != page_zone(page)) \ -) +void __init set_num_colors(void) +{ + char *size_ptr; + unsigned int cache_size = 0; + unsigned int i, tmp_colors; + + size_ptr = strstr(saved_command_line, "page_color="); + if (size_ptr) { + cache_size = simple_strtoul(size_ptr+11, (char **)NULL, 10); + cache_size *= 1024; + } + + if (cache_size >= 65536) { + printk("page_color: using %dkB external cache (%d colors)\n", + cache_size >> 10, cache_size >> PAGE_SHIFT); + } + else { + /* Cache size absent or invalid; guess a value */ + + if (cache_size < 65536) +#if defined(__i386__) + cache_size = 256 * 1024; +#elif defined(__parisc__) || defined(__powerpc__) + cache_size = 1024 * 1024; +#else + cache_size = 2048 * 1024; +#endif + printk("page_color: assuming %dkB external cache (%d colors)\n", + cache_size >> 10, cache_size >> PAGE_SHIFT); + printk("override this with \"page_color=X\" (in kB)\n"); + } + + num_colors = cache_size >> PAGE_SHIFT; + tmp_colors = num_colors; + for(i = 0; i < MAX_ORDER; i++) { + num_queues += tmp_colors; + if (tmp_colors > 1) + tmp_colors = (tmp_colors + 1) >> 1; + } +} + +void __init page_color_init(void) +{ +#ifdef CONFIG_PROC_FS + create_proc_info_entry("page_color", 0, NULL, page_color_getinfo); +#endif +} + +static unsigned int rand_seed = 0x01234567; + +unsigned long get_start_color(void) +{ + rand_seed = rand_seed * 1664525 + 1013904223; + return COLOR(rand_seed >> 16); +} /* * Freeing function for a buddy system allocator. @@ -88,6 +196,7 @@ free_area_t *area; struct page *base; zone_t *zone; + unsigned long free_idx; /* * Yes, think what happens when other parts of the kernel take @@ -148,18 +257,16 @@ */ buddy1 = base + (page_idx ^ -mask); buddy2 = base + page_idx; - if (BAD_RANGE(zone,buddy1)) - BUG(); - if (BAD_RANGE(zone,buddy2)) - BUG(); - + order++; list_del(&buddy1->list); mask <<= 1; area++; index >>= 1; page_idx &= mask; } - list_add(&(base + page_idx)->list, &area->free_list); + free_idx = (zone->zone_start_paddr >> PAGE_SHIFT) + page_idx; + list_add(&(base + page_idx)->list, area->qlist + + (COLOR(free_idx) >> order)); spin_unlock_irqrestore(&zone->lock, flags); return; @@ -175,81 +282,124 @@ current->nr_local_pages++; } -#define MARK_USED(index, order, area) \ - __change_bit((index) >> (1+(order)), (area)->map) +static FASTCALL(struct page * rmqueue(zone_t *zone, unsigned int order, + unsigned int *pfn)); +static struct page * rmqueue(zone_t *zone, unsigned int order, + unsigned int *pfn) +{ + unsigned int i, j; + unsigned int mask, color; + unsigned long flags, page_idx, alloc_idx; + free_area_t *area; + struct list_head *curr, *head; + struct page *page; -static inline struct page * expand (zone_t *zone, struct page *page, - unsigned long index, int low, int high, free_area_t * area) -{ - unsigned long size = 1 << high; + spin_lock_irqsave(&zone->lock, flags); - while (high > low) { - if (BAD_RANGE(zone,page)) - BUG(); - area--; - high--; - size >>= 1; - list_add(&(page)->list, &(area)->free_list); - MARK_USED(index, high, area); - index += size; - page += size; + if (pfn == NULL) { + pfn = &zone->color; + page_anon_alloc_count++; } - if (BAD_RANGE(zone,page)) - BUG(); - return page; -} -static FASTCALL(struct page * rmqueue(zone_t *zone, unsigned int order)); -static struct page * rmqueue(zone_t *zone, unsigned int order) -{ - free_area_t * area = zone->free_area + order; - unsigned int curr_order = order; - struct list_head *head, *curr; - unsigned long flags; - struct page *page; + mask = (1 << order) - 1; + color = COLOR((*pfn + mask) & ~mask); - spin_lock_irqsave(&zone->lock, flags); - do { - head = &area->free_list; + /* Make one attempt to find a contiguous group + * of pages that contains the specified color + */ + for(i = order; i < MAX_ORDER; i++) { + area = zone->free_area + i; + head = area->qlist + (color >> i); curr = head->next; + if (curr != head) + goto page_found; + } - if (curr != head) { - unsigned int index; + /* Memory is either very scarce or very + * fragmented. To avoid making the situation + * worse, switch to breadth-first search of + * the free list and choose the first group + * of pages that's available. Begin the search + * from the current color to randomize the + * process a little. + */ - page = list_entry(curr, struct page, list); - if (BAD_RANGE(zone,page)) - BUG(); - list_del(curr); - index = page - zone->zone_mem_map; - if (curr_order != MAX_ORDER-1) - MARK_USED(index, curr_order, area); - zone->free_pages -= 1UL << order; + page_miss_count++; + for(i = order; i < MAX_ORDER; i++) { + area = zone->free_area + i; + for(j = 0; j < ((area+1)->qlist - area->qlist); j++) { + head = area->qlist + (color >> i); + curr = head->next; + if (curr != head) + goto page_found; + + color += 1 << i; + if (color >= num_colors) + color = 0; + } + } + spin_unlock_irqrestore(&zone->lock, flags); + return NULL; - page = expand(zone, page, index, order, curr_order, area); - spin_unlock_irqrestore(&zone->lock, flags); +page_found: + page = list_entry(curr, struct page, list); + list_del(curr); + page_idx = page - zone->zone_mem_map; + alloc_idx = (zone->zone_start_paddr >> PAGE_SHIFT) + page_idx; + zone->free_pages -= 1 << order; + + if (i < (MAX_ORDER - 1)) + __change_bit(page_idx >> (1+i), area->map); + + while (i > order) { + /* Return 1<> (1+i), area->map); + if (color & mask) { + list_add(&page->list, area->qlist + + (COLOR(alloc_idx) >> i)); + page_idx += mask; + alloc_idx += mask; + page += mask; + } + else { + list_add(&(page + mask)->list, area->qlist + + (COLOR(alloc_idx + mask) >> i)); } - curr_order++; - area++; - } while (curr_order < MAX_ORDER); - spin_unlock_irqrestore(&zone->lock, flags); + } - return NULL; + set_page_count(page, 1); + if (PageLRU(page)) + BUG(); + if (PageActive(page)) + BUG(); + + *pfn = COLOR(color + (1<lock, flags); + return page; } #ifndef CONFIG_DISCONTIGMEM -struct page *_alloc_pages(unsigned int gfp_mask, unsigned int order) +struct page *_alloc_pages_color(unsigned int gfp_mask, unsigned int order, + unsigned int *pfn) { - return __alloc_pages(gfp_mask, order, - contig_page_data.node_zonelists+(gfp_mask & GFP_ZONEMASK)); + return __alloc_pages_color(gfp_mask, order, + contig_page_data.node_zonelists+(gfp_mask & GFP_ZONEMASK), pfn); } #endif @@ -332,7 +482,7 @@ /* * This is the 'heart' of the zoned buddy allocator: */ -struct page * __alloc_pages(unsigned int gfp_mask, unsigned int order, zonelist_t *zonelist) +struct page * __alloc_pages_color(unsigned int gfp_mask, unsigned int order, zonelist_t *zonelist, unsigned int *pfn) { zone_t **zone, * classzone; struct page * page; @@ -348,7 +498,7 @@ break; if (zone_free_pages(z, order) > z->watermarks[class_idx].low) { - page = rmqueue(z, order); + page = rmqueue(z, order, pfn); if (page) return page; } @@ -370,7 +520,7 @@ if (!(gfp_mask & __GFP_WAIT)) min >>= 2; if (zone_free_pages(z, order) > min) { - page = rmqueue(z, order); + page = rmqueue(z, order, pfn); if (page) return page; } @@ -385,7 +535,7 @@ if (!z) break; - page = rmqueue(z, order); + page = rmqueue(z, order, pfn); if (page) return page; } @@ -409,7 +559,7 @@ break; if (zone_free_pages(z, order) > z->watermarks[class_idx].min) { - page = rmqueue(z, order); + page = rmqueue(z, order, pfn); if (page) return page; } @@ -426,7 +576,7 @@ break; if (zone_free_pages(z, order) > z->watermarks[class_idx].high) { - page = rmqueue(z, order); + page = rmqueue(z, order, pfn); if (page) return page; } @@ -444,21 +594,22 @@ /* * Common helper functions. */ -unsigned long __get_free_pages(unsigned int gfp_mask, unsigned int order) +unsigned long __get_free_pages_color(unsigned int gfp_mask, unsigned int order, + unsigned int *pfn) { struct page * page; - page = alloc_pages(gfp_mask, order); + page = alloc_pages_color(gfp_mask, order, pfn); if (!page) return 0; return (unsigned long) page_address(page); } -unsigned long get_zeroed_page(unsigned int gfp_mask) +unsigned long get_zeroed_page_color(unsigned int gfp_mask, unsigned int *pfn) { struct page * page; - page = alloc_pages(gfp_mask, 0); + page = alloc_pages_color(gfp_mask, 0, pfn); if (page) { void *address = page_address(page); clear_page(address); @@ -576,14 +727,17 @@ total = 0; if (zone->size) { spin_lock_irqsave(&zone->lock, flags); + head = zone->free_area[0].qlist; for (order = 0; order < MAX_ORDER; order++) { - head = &(zone->free_area + order)->free_list; - curr = head; nr = 0; - for (;;) { - if ((curr = curr->next) == head) - break; - nr++; + while(head < zone->free_area[order+1].qlist) { + curr = head; + for (;;) { + if ((curr = curr->next) == head) + break; + nr++; + } + head++; } total += nr * (1 << order); printk("%lu*%lukB ", nr, K(1UL) << order); @@ -709,6 +863,11 @@ unsigned long map_size; unsigned long totalpages, offset, realtotalpages; const unsigned long zone_required_alignment = 1UL << (MAX_ORDER-1); + unsigned int alloc_size, tmp_colors; + free_area_t *area; + + if (!num_colors) + set_num_colors(); if (zone_start_paddr & ~PAGE_MASK) BUG(); @@ -764,12 +923,31 @@ zone->zone_pgdat = pgdat; zone->free_pages = 0; zone->need_balance = 0; - zone->nr_active_pages = zone->nr_inactive_pages = 0; - + zone->nr_active_pages = zone->nr_inactive_pages = 0; + zone->color = 0; if (!size) continue; + /* Allocate bootmem to hold the queues needed + * by the page coloring implementation + */ + alloc_size = num_queues * sizeof(struct list_head); + area = zone->free_area; + area->qlist = (struct list_head *)__alloc_bootmem_node( + pgdat, alloc_size, sizeof(void *), + __pa(MAX_DMA_ADDRESS)); + + for(i = 0; i < num_queues; i++) + INIT_LIST_HEAD(area->qlist + i); + + tmp_colors = num_colors; + for(i = 1; i < (MAX_ORDER + 1); i++) { + area[i].qlist = area[i-1].qlist + tmp_colors; + if (tmp_colors > 1) + tmp_colors = (tmp_colors + 1) >> 1; + } + /* * The per-page waitqueue mechanism uses hashed waitqueues * per zone. @@ -842,7 +1020,6 @@ for (i = 0; ; i++) { unsigned long bitmap_size; - INIT_LIST_HEAD(&zone->free_area[i].free_list); if (i == MAX_ORDER-1) { zone->free_area[i].map = NULL; break; diff -ruN linux-2.4.23/mm/shmem.c linux-2.4.23a/mm/shmem.c --- linux-2.4.23/mm/shmem.c Sun Dec 7 12:31:05 2003 +++ linux-2.4.23a/mm/shmem.c Thu Jan 1 20:09:57 2004 @@ -749,7 +749,7 @@ if (!filepage) { spin_unlock(&info->lock); - filepage = page_cache_alloc(mapping); + filepage = page_cache_alloc(inode); if (!filepage) { shmem_free_block(inode); error = -ENOMEM; diff -ruN linux-2.4.23/mm/slab.c linux-2.4.23a/mm/slab.c --- linux-2.4.23/mm/slab.c Sun Dec 7 12:31:05 2003 +++ linux-2.4.23a/mm/slab.c Thu Jan 1 22:02:38 2004 @@ -196,6 +196,7 @@ unsigned int objsize; unsigned int flags; /* constant flags */ unsigned int num; /* # of objs per slab */ + unsigned int color; /* current page color for this cache */ spinlock_t spinlock; #ifdef CONFIG_SMP unsigned int batchcount; @@ -493,7 +494,8 @@ * would be relatively rare and ignorable. */ flags |= cachep->gfpflags; - addr = (void*) __get_free_pages(flags, cachep->gfporder); + addr = (void*) __get_free_pages_color(flags, + cachep->gfporder, &cachep->color); /* Assume that now we have the pages no one else can legally * messes with the 'struct page's. * However vm_scan() might try to test the structure to see if @@ -795,6 +797,7 @@ cachep->slabp_cache = kmem_find_general_cachep(slab_size,0); cachep->ctor = ctor; cachep->dtor = dtor; + cachep->color = get_start_color(); /* Copy name over so we don't have problems with unloaded modules */ strcpy(cachep->name, name);