diff -urN linux-2.4.26-rc1/Documentation/Configure.help linux-2.4.26-rc1-ccache/Documentation/Configure.help --- linux-2.4.26-rc1/Documentation/Configure.help Sun Mar 28 16:12:41 2004 +++ linux-2.4.26-rc1-ccache/Documentation/Configure.help Sun Mar 28 17:12:21 2004 @@ -426,6 +426,84 @@ If unsure, say N. +Compressed cache (EXPERIMENTAL) +CONFIG_COMP_CACHE + Select this if you want to enable compressed cache support. If this + option is chosen, your memory will be split into two caches: + uncompressed cache, which holds pages in normal state; and + compressed cache, which will store the pages in compressed + form. This cache aims to reduce disk paging and improve overall + system performance. + + The maximum number of pages reserved for compressed cache is set by + the kernel parameter "compsize=N", where N is a memory size like the + input accepted by "mem=" parameter. For example "compsize=48M" sets + the maximum compressed cache size to 48 megabytes. It resizes by + itself as much as it is actually needed (ie, resizes on demand). + + If unsure, say N here. + +Support for Page Cache compression +CONFIG_COMP_PAGE_CACHE + Select this option in case you want compressed cache to store also + pages from page cache, ie file mapped pages, and to take into + account these pages when adapting compressed cache to recent + behaviour. If you don't select this option, compressed cache will + store only anonymous pages, ie pages not mapped to files. + + If unsure, say Y here. + +Double Page Size +CONFIG_COMP_DOUBLE_PAGE + + Select this option to make compressed cache use double sized pages + (two pages contiguos) instead of single memory pages. That increases + the effect of the compressed cache use even when the compression + ratio isn't very good. On i386, compressed cache pages will have + 8KiB instead of the regular 4KiB. + + The size of the compressed cache page is listed in the output of + /proc/comp_cache_stat. + + If unsure, say Y here. + +Compressed Swap +CONFIG_COMP_SWAP + + If you want to write many pages together in a block on the swap + device, say Y here. The compressed cache will keep swapping out the + pages in compressed form, and will group them to save swap + space. This is likely to decrease the number of IO performed to the + swap (dependant on the compression ratio). + + Notice that this option introduces a memory overhead due to the data + structures need for the new swap addressing (dependant on the swap + space). + +Disable Adaptability +CONFIG_COMP_DIS_ADAPT + + Select this option if you want to disable compressed cache + adaptability policy. In this case, compressed cache is known as + static, because it has a fixed size, and does not change it at run + time. When adaptability is disable, the "compsize=" kernel option + will select, rather than the maximum size, the static compressed + cache size. This option disables clean pages adaptability too. + + If unsure, say N here. + +Disable Clean Page Adaptability +CONFIG_COMP_DIS_CLEAN + + Clean page adaptability attempts to detect when compressign clean + pages is not worthwhile, disabling this compression. When the + compression of clean pages is disabled, it keeps track of new + evicted clean pages in order to decide when compressed cache should + resume compressing them. Say Y here if you want to disable clean + page adaptability. + + If unsure, say N here. + Normal floppy disk support CONFIG_BLK_DEV_FD If you want to use the floppy disk drive(s) of your PC under Linux, diff -urN linux-2.4.26-rc1/MAINTAINERS linux-2.4.26-rc1-ccache/MAINTAINERS --- linux-2.4.26-rc1/MAINTAINERS Sun Mar 28 16:12:40 2004 +++ linux-2.4.26-rc1-ccache/MAINTAINERS Sun Mar 28 17:12:21 2004 @@ -466,6 +466,13 @@ L: iss_storagedev@hp.com S: Supported +COMPRESSED CACHE +P: Rodrigo S. de Castro +M: rcastro@ime.usp.br +L: linuxcompressed@lists.sourceforge.net +W: http://linuxcompressed.sourceforge.net +S: Maintained + COMPUTONE INTELLIPORT MULTIPORT CARD P: Michael H. Warfield M: Michael H. Warfield diff -urN linux-2.4.26-rc1/arch/i386/config.in linux-2.4.26-rc1-ccache/arch/i386/config.in --- linux-2.4.26-rc1/arch/i386/config.in Fri Feb 20 07:38:25 2004 +++ linux-2.4.26-rc1-ccache/arch/i386/config.in Sun Mar 28 17:12:21 2004 @@ -266,6 +266,21 @@ mainmenu_option next_comment comment 'General setup' +if [ "$CONFIG_SMP" != "y" ]; then + dep_bool 'Compressed cache (EXPERIMENTAL)' CONFIG_COMP_CACHE $CONFIG_EXPERIMENTAL + if [ "$CONFIG_COMP_CACHE" = "y" ]; then + bool ' Support for Page Cache compression' CONFIG_COMP_PAGE_CACHE + bool ' Double Page Size' CONFIG_COMP_DOUBLE_PAGE + bool ' Compressed Swap' CONFIG_COMP_SWAP + bool ' Disable Adaptability' CONFIG_COMP_DIS_ADAPT + if [ "$CONFIG_COMP_DIS_ADAPT" = "y" ]; then + define_bool CONFIG_COMP_DIS_CLEAN y + else + bool ' Disable Clean Page Adaptability' CONFIG_COMP_DIS_CLEAN + fi + fi +fi + bool 'Networking support' CONFIG_NET # Visual Workstation support is utterly broken. diff -urN linux-2.4.26-rc1/fs/buffer.c linux-2.4.26-rc1-ccache/fs/buffer.c --- linux-2.4.26-rc1/fs/buffer.c Fri Feb 20 07:38:31 2004 +++ linux-2.4.26-rc1-ccache/fs/buffer.c Sun Mar 28 17:13:09 2004 @@ -47,6 +47,7 @@ #include #include #include +#include #include #include @@ -853,7 +854,6 @@ SetPageUptodate(page); UnlockPage(page); - return; still_busy: diff -urN linux-2.4.26-rc1/fs/inode.c linux-2.4.26-rc1-ccache/fs/inode.c --- linux-2.4.26-rc1/fs/inode.c Sun Mar 28 16:12:41 2004 +++ linux-2.4.26-rc1-ccache/fs/inode.c Sun Mar 28 17:14:13 2004 @@ -156,6 +156,11 @@ INIT_LIST_HEAD(&inode->i_data.clean_pages); INIT_LIST_HEAD(&inode->i_data.dirty_pages); INIT_LIST_HEAD(&inode->i_data.locked_pages); +#ifdef CONFIG_COMP_CACHE + INIT_LIST_HEAD(&inode->i_data.clean_comp_pages); + INIT_LIST_HEAD(&inode->i_data.dirty_comp_pages); + INIT_LIST_HEAD(&inode->i_data.locked_comp_pages); +#endif INIT_LIST_HEAD(&inode->i_dentry); INIT_LIST_HEAD(&inode->i_dirty_buffers); INIT_LIST_HEAD(&inode->i_dirty_data_buffers); diff -urN linux-2.4.26-rc1/fs/ncpfs/dir.c linux-2.4.26-rc1-ccache/fs/ncpfs/dir.c --- linux-2.4.26-rc1/fs/ncpfs/dir.c Fri Feb 20 07:38:32 2004 +++ linux-2.4.26-rc1-ccache/fs/ncpfs/dir.c Sun Mar 28 17:12:21 2004 @@ -22,6 +22,7 @@ #include #include #include +#include #include @@ -458,6 +459,9 @@ if (!ctl.page) goto invalid_cache; ctl.cache = kmap(ctl.page); +#ifdef CONFIG_COMP_PAGE_CACHE + flush_comp_cache(ctl.page); +#endif if (!Page_Uptodate(ctl.page)) goto invalid_cache; } diff -urN linux-2.4.26-rc1/fs/proc/proc_misc.c linux-2.4.26-rc1-ccache/fs/proc/proc_misc.c --- linux-2.4.26-rc1/fs/proc/proc_misc.c Sat Dec 6 08:14:48 2003 +++ linux-2.4.26-rc1-ccache/fs/proc/proc_misc.c Sun Mar 28 17:17:23 2004 @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -186,6 +187,10 @@ "MemShared: %8lu kB\n" "Buffers: %8lu kB\n" "Cached: %8lu kB\n" +#ifdef CONFIG_COMP_CACHE + "CCacheAlloc: %8lu kB\n" + "CCacheUsed: %8lu kB\n" +#endif "SwapCached: %8lu kB\n" "Active: %8u kB\n" "Inactive: %8u kB\n" @@ -199,8 +204,15 @@ K(i.freeram), K(i.sharedram), K(i.bufferram), +#ifdef CONFIG_COMP_CACHE + K(pg_size + num_swapper_fragments - swapper_space.nrpages), + K(num_comp_pages << COMP_PAGE_ORDER), + comp_cache_used_space/1024, + K(swapper_space.nrpages - num_swapper_fragments), +#else K(pg_size - swapper_space.nrpages), K(swapper_space.nrpages), +#endif K(nr_active_pages), K(nr_inactive_pages), K(i.totalhigh), @@ -608,6 +620,11 @@ {"modules", modules_read_proc}, #endif {"stat", kstat_read_proc}, +#ifdef CONFIG_COMP_CACHE + {"comp_cache_stat", comp_cache_stat_read_proc}, + {"comp_cache_hist", comp_cache_hist_read_proc}, + {"comp_cache_frag", comp_cache_frag_read_proc}, +#endif {"devices", devices_read_proc}, #if !defined(CONFIG_ARCH_S390) && !defined(CONFIG_X86) {"interrupts", interrupts_read_proc}, diff -urN linux-2.4.26-rc1/fs/smbfs/dir.c linux-2.4.26-rc1-ccache/fs/smbfs/dir.c --- linux-2.4.26-rc1/fs/smbfs/dir.c Fri Feb 20 07:38:32 2004 +++ linux-2.4.26-rc1-ccache/fs/smbfs/dir.c Sun Mar 28 17:12:21 2004 @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -154,6 +155,9 @@ if (!ctl.page) goto invalid_cache; ctl.cache = kmap(ctl.page); +#ifdef CONFIG_COMP_PAGE_CACHE + flush_comp_cache(ctl.page); +#endif if (!Page_Uptodate(ctl.page)) goto invalid_cache; } diff -urN linux-2.4.26-rc1/include/linux/WK4x4.h linux-2.4.26-rc1-ccache/include/linux/WK4x4.h --- linux-2.4.26-rc1/include/linux/WK4x4.h Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/include/linux/WK4x4.h Sun Mar 28 17:12:21 2004 @@ -0,0 +1,354 @@ +/* + WK4x4-v0.2 -- Wilson-Kaplan-4x4 version 0.2 + + Compresses buffers using a dictionary based match and partial match + scheme. + + Scott F. Kaplan -- sfkaplan@cs.utexas.edu + September 1997 */ + +/* ============================================================ */ +/* Preprocessor directives to avoid multiple inclusion */ +#ifndef _WK4x4_H +#define _WK4x4_H + +/* This macro defines the contents of the hash table, into which the + hash function will look to determine in which set to search for a + pattern. The table contents has been generated through the + pseudo-random number function lrand48(). */ +#define HASH_LOOKUP_TABLE_CONTENTS_WK4x4 { \ + 8, \ + 0, \ + 8, \ + 4, \ + 0, \ + 8, \ + 4, \ + 8, \ + 8, \ + 0, \ + 0, \ + 12, \ + 12, \ + 0, \ + 8, \ + 8, \ + 12, \ + 4, \ + 0, \ + 4, \ + 8, \ + 4, \ + 4, \ + 4, \ + 8, \ + 4, \ + 8, \ + 0, \ + 12, \ + 4, \ + 12, \ + 8, \ + 12, \ + 0, \ + 8, \ + 0, \ + 12, \ + 4, \ + 0, \ + 0, \ + 12, \ + 0, \ + 8, \ + 4, \ + 8, \ + 8, \ + 4, \ + 8, \ + 12, \ + 12, \ + 8, \ + 4, \ + 0, \ + 8, \ + 8, \ + 8, \ + 0, \ + 0, \ + 4, \ + 12, \ + 0, \ + 12, \ + 8, \ + 8, \ + 8, \ + 8, \ + 12, \ + 4, \ + 8, \ + 8, \ + 12, \ + 12, \ + 8, \ + 12, \ + 12, \ + 4, \ + 8, \ + 4, \ + 4, \ + 8, \ + 4, \ + 8, \ + 0, \ + 0, \ + 0, \ + 0, \ + 0, \ + 4, \ + 4, \ + 4, \ + 4, \ + 4, \ + 0, \ + 12, \ + 0, \ + 12, \ + 4, \ + 12, \ + 8, \ + 0, \ + 4, \ + 4, \ + 8, \ + 4, \ + 12, \ + 4, \ + 8, \ + 12, \ + 12, \ + 8, \ + 8, \ + 0, \ + 12, \ + 4, \ + 4, \ + 8, \ + 8, \ + 8, \ + 0, \ + 4, \ + 12, \ + 8, \ + 4, \ + 0, \ + 8, \ + 0, \ + 4, \ + 0, \ + 8, \ + 4, \ + 8, \ + 12, \ + 12, \ + 12, \ + 4, \ + 0, \ + 8, \ + 12, \ + 0, \ + 0, \ + 0, \ + 12, \ + 0, \ + 4, \ + 0, \ + 12, \ + 4, \ + 0, \ + 8, \ + 4, \ + 12, \ + 0, \ + 4, \ + 0, \ + 4, \ + 12, \ + 12, \ + 8, \ + 4, \ + 8, \ + 0, \ + 4, \ + 0, \ + 0, \ + 4, \ + 0, \ + 0, \ + 0, \ + 0, \ + 12, \ + 0, \ + 12, \ + 8, \ + 0, \ + 8, \ + 4, \ + 0, \ + 8, \ + 4, \ + 4, \ + 12, \ + 0, \ + 12, \ + 0, \ + 12, \ + 0, \ + 8, \ + 12, \ + 8, \ + 8, \ + 12, \ + 4, \ + 12, \ + 12, \ + 12, \ + 8, \ + 8, \ + 12, \ + 8, \ + 12, \ + 12, \ + 12, \ + 8, \ + 0, \ + 0, \ + 4, \ + 8, \ + 0, \ + 12, \ + 4, \ + 0, \ + 4, \ + 4, \ + 0, \ + 4, \ + 12, \ + 4, \ + 0, \ + 12, \ + 12, \ + 4, \ + 0, \ + 4, \ + 0, \ + 12, \ + 0, \ + 12, \ + 8, \ + 12, \ + 0, \ + 4, \ + 12, \ + 4, \ + 4, \ + 12, \ + 12, \ + 12, \ + 0, \ + 12, \ + 4, \ + 8, \ + 0, \ + 12, \ + 4, \ + 8, \ + 0, \ + 4, \ + 4, \ + 12, \ + 8, \ + 4, \ + 12, \ + 8, \ + 12, \ + 8, \ + 0 \ +} + + +/* ============================================================ */ +/* Pre-processor constants */ + +#define BITS_PER_WORD 32 +#define BYTES_PER_WORD 4 +#define NUMBER_OF_SETS 4 +#define ENTRIES_PER_SET 4 +#define ALL_ONES_MASK 0xFFFFFFFF + +#define ZERO_TAG 0x0 +#define PARTIAL_TAG 0x1 +#define MISS_TAG 0x2 +#define EXACT_TAG 0x3 + +#define BITS_PER_BYTE 8 + +/* ============================================================ */ +/* Global macros */ + +/* Strip the low bits of a pattern to give the high bits pattern. + The stripped patterns are used for initial tests of partial + matches. */ +#define STRIP_LOW_BITS(pattern) (pattern & 0xFFFFFC00) + +/* String the high bits of a pattern so the low order bits can + be included in an encoding of a partial match. */ +#define STRIP_HIGH_BITS(pattern) (pattern & 0x3FF) + +/* This macro defined the contents of the initial index table. The + sets that compose the dictionary are stored in a single array. + Given an index to some element in the dictionary, that index can be + used with this table to obtain the index of the first element of + that set in the dictionary array. */ +#define INITIAL_INDEX_TABLE_CONTENTS { \ + 0, 0, 0, 0, \ + 4, 4, 4, 4, \ + 8, 8, 8, 8, \ + 12, 12, 12, 12 \ +} + +/* Given pointers to source buffer (sourceBuffer) of uncompressed data + and a destination buffer (destinationBuffer) into which compressed + data can be placed, as well as the number of words in the source + buffer (words), compress the contents of the source and place the + results in the destination. Return the number of bytes placed into + the destination. */ +unsigned int WK4x4_compress (WK_word* sourceBuffer, + WK_word* destinationBuffer, + unsigned int words, + struct comp_alg_data * data); + +/* Given a pointer to a source buffer (sourceBuffer) of compressed + data and a pointer to a destination buffer (destinationBuffer) into + which uncompressed data can be placed, as well as the number of + uncompressed words that will be written to the destination buffer + (words), decompress the data into the destination buffer. */ +void WK4x4_decompress (WK_word* sourceBuffer, + WK_word* destinationPage, + unsigned int words, + struct comp_alg_data * data); + +/* Given a pointer to a source buffer (sourceBuffer) of uncompressed + data, the number of words stored in that buffer (words), and two + arrays for counting the number of exact (exactMatchCountArray) and + partial (partialMatchCountArray) matches at each LRU position, + compress the source and record what types of hits (partial or + exact) occurred at each queue position. Return the number of words + that would be placed into a destination buffer (if there were + one). */ +unsigned int +WK4x4_measureCompress (WK_word* sourceBuffer, + unsigned int words, + unsigned int* exactMatchCountArray, + unsigned int* partialMatchCountArray); + +#endif /* _WK4x4_H */ diff -urN linux-2.4.26-rc1/include/linux/WKcommon.h linux-2.4.26-rc1-ccache/include/linux/WKcommon.h --- linux-2.4.26-rc1/include/linux/WKcommon.h Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/include/linux/WKcommon.h Sun Mar 28 17:12:21 2004 @@ -0,0 +1,22 @@ +#ifndef _WKCOMMON_H +#define _WKCOMMON_H + +/* A manipulable type that is the machine word size. */ +typedef unsigned long WK_word; + +/* A structure to store each element of the dictionary. */ +typedef WK_word DictionaryElement; + + +#define PAGE_COMPRESS_WORDS_PER_PAGE 1024 +#define DICTIONARY_SIZE 16 + +static inline unsigned int +ceil (double x) { + if (!(x - ((int) x))) + return ((unsigned int) x); + + return ((unsigned int) (x + 1)); +} + +#endif diff -urN linux-2.4.26-rc1/include/linux/WKdm.h linux-2.4.26-rc1-ccache/include/linux/WKdm.h --- linux-2.4.26-rc1/include/linux/WKdm.h Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/include/linux/WKdm.h Sun Mar 28 17:12:21 2004 @@ -0,0 +1,81 @@ +/* + WKdm-v0.1 -- Wilson-Kaplan-dm (direct-mapped) version 0.1 + + Compresses buffers using a dictionary based match and partial match + scheme. + + Paul Wilson -- wilson@cs.utexas.edu + Scott F. Kaplan -- sfkaplan@cs.utexas.edu + September 1997 */ + +/* ============================================================ */ +/* Preprocessor directives to avoid multiple inclusion */ +#ifndef _LINUX_WKDM_H +#define _LINUX_WKDM_H + +/* these are the constants for the hash function lookup table. + * Only zero maps to zero. The rest of the tabale is the result + * of appending 17 randomizations of the multiples of 4 from + * 4 to 56. Generated by a Scheme script in hash.scm. + */ +#define HASH_LOOKUP_TABLE_CONTENTS_WKDM { \ + 0, 52, 8, 56, 16, 12, 28, 20, 4, 36, 48, 24, 44, 40, 32, 60, \ + 8, 12, 28, 20, 4, 60, 16, 36, 24, 48, 44, 32, 52, 56, 40, 12, \ + 8, 48, 16, 52, 60, 28, 56, 32, 20, 24, 36, 40, 44, 4, 8, 40, \ + 60, 32, 20, 44, 4, 36, 52, 24, 16, 56, 48, 12, 28, 16, 8, 40, \ + 36, 28, 32, 12, 4, 44, 52, 20, 24, 48, 60, 56, 40, 48, 8, 32, \ + 28, 36, 4, 44, 20, 56, 60, 24, 52, 16, 12, 12, 4, 48, 20, 8, \ + 52, 16, 60, 24, 36, 44, 28, 56, 40, 32, 36, 20, 24, 60, 40, 44, \ + 52, 16, 32, 4, 48, 8, 28, 56, 12, 28, 32, 40, 52, 36, 16, 20, \ + 48, 8, 4, 60, 24, 56, 44, 12, 8, 36, 24, 28, 16, 60, 20, 56, \ + 32, 40, 48, 12, 4, 44, 52, 44, 40, 12, 56, 8, 36, 24, 60, 28, \ + 48, 4, 32, 20, 16, 52, 60, 12, 24, 36, 8, 4, 16, 56, 48, 44, \ + 40, 52, 32, 20, 28, 32, 12, 36, 28, 24, 56, 40, 16, 52, 44, 4, \ + 20, 60, 8, 48, 48, 52, 12, 20, 32, 44, 36, 28, 4, 40, 24, 8, \ + 56, 60, 16, 36, 32, 8, 40, 4, 52, 24, 44, 20, 12, 28, 48, 56, \ + 16, 60, 4, 52, 60, 48, 20, 16, 56, 44, 24, 8, 40, 12, 32, 28, \ + 36, 24, 32, 12, 4, 20, 16, 60, 36, 28, 8, 52, 40, 48, 44, 56 \ +} + + +/* ============================================================ */ +/* Interface */ + +/* Given pointers to source buffer (sourceBuffer) of uncompressed data + and a destination buffer (destinationBuffer) into which compressed + data can be placed, as well as the number of words in the source + buffer (words), compress the contents of the source and place the + results in the destination. Return the number of bytes placed into + the destination. */ +unsigned int +WKdm_compress (WK_word* sourceBuffer, + WK_word* destinationBuffer, + unsigned int words, + struct comp_alg_data * data); + +/* Given a pointer to a source buffer (sourceBuffer) of compressed + data and a pointer to a destination buffer (destinationBuffer) into + which uncompressed data can be placed, as well as the number of + uncompressed words that will be written to the destination buffer + (words), decompress the data into the destination buffer. */ +void +WKdm_decompress (WK_word* sourceBuffer, + WK_word* destinationPage, + unsigned int words, + struct comp_alg_data * data); + +/* Given a pointer to a source buffer (sourceBuffer) of uncompressed + data, the number of words stored in that buffer (words), and two + arrays for counting the number of exact (exactMatchCountArray) and + partial (partialMatchCountArray) matches at each LRU position, + compress the source and record what types of hits (partial or + exact) occurred at each queue position. Return the number of words + that would be placed into a destination buffer (if there were + one). */ +unsigned int +WKdm_measureCompress (WK_word* sourceBuffer, + unsigned int words, + unsigned int* exactMatchCountArray, + unsigned int* partialMatchCountArray); + +#endif /* _LINUX_WKDM_H */ diff -urN linux-2.4.26-rc1/include/linux/comp_cache.h linux-2.4.26-rc1-ccache/include/linux/comp_cache.h --- linux-2.4.26-rc1/include/linux/comp_cache.h Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/include/linux/comp_cache.h Sun Mar 28 17:12:21 2004 @@ -0,0 +1,600 @@ +/* + * linux/mm/comp_cache.h + * + * Time-stamp: <2002-11-21 16:46:32 rcastro> + * + * Linux Virtual Memory Compressed Cache + * + * Author: Rodrigo S. de Castro + * + * 2001 + */ + +#ifndef _LINUX_COMP_CACHE_H +#define _LINUX_COMP_CACHE_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#define COMP_CACHE_VERSION "0.24pre5" + +/* maximum compressed size of a page */ +#define MAX_COMPRESSED_SIZE 4500 + +/* compressed cache metadata */ +extern unsigned long num_fragments, num_active_fragments; +extern unsigned long num_swapper_fragments, num_clean_fragments; + +extern unsigned long num_comp_pages, zone_num_comp_pages; +extern unsigned long min_num_comp_pages, max_num_comp_pages, max_used_num_comp_pages; + +extern kmem_cache_t * fragment_cachep; + +struct pte_list { + struct pte_list * next; + pte_t * ptep; +}; + +struct comp_cache_fragment { + /* list of fragments in a comp page*/ + struct list_head list; + + /* list for lru queue ordering */ + struct list_head lru_queue; + + /* list of comp pages in the mapping */ + struct list_head mapping_list; + + /* usage count */ + atomic_t count; + + unsigned long index; + struct address_space * mapping; + + struct swp_buffer * swp_buffer; + + /* offset in the compressed cache we are stored in */ + unsigned short offset; + + unsigned short compressed_size; + unsigned long flags; + + struct comp_cache_page * comp_page; + + struct comp_cache_fragment * next_hash; + struct comp_cache_fragment ** pprev_hash; +}; + +struct comp_cache_page { + struct page * page; + + /* fields for compression structure */ + unsigned short free_offset; + + /* free space that can used right away */ + short free_space; + + /* total free space = free_space + fragmented space, ie the + * sum of all Freed fragments waiting to be merged */ + short total_free_space; + + struct list_head fragments; + + /* free space hash table */ + struct comp_cache_page * next_hash_fs; + struct comp_cache_page ** pprev_hash_fs; + + /* total free space hash table */ + struct comp_cache_page * next_hash_tfs; + struct comp_cache_page ** pprev_hash_tfs; +}; + +#define alloc_fragment() \ + ((struct comp_cache_fragment *) kmem_cache_alloc(fragment_cachep, SLAB_ATOMIC)) + +#define alloc_comp_cache() \ + ((struct comp_cache_page *) kmem_cache_alloc(comp_cachep, SLAB_ATOMIC)) + +#define alloc_vswap() \ + ((struct vswap_address *) kmem_cache_alloc(vswap_cachep, SLAB_ATOMIC)) + +#define alloc_swp_buffer() \ + ((struct swp_buffer *) kmem_cache_alloc(comp_cachep, SLAB_ATOMIC)) + +extern int shmem_page(struct page *); +extern void comp_cache_fix_watermarks(int); + +/* adaptivity.c */ +#ifdef CONFIG_COMP_CACHE +extern unsigned long failed_comp_page_allocs; +extern int growth_lock; + +int grow_on_demand(void); +int shrink_on_demand(struct comp_cache_page *); +void compact_comp_cache(void); + +#ifdef CONFIG_COMP_DIS_CLEAN +static inline void hit_clean_page(struct page * page) { }; +static inline void add_clean_page(struct page * page) { }; +#else +void hit_clean_page(struct page *); +void add_clean_page(struct page *); +#endif + +#ifdef CONFIG_COMP_DIS_ADAPT +static inline void balance_lru_queues(void) { }; +#else +void balance_lru_queues(void); +#endif + +#else +static inline int grow_on_demand(void) { return 0; } +static inline int shrink_on_demand(struct comp_cache_page * comp_page) { return 0; } +#endif + +extern unsigned long clean_page_hash_size; +extern unsigned int clean_page_hash_bits; + +struct clean_page_data { + struct list_head list; + + struct address_space * mapping; + unsigned long index; + + struct clean_page_data * next_hash; + struct clean_page_data ** pprev_hash; +}; + +static inline unsigned long +clean_page_hashfn(struct address_space * mapping, unsigned long index) +{ +#define i (((unsigned long) mapping)/(sizeof(struct inode) & ~ (sizeof(struct inode) - 1))) +#define s(x) ((x)+((x) >> clean_page_hash_bits)) + return s(i+index) & (clean_page_hash_size - 1); +#undef i +#undef s +} + +/* swapout.c */ +extern struct list_head swp_free_buffer_head; + +int writeout_fragments(unsigned int, int, int); + +/* -- Fragment Flags */ + +/* CF_Dirty: is the fragment dirty? */ +#define CF_Dirty 0 + +#define CF_ToBeFreed 1 +#define CF_Active 2 +#define CF_End 3 + +#define CompFragmentDirty(fragment) test_bit(CF_Dirty, &(fragment)->flags) +#define CompFragmentSetDirty(fragment) set_bit(CF_Dirty, &(fragment)->flags) +#define CompFragmentTestandSetDirty(fragment) test_and_set_bit(CF_Dirty, &(fragment)->flags) +#define CompFragmentTestandClearDirty(fragment) test_and_clear_bit(CF_Dirty, &(fragment)->flags) +#define CompFragmentClearDirty(fragment) clear_bit(CF_Dirty, &(fragment)->flags) + +#define CompFragmentToBeFreed(fragment) test_bit(CF_ToBeFreed, &(fragment)->flags) +#define CompFragmentSetToBeFreed(fragment) set_bit(CF_ToBeFreed, &(fragment)->flags) +#define CompFragmentTestandSetToBeFreed(fragment) test_and_set_bit(CF_ToBeFreed, &(fragment)->flags) + +#define CompFragmentActive(fragment) test_bit(CF_Active, &(fragment)->flags) +#define CompFragmentSetActive(fragment) set_bit(CF_Active, &(fragment)->flags) +#define CompFragmentTestandSetActive(fragment) test_and_set_bit(CF_Active, &(fragment)->flags) +#define CompFragmentTestandClearActive(fragment) test_and_clear_bit(CF_Active, &(fragment)->flags) +#define CompFragmentClearActive(fragment) clear_bit(CF_Active, &(fragment)->flags) + +#define CompFragmentEnd(fragment) test_bit(CF_End, &(fragment)->flags) +#define CompFragmentSetEnd(fragment) set_bit(CF_End, &(fragment)->flags) +#define CompFragmentTestandSetEnd(fragment) test_and_set_bit(CF_End, &(fragment)->flags) +#define CompFragmentTestandClearEnd(fragment) test_and_clear_bit(CF_End, &(fragment)->flags) +#define CompFragmentClearEnd(fragment) clear_bit(CF_End, &(fragment)->flags) + +/* general */ +#define get_fragment(f) do { \ + if (atomic_read(&(f)->count) == 0) \ + BUG(); \ + atomic_inc(&(f)->count); \ +} while(0); + +#define put_fragment(f) __comp_cache_free(f) +#define put_fragment_testzero(f) atomic_dec_and_test(&(f)->count) +#define fragment_count(f) atomic_read(&(f)->count) +#define set_fragment_count(f,v) atomic_set(&(f)->count, v) + +extern int __comp_cache_free(struct comp_cache_fragment *); + +static inline int +drop_fragment(struct comp_cache_fragment * fragment) +{ + int err = 0; + if (!CompFragmentTestandSetToBeFreed(fragment)) + err = put_fragment(fragment); + return err; +} + +#define INF 0xffffffff + +#define NUM_SWP_BUFFERS 32 + +/* do not change the fields order */ +struct swp_buffer { + struct list_head list; + + struct page * page; /* page for IO */ + struct comp_cache_fragment * fragment; /* pointer to the fragment we are doing IO */ +}; + +#define NUM_MEAN_PAGES 100 + +#define for_each_fragment_safe(p, p_tmp, comp_page) list_for_each_safe(p, p_tmp, &(comp_page->fragments)) +#define for_each_fragment(p, comp_page) list_for_each(p, &(comp_page->fragments)) + +#define member_offset(key) ((unsigned long) (&((struct comp_cache_page *)0)->key)) + +#define apply_key_offset(node, offset) (* (unsigned short *)((char *) node + offset)) +#define apply_info_offset(node, offset) ((struct avl_info *)((char *) node + offset)) + +#define compressed(fragment) ((fragment)->compressed_size < PAGE_SIZE) +#define not_compressed(fragment) ((fragment)->compressed_size == PAGE_SIZE) + +#define mapped(page) (!page->buffers && page_count(page) > 2) + +#define DISCARD_MARK 0.80 + +struct comp_alg_data { + /* WKdm and WK4x4 */ + WK_word *tempTagsArray; + WK_word *tempQPosArray; + WK_word *tempLowBitsArray; + + /* LZO */ + lzo_byte * wrkmem; + unsigned short compressed_size; + + /* Compressed Swap */ + struct page * decompress_buffer; +}; + +typedef unsigned int (compress_function_t)(unsigned long* source, + unsigned long* destination, + unsigned int words, + struct comp_alg_data * data); + +typedef void (decompress_function_t)(unsigned long* source, + unsigned long* destination, + unsigned int words, + struct comp_alg_data * data); + +/* proc.c */ +extern int clean_page_compress_lock; + +#ifdef CONFIG_COMP_CACHE +void decompress_fragment_to_page(struct comp_cache_fragment *, struct page *); +void decompress_swap_cache_page(struct page *); +int compress(struct page *, void *, int); + +void __init comp_cache_algorithms_init(void); +#else +static inline void decompress_swap_cache_page(struct page * page) { }; +#endif + +#ifdef CONFIG_COMP_SWAP +void get_comp_data(struct page *, unsigned short *, unsigned short *); +#else +static inline void +get_comp_data(struct page * page, unsigned short * size, unsigned short * offset) +{ + *size = *((unsigned short *) page_address(page)); + *offset = sizeof(unsigned short); +} +#endif + + +/* swapin.c */ +#ifdef CONFIG_COMP_CACHE +extern int FASTCALL(flush_comp_cache(struct page *)); + +#define read_comp_cache(mapping, index, page) __read_comp_cache(mapping, index, page, CLEAN_PAGE) +#define read_dirty_comp_cache(mapping, index, page) __read_comp_cache(mapping, index, page, DIRTY_PAGE) + +int __read_comp_cache(struct address_space *, unsigned long, struct page *, int); +int invalidate_comp_cache(struct address_space *, unsigned long); +void invalidate_comp_pages(struct address_space *); +void truncate_comp_pages(struct address_space *, unsigned long, unsigned); +void wait_comp_pages(struct address_space *); +void lookup_comp_pages(struct address_space *); +#define there_are_dirty_comp_pages(mapping) (!list_empty(&(mapping)->dirty_comp_pages)) +#define there_are_locked_comp_pages(mapping) (!list_empty(&(mapping)->locked_comp_pages)) +#else +static inline int read_comp_cache(struct address_space * mapping, unsigned long offset, struct page * page) { return -ENOENT; } +static inline int invalidate_comp_cache(struct address_space * mapping, unsigned long offset) { return -ENOENT; } +static inline int flush_comp_cache(struct page * page) { return -ENOENT; } +static inline void invalidate_comp_pages(struct address_space * mapping) { }; +static inline void truncate_comp_pages(struct address_space * mapping, unsigned long start, unsigned partial) { }; +static inline void wait_comp_pages(struct address_space * mapping) { }; +static inline void lookup_comp_pages(struct address_space * mapping) { }; +#define there_are_dirty_comp_pages(mapping) 0 +#define there_are_locked_comp_pages(mapping) 0 +#endif + +/* main.c */ +#ifdef CONFIG_COMP_CACHE +void comp_cache_init(void); +inline int init_comp_page(struct comp_cache_page **,struct page *); +int compress_dirty_page(struct page *, int (*writepage)(struct page *), unsigned int, int); +int compress_clean_page(struct page *, unsigned int, int); + +#define CLEAN_PAGE 0 +#define DIRTY_PAGE 1 + +#ifdef CONFIG_COMP_DOUBLE_PAGE +#define COMP_PAGE_ORDER 1 +#else +#define COMP_PAGE_ORDER 0 +#endif + +#define COMP_PAGE_SIZE (PAGE_SIZE << COMP_PAGE_ORDER) +#define comp_cache_used_space ((num_comp_pages * COMP_PAGE_SIZE) - comp_cache_free_space) + +#define page_to_comp_page(n) ((n) >> COMP_PAGE_ORDER) +#define comp_page_to_page(n) ((n) << COMP_PAGE_ORDER) + +extern unsigned long comp_cache_free_space; +extern spinlock_t comp_cache_lock; +extern struct comp_cache_fragment * last_checked_inactive; +#else +static inline void comp_cache_init(void) {}; +static inline int compress_dirty_page(struct page * page, int (*writepage)(struct page *), unsigned int gfp_mask, int priority) { writepage(page); return 0; } +static inline int compress_clean_page(struct page * page, unsigned int gfp_mask, int priority) { return 1; } +#endif + +/* vswap.c */ +struct vswap_address { + struct list_head list; + + /* how many ptes are set to this vswap address */ + unsigned int swap_count; + /* number of faults being serviced at a given moment */ + unsigned int fault_count; + /* offset within the vswap table */ + unsigned long offset; + + struct comp_cache_fragment * fragment; + + struct page * swap_cache_page; + + struct pte_list * pte_list; +}; + +extern unsigned long comp_cache_freeable_space; + +extern struct vswap_address ** vswap_address; +extern struct list_head vswap_address_free_head; +extern struct list_head vswap_address_used_head; + +extern unsigned long vswap_num_reserved_entries; +extern unsigned long vswap_current_num_entries; +extern unsigned long vswap_num_used_entries; +extern unsigned long vswap_num_swap_cache; +extern unsigned int vswap_last_used; +extern int last_vswap_allocated; + +extern unsigned short * last_page_size; +extern unsigned short last_page; + +#define NUM_VSWAP_ENTRIES (3 * comp_page_to_page(num_comp_pages)) +#define COMP_CACHE_SWP_TYPE MAX_SWAPFILES +#define VSWAP_RESERVED ((struct comp_cache_fragment *) 0xffffffff) + +#define VSWAP_ALLOCATING ((struct page *) 0xffffffff) + +extern spinlock_t virtual_swap_list; + +#ifdef CONFIG_COMP_CACHE +#define vswap_info_struct(p) (p == &swap_info[COMP_CACHE_SWP_TYPE]) +#define vswap_address(entry) (SWP_TYPE(entry) == COMP_CACHE_SWP_TYPE) +#define reserved(offset) (vswap_address[offset]->fragment == VSWAP_RESERVED) + +int virtual_swap_duplicate(swp_entry_t); +int __virtual_swap_free(unsigned long); +int virtual_swap_free(unsigned long); +swp_entry_t get_virtual_swap_page(void); +void add_fragment_vswap(struct comp_cache_fragment *); + +int comp_cache_available_space(void); + +extern void FASTCALL(add_pte_vswap(pte_t *, swp_entry_t)); +extern void FASTCALL(remove_pte_vswap(pte_t *)); +extern void FASTCALL(add_swap_cache_page_vswap(struct page *, swp_entry_t)); +extern void FASTCALL(del_swap_cache_page_vswap(struct page *)); +extern int FASTCALL(free_pte_list(struct pte_list *, unsigned long)); +extern void FASTCALL(get_vswap(swp_entry_t)); +extern void FASTCALL(put_vswap(swp_entry_t)); + +int vswap_alloc_and_init(struct vswap_address **, unsigned long); + +#else + +#define vswap_address_available() (0) +#define vswap_info_struct(p) (0) +#define vswap_address(entry) (0) + +static inline int virtual_swap_duplicate(swp_entry_t entry) { return 0; }; +static inline int virtual_swap_free(unsigned long offset) { return 0; } +static inline swp_entry_t get_virtual_swap_page(void) { return (swp_entry_t) { 0 }; } + +static inline int comp_cache_available_space(void) { return 0; } + +static inline void add_pte_vswap(pte_t * ptep, swp_entry_t entry) {}; +static inline void remove_pte_vswap(pte_t * ptep) {}; +static inline void add_swap_cache_page_vswap(struct page * page, swp_entry_t entry) {}; +static inline void del_swap_cache_page_vswap(struct page * page) {}; +static inline int free_pte_list(struct pte_list * pte_list, unsigned long offset) { return 0; } +static inline void get_vswap(swp_entry_t entry) {}; +static inline void put_vswap(swp_entry_t entry) {}; +#endif + +/* free.c */ +#ifdef CONFIG_COMP_CACHE + +#define fragment_freed(fragment) (!fragment_count(fragment) && !fragment->mapping) + +int comp_cache_use_address(swp_entry_t); +int __comp_cache_free(struct comp_cache_fragment *); + +/* from Riel's rmap patch */ +static inline void pgtable_add_rmap(pte_t * ptep, struct mm_struct * mm, unsigned long address) +{ + struct page * page = virt_to_page(ptep); + + page->mapping = (void *)mm; + page->index = address & ~((PTRS_PER_PTE * PAGE_SIZE) - 1); +} + +static inline void pgtable_remove_rmap(pte_t * ptep) +{ + struct page * page = virt_to_page(ptep); + + page->mapping = NULL; + page->index = 0; +} + +static inline struct mm_struct * ptep_to_mm(pte_t * ptep) +{ + struct page * page = virt_to_page(ptep); + + return (struct mm_struct *) page->mapping; +} + +static inline unsigned long ptep_to_address(pte_t * ptep) +{ + struct page * page = virt_to_page(ptep); + unsigned long low_bits; + + low_bits = ((unsigned long)ptep & ~PAGE_MASK) * PTRS_PER_PTE; + return page->index + low_bits; +} + +void compact_fragments(struct comp_cache_page *); + +#else +static inline int comp_cache_use_address(swp_entry_t entry) { return 0; } +static inline void pgtable_add_rmap(pte_t * ptep, struct mm_struct * mm, unsigned long address) { } +static inline void pgtable_remove_rmap(pte_t * ptep) { } +static inline struct mm_struct * ptep_to_mm(pte_t * ptep) { return 0; } +static inline unsigned long ptep_to_address(pte_t * ptep) { return 0; } +#endif + +/* aux.c */ +unsigned long long big_division(unsigned long long, unsigned long long); +inline void set_comp_page(struct comp_cache_page *, struct page *); +inline void check_all_fragments(struct comp_cache_page *); +void add_to_comp_page_list(struct comp_cache_page *, struct comp_cache_fragment *); + + +extern struct comp_cache_fragment ** fragment_hash; +extern unsigned long fragment_hash_size; +extern unsigned long fragment_hash_used; +extern unsigned int fragment_hash_order; +extern unsigned int fragment_hash_bits; + +#define NUM_FRAG_HASH_ENTRIES (3 * comp_page_to_page(num_comp_pages)) + +/* hash function adapted from _page_hashfn:pagemap.h since our + * parameters for hash table are the same: mapping and index */ +static inline unsigned long +__fragment_hashfn(struct address_space * mapping, unsigned long index, unsigned int hash_size, unsigned int hash_bits) +{ +#define i (((unsigned long) mapping)/(sizeof(struct inode) & ~ (sizeof(struct inode) - 1))) +#define s(x) ((x) + ((x) >> hash_bits)) + return s(i + index) & (hash_size - 1); +#undef i +#undef s +} + +inline void __add_fragment_to_hash_table(struct comp_cache_fragment **, unsigned int, struct comp_cache_fragment *); +inline void remove_fragment_from_hash_table(struct comp_cache_fragment *); + +static inline void add_fragment_to_hash_table(struct comp_cache_fragment * fragment) { + __add_fragment_to_hash_table(fragment_hash, __fragment_hashfn(fragment->mapping, fragment->index, fragment_hash_size, fragment_hash_bits), fragment); +} + +extern struct comp_cache_page ** free_space_hash; +extern unsigned int free_space_hash_size; +extern unsigned int free_space_interval; + +extern struct comp_cache_page ** total_free_space_hash; +extern unsigned int total_free_space_hash_size; +extern unsigned int total_free_space_interval; + +static inline int free_space_hashfn(int free_space) +{ + if (!free_space) + return 0; + + free_space -= (free_space % free_space_interval); + + return (free_space/free_space_interval + 1); +} + +inline void add_comp_page_to_hash_table(struct comp_cache_page *); +inline void remove_comp_page_from_hash_table(struct comp_cache_page *); +int set_pte_list_to_entry(struct pte_list *, swp_entry_t, swp_entry_t); + +struct comp_cache_page * FASTCALL(search_comp_page(struct comp_cache_page ** hash_table, int free_space)); + +struct comp_cache_fragment ** create_fragment_hash(unsigned long *, unsigned int *, unsigned int *); +extern struct list_head active_lru_queue, inactive_lru_queue; + +inline void add_fragment_to_active_lru_queue(struct comp_cache_fragment *); +inline void add_fragment_to_active_lru_queue_tail(struct comp_cache_fragment *); +inline void add_fragment_to_inactive_lru_queue(struct comp_cache_fragment *); +inline void add_fragment_to_inactive_lru_queue_tail(struct comp_cache_fragment *); +inline void remove_fragment_from_lru_queue(struct comp_cache_fragment *); + +unsigned long free_space_count(int, unsigned long *); +unsigned long fragmentation_count(int, unsigned long *, int); + +/* enough memory functions */ +#ifdef CONFIG_COMP_CACHE +extern int FASTCALL(find_comp_page(struct address_space *, unsigned long, struct comp_cache_fragment **)); +inline int in_comp_cache(struct address_space *, unsigned long); +#else +static inline int find_comp_page(struct address_space * mapping, unsigned long offset, struct comp_cache_fragment ** fragment) { return -ENOENT; } +static inline int in_comp_cache(struct address_space * mapping, unsigned long offset) { return 0; } +#endif + +/* proc.c */ +int comp_cache_stat_read_proc(char *, char **, off_t, int, int *, void *); +int comp_cache_hist_read_proc(char *, char **, off_t, int, int *, void *); +int comp_cache_frag_read_proc(char *, char **, off_t, int, int *, void *); + + +#endif /* _LINUX_COMP_CACHE_H */ + +/* + * Overrides for Emacs so that we follow Linus's tabbing style. + * Emacs will notice this stuff at the end of the file and automatically + * adjust the settings for this buffer only. This must remain at the end + * of the file. + * --------------------------------------------------------------------------- + * Local variables: + * c-file-style: "linux" + * End: + */ diff -urN linux-2.4.26-rc1/include/linux/fs.h linux-2.4.26-rc1-ccache/include/linux/fs.h --- linux-2.4.26-rc1/include/linux/fs.h Sun Mar 28 13:50:54 2004 +++ linux-2.4.26-rc1-ccache/include/linux/fs.h Sun Mar 28 17:12:21 2004 @@ -409,6 +409,11 @@ struct list_head clean_pages; /* list of clean pages */ struct list_head dirty_pages; /* list of dirty pages */ struct list_head locked_pages; /* list of locked pages */ +#ifdef CONFIG_COMP_CACHE + struct list_head clean_comp_pages; /* list of clean compressed pages */ + struct list_head dirty_comp_pages; /* list of dirty compressed pages */ + struct list_head locked_comp_pages; /* list of locked compressed pages */ +#endif unsigned long nrpages; /* number of total pages */ struct address_space_operations *a_ops; /* methods */ struct inode *host; /* owner: inode, block_device */ diff -urN linux-2.4.26-rc1/include/linux/lzoconf.h linux-2.4.26-rc1-ccache/include/linux/lzoconf.h --- linux-2.4.26-rc1/include/linux/lzoconf.h Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/include/linux/lzoconf.h Sun Mar 28 17:12:21 2004 @@ -0,0 +1,418 @@ +/* lzoconf.h -- configuration for the LZO real-time data compression library + + This file is part of the LZO real-time data compression library. + + Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer + Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer + Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer + Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer + Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer + + The LZO library is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + The LZO library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with the LZO library; see the file COPYING. + If not, write to the Free Software Foundation, Inc., + 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + Markus F.X.J. Oberhumer + + http://wildsau.idv.uni-linz.ac.at/mfx/lzo.html + */ + + +#ifndef __LZOCONF_H +#define __LZOCONF_H + +#define LZO_VERSION 0x1070 +#define LZO_VERSION_STRING "1.07" +#define LZO_VERSION_DATE "Oct 18 2000" + +/* internal Autoconf configuration file - only used when building LZO */ +#if defined(LZO_HAVE_CONFIG_H) +# include +#endif + +/* definitions from limits.h */ +#define CHAR_BIT 8 +#define UCHAR_MAX 255 + +#ifndef __INT_MAX__ +#define __INT_MAX__ 2147483647 +#endif +#undef INT_MIN +#define INT_MIN (-INT_MAX-1) +#undef INT_MAX +#define INT_MAX __INT_MAX__ + +#undef UINT_MAX +#define UINT_MAX (INT_MAX * 2U + 1) + +#ifndef __LONG_MAX__ +#if defined (__alpha__) || (defined (_ARCH_PPC) && defined (__64BIT__)) || defined (__sparc_v9__) || defined (__sparcv9) +#define __LONG_MAX__ 9223372036854775807L +#else +#define __LONG_MAX__ 2147483647L +#endif /* __alpha__ || sparc64 */ +#endif +#undef LONG_MIN +#define LONG_MIN (-LONG_MAX-1) +#undef LONG_MAX +#define LONG_MAX __LONG_MAX__ + +#undef ULONG_MAX +#define ULONG_MAX (LONG_MAX * 2UL + 1) + +#define USHRT_MAX 65535 +#define SHRT_MAX 32767 + +#ifdef __cplusplus +extern "C" { +#endif + + +/*********************************************************************** +// LZO requires a conforming +************************************************************************/ + +#if !defined(CHAR_BIT) || (CHAR_BIT != 8) +# error "invalid CHAR_BIT" +#endif +#if !defined(UCHAR_MAX) || !defined(UINT_MAX) || !defined(ULONG_MAX) +# error "check your compiler installation" +#endif +#if (USHRT_MAX < 1) || (UINT_MAX < 1) || (ULONG_MAX < 1) +# error "your limits.h macros are broken" +#endif + +/* workaround a cpp bug under hpux 10.20 */ +#define LZO_0xffffffffL 4294967295ul + + +/*********************************************************************** +// architecture defines +************************************************************************/ + +#if !defined(__LZO_WIN) && !defined(__LZO_DOS) && !defined(__LZO_OS2) +# if defined(__WINDOWS__) || defined(_WINDOWS) || defined(_Windows) +# define __LZO_WIN +# elif defined(__WIN32__) || defined(_WIN32) || defined(WIN32) +# define __LZO_WIN +# elif defined(__NT__) || defined(__NT_DLL__) || defined(__WINDOWS_386__) +# define __LZO_WIN +# elif defined(__DOS__) || defined(__MSDOS__) || defined(MSDOS) +# define __LZO_DOS +# elif defined(__OS2__) || defined(__OS2V2__) || defined(OS2) +# define __LZO_OS2 +# elif defined(__palmos__) +# define __LZO_PALMOS +# elif defined(__TOS__) || defined(__atarist__) +# define __LZO_TOS +# endif +#endif + +#if (UINT_MAX < LZO_0xffffffffL) +# if defined(__LZO_WIN) +# define __LZO_WIN16 +# elif defined(__LZO_DOS) +# define __LZO_DOS16 +# elif defined(__LZO_PALMOS) +# define __LZO_PALMOS16 +# elif defined(__LZO_TOS) +# define __LZO_TOS16 +# elif defined(__C166__) +# else +# error "16-bit target not supported - contact me for porting hints" +# endif +#endif + +#if !defined(__LZO_i386) +# if defined(__LZO_DOS) || defined(__LZO_WIN16) +# define __LZO_i386 +# elif defined(__i386__) || defined(__386__) || defined(_M_IX86) +# define __LZO_i386 +# endif +#endif + +#if defined(__LZO_STRICT_16BIT) +# if (UINT_MAX < LZO_0xffffffffL) +# include +# endif +#endif + +/* memory checkers */ +#if !defined(__LZO_CHECKER) +# if defined(__BOUNDS_CHECKING_ON) +# define __LZO_CHECKER +# elif defined(__CHECKER__) +# define __LZO_CHECKER +# elif defined(__INSURE__) +# define __LZO_CHECKER +# elif defined(__PURIFY__) +# define __LZO_CHECKER +# endif +#endif + + +/*********************************************************************** +// integral and pointer types +************************************************************************/ + +/* Integral types with 32 bits or more */ +#if !defined(LZO_UINT32_MAX) +# if (UINT_MAX >= LZO_0xffffffffL) + typedef unsigned int lzo_uint32; + typedef int lzo_int32; +# define LZO_UINT32_MAX UINT_MAX +# define LZO_INT32_MAX INT_MAX +# define LZO_INT32_MIN INT_MIN +# elif (ULONG_MAX >= LZO_0xffffffffL) + typedef unsigned long lzo_uint32; + typedef long lzo_int32; +# define LZO_UINT32_MAX ULONG_MAX +# define LZO_INT32_MAX LONG_MAX +# define LZO_INT32_MIN LONG_MIN +# else +# error "lzo_uint32" +# endif +#endif + +/* lzo_uint is used like size_t */ +#if !defined(LZO_UINT_MAX) +# if (UINT_MAX >= LZO_0xffffffffL) + typedef unsigned int lzo_uint; + typedef int lzo_int; +# define LZO_UINT_MAX UINT_MAX +# define LZO_INT_MAX INT_MAX +# define LZO_INT_MIN INT_MIN +# elif (ULONG_MAX >= LZO_0xffffffffL) + typedef unsigned long lzo_uint; + typedef long lzo_int; +# define LZO_UINT_MAX ULONG_MAX +# define LZO_INT_MAX LONG_MAX +# define LZO_INT_MIN LONG_MIN +# else +# error "lzo_uint" +# endif +#endif + + +/* Memory model that allows to access memory at offsets of lzo_uint. */ +#if !defined(__LZO_MMODEL) +# if (LZO_UINT_MAX <= UINT_MAX) +# define __LZO_MMODEL +# elif defined(__LZO_DOS16) || defined(__LZO_WIN16) +# define __LZO_MMODEL __huge +# define LZO_999_UNSUPPORTED +# elif defined(__LZO_PALMOS16) || defined(__LZO_TOS16) +# define __LZO_MMODEL +# else +# error "__LZO_MMODEL" +# endif +#endif + +/* no typedef here because of const-pointer issues */ +#define lzo_byte unsigned char __LZO_MMODEL +#define lzo_bytep unsigned char __LZO_MMODEL * +#define lzo_charp char __LZO_MMODEL * +#define lzo_voidp void __LZO_MMODEL * +#define lzo_shortp short __LZO_MMODEL * +#define lzo_ushortp unsigned short __LZO_MMODEL * +#define lzo_uint32p lzo_uint32 __LZO_MMODEL * +#define lzo_int32p lzo_int32 __LZO_MMODEL * +#define lzo_uintp lzo_uint __LZO_MMODEL * +#define lzo_intp lzo_int __LZO_MMODEL * +#define lzo_voidpp lzo_voidp __LZO_MMODEL * +#define lzo_bytepp lzo_bytep __LZO_MMODEL * + +typedef int lzo_bool; + +#ifndef lzo_sizeof_dict_t +# define lzo_sizeof_dict_t sizeof(lzo_bytep) +#endif + + +/*********************************************************************** +// function types +************************************************************************/ + +/* linkage */ +#if !defined(__LZO_EXTERN_C) +# ifdef __cplusplus +# define __LZO_EXTERN_C extern "C" +# else +# define __LZO_EXTERN_C extern +# endif +#endif + +/* calling conventions */ +#if !defined(__LZO_CDECL) +# if defined(__LZO_DOS16) || defined(__LZO_WIN16) +# define __LZO_CDECL __far __cdecl +# elif defined(__LZO_i386) && defined(_MSC_VER) +# define __LZO_CDECL __cdecl +# elif defined(__LZO_i386) && defined(__WATCOMC__) +# define __LZO_CDECL __near __cdecl +# else +# define __LZO_CDECL +# endif +#endif +#if !defined(__LZO_ENTRY) +# define __LZO_ENTRY __LZO_CDECL +#endif + +/* DLL export information */ +#if !defined(__LZO_EXPORT1) +# define __LZO_EXPORT1 +#endif +#if !defined(__LZO_EXPORT2) +# define __LZO_EXPORT2 +#endif + +/* calling convention for C functions */ +#if !defined(LZO_PUBLIC) +# define LZO_PUBLIC(_rettype) __LZO_EXPORT1 _rettype __LZO_EXPORT2 __LZO_ENTRY +#endif +#if !defined(LZO_EXTERN) +# define LZO_EXTERN(_rettype) __LZO_EXTERN_C LZO_PUBLIC(_rettype) +#endif +#if !defined(LZO_PRIVATE) +# define LZO_PRIVATE(_rettype) static _rettype __LZO_ENTRY +#endif + +/* cdecl calling convention for assembler functions */ +#if !defined(LZO_PUBLIC_CDECL) +# define LZO_PUBLIC_CDECL(_rettype) \ + __LZO_EXPORT1 _rettype __LZO_EXPORT2 __LZO_CDECL +#endif +#if !defined(LZO_EXTERN_CDECL) +# define LZO_EXTERN_CDECL(_rettype) __LZO_EXTERN_C LZO_PUBLIC_CDECL(_rettype) +#endif + + +typedef int +(__LZO_ENTRY *lzo_compress_t) ( const lzo_byte *src, lzo_uint src_len, + lzo_byte *dst, lzo_uint *dst_len, + lzo_voidp wrkmem ); + +typedef int +(__LZO_ENTRY *lzo_decompress_t) ( const lzo_byte *src, lzo_uint src_len, + lzo_byte *dst, lzo_uint *dst_len, + lzo_voidp wrkmem ); + +typedef int +(__LZO_ENTRY *lzo_optimize_t) ( lzo_byte *src, lzo_uint src_len, + lzo_byte *dst, lzo_uint *dst_len, + lzo_voidp wrkmem ); + +typedef int +(__LZO_ENTRY *lzo_compress_dict_t)(const lzo_byte *src, lzo_uint src_len, + lzo_byte *dst, lzo_uint *dst_len, + lzo_voidp wrkmem, + const lzo_byte *dict, lzo_uint dict_len ); + +typedef int +(__LZO_ENTRY *lzo_decompress_dict_t)(const lzo_byte *src, lzo_uint src_len, + lzo_byte *dst, lzo_uint *dst_len, + lzo_voidp wrkmem, + const lzo_byte *dict, lzo_uint dict_len ); + + +/* a progress indicator callback function */ +typedef void (__LZO_ENTRY *lzo_progress_callback_t) (lzo_uint, lzo_uint); + + +/*********************************************************************** +// error codes and prototypes +************************************************************************/ + +/* Error codes for the compression/decompression functions. Negative + * values are errors, positive values will be used for special but + * normal events. + */ +#define LZO_E_OK 0 +#define LZO_E_ERROR (-1) +#define LZO_E_OUT_OF_MEMORY (-2) /* not used right now */ +#define LZO_E_NOT_COMPRESSIBLE (-3) /* not used right now */ +#define LZO_E_INPUT_OVERRUN (-4) +#define LZO_E_OUTPUT_OVERRUN (-5) +#define LZO_E_LOOKBEHIND_OVERRUN (-6) +#define LZO_E_EOF_NOT_FOUND (-7) +#define LZO_E_INPUT_NOT_CONSUMED (-8) + + +/* lzo_init() should be the first function you call. + * Check the return code ! + * + * lzo_init() is a macro to allow checking that the library and the + * compiler's view of various types are consistent. + */ +#define lzo_init() __lzo_init2(LZO_VERSION,(int)sizeof(short),(int)sizeof(int),\ + (int)sizeof(long),(int)sizeof(lzo_uint32),(int)sizeof(lzo_uint),\ + (int)lzo_sizeof_dict_t,(int)sizeof(char *),(int)sizeof(lzo_voidp),\ + (int)sizeof(lzo_compress_t)) +LZO_EXTERN(int) __lzo_init2(unsigned,int,int,int,int,int,int,int,int,int); + +/* version functions (useful for shared libraries) */ +LZO_EXTERN(unsigned) lzo_version(void); +LZO_EXTERN(const char *) lzo_version_string(void); +LZO_EXTERN(const char *) lzo_version_date(void); +LZO_EXTERN(const lzo_charp) _lzo_version_string(void); +LZO_EXTERN(const lzo_charp) _lzo_version_date(void); + +/* string functions */ +LZO_EXTERN(int) +lzo_memcmp(const lzo_voidp _s1, const lzo_voidp _s2, lzo_uint _len); +LZO_EXTERN(lzo_voidp) +lzo_memcpy(lzo_voidp _dest, const lzo_voidp _src, lzo_uint _len); +LZO_EXTERN(lzo_voidp) +lzo_memmove(lzo_voidp _dest, const lzo_voidp _src, lzo_uint _len); +LZO_EXTERN(lzo_voidp) +lzo_memset(lzo_voidp _s, int _c, lzo_uint _len); + +/* checksum functions */ +LZO_EXTERN(lzo_uint32) +lzo_adler32(lzo_uint32 _adler, const lzo_byte *_buf, lzo_uint _len); +LZO_EXTERN(lzo_uint32) +lzo_crc32(lzo_uint32 _c, const lzo_byte *_buf, lzo_uint _len); + +/* memory allocation functions */ +LZO_EXTERN(lzo_bytep) lzo_alloc(lzo_uint _nelems, lzo_uint _size); +LZO_EXTERN(lzo_bytep) lzo_malloc(lzo_uint _size); +LZO_EXTERN(void) lzo_free(lzo_voidp _ptr); + +typedef lzo_bytep (__LZO_ENTRY *lzo_alloc_hook_t) (lzo_uint, lzo_uint); +typedef void (__LZO_ENTRY *lzo_free_hook_t) (lzo_voidp); + +extern lzo_alloc_hook_t lzo_alloc_hook; +extern lzo_free_hook_t lzo_free_hook; + +/* misc. */ +LZO_EXTERN(lzo_bool) lzo_assert(int _expr); +LZO_EXTERN(int) _lzo_config_check(void); +typedef union { lzo_bytep p; lzo_uint u; } __lzo_pu_u; +typedef union { lzo_bytep p; lzo_uint32 u32; } __lzo_pu32_u; + +/* align a char pointer on a boundary that is a multiple of `size' */ +LZO_EXTERN(unsigned) __lzo_align_gap(const lzo_voidp _ptr, lzo_uint _size); +#define LZO_PTR_ALIGN_UP(_ptr,_size) \ + ((_ptr) + (lzo_uint) __lzo_align_gap((const lzo_voidp)(_ptr),(lzo_uint)(_size))) + +/* deprecated - only for backward compatibility */ +#define LZO_ALIGN(_ptr,_size) LZO_PTR_ALIGN_UP(_ptr,_size) + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* already included */ + diff -urN linux-2.4.26-rc1/include/linux/minilzo.h linux-2.4.26-rc1-ccache/include/linux/minilzo.h --- linux-2.4.26-rc1/include/linux/minilzo.h Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/include/linux/minilzo.h Sun Mar 28 17:12:21 2004 @@ -0,0 +1,97 @@ +/* minilzo.h -- mini subset of the LZO real-time data compression library + + This file is part of the LZO real-time data compression library. + + Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer + Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer + Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer + Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer + Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer + + The LZO library is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + The LZO library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with the LZO library; see the file COPYING. + If not, write to the Free Software Foundation, Inc., + 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + Markus F.X.J. Oberhumer + + http://wildsau.idv.uni-linz.ac.at/mfx/lzo.html + */ + +/* + * NOTE: + * the full LZO package can be found at + * http://wildsau.idv.uni-linz.ac.at/mfx/lzo.html + */ + + +#ifndef __MINILZO_H +#define __MINILZO_H + +#define MINILZO_VERSION 0x1070 + +#ifdef __LZOCONF_H +# error "you cannot use both LZO and miniLZO" +#endif + +#undef LZO_HAVE_CONFIG_H +#include + +#if !defined(LZO_VERSION) || (LZO_VERSION != MINILZO_VERSION) +# error "version mismatch in header files" +#endif + + +#ifdef __cplusplus +extern "C" { +#endif + + +/*********************************************************************** +// +************************************************************************/ + +/* Memory required for the wrkmem parameter. + * When the required size is 0, you can also pass a NULL pointer. + */ + +#define LZO1X_MEM_COMPRESS LZO1X_1_MEM_COMPRESS +#define LZO1X_1_MEM_COMPRESS ((lzo_uint32) (16384L * lzo_sizeof_dict_t)) +#define LZO1X_MEM_DECOMPRESS (0) + + +/* compression */ +LZO_EXTERN(int) +lzo1x_1_compress ( const lzo_byte *src, lzo_uint src_len, + lzo_byte *dst, lzo_uint *dst_len, + lzo_voidp wrkmem ); + +/* decompression */ +LZO_EXTERN(int) +lzo1x_decompress ( const lzo_byte *src, lzo_uint src_len, + lzo_byte *dst, lzo_uint *dst_len, + lzo_voidp wrkmem /* NOT USED */ ); + +/* safe decompression with overrun testing */ +LZO_EXTERN(int) +lzo1x_decompress_safe ( const lzo_byte *src, lzo_uint src_len, + lzo_byte *dst, lzo_uint *dst_len, + lzo_voidp wrkmem /* NOT USED */ ); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* already included */ + diff -urN linux-2.4.26-rc1/include/linux/mm.h linux-2.4.26-rc1-ccache/include/linux/mm.h --- linux-2.4.26-rc1/include/linux/mm.h Sun Mar 28 13:50:54 2004 +++ linux-2.4.26-rc1-ccache/include/linux/mm.h Sun Mar 28 17:30:27 2004 @@ -300,6 +300,8 @@ #define PG_reserved 14 #define PG_launder 15 /* written out by VM pressure.. */ #define PG_fs_1 16 /* Filesystem specific */ +#define PG_comp_cache 17 /* page with a fragment in compressed cache */ +#define PG_compressed 18 /* swapped in page with compressed data */ #ifndef arch_set_page_uptodate #define arch_set_page_uptodate(page) @@ -383,6 +385,9 @@ #endif /* CONFIG_HIGHMEM || WANT_PAGE_VIRTUAL */ extern void FASTCALL(set_page_dirty(struct page *)); +#ifdef CONFIG_COMP_CACHE +extern void FASTCALL(__set_page_dirty(struct page *)); +#endif /* * The first mb is necessary to safely close the critical section opened by the @@ -401,6 +406,32 @@ #define PageSetSlab(page) set_bit(PG_slab, &(page)->flags) #define PageClearSlab(page) clear_bit(PG_slab, &(page)->flags) #define PageReserved(page) test_bit(PG_reserved, &(page)->flags) + +#ifdef CONFIG_COMP_CACHE +#define PageCompCache(page) test_bit(PG_comp_cache, &(page)->flags) +#define PageCompressed(page) test_bit(PG_compressed, &(page)->flags) +#define SetPageCompCache(page) set_bit(PG_comp_cache, &(page)->flags) +#define SetPageCompressed(page) set_bit(PG_compressed, &(page)->flags) +#define ClearPageCompCache(page) clear_bit(PG_comp_cache, &(page)->flags) +#define ClearPageCompressed(page) clear_bit(PG_compressed, &(page)->flags) +#else +#define PageCompCache(page) 0 +#define PageCompressed(page) 0 +#define SetPageCompCache(page) do { } while (0) +#define SetPageCompressed(page) do { } while (0) +#define ClearPageCompCache(page) do { } while (0) +#define ClearPageCompressed(page) do { } while (0) +#endif + +#define PageSetCompCache(page) set_bit(PG_comp_cache, &(page)->flags) +#define PageClearCompCache(page) clear_bit(PG_comp_cache, &(page)->flags) +#define PageTestandSetCompCache(page) test_and_set_bit(PG_comp_cache, &(page)->flags) +#define PageTestandClearCompCache(page) test_and_clear_bit(PG_comp_cache, &(page)->flags) + +#define PageSetCompressed(page) set_bit(PG_compressed, &(page)->flags) +#define PageClearCompressed(page) clear_bit(PG_compressed, &(page)->flags) +#define PageTestandSetCompressed(page) test_and_set_bit(PG_compressed, &(page)->flags) +#define PageTestandClearCompressed(page) test_and_clear_bit(PG_compressed, &(page)->flags) #define PageActive(page) test_bit(PG_active, &(page)->flags) #define SetPageActive(page) set_bit(PG_active, &(page)->flags) diff -urN linux-2.4.26-rc1/include/linux/swap.h linux-2.4.26-rc1-ccache/include/linux/swap.h --- linux-2.4.26-rc1/include/linux/swap.h Sun Mar 28 13:50:54 2004 +++ linux-2.4.26-rc1-ccache/include/linux/swap.h Sun Mar 28 17:12:21 2004 @@ -8,7 +8,17 @@ #define SWAP_FLAG_PRIO_MASK 0x7fff #define SWAP_FLAG_PRIO_SHIFT 0 +#ifdef CONFIG_COMP_CACHE + +/* Some architectures may deal with SWP_ENTRY differently, such as + * UML. Using MAX_SWAPFILES for COMP_CACHE_SWP_TYPE may cause + * problems, so let's decrease the maximum number in order to use + * safely the last swap file type (in this case 31) */ + +#define MAX_SWAPFILES 31 +#else #define MAX_SWAPFILES 32 +#endif /* * Magic header for a swap area. The first part of the union is @@ -55,8 +65,24 @@ #define SWAP_CLUSTER_MAX 32 -#define SWAP_MAP_MAX 0x7fff -#define SWAP_MAP_BAD 0x8000 +#ifdef CONFIG_COMP_CACHE +#define SWAP_MAP_MAX 0x3fff +#define SWAP_MAP_BAD 0x4000 +#define SWAP_MAP_COMP_BIT 0x8000 +#define SWAP_MAP_COMP_BIT_MASK 0x7fff +#define swap_map_count(swap) (swap & 0x7fff) +#else +#define SWAP_MAP_MAX 0x7fff +#define SWAP_MAP_BAD 0x8000 +#define SWAP_MAP_COMP_BIT 0x0000 +#define SWAP_MAP_COMP_BIT_MASK 0x0000 +#define swap_map_count(swap) (swap) +#endif + +#ifdef CONFIG_COMP_SWAP +#define COMP_SWAP_MAP_MAX 0x7fff +#define COMP_SWAP_MAP_BAD 0x8000 +#endif /* * The in-memory structure used to track swap areas. @@ -72,6 +98,14 @@ unsigned int highest_bit; unsigned int cluster_next; unsigned int cluster_nr; +#ifdef CONFIG_COMP_SWAP + unsigned long * real_swap; + unsigned short * real_swap_map; + unsigned int real_lowest_bit; + unsigned int real_highest_bit; + unsigned int real_cluster_next; + unsigned int real_cluster_nr; +#endif int prio; /* swap priority */ int pages; unsigned long max; @@ -134,7 +168,9 @@ extern void delete_from_swap_cache(struct page *page); extern void free_page_and_swap_cache(struct page *page); extern struct page * lookup_swap_cache(swp_entry_t); -extern struct page * read_swap_cache_async(swp_entry_t); +extern struct page * __read_swap_cache_async(swp_entry_t, int); +#define read_swap_cache_async(entry) __read_swap_cache_async(entry, 0) +#define read_swap_cache_async_ahead(entry) __read_swap_cache_async(entry, 1) /* linux/mm/oom_kill.c */ extern void out_of_memory(void); @@ -159,6 +195,20 @@ extern struct swap_list_t swap_list; asmlinkage long sys_swapoff(const char *); asmlinkage long sys_swapon(const char *, int); + +#ifdef CONFIG_COMP_SWAP +swp_entry_t get_real_swap_page(swp_entry_t); +swp_entry_t get_map(swp_entry_t); + +void map_swap(swp_entry_t, swp_entry_t); +#endif + +#ifdef CONFIG_COMP_CACHE +void set_swap_compressed(swp_entry_t, int); +int get_swap_compressed(swp_entry_t); +#else +static inline int get_swap_compressed(swp_entry_t entry) { return 0; } +#endif extern spinlock_cacheline_t pagemap_lru_lock_cacheline; #define pagemap_lru_lock pagemap_lru_lock_cacheline.lock diff -urN linux-2.4.26-rc1/mm/Makefile linux-2.4.26-rc1-ccache/mm/Makefile --- linux-2.4.26-rc1/mm/Makefile Sun Aug 4 12:05:14 2002 +++ linux-2.4.26-rc1-ccache/mm/Makefile Sun Mar 28 17:12:21 2004 @@ -18,4 +18,10 @@ obj-$(CONFIG_HIGHMEM) += highmem.o +ifeq ($(CONFIG_COMP_CACHE),y) +obj-y += comp_cache/comp_cache.o +endif + +subdir-$(CONFIG_COMP_CACHE) += comp_cache + include $(TOPDIR)/Rules.make diff -urN linux-2.4.26-rc1/mm/comp_cache/Makefile linux-2.4.26-rc1-ccache/mm/comp_cache/Makefile --- linux-2.4.26-rc1/mm/comp_cache/Makefile Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/mm/comp_cache/Makefile Sun Mar 28 17:12:21 2004 @@ -0,0 +1,15 @@ +# +# Makefile for the linux compressed cache +# + +O_TARGET := comp_cache.o + +export-objs := swapin.o + +obj-y := main.o vswap.o free.o swapout.o swapin.o aux.o proc.o WK4x4.o WKdm.o minilzo.o + +ifneq ($(CONFIG_COMP_DIS_ADAPT),y) +obj-y += adaptivity.o +endif + +include $(TOPDIR)/Rules.make diff -urN linux-2.4.26-rc1/mm/comp_cache/WK4x4.c linux-2.4.26-rc1-ccache/mm/comp_cache/WK4x4.c --- linux-2.4.26-rc1/mm/comp_cache/WK4x4.c Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/mm/comp_cache/WK4x4.c Sun Mar 28 17:12:21 2004 @@ -0,0 +1,696 @@ +/* + WK4x4-v0.2 -- Wilson-Kaplan-4x4 version 0.2 + + Compresses buffers using a dictionary based match and partial match + scheme. + + Scott F. Kaplan -- sfkaplan@cs.utexas.edu + September 1997 */ + +/* ============================================================ */ +/* Included files */ + +#include +#include +#include + +/* Some output macros for debugging. These macros are pre-processed + into nothing if the DEBUG_PAGE_COMPRESS symbol is not defined. */ +#if defined DEBUG_WK +#define DEBUG_PRINT_1(string) printf (string) +#define DEBUG_PRINT_2(string,value) printf(string, value) +#else +#define DEBUG_PRINT_1(string) +#define DEBUG_PRINT_2(string, value) +#endif + +/* ============================================================ */ +/* Macros for WK4x4_compress and WK4x4_decompress only */ + +/* Set up the dictionary before performing compression or + decompression. Each element is loaded with some value, the + high-bits version of that value, and a next pointer. */ +#define PRELOAD_DICTIONARY { \ + dictionary[0] = 1; \ + dictionary[1] = 1; \ + dictionary[2] = 1; \ + dictionary[3] = 1; \ + dictionary[4] = 1; \ + dictionary[5] = 1; \ + dictionary[6] = 1; \ + dictionary[7] = 1; \ + dictionary[8] = 1; \ + dictionary[9] = 1; \ + dictionary[10] = 1; \ + dictionary[11] = 1; \ + dictionary[12] = 1; \ + dictionary[13] = 1; \ + dictionary[14] = 1; \ + dictionary[15] = 1; \ +} + +/* In the set the starts at index initialIndex, copy the zeroth item + into the first slot, leaving the zeroth slot open. */ +#define SLIDE_TOP_ONE_DOWN \ + dictionary[initialIndex + 1] = \ + dictionary[initialIndex] + +/* In the set that starts at index initialIndex, copy the first item + into the second slot, and then the zeroth item into the first slot, + leaving the zeroth slot open. */ +#define SLIDE_TOP_TWO_DOWN { \ + dictionary[initialIndex + 2] = \ + dictionary[initialIndex + 1]; \ + SLIDE_TOP_ONE_DOWN; \ +} + +/* In the set that starts at index initialIndex, copy the second item + into the third slot, the first item into the second slot, and then + the zeroth item into the first slot, leaving the zeroth slot + open. */ +#define SLIDE_TOP_THREE_DOWN { \ + dictionary[initialIndex + 3] = \ + dictionary[initialIndex + 2]; \ + SLIDE_TOP_TWO_DOWN; \ +} + +/* ============================================================ */ +/* WK4x4_compress macros and procedure */ + +/* When a word that missed completely is encountered, it needs to be + inserted into the dictionary in favor of the LRU entry in that + set. Slide all other items in that set down, and insert this new + pattern as the first entry in that set. + Note that this macro is not the same one used to perform the same + task when decompressing, as extra information (the high bits + pattern and the set) are already available during compression, but + will have to be calculated for decompression. */ +#define INSERT_NEW_COMPRESSED_PATTERN { \ + SLIDE_TOP_THREE_DOWN; \ + dictionary[initialIndex] = sourcePattern; \ +} + +/* Given a word with its low bits set to zero, hash that word into an + index (corresponding to the zero-th entry the appropriate set. The + hash into the table is determine by bits 10 through 21. */ +#define HASH(pattern) \ + hashTable[((pattern) >> 10) & 0x000000FF] + +/* Add the bits given by bitsToEncodeExpr to the destination buffer. */ +#define ADD_TO_DESTINATION(bitsToEncodeExpr, numberOfBitsToEncode) { \ + if (numberOfBitsToEncode > remainingBits) { \ + /* The number of bits to be encoded exceeds the number of bits + available in the current encoding word. So, append as many bits + as will fit into the current word, store that word, and then + begin a new encoding word with the remaining bits. */ \ + unsigned int bitsInSecondWord = numberOfBitsToEncode - remainingBits; \ + WK_word bitsToEncode = (bitsToEncodeExpr); \ + *destinationBuffer = ((encodingWord << remainingBits) | \ + (bitsToEncode >> bitsInSecondWord)); \ + DEBUG_PRINT_2("Crossing word boundary: %8x\n", *destinationBuffer); \ + destinationBuffer++; \ + /* It's safe not to mask out the high bits already stored in the + previous encoding word, as those high bits will be shifted out + later, as this encoding word is filled. */ \ + encodingWord = bitsToEncode; \ + remainingBits = BITS_PER_WORD - bitsInSecondWord; \ + } else if (numberOfBitsToEncode < remainingBits) { \ + /* The number of bits to be encoded is less than the number of + bits remaining in the current encoding word. Simply append + all the bits to the current encoding word. */ \ + DEBUG_PRINT_1("Within word boundary\n"); \ + encodingWord <<= numberOfBitsToEncode; \ + encodingWord |= (bitsToEncodeExpr); \ + remainingBits -= numberOfBitsToEncode; \ + } else { \ + /* The number of bits to be encoded is exactly the number of + bits remaining in the current encoding word. Append the bits + to the current encoding word, store that word, and initialize + a new encoding word by resetting the remaining bits counter. */ \ + if (numberOfBitsToEncode == BITS_PER_WORD) { \ + *destinationBuffer = (bitsToEncodeExpr); \ + } else { \ + *destinationBuffer = ((encodingWord << remainingBits) | \ + (bitsToEncodeExpr)); \ + } \ + DEBUG_PRINT_2("Hitting word boundary: %8x\n", *destinationBuffer); \ + destinationBuffer++; \ + remainingBits = BITS_PER_WORD; \ + } \ +} + + +/* Create the encoding for a word that did not match any dictionary + entry and add that information to the end of the desintation buffer. */ +#define ENCODE_MISS { \ + DEBUG_PRINT_1("Encoding miss\n"); \ + ADD_TO_DESTINATION(0x2,2); \ + ADD_TO_DESTINATION(sourcePattern,BITS_PER_WORD); \ +} + +/* Create the encoding of a partial match and add that information + to the end of the destination buffer. */ +#define ENCODE_PARTIAL { \ + DEBUG_PRINT_2("Encoding partial: %1x\n", currentIndex); \ + ADD_TO_DESTINATION(0x4000 | \ + (currentIndex << 10) | \ + STRIP_HIGH_BITS(sourcePattern), \ + 16); \ +} + +/* Create the encoding of a partial match and add that information + to the end of the destination buffer. */ +#define ENCODE_EXACT { \ + DEBUG_PRINT_2("Encoding exact: %1x\n", currentIndex); \ + ADD_TO_DESTINATION(0x30 | currentIndex, 6); \ +} + +/* Create the encoding of an exact match at the first position and add + that information to the end of the destination buffer. */ +#define ENCODE_ZERO { \ + DEBUG_PRINT_1("Encoding zero\n"); \ + ADD_TO_DESTINATION(ZERO_TAG, 2); \ +} + +#define COPY_SOURCE_TO_ZEROTH_POSITION \ + dictionary[initialIndex] = sourcePattern + + +/* Attempt a partial match and, if needed, an exact match between + the given pattern and the zeroth dictionary element in the given + set. */ +#define COMPARE_ZEROTH_ELEMENT { \ + currentIndex = initialIndex; \ + if (STRIP_LOW_BITS(dictionary[currentIndex]) == \ + sourceHighBitsPattern) { \ + if (dictionary[currentIndex] == sourcePattern) { \ + ENCODE_EXACT; \ + } else { \ + ENCODE_PARTIAL; \ + dictionary[currentIndex] = sourcePattern; \ + } \ + continue; \ + } \ +} + +/* Attempt a partial match and, if needed, an exact match between + the given pattern and the first dictionary element in the given + set. Re-order the set if a match occurs. */ +#define COMPARE_FIRST_ELEMENT { \ + currentIndex++; \ + if (STRIP_LOW_BITS(dictionary[currentIndex]) == \ + sourceHighBitsPattern) { \ + if (dictionary[currentIndex] == sourcePattern) { \ + ENCODE_EXACT; \ + } else { \ + ENCODE_PARTIAL; \ + } \ + SLIDE_TOP_ONE_DOWN; \ + COPY_SOURCE_TO_ZEROTH_POSITION; \ + continue; \ + } \ +} + +/* Attempt a partial match and, if needed, an exact match between + the given pattern and the zeroth dictionary element in the given + set. Re-order the set if a match occurs. */ +#define COMPARE_SECOND_ELEMENT { \ + currentIndex++; \ + if (STRIP_LOW_BITS(dictionary[currentIndex]) == \ + sourceHighBitsPattern) { \ + if (dictionary[currentIndex] == sourcePattern) { \ + ENCODE_EXACT; \ + } else { \ + ENCODE_PARTIAL; \ + } \ + SLIDE_TOP_TWO_DOWN; \ + COPY_SOURCE_TO_ZEROTH_POSITION; \ + continue; \ + } \ +} + +/* Attempt a partial match and, if needed, an exact match between + the given pattern and the zeroth dictionary element in the given + set. Re-order the set if a match occurs. */ +#define COMPARE_THIRD_ELEMENT { \ + currentIndex++; \ + if (STRIP_LOW_BITS(dictionary[currentIndex]) == \ + sourceHighBitsPattern) { \ + if (dictionary[currentIndex] == sourcePattern) { \ + ENCODE_EXACT; \ + } else { \ + ENCODE_PARTIAL; \ + } \ + SLIDE_TOP_THREE_DOWN; \ + COPY_SOURCE_TO_ZEROTH_POSITION; \ + continue; \ + } \ +} + +/* Given pointers to source buffer (sourceBuffer) of uncompressed data + and a destination buffer (destinationBuffer) into which compressed + data can be placed, as well as the number of words in the source + buffer (words), compress the contents of the source and place the + results in the destination. Return the number of bytes placed into + the destination. */ +unsigned int +WK4x4_compress (WK_word* sourceBuffer, + WK_word* destinationBuffer, + unsigned int words, + struct comp_alg_data * data) +{ + DictionaryElement dictionary[DICTIONARY_SIZE]; + unsigned int hashTable [] = HASH_LOOKUP_TABLE_CONTENTS_WK4x4; + int wordIndex; + unsigned int remainingBits = BITS_PER_WORD; + WK_word encodingWord = 0; + WK_word* destinationBufferStartingPoint = destinationBuffer; + + /* Initialize the elements with values and set its pointers. */ + PRELOAD_DICTIONARY; + + /* Loop through each word in the source page. */ + for (wordIndex = 0; wordIndex < words; wordIndex++) { + register WK_word sourcePattern; + register WK_word sourceHighBitsPattern; + unsigned int initialIndex; + unsigned int currentIndex; + + /* Load the current pattern into a register. */ + sourcePattern = sourceBuffer[wordIndex]; + + /* If this word is all zeros, encode it as such. */ + if (sourcePattern == 0) { + ENCODE_ZERO; + continue; + } + + /* Load a partial matching pattern (i.e. the low bits have all + been set to zero) into another register. */ + sourceHighBitsPattern = STRIP_LOW_BITS(sourcePattern); + + /* Determine which set to search in the dictionary through a hash + function. Note that the hash function returns the array index + at which we begin searching, since the sets are stored linearly + in an array. */ + initialIndex = HASH(sourcePattern); + currentIndex = initialIndex; + + /* Compare the source word to each element in the set by comparing + first the high bits and then the full pattern. If a match is + made, the encoding will be performed and a "continue" statement + will cause a skip to the next word in the source page. */ + COMPARE_ZEROTH_ELEMENT; /* 0 */ + COMPARE_FIRST_ELEMENT; /* 1 */ + COMPARE_SECOND_ELEMENT; /* 2 */ + COMPARE_THIRD_ELEMENT; /* 3 */ + + /* If we've reached this point, we've missed at every element + of the dictionary. Encode the miss and update the + dictionary. */ + ENCODE_MISS; + INSERT_NEW_COMPRESSED_PATTERN; + + } + + /* If there are any bits stored in the current encoding word, shift + those bits to the top of the word and store them. */ + if (remainingBits < BITS_PER_WORD) { + *destinationBuffer = (encodingWord << remainingBits); + DEBUG_PRINT_2("Flushing last word: %8x\n", *destinationBuffer); + destinationBuffer++; + } + + /* Return the number of bytes placed into the compression buffer. */ + return ((unsigned int)(destinationBuffer - + destinationBufferStartingPoint)) * + BYTES_PER_WORD; + +} + +/* ============================================================ */ +/* WK4x4_decompress macros and procedure */ + +/* When a word that missed completely is encountered, it needs to be + inserted into the dictionary in favor of the LRU entry in that + set. Slide all other items in that set down, and insert this new + pattern as the first entry in that set. + Note that this macro is not the same one used to perform the same + task when compressing, as extra information (the high bits + pattern and the set) are already available during compression, but + will have to be calculated for decompression. */ +#define INSERT_NEW_DECOMPRESSED_PATTERN { \ + initialIndex = HASH(*destinationBuffer); \ + SLIDE_TOP_THREE_DOWN; \ + dictionary[initialIndex] = *destinationBuffer; \ +} + +/* Based on some current index and the initial index for the set in + which the current index resides, slide the elements of that set + down such that the current index's contents are clobbered, and the + zeroth slot in the set is open. */ +#define SLIDE_SOME_NUMBER_OF_ELEMENTS \ + switch (currentIndex - initialIndex) { \ + case 3: \ + dictionary[initialIndex + 3] = \ + dictionary[initialIndex + 2]; \ + case 2: \ + dictionary[initialIndex + 2] = \ + dictionary[initialIndex + 1]; \ + case 1: \ + dictionary[initialIndex + 1] = \ + dictionary[initialIndex]; \ + } \ + +/* Extract the given number of bits from the source buffer and + place those bits, by themselves, into the target variable. */ +#define EXTRACT_BITS(numberOfBitsToExtract, targetVariable) { \ + if (numberOfBitsToExtract > remainingBits) { \ + /* The set of bits to be extracted are split across the current + decoding word and the next one. Extract the bits available in + the current word and place them into their final position in + the target variable. Advance to the next decoding word in + the source buffer and extract the remaining needed bits into + the target variable. */ \ + unsigned int bitsInSecondWord = numberOfBitsToExtract - remainingBits; \ + targetVariable = (decodingWord >> \ + (BITS_PER_WORD - numberOfBitsToExtract)); \ + sourceBuffer++; \ + decodingWord = *sourceBuffer; \ + DEBUG_PRINT_2("Crossing word boundary: %8x\n", decodingWord); \ + targetVariable |= (decodingWord >> (BITS_PER_WORD - bitsInSecondWord)); \ + decodingWord <<= bitsInSecondWord; \ + remainingBits = BITS_PER_WORD - bitsInSecondWord; \ + } else if (numberOfBitsToExtract < remainingBits) { \ + /* All of the bits to be extracted are available in the current + decoding word. Extract those bits, place them into the target, + and update the current decoding word accordingly. */ \ + DEBUG_PRINT_1("Within word boundary\n"); \ + targetVariable = (decodingWord >> \ + (BITS_PER_WORD - numberOfBitsToExtract)); \ + decodingWord <<= numberOfBitsToExtract; \ + remainingBits -= numberOfBitsToExtract; \ + } else { \ + /* The number of bits requested is exactly the number of bits left in + the current decoding word. Copy those bits and advance the + current decoding word to the next word in the source buffer. */ \ + targetVariable = (decodingWord >> \ + (BITS_PER_WORD - numberOfBitsToExtract)); \ + sourceBuffer++; \ + decodingWord = *sourceBuffer; \ + DEBUG_PRINT_2("Hitting word boundary: %8x\n", decodingWord); \ + remainingBits = BITS_PER_WORD; \ + } \ +} + +/* Given a pointer to a source buffer (sourceBuffer) of compressed + data and a pointer to a destination buffer (destinationBuffer) into + which uncompressed data can be placed, as well as the number of + uncompressed words that will be written to the destination buffer + (words), decompress the data into the destination buffer. */ +void +WK4x4_decompress (WK_word* sourceBuffer, + WK_word* destinationBuffer, + unsigned int words, + struct comp_alg_data * data) +{ + /* The dictionary array is divided into sets. Each entry in the + dictionary array is really an entry in one of the dictionary's + sets. Each set begins at a particular offset within the + dictionary array. Given an index into the dictionary array (and + thus into a set), the initial index table provides the index at + which that set begins in the dictionary array. */ + + DictionaryElement dictionary[DICTIONARY_SIZE]; + unsigned int hashTable [] = HASH_LOOKUP_TABLE_CONTENTS_WK4x4; + + unsigned int initialIndexTable [] = INITIAL_INDEX_TABLE_CONTENTS; + + unsigned int wordIndex; + unsigned int remainingBits = BITS_PER_WORD; + WK_word decodingWord = *sourceBuffer; + + DEBUG_PRINT_2("\n-----\nBeginning decompression: %8x\n", decodingWord); + + /* Initialize the elements with values and set its pointers. */ + PRELOAD_DICTIONARY; + + /* Loop through each encoded word in the source buffer, and iterate + through the words of the destination buffer as decompression + occurs. */ + for (wordIndex = 0; + wordIndex < words; + wordIndex++, destinationBuffer++) { + unsigned int tag; + unsigned int currentIndex; + unsigned int initialIndex; + register WK_word lowBitsWord; + register WK_word highBitsWord; + + /* Extract the tag from the current word. */ + EXTRACT_BITS(2, tag); + + /* Based on that tag, determine how to decode the rest of the + word. */ + switch (tag) { + + case ZERO_TAG: + /* Append a zero word to the destination. */ + *destinationBuffer = 0; + break; + + case MISS_TAG: + DEBUG_PRINT_1("Decoding miss\n"); + /* Extract the whole word and append it to the destination + page. */ + EXTRACT_BITS(BITS_PER_WORD, *destinationBuffer); + /* Update the dictionary by inserting this pattern in place of + the oldest pattern in the LRU queue, and making that position + the new head. */ + INSERT_NEW_DECOMPRESSED_PATTERN; + break; + + case PARTIAL_TAG: + /* Extract the index of the dictionary entry that this word + matched. */ + EXTRACT_BITS(4, currentIndex); + DEBUG_PRINT_2("Decoding partial: %1x\n", currentIndex); + /* Extract the low bits. */ + EXTRACT_BITS(10, lowBitsWord); + /* Grab the high bits pattern from the proper dictionary + entry. */ + highBitsWord = + STRIP_LOW_BITS(dictionary[currentIndex]); + /* Append to the destination page the combination of the high + and low bits words. */ + *destinationBuffer = highBitsWord | lowBitsWord; + /* Update the dictionary by moving the accessed entry to the + head of its set. Note that we also update the full pattern + with the word that was just reconstructed. */ + initialIndex = initialIndexTable[currentIndex]; + SLIDE_SOME_NUMBER_OF_ELEMENTS; + dictionary[initialIndex] = *destinationBuffer; + break; + + case EXACT_TAG: + /* Extract the index of the dictionary entry that this word + matched. */ + EXTRACT_BITS(4, currentIndex); + DEBUG_PRINT_2("Decoding exact: %1x\n", currentIndex); + /* Grab the pattern from the given entry in the dictionary + and append it to the destination page. */ + *destinationBuffer = dictionary[currentIndex]; + /* Update the dictionary by moving the accessed entry to the + head of its set. */ + initialIndex = initialIndexTable[currentIndex]; + SLIDE_SOME_NUMBER_OF_ELEMENTS; + dictionary[initialIndex] = *destinationBuffer; + break; + + default: + DEBUG_PRINT_2("***Bad tag value: %1x\n", tag); + //exit (-1); + } + } +} + +/* ============================================================ */ +/* WK4x4_measureCompress macros and procedure */ + +/* Add to the exact or partial count for this queue position. */ +#define COUNT_EXACT { \ + exactMatchCountArray[currentIndex]++; \ + bitsUsedForEncoding += 6; \ +} + +#define COUNT_ZERO bitsUsedForEncoding += 2 + +#define COUNT_PARTIAL { \ + partialMatchCountArray[currentIndex]++; \ + bitsUsedForEncoding += 16; \ +} + +#define COUNT_MISS bitsUsedForEncoding += 34 + +/* Attempt a partial match and, if needed, an exact match between + the given pattern and the zeroth dictionary element in the given + set. */ +#define COMPARE_ZEROTH_ELEMENT_FOR_MEASUREMENT { \ + currentIndex = initialIndex; \ + if (STRIP_LOW_BITS(dictionary[currentIndex]) == \ + sourceHighBitsPattern) { \ + if (dictionary[currentIndex] == sourcePattern) { \ + COUNT_EXACT; \ + } else { \ + COUNT_PARTIAL; \ + dictionary[currentIndex] = sourcePattern; \ + } \ + continue; \ + } \ +} + +/* Attempt a partial match and, if needed, an exact match between + the given pattern and the first dictionary element in the given + set. Re-order the set if a match occurs. */ +#define COMPARE_FIRST_ELEMENT_FOR_MEASUREMENT { \ + currentIndex++; \ + if (STRIP_LOW_BITS(dictionary[currentIndex]) == \ + sourceHighBitsPattern) { \ + if (dictionary[currentIndex] == sourcePattern) { \ + COUNT_EXACT; \ + } else { \ + COUNT_PARTIAL; \ + } \ + SLIDE_TOP_ONE_DOWN; \ + COPY_SOURCE_TO_ZEROTH_POSITION; \ + continue; \ + } \ +} + +/* Attempt a partial match and, if needed, an exact match between + the given pattern and the zeroth dictionary element in the given + set. Re-order the set if a match occurs. */ +#define COMPARE_SECOND_ELEMENT_FOR_MEASUREMENT { \ + currentIndex++; \ + if (STRIP_LOW_BITS(dictionary[currentIndex]) == \ + sourceHighBitsPattern) { \ + if (dictionary[currentIndex] == sourcePattern) { \ + COUNT_EXACT; \ + } else { \ + COUNT_PARTIAL; \ + } \ + SLIDE_TOP_TWO_DOWN; \ + COPY_SOURCE_TO_ZEROTH_POSITION; \ + continue; \ + } \ +} + +/* Attempt a partial match and, if needed, an exact match between + the given pattern and the zeroth dictionary element in the given + set. Re-order the set if a match occurs. */ +#define COMPARE_THIRD_ELEMENT_FOR_MEASUREMENT { \ + currentIndex++; \ + if (STRIP_LOW_BITS(dictionary[currentIndex]) == \ + sourceHighBitsPattern) { \ + if (dictionary[currentIndex] == sourcePattern) { \ + COUNT_EXACT; \ + } else { \ + COUNT_PARTIAL; \ + } \ + SLIDE_TOP_THREE_DOWN; \ + COPY_SOURCE_TO_ZEROTH_POSITION; \ + continue; \ + } \ +} + +/* Given a pointer to a source buffer (sourceBuffer) of uncompressed + data, the number of words stored in that buffer (words), and two + arrays for counting the number of exact (exactMatchCountArray) and + partial (partialMatchCountArray) matches at each LRU position, + compress the source and record what types of hits (partial or + exact) occurred at each queue position. Return the number of words + that would be placed into a destination buffer (if there were + one). */ +unsigned int +WK4x4_measureCompress (WK_word* sourceBuffer, + unsigned int words, + unsigned int* exactMatchCountArray, + unsigned int* partialMatchCountArray) { + DictionaryElement dictionary[DICTIONARY_SIZE]; + unsigned int hashTable [] = HASH_LOOKUP_TABLE_CONTENTS_WK4x4; + unsigned int wordIndex; + unsigned int index; + unsigned int bitsUsedForEncoding = 0; + + /* Initialize the elements with values and set its pointers. */ + PRELOAD_DICTIONARY; + + /* Loop through the match information arrays and clear their + values. */ + for (index = 0; index < DICTIONARY_SIZE; index++) { + exactMatchCountArray[index] = 0; + partialMatchCountArray[index] = 0; + } + + /* Loop through each word in the source page. */ + for (wordIndex = 0; wordIndex < words; wordIndex++) { + register WK_word sourcePattern; + register WK_word sourceHighBitsPattern; + unsigned int initialIndex; + unsigned int currentIndex; + + /* Load the current pattern into a register. */ + sourcePattern = sourceBuffer[wordIndex]; + + /* If this word is all zeros, encode it as such. */ + if (sourcePattern == 0) { + COUNT_ZERO; + continue; + } + + /* Load a partial matching pattern (i.e. the low bits have all + been set to zero) into another register. */ + sourceHighBitsPattern = STRIP_LOW_BITS(sourcePattern); + + /* Determine which set to search in the dictionary through a hash + function. Note that the hash function returns the array index + at which we begin searching, since the sets are stored linearly + in an array. */ + initialIndex = HASH(sourcePattern); + currentIndex = initialIndex; + + /* Compare the source word to each element in the set by comparing + first the high bits and then the full pattern. If a match is + made, the encoding will be performed and a "continue" statement + will cause a skip to the next word in the source page. */ + COMPARE_ZEROTH_ELEMENT_FOR_MEASUREMENT; /* 0 */ + COMPARE_FIRST_ELEMENT_FOR_MEASUREMENT; /* 1 */ + COMPARE_SECOND_ELEMENT_FOR_MEASUREMENT; /* 2 */ + COMPARE_THIRD_ELEMENT_FOR_MEASUREMENT; /* 3 */ + + /* If we've reached this point, we've missed at every element + of the dictionary. Encode the miss and update the + dictionary. */ + COUNT_MISS; + INSERT_NEW_COMPRESSED_PATTERN; + + } + + /* Return the number of bytes used to store the compressed + region. */ + return ((unsigned int)ceil((double)bitsUsedForEncoding / + (double)BITS_PER_WORD)) * + BYTES_PER_WORD; + +} + +/* + * Overrides for Emacs so that we follow Linus's tabbing style. + * Emacs will notice this stuff at the end of the file and automatically + * adjust the settings for this buffer only. This must remain at the end + * of the file. + * --------------------------------------------------------------------------- + * Local variables: + * c-file-style: "linux" + * End: + */ diff -urN linux-2.4.26-rc1/mm/comp_cache/WKdm.c linux-2.4.26-rc1-ccache/mm/comp_cache/WKdm.c --- linux-2.4.26-rc1/mm/comp_cache/WKdm.c Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/mm/comp_cache/WKdm.c Sun Mar 28 17:12:21 2004 @@ -0,0 +1,876 @@ +/* direct-mapped partial matching compressor with simple 22/10 split + * + * Compresses buffers using a dictionary based match and partial match + * (high bits only or full match) scheme. + * + * Paul Wilson -- wilson@cs.utexas.edu + * Scott F. Kaplan -- sfkaplan@cs.utexas.edu + * September 1997 + */ + +/* compressed output format, in memory order + * 1. a four-word HEADER containing four one-word values: + * i. a one-word code saying what algorithm compressed the data + * ii. an integer WORD offset into the page saying + * where the queue position area starts + * iii. an integer WORD offset into the page saying where + * the low-bits area starts + * iv. an integer WORD offset into the page saying where the + * low-bits area ends + * + * 2. a 64-word TAGS AREA holding one two-bit tag for each word in + * the original (1024-word) page, packed 16 per word + * + * 3. a variable-sized FULL WORDS AREA (always word aligned and an + * integral number of words) holding full-word patterns that + * were not in the dictionary when encoded (i.e., dictionary misses) + * + * 4. a variable-sized QUEUE POSITIONS AREA (always word aligned and + * an integral number of words) holding four-bit queue positions, + * packed eight per word. + * + * 5. a variable-sized LOW BITS AREA (always word aligned and an + * integral number of words) holding ten-bit low-bit patterns + * (from partial matches), packed three per word. + */ + + + +/* ============================================================ */ +/* Included files */ + +#include +#include +#include +#include + +/* at the moment we have dependencies on the page size. That should + * be changed to work for any power-of-two size that's at least 16 + * words, or something like that + */ + +#define PAGE_SIZE_IN_WORDS 1024 +#define PAGE_SIZE_IN_BYTES 4096 + +#define DICTIONARY_SIZE 16 + +/* + * macros defining the basic layout of stuff in a page + */ +#define HEADER_SIZE_IN_WORDS 4 +#define TAGS_AREA_OFFSET 4 +#define TAGS_AREA_SIZE 64 + +/* the next few are used during compression to write the header */ +#define SET_QPOS_AREA_START(compr_dest_buf,qpos_start_addr) \ + (compr_dest_buf[1] = qpos_start_addr - compr_dest_buf) +#define SET_LOW_BITS_AREA_START(compr_dest_buf,lb_start_addr) \ + (compr_dest_buf[2] = lb_start_addr - compr_dest_buf) +#define SET_LOW_BITS_AREA_END(compr_dest_buf,lb_end_addr) \ + (compr_dest_buf[3] = lb_end_addr - compr_dest_buf) + +/* the next few are only use during decompression to read the header */ +#define TAGS_AREA_START(decomp_src_buf) \ + (decomp_src_buf + TAGS_AREA_OFFSET) +#define TAGS_AREA_END(decomp_src_buf) \ + (TAGS_AREA_START(decomp_src_buf) + TAGS_AREA_SIZE) +#define FULL_WORD_AREA_START(the_buf) TAGS_AREA_END(the_buf) +#define QPOS_AREA_START(decomp_src_buf) \ + (decomp_src_buf + decomp_src_buf[1]) +#define LOW_BITS_AREA_START(decomp_src_buf) \ + (decomp_src_buf + (decomp_src_buf[2])) +#define QPOS_AREA_END(the_buf) LOW_BITS_AREA_START(the_buf) +#define LOW_BITS_AREA_END(decomp_src_buf) \ + (decomp_src_buf + (decomp_src_buf[3])) + +/* ============================================================ */ +/* Types and structures */ + +/* A structure to store each element of the dictionary. */ +/* typedef WK_word DictionaryElement; */ + +/* ============================================================ */ +/* Misc constants */ + +#define BITS_PER_WORD 32 +#define BYTES_PER_WORD 4 +#define NUM_LOW_BITS 10 +#define LOW_BITS_MASK 0x3FF +#define ALL_ONES_MASK 0xFFFFFFFF + +#define TWO_BITS_PACKING_MASK 0x03030303 +#define FOUR_BITS_PACKING_MASK 0x0F0F0F0F +#define TEN_LOW_BITS_MASK 0x000003FF +#define TWENTY_TWO_HIGH_BITS_MASK 0xFFFFFC00 + +/* Tag values. NOTE THAT CODE MAY DEPEND ON THE NUMBERS USED. + * Check for conditionals doing arithmetic on these things + * before changing them + */ +#define ZERO_TAG 0x0 +#define PARTIAL_TAG 0x1 +#define MISS_TAG 0x2 +#define EXACT_TAG 0x3 + +#define BITS_PER_BYTE 8 + +/* ============================================================ */ +/* Global macros */ + +/* Shift out the low bits of a pattern to give the high bits pattern. + The stripped patterns are used for initial tests of partial + matches. */ +#define HIGH_BITS(word_pattern) (word_pattern >> NUM_LOW_BITS) + +/* String the high bits of a pattern so the low order bits can + be included in an encoding of a partial match. */ +#define LOW_BITS(word_pattern) (word_pattern & LOW_BITS_MASK) + +#if defined DEBUG_WK +#define DEBUG_PRINT_1(string) printf (string) +#define DEBUG_PRINT_2(string,value) printf(string, value) +#else +#define DEBUG_PRINT_1(string) +#define DEBUG_PRINT_2(string, value) +#endif + +/* Set up the dictionary before performing compression or + decompression. Each element is loaded with some value, the + high-bits version of that value, and a next pointer. */ +#define PRELOAD_DICTIONARY { \ + dictionary[0] = 1; \ + dictionary[1] = 1; \ + dictionary[2] = 1; \ + dictionary[3] = 1; \ + dictionary[4] = 1; \ + dictionary[5] = 1; \ + dictionary[6] = 1; \ + dictionary[7] = 1; \ + dictionary[8] = 1; \ + dictionary[9] = 1; \ + dictionary[10] = 1; \ + dictionary[11] = 1; \ + dictionary[12] = 1; \ + dictionary[13] = 1; \ + dictionary[14] = 1; \ + dictionary[15] = 1; \ +} + +#define HASH_TO_DICT_BYTE_OFFSET(pattern) \ + (hashLookupTable[((pattern) >> 10) & 0xFF]) + +/* EMIT... macros emit bytes or words into the intermediate arrays + */ + +#define EMIT_BYTE(fill_ptr, byte_value) {*fill_ptr = byte_value; fill_ptr++;} +#define EMIT_WORD(fill_ptr,word_value) {*fill_ptr = word_value; fill_ptr++;} + +/* RECORD... macros record the results of modeling in the intermediate + * arrays + */ + +#define RECORD_ZERO { EMIT_BYTE(next_tag,ZERO_TAG); } + +#define RECORD_EXACT(queue_posn) EMIT_BYTE(next_tag,EXACT_TAG); \ + EMIT_BYTE(next_qp,(queue_posn)); + +#define RECORD_PARTIAL(queue_posn,low_bits_pattern) { \ + EMIT_BYTE(next_tag,PARTIAL_TAG); \ + EMIT_BYTE(next_qp,(queue_posn)); \ + EMIT_WORD(next_low_bits,(low_bits_pattern)) } + +#define RECORD_MISS(word_pattern) EMIT_BYTE(next_tag,MISS_TAG); \ + EMIT_WORD(next_full_patt,(word_pattern)); + +/*********************************************************************** + * THE PACKING ROUTINES + */ + +/* WK_pack_2bits() + * Pack some multiple of four words holding two-bit tags (in the low + * two bits of each byte) into an integral number of words, i.e., + * one fourth as many. + * NOTE: Pad the input out with zeroes to a multiple of four words! + */ +WK_word* +WK_pack_2bits(WK_word* source_buf, + WK_word* source_end, + WK_word* dest_buf) { + register WK_word* src_next = source_buf; + WK_word* dest_next = dest_buf; + + while (src_next < source_end) { + register WK_word temp = src_next[0]; + temp |= (src_next[1] << 2); + temp |= (src_next[2] << 4); + temp |= (src_next[3] << 6); + + dest_next[0] = temp; + dest_next++; + src_next += 4; + } + + return dest_next; + +} + +/* WK_pack_4bits() + * Pack an even number of words holding 4-bit patterns in the low bits + * of each byte into half as many words. + * note: pad out the input with zeroes to an even number of words! + */ + +WK_word* +WK_pack_4bits(WK_word* source_buf, + WK_word* source_end, + WK_word* dest_buf) { + register WK_word* src_next = source_buf; + WK_word* dest_next = dest_buf; + + /* this loop should probably be unrolled */ + while (src_next < source_end) { + register WK_word temp = src_next[0]; + temp |= (src_next[1] << 4); + + dest_next[0] = temp; + dest_next++; + src_next += 2; + } + + return dest_next; + +} + +/* pack_3_tenbits() + * Pack a sequence of three ten bit items into one word. + * note: pad out the input with zeroes to an even number of words! + */ +WK_word* +WK_pack_3_tenbits(WK_word* source_buf, + WK_word* source_end, + WK_word* dest_buf) { + + register WK_word* src_next = source_buf; + WK_word* dest_next = dest_buf; + + /* this loop should probably be unrolled */ + while (src_next < source_end) { + register WK_word temp = src_next[0]; + temp |= (src_next[1] << 10); + temp |= (src_next[2] << 20); + + dest_next[0] = temp; + dest_next++; + src_next += 3; + } + + return dest_next; + +} + +/*************************************************************************** + * THE UNPACKING ROUTINES should GO HERE + */ + + +#define GET_NEXT_TAG tags[tagsIndex++] +#define GET_NEXT_FULL_PATTERN fullPatterns[fullPatternsIndex++] +#define GET_NEXT_LOW_BITS lowBits[lowBitsIndex++] +#define GET_NEXT_DICTIONARY_INDEX dictionaryIndices[dictionaryIndicesIndex++] + +/* WK_unpack_2bits takes any number of words containing 16 two-bit values + * and unpacks them into four times as many words containg those + * two bit values as bytes (with the low two bits of each byte holding + * the actual value. + */ +WK_word* +WK_unpack_2bits(WK_word *input_buf, + WK_word *input_end, + WK_word *output_buf) { + + register WK_word *input_next = input_buf; + register WK_word *output_next = output_buf; + register WK_word packing_mask = TWO_BITS_PACKING_MASK; + + /* loop to repeatedly grab one input word and unpack it into + * 4 output words. This loop could be unrolled a little---it's + * designed to be easy to do that. + */ + while (input_next < input_end) { + register WK_word temp = input_next[0]; + DEBUG_PRINT_2("Unpacked tags word: %.8x\n", temp); + output_next[0] = temp & packing_mask; + output_next[1] = (temp >> 2) & packing_mask; + output_next[2] = (temp >> 4) & packing_mask; + output_next[3] = (temp >> 6) & packing_mask; + + output_next += 4; + input_next++; + } + + return output_next; + +} + +/* unpack four bits consumes any number of words (between input_buf + * and input_end) holding 8 4-bit values per word, and unpacks them + * into twice as many words, with each value in a separate byte. + * (The four-bit values occupy the low halves of the bytes in the + * result). + */ +WK_word* +WK_unpack_4bits(WK_word *input_buf, + WK_word *input_end, + WK_word *output_buf) { + + register WK_word *input_next = input_buf; + register WK_word *output_next = output_buf; + register WK_word packing_mask = FOUR_BITS_PACKING_MASK; + + + /* loop to repeatedly grab one input word and unpack it into + * 4 output words. This loop should probably be unrolled + * a little---it's designed to be easy to do that. + */ + while (input_next < input_end) { + register WK_word temp = input_next[0]; + DEBUG_PRINT_2("Unpacked dictionary indices word: %.8x\n", temp); + output_next[0] = temp & packing_mask; + output_next[1] = (temp >> 4) & packing_mask; + + output_next += 2; + input_next++; + } + return output_next; + +} + +/* unpack_3_tenbits unpacks three 10-bit items from (the low 30 bits of) + * a 32-bit word + */ +WK_word* +WK_unpack_3_tenbits(WK_word *input_buf, + WK_word *input_end, + WK_word *output_buf) { + + register WK_word *input_next = input_buf; + register WK_word *output_next = output_buf; +/* register WK_word packing_mask = LOW_BITS_MASK; */ + + /* loop to fetch words of input, splitting each into three + * words of output with 10 meaningful low bits. This loop + * probably ought to be unrolled and maybe coiled + */ + while (input_next < input_end) { + register WK_word temp = input_next[0]; + + output_next[0] = temp & LOW_BITS_MASK; + output_next[1] = (temp >> 10) & LOW_BITS_MASK; + output_next[2] = temp >> 20; + + input_next++; + output_next += 3; + } + + return output_next; + +} +/*************************************************************************** + * WKdm_compress()---THE COMPRESSOR + */ + +unsigned int +WKdm_compress (WK_word* src_buf, + WK_word* dest_buf, + unsigned int num_input_words, + struct comp_alg_data * data) +{ + DictionaryElement dictionary[DICTIONARY_SIZE]; + char hashLookupTable [] = HASH_LOOKUP_TABLE_CONTENTS_WKDM; + + /* arrays that hold output data in intermediate form during modeling */ + /* and whose contents are packed into the actual output after modeling */ + + /* sizes of these arrays should be increased if you want to compress + * pages larger than 4KB + */ + WK_word * tempTagsArray = data->tempTagsArray; + WK_word * tempQPosArray = data->tempQPosArray; + WK_word * tempLowBitsArray = data->tempLowBitsArray; + + /* boundary_tmp will be used for keeping track of what's where in + * the compressed page during packing + */ + WK_word* boundary_tmp; + + /* Fill pointers for filling intermediate arrays (of queue positions + * and low bits) during encoding. + * Full words go straight to the destination buffer area reserved + * for them. (Right after where the tags go.) + */ + WK_word* next_full_patt; + char* next_tag = (char *) tempTagsArray; + char* next_qp = (char *) tempQPosArray; + WK_word* next_low_bits = tempLowBitsArray; + + WK_word* next_input_word = src_buf; + WK_word* end_of_input = src_buf + num_input_words; + + PRELOAD_DICTIONARY; + + next_full_patt = dest_buf + TAGS_AREA_OFFSET + (num_input_words / 16); + +#ifdef WK_DEBUG + printf("\nIn WKdm_compress\n"); + printf("About to actually compress, src_buf is %u\n", src_buf); + printf("dictionary is at %u\n", dictionary); + printf("dest_buf is %u next_full_patt is %u\n", dest_buf, next_full_patt); + fflush(stdout); +#endif + + while (next_input_word < end_of_input) + { + WK_word *dict_location; + WK_word dict_word; + WK_word input_word = *next_input_word; + + /* compute hash value, which is a byte offset into the dictionary, + * and add it to the base address of the dictionary. Cast back and + * forth to/from char * so no shifts are needed + */ + dict_location = + (WK_word *) + (((char*) dictionary) + HASH_TO_DICT_BYTE_OFFSET(input_word)); + + dict_word = *dict_location; + + if (input_word == dict_word) + { + RECORD_EXACT(dict_location - dictionary); + } + else if (input_word == 0) { + RECORD_ZERO; + } + else + { + WK_word input_high_bits = HIGH_BITS(input_word); + if (input_high_bits == HIGH_BITS(dict_word)) { + RECORD_PARTIAL(dict_location - dictionary, LOW_BITS(input_word)); + *dict_location = input_word; + } + else { + RECORD_MISS(input_word); + *dict_location = input_word; + } + } + next_input_word++; + } + +#ifdef WK_DEBUG + printf("AFTER MODELING in WKdm_compress()\n"); fflush(stdout); + printf("tempTagsArray holds %u bytes\n", + next_tag - (char *) tempTagsArray); + printf("tempQPosArray holds %u bytes\n", + next_qp - (char *) tempQPosArray); + printf("tempLowBitsArray holds %u bytes\n", + (char *) next_low_bits - (char *) tempLowBitsArray); + + printf("next_full_patt is %u\n", + (unsigned long) next_full_patt); + + printf(" i.e., there are %u full patterns\n", + next_full_patt - (dest_buf + TAGS_AREA_OFFSET + (num_input_words / 16))); + fflush(stdout); + + { int i; + WK_word *arr =(dest_buf + TAGS_AREA_OFFSET + (num_input_words / 16)); + + printf(" first 20 full patterns are: \n"); + for (i = 0; i < 20; i++) { + printf(" %d", arr[i]); + } + printf("\n"); + } +#endif + + /* Record (into the header) where we stopped writing full words, + * which is where we will pack the queue positions. (Recall + * that we wrote the full words directly into the dest buffer + * during modeling. + */ + + SET_QPOS_AREA_START(dest_buf,next_full_patt); + + /* Pack the tags into the tags area, between the page header + * and the full words area. We don't pad for the packer + * because we assume that the page size is a multiple of 16. + */ + +#ifdef WK_DEBUG + printf("about to pack %u bytes holding tags\n", + next_tag - (char *) tempTagsArray); + + { int i; + char* arr = (char *) tempTagsArray; + + printf(" first 200 tags are: \n"); + for (i = 0; i < 200; i++) { + printf(" %d", arr[i]); + } + printf("\n"); + } +#endif + + boundary_tmp = WK_pack_2bits(tempTagsArray, + (WK_word *) next_tag, + dest_buf + HEADER_SIZE_IN_WORDS); + +#ifdef WK_DEBUG + printf("packing tags stopped at %u\n", boundary_tmp); +#endif + + /* Pack the queue positions into the area just after + * the full words. We have to round up the source + * region to a multiple of two words. + */ + + { + unsigned int num_bytes_to_pack = next_qp - (char *) tempQPosArray; + unsigned int num_packed_words = ceil((double) num_bytes_to_pack / 8); + unsigned int num_source_words = num_packed_words * 2; + WK_word* endQPosArray = tempQPosArray + num_source_words; + + /* Pad out the array with zeros to avoid corrupting real packed + values. */ + for (; /* next_qp is already set as desired */ + next_qp < (char*)endQPosArray; + next_qp++) { + *next_qp = 0; + } + +#ifdef WK_DEBUG + printf("about to pack %u (bytes holding) queue posns.\n", + num_bytes_to_pack); + printf("packing them from %u words into %u words\n", + num_source_words, num_packed_words); + printf("dest is range %u to %u\n", + next_full_patt, next_full_patt + num_packed_words); + { int i; + char *arr = (char *) tempQPosArray; + printf(" first 200 queue positions are: \n"); + for (i = 0; i < 200; i++) { + printf(" %d", arr[i]); + } + printf("\n"); + } +#endif + + boundary_tmp = WK_pack_4bits(tempQPosArray, + endQPosArray, + next_full_patt); +#ifdef WK_DEBUG + printf("Packing of queue positions stopped at %u\n", boundary_tmp); +#endif + + /* Record (into the header) where we stopped packing queue positions, + * which is where we will start packing low bits. + */ + SET_LOW_BITS_AREA_START(dest_buf,boundary_tmp); + + } + + /* Pack the low bit patterns into the area just after + * the queue positions. We have to round up the source + * region to a multiple of three words. + */ + + { + unsigned int num_tenbits_to_pack = + next_low_bits - tempLowBitsArray; + unsigned int num_packed_words = ceil((double) num_tenbits_to_pack / 3); + unsigned int num_source_words = num_packed_words * 3; + WK_word* endLowBitsArray = tempLowBitsArray + num_source_words; + + /* Pad out the array with zeros to avoid corrupting real packed + values. */ + + for (; /* next_low_bits is already set as desired */ + next_low_bits < endLowBitsArray; + next_low_bits++) { + *next_low_bits = 0; + } + +#ifdef WK_DEBUG + printf("about to pack low bits\n"); + printf("num_tenbits_to_pack is %u\n", num_tenbits_to_pack); + printf("endLowBitsArray is %u\n", endLowBitsArray); +#endif + + boundary_tmp = WK_pack_3_tenbits (tempLowBitsArray, + endLowBitsArray, + boundary_tmp); + + SET_LOW_BITS_AREA_END(dest_buf,boundary_tmp); + + } + return ((char *) boundary_tmp - (char *) dest_buf); + +} + +/********************************************************************* + * WKdm_decompress --- THE DECOMPRESSOR + * Expects WORD pointers to the source and destination buffers + * and a page size in words. The page size had better be 1024 unless + * somebody finds the places that are dependent on the page size and + * fixes them + */ + +void /* unsigned long int */ +WKdm_decompress (WK_word* src_buf, + WK_word* dest_buf, + unsigned int words, + struct comp_alg_data * data) +{ + + DictionaryElement dictionary[DICTIONARY_SIZE]; + unsigned int hashLookupTable [] = HASH_LOOKUP_TABLE_CONTENTS_WKDM; + + /* arrays that hold output data in intermediate form during modeling */ + /* and whose contents are packed into the actual output after modeling */ + + /* sizes of these arrays should be increased if you want to compress + * pages larger than 4KB + */ + WK_word * tempTagsArray = data->tempTagsArray; + WK_word * tempQPosArray = data->tempQPosArray; + WK_word * tempLowBitsArray = data->tempLowBitsArray; + + + PRELOAD_DICTIONARY; + + if (!tempTagsArray) + BUG(); + if (!tempQPosArray) + BUG(); + if (!tempLowBitsArray) + BUG(); + +#ifdef WK_DEBUG + printf("\nIn DECOMPRESSOR\n"); + printf("tempTagsArray is at %u\n", (unsigned long int) tempTagsArray); + printf("tempQPosArray is at %u\n", (unsigned long int) tempQPosArray); + printf("tempLowBitsArray is at %u\n", (unsigned long int) tempLowBitsArray); + + printf(" first four words of source buffer are:\n"); + printf(" %u\n %u\n %u\n %u\n", + src_buf[0], src_buf[1], src_buf[2], src_buf[3]); + + { int i; + WK_word *arr =(src_buf + TAGS_AREA_OFFSET + (PAGE_SIZE_IN_WORDS / 16)); + + printf(" first 20 full patterns are: \n"); + for (i = 0; i < 20; i++) { + printf(" %d", arr[i]); + } + printf("\n"); + } +#endif + + WK_unpack_2bits(TAGS_AREA_START(src_buf), + TAGS_AREA_END(src_buf), + tempTagsArray); + +#ifdef WK_DEBUG + { int i; + char* arr = (char *) tempTagsArray; + + printf(" first 200 tags are: \n"); + for (i = 0; i < 200; i++) { + printf(" %d", arr[i]); + } + printf("\n"); + } +#endif + + WK_unpack_4bits(QPOS_AREA_START(src_buf), + QPOS_AREA_END(src_buf), + tempQPosArray); + +#ifdef WK_DEBUG + { int i; + char* arr = (char *) tempQPosArray; + + printf(" first 200 queue positions are: \n"); + for (i = 0; i < 200; i++) { + printf(" %d", arr[i]); + } + printf("\n"); + } +#endif + + WK_unpack_3_tenbits(LOW_BITS_AREA_START(src_buf), + LOW_BITS_AREA_END(src_buf), + tempLowBitsArray); + +#ifdef WK_DEBUG + printf("AFTER UNPACKING, about to enter main block \n"); +#endif + + { + register char *next_tag = (char *) tempTagsArray; + char *tags_area_end = + ((char *) tempTagsArray) + PAGE_SIZE_IN_WORDS; + char *next_q_pos = (char *) tempQPosArray; + WK_word *next_low_bits = tempLowBitsArray; + WK_word *next_full_word = FULL_WORD_AREA_START(src_buf); + + WK_word *next_output = dest_buf; + +#ifdef WK_DEBUG + printf("next_output is %u\n", next_output); + + printf("next_tag is %u \n", next_tag); + printf("tags_area_end is %u\n", tags_area_end); + printf("next_q_pos is %u\n", next_q_pos); + printf("next_low_bits is %u\n", next_low_bits); + printf("next_full_word is %u\n", next_full_word); +#endif + + /* this loop should probably be unrolled. Maybe we should unpack + * as 4 bit values, giving two consecutive tags, and switch on + * that 16 ways to decompress 2 words at a whack + */ + while (next_tag < tags_area_end) { + + char tag = next_tag[0]; + + switch(tag) { + + case ZERO_TAG: { + *next_output = 0; + break; + } + case EXACT_TAG: { + WK_word *dict_location = dictionary + *(next_q_pos++); + /* no need to replace dict. entry if matched exactly */ + *next_output = *dict_location; + break; + } + case PARTIAL_TAG: { + WK_word *dict_location = dictionary + *(next_q_pos++); + { + WK_word temp = *dict_location; + + /* strip out low bits */ + temp = ((temp >> NUM_LOW_BITS) << NUM_LOW_BITS); + + /* add in stored low bits from temp array */ + temp = temp | *(next_low_bits++); + + *dict_location = temp; /* replace old value in dict. */ + *next_output = temp; /* and echo it to output */ + } + break; + } + case MISS_TAG: { + WK_word missed_word = *(next_full_word++); + WK_word *dict_location = + (WK_word *) + (((char *) dictionary) + HASH_TO_DICT_BYTE_OFFSET(missed_word)); + *dict_location = missed_word; + *next_output = missed_word; + break; + } + } + next_tag++; + next_output++; + } + +#ifdef WK_DEBUG + printf("AFTER DECOMPRESSING\n"); + printf("next_output is %u\n", (unsigned long int) next_output); + printf("next_tag is %u\n", (unsigned long int) next_tag); + printf("next_full_word is %u\n", (unsigned long int) next_full_word); + printf("next_q_pos is %u\n", (unsigned long int) next_q_pos); +#endif + } +} + +#ifdef WK_DEBUG +main() +{ + + WK_word* source_buf; + WK_word* dest_buf; + WK_word* udest_buf; + + int i; + + source_buf = (WK_word*) malloc(4096 * sizeof(WK_word)); + dest_buf = (WK_word*) malloc(4096 * sizeof(WK_word)); + udest_buf = (WK_word*) malloc(4096 * sizeof(WK_word)); + + for (i = 0; i < 1024; i++) + { + source_buf[i] = i * 15; + } + + source_buf[1025] = 99999; + + for (i = 0; i < 1024; i += 2) + { + source_buf[i] = i * 1024; + } + + for (i = 0; i < 1024; i += 17) + { + source_buf[i] = 0; + } + + for (i = 0; i < 1024; i++) + { + udest_buf[i] = 666; + } + + udest_buf[1025] = 55555; + + printf("source_buf is %u, dest_buf is %u, udest_buf is %u\n", + source_buf, dest_buf, udest_buf); + + printf("first 50 words of source_buf are:\n"); + for (i = 0; i < 50; i++) + printf(" %d", source_buf[i]); + + i = WKdm_compress(source_buf,dest_buf,1024); + printf("WKdm_compress returned %u\n", i); + + printf("redzone value at end of source buf (shd be 99999) is %u\n", + source_buf[1025]); + + i = WKdm_decompress(dest_buf,udest_buf,1024); + + printf("WKdm_decompress returned %u\n", i); + printf("redzone value at end of udest buf (shd be 55555) is %u\n", + udest_buf[1025]); + + printf("first 50 words of udest_buf are:\n"); + for (i = 0; i < 50; i++) + printf(" %d", udest_buf[i]); + + i = bcmp(source_buf, udest_buf, 100); + + printf("bcmp of orig. and compr'd/decompr'd copy (shd be 0) is %u\n", i); + +} +#endif + +/* + * Overrides for Emacs so that we follow Linus's tabbing style. + * Emacs will notice this stuff at the end of the file and automatically + * adjust the settings for this buffer only. This must remain at the end + * of the file. + * --------------------------------------------------------------------------- + * Local variables: + * c-file-style: "linux" + * End: + */ diff -urN linux-2.4.26-rc1/mm/comp_cache/adaptivity.c linux-2.4.26-rc1-ccache/mm/comp_cache/adaptivity.c --- linux-2.4.26-rc1/mm/comp_cache/adaptivity.c Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/mm/comp_cache/adaptivity.c Sun Mar 28 17:12:21 2004 @@ -0,0 +1,947 @@ +/* + * linux/mm/comp_cache/adaptivity.c + * + * Time-stamp: <2002-11-21 17:28:13 rcastro> + * + * Linux Virtual Memory Compressed Cache + * - Adaptivity Routines + * + * Author: Rodrigo S. de Castro + * Licensed under the GPL - 2001 + */ + +#include +#include +#include + +extern kmem_cache_t * comp_cachep; +static int fragment_failed_alloc = 0, vswap_failed_alloc = 0; +int growth_lock = 0; + +/* clean page hash */ +kmem_cache_t * clean_page_cachep; +struct clean_page_data ** clean_page_hash; +unsigned long clean_page_hash_size; +unsigned int clean_page_hash_bits; + +/* clean page list */ +LIST_HEAD(clean_page_list); +unsigned long nr_clean_page_hash = 0; + +unsigned long nr_clean_page_hits = 0; +unsigned long old_nr_clean_page_hits = 0; + +/* semaphore used to avoid two concurrent instances of + * {grow,shrink}_vswap() functions to run together */ +static struct semaphore vswap_resize_semaphore; + +void +resize_fragment_hash_table(void) { + struct comp_cache_fragment ** new_fragment_hash, * fragment, * next_fragment; + unsigned long new_fragment_hash_size; + unsigned int i, new_fragment_hash_bits, new_fragment_hash_order, hash_index; + + new_fragment_hash_size = NUM_FRAG_HASH_ENTRIES * sizeof(struct comp_cache_fragment *); + new_fragment_hash = create_fragment_hash(&new_fragment_hash_size, &new_fragment_hash_bits, &new_fragment_hash_order); + + if (!new_fragment_hash) { + fragment_failed_alloc = 1; + return; + } + + /* if we are growing the hash, but couldn't allocate a bigger + * chunk, let's back out and keep the current one */ + if (NUM_FRAG_HASH_ENTRIES > fragment_hash_size && + new_fragment_hash_order <= fragment_hash_order) + goto free_new_hash; + + /* let's rehash all the entries */ + for(i = 0; i < fragment_hash_size; i++) { + fragment = fragment_hash[i]; + + while (fragment) { + next_fragment = fragment->next_hash; + + remove_fragment_from_hash_table(fragment); + + hash_index = __fragment_hashfn(fragment->mapping, fragment->index, new_fragment_hash_size, new_fragment_hash_bits); + __add_fragment_to_hash_table(new_fragment_hash, hash_index, fragment); + + fragment = next_fragment; + } + } + + free_pages((unsigned long) fragment_hash, fragment_hash_order); + +#if 0 + printk("FRAGMENT HASH TABLE - resized from %lu to %lu\n", fragment_hash_size, new_fragment_hash_size); +#endif + fragment_hash = new_fragment_hash; + fragment_hash_size = new_fragment_hash_size; + fragment_hash_bits = new_fragment_hash_bits; + fragment_hash_order = new_fragment_hash_order; + fragment_failed_alloc = 0; + return; + + free_new_hash: + free_pages((unsigned long) new_fragment_hash, new_fragment_hash_order); +} + +extern void add_page_to_hash_queue(struct page * page, struct page **p); +extern inline void remove_page_from_hash_queue(struct page * page); +extern void change_index_hash_queue(struct page * page, unsigned long new_index); + +extern kmem_cache_t * vswap_cachep; +extern unsigned long nr_free_vswap; + +/*** + * shrink_vswap(void) - shrinks vswap adressing table from its current + * size (vswap_current_num_entries) to NUM_VSWAP_ENTRIES, its new size + * in function of num_comp_pages. + * + * we try to shrink the vswap at once, but that will depend on getting + * the lock on fragment->comp_page->page and swap cache page. The + * procedure is pretty simple: + * + * (a) unused entries: deallocate them, nulling vswap_address[i]. + * + * (b) used entries: we move them to an index below the new size + * (NUM_VSWAP_ENTRIES), ie, we compact all the used entries. For + * that, we update the fragment (if any), the swap cache page (if + * any) and all the ptes set to this address. + * + * (c) used entry && !vma: if we can't find a vma related to a pte, + * we backout all changes done to previous ptes and don't change + * anything else. + * + * - compacted all the entries, ie, the last used vswap entry is below + * NUM_VSWAP_ENTRIES? Let's just proceed to allocating the new vswap + * table. + * + * - no success compacting all the entries? so either some page + * couldn't be locked or ptes couldn't be set. Therefore we will wait + * a number of times before trying to shrink the vswap again. + */ +void +shrink_vswap(void) { + struct page * swap_cache_page; + struct comp_cache_fragment * fragment; + struct vswap_address ** new_vswap_address; + unsigned long index, new_index, vswap_new_num_entries = NUM_VSWAP_ENTRIES; + swp_entry_t old_entry, entry; + int failed_alloc = 0, ret; + + if (!vswap_address) + return; + + if (down_trylock(&vswap_resize_semaphore)) + return; + + spin_lock(&virtual_swap_list); + + if (vswap_current_num_entries <= 1.10 * NUM_VSWAP_ENTRIES) + goto out_unlock; + + /* more used entries than the new size? can't shrink */ + if (vswap_num_used_entries >= NUM_VSWAP_ENTRIES) + goto out_unlock; + +#if 0 + printk("shrinking\n"); +#endif + + if (vswap_current_num_entries <= vswap_new_num_entries) + BUG(); + + /* shrinking... it's good that we don't have to rush to shrink + * the cache since the estimated_* variables will make sure we + * won't use many vswap address */ + if (vswap_last_used < vswap_new_num_entries) + goto allocate_new_vswap; + + /* sanity check */ + if (vswap_last_used >= vswap_current_num_entries) + BUG(); + + new_index = vswap_new_num_entries - 1; + + for (index = vswap_last_used; index >= vswap_new_num_entries; index--) { + /* this vswap entry has already been freed, has been + * sucessfully allocated or has any page fault being + * serviced, so we are unable to move it in the vswap + * table */ + if (!vswap_address[index] || vswap_address[index]->fault_count) + continue; + + /* unused entry? let's only free it */ + if (!vswap_address[index]->swap_count) { + list_del(&(vswap_address[index]->list)); + nr_free_vswap--; + kmem_cache_free(vswap_cachep, (vswap_address[index])); + vswap_address[index] = NULL; + + /* time to fix the last_vswap_allocated (we + * may not reach the point where it will be + * updated) */ + if (index <= last_vswap_allocated) + last_vswap_allocated = index - 1; +#if 0 + printk("null %d\n", index); +#endif + continue; + } + + /* used entry? try to lock its swap cache page */ + swap_cache_page = vswap_address[index]->swap_cache_page; + + if (swap_cache_page && TryLockPage(swap_cache_page)) + continue; + + /* now try to lock the comp_page which has the + * fragment set to this vswap entry */ + fragment = vswap_address[index]->fragment; + + if (fragment && !reserved(index) && TryLockPage(fragment->comp_page->page)) { + if (swap_cache_page) + UnlockPage(swap_cache_page); + continue; + } + + /* find a unused entry below vswap_new_num_entries + * boundary to link this used entry we are moving + * down */ + while (new_index > 0) { + if (!vswap_address[new_index]) + break; + + if (!vswap_address[new_index]->swap_count) + break; + + new_index--; + } + + if (!new_index) + goto backout; + + old_entry = SWP_ENTRY(COMP_CACHE_SWP_TYPE, index); + entry = SWP_ENTRY(COMP_CACHE_SWP_TYPE, new_index); + + /* let's fix the ptes */ + spin_unlock(&virtual_swap_list); + ret = set_pte_list_to_entry(vswap_address[index]->pte_list, old_entry, entry); + spin_lock(&virtual_swap_list); + + if (!ret) + goto backout; + + /* let's fix the swap cache page */ + if (swap_cache_page) { + change_index_hash_queue(swap_cache_page, entry.val); + UnlockPage(swap_cache_page); + } + + /* now we have to fix the fragment in the hash table */ + if (fragment && !reserved(index)) { + remove_fragment_from_hash_table(fragment); + fragment->index = entry.val; + add_fragment_to_hash_table(fragment); + + UnlockPage(fragment->comp_page->page); + } + + if (vswap_address[new_index]) { + list_del(&(vswap_address[new_index]->list)); + nr_free_vswap--; + kmem_cache_free(vswap_cachep, (vswap_address[new_index])); + vswap_address[new_index] = NULL; + } + +#if 0 + printk("vswap %lu -> %lu\n", index, new_index); +#endif + + vswap_address[new_index] = vswap_address[index]; + vswap_address[new_index]->offset = new_index; + + vswap_address[index] = NULL; + + continue; + + backout: + if (swap_cache_page) + UnlockPage(swap_cache_page); + if (fragment && !reserved(index)) + UnlockPage(fragment->comp_page->page); + break; + } + + /* let's fix the vswap_last_used variable */ + for (; vswap_last_used > 0; vswap_last_used--) { + if (!vswap_address[vswap_last_used]) + continue; + + if (!vswap_address[vswap_last_used]->swap_count + && vswap_last_used >= vswap_new_num_entries) + BUG(); + + break; + } + + if (vswap_last_used >= vswap_new_num_entries) + goto out_unlock; + + allocate_new_vswap: + spin_unlock(&virtual_swap_list); + new_vswap_address = (struct vswap_address **) vmalloc(vswap_new_num_entries * sizeof(struct vswap_address*)); + spin_lock(&virtual_swap_list); + + if (!new_vswap_address) { + vswap_failed_alloc = 1; + goto out_unlock; + } + + for (index = 0; index <= vswap_last_used; index++) { + /* we are shrinking the cache after tried to shrink it + * to a _smaller_ value, so we may have null entries + * in indexes lower than vswap_last_used that have to + * be reallocated */ + if (!vswap_address[index]) { + if (!vswap_alloc_and_init(new_vswap_address, index) && !failed_alloc) { + failed_alloc = 1; + last_vswap_allocated = index - 1; + } + continue; + } + new_vswap_address[index] = vswap_address[index]; + } + + /* debug purposes */ + for (index = vswap_last_used; index < vswap_current_num_entries; index++) { + if (!vswap_address[index]) + continue; + + if (index < vswap_new_num_entries) + continue; + + BUG(); + } + + for (index = vswap_last_used + 1; index < vswap_new_num_entries; index++) { + if (!vswap_alloc_and_init(new_vswap_address, index) && !failed_alloc) { + failed_alloc = 1; + last_vswap_allocated = index - 1; + } + } + + if (!failed_alloc) + last_vswap_allocated = vswap_new_num_entries - 1; + + vfree(vswap_address); + vswap_address = new_vswap_address; + +#if 0 + printk("VSWAP - resized from %ld to %ld (copied until %d)\n", vswap_current_num_entries, vswap_new_num_entries, vswap_last_used); +#endif + vswap_current_num_entries = vswap_new_num_entries; + vswap_last_used = vswap_new_num_entries - 1; + vswap_failed_alloc = 0; + out_unlock: + spin_unlock(&virtual_swap_list); + up(&vswap_resize_semaphore); +} + +/*** + * grow_vswap(void) - grows vswap adressing table from its current + * size (vswap_current_num_entries) to NUM_VSWAP_ENTRIES, its new size + * in function of num_comp_pages. + * + * we allocate the new vswap table and allocates the needed vswap + * entries (empty ones, due to a previous unsuccesful shrink, or the + * new ones), updating some control variables to conclude. + */ +static void +grow_vswap(void) { + struct vswap_address ** new_vswap_address; + unsigned int i, failed_alloc = 0; + unsigned long vswap_new_num_entries = NUM_VSWAP_ENTRIES; + + if (!vswap_address) + return; + + if (down_trylock(&vswap_resize_semaphore)) + return; + + spin_lock(&virtual_swap_list); + + /* using vswap_last_used instead of vswap_current_num_entries + * forces us to grow the cache even if we started shrinking + * it, but one set comp cache to the original size */ + if (vswap_last_used >= 0.90 * (NUM_VSWAP_ENTRIES - 1)) + goto out_unlock; + +#if 0 + printk("growing\n"); +#endif + + if (vswap_last_used >= vswap_new_num_entries - 1) + BUG(); + + if (vswap_current_num_entries == vswap_new_num_entries) + goto fix_old_vswap; + + spin_unlock(&virtual_swap_list); + new_vswap_address = (struct vswap_address **) vmalloc(vswap_new_num_entries * sizeof(struct vswap_address*)); + spin_lock(&virtual_swap_list); + + if (!new_vswap_address) { + vswap_failed_alloc = 1; + goto out_unlock; + } + + if (vswap_last_used >= vswap_current_num_entries) + BUG(); + + for (i = 0; i <= vswap_last_used; i++) { + /* we are growing the cache after tried to shrink it + * (but couldn't do it at once), so we may have null + * vswap_address entries in indexes lower than + * vswap_last_used that have to be reallocated */ + if (!vswap_address[i]) { + if (!vswap_alloc_and_init(new_vswap_address, i) && !failed_alloc) { + failed_alloc = 1; + last_vswap_allocated = i - 1; + } + continue; + } + new_vswap_address[i] = vswap_address[i]; + } + + /* we are growing the cache after tried to shrink it (but + * couldn't do it at once), vswap_last_used will be smaller + * than vswap_new_num_entries - 1, so we have to reallocate the + * missing entries */ + for (i = vswap_last_used + 1; i < vswap_new_num_entries; i++) { + if (!vswap_alloc_and_init(new_vswap_address, i) && !failed_alloc) { + failed_alloc = 1; + last_vswap_allocated = i - 1; + } + } + + if (!failed_alloc) + last_vswap_allocated = vswap_new_num_entries - 1; + + vfree(vswap_address); + vswap_address = new_vswap_address; + +#if 0 + printk("VSWAP - resized from %ld to %ld\n", vswap_current_num_entries, vswap_new_num_entries); +#endif + vswap_current_num_entries = vswap_new_num_entries; + vswap_last_used = vswap_new_num_entries - 1; + vswap_failed_alloc = 0; + goto out_unlock; + + fix_old_vswap: + /* that will happen if we tried to shrink, but couldn't finish + * it at once, and in the meanwhile one set comp cache to the + * original size. That way, we don't have to allocate a new + * vswap, only reallocate the empty entries */ + for (i = 0; i < vswap_current_num_entries; i++) { + if (!vswap_address[i]) { + if (!vswap_alloc_and_init(vswap_address, i) && !failed_alloc) { + failed_alloc = 1; + last_vswap_allocated = i - 1; + } + } + } + + if (!failed_alloc) + last_vswap_allocated = vswap_new_num_entries - 1; + vswap_last_used = vswap_current_num_entries - 1; + out_unlock: + spin_unlock(&virtual_swap_list); + up(&vswap_resize_semaphore); +} + +static inline void +shrink_fragment_hash_table(void) { + unsigned long new_fragment_hash_size = NUM_FRAG_HASH_ENTRIES * sizeof(struct comp_cache_fragment *); + + /* if we shrink the hash table an order, will the data fit in + * there? if they won't, no need to shrink the hash table */ + if ((PAGE_SIZE << (fragment_hash_order - 1)) < new_fragment_hash_size) + return; + + resize_fragment_hash_table(); +} + +static inline void +shrink_zone_watermarks(void) +{ + if (zone_num_comp_pages <= num_comp_pages) + return; + + comp_cache_fix_watermarks(num_comp_pages); +} + +/*** + * shrink_comp_cache(comp_page) - given a "comp_page" entry, check if + * this page does not have fragments, trying to release it to the + * system in this case. After the page is released, all the compressed + * cache data structures must be fixed accordingly. + * + * caller must hold comp_cache_lock lock + */ +static int +shrink_comp_cache(struct comp_cache_page * comp_page) +{ + int retval = 0; + + spin_lock(&comp_cache_lock); + + if (!comp_page->page) + BUG(); + + /* if the comp_page is not empty, can't free it */ + if (!list_empty(&(comp_page->fragments))) { + UnlockPage(comp_page->page); + goto out; + } + + /* we have an empty page, so let's do it */ + retval = 1; + + remove_comp_page_from_hash_table(comp_page); + + if (page_count(comp_page->page) != 1) + BUG(); + UnlockPage(comp_page->page); + __free_pages(comp_page->page, COMP_PAGE_ORDER); + + set_comp_page(comp_page, NULL); + kmem_cache_free(comp_cachep, comp_page); + num_comp_pages--; + + /* only change the zone watermarks if we shrunk the cache */ + shrink_zone_watermarks(); + out: + shrink_fragment_hash_table(); + shrink_vswap(); + spin_unlock(&comp_cache_lock); + return retval; +} + +/*** + * shrink_on_demand(comp_page) - called by comp_cache_free(), it will + * try to shrink the compressed cache by one entry (comp_page). The + * comp_cache_free() function is called by every place that free a + * compressed cache fragment but swap out functions. + */ +int +shrink_on_demand(struct comp_cache_page * comp_page) +{ + /* don't shrink a comp cache that has reached the min size */ + if (num_comp_pages == min_num_comp_pages) { + UnlockPage(comp_page->page); + return 0; + } + + if (shrink_comp_cache(comp_page)) + return 1; + return 0; +} + +static inline void +grow_fragment_hash_table(void) { + unsigned long new_fragment_hash_size = NUM_FRAG_HASH_ENTRIES * sizeof(struct comp_cache_fragment *); + + /* do we really need a bigger hash table? */ + if ((PAGE_SIZE << fragment_hash_order) >= new_fragment_hash_size) + return; + + resize_fragment_hash_table(); +} + +static inline void +grow_zone_watermarks(void) +{ + if (zone_num_comp_pages >= num_comp_pages) + return; + + comp_cache_fix_watermarks(num_comp_pages); +} + +/*** + * grow_comp_cache(void) - try to allocate a compressed cache page + * (may be 1 or 2 memory pages). If it is successful, initialize it, + * adding to the compressed cache. + */ +static int +grow_comp_cache(void) +{ + struct comp_cache_page * comp_page; + struct page * page; + int retval = 0; + + spin_lock(&comp_cache_lock); + + page = alloc_pages(GFP_ATOMIC, COMP_PAGE_ORDER); + + /* couldn't allocate the page */ + if (!page) { + failed_comp_page_allocs++; + goto out_unlock; + } + + if (!init_comp_page(&comp_page, page)) { + __free_pages(page, COMP_PAGE_ORDER); + goto out_unlock; + } + + retval = 1; + + comp_cache_freeable_space += COMP_PAGE_SIZE; + comp_cache_free_space += COMP_PAGE_SIZE; + num_comp_pages++; + + if (num_comp_pages > max_used_num_comp_pages) + max_used_num_comp_pages = num_comp_pages; + + grow_zone_watermarks(); + grow_fragment_hash_table(); + grow_vswap(); + out_unlock: + spin_unlock(&comp_cache_lock); + return retval; +} + +/*** + * grow_on_demand(void) - called by get_comp_cache_page() when it + * cannot find space in the compressed cache. If compressed cache has + * not yet reached the maximum size, we try to grow compressed cache + * by one new entry. + */ +int +grow_on_demand(void) +{ + /* don't grow a comp cache that has reached the max size */ + if (num_comp_pages == max_num_comp_pages) + return 0; + + /* if adaptability policy locked the growth, return */ + if (growth_lock) + return 0; + + if (grow_comp_cache()) + return 1; + + return 0; +} + +#define writeout_one_fragment(gfp_mask) writeout_fragments(gfp_mask, 1, 6) + +void +compact_comp_cache(void) +{ + struct comp_cache_page * comp_page, * previous_comp_page = NULL, * new_comp_page, ** hash_table; + struct comp_cache_fragment * fragment, * new_fragment; + struct list_head * fragment_lh; + int i, fail; + + next_fragment: + hash_table = free_space_hash; + + i = free_space_hash_size - 1; + do { + comp_page = hash_table[i--]; + } while(i > 0 && !comp_page); + + if (previous_comp_page && previous_comp_page != comp_page) + return; + + if (!comp_page || TryLockPage(comp_page->page)) + goto writeout; + + if (list_empty(&comp_page->fragments)) { + shrink_on_demand(comp_page); + return; + } + + fragment_lh = comp_page->fragments.prev; + fail = 0; + while (1) { + fragment = list_entry(fragment_lh, struct comp_cache_fragment, list); + if (fragment_count(fragment) != 1) { + fail = 1; + goto next; + } + if (!CompFragmentToBeFreed(fragment)) + break; + next: + fragment_lh = fragment_lh->prev; + if (fragment_lh == &comp_page->fragments) { + if (fail) + goto out2_failed; + UnlockPage(comp_page->page); + return; + } + } + search_again: + new_comp_page = search_comp_page(hash_table, fragment->compressed_size); + + if (new_comp_page && !TryLockPage(new_comp_page->page)) + goto got_page; + + if (hash_table == free_space_hash) { + hash_table = total_free_space_hash; + goto search_again; + } + goto out2_failed; + + got_page: + if (hash_table == total_free_space_hash) + compact_fragments(new_comp_page); + + remove_comp_page_from_hash_table(new_comp_page); + + /* allocate the new fragment */ + new_fragment = alloc_fragment(); + + if (!new_fragment) { + UnlockPage(comp_page->page); + goto out_failed; + } + + new_fragment->index = fragment->index; + new_fragment->mapping = fragment->mapping; + new_fragment->offset = new_comp_page->free_offset; + new_fragment->compressed_size = fragment->compressed_size; + new_fragment->flags = fragment->flags; + new_fragment->comp_page = new_comp_page; + set_fragment_count(new_fragment, fragment_count(fragment)); + + if ((new_fragment->swp_buffer = fragment->swp_buffer)) + new_fragment->swp_buffer->fragment = new_fragment; + + memcpy(page_address(new_comp_page->page) + new_fragment->offset, page_address(comp_page->page) + fragment->offset, fragment->compressed_size); + + previous_comp_page = comp_page; + + UnlockPage(comp_page->page); + if (!drop_fragment(fragment)) { + if (fragment->swp_buffer) + fragment->swp_buffer->fragment = fragment; + kmem_cache_free(fragment_cachep, new_fragment); + goto out_failed; + } + + /* let's update some important fields */ + new_comp_page->free_space -= new_fragment->compressed_size; + new_comp_page->total_free_space -= new_fragment->compressed_size; + new_comp_page->free_offset += new_fragment->compressed_size; + + add_to_comp_page_list(new_comp_page, new_fragment); + add_fragment_vswap(new_fragment); + add_fragment_to_hash_table(new_fragment); + + if (CompFragmentActive(new_fragment)) + add_fragment_to_active_lru_queue(new_fragment); + else + add_fragment_to_inactive_lru_queue(new_fragment); + + if (PageSwapCache(new_fragment)) + num_swapper_fragments++; + num_fragments++; + + new_fragment->mapping->nrpages++; + if (CompFragmentDirty(new_fragment)) + list_add(&new_fragment->mapping_list, &new_fragment->mapping->dirty_comp_pages); + else { + list_add(&new_fragment->mapping_list, &new_fragment->mapping->clean_comp_pages); + num_clean_fragments++; + } + + balance_lru_queues(); + + add_comp_page_to_hash_table(new_comp_page); + UnlockPage(new_comp_page->page); + goto next_fragment; + + writeout: + writeout_one_fragment(GFP_KERNEL); + return; + + out_failed: + add_comp_page_to_hash_table(new_comp_page); + UnlockPage(new_comp_page->page); + goto writeout; + + out2_failed: + UnlockPage(comp_page->page); + goto writeout; +} + +void +balance_lru_queues(void) +{ + struct comp_cache_fragment * fragment; + unsigned long num_memory_pages; + + /* while condition: + * + * (num_active_fragments * 100)/num_fragments > ((num_comp_pages << COMP_PAGE_ORDER) * 100)/num_fragments + */ + num_memory_pages = (num_comp_pages << COMP_PAGE_ORDER); + while (num_active_fragments > num_memory_pages) { + fragment = list_entry(active_lru_queue.prev, struct comp_cache_fragment, lru_queue); + + remove_fragment_from_lru_queue(fragment); + add_fragment_to_inactive_lru_queue(fragment); + } +} + +#ifndef CONFIG_COMP_DIS_CLEAN +void +hit_clean_page(struct page * page) +{ + struct clean_page_data * clpage; + + clpage = clean_page_hash[clean_page_hashfn(page->mapping, page->index)]; + + goto inside; + + for (;;) { + clpage = clpage->next_hash; + inside: + if (!clpage) + return; + if (clpage->mapping != page->mapping) + continue; + if (clpage->index == page->index) + break; + } + + /* mark it as hit */ + clpage->mapping = NULL; + nr_clean_page_hits++; + + /* if too many hits, try to store the clean pages */ + if (nr_clean_page_hits * 10 > clean_page_hash_size) { + clean_page_compress_lock = 0; + old_nr_clean_page_hits += nr_clean_page_hits; + nr_clean_page_hits = 0; + } +} + +void +add_clean_page(struct page * page) +{ + struct clean_page_data * clpage, **old_clpage; + unsigned long hash_index; + + /* allocate a new structure */ + clpage = ((struct clean_page_data *) kmem_cache_alloc(clean_page_cachep, SLAB_ATOMIC)); + + if (unlikely(!clpage)) + return; + + clpage->mapping = page->mapping; + clpage->index = page->index; + + /* add to hash table...*/ + hash_index = clean_page_hashfn(page->mapping, page->index); + old_clpage = &clean_page_hash[hash_index]; + + if ((clpage->next_hash = *old_clpage)) + (*old_clpage)->pprev_hash = &clpage->next_hash; + + *old_clpage = clpage; + clpage->pprev_hash = old_clpage; + + /* and to the list */ + list_add(&clpage->list, &clean_page_list); + nr_clean_page_hash++; + + if (nr_clean_page_hash > clean_page_hash_size * 2) { + struct clean_page_data *next; + struct clean_page_data **pprev; + + clpage = list_entry(clean_page_list.prev, struct clean_page_data, list); + + /* remove from the list... */ + list_del(clean_page_list.prev); + + if (!clpage->mapping) { + if (old_nr_clean_page_hits) + old_nr_clean_page_hits--; + else + nr_clean_page_hits--; + } + + /* and from the hash table */ + next = clpage->next_hash; + pprev = clpage->pprev_hash; + + if (next) + next->pprev_hash = pprev; + *pprev = next; + clpage->pprev_hash = NULL; + + /* free the old structure */ + kmem_cache_free(clean_page_cachep, clpage); + + nr_clean_page_hash--; + } + + if (num_clean_fragments * 10 > num_fragments * 3) + compact_comp_cache(); +} +#endif + +void __init +comp_cache_adaptivity_init(void) +{ + unsigned int order; + + init_MUTEX(&vswap_resize_semaphore); + +#ifndef CONFIG_COMP_DIS_CLEAN + /* clean pages hash table */ + clean_page_hash_size = comp_page_to_page(max_num_comp_pages)/7; + + for (order = 0; (PAGE_SIZE << order) < clean_page_hash_size; order++); + + do { + unsigned long tmp = (PAGE_SIZE << order)/sizeof(struct clean_page_data *); + + clean_page_hash_bits = 0; + while((tmp >>= 1UL) != 0UL) + clean_page_hash_bits++; + + clean_page_hash = (struct clean_page_data **) __get_free_pages(GFP_ATOMIC, order); + } while(clean_page_hash == NULL && --order > 0); + + clean_page_hash_size = 1 << clean_page_hash_bits; + + if (!clean_page_hash) + panic("comp_cache_adaptivity_init(): couldn't allocate clean page hash table\n"); + + memset((void *) clean_page_hash, 0, clean_page_hash_size * sizeof(struct clean_page_data *)); + + clean_page_cachep = kmem_cache_create("comp_cache_clean", sizeof(struct clean_page_data), 0, SLAB_HWCACHE_ALIGN, NULL, NULL); + + printk("Compressed Cache: adaptivity\n" + "Compressed Cache: clean page (%lu entries = %luB)\n", clean_page_hash_size, PAGE_SIZE << order); +#endif +} + +/* + * Overrides for Emacs so that we follow Linus's tabbing style. + * Emacs will notice this stuff at the end of the file and automatically + * adjust the settings for this buffer only. This must remain at the end + * of the file. + * --------------------------------------------------------------------------- + * Local variables: + * c-file-style: "linux" + * End: + */ diff -urN linux-2.4.26-rc1/mm/comp_cache/aux.c linux-2.4.26-rc1-ccache/mm/comp_cache/aux.c --- linux-2.4.26-rc1/mm/comp_cache/aux.c Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/mm/comp_cache/aux.c Sun Mar 28 17:12:21 2004 @@ -0,0 +1,659 @@ +/* + * linux/mm/comp_cache/aux.c + * + * Time-stamp: <2002-10-28 21:13:03 rcastro> + * + * Linux Virtual Memory Compressed Cache + * + * Author: Rodrigo S. de Castro + * Licensed under the GPL - 2001 + */ + +#include +#include +#include +#include + +struct comp_cache_fragment ** fragment_hash; +unsigned long fragment_hash_size; +unsigned long fragment_hash_used; +unsigned int fragment_hash_order; +unsigned int fragment_hash_bits; + +struct comp_cache_page ** free_space_hash; +unsigned int free_space_hash_size; +unsigned int free_space_interval; + +struct comp_cache_page ** total_free_space_hash; +unsigned int total_free_space_hash_size; +unsigned int total_free_space_interval; + +/* computes (unsigned long long x) / (unsigned long long y) */ +unsigned long long +big_division(unsigned long long x, unsigned long long y) +{ + unsigned long long q; /* x / y */ + unsigned long long limit; + int k; /* log q */ + + if (y == 0) + return 0xffffffffffffffff; /* -1 */ + + if (x == 0) + return (unsigned long long) 0; + + if (x <= 0x7fffffffffffffff) + limit = x; + else + limit = 0x7fffffffffffffff; + + k = 0; + while ( y <= limit ) { + y <<= 1; + k++; + } + if (y > x) { + y >>= 1; + k--; + } + /* x/2 < y <= x */ + + q = 1; x -= y; + while ( k-- ) { + y >>= 1; + q <<= 1; + if (y <= x) { + x -= y; + q++; + } + } + + return q; +} + +inline void +set_comp_page(struct comp_cache_page * comp_page, struct page * page) +{ + if (!comp_page) + BUG(); + if (comp_page->page) { + if (page) + goto out; + comp_cache_freeable_space -= COMP_PAGE_SIZE; + comp_cache_free_space -= COMP_PAGE_SIZE; + goto out; + } + + if (!page) + BUG(); + + comp_cache_freeable_space += COMP_PAGE_SIZE; + comp_cache_free_space += COMP_PAGE_SIZE; + + out: + comp_page->page = page; +} + +static void +backout_pte_changes(struct pte_list * start_pte_list, struct pte_list * failed_pte_list, swp_entry_t old_entry) +{ + struct mm_struct * mm; + struct vm_area_struct * vma; + struct pte_list * pte_list; + unsigned long address; + pte_t pte; + + pte_list = start_pte_list; + + while (pte_list != failed_pte_list) { + mm = ptep_to_mm(pte_list->ptep); + spin_lock(&mm->page_table_lock); + + address = ptep_to_address(pte_list->ptep); + vma = find_vma(mm, address); + + if (!vma) + BUG(); + + pte = ptep_get_and_clear(pte_list->ptep); + flush_tlb_page(vma, address); + flush_cache_page(vma, address); + + set_pte(pte_list->ptep, swp_entry_to_pte(old_entry)); + + spin_unlock(&mm->page_table_lock); + pte_list = pte_list->next; + } +} + +/* unlikely, but list can still be corrupted since it is not protected by any lock */ +int +set_pte_list_to_entry(struct pte_list * start_pte_list, swp_entry_t old_entry, swp_entry_t entry) +{ + struct mm_struct * mm; + unsigned long address; + struct vm_area_struct * vma; + struct pte_list * pte_list; + pte_t * ptep, pte; + + pte_list = start_pte_list; + + while (pte_list) { + ptep = pte_list->ptep; + mm = ptep_to_mm(ptep); + spin_lock(&mm->page_table_lock); + + address = ptep_to_address(ptep); + vma = find_vma(mm, address); + + if (!vma) + goto error; + + pte = ptep_get_and_clear(ptep); + flush_tlb_page(vma, address); + flush_cache_page(vma, address); + + set_pte(ptep, swp_entry_to_pte(entry)); + + spin_unlock(&mm->page_table_lock); + pte_list = pte_list->next; + } + + return 1; + + error: + spin_unlock(&mm->page_table_lock); + backout_pte_changes(start_pte_list, pte_list, old_entry); + return 0; +} + +inline void +__add_fragment_to_hash_table(struct comp_cache_fragment ** hash_table, unsigned int hash_index, struct comp_cache_fragment * new_fragment) { + struct comp_cache_fragment ** fragment; + + fragment = &hash_table[hash_index]; + + if ((new_fragment->next_hash = *fragment)) + (*fragment)->pprev_hash = &new_fragment->next_hash; + + *fragment = new_fragment; + new_fragment->pprev_hash = fragment; + + fragment_hash_used++; +} + +inline void +remove_fragment_from_hash_table(struct comp_cache_fragment * fragment) { + struct comp_cache_fragment *next = fragment->next_hash; + struct comp_cache_fragment **pprev = fragment->pprev_hash; + + if (next) + next->pprev_hash = pprev; + *pprev = next; + fragment->pprev_hash = NULL; + + fragment_hash_used--; +} + +unsigned long +free_space_count(int index, unsigned long * num_fragments) { + struct comp_cache_page * comp_page; + struct comp_cache_fragment * fragment; + unsigned long total, total_fragments; + struct list_head * fragment_lh; + + if (index < 0) + BUG(); + + if (index >= free_space_hash_size) + BUG(); + + for (comp_page = free_space_hash[index], total = 0; comp_page; comp_page = comp_page->next_hash_fs, total++) { + total_fragments = 0; + + for_each_fragment(fragment_lh, comp_page) { + fragment = list_entry(fragment_lh, struct comp_cache_fragment, list); + + if (!fragment_freed(fragment)) + total_fragments++; + } + +#if 0 + if (comp_page->page) { + if (!comp_page->page->buffers && page_count(comp_page->page) != 1) + BUG(); + if (PageDirty(comp_page->page)) + BUG(); + } + else { + if (index != free_space_hash_size - 1) + BUG(); + if (comp_page->free_space != COMP_PAGE_SIZE) + BUG(); + } +#endif + + switch(total_fragments) + { + case 0: + num_fragments[0]++; + break; + case 1: + num_fragments[1]++; + break; + default: + if (total_fragments > 6) + num_fragments[7]++; + else + num_fragments[total_fragments]++; + break; + } + } + + return total; +} + +unsigned long +fragmentation_count(int index, unsigned long * frag_space, int interval) { + struct comp_cache_page * comp_page; + struct comp_cache_fragment * fragment; + struct list_head * fragment_lh; + unsigned long total, fragmented_space; + + if (index < 0) + BUG(); + + if (index >= free_space_hash_size) + BUG(); + + for (comp_page = free_space_hash[index], total = 0; comp_page; comp_page = comp_page->next_hash_fs, total++) { + fragmented_space = 0; + + for_each_fragment(fragment_lh, comp_page) { + fragment = list_entry(fragment_lh, struct comp_cache_fragment, list); + + if (fragment_freed(fragment)) + fragmented_space += fragment->compressed_size; + } + + if (fragmented_space + comp_page->free_space != comp_page->total_free_space) + BUG(); + + frag_space[(int) fragmented_space/interval]++; + } + + return total; +} + +inline void +add_comp_page_to_hash_table(struct comp_cache_page * new_comp_page) { + struct comp_cache_page ** comp_page; + + /* add to free space hash table */ + comp_page = &free_space_hash[free_space_hashfn(new_comp_page->free_space)]; + + if ((new_comp_page->next_hash_fs = *comp_page)) + (*comp_page)->pprev_hash_fs = &new_comp_page->next_hash_fs; + + *comp_page = new_comp_page; + new_comp_page->pprev_hash_fs = comp_page; + + /* add to total free space hash table */ + comp_page = &total_free_space_hash[free_space_hashfn(new_comp_page->total_free_space)]; + + if ((new_comp_page->next_hash_tfs = *comp_page)) + (*comp_page)->pprev_hash_tfs = &new_comp_page->next_hash_tfs; + + *comp_page = new_comp_page; + new_comp_page->pprev_hash_tfs = comp_page; +} + +inline void +remove_comp_page_from_hash_table(struct comp_cache_page * comp_page) { + struct comp_cache_page *next; + struct comp_cache_page **pprev; + + /* remove from free space hash table */ + next = comp_page->next_hash_fs; + pprev = comp_page->pprev_hash_fs; + + if (next) + next->pprev_hash_fs = pprev; + *pprev = next; + comp_page->pprev_hash_fs = NULL; + + /* remove from total free space hash table */ + next = comp_page->next_hash_tfs; + pprev = comp_page->pprev_hash_tfs; + + if (next) + next->pprev_hash_tfs = pprev; + *pprev = next; + comp_page->pprev_hash_tfs = NULL; +} + +void +add_to_comp_page_list(struct comp_cache_page * comp_page, struct comp_cache_fragment * fragment) +{ + struct list_head * fragment_lh; + struct comp_cache_fragment * previous_fragment = NULL; + + /* add the fragment to the comp_page list of fragments */ + if (list_empty(&(comp_page->fragments))) { + list_add(&(fragment->list), &(comp_page->fragments)); + return; + } + + previous_fragment = list_entry(comp_page->fragments.prev, struct comp_cache_fragment, list); + + if (previous_fragment->offset + previous_fragment->compressed_size == fragment->offset) { + list_add_tail(&(fragment->list), &(comp_page->fragments)); + return; + } + + /* let's search for the correct place in the comp_page list */ + previous_fragment = NULL; + + for_each_fragment(fragment_lh, comp_page) { + struct comp_cache_fragment * aux_fragment; + + aux_fragment = list_entry(fragment_lh, struct comp_cache_fragment, list); + + if (aux_fragment->offset + aux_fragment->compressed_size > fragment->offset) + break; + + previous_fragment = aux_fragment; + } + + if (previous_fragment) + list_add(&(fragment->list), &(previous_fragment->list)); + else + list_add(&(fragment->list), &(comp_page->fragments)); +} + +struct comp_cache_page * +FASTCALL(search_comp_page(struct comp_cache_page ** hash_table, int free_space)); +struct comp_cache_page * +search_comp_page(struct comp_cache_page ** hash_table, int free_space) { + struct comp_cache_page * comp_page; + int idx, i; + + idx = free_space_hashfn(free_space); + + /* first of all let's try to get at once a comp page whose + * free space is surely bigger than what need */ + for (i = idx + 1; i < free_space_hash_size; i++) { + if (hash_table[i]) + return hash_table[i]; + } + + comp_page = hash_table[idx]; + if (hash_table == free_space_hash) + goto inside_fs; + goto inside_tfs; + + for (;;) { + comp_page = comp_page->next_hash_fs; + inside_fs: + if (!comp_page) + return NULL; + if (comp_page->free_space >= free_space) + return comp_page; + } + + for (;;) { + comp_page = comp_page->next_hash_tfs; + inside_tfs: + if (!comp_page) + return NULL; + if (comp_page->total_free_space >= free_space) + return comp_page; + } +} + +inline void +add_fragment_to_active_lru_queue(struct comp_cache_fragment * fragment) { + if (!fragment) + BUG(); + + if (PageSwapCache(fragment)) { + swp_entry_t entry; + entry.val = fragment->index; + if (vswap_address(entry)) + return; + } + + list_add(&(fragment->lru_queue), &active_lru_queue); + CompFragmentSetActive(fragment); + num_active_fragments++; +} + +inline void +add_fragment_to_inactive_lru_queue(struct comp_cache_fragment * fragment) { + if (!fragment) + BUG(); + + if (PageSwapCache(fragment)) { + swp_entry_t entry; + entry.val = fragment->index; + if (vswap_address(entry)) + return; + } + + list_add(&(fragment->lru_queue), &inactive_lru_queue); +} + +inline void +remove_fragment_from_lru_queue(struct comp_cache_fragment * fragment) { + if (!fragment) + BUG(); + + if (PageSwapCache(fragment)) { + swp_entry_t entry; + entry.val = fragment->index; + if (vswap_address(entry)) + return; + } + + list_del_init(&(fragment->lru_queue)); + if (CompFragmentTestandClearActive(fragment)) + num_active_fragments--; +} + +/* adapted version of __find_page_nolock:filemap.c + * caller must hold comp_cache_hold */ +int FASTCALL(find_comp_page(struct address_space *, unsigned long, struct comp_cache_fragment **)); +int find_comp_page(struct address_space * mapping, unsigned long offset, struct comp_cache_fragment ** fragment) +{ + struct comp_cache_fragment * fhash; + int err = -ENOENT; + + *fragment = NULL; + + if (list_empty(&mapping->clean_comp_pages) && list_empty(&mapping->dirty_comp_pages)) + goto not_found; + + fhash = fragment_hash[__fragment_hashfn(mapping, offset, fragment_hash_size, fragment_hash_bits)]; + + goto inside; + + for (;;) { + fhash = fhash->next_hash; + inside: + if (!fhash) + goto not_found; + if (fhash->mapping != mapping) + continue; + if (fhash->index == offset) { + *fragment = fhash; + err = 0; + break; + } + } + + not_found: + return err; +} + +inline int +in_comp_cache(struct address_space * mapping, unsigned long offset) +{ + struct comp_cache_fragment * fragment; + int ret; + + spin_lock(&comp_cache_lock); + ret = !find_comp_page(mapping, offset, &fragment); + spin_unlock(&comp_cache_lock); + return ret; +} + + +inline void +print_all_fragments (struct comp_cache_page * comp_page) +{ + struct list_head * fragment_lh; + struct comp_cache_fragment * fragment; + + printk("DEBUG: fragment List for %08lx\n", (unsigned long) comp_page); + + for_each_fragment(fragment_lh, comp_page) { + fragment = list_entry(fragment_lh, struct comp_cache_fragment, list); + printk(" %08lx (entry: %08lx offset: %d compressed_size: %d\n", (unsigned long) fragment, fragment->index, fragment->offset, fragment->compressed_size); + } +} + +inline void +check_all_fragments(struct comp_cache_page * comp_page) +{ + struct comp_cache_fragment * fragment, * aux_fragment; + struct list_head * fragment_lh, * aux_fragment_lh; + int used_space = 0; + swp_entry_t entry; + + if (!PageLocked(comp_page->page)) + BUG(); + + for_each_fragment(fragment_lh, comp_page) { + fragment = list_entry(fragment_lh, struct comp_cache_fragment, list); + + if (fragment->comp_page != comp_page) + BUG(); + + entry.val = fragment->index; + + /* free space overlapping */ + if (comp_page->free_offset < fragment->offset && comp_page->free_offset + comp_page->free_space > fragment->offset) + BUG(); + + if (fragment->compressed_size <= 0 || fragment->compressed_size > PAGE_SIZE) + BUG(); + + used_space += fragment->compressed_size; + } + + if (comp_page->free_space != COMP_PAGE_SIZE - used_space) + BUG(); + + for_each_fragment(fragment_lh, comp_page) { + fragment = list_entry(fragment_lh, struct comp_cache_fragment, list); + + for_each_fragment(aux_fragment_lh, comp_page) { + aux_fragment = list_entry(aux_fragment_lh, struct comp_cache_fragment, list); + + if (aux_fragment == fragment) + continue; + + if (aux_fragment->offset == fragment->offset) + BUG(); + + if (aux_fragment->index == fragment->index) + BUG(); + + if (aux_fragment->offset < fragment->offset) { + if (aux_fragment->offset + aux_fragment->compressed_size > fragment->offset) + BUG(); + } + else { + if (fragment->offset + fragment->compressed_size > aux_fragment->offset) + BUG(); + } + } + } +} + +struct comp_cache_fragment ** +create_fragment_hash(unsigned long * fragment_hash_size, unsigned int * bits, unsigned int * order) { + struct comp_cache_fragment ** hash_table; + + for (*order = 0; (PAGE_SIZE << *order) < *fragment_hash_size; (*order)++); + + do { + unsigned long tmp = (PAGE_SIZE << *order)/sizeof(struct comp_cache_fragment *); + + *bits = 0; + while((tmp >>= 1UL) != 0UL) + (*bits)++; + + hash_table = (struct comp_cache_fragment **) __get_free_pages(GFP_ATOMIC, *order); + } while(hash_table == NULL && --(*order) > 0); + + *fragment_hash_size = 1 << (*bits); + + if (hash_table) + memset((void *) hash_table, 0, *fragment_hash_size * sizeof(struct comp_cache_fragment *)); + + return hash_table; +} + +void __init +comp_cache_hash_init(void) +{ + /* fragment hash table (code heavily based on + * page_cache_init():filemap.c */ + fragment_hash_size = NUM_FRAG_HASH_ENTRIES * sizeof(struct comp_cache_fragment *); + fragment_hash_used = 0; + fragment_hash = create_fragment_hash(&fragment_hash_size, &fragment_hash_bits, &fragment_hash_order); + + if (!fragment_hash) + panic("comp_cache_hash_init(): couldn't allocate fragment hash table\n"); + + printk("Compressed Cache: hash table\n" + "Compressed Cache: fragment (%lu entries = %luB)\n", fragment_hash_size, (PAGE_SIZE << fragment_hash_order)); + + /* inits comp cache free space hash table */ + free_space_interval = 100 * (COMP_PAGE_ORDER + 1); + free_space_hash_size = (int) (COMP_PAGE_SIZE/free_space_interval) + 2; + + free_space_hash = (struct comp_cache_page **) kmalloc(free_space_hash_size * sizeof(struct comp_cache_page *), GFP_ATOMIC); + + printk("Compressed Cache: free space (%u entries = %uB)\n", free_space_hash_size, free_space_hash_size * sizeof(struct comp_cache_page *)); + + if (!free_space_hash) + panic("comp_cache_hash_init(): couldn't allocate free space hash table\n"); + + memset((void *) free_space_hash, 0, free_space_hash_size * sizeof(struct comp_cache_page *)); + + /* inits comp cache total free space hash table */ + total_free_space_interval = 100 * (COMP_PAGE_ORDER + 1); + total_free_space_hash_size = (int) (COMP_PAGE_SIZE/free_space_interval) + 2; + + total_free_space_hash = (struct comp_cache_page **) kmalloc(total_free_space_hash_size * sizeof(struct comp_cache_page *), GFP_ATOMIC); + + printk("Compressed Cache: total free space (%u entries = %uB)\n", total_free_space_hash_size, total_free_space_hash_size * sizeof(struct comp_cache_page *)); + + if (!total_free_space_hash) + panic("comp_cache_hash_init(): couldn't allocate total free space hash table\n"); + + memset((void *) total_free_space_hash, 0, total_free_space_hash_size * sizeof(struct comp_cache_page *)); +} + +/* + * Overrides for Emacs so that we follow Linus's tabbing style. + * Emacs will notice this stuff at the end of the file and automatically + * adjust the settings for this buffer only. This must remain at the end + * of the file. + * --------------------------------------------------------------------------- + * Local variables: + * c-file-style: "linux" + * End: + */ diff -urN linux-2.4.26-rc1/mm/comp_cache/free.c linux-2.4.26-rc1-ccache/mm/comp_cache/free.c --- linux-2.4.26-rc1/mm/comp_cache/free.c Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/mm/comp_cache/free.c Sun Mar 28 17:12:21 2004 @@ -0,0 +1,410 @@ +/* + * linux/mm/comp_cache/free.c + * + * Time-stamp: <2002-10-25 11:26:26 rcastro> + * + * Linux Virtual Memory Compressed Cache + * - Free Comp Cache Entries Routines + * + * Author: Rodrigo S. de Castro + * Licensed under the GPL - 2001 + */ + +#include +#include +#include +#include +#include + +extern kmem_cache_t * fragment_cachep; + +extern void remove_fragment_vswap(struct comp_cache_fragment *); +extern void add_fragment_vswap(struct comp_cache_fragment *); + +/* is fragment1 the left neighbour of fragment2? */ +#define left_fragment(fragment1, fragment2) \ + (fragment1)->offset + (fragment1)->compressed_size == (fragment2)->offset + +/* is fragment1 the right neighbour of fragment2? */ +#define right_fragment(fragment1, fragment2) left_fragment(fragment2, fragment1) + +static inline void +merge_right_neighbour(struct comp_cache_fragment * fragment_to_free, struct comp_cache_fragment * right_fragment) +{ + if (!right_fragment) + return; + + if (fragment_freed(right_fragment)) { + fragment_to_free->compressed_size += right_fragment->compressed_size; + list_del(&(right_fragment->list)); + kmem_cache_free(fragment_cachep, (right_fragment)); + } +} + +static inline void +merge_left_neighbour(struct comp_cache_fragment * fragment_to_free, struct comp_cache_fragment * left_fragment) +{ + if (!left_fragment) + return; + + if (fragment_freed(left_fragment)) { + fragment_to_free->offset = left_fragment->offset; + fragment_to_free->compressed_size += left_fragment->compressed_size; + + list_del(&(left_fragment->list)); + kmem_cache_free(fragment_cachep, (left_fragment)); + } +} + +static inline void +remove_fragment_from_comp_cache(struct comp_cache_fragment * fragment) +{ + if (!fragment->mapping) + BUG(); + + /* fragments that have already been submitted to IO have a + * non-null swp_buffer. Let's warn the swap buffer that this + * page has been already removed by setting its fragment field + * to NULL. */ + if (fragment->swp_buffer) + fragment->swp_buffer->fragment = NULL; + + /* compressed fragments of swap cache are accounted in + * swapper_space.nrpages, so we need to account them + * separately to display sane values in /proc/meminfo */ + if (PageSwapCache(fragment)) + num_swapper_fragments--; + if (!CompFragmentDirty(fragment)) + num_clean_fragments--; + num_fragments--; + + /* total free space in compressed cache */ + comp_cache_free_space += fragment->compressed_size; + + /* used to know a fragment with zero count is actually freed + * or waiting to be freed (like when sleeping to lock its page + * in comp_cache_free()) */ + fragment->mapping = NULL; + + /* debug */ + fragment->index = 0; +} + +void +compact_fragments(struct comp_cache_page * comp_page) +{ + struct comp_cache_fragment * fragment, * min_fragment = NULL; + struct list_head * fragment_lh, * tmp_lh, aux_fragment_list; + int min_offset = COMP_PAGE_SIZE + 1, num_fragments = 0, next_offset = 0; + + INIT_LIST_HEAD(&aux_fragment_list); + + /* remove all the freed fragments */ + for_each_fragment_safe(fragment_lh, tmp_lh, comp_page) { + fragment = list_entry(fragment_lh, struct comp_cache_fragment, list); + + if (fragment_freed(fragment)) { + list_del(&(fragment->list)); + comp_page->free_space += fragment->compressed_size; + kmem_cache_free(fragment_cachep, (fragment)); + continue; + } + + /* fragment not yet freed */ + if (fragment->offset < min_offset) { + min_offset = fragment->offset; + min_fragment = fragment; + } + num_fragments++; + } + + /* compact the other fragments */ + while (num_fragments--) { + list_del(&min_fragment->list); + list_add(&min_fragment->list, &aux_fragment_list); + + memmove(page_address(comp_page->page) + next_offset, page_address(comp_page->page) + min_fragment->offset, min_fragment->compressed_size); + min_fragment->offset = next_offset; + + min_offset = COMP_PAGE_SIZE + 1; + next_offset += min_fragment->compressed_size; + + for_each_fragment(fragment_lh, comp_page) { + fragment = list_entry(fragment_lh, struct comp_cache_fragment, list); + if (fragment->offset < min_offset) { + min_offset = fragment->offset; + min_fragment = fragment; + } + } + } + + comp_page->free_offset = next_offset; + + list_for_each_safe(fragment_lh, tmp_lh, &aux_fragment_list) { + list_del(fragment_lh); + list_add(fragment_lh, &(comp_page->fragments)); + } + + + if (comp_page->free_space != comp_page->total_free_space) + BUG(); +} + +/* caller must hold comp_cache_lock lock */ +static void +comp_cache_free_locked(struct comp_cache_fragment * fragment) +{ + struct comp_cache_page * comp_page; + struct comp_cache_fragment * next_fragment, * previous_fragment; + + if (!fragment) + BUG(); + comp_page = fragment->comp_page; + if (!comp_page) + BUG(); + + /* remove from the free space hash table to update it */ + remove_comp_page_from_hash_table(comp_page); + + /* fragment is added in the correct location to the comp_page + * list (see get_comp_cache_page():swapout.c) */ + next_fragment = NULL; + if (fragment->list.next != &(comp_page->fragments)) + next_fragment = list_entry(fragment->list.next, struct comp_cache_fragment, list); + + previous_fragment = NULL; + if (fragment->list.prev != &(comp_page->fragments)) + previous_fragment = list_entry(fragment->list.prev, struct comp_cache_fragment, list); + + remove_fragment_from_comp_cache(fragment); + fragment->comp_page->total_free_space += fragment->compressed_size; + + if (fragment->mapping) + BUG(); + + /* simple case - no free space + * 1 - one not compressed page + * 2 - sum of all fragments = COMP_PAGE_SIZE */ + if (!comp_page->free_space) { + comp_page->free_offset = fragment->offset; + goto remove; + } + + /* this fragment has the free space as its left neighbour */ + if (comp_page->free_offset + comp_page->free_space == fragment->offset) { + merge_right_neighbour(fragment, next_fragment); + goto remove; + } + + /* this fragment has the free space as its right neighbour */ + if (fragment->offset + fragment->compressed_size == comp_page->free_offset) { + merge_left_neighbour(fragment, previous_fragment); + comp_page->free_offset = fragment->offset; + goto remove; + } + + merge_right_neighbour(fragment, next_fragment); + merge_left_neighbour(fragment, previous_fragment); + + goto out; + + remove: + /* remove the fragment from comp page */ + list_del(&(fragment->list)); + + /* careful: that's not only the compressed size from this + * fragment, but also the fragments that might have been + * merged in merge_*_neighbour() functions above */ + comp_page->free_space += fragment->compressed_size; + + kmem_cache_free(fragment_cachep, (fragment)); + out: + add_comp_page_to_hash_table(comp_page); +} + +/* caller must hold comp_cache_lock lock */ +static void +comp_cache_free(struct comp_cache_fragment * fragment) { + struct comp_cache_page * comp_page; + + if (!fragment) + BUG(); + + /* remove from mapping->{clean,dirty}_comp_pages */ + spin_lock(&pagecache_lock); + list_del_init(&fragment->mapping_list); + fragment->mapping->nrpages--; + spin_unlock(&pagecache_lock); + + remove_fragment_vswap(fragment); + remove_fragment_from_hash_table(fragment); + remove_fragment_from_lru_queue(fragment); + comp_page = fragment->comp_page; + + spin_unlock(&comp_cache_lock); + lock_page(comp_page->page); + spin_lock(&comp_cache_lock); + + comp_cache_free_locked(fragment); + +#ifdef CONFIG_COMP_DIS_ADAPT + UnlockPage(comp_page->page); +#else + /* *** adaptability policy *** + * + * Release the page to the system if it doesn't have another + * fragments after the above fragment got just freed. + */ + shrink_on_demand(comp_page); +#endif +} + +/* caller must hold comp_cache_lock lock */ +int +__comp_cache_free(struct comp_cache_fragment * fragment) { + int zero; + + if (!fragment_count(fragment)) + BUG(); + + if ((zero = put_fragment_testzero(fragment))) + comp_cache_free(fragment); + return zero; +} + +extern struct rw_semaphore vswap_resize_sem; + +int +comp_cache_use_address(swp_entry_t entry) +{ + struct comp_cache_fragment * fragment = NULL; + struct vswap_address * vswap; + struct list_head * vswap_lh; + swp_entry_t old_entry; + int num_freed_ptes, ret = 0; + + spin_lock(&virtual_swap_list); + + /* no virtual swap entry with a compressed page */ + if (list_empty(&vswap_address_used_head)) + goto out_unlock; + + vswap_lh = &vswap_address_used_head; + +try_again: + /* we are not supposed to sleep to get a lock in this + * function, so let's try to lock one of the virtual swap + * address pages without sleeping, ie at once, with + * TryLockPage() */ + while ((vswap_lh = vswap_lh->next) != &vswap_address_used_head) { + vswap = list_entry(vswap_lh, struct vswap_address, list); + + if (!vswap->fragment) + BUG(); + + old_entry = SWP_ENTRY(COMP_CACHE_SWP_TYPE, vswap->offset); + + if (vswap->fault_count) + continue; + + if (TryLockPage(vswap->fragment->comp_page->page)) + continue; + + /* no swap cache page? ok, let's assign the real entry */ + if (!vswap->swap_cache_page) + break; + + if (!TryLockPage(vswap->swap_cache_page)) + break; + + /* couldn't lock the swap cache page, let's try + * another page */ + UnlockPage(vswap->fragment->comp_page->page); + } + + /* no page could be locked for changes */ + if (vswap_lh == &vswap_address_used_head) + goto out_unlock; + + fragment = vswap->fragment; + + /* set old virtual addressed ptes to the real swap entry */ + spin_unlock(&virtual_swap_list); + ret = set_pte_list_to_entry(vswap->pte_list, old_entry, entry); + spin_lock(&virtual_swap_list); + + /* if we set all the pte list, but while setting to the new + * entry, a pte has faulted in, back out the changes so + * hopefully the page fault can be serviced */ + if (!ret || vswap->fault_count) + goto backout; + + spin_lock(&comp_cache_lock); + remove_fragment_vswap(fragment); + remove_fragment_from_hash_table(fragment); + spin_unlock(&comp_cache_lock); + + /* remove all those ptes from vswap struct */ + num_freed_ptes = free_pte_list(vswap->pte_list, vswap->offset); + + /* let's proceed to fix swap counter for either entries */ + for(; num_freed_ptes > 0; --num_freed_ptes) { + __virtual_swap_free(vswap->offset); + swap_duplicate(entry); + } + + /* let's fix swap cache page address (if any) */ + if (vswap->swap_cache_page) { + struct page * swap_cache_page = vswap->swap_cache_page; + + if (!PageLocked(swap_cache_page)) + BUG(); + + page_cache_get(swap_cache_page); + + spin_lock(&pagecache_lock); + __delete_from_swap_cache(swap_cache_page); + spin_unlock(&pagecache_lock); + __virtual_swap_free(vswap->offset); + + add_to_swap_cache(swap_cache_page, entry); + + UnlockPage(swap_cache_page); + page_cache_release(swap_cache_page); + } + + /* Even if the swap cache page has been removed but the + * swap_count not yet decremented, the maximum value of + * swap_count is 1. This vswap entry will be added to free + * list as soon as swap_count gets to zero. */ + if (vswap->swap_count > 1) + BUG(); + + fragment->index = entry.val; + + spin_lock(&comp_cache_lock); + add_fragment_to_active_lru_queue(fragment); + add_fragment_to_hash_table(fragment); + UnlockPage(fragment->comp_page->page); + spin_unlock(&comp_cache_lock); + out_unlock: + spin_unlock(&virtual_swap_list); + return ret; + + backout: + if (vswap->swap_cache_page) + UnlockPage(vswap->swap_cache_page); + UnlockPage(fragment->comp_page->page); + goto try_again; +} + +/* + * Overrides for Emacs so that we follow Linus's tabbing style. + * Emacs will notice this stuff at the end of the file and automatically + * adjust the settings for this buffer only. This must remain at the end + * of the file. + * --------------------------------------------------------------------------- + * Local variables: + * c-file-style: "linux" + * End: + */ diff -urN linux-2.4.26-rc1/mm/comp_cache/main.c linux-2.4.26-rc1-ccache/mm/comp_cache/main.c --- linux-2.4.26-rc1/mm/comp_cache/main.c Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/mm/comp_cache/main.c Sun Mar 28 17:12:21 2004 @@ -0,0 +1,307 @@ +/* + * linux/mm/comp_cache/main.c + * + * Time-stamp: <2002-10-25 08:54:11 rcastro> + * + * Linux Virtual Memory Compressed Cache + * + * Author: Rodrigo S. de Castro + * Licensed under the GPL - 2001 + */ + +#include +#include +#include +#include +#include + +#include + +/* compressed cache control variables */ +unsigned long num_comp_pages = 0; +unsigned long num_fragments = 0; +unsigned long num_swapper_fragments = 0; +unsigned long num_active_fragments = 0; +unsigned long num_clean_fragments = 0; +unsigned long failed_comp_page_allocs = 0; + +/* maximum number of pages that the compressed cache can use */ +unsigned long max_num_comp_pages = 0; +/* minimum number of pages that the compressed cache can use */ +unsigned long min_num_comp_pages = 0; +/* maximum number of pages ever used by the compressed cache */ +unsigned long max_used_num_comp_pages = 0; + +/* stores the last number of compressed pages that has been used to + * fix the normal zone watermarks */ +unsigned long zone_num_comp_pages = 0; + +unsigned long comp_cache_free_space; + +kmem_cache_t * comp_cachep; +kmem_cache_t * fragment_cachep; + +extern unsigned long num_physpages; +extern struct comp_cache_page * get_comp_cache_page(struct page *, unsigned short, struct comp_cache_fragment **, unsigned int, int); + +/* ugly global comp_cache_lock (only to start make it SMP-safe) */ +spinlock_t comp_cache_lock __cacheline_aligned = SPIN_LOCK_UNLOCKED; + +static int +compress_page(struct page * page, int state, unsigned int gfp_mask, int priority) +{ + struct comp_cache_page * comp_page; + struct comp_cache_fragment * fragment; + unsigned short comp_size, comp_offset; + + static struct page * current_compressed_page; + static char buffer_compressed1[MAX_COMPRESSED_SIZE]; + static char buffer_compressed2[MAX_COMPRESSED_SIZE]; + unsigned long * buffer_compressed = NULL; + + + if (!page) + BUG(); + if (!PageLocked(page)) + BUG(); + if (PageCompCache(page)) { + if (!find_comp_page(page->mapping, page->index, &fragment)) { + if (!CompFragmentToBeFreed(fragment)) + BUG(); + return 0; + } + PageClearCompCache(page); + } + + try_again: + /* don't compress a page already compressed */ + if (PageCompressed(page)) + get_comp_data(page, &comp_size, &comp_offset); + else + comp_size = compress(current_compressed_page = page, buffer_compressed = (unsigned long *) &buffer_compressed1, state); + if (comp_size > PAGE_SIZE) + BUG(); + comp_page = get_comp_cache_page(page, comp_size, &fragment, gfp_mask, priority); + + /* if comp_page == NULL, get_comp_cache_page() gave up + * reserving a swap entry for this page, so we should return + * right now, because it won't be compressed. Its dirty bit + * has been set back in get_comp_cache_page() since it's still + * dirty and needs to be cleaned. */ + if (!comp_page) + return 0; + + if (fragment->offset + fragment->compressed_size > COMP_PAGE_SIZE) + BUG(); + + /* fix mapping stuff - clean fragment */ + page->mapping->nrpages++; + if (state != DIRTY_PAGE) { + list_add(&fragment->mapping_list, &fragment->mapping->clean_comp_pages); + num_clean_fragments++; + goto copy_page; + } + + /* dirty fragment */ + CompFragmentSetDirty(fragment); + list_add(&fragment->mapping_list, &fragment->mapping->dirty_comp_pages); + + /* the inode might have been synced in the meanwhile (if we + * slept to get a free comp cache entry above), so dirty it */ + if (page->mapping->host) + mark_inode_dirty_pages(page->mapping->host); + + copy_page: + /* if the page is already compressed, we just copy it */ + if (PageCompressed(page)) { + memcpy(page_address(comp_page->page) + fragment->offset, page_address(page) + comp_offset, comp_size); + goto out; + } + + if (compressed(fragment)) { + if (current_compressed_page != page) { + comp_size = compress(page, buffer_compressed = (unsigned long *) &buffer_compressed2, state); + if (comp_size != fragment->compressed_size) { + UnlockPage(comp_page->page); + drop_fragment(fragment); + goto try_again; + } + } + memcpy(page_address(comp_page->page) + fragment->offset, buffer_compressed , fragment->compressed_size); + } else + memcpy(page_address(comp_page->page) + fragment->offset, page_address(page), PAGE_SIZE); + + out: + if (PageTestandSetCompCache(page)) + BUG(); + UnlockPage(comp_page->page); + return 1; +} + +int +compress_dirty_page(struct page * page, int (*writepage)(struct page *), unsigned int gfp_mask, int priority) +{ + int write = 0, ret = 0; + + if (page->buffers) + write = !try_to_free_buffers(page, 0); +#ifdef CONFIG_COMP_PAGE_CACHE + write |= shmem_page(page); +#else + write |= !PageSwapCache(page); +#endif + if (write) { + /* if gfp_mask does not allow us to write out the + * page, unlock the page and set all the bits back */ + if (!(gfp_mask & __GFP_FS)) + goto set_bits_back; + writepage(page); + return 0; + } + + if (page->buffers) + BUG(); + + spin_lock(&comp_cache_lock); + ret = compress_page(page, DIRTY_PAGE, gfp_mask, priority); + spin_unlock(&comp_cache_lock); + + /* failed to compress the dirty page? set the bits back */ + if (!ret) { + set_bits_back: + spin_lock(&pagemap_lru_lock); + SetPageDirty(page); + ClearPageLaunder(page); + UnlockPage(page); + spin_unlock(&pagemap_lru_lock); + } + return ret; +} + +int +compress_clean_page(struct page * page, unsigned int gfp_mask, int priority) +{ + int ret; + + if (page->buffers) + BUG(); + +#ifndef CONFIG_COMP_PAGE_CACHE + if (!PageSwapCache(page)) + return 1; +#endif + spin_lock(&comp_cache_lock); + ret = compress_page(page, CLEAN_PAGE, gfp_mask, priority); + spin_unlock(&comp_cache_lock); + + return ret; +} + +extern void __init comp_cache_hash_init(void); +extern void __init comp_cache_swp_buffer_init(void); +extern void __init comp_cache_vswap_init(void); +extern void __init comp_cache_adaptivity_init(void); + +LIST_HEAD(active_lru_queue); +LIST_HEAD(inactive_lru_queue); + +inline int +init_comp_page(struct comp_cache_page ** comp_page,struct page * page) { + *comp_page = alloc_comp_cache(); + + if (!(*comp_page)) + return 0; + + (*comp_page)->free_space = (*comp_page)->total_free_space = (COMP_PAGE_ORDER + 1) * PAGE_SIZE; + (*comp_page)->free_offset = 0; + (*comp_page)->page = page; + INIT_LIST_HEAD(&((*comp_page)->fragments)); + add_comp_page_to_hash_table((*comp_page)); + return 1; +} + +void __init +comp_cache_init(void) +{ + struct comp_cache_page * comp_page; + struct page * page; + int i; + + printk("Compressed Cache: %s\n", COMP_CACHE_VERSION); + +#ifdef CONFIG_COMP_DIS_ADAPT + /* static compressed cache */ + min_num_comp_pages = page_to_comp_page(48); + + if (!max_num_comp_pages || max_num_comp_pages < min_num_comp_pages || max_num_comp_pages > num_physpages * 0.5) + max_num_comp_pages = page_to_comp_page((unsigned long) (num_physpages * 0.5)); + + max_used_num_comp_pages = num_comp_pages = max_num_comp_pages; + printk("Compressed Cache: static size\n"); +#else + /* adaptive compressed cache */ + max_used_num_comp_pages = min_num_comp_pages = num_comp_pages = page_to_comp_page(48); + + if (!max_num_comp_pages || max_num_comp_pages < min_num_comp_pages || max_num_comp_pages > num_physpages * 0.5) + max_num_comp_pages = page_to_comp_page((unsigned long) (num_physpages * 0.5)); + + printk("Compressed Cache: maximum size\n"); +#endif + printk("Compressed Cache: %lu pages = %luKiB\n", + max_num_comp_pages, (max_num_comp_pages * COMP_PAGE_SIZE) >> 10); + + /* fiz zone watermarks */ + comp_cache_fix_watermarks(num_comp_pages); + + /* create slab caches */ + comp_cachep = kmem_cache_create("comp_cache_struct", sizeof(struct comp_cache_page), 0, SLAB_HWCACHE_ALIGN, NULL, NULL); + fragment_cachep = kmem_cache_create("comp_cache_frag", sizeof(struct comp_cache_fragment), 0, SLAB_HWCACHE_ALIGN, NULL, NULL); + + comp_cache_hash_init(); + + /* a buffer to be used to decompress page data */ + comp_cache_swp_buffer_init(); + + comp_cache_vswap_init(); + + /* initialize each comp cache entry */ + for (i = 0; i < num_comp_pages; i++) { + page = alloc_pages(GFP_KERNEL, COMP_PAGE_ORDER); + + if (!init_comp_page(&comp_page, page)) + __free_pages(page, COMP_PAGE_ORDER); + } + comp_cache_free_space = num_comp_pages * COMP_PAGE_SIZE; + + /* initialize our algorithms statistics array */ + comp_cache_algorithms_init(); + +#ifndef CONFIG_COMP_DIS_ADAPT + comp_cache_adaptivity_init(); +#endif +} + + +static int __init comp_cache_size(char *str) +{ + unsigned long nr_pages; + char * endp; + + nr_pages = memparse(str, &endp) >> (PAGE_SHIFT + COMP_PAGE_ORDER); + + max_num_comp_pages = nr_pages; + return 1; +} + +__setup("compsize=", comp_cache_size); + +/* + * Overrides for Emacs so that we follow Linus's tabbing style. + * Emacs will notice this stuff at the end of the file and automatically + * adjust the settings for this buffer only. This must remain at the end + * of the file. + * --------------------------------------------------------------------------- + * Local variables: + * c-file-style: "linux" + * End: + */ diff -urN linux-2.4.26-rc1/mm/comp_cache/minilzo.c linux-2.4.26-rc1-ccache/mm/comp_cache/minilzo.c --- linux-2.4.26-rc1/mm/comp_cache/minilzo.c Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/mm/comp_cache/minilzo.c Sun Mar 28 17:12:21 2004 @@ -0,0 +1,2849 @@ +/* minilzo.c -- mini subset of the LZO real-time data compression library + + This file is part of the LZO real-time data compression library. + + Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer + Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer + Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer + Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer + Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer + + The LZO library is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + The LZO library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with the LZO library; see the file COPYING. + If not, write to the Free Software Foundation, Inc., + 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + Markus F.X.J. Oberhumer + + http://wildsau.idv.uni-linz.ac.at/mfx/lzo.html + */ + +/* + * NOTE: + * the full LZO package can be found at + * http://wildsau.idv.uni-linz.ac.at/mfx/lzo.html + */ + +#define __LZO_IN_MINILZO + +//#ifdef MINILZO_HAVE_CONFIG_H +//# include +//#endif + +#undef LZO_HAVE_CONFIG_H +#include +#include +#include + +#if !defined(MINILZO_VERSION) || (MINILZO_VERSION != 0x1070) +# error "version mismatch in miniLZO source files" +#endif + +#ifdef MINILZO_HAVE_CONFIG_H +# define LZO_HAVE_CONFIG_H +#endif + +/* #if !defined(LZO_NO_SYS_TYPES_H) */ +/* # include */ +/* #endif */ +/* #include */ + +#ifndef __LZO_CONF_H +#define __LZO_CONF_H + +#if !defined(__LZO_IN_MINILZO) +# ifndef __LZOCONF_H +# include +# endif +#endif + +#if defined(__BOUNDS_CHECKING_ON) +//# include +#else +# define BOUNDS_CHECKING_OFF_DURING(stmt) stmt +# define BOUNDS_CHECKING_OFF_IN_EXPR(expr) (expr) +#endif + +#if !defined(LZO_HAVE_CONFIG_H) +# include +# include +# define HAVE_MEMCMP +# define HAVE_MEMCPY +# define HAVE_MEMMOVE +# define HAVE_MEMSET +#else +# include +# if defined(STDC_HEADERS) +# include +//# include +# endif +# if defined(HAVE_STDDEF_H) +# include +# endif +# if defined(HAVE_MEMORY_H) +//# include +# endif +#endif + +#if defined(__LZO_DOS16) || defined(__LZO_WIN16) +# define HAVE_MALLOC_H +# define HAVE_HALLOC +#endif + +#undef NDEBUG +#if !defined(LZO_DEBUG) +# define NDEBUG +#endif +#if defined(LZO_DEBUG) || !defined(NDEBUG) +# if !defined(NO_STDIO_H) +//# include +# endif +#endif +//#include + +#define assert(condition) do { if (unlikely(!(condition))) BUG(); } while(0) + +#if !defined(LZO_UNUSED) +# define LZO_UNUSED(parm) (parm = parm) +#endif + +#if !defined(__inline__) && !defined(__GNUC__) +# if defined(__cplusplus) +# define __inline__ inline +# else +# define __inline__ +# endif +#endif + +#if defined(NO_MEMCMP) +# undef HAVE_MEMCMP +#endif + +#if !defined(HAVE_MEMCMP) +# undef memcmp +# define memcmp lzo_memcmp +#endif +#if !defined(HAVE_MEMCPY) +# undef memcpy +# define memcpy lzo_memcpy +#endif +#if !defined(HAVE_MEMMOVE) +# undef memmove +# define memmove lzo_memmove +#endif +#if !defined(HAVE_MEMSET) +# undef memset +# define memset lzo_memset +#endif + +#if 1 +# define LZO_BYTE(x) ((unsigned char) (x)) +#else +# define LZO_BYTE(x) ((unsigned char) ((x) & 0xff)) +#endif +#if 0 +# define LZO_USHORT(x) ((unsigned short) (x)) +#else +# define LZO_USHORT(x) ((unsigned short) ((x) & 0xffff)) +#endif + +#define LZO_MAX(a,b) ((a) >= (b) ? (a) : (b)) +#define LZO_MIN(a,b) ((a) <= (b) ? (a) : (b)) +#define LZO_MAX3(a,b,c) ((a) >= (b) ? LZO_MAX(a,c) : LZO_MAX(b,c)) +#define LZO_MIN3(a,b,c) ((a) <= (b) ? LZO_MIN(a,c) : LZO_MIN(b,c)) + +#define lzo_sizeof(type) ((lzo_uint) (sizeof(type))) + +#define LZO_HIGH(array) ((lzo_uint) (sizeof(array)/sizeof(*(array)))) + +#define LZO_SIZE(bits) (1u << (bits)) +#define LZO_MASK(bits) (LZO_SIZE(bits) - 1) + +#define LZO_LSIZE(bits) (1ul << (bits)) +#define LZO_LMASK(bits) (LZO_LSIZE(bits) - 1) + +#define LZO_USIZE(bits) ((lzo_uint) 1 << (bits)) +#define LZO_UMASK(bits) (LZO_USIZE(bits) - 1) + +#define LZO_STYPE_MAX(b) (((1l << (8*(b)-2)) - 1l) + (1l << (8*(b)-2))) +#define LZO_UTYPE_MAX(b) (((1ul << (8*(b)-1)) - 1ul) + (1ul << (8*(b)-1))) + +#if !defined(SIZEOF_UNSIGNED) +# if (UINT_MAX == 0xffff) +# define SIZEOF_UNSIGNED 2 +# elif (UINT_MAX == LZO_0xffffffffL) +# define SIZEOF_UNSIGNED 4 +# elif (UINT_MAX >= LZO_0xffffffffL) +# define SIZEOF_UNSIGNED 8 +# else +# error SIZEOF_UNSIGNED +# endif +#endif + +#if !defined(SIZEOF_UNSIGNED_LONG) +# if (ULONG_MAX == LZO_0xffffffffL) +# define SIZEOF_UNSIGNED_LONG 4 +# elif (ULONG_MAX >= LZO_0xffffffffL) +# define SIZEOF_UNSIGNED_LONG 8 +# else +# error SIZEOF_UNSIGNED_LONG +# endif +#endif + +#if !defined(SIZEOF_SIZE_T) +# define SIZEOF_SIZE_T SIZEOF_UNSIGNED +#endif +#if !defined(SIZE_T_MAX) +# define SIZE_T_MAX LZO_UTYPE_MAX(SIZEOF_SIZE_T) +#endif + +#if 1 && defined(__LZO_i386) && (UINT_MAX == LZO_0xffffffffL) +# if !defined(LZO_UNALIGNED_OK_2) && (USHRT_MAX == 0xffff) +# define LZO_UNALIGNED_OK_2 +# endif +# if !defined(LZO_UNALIGNED_OK_4) && (LZO_UINT32_MAX == LZO_0xffffffffL) +# define LZO_UNALIGNED_OK_4 +# endif +#endif + +#if defined(LZO_UNALIGNED_OK_2) || defined(LZO_UNALIGNED_OK_4) +# if !defined(LZO_UNALIGNED_OK) +# define LZO_UNALIGNED_OK +# endif +#endif + +#if defined(__LZO_NO_UNALIGNED) +# undef LZO_UNALIGNED_OK +# undef LZO_UNALIGNED_OK_2 +# undef LZO_UNALIGNED_OK_4 +#endif + +#if defined(LZO_UNALIGNED_OK_2) && (USHRT_MAX != 0xffff) +# error "LZO_UNALIGNED_OK_2 must not be defined on this system" +#endif +#if defined(LZO_UNALIGNED_OK_4) && (LZO_UINT32_MAX != LZO_0xffffffffL) +# error "LZO_UNALIGNED_OK_4 must not be defined on this system" +#endif + +#if defined(__LZO_NO_ALIGNED) +# undef LZO_ALIGNED_OK_4 +#endif + +#if defined(LZO_ALIGNED_OK_4) && (LZO_UINT32_MAX != LZO_0xffffffffL) +# error "LZO_ALIGNED_OK_4 must not be defined on this system" +#endif + +#define LZO_LITTLE_ENDIAN 1234 +#define LZO_BIG_ENDIAN 4321 +#define LZO_PDP_ENDIAN 3412 + +#if !defined(LZO_BYTE_ORDER) +# if defined(MFX_BYTE_ORDER) +# define LZO_BYTE_ORDER MFX_BYTE_ORDER +# elif defined(__LZO_i386) +# define LZO_BYTE_ORDER LZO_LITTLE_ENDIAN +# elif defined(BYTE_ORDER) +# define LZO_BYTE_ORDER BYTE_ORDER +# elif defined(__BYTE_ORDER) +# define LZO_BYTE_ORDER __BYTE_ORDER +# endif +#endif + +#if defined(LZO_BYTE_ORDER) +# if (LZO_BYTE_ORDER != LZO_LITTLE_ENDIAN) && \ + (LZO_BYTE_ORDER != LZO_BIG_ENDIAN) +# error "invalid LZO_BYTE_ORDER" +# endif +#endif + +#if defined(LZO_UNALIGNED_OK) && !defined(LZO_BYTE_ORDER) +# error "LZO_BYTE_ORDER is not defined" +#endif + +#define LZO_OPTIMIZE_GNUC_i386_IS_BUGGY + +#if defined(NDEBUG) && !defined(LZO_DEBUG) && !defined(__LZO_CHECKER) +# if defined(__GNUC__) && defined(__i386__) +# if !defined(LZO_OPTIMIZE_GNUC_i386_IS_BUGGY) +# define LZO_OPTIMIZE_GNUC_i386 +# endif +# endif +#endif + +__LZO_EXTERN_C int __lzo_init_done; +__LZO_EXTERN_C const lzo_byte __lzo_copyright[]; +LZO_EXTERN(const lzo_byte *) lzo_copyright(void); +__LZO_EXTERN_C const lzo_uint32 _lzo_crc32_table[256]; + +#define _LZO_STRINGIZE(x) #x +#define _LZO_MEXPAND(x) _LZO_STRINGIZE(x) + +#define _LZO_CONCAT2(a,b) a ## b +#define _LZO_CONCAT3(a,b,c) a ## b ## c +#define _LZO_CONCAT4(a,b,c,d) a ## b ## c ## d +#define _LZO_CONCAT5(a,b,c,d,e) a ## b ## c ## d ## e + +#define _LZO_ECONCAT2(a,b) _LZO_CONCAT2(a,b) +#define _LZO_ECONCAT3(a,b,c) _LZO_CONCAT3(a,b,c) +#define _LZO_ECONCAT4(a,b,c,d) _LZO_CONCAT4(a,b,c,d) +#define _LZO_ECONCAT5(a,b,c,d,e) _LZO_CONCAT5(a,b,c,d,e) + +#if 0 + +#define __LZO_IS_COMPRESS_QUERY(i,il,o,ol,w) ((lzo_voidp)(o) == (w)) +#define __LZO_QUERY_COMPRESS(i,il,o,ol,w,n,s) \ + (*ol = (n)*(s), LZO_E_OK) + +#define __LZO_IS_DECOMPRESS_QUERY(i,il,o,ol,w) ((lzo_voidp)(o) == (w)) +#define __LZO_QUERY_DECOMPRESS(i,il,o,ol,w,n,s) \ + (*ol = (n)*(s), LZO_E_OK) + +#define __LZO_IS_OPTIMIZE_QUERY(i,il,o,ol,w) ((lzo_voidp)(o) == (w)) +#define __LZO_QUERY_OPTIMIZE(i,il,o,ol,w,n,s) \ + (*ol = (n)*(s), LZO_E_OK) + +#endif + +#ifndef __LZO_PTR_H +#define __LZO_PTR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(__LZO_DOS16) || defined(__LZO_WIN16) +//# include +#if 1 && defined(__WATCOMC__) +//# include + __LZO_EXTERN_C unsigned char _HShift; +# define __LZO_HShift _HShift +# elif 1 && defined(_MSC_VER) + __LZO_EXTERN_C unsigned short __near _AHSHIFT; +# define __LZO_HShift ((unsigned) &_AHSHIFT) +# elif defined(__LZO_WIN16) +# define __LZO_HShift 3 +# else +# define __LZO_HShift 12 +# endif +# if !defined(_FP_SEG) && defined(FP_SEG) +# define _FP_SEG FP_SEG +# endif +# if !defined(_FP_OFF) && defined(FP_OFF) +# define _FP_OFF FP_OFF +# endif +#endif + +#if (UINT_MAX >= LZO_0xffffffffL) + typedef ptrdiff_t lzo_ptrdiff_t; +#else + typedef long lzo_ptrdiff_t; +#endif + +#if !defined(__LZO_HAVE_PTR_T) +# if defined(lzo_ptr_t) +# define __LZO_HAVE_PTR_T +# endif +#endif +#if !defined(__LZO_HAVE_PTR_T) +# if defined(SIZEOF_CHAR_P) && defined(SIZEOF_UNSIGNED_LONG) +# if (SIZEOF_CHAR_P == SIZEOF_UNSIGNED_LONG) + typedef unsigned long lzo_ptr_t; + typedef long lzo_sptr_t; +# define __LZO_HAVE_PTR_T +# endif +# endif +#endif +#if !defined(__LZO_HAVE_PTR_T) +# if defined(SIZEOF_CHAR_P) && defined(SIZEOF_UNSIGNED) +# if (SIZEOF_CHAR_P == SIZEOF_UNSIGNED) + typedef unsigned int lzo_ptr_t; + typedef int lzo_sptr_t; +# define __LZO_HAVE_PTR_T +# endif +# endif +#endif +#if !defined(__LZO_HAVE_PTR_T) +# if defined(SIZEOF_CHAR_P) && defined(SIZEOF_UNSIGNED_SHORT) +# if (SIZEOF_CHAR_P == SIZEOF_UNSIGNED_SHORT) + typedef unsigned short lzo_ptr_t; + typedef short lzo_sptr_t; +# define __LZO_HAVE_PTR_T +# endif +# endif +#endif +#if !defined(__LZO_HAVE_PTR_T) +# if defined(LZO_HAVE_CONFIG_H) || defined(SIZEOF_CHAR_P) +# error "no suitable type for lzo_ptr_t" +# else + typedef unsigned long lzo_ptr_t; + typedef long lzo_sptr_t; +# define __LZO_HAVE_PTR_T +# endif +#endif + +#if defined(__LZO_DOS16) || defined(__LZO_WIN16) +#define PTR(a) ((lzo_bytep) (a)) +#define PTR_ALIGNED_4(a) ((_FP_OFF(a) & 3) == 0) +#define PTR_ALIGNED2_4(a,b) (((_FP_OFF(a) | _FP_OFF(b)) & 3) == 0) +#else +#define PTR(a) ((lzo_ptr_t) (a)) +#define PTR_LINEAR(a) PTR(a) +#define PTR_ALIGNED_4(a) ((PTR_LINEAR(a) & 3) == 0) +#define PTR_ALIGNED_8(a) ((PTR_LINEAR(a) & 7) == 0) +#define PTR_ALIGNED2_4(a,b) (((PTR_LINEAR(a) | PTR_LINEAR(b)) & 3) == 0) +#define PTR_ALIGNED2_8(a,b) (((PTR_LINEAR(a) | PTR_LINEAR(b)) & 7) == 0) +#endif + +#define PTR_LT(a,b) (PTR(a) < PTR(b)) +#define PTR_GE(a,b) (PTR(a) >= PTR(b)) +#define PTR_DIFF(a,b) ((lzo_ptrdiff_t) (PTR(a) - PTR(b))) + +LZO_EXTERN(lzo_ptr_t) +__lzo_ptr_linear(const lzo_voidp ptr); + +typedef union +{ + char a_char; + unsigned char a_uchar; + short a_short; + unsigned short a_ushort; + int a_int; + unsigned int a_uint; + long a_long; + unsigned long a_ulong; + lzo_int a_lzo_int; + lzo_uint a_lzo_uint; + lzo_int32 a_lzo_int32; + lzo_uint32 a_lzo_uint32; + ptrdiff_t a_ptrdiff_t; + lzo_ptrdiff_t a_lzo_ptrdiff_t; + lzo_ptr_t a_lzo_ptr_t; + char * a_charp; + lzo_bytep a_lzo_bytep; + lzo_bytepp a_lzo_bytepp; +} +lzo_align_t; + +#ifdef __cplusplus +} +#endif + +#endif + +#define LZO_DETERMINISTIC + +#define LZO_DICT_USE_PTR +#if defined(__LZO_DOS16) || defined(__LZO_WIN16) || defined(__LZO_STRICT_16BIT) +# undef LZO_DICT_USE_PTR +#endif + +#if defined(LZO_DICT_USE_PTR) +# define lzo_dict_t const lzo_bytep +# define lzo_dict_p lzo_dict_t __LZO_MMODEL * +#else +# define lzo_dict_t lzo_uint +# define lzo_dict_p lzo_dict_t __LZO_MMODEL * +#endif + +#if !defined(lzo_moff_t) +#define lzo_moff_t lzo_uint +#endif + +#endif + +LZO_PUBLIC(lzo_ptr_t) +__lzo_ptr_linear(const lzo_voidp ptr) +{ + lzo_ptr_t p; + +#if defined(__LZO_DOS16) || defined(__LZO_WIN16) + p = (((lzo_ptr_t)(_FP_SEG(ptr))) << (16 - __LZO_HShift)) + (_FP_OFF(ptr)); +#else + p = PTR_LINEAR(ptr); +#endif + + return p; +} + +LZO_PUBLIC(unsigned) +__lzo_align_gap(const lzo_voidp ptr, lzo_uint size) +{ + lzo_ptr_t p, s, n; + + assert(size > 0); + + p = __lzo_ptr_linear(ptr); + s = (lzo_ptr_t) (size - 1); +#if 0 + assert((size & (size - 1)) == 0); + n = ((p + s) & ~s) - p; +#else + n = (((p + s) / size) * size) - p; +#endif + + assert((long)n >= 0); + assert(n <= s); + + return (unsigned)n; +} + +#ifndef __LZO_UTIL_H +#define __LZO_UTIL_H + +#ifndef __LZO_CONF_H +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#if 1 && defined(HAVE_MEMCPY) +#if !defined(__LZO_DOS16) && !defined(__LZO_WIN16) + +#define MEMCPY8_DS(dest,src,len) \ + memcpy(dest,src,len); \ + dest += len; \ + src += len + +#endif +#endif + +#if 0 && !defined(MEMCPY8_DS) + +#define MEMCPY8_DS(dest,src,len) \ + { do { \ + *dest++ = *src++; \ + *dest++ = *src++; \ + *dest++ = *src++; \ + *dest++ = *src++; \ + *dest++ = *src++; \ + *dest++ = *src++; \ + *dest++ = *src++; \ + *dest++ = *src++; \ + len -= 8; \ + } while (len > 0); } + +#endif + +#if !defined(MEMCPY8_DS) + +#define MEMCPY8_DS(dest,src,len) \ + { register lzo_uint __l = (len) / 8; \ + do { \ + *dest++ = *src++; \ + *dest++ = *src++; \ + *dest++ = *src++; \ + *dest++ = *src++; \ + *dest++ = *src++; \ + *dest++ = *src++; \ + *dest++ = *src++; \ + *dest++ = *src++; \ + } while (--__l > 0); } + +#endif + +#define MEMCPY_DS(dest,src,len) \ + do *dest++ = *src++; \ + while (--len > 0) + +#define MEMMOVE_DS(dest,src,len) \ + do *dest++ = *src++; \ + while (--len > 0) + +#if 0 && defined(LZO_OPTIMIZE_GNUC_i386) + +#define BZERO8_PTR(s,l,n) \ +__asm__ __volatile__( \ + "movl %0,%%eax \n" \ + "movl %1,%%edi \n" \ + "movl %2,%%ecx \n" \ + "cld \n" \ + "rep \n" \ + "stosl %%eax,(%%edi) \n" \ + : \ + :"g" (0),"g" (s),"g" (n) \ + :"eax","edi","ecx", "memory", "cc" \ +) + +#elif (LZO_UINT_MAX <= SIZE_T_MAX) && defined(HAVE_MEMSET) + +#if 1 +#define BZERO8_PTR(s,l,n) memset((s),0,(lzo_uint)(l)*(n)) +#else +#define BZERO8_PTR(s,l,n) memset((lzo_voidp)(s),0,(lzo_uint)(l)*(n)) +#endif + +#else + +#define BZERO8_PTR(s,l,n) \ + lzo_memset((lzo_voidp)(s),0,(lzo_uint)(l)*(n)) + +#endif + +#if 0 +#if defined(__GNUC__) && defined(__i386__) + +unsigned char lzo_rotr8(unsigned char value, int shift); +extern __inline__ unsigned char lzo_rotr8(unsigned char value, int shift) +{ + unsigned char result; + + __asm__ __volatile__ ("movb %b1, %b0; rorb %b2, %b0" + : "=a"(result) : "g"(value), "c"(shift)); + return result; +} + +unsigned short lzo_rotr16(unsigned short value, int shift); +extern __inline__ unsigned short lzo_rotr16(unsigned short value, int shift) +{ + unsigned short result; + + __asm__ __volatile__ ("movw %b1, %b0; rorw %b2, %b0" + : "=a"(result) : "g"(value), "c"(shift)); + return result; +} + +#endif +#endif + +#ifdef __cplusplus +} +#endif + +#endif + +LZO_PUBLIC(lzo_bool) +lzo_assert(int expr) +{ + return (expr) ? 1 : 0; +} + +/* If you use the LZO library in a product, you *must* keep this + * copyright string in the executable of your product. + */ + +const lzo_byte __lzo_copyright[] = +#if !defined(__LZO_IN_MINLZO) + LZO_VERSION_STRING; +#else + "\n\n\n" + "LZO real-time data compression library.\n" + "Copyright (C) 1996, 1997, 1998, 1999, 2000 Markus Franz Xaver Johannes Oberhumer\n" + "\n" + "http://wildsau.idv.uni-linz.ac.at/mfx/lzo.html\n" + "\n" + "LZO version: v" LZO_VERSION_STRING ", " LZO_VERSION_DATE "\n" + "LZO build date: " __DATE__ " " __TIME__ "\n\n" + "LZO special compilation options:\n" +#ifdef __cplusplus + " __cplusplus\n" +#endif +#if defined(__PIC__) + " __PIC__\n" +#elif defined(__pic__) + " __pic__\n" +#endif +#if (UINT_MAX < LZO_0xffffffffL) + " 16BIT\n" +#endif +#if defined(__LZO_STRICT_16BIT) + " __LZO_STRICT_16BIT\n" +#endif +#if (UINT_MAX > LZO_0xffffffffL) + " UINT_MAX=" _LZO_MEXPAND(UINT_MAX) "\n" +#endif +#if (ULONG_MAX > LZO_0xffffffffL) + " ULONG_MAX=" _LZO_MEXPAND(ULONG_MAX) "\n" +#endif +#if defined(LZO_BYTE_ORDER) + " LZO_BYTE_ORDER=" _LZO_MEXPAND(LZO_BYTE_ORDER) "\n" +#endif +#if defined(LZO_UNALIGNED_OK_2) + " LZO_UNALIGNED_OK_2\n" +#endif +#if defined(LZO_UNALIGNED_OK_4) + " LZO_UNALIGNED_OK_4\n" +#endif +#if defined(LZO_ALIGNED_OK_4) + " LZO_ALIGNED_OK_4\n" +#endif +#if defined(LZO_DICT_USE_PTR) + " LZO_DICT_USE_PTR\n" +#endif +#if defined(__LZO_QUERY_COMPRESS) + " __LZO_QUERY_COMPRESS\n" +#endif +#if defined(__LZO_QUERY_DECOMPRESS) + " __LZO_QUERY_DECOMPRESS\n" +#endif +#if defined(__LZO_IN_MINILZO) + " __LZO_IN_MINILZO\n" +#endif + "\n\n" + "$Id: LZO " LZO_VERSION_STRING " built " __DATE__ " " __TIME__ +#if defined(__GNUC__) && defined(__VERSION__) + " by gcc " __VERSION__ +#elif defined(__BORLANDC__) + " by Borland C " _LZO_MEXPAND(__BORLANDC__) +#elif defined(_MSC_VER) + " by Microsoft C " _LZO_MEXPAND(_MSC_VER) +#elif defined(__PUREC__) + " by Pure C " _LZO_MEXPAND(__PUREC__) +#elif defined(__SC__) + " by Symantec C " _LZO_MEXPAND(__SC__) +#elif defined(__TURBOC__) + " by Turbo C " _LZO_MEXPAND(__TURBOC__) +#elif defined(__WATCOMC__) + " by Watcom C " _LZO_MEXPAND(__WATCOMC__) +#endif + " $\n" + "$Copyright: LZO (C) 1996, 1997, 1998, 1999, 2000 Markus Franz Xaver Johannes Oberhumer $\n"; +#endif + +LZO_PUBLIC(const lzo_byte *) +lzo_copyright(void) +{ + return __lzo_copyright; +} + +LZO_PUBLIC(unsigned) +lzo_version(void) +{ + return LZO_VERSION; +} + +LZO_PUBLIC(const char *) +lzo_version_string(void) +{ + return LZO_VERSION_STRING; +} + +LZO_PUBLIC(const char *) +lzo_version_date(void) +{ + return LZO_VERSION_DATE; +} + +LZO_PUBLIC(const lzo_charp) +_lzo_version_string(void) +{ + return LZO_VERSION_STRING; +} + +LZO_PUBLIC(const lzo_charp) +_lzo_version_date(void) +{ + return LZO_VERSION_DATE; +} + +#define LZO_BASE 65521u +#define LZO_NMAX 5552 + +#define LZO_DO1(buf,i) {s1 += buf[i]; s2 += s1;} +#define LZO_DO2(buf,i) LZO_DO1(buf,i); LZO_DO1(buf,i+1); +#define LZO_DO4(buf,i) LZO_DO2(buf,i); LZO_DO2(buf,i+2); +#define LZO_DO8(buf,i) LZO_DO4(buf,i); LZO_DO4(buf,i+4); +#define LZO_DO16(buf,i) LZO_DO8(buf,i); LZO_DO8(buf,i+8); + +LZO_PUBLIC(lzo_uint32) +lzo_adler32(lzo_uint32 adler, const lzo_byte *buf, lzo_uint len) +{ + lzo_uint32 s1 = adler & 0xffff; + lzo_uint32 s2 = (adler >> 16) & 0xffff; + int k; + + if (buf == NULL) + return 1; + + while (len > 0) + { + k = len < LZO_NMAX ? (int) len : LZO_NMAX; + len -= k; + if (k >= 16) do + { + LZO_DO16(buf,0); + buf += 16; + k -= 16; + } while (k >= 16); + if (k != 0) do + { + s1 += *buf++; + s2 += s1; + } while (--k > 0); + s1 %= LZO_BASE; + s2 %= LZO_BASE; + } + return (s2 << 16) | s1; +} + +LZO_PUBLIC(int) +lzo_memcmp(const lzo_voidp s1, const lzo_voidp s2, lzo_uint len) +{ +#if (LZO_UINT_MAX <= SIZE_T_MAX) && defined(HAVE_MEMCMP) + return memcmp(s1,s2,len); +#else + const lzo_byte *p1 = (const lzo_byte *) s1; + const lzo_byte *p2 = (const lzo_byte *) s2; + int d; + + if (len > 0) do + { + d = *p1 - *p2; + if (d != 0) + return d; + p1++; + p2++; + } + while (--len > 0); + return 0; +#endif +} + +LZO_PUBLIC(lzo_voidp) +lzo_memcpy(lzo_voidp dest, const lzo_voidp src, lzo_uint len) +{ +#if (LZO_UINT_MAX <= SIZE_T_MAX) && defined(HAVE_MEMCPY) + return memcpy(dest,src,len); +#else + lzo_byte *p1 = (lzo_byte *) dest; + const lzo_byte *p2 = (const lzo_byte *) src; + + if (len <= 0 || p1 == p2) + return dest; + do + *p1++ = *p2++; + while (--len > 0); + return dest; +#endif +} + +LZO_PUBLIC(lzo_voidp) +lzo_memmove(lzo_voidp dest, const lzo_voidp src, lzo_uint len) +{ +#if (LZO_UINT_MAX <= SIZE_T_MAX) && defined(HAVE_MEMMOVE) + return memmove(dest,src,len); +#else + lzo_byte *p1 = (lzo_byte *) dest; + const lzo_byte *p2 = (const lzo_byte *) src; + + if (len <= 0 || p1 == p2) + return dest; + + if (p1 < p2) + { + do + *p1++ = *p2++; + while (--len > 0); + } + else + { + p1 += len; + p2 += len; + do + *--p1 = *--p2; + while (--len > 0); + } + return dest; +#endif +} + +LZO_PUBLIC(lzo_voidp) +lzo_memset(lzo_voidp s, int c, lzo_uint len) +{ +#if (LZO_UINT_MAX <= SIZE_T_MAX) && defined(HAVE_MEMSET) + return memset(s,c,len); +#else + lzo_byte *p = (lzo_byte *) s; + + if (len > 0) do + *p++ = LZO_BYTE(c); + while (--len > 0); + return s; +#endif +} + +//#include + +#if 0 +# define IS_SIGNED(type) (((type) (1ul << (8 * sizeof(type) - 1))) < 0) +# define IS_UNSIGNED(type) (((type) (1ul << (8 * sizeof(type) - 1))) > 0) +#else +# define IS_SIGNED(type) (((type) (-1)) < ((type) 0)) +# define IS_UNSIGNED(type) (((type) (-1)) > ((type) 0)) +#endif + +static lzo_bool schedule_insns_bug(void); +static lzo_bool strength_reduce_bug(int *); + +#if 0 || defined(LZO_DEBUG) +static lzo_bool __lzo_assert_fail(const char *s, unsigned line) +{ +#if defined(__palmos__) + printf("LZO assertion failed in line %u: '%s'\n",line,s); +#else + fprintf(stderr,"LZO assertion failed in line %u: '%s'\n",line,s); +#endif + return 0; +} +# define __lzo_assert(x) ((x) ? 1 : __lzo_assert_fail(#x,__LINE__)) +#else +# define __lzo_assert(x) ((x) ? 1 : 0) +#endif + +static lzo_bool basic_integral_check(void) +{ + lzo_bool r = 1; + lzo_bool sanity; + + r &= __lzo_assert(CHAR_BIT == 8); + r &= __lzo_assert(sizeof(char) == 1); + r &= __lzo_assert(sizeof(short) >= 2); + r &= __lzo_assert(sizeof(long) >= 4); + r &= __lzo_assert(sizeof(int) >= sizeof(short)); + r &= __lzo_assert(sizeof(long) >= sizeof(int)); + + r &= __lzo_assert(sizeof(lzo_uint32) >= 4); + r &= __lzo_assert(sizeof(lzo_uint32) >= sizeof(unsigned)); +#if defined(__LZO_STRICT_16BIT) + r &= __lzo_assert(sizeof(lzo_uint) == 2); +#else + r &= __lzo_assert(sizeof(lzo_uint) >= 4); + r &= __lzo_assert(sizeof(lzo_uint) >= sizeof(unsigned)); +#endif + +#if defined(SIZEOF_UNSIGNED) + r &= __lzo_assert(SIZEOF_UNSIGNED == sizeof(unsigned)); +#endif +#if defined(SIZEOF_UNSIGNED_LONG) + r &= __lzo_assert(SIZEOF_UNSIGNED_LONG == sizeof(unsigned long)); +#endif +#if defined(SIZEOF_UNSIGNED_SHORT) + r &= __lzo_assert(SIZEOF_UNSIGNED_SHORT == sizeof(unsigned short)); +#endif +#if !defined(__LZO_IN_MINILZO) +#if defined(SIZEOF_SIZE_T) + r &= __lzo_assert(SIZEOF_SIZE_T == sizeof(size_t)); +#endif +#endif + + sanity = IS_UNSIGNED(unsigned short) && IS_UNSIGNED(unsigned) && + IS_UNSIGNED(unsigned long) && + IS_SIGNED(short) && IS_SIGNED(int) && IS_SIGNED(long); + if (sanity) + { + r &= __lzo_assert(IS_UNSIGNED(lzo_uint32)); + r &= __lzo_assert(IS_UNSIGNED(lzo_uint)); + r &= __lzo_assert(IS_SIGNED(lzo_int32)); + r &= __lzo_assert(IS_SIGNED(lzo_int)); + + r &= __lzo_assert(INT_MAX == LZO_STYPE_MAX(sizeof(int))); + r &= __lzo_assert(UINT_MAX == LZO_UTYPE_MAX(sizeof(unsigned))); + r &= __lzo_assert(LONG_MAX == LZO_STYPE_MAX(sizeof(long))); + r &= __lzo_assert(ULONG_MAX == LZO_UTYPE_MAX(sizeof(unsigned long))); + r &= __lzo_assert(SHRT_MAX == LZO_STYPE_MAX(sizeof(short))); + r &= __lzo_assert(USHRT_MAX == LZO_UTYPE_MAX(sizeof(unsigned short))); + r &= __lzo_assert(LZO_UINT32_MAX == LZO_UTYPE_MAX(sizeof(lzo_uint32))); + r &= __lzo_assert(LZO_UINT_MAX == LZO_UTYPE_MAX(sizeof(lzo_uint))); +#if !defined(__LZO_IN_MINILZO) + r &= __lzo_assert(SIZE_T_MAX == LZO_UTYPE_MAX(sizeof(size_t))); +#endif + } + +#if 0 + r &= __lzo_assert(LZO_BYTE(257) == 1); + r &= __lzo_assert(LZO_USHORT(65537L) == 1); +#endif + + return r; +} + +static lzo_bool basic_ptr_check(void) +{ + lzo_bool r = 1; + lzo_bool sanity; + + r &= __lzo_assert(sizeof(char *) >= sizeof(int)); + r &= __lzo_assert(sizeof(lzo_byte *) >= sizeof(char *)); + + r &= __lzo_assert(sizeof(lzo_voidp) == sizeof(lzo_byte *)); + r &= __lzo_assert(sizeof(lzo_voidp) == sizeof(lzo_voidpp)); + r &= __lzo_assert(sizeof(lzo_voidp) == sizeof(lzo_bytepp)); + r &= __lzo_assert(sizeof(lzo_voidp) >= sizeof(lzo_uint)); + + r &= __lzo_assert(sizeof(lzo_ptr_t) == sizeof(lzo_voidp)); + r &= __lzo_assert(sizeof(lzo_ptr_t) >= sizeof(lzo_uint)); + + r &= __lzo_assert(sizeof(lzo_ptrdiff_t) >= 4); + r &= __lzo_assert(sizeof(lzo_ptrdiff_t) >= sizeof(ptrdiff_t)); + +#if defined(SIZEOF_CHAR_P) + r &= __lzo_assert(SIZEOF_CHAR_P == sizeof(char *)); +#endif +#if defined(SIZEOF_PTRDIFF_T) + r &= __lzo_assert(SIZEOF_PTRDIFF_T == sizeof(ptrdiff_t)); +#endif + + sanity = IS_UNSIGNED(unsigned short) && IS_UNSIGNED(unsigned) && + IS_UNSIGNED(unsigned long) && + IS_SIGNED(short) && IS_SIGNED(int) && IS_SIGNED(long); + if (sanity) + { + r &= __lzo_assert(IS_UNSIGNED(lzo_ptr_t)); + r &= __lzo_assert(IS_UNSIGNED(lzo_moff_t)); + r &= __lzo_assert(IS_SIGNED(lzo_ptrdiff_t)); + r &= __lzo_assert(IS_SIGNED(lzo_sptr_t)); + } + + return r; +} + +static lzo_bool ptr_check(void) +{ + lzo_bool r = 1; + int i; + char _wrkmem[10 * sizeof(lzo_byte *) + sizeof(lzo_align_t)]; + lzo_byte *wrkmem; + lzo_bytepp dict; + unsigned char x[4 * sizeof(lzo_align_t)]; + long d; + lzo_align_t a; + + for (i = 0; i < (int) sizeof(x); i++) + x[i] = LZO_BYTE(i); + + wrkmem = LZO_PTR_ALIGN_UP((lzo_byte *)_wrkmem,sizeof(lzo_align_t)); + dict = (lzo_bytepp) wrkmem; + + d = (long) ((const lzo_bytep) dict - (const lzo_bytep) _wrkmem); + r &= __lzo_assert(d >= 0); + r &= __lzo_assert(d < (long) sizeof(lzo_align_t)); + + memset(&a,0xff,sizeof(a)); + r &= __lzo_assert(a.a_ushort == USHRT_MAX); + r &= __lzo_assert(a.a_uint == UINT_MAX); + r &= __lzo_assert(a.a_ulong == ULONG_MAX); + r &= __lzo_assert(a.a_lzo_uint == LZO_UINT_MAX); + + if (r == 1) + { + for (i = 0; i < 8; i++) + r &= __lzo_assert((const lzo_voidp) (&dict[i]) == (const lzo_voidp) (&wrkmem[i * sizeof(lzo_byte *)])); + } + + memset(&a,0,sizeof(a)); + r &= __lzo_assert(a.a_charp == NULL); + r &= __lzo_assert(a.a_lzo_bytep == NULL); + r &= __lzo_assert(NULL == 0); + if (r == 1) + { + for (i = 0; i < 10; i++) + dict[i] = wrkmem; + BZERO8_PTR(dict+1,sizeof(dict[0]),8); + r &= __lzo_assert(dict[0] == wrkmem); + for (i = 1; i < 9; i++) + r &= __lzo_assert(dict[i] == NULL); + r &= __lzo_assert(dict[9] == wrkmem); + } + + if (r == 1) + { + unsigned k = 1; + const unsigned n = (unsigned) sizeof(lzo_uint32); + lzo_byte *p0; + lzo_byte *p1; + + k += __lzo_align_gap(&x[k],n); + p0 = (lzo_bytep) &x[k]; +#if defined(PTR_LINEAR) + r &= __lzo_assert((PTR_LINEAR(p0) & (n-1)) == 0); +#else + r &= __lzo_assert(n == 4); + r &= __lzo_assert(PTR_ALIGNED_4(p0)); +#endif + + r &= __lzo_assert(k >= 1); + p1 = (lzo_bytep) &x[1]; + r &= __lzo_assert(PTR_GE(p0,p1)); + + r &= __lzo_assert(k < 1+n); + p1 = (lzo_bytep) &x[1+n]; + r &= __lzo_assert(PTR_LT(p0,p1)); + + if (r == 1) + { + lzo_uint32 v0 = * (lzo_uint32 *) &x[k]; + lzo_uint32 v1 = * (lzo_uint32 *) &x[k+n]; + + r &= __lzo_assert(v0 > 0); + r &= __lzo_assert(v1 > 0); + } + } + + return r; +} + +LZO_PUBLIC(int) +_lzo_config_check(void) +{ + lzo_bool r = 1; + int i; + union { + lzo_uint32 a; + unsigned short b; + lzo_uint32 aa[4]; + unsigned char x[4*sizeof(lzo_align_t)]; + } u; + +#if 0 + r &= __lzo_assert((const void *)&u == (const void *)&u.a); + r &= __lzo_assert((const void *)&u == (const void *)&u.b); + r &= __lzo_assert((const void *)&u == (const void *)&u.x[0]); + r &= __lzo_assert((const void *)&u == (const void *)&u.aa[0]); +#endif + + r &= basic_integral_check(); + r &= basic_ptr_check(); + if (r != 1) + return LZO_E_ERROR; + + u.a = 0; u.b = 0; + for (i = 0; i < (int) sizeof(u.x); i++) + u.x[i] = LZO_BYTE(i); + +#if 0 + r &= __lzo_assert( (int) (unsigned char) ((char) -1) == 255); +#endif + +#if defined(LZO_BYTE_ORDER) + if (r == 1) + { +# if (LZO_BYTE_ORDER == LZO_LITTLE_ENDIAN) + lzo_uint32 a = (lzo_uint32) (u.a & LZO_0xffffffffL); + unsigned short b = (unsigned short) (u.b & 0xffff); + r &= __lzo_assert(a == 0x03020100L); + r &= __lzo_assert(b == 0x0100); +# elif (LZO_BYTE_ORDER == LZO_BIG_ENDIAN) + lzo_uint32 a = u.a >> (8 * sizeof(u.a) - 32); + unsigned short b = u.b >> (8 * sizeof(u.b) - 16); + r &= __lzo_assert(a == 0x00010203L); + r &= __lzo_assert(b == 0x0001); +# else +# error invalid LZO_BYTE_ORDER +# endif + } +#endif + +#if defined(LZO_UNALIGNED_OK_2) + r &= __lzo_assert(sizeof(short) == 2); + if (r == 1) + { + unsigned short b[4]; + + for (i = 0; i < 4; i++) + b[i] = * (const unsigned short *) &u.x[i]; + +# if (LZO_BYTE_ORDER == LZO_LITTLE_ENDIAN) + r &= __lzo_assert(b[0] == 0x0100); + r &= __lzo_assert(b[1] == 0x0201); + r &= __lzo_assert(b[2] == 0x0302); + r &= __lzo_assert(b[3] == 0x0403); +# elif (LZO_BYTE_ORDER == LZO_BIG_ENDIAN) + r &= __lzo_assert(b[0] == 0x0001); + r &= __lzo_assert(b[1] == 0x0102); + r &= __lzo_assert(b[2] == 0x0203); + r &= __lzo_assert(b[3] == 0x0304); +# endif + } +#endif + +#if defined(LZO_UNALIGNED_OK_4) + r &= __lzo_assert(sizeof(lzo_uint32) == 4); + if (r == 1) + { + lzo_uint32 a[4]; + + for (i = 0; i < 4; i++) + a[i] = * (const lzo_uint32 *) &u.x[i]; + +# if (LZO_BYTE_ORDER == LZO_LITTLE_ENDIAN) + r &= __lzo_assert(a[0] == 0x03020100L); + r &= __lzo_assert(a[1] == 0x04030201L); + r &= __lzo_assert(a[2] == 0x05040302L); + r &= __lzo_assert(a[3] == 0x06050403L); +# elif (LZO_BYTE_ORDER == LZO_BIG_ENDIAN) + r &= __lzo_assert(a[0] == 0x00010203L); + r &= __lzo_assert(a[1] == 0x01020304L); + r &= __lzo_assert(a[2] == 0x02030405L); + r &= __lzo_assert(a[3] == 0x03040506L); +# endif + } +#endif + +#if defined(LZO_ALIGNED_OK_4) + r &= __lzo_assert(sizeof(lzo_uint32) == 4); +#endif + + r &= __lzo_assert(lzo_sizeof_dict_t == sizeof(lzo_dict_t)); + +#if defined(__LZO_IN_MINLZO) + if (r == 1) + { + lzo_uint32 adler; + adler = lzo_adler32(0, NULL, 0); + adler = lzo_adler32(adler, lzo_copyright(), 200); + r &= __lzo_assert(adler == 0x7ea34377L); + } +#endif + + if (r == 1) + { + r &= __lzo_assert(!schedule_insns_bug()); + } + + if (r == 1) + { + static int x[3]; + static unsigned xn = 3; + register unsigned j; + + for (j = 0; j < xn; j++) + x[j] = (int)j - 3; + r &= __lzo_assert(!strength_reduce_bug(x)); + } + + if (r == 1) + { + r &= ptr_check(); + } + + return r == 1 ? LZO_E_OK : LZO_E_ERROR; +} + +static lzo_bool schedule_insns_bug(void) +{ +#if defined(__LZO_CHECKER) + return 0; +#else + const int clone[] = {1, 2, 0}; + const int *q; + q = clone; + return (*q) ? 0 : 1; +#endif +} + +static lzo_bool strength_reduce_bug(int *x) +{ + return x[0] != -3 || x[1] != -2 || x[2] != -1; +} + +int __lzo_init_done = 0; + +LZO_PUBLIC(int) +__lzo_init2(unsigned v, int s1, int s2, int s3, int s4, int s5, + int s6, int s7, int s8, int s9) +{ + int r; + + __lzo_init_done = 1; + + if (v == 0) + return LZO_E_ERROR; + + r = (s1 == -1 || s1 == (int) sizeof(short)) && + (s2 == -1 || s2 == (int) sizeof(int)) && + (s3 == -1 || s3 == (int) sizeof(long)) && + (s4 == -1 || s4 == (int) sizeof(lzo_uint32)) && + (s5 == -1 || s5 == (int) sizeof(lzo_uint)) && + (s6 == -1 || s6 == (int) lzo_sizeof_dict_t) && + (s7 == -1 || s7 == (int) sizeof(char *)) && + (s8 == -1 || s8 == (int) sizeof(lzo_voidp)) && + (s9 == -1 || s9 == (int) sizeof(lzo_compress_t)); + if (!r) + return LZO_E_ERROR; + + r = _lzo_config_check(); + if (r != LZO_E_OK) + return r; + + return r; +} + +#if !defined(__LZO_IN_MINILZO) + +LZO_EXTERN(int) +__lzo_init(unsigned v,int s1,int s2,int s3,int s4,int s5,int s6,int s7); + +LZO_PUBLIC(int) +__lzo_init(unsigned v,int s1,int s2,int s3,int s4,int s5,int s6,int s7) +{ + if (v == 0 || v > 0x1010) + return LZO_E_ERROR; + return __lzo_init2(v,s1,s2,s3,s4,s5,-1,-1,s6,s7); +} + +#endif + +#define do_compress _lzo1x_1_do_compress + +#define LZO_NEED_DICT_H +#define D_BITS 14 +#define D_INDEX1(d,p) d = DM((0x21*DX3(p,5,5,6)) >> 5) +#define D_INDEX2(d,p) d = (d & (D_MASK & 0x7ff)) ^ (D_HIGH | 0x1f) + +#ifndef __LZO_CONFIG1X_H +#define __LZO_CONFIG1X_H + +#if !defined(LZO1X) && !defined(LZO1Y) && !defined(LZO1Z) +# define LZO1X +#endif + +#if !defined(__LZO_IN_MINILZO) +//#include +#endif + +#define LZO_EOF_CODE +#undef LZO_DETERMINISTIC + +#define M1_MAX_OFFSET 0x0400 +#ifndef M2_MAX_OFFSET +#define M2_MAX_OFFSET 0x0800 +#endif +#define M3_MAX_OFFSET 0x4000 +#define M4_MAX_OFFSET 0xbfff + +#define MX_MAX_OFFSET (M1_MAX_OFFSET + M2_MAX_OFFSET) + +#define M1_MIN_LEN 2 +#define M1_MAX_LEN 2 +#define M2_MIN_LEN 3 +#ifndef M2_MAX_LEN +#define M2_MAX_LEN 8 +#endif +#define M3_MIN_LEN 3 +#define M3_MAX_LEN 33 +#define M4_MIN_LEN 3 +#define M4_MAX_LEN 9 + +#define M1_MARKER 0 +#define M2_MARKER 64 +#define M3_MARKER 32 +#define M4_MARKER 16 + +#ifndef MIN_LOOKAHEAD +#define MIN_LOOKAHEAD (M2_MAX_LEN + 1) +#endif + +#if defined(LZO_NEED_DICT_H) + +#ifndef LZO_HASH +#define LZO_HASH LZO_HASH_LZO_INCREMENTAL_B +#endif +#define DL_MIN_LEN M2_MIN_LEN + +#ifndef __LZO_DICT_H +#define __LZO_DICT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#if !defined(D_BITS) && defined(DBITS) +# define D_BITS DBITS +#endif +#if !defined(D_BITS) +# error D_BITS is not defined +#endif +#if (D_BITS < 16) +# define D_SIZE LZO_SIZE(D_BITS) +# define D_MASK LZO_MASK(D_BITS) +#else +# define D_SIZE LZO_USIZE(D_BITS) +# define D_MASK LZO_UMASK(D_BITS) +#endif +#define D_HIGH ((D_MASK >> 1) + 1) + +#if !defined(DD_BITS) +# define DD_BITS 0 +#endif +#define DD_SIZE LZO_SIZE(DD_BITS) +#define DD_MASK LZO_MASK(DD_BITS) + +#if !defined(DL_BITS) +# define DL_BITS (D_BITS - DD_BITS) +#endif +#if (DL_BITS < 16) +# define DL_SIZE LZO_SIZE(DL_BITS) +# define DL_MASK LZO_MASK(DL_BITS) +#else +# define DL_SIZE LZO_USIZE(DL_BITS) +# define DL_MASK LZO_UMASK(DL_BITS) +#endif + +#if (D_BITS != DL_BITS + DD_BITS) +# error D_BITS does not match +#endif +#if (D_BITS < 8 || D_BITS > 18) +# error invalid D_BITS +#endif +#if (DL_BITS < 8 || DL_BITS > 20) +# error invalid DL_BITS +#endif +#if (DD_BITS < 0 || DD_BITS > 6) +# error invalid DD_BITS +#endif + +#if !defined(DL_MIN_LEN) +# define DL_MIN_LEN 3 +#endif +#if !defined(DL_SHIFT) +# define DL_SHIFT ((DL_BITS + (DL_MIN_LEN - 1)) / DL_MIN_LEN) +#endif + +#define LZO_HASH_GZIP 1 +#define LZO_HASH_GZIP_INCREMENTAL 2 +#define LZO_HASH_LZO_INCREMENTAL_A 3 +#define LZO_HASH_LZO_INCREMENTAL_B 4 + +#if !defined(LZO_HASH) +# error choose a hashing strategy +#endif + +#if (DL_MIN_LEN == 3) +# define _DV2_A(p,shift1,shift2) \ + (((( (lzo_uint32)((p)[0]) << shift1) ^ (p)[1]) << shift2) ^ (p)[2]) +# define _DV2_B(p,shift1,shift2) \ + (((( (lzo_uint32)((p)[2]) << shift1) ^ (p)[1]) << shift2) ^ (p)[0]) +# define _DV3_B(p,shift1,shift2,shift3) \ + ((_DV2_B((p)+1,shift1,shift2) << (shift3)) ^ (p)[0]) +#elif (DL_MIN_LEN == 2) +# define _DV2_A(p,shift1,shift2) \ + (( (lzo_uint32)(p[0]) << shift1) ^ p[1]) +# define _DV2_B(p,shift1,shift2) \ + (( (lzo_uint32)(p[1]) << shift1) ^ p[2]) +#else +# error invalid DL_MIN_LEN +#endif +#define _DV_A(p,shift) _DV2_A(p,shift,shift) +#define _DV_B(p,shift) _DV2_B(p,shift,shift) +#define DA2(p,s1,s2) \ + (((((lzo_uint32)((p)[2]) << (s2)) + (p)[1]) << (s1)) + (p)[0]) +#define DS2(p,s1,s2) \ + (((((lzo_uint32)((p)[2]) << (s2)) - (p)[1]) << (s1)) - (p)[0]) +#define DX2(p,s1,s2) \ + (((((lzo_uint32)((p)[2]) << (s2)) ^ (p)[1]) << (s1)) ^ (p)[0]) +#define DA3(p,s1,s2,s3) ((DA2((p)+1,s2,s3) << (s1)) + (p)[0]) +#define DS3(p,s1,s2,s3) ((DS2((p)+1,s2,s3) << (s1)) - (p)[0]) +#define DX3(p,s1,s2,s3) ((DX2((p)+1,s2,s3) << (s1)) ^ (p)[0]) +#define DMS(v,s) ((lzo_uint) (((v) & (D_MASK >> (s))) << (s))) +#define DM(v) DMS(v,0) + +#if (LZO_HASH == LZO_HASH_GZIP) +# define _DINDEX(dv,p) (_DV_A((p),DL_SHIFT)) + +#elif (LZO_HASH == LZO_HASH_GZIP_INCREMENTAL) +# define __LZO_HASH_INCREMENTAL +# define DVAL_FIRST(dv,p) dv = _DV_A((p),DL_SHIFT) +# define DVAL_NEXT(dv,p) dv = (((dv) << DL_SHIFT) ^ p[2]) +# define _DINDEX(dv,p) (dv) +# define DVAL_LOOKAHEAD DL_MIN_LEN + +#elif (LZO_HASH == LZO_HASH_LZO_INCREMENTAL_A) +# define __LZO_HASH_INCREMENTAL +# define DVAL_FIRST(dv,p) dv = _DV_A((p),5) +# define DVAL_NEXT(dv,p) \ + dv ^= (lzo_uint32)(p[-1]) << (2*5); dv = (((dv) << 5) ^ p[2]) +# define _DINDEX(dv,p) ((0x9f5f * (dv)) >> 5) +# define DVAL_LOOKAHEAD DL_MIN_LEN + +#elif (LZO_HASH == LZO_HASH_LZO_INCREMENTAL_B) +# define __LZO_HASH_INCREMENTAL +# define DVAL_FIRST(dv,p) dv = _DV_B((p),5) +# define DVAL_NEXT(dv,p) \ + dv ^= p[-1]; dv = (((dv) >> 5) ^ ((lzo_uint32)(p[2]) << (2*5))) +# define _DINDEX(dv,p) ((0x9f5f * (dv)) >> 5) +# define DVAL_LOOKAHEAD DL_MIN_LEN + +#else +# error choose a hashing strategy +#endif + +#ifndef DINDEX +#define DINDEX(dv,p) ((lzo_uint)((_DINDEX(dv,p)) & DL_MASK) << DD_BITS) +#endif +#if !defined(DINDEX1) && defined(D_INDEX1) +#define DINDEX1 D_INDEX1 +#endif +#if !defined(DINDEX2) && defined(D_INDEX2) +#define DINDEX2 D_INDEX2 +#endif + +#if !defined(__LZO_HASH_INCREMENTAL) +# define DVAL_FIRST(dv,p) ((void) 0) +# define DVAL_NEXT(dv,p) ((void) 0) +# define DVAL_LOOKAHEAD 0 +#endif + +#if !defined(DVAL_ASSERT) +#if defined(__LZO_HASH_INCREMENTAL) && !defined(NDEBUG) +static void DVAL_ASSERT(lzo_uint32 dv, const lzo_byte *p) +{ + lzo_uint32 df; + DVAL_FIRST(df,(p)); + assert(DINDEX(dv,p) == DINDEX(df,p)); +} +#else +# define DVAL_ASSERT(dv,p) ((void) 0) +#endif +#endif + +#if defined(LZO_DICT_USE_PTR) +# define DENTRY(p,in) (p) +# define GINDEX(m_pos,m_off,dict,dindex,in) m_pos = dict[dindex] +#else +# define DENTRY(p,in) ((lzo_uint) ((p)-(in))) +# define GINDEX(m_pos,m_off,dict,dindex,in) m_off = dict[dindex] +#endif + +#if (DD_BITS == 0) + +# define UPDATE_D(dict,drun,dv,p,in) dict[ DINDEX(dv,p) ] = DENTRY(p,in) +# define UPDATE_I(dict,drun,index,p,in) dict[index] = DENTRY(p,in) +# define UPDATE_P(ptr,drun,p,in) (ptr)[0] = DENTRY(p,in) + +#else + +# define UPDATE_D(dict,drun,dv,p,in) \ + dict[ DINDEX(dv,p) + drun++ ] = DENTRY(p,in); drun &= DD_MASK +# define UPDATE_I(dict,drun,index,p,in) \ + dict[ (index) + drun++ ] = DENTRY(p,in); drun &= DD_MASK +# define UPDATE_P(ptr,drun,p,in) \ + (ptr) [ drun++ ] = DENTRY(p,in); drun &= DD_MASK + +#endif + +#if defined(LZO_DICT_USE_PTR) + +#define LZO_CHECK_MPOS_DET(m_pos,m_off,in,ip,max_offset) \ + (m_pos == NULL || (m_off = (lzo_moff_t) (ip - m_pos)) > max_offset) + +#define LZO_CHECK_MPOS_NON_DET(m_pos,m_off,in,ip,max_offset) \ + (BOUNDS_CHECKING_OFF_IN_EXPR( \ + (PTR_LT(m_pos,in) || \ + (m_off = (lzo_moff_t) PTR_DIFF(ip,m_pos)) <= 0 || \ + m_off > max_offset) )) + +#else + +#define LZO_CHECK_MPOS_DET(m_pos,m_off,in,ip,max_offset) \ + (m_off == 0 || \ + ((m_off = (lzo_moff_t) ((ip)-(in)) - m_off) > max_offset) || \ + (m_pos = (ip) - (m_off), 0) ) + +#define LZO_CHECK_MPOS_NON_DET(m_pos,m_off,in,ip,max_offset) \ + ((lzo_moff_t) ((ip)-(in)) <= m_off || \ + ((m_off = (lzo_moff_t) ((ip)-(in)) - m_off) > max_offset) || \ + (m_pos = (ip) - (m_off), 0) ) + +#endif + +#if defined(LZO_DETERMINISTIC) +# define LZO_CHECK_MPOS LZO_CHECK_MPOS_DET +#else +# define LZO_CHECK_MPOS LZO_CHECK_MPOS_NON_DET +#endif + +#ifdef __cplusplus +} +#endif + +#endif + +#endif + +#endif + +#define DO_COMPRESS lzo1x_1_compress + +static +lzo_uint do_compress ( const lzo_byte *in , lzo_uint in_len, + lzo_byte *out, lzo_uint *out_len, + lzo_voidp wrkmem ) +{ +#if 0 && defined(__GNUC__) && defined(__i386__) + register const lzo_byte *ip __asm__("%esi"); +#else + register const lzo_byte *ip; +#endif + lzo_byte *op; + const lzo_byte * const in_end = in + in_len; + const lzo_byte * const ip_end = in + in_len - M2_MAX_LEN - 5; + const lzo_byte *ii; + lzo_dict_p const dict = (lzo_dict_p) wrkmem; + + op = out; + ip = in; + ii = ip; + + ip += 4; + for (;;) + { +#if 0 && defined(__GNUC__) && defined(__i386__) + register const lzo_byte *m_pos __asm__("%edi"); +#else + register const lzo_byte *m_pos; +#endif + lzo_moff_t m_off; + lzo_uint m_len; + lzo_uint dindex; + + DINDEX1(dindex,ip); + GINDEX(m_pos,m_off,dict,dindex,in); + if (LZO_CHECK_MPOS_NON_DET(m_pos,m_off,in,ip,M4_MAX_OFFSET)) + goto literal; +#if 1 + if (m_off <= M2_MAX_OFFSET || m_pos[3] == ip[3]) + goto try_match; + DINDEX2(dindex,ip); +#endif + GINDEX(m_pos,m_off,dict,dindex,in); + if (LZO_CHECK_MPOS_NON_DET(m_pos,m_off,in,ip,M4_MAX_OFFSET)) + goto literal; + if (m_off <= M2_MAX_OFFSET || m_pos[3] == ip[3]) + goto try_match; + goto literal; + +try_match: +#if 1 && defined(LZO_UNALIGNED_OK_2) + if (* (const lzo_ushortp) m_pos != * (const lzo_ushortp) ip) +#else + if (m_pos[0] != ip[0] || m_pos[1] != ip[1]) +#endif + { + } + else + { + if (m_pos[2] == ip[2]) + { +#if 0 + if (m_off <= M2_MAX_OFFSET) + goto match; + if (lit <= 3) + goto match; + if (lit == 3) + { + assert(op - 2 > out); op[-2] |= LZO_BYTE(3); + *op++ = *ii++; *op++ = *ii++; *op++ = *ii++; + goto code_match; + } + if (m_pos[3] == ip[3]) +#endif + goto match; + } + else + { +#if 0 +#if 0 + if (m_off <= M1_MAX_OFFSET && lit > 0 && lit <= 3) +#else + if (m_off <= M1_MAX_OFFSET && lit == 3) +#endif + { + register lzo_uint t; + + t = lit; + assert(op - 2 > out); op[-2] |= LZO_BYTE(t); + do *op++ = *ii++; while (--t > 0); + assert(ii == ip); + m_off -= 1; + *op++ = LZO_BYTE(M1_MARKER | ((m_off & 3) << 2)); + *op++ = LZO_BYTE(m_off >> 2); + ip += 2; + goto match_done; + } +#endif + } + } + +literal: + UPDATE_I(dict,0,dindex,ip,in); + ++ip; + if (ip >= ip_end) + break; + continue; + +match: + UPDATE_I(dict,0,dindex,ip,in); + if (ip - ii > 0) + { + register lzo_uint t = ip - ii; + + if (t <= 3) + { + assert(op - 2 > out); + op[-2] |= LZO_BYTE(t); + } + else if (t <= 18) + *op++ = LZO_BYTE(t - 3); + else + { + register lzo_uint tt = t - 18; + + *op++ = 0; + while (tt > 255) + { + tt -= 255; + *op++ = 0; + } + assert(tt > 0); + *op++ = LZO_BYTE(tt); + } + do *op++ = *ii++; while (--t > 0); + } + + assert(ii == ip); + ip += 3; + if (m_pos[3] != *ip++ || m_pos[4] != *ip++ || m_pos[5] != *ip++ || + m_pos[6] != *ip++ || m_pos[7] != *ip++ || m_pos[8] != *ip++ +#ifdef LZO1Y + || m_pos[ 9] != *ip++ || m_pos[10] != *ip++ || m_pos[11] != *ip++ + || m_pos[12] != *ip++ || m_pos[13] != *ip++ || m_pos[14] != *ip++ +#endif + ) + { + --ip; + m_len = ip - ii; + assert(m_len >= 3); assert(m_len <= M2_MAX_LEN); + + if (m_off <= M2_MAX_OFFSET) + { + m_off -= 1; +#if defined(LZO1X) + *op++ = LZO_BYTE(((m_len - 1) << 5) | ((m_off & 7) << 2)); + *op++ = LZO_BYTE(m_off >> 3); +#elif defined(LZO1Y) + *op++ = LZO_BYTE(((m_len + 1) << 4) | ((m_off & 3) << 2)); + *op++ = LZO_BYTE(m_off >> 2); +#endif + } + else if (m_off <= M3_MAX_OFFSET) + { + m_off -= 1; + *op++ = LZO_BYTE(M3_MARKER | (m_len - 2)); + goto m3_m4_offset; + } + else +#if defined(LZO1X) + { + m_off -= 0x4000; + assert(m_off > 0); assert(m_off <= 0x7fff); + *op++ = LZO_BYTE(M4_MARKER | + ((m_off & 0x4000) >> 11) | (m_len - 2)); + goto m3_m4_offset; + } +#elif defined(LZO1Y) + goto m4_match; +#endif + } + else + { + { + const lzo_byte *end = in_end; + const lzo_byte *m = m_pos + M2_MAX_LEN + 1; + while (ip < end && *m == *ip) + m++, ip++; + m_len = (ip - ii); + } + assert(m_len > M2_MAX_LEN); + + if (m_off <= M3_MAX_OFFSET) + { + m_off -= 1; + if (m_len <= 33) + *op++ = LZO_BYTE(M3_MARKER | (m_len - 2)); + else + { + m_len -= 33; + *op++ = M3_MARKER | 0; + goto m3_m4_len; + } + } + else + { +#if defined(LZO1Y) +m4_match: +#endif + m_off -= 0x4000; + assert(m_off > 0); assert(m_off <= 0x7fff); + if (m_len <= M4_MAX_LEN) + *op++ = LZO_BYTE(M4_MARKER | + ((m_off & 0x4000) >> 11) | (m_len - 2)); + else + { + m_len -= M4_MAX_LEN; + *op++ = LZO_BYTE(M4_MARKER | ((m_off & 0x4000) >> 11)); +m3_m4_len: + while (m_len > 255) + { + m_len -= 255; + *op++ = 0; + } + assert(m_len > 0); + *op++ = LZO_BYTE(m_len); + } + } + +m3_m4_offset: + *op++ = LZO_BYTE((m_off & 63) << 2); + *op++ = LZO_BYTE(m_off >> 6); + } + +#if 0 +match_done: +#endif + ii = ip; + if (ip >= ip_end) + break; + } + + *out_len = op - out; + return (lzo_uint) (in_end - ii); +} + +LZO_PUBLIC(int) +DO_COMPRESS ( const lzo_byte *in , lzo_uint in_len, + lzo_byte *out, lzo_uint *out_len, + lzo_voidp wrkmem ) +{ + lzo_byte *op = out; + lzo_uint t; + +#if defined(__LZO_QUERY_COMPRESS) + if (__LZO_IS_COMPRESS_QUERY(in,in_len,out,out_len,wrkmem)) + return __LZO_QUERY_COMPRESS(in,in_len,out,out_len,wrkmem,D_SIZE,lzo_sizeof(lzo_dict_t)); +#endif + + if (in_len <= M2_MAX_LEN + 5) + t = in_len; + else + { + t = do_compress(in,in_len,op,out_len,wrkmem); + op += *out_len; + } + + if (t > 0) + { + const lzo_byte *ii = in + in_len - t; + + if (op == out && t <= 238) + *op++ = LZO_BYTE(17 + t); + else if (t <= 3) + op[-2] |= LZO_BYTE(t); + else if (t <= 18) + *op++ = LZO_BYTE(t - 3); + else + { + lzo_uint tt = t - 18; + + *op++ = 0; + while (tt > 255) + { + tt -= 255; + *op++ = 0; + } + assert(tt > 0); + *op++ = LZO_BYTE(tt); + } + do *op++ = *ii++; while (--t > 0); + } + + *op++ = M4_MARKER | 1; + *op++ = 0; + *op++ = 0; + + *out_len = op - out; + return LZO_E_OK; +} + +#undef do_compress +#undef DO_COMPRESS +#undef LZO_HASH + +#undef LZO_TEST_DECOMPRESS_OVERRUN +#undef LZO_TEST_DECOMPRESS_OVERRUN_INPUT +#undef LZO_TEST_DECOMPRESS_OVERRUN_OUTPUT +#undef LZO_TEST_DECOMPRESS_OVERRUN_LOOKBEHIND +#undef DO_DECOMPRESS +#define DO_DECOMPRESS lzo1x_decompress + +#if defined(LZO_TEST_DECOMPRESS_OVERRUN) +# if !defined(LZO_TEST_DECOMPRESS_OVERRUN_INPUT) +# define LZO_TEST_DECOMPRESS_OVERRUN_INPUT 2 +# endif +# if !defined(LZO_TEST_DECOMPRESS_OVERRUN_OUTPUT) +# define LZO_TEST_DECOMPRESS_OVERRUN_OUTPUT 2 +# endif +# if !defined(LZO_TEST_DECOMPRESS_OVERRUN_LOOKBEHIND) +# define LZO_TEST_DECOMPRESS_OVERRUN_LOOKBEHIND +# endif +#endif + +#undef TEST_IP +#undef TEST_OP +#undef TEST_LOOKBEHIND +#undef NEED_IP +#undef NEED_OP +#undef HAVE_TEST_IP +#undef HAVE_TEST_OP +#undef HAVE_NEED_IP +#undef HAVE_NEED_OP +#undef HAVE_ANY_IP +#undef HAVE_ANY_OP + +#if defined(LZO_TEST_DECOMPRESS_OVERRUN_INPUT) +# if (LZO_TEST_DECOMPRESS_OVERRUN_INPUT >= 1) +# define TEST_IP (ip < ip_end) +# endif +# if (LZO_TEST_DECOMPRESS_OVERRUN_INPUT >= 2) +# define NEED_IP(x) \ + if ((lzo_uint)(ip_end - ip) < (lzo_uint)(x)) goto input_overrun +# endif +#endif + +#if defined(LZO_TEST_DECOMPRESS_OVERRUN_OUTPUT) +# if (LZO_TEST_DECOMPRESS_OVERRUN_OUTPUT >= 1) +# define TEST_OP (op <= op_end) +# endif +# if (LZO_TEST_DECOMPRESS_OVERRUN_OUTPUT >= 2) +# undef TEST_OP +# define NEED_OP(x) \ + if ((lzo_uint)(op_end - op) < (lzo_uint)(x)) goto output_overrun +# endif +#endif + +#if defined(LZO_TEST_DECOMPRESS_OVERRUN_LOOKBEHIND) +# define TEST_LOOKBEHIND(m_pos,out) if (m_pos < out) goto lookbehind_overrun +#else +# define TEST_LOOKBEHIND(m_pos,op) ((void) 0) +#endif + +#if !defined(LZO_EOF_CODE) && !defined(TEST_IP) +# define TEST_IP (ip < ip_end) +#endif + +#if defined(TEST_IP) +# define HAVE_TEST_IP +#else +# define TEST_IP 1 +#endif +#if defined(TEST_OP) +# define HAVE_TEST_OP +#else +# define TEST_OP 1 +#endif + +#if defined(NEED_IP) +# define HAVE_NEED_IP +#else +# define NEED_IP(x) ((void) 0) +#endif +#if defined(NEED_OP) +# define HAVE_NEED_OP +#else +# define NEED_OP(x) ((void) 0) +#endif + +#if defined(HAVE_TEST_IP) || defined(HAVE_NEED_IP) +# define HAVE_ANY_IP +#endif +#if defined(HAVE_TEST_OP) || defined(HAVE_NEED_OP) +# define HAVE_ANY_OP +#endif + +#if defined(DO_DECOMPRESS) +LZO_PUBLIC(int) +DO_DECOMPRESS ( const lzo_byte *in , lzo_uint in_len, + lzo_byte *out, lzo_uint *out_len, + lzo_voidp wrkmem ) +#endif +{ + register lzo_byte *op; + register const lzo_byte *ip; + register lzo_uint t; +#if defined(COPY_DICT) + lzo_uint m_off; + const lzo_byte *dict_end; +#else + register const lzo_byte *m_pos; +#endif + + const lzo_byte * const ip_end = in + in_len; +#if defined(HAVE_ANY_OP) + lzo_byte * const op_end = out + *out_len; +#endif +#if defined(LZO1Z) + lzo_uint last_m_off = 0; +#endif + + LZO_UNUSED(wrkmem); + +#if defined(__LZO_QUERY_DECOMPRESS) + if (__LZO_IS_DECOMPRESS_QUERY(in,in_len,out,out_len,wrkmem)) + return __LZO_QUERY_DECOMPRESS(in,in_len,out,out_len,wrkmem,0,0); +#endif + +#if defined(COPY_DICT) + if (dict) + { + if (dict_len > M4_MAX_OFFSET) + { + dict += dict_len - M4_MAX_OFFSET; + dict_len = M4_MAX_OFFSET; + } + dict_end = dict + dict_len; + } + else + { + dict_len = 0; + dict_end = NULL; + } +#endif + + *out_len = 0; + + op = out; + ip = in; + + if (*ip > 17) + { + t = *ip++ - 17; + if (t < 4) + goto match_next; + assert(t > 0); NEED_OP(t); NEED_IP(t+1); + do *op++ = *ip++; while (--t > 0); + goto first_literal_run; + } + + while (TEST_IP && TEST_OP) + { + t = *ip++; + if (t >= 16) + goto match; + if (t == 0) + { + NEED_IP(1); + while (*ip == 0) + { + t += 255; + ip++; + NEED_IP(1); + } + t += 15 + *ip++; + } + assert(t > 0); NEED_OP(t+3); NEED_IP(t+4); +#if defined(LZO_UNALIGNED_OK_4) || defined(LZO_ALIGNED_OK_4) +#if !defined(LZO_UNALIGNED_OK_4) + if (PTR_ALIGNED2_4(op,ip)) + { +#endif + * (lzo_uint32p) op = * (const lzo_uint32p) ip; + op += 4; ip += 4; + if (--t > 0) + { + if (t >= 4) + { + do { + * (lzo_uint32p) op = * (const lzo_uint32p) ip; + op += 4; ip += 4; t -= 4; + } while (t >= 4); + if (t > 0) do *op++ = *ip++; while (--t > 0); + } + else + do *op++ = *ip++; while (--t > 0); + } +#if !defined(LZO_UNALIGNED_OK_4) + } + else +#endif +#endif +#if !defined(LZO_UNALIGNED_OK_4) + { + *op++ = *ip++; *op++ = *ip++; *op++ = *ip++; + do *op++ = *ip++; while (--t > 0); + } +#endif + +first_literal_run: + + t = *ip++; + if (t >= 16) + goto match; +#if defined(COPY_DICT) +#if defined(LZO1Z) + m_off = (1 + M2_MAX_OFFSET) + (t << 6) + (*ip++ >> 2); + last_m_off = m_off; +#else + m_off = (1 + M2_MAX_OFFSET) + (t >> 2) + (*ip++ << 2); +#endif + NEED_OP(3); + t = 3; COPY_DICT(t,m_off) +#else +#if defined(LZO1Z) + t = (1 + M2_MAX_OFFSET) + (t << 6) + (*ip++ >> 2); + m_pos = op - t; + last_m_off = t; +#else + m_pos = op - (1 + M2_MAX_OFFSET); + m_pos -= t >> 2; + m_pos -= *ip++ << 2; +#endif + TEST_LOOKBEHIND(m_pos,out); NEED_OP(3); + *op++ = *m_pos++; *op++ = *m_pos++; *op++ = *m_pos; +#endif + goto match_done; + + while (TEST_IP && TEST_OP) + { +match: + if (t >= 64) + { +#if defined(COPY_DICT) +#if defined(LZO1X) + m_off = 1 + ((t >> 2) & 7) + (*ip++ << 3); + t = (t >> 5) - 1; +#elif defined(LZO1Y) + m_off = 1 + ((t >> 2) & 3) + (*ip++ << 2); + t = (t >> 4) - 3; +#elif defined(LZO1Z) + m_off = t & 0x1f; + if (m_off >= 0x1c) + m_off = last_m_off; + else + { + m_off = 1 + (m_off << 6) + (*ip++ >> 2); + last_m_off = m_off; + } + t = (t >> 5) - 1; +#endif +#else +#if defined(LZO1X) + m_pos = op - 1; + m_pos -= (t >> 2) & 7; + m_pos -= *ip++ << 3; + t = (t >> 5) - 1; +#elif defined(LZO1Y) + m_pos = op - 1; + m_pos -= (t >> 2) & 3; + m_pos -= *ip++ << 2; + t = (t >> 4) - 3; +#elif defined(LZO1Z) + { + lzo_uint off = t & 0x1f; + m_pos = op; + if (off >= 0x1c) + { + assert(last_m_off > 0); + m_pos -= last_m_off; + } + else + { + off = 1 + (off << 6) + (*ip++ >> 2); + m_pos -= off; + last_m_off = off; + } + } + t = (t >> 5) - 1; +#endif + TEST_LOOKBEHIND(m_pos,out); assert(t > 0); NEED_OP(t+3-1); + goto copy_match; +#endif + } + else if (t >= 32) + { + t &= 31; + if (t == 0) + { + NEED_IP(1); + while (*ip == 0) + { + t += 255; + ip++; + NEED_IP(1); + } + t += 31 + *ip++; + } +#if defined(COPY_DICT) +#if defined(LZO1Z) + m_off = 1 + (ip[0] << 6) + (ip[1] >> 2); + last_m_off = m_off; +#else + m_off = 1 + (ip[0] >> 2) + (ip[1] << 6); +#endif +#else +#if defined(LZO1Z) + { + lzo_uint off = 1 + (ip[0] << 6) + (ip[1] >> 2); + m_pos = op - off; + last_m_off = off; + } +#elif defined(LZO_UNALIGNED_OK_2) && (LZO_BYTE_ORDER == LZO_LITTLE_ENDIAN) + m_pos = op - 1; + m_pos -= (* (const lzo_ushortp) ip) >> 2; +#else + m_pos = op - 1; + m_pos -= (ip[0] >> 2) + (ip[1] << 6); +#endif +#endif + ip += 2; + } + else if (t >= 16) + { +#if defined(COPY_DICT) + m_off = (t & 8) << 11; +#else + m_pos = op; + m_pos -= (t & 8) << 11; +#endif + t &= 7; + if (t == 0) + { + NEED_IP(1); + while (*ip == 0) + { + t += 255; + ip++; + NEED_IP(1); + } + t += 7 + *ip++; + } +#if defined(COPY_DICT) +#if defined(LZO1Z) + m_off += (ip[0] << 6) + (ip[1] >> 2); +#else + m_off += (ip[0] >> 2) + (ip[1] << 6); +#endif + ip += 2; + if (m_off == 0) + goto eof_found; + m_off += 0x4000; +#if defined(LZO1Z) + last_m_off = m_off; +#endif +#else +#if defined(LZO1Z) + m_pos -= (ip[0] << 6) + (ip[1] >> 2); +#elif defined(LZO_UNALIGNED_OK_2) && (LZO_BYTE_ORDER == LZO_LITTLE_ENDIAN) + m_pos -= (* (const lzo_ushortp) ip) >> 2; +#else + m_pos -= (ip[0] >> 2) + (ip[1] << 6); +#endif + ip += 2; + if (m_pos == op) + goto eof_found; + m_pos -= 0x4000; +#if defined(LZO1Z) + last_m_off = op - m_pos; +#endif +#endif + } + else + { +#if defined(COPY_DICT) +#if defined(LZO1Z) + m_off = 1 + (t << 6) + (*ip++ >> 2); + last_m_off = m_off; +#else + m_off = 1 + (t >> 2) + (*ip++ << 2); +#endif + NEED_OP(2); + t = 2; COPY_DICT(t,m_off) +#else +#if defined(LZO1Z) + t = 1 + (t << 6) + (*ip++ >> 2); + m_pos = op - t; + last_m_off = t; +#else + m_pos = op - 1; + m_pos -= t >> 2; + m_pos -= *ip++ << 2; +#endif + TEST_LOOKBEHIND(m_pos,out); NEED_OP(2); + *op++ = *m_pos++; *op++ = *m_pos; +#endif + goto match_done; + } + +#if defined(COPY_DICT) + + NEED_OP(t+3-1); + t += 3-1; COPY_DICT(t,m_off) + +#else + + TEST_LOOKBEHIND(m_pos,out); assert(t > 0); NEED_OP(t+3-1); +#if defined(LZO_UNALIGNED_OK_4) || defined(LZO_ALIGNED_OK_4) +#if !defined(LZO_UNALIGNED_OK_4) + if (t >= 2 * 4 - (3 - 1) && PTR_ALIGNED2_4(op,m_pos)) + { + assert((op - m_pos) >= 4); +#else + if (t >= 2 * 4 - (3 - 1) && (op - m_pos) >= 4) + { +#endif + * (lzo_uint32p) op = * (const lzo_uint32p) m_pos; + op += 4; m_pos += 4; t -= 4 - (3 - 1); + do { + * (lzo_uint32p) op = * (const lzo_uint32p) m_pos; + op += 4; m_pos += 4; t -= 4; + } while (t >= 4); + if (t > 0) do *op++ = *m_pos++; while (--t > 0); + } + else +#endif + { +copy_match: + *op++ = *m_pos++; *op++ = *m_pos++; + do *op++ = *m_pos++; while (--t > 0); + } + +#endif + +match_done: +#if defined(LZO1Z) + t = ip[-1] & 3; +#else + t = ip[-2] & 3; +#endif + if (t == 0) + break; + +match_next: + assert(t > 0); NEED_OP(t); NEED_IP(t+1); + do *op++ = *ip++; while (--t > 0); + t = *ip++; + } + } + +#if defined(HAVE_TEST_IP) || defined(HAVE_TEST_OP) + *out_len = op - out; + return LZO_E_EOF_NOT_FOUND; +#endif + +eof_found: + assert(t == 1); + *out_len = op - out; + return (ip == ip_end ? LZO_E_OK : + (ip < ip_end ? LZO_E_INPUT_NOT_CONSUMED : LZO_E_INPUT_OVERRUN)); + +#if defined(HAVE_NEED_IP) +input_overrun: + *out_len = op - out; + return LZO_E_INPUT_OVERRUN; +#endif + +#if defined(HAVE_NEED_OP) +output_overrun: + *out_len = op - out; + return LZO_E_OUTPUT_OVERRUN; +#endif + +#if defined(LZO_TEST_DECOMPRESS_OVERRUN_LOOKBEHIND) +lookbehind_overrun: + *out_len = op - out; + return LZO_E_LOOKBEHIND_OVERRUN; +#endif +} + +#define LZO_TEST_DECOMPRESS_OVERRUN +#undef DO_DECOMPRESS +#define DO_DECOMPRESS lzo1x_decompress_safe + +#if defined(LZO_TEST_DECOMPRESS_OVERRUN) +# if !defined(LZO_TEST_DECOMPRESS_OVERRUN_INPUT) +# define LZO_TEST_DECOMPRESS_OVERRUN_INPUT 2 +# endif +# if !defined(LZO_TEST_DECOMPRESS_OVERRUN_OUTPUT) +# define LZO_TEST_DECOMPRESS_OVERRUN_OUTPUT 2 +# endif +# if !defined(LZO_TEST_DECOMPRESS_OVERRUN_LOOKBEHIND) +# define LZO_TEST_DECOMPRESS_OVERRUN_LOOKBEHIND +# endif +#endif + +#undef TEST_IP +#undef TEST_OP +#undef TEST_LOOKBEHIND +#undef NEED_IP +#undef NEED_OP +#undef HAVE_TEST_IP +#undef HAVE_TEST_OP +#undef HAVE_NEED_IP +#undef HAVE_NEED_OP +#undef HAVE_ANY_IP +#undef HAVE_ANY_OP + +#if defined(LZO_TEST_DECOMPRESS_OVERRUN_INPUT) +# if (LZO_TEST_DECOMPRESS_OVERRUN_INPUT >= 1) +# define TEST_IP (ip < ip_end) +# endif +# if (LZO_TEST_DECOMPRESS_OVERRUN_INPUT >= 2) +# define NEED_IP(x) \ + if ((lzo_uint)(ip_end - ip) < (lzo_uint)(x)) goto input_overrun +# endif +#endif + +#if defined(LZO_TEST_DECOMPRESS_OVERRUN_OUTPUT) +# if (LZO_TEST_DECOMPRESS_OVERRUN_OUTPUT >= 1) +# define TEST_OP (op <= op_end) +# endif +# if (LZO_TEST_DECOMPRESS_OVERRUN_OUTPUT >= 2) +# undef TEST_OP +# define NEED_OP(x) \ + if ((lzo_uint)(op_end - op) < (lzo_uint)(x)) goto output_overrun +# endif +#endif + +#if defined(LZO_TEST_DECOMPRESS_OVERRUN_LOOKBEHIND) +# define TEST_LOOKBEHIND(m_pos,out) if (m_pos < out) goto lookbehind_overrun +#else +# define TEST_LOOKBEHIND(m_pos,op) ((void) 0) +#endif + +#if !defined(LZO_EOF_CODE) && !defined(TEST_IP) +# define TEST_IP (ip < ip_end) +#endif + +#if defined(TEST_IP) +# define HAVE_TEST_IP +#else +# define TEST_IP 1 +#endif +#if defined(TEST_OP) +# define HAVE_TEST_OP +#else +# define TEST_OP 1 +#endif + +#if defined(NEED_IP) +# define HAVE_NEED_IP +#else +# define NEED_IP(x) ((void) 0) +#endif +#if defined(NEED_OP) +# define HAVE_NEED_OP +#else +# define NEED_OP(x) ((void) 0) +#endif + +#if defined(HAVE_TEST_IP) || defined(HAVE_NEED_IP) +# define HAVE_ANY_IP +#endif +#if defined(HAVE_TEST_OP) || defined(HAVE_NEED_OP) +# define HAVE_ANY_OP +#endif + +#if defined(DO_DECOMPRESS) +LZO_PUBLIC(int) +DO_DECOMPRESS ( const lzo_byte *in , lzo_uint in_len, + lzo_byte *out, lzo_uint *out_len, + lzo_voidp wrkmem ) +#endif +{ + register lzo_byte *op; + register const lzo_byte *ip; + register lzo_uint t; +#if defined(COPY_DICT) + lzo_uint m_off; + const lzo_byte *dict_end; +#else + register const lzo_byte *m_pos; +#endif + + const lzo_byte * const ip_end = in + in_len; +#if defined(HAVE_ANY_OP) + lzo_byte * const op_end = out + *out_len; +#endif +#if defined(LZO1Z) + lzo_uint last_m_off = 0; +#endif + + LZO_UNUSED(wrkmem); + +#if defined(__LZO_QUERY_DECOMPRESS) + if (__LZO_IS_DECOMPRESS_QUERY(in,in_len,out,out_len,wrkmem)) + return __LZO_QUERY_DECOMPRESS(in,in_len,out,out_len,wrkmem,0,0); +#endif + +#if defined(COPY_DICT) + if (dict) + { + if (dict_len > M4_MAX_OFFSET) + { + dict += dict_len - M4_MAX_OFFSET; + dict_len = M4_MAX_OFFSET; + } + dict_end = dict + dict_len; + } + else + { + dict_len = 0; + dict_end = NULL; + } +#endif + + *out_len = 0; + + op = out; + ip = in; + + if (*ip > 17) + { + t = *ip++ - 17; + if (t < 4) + goto match_next; + assert(t > 0); NEED_OP(t); NEED_IP(t+1); + do *op++ = *ip++; while (--t > 0); + goto first_literal_run; + } + + while (TEST_IP && TEST_OP) + { + t = *ip++; + if (t >= 16) + goto match; + if (t == 0) + { + NEED_IP(1); + while (*ip == 0) + { + t += 255; + ip++; + NEED_IP(1); + } + t += 15 + *ip++; + } + assert(t > 0); NEED_OP(t+3); NEED_IP(t+4); +#if defined(LZO_UNALIGNED_OK_4) || defined(LZO_ALIGNED_OK_4) +#if !defined(LZO_UNALIGNED_OK_4) + if (PTR_ALIGNED2_4(op,ip)) + { +#endif + * (lzo_uint32p) op = * (const lzo_uint32p) ip; + op += 4; ip += 4; + if (--t > 0) + { + if (t >= 4) + { + do { + * (lzo_uint32p) op = * (const lzo_uint32p) ip; + op += 4; ip += 4; t -= 4; + } while (t >= 4); + if (t > 0) do *op++ = *ip++; while (--t > 0); + } + else + do *op++ = *ip++; while (--t > 0); + } +#if !defined(LZO_UNALIGNED_OK_4) + } + else +#endif +#endif +#if !defined(LZO_UNALIGNED_OK_4) + { + *op++ = *ip++; *op++ = *ip++; *op++ = *ip++; + do *op++ = *ip++; while (--t > 0); + } +#endif + +first_literal_run: + + t = *ip++; + if (t >= 16) + goto match; +#if defined(COPY_DICT) +#if defined(LZO1Z) + m_off = (1 + M2_MAX_OFFSET) + (t << 6) + (*ip++ >> 2); + last_m_off = m_off; +#else + m_off = (1 + M2_MAX_OFFSET) + (t >> 2) + (*ip++ << 2); +#endif + NEED_OP(3); + t = 3; COPY_DICT(t,m_off) +#else +#if defined(LZO1Z) + t = (1 + M2_MAX_OFFSET) + (t << 6) + (*ip++ >> 2); + m_pos = op - t; + last_m_off = t; +#else + m_pos = op - (1 + M2_MAX_OFFSET); + m_pos -= t >> 2; + m_pos -= *ip++ << 2; +#endif + TEST_LOOKBEHIND(m_pos,out); NEED_OP(3); + *op++ = *m_pos++; *op++ = *m_pos++; *op++ = *m_pos; +#endif + goto match_done; + + while (TEST_IP && TEST_OP) + { +match: + if (t >= 64) + { +#if defined(COPY_DICT) +#if defined(LZO1X) + m_off = 1 + ((t >> 2) & 7) + (*ip++ << 3); + t = (t >> 5) - 1; +#elif defined(LZO1Y) + m_off = 1 + ((t >> 2) & 3) + (*ip++ << 2); + t = (t >> 4) - 3; +#elif defined(LZO1Z) + m_off = t & 0x1f; + if (m_off >= 0x1c) + m_off = last_m_off; + else + { + m_off = 1 + (m_off << 6) + (*ip++ >> 2); + last_m_off = m_off; + } + t = (t >> 5) - 1; +#endif +#else +#if defined(LZO1X) + m_pos = op - 1; + m_pos -= (t >> 2) & 7; + m_pos -= *ip++ << 3; + t = (t >> 5) - 1; +#elif defined(LZO1Y) + m_pos = op - 1; + m_pos -= (t >> 2) & 3; + m_pos -= *ip++ << 2; + t = (t >> 4) - 3; +#elif defined(LZO1Z) + { + lzo_uint off = t & 0x1f; + m_pos = op; + if (off >= 0x1c) + { + assert(last_m_off > 0); + m_pos -= last_m_off; + } + else + { + off = 1 + (off << 6) + (*ip++ >> 2); + m_pos -= off; + last_m_off = off; + } + } + t = (t >> 5) - 1; +#endif + TEST_LOOKBEHIND(m_pos,out); assert(t > 0); NEED_OP(t+3-1); + goto copy_match; +#endif + } + else if (t >= 32) + { + t &= 31; + if (t == 0) + { + NEED_IP(1); + while (*ip == 0) + { + t += 255; + ip++; + NEED_IP(1); + } + t += 31 + *ip++; + } +#if defined(COPY_DICT) +#if defined(LZO1Z) + m_off = 1 + (ip[0] << 6) + (ip[1] >> 2); + last_m_off = m_off; +#else + m_off = 1 + (ip[0] >> 2) + (ip[1] << 6); +#endif +#else +#if defined(LZO1Z) + { + lzo_uint off = 1 + (ip[0] << 6) + (ip[1] >> 2); + m_pos = op - off; + last_m_off = off; + } +#elif defined(LZO_UNALIGNED_OK_2) && (LZO_BYTE_ORDER == LZO_LITTLE_ENDIAN) + m_pos = op - 1; + m_pos -= (* (const lzo_ushortp) ip) >> 2; +#else + m_pos = op - 1; + m_pos -= (ip[0] >> 2) + (ip[1] << 6); +#endif +#endif + ip += 2; + } + else if (t >= 16) + { +#if defined(COPY_DICT) + m_off = (t & 8) << 11; +#else + m_pos = op; + m_pos -= (t & 8) << 11; +#endif + t &= 7; + if (t == 0) + { + NEED_IP(1); + while (*ip == 0) + { + t += 255; + ip++; + NEED_IP(1); + } + t += 7 + *ip++; + } +#if defined(COPY_DICT) +#if defined(LZO1Z) + m_off += (ip[0] << 6) + (ip[1] >> 2); +#else + m_off += (ip[0] >> 2) + (ip[1] << 6); +#endif + ip += 2; + if (m_off == 0) + goto eof_found; + m_off += 0x4000; +#if defined(LZO1Z) + last_m_off = m_off; +#endif +#else +#if defined(LZO1Z) + m_pos -= (ip[0] << 6) + (ip[1] >> 2); +#elif defined(LZO_UNALIGNED_OK_2) && (LZO_BYTE_ORDER == LZO_LITTLE_ENDIAN) + m_pos -= (* (const lzo_ushortp) ip) >> 2; +#else + m_pos -= (ip[0] >> 2) + (ip[1] << 6); +#endif + ip += 2; + if (m_pos == op) + goto eof_found; + m_pos -= 0x4000; +#if defined(LZO1Z) + last_m_off = op - m_pos; +#endif +#endif + } + else + { +#if defined(COPY_DICT) +#if defined(LZO1Z) + m_off = 1 + (t << 6) + (*ip++ >> 2); + last_m_off = m_off; +#else + m_off = 1 + (t >> 2) + (*ip++ << 2); +#endif + NEED_OP(2); + t = 2; COPY_DICT(t,m_off) +#else +#if defined(LZO1Z) + t = 1 + (t << 6) + (*ip++ >> 2); + m_pos = op - t; + last_m_off = t; +#else + m_pos = op - 1; + m_pos -= t >> 2; + m_pos -= *ip++ << 2; +#endif + TEST_LOOKBEHIND(m_pos,out); NEED_OP(2); + *op++ = *m_pos++; *op++ = *m_pos; +#endif + goto match_done; + } + +#if defined(COPY_DICT) + + NEED_OP(t+3-1); + t += 3-1; COPY_DICT(t,m_off) + +#else + + TEST_LOOKBEHIND(m_pos,out); assert(t > 0); NEED_OP(t+3-1); +#if defined(LZO_UNALIGNED_OK_4) || defined(LZO_ALIGNED_OK_4) +#if !defined(LZO_UNALIGNED_OK_4) + if (t >= 2 * 4 - (3 - 1) && PTR_ALIGNED2_4(op,m_pos)) + { + assert((op - m_pos) >= 4); +#else + if (t >= 2 * 4 - (3 - 1) && (op - m_pos) >= 4) + { +#endif + * (lzo_uint32p) op = * (const lzo_uint32p) m_pos; + op += 4; m_pos += 4; t -= 4 - (3 - 1); + do { + * (lzo_uint32p) op = * (const lzo_uint32p) m_pos; + op += 4; m_pos += 4; t -= 4; + } while (t >= 4); + if (t > 0) do *op++ = *m_pos++; while (--t > 0); + } + else +#endif + { +copy_match: + *op++ = *m_pos++; *op++ = *m_pos++; + do *op++ = *m_pos++; while (--t > 0); + } + +#endif + +match_done: +#if defined(LZO1Z) + t = ip[-1] & 3; +#else + t = ip[-2] & 3; +#endif + if (t == 0) + break; + +match_next: + assert(t > 0); NEED_OP(t); NEED_IP(t+1); + do *op++ = *ip++; while (--t > 0); + t = *ip++; + } + } + +#if defined(HAVE_TEST_IP) || defined(HAVE_TEST_OP) + *out_len = op - out; + return LZO_E_EOF_NOT_FOUND; +#endif + +eof_found: + assert(t == 1); + *out_len = op - out; + return (ip == ip_end ? LZO_E_OK : + (ip < ip_end ? LZO_E_INPUT_NOT_CONSUMED : LZO_E_INPUT_OVERRUN)); + +#if defined(HAVE_NEED_IP) +input_overrun: + *out_len = op - out; + return LZO_E_INPUT_OVERRUN; +#endif + +#if defined(HAVE_NEED_OP) +output_overrun: + *out_len = op - out; + return LZO_E_OUTPUT_OVERRUN; +#endif + +#if defined(LZO_TEST_DECOMPRESS_OVERRUN_LOOKBEHIND) +lookbehind_overrun: + *out_len = op - out; + return LZO_E_LOOKBEHIND_OVERRUN; +#endif +} + +/***** End of minilzo.c *****/ + diff -urN linux-2.4.26-rc1/mm/comp_cache/proc.c linux-2.4.26-rc1-ccache/mm/comp_cache/proc.c --- linux-2.4.26-rc1/mm/comp_cache/proc.c Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/mm/comp_cache/proc.c Sun Mar 28 17:12:21 2004 @@ -0,0 +1,528 @@ +/* + * linux/mm/comp_cache/proc.c + * + * Time-stamp: <2002-10-21 16:26:52 rcastro> + * + * Linux Virtual Memory Compressed Cache + * + * Author: Rodrigo S. de Castro + * Licensed under the GPL - 2001 + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#define NUM_ALGORITHMS 3 +#define WKDM_IDX 0 +#define WK4X4_IDX 1 +#define LZO_IDX 2 + +extern unsigned long new_num_comp_pages, max_num_comp_pages, min_num_comp_pages; + +struct stats_summary { + /* swap cache */ + unsigned long long comp_size_sum_swap; + unsigned long comp_swap; + unsigned long decomp_swap; + unsigned long read_swap; + unsigned long written_swap; + + + /* page cache */ + unsigned long long comp_size_sum_page; + unsigned long comp_page; + unsigned long decomp_page; + unsigned long read_page; + unsigned long written_page; + +}; + +static struct comp_alg { + char name[6]; + compress_function_t * comp; + decompress_function_t * decomp; + struct stats_summary stats; +} compression_algorithm; + +static int algorithm_min = WKDM_IDX; +static int algorithm_max = LZO_IDX; +static int algorithm_idx = -1; +struct stats_summary * stats = &compression_algorithm.stats; + +static struct comp_alg_data comp_data; + +static spinlock_t comp_data_lock __cacheline_aligned = SPIN_LOCK_UNLOCKED; + +int clean_page_compress_lock = 0; + +inline void +comp_cache_update_read_stats(struct comp_cache_fragment * fragment) +{ +#ifdef CONFIG_COMP_PAGE_CACHE + if (!PageSwapCache(fragment)) { + stats->read_page++; + return; + } +#endif + stats->read_swap++; +} + +inline void +comp_cache_update_written_stats(struct comp_cache_fragment * fragment) +{ +#ifdef CONFIG_COMP_PAGE_CACHE + if (!PageSwapCache(fragment)) { + stats->written_page++; + return; + } +#endif + stats->written_swap++; +} + +static inline void +comp_cache_update_decomp_stats(struct comp_cache_fragment * fragment) +{ +#ifdef CONFIG_COMP_PAGE_CACHE + if (!PageSwapCache(fragment)) { + stats->decomp_page++; + return; + } +#endif + stats->decomp_swap++; +} + +static inline void +comp_cache_update_comp_stats(unsigned int comp_size, struct page * page) +{ + /* update compressed size statistics */ + if (!comp_size) + BUG(); + +#ifdef CONFIG_COMP_PAGE_CACHE + if (!PageSwapCache(page)) { + stats->comp_page++; + stats->comp_size_sum_page += comp_size; + return; + } +#endif + + stats->comp_swap++; + stats->comp_size_sum_swap += comp_size; +} + +static unsigned int +lzo_wrapper_compress(unsigned long * from, unsigned long * to, unsigned int words, struct comp_alg_data * data) +{ + int error; + lzo_uint out_len; + + error = lzo1x_1_compress((lzo_byte *) from, words * sizeof(unsigned long), (lzo_byte *) to, &out_len, data->wrkmem); + + if (error != LZO_E_OK) + BUG(); + + return out_len; +} + +static void +lzo_wrapper_decompress(unsigned long * from, unsigned long * to, unsigned int words, struct comp_alg_data * data) +{ + int error; + lzo_uint new_len; + + error = lzo1x_decompress((lzo_byte *) from, data->compressed_size, (lzo_byte *) to, &new_len, NULL); + + if (error != LZO_E_OK || new_len != PAGE_SIZE) { + printk(KERN_EMERG "internal error - decompression failed: %d\n", error); + BUG(); + } +} + +int +compress(struct page * page, void * to, int state) +{ + unsigned int comp_size; + void * from = page_address(page); + +#if 0 + if (state == CLEAN_PAGE && clean_page_compress_lock) { // && (num_clean_fragments * 5 < num_fragments)) { + comp_size = PAGE_SIZE; + comp_cache_update_comp_stats(comp_size, page); + return comp_size; + } +#endif + + spin_lock(&comp_data_lock); + comp_size = compression_algorithm.comp(from, to, PAGE_SIZE/4, &comp_data); + spin_unlock(&comp_data_lock); + comp_cache_update_comp_stats(comp_size, page); + + if (comp_size > PAGE_SIZE) + comp_size = PAGE_SIZE; + + return comp_size; +} + +void +decompress_fragment_to_page(struct comp_cache_fragment * fragment, struct page * page) +{ + struct comp_cache_page * comp_page; + void * from = page_address(fragment->comp_page->page) + fragment->offset; + void * to = page_address(page); + + if (!fragment) + BUG(); + if (!fragment_count(fragment)) + BUG(); + comp_page = fragment->comp_page; + if (!comp_page->page) + BUG(); + if (!PageLocked(page)) + BUG(); + if (!PageLocked(comp_page->page)) + BUG(); + + SetPageUptodate(page); + + if (!compressed(fragment)) { + copy_page(to, from); + return; + } + + /* regular compressed fragment */ + spin_lock(&comp_data_lock); + comp_data.compressed_size = fragment->compressed_size; + compression_algorithm.decomp(from, to, PAGE_SIZE/4, &comp_data); + spin_unlock(&comp_data_lock); + comp_cache_update_decomp_stats(fragment); +} + +#ifdef CONFIG_COMP_SWAP +void +get_comp_data(struct page * page, unsigned short * size, unsigned short * offset) +{ + unsigned short counter, metadata_offset; + unsigned long fragment_index; + + counter = *((unsigned short *) page_address(page)); + metadata_offset = *((unsigned short *) (page_address(page) + 2)); + + fragment_index = 0; + + while (counter-- && fragment_index != page->index) { + fragment_index = *((unsigned long *) (page_address(page) + metadata_offset + 4)); + metadata_offset += 8; + } + + if (!fragment_index) + BUG(); + if (fragment_index != page->index) + BUG(); + + metadata_offset -= 8; + *size = *((unsigned short *) (page_address(page) + metadata_offset)); + *offset = *((unsigned short *) (page_address(page) + metadata_offset + 2)); +} +#endif + +void +decompress_swap_cache_page(struct page * page) +{ + unsigned short comp_size, comp_offset; + + if (!PageLocked(page)) + BUG(); + + spin_lock(&comp_data_lock); + get_comp_data(page, &comp_size, &comp_offset); + + if (comp_size > PAGE_SIZE) + BUG(); + + memcpy(page_address(comp_data.decompress_buffer), page_address(page) + comp_offset, comp_size); + + comp_data.compressed_size = comp_size; + compression_algorithm.decomp(page_address(comp_data.decompress_buffer), page_address(page), PAGE_SIZE/4, &comp_data); + + spin_unlock(&comp_data_lock); + + stats->decomp_swap++; + PageClearCompressed(page); +} + +void __init +comp_cache_algorithms_init(void) +{ + if (algorithm_idx == -1 || algorithm_idx < algorithm_min || algorithm_idx > algorithm_max) + algorithm_idx = LZO_IDX; + + /* data structure for compression algorithms */ + switch(algorithm_idx) { + case WKDM_IDX: + case WK4X4_IDX: + comp_data.tempTagsArray = kmalloc(300 * sizeof(WK_word), GFP_ATOMIC); + comp_data.tempQPosArray = kmalloc(300 * sizeof(WK_word), GFP_ATOMIC); + comp_data.tempLowBitsArray = kmalloc(1200 * sizeof(WK_word), GFP_ATOMIC); + + if (!comp_data.tempTagsArray || !comp_data.tempQPosArray || !comp_data.tempLowBitsArray) + panic("comp_cache_algorithms_init(): cannot allocate structures for WKdm/WK4x4"); + break; + case LZO_IDX: + comp_data.wrkmem = (lzo_byte *) kmalloc(LZO1X_1_MEM_COMPRESS, GFP_ATOMIC); + if (!comp_data.wrkmem) + panic("comp_cache_algorithms_init(): cannot allocate dictionary for LZO"); + break; + } + comp_data.decompress_buffer = alloc_page(GFP_ATOMIC); + if (!comp_data.decompress_buffer) + panic("comp_cache_algorithms_init(): cannot allocate decompression buffer"); + + /* stats for algorithm */ + memset((void *) &compression_algorithm, 0, sizeof(struct stats_summary)); + + /* compression algorithms */ + switch(algorithm_idx) { + case WKDM_IDX: + strcpy(compression_algorithm.name, "WKdm"); + compression_algorithm.comp = WKdm_compress; + compression_algorithm.decomp = WKdm_decompress; + break; + case WK4X4_IDX: + strcpy(compression_algorithm.name, "WK4x4"); + compression_algorithm.comp = WK4x4_compress; + compression_algorithm.decomp = WK4x4_decompress; + break; + case LZO_IDX: + strcpy(compression_algorithm.name, "LZO"); + compression_algorithm.comp = lzo_wrapper_compress; + compression_algorithm.decomp = lzo_wrapper_decompress; + break; + } + + printk("Compressed Cache: compression algorithm: %s\n", compression_algorithm.name); +} + +static int proc_calc_metrics(char *page, char **start, off_t off, + int count, int *eof, int len) +{ + if (len <= off+count) *eof = 1; + *start = page + off; + len -= off; + if (len>count) len = count; + if (len<0) len = 0; + return len; +} + +#define proportion(part, total) (total?((unsigned int) ((part * 100)/(total))):0) + +static void +print_comp_cache_stats(char * page, int * length) +{ + unsigned int compression_ratio_swap, compression_ratio_page, compression_ratio_total; + unsigned long long total_sum_comp_pages; + unsigned long total_comp_pages; + + /* swap cache */ + total_comp_pages = stats->comp_swap; + total_sum_comp_pages = stats->comp_size_sum_swap; + + compression_ratio_swap = 0; + if (stats->comp_swap) + compression_ratio_swap = ((big_division(stats->comp_size_sum_swap, stats->comp_swap)*100)/PAGE_SIZE); + + /* page cache */ + total_comp_pages += stats->comp_page; + total_sum_comp_pages += stats->comp_size_sum_page; + + compression_ratio_page = 0; + if (stats->comp_page) + compression_ratio_page = ((big_division(stats->comp_size_sum_page, stats->comp_page)*100)/PAGE_SIZE); + + /* total */ + compression_ratio_total = 0; + if (total_comp_pages) + compression_ratio_total = ((big_division(total_sum_comp_pages, total_comp_pages)*100)/PAGE_SIZE); + + *length += sprintf(page + *length, + " algorithm %s\n" + " - (C) compressed pages: %8lu (S: %3d%% P: %3d%%)\n" + " - (D) decompressed pages: %8lu (S: %3d%% P: %3d%%) D/C: %3u%%\n" + " - (R) read pages: %8lu (S: %3d%% P: %3d%%) R/C: %3u%%\n" + " - (W) written pages: %8lu (S: %3d%% P: %3d%%) W/C: %3u%%\n" + " compression ratio: %8u%% (S: %3u%% P: %3u%%)\n", + compression_algorithm.name, + total_comp_pages, + proportion(stats->comp_swap, total_comp_pages), + proportion(stats->comp_page, total_comp_pages), + stats->decomp_swap + stats->decomp_page, + proportion(stats->decomp_swap, stats->decomp_swap + stats->decomp_page), + proportion(stats->decomp_page, stats->decomp_swap + stats->decomp_page), + total_comp_pages?((unsigned int) (((stats->decomp_swap + stats->decomp_page) * 100)/total_comp_pages)):0, + stats->read_swap + stats->read_page, + proportion(stats->read_swap, stats->read_swap + stats->read_page), + proportion(stats->read_page, stats->read_swap + stats->read_page), + total_comp_pages?((unsigned int) (((stats->read_swap + stats->read_page) * 100)/total_comp_pages)):0, + stats->written_swap + stats->written_page, + proportion(stats->written_swap, stats->written_swap + stats->written_page), + proportion(stats->written_page, stats->written_swap + stats->written_page), + total_comp_pages?((unsigned int) (((stats->written_swap + stats->written_page) * 100)/total_comp_pages)):0, + compression_ratio_total, + compression_ratio_swap, + compression_ratio_page); +} + +#define HIST_PRINTK \ + array_num_fragments[0], array_num_fragments[1], array_num_fragments[2], array_num_fragments[3], \ + array_num_fragments[4], array_num_fragments[5], array_num_fragments[6], array_num_fragments[7] +#define HIST_COUNT 8 + +int +comp_cache_hist_read_proc(char *page, char **start, off_t off, int count, int *eof, void *data) +{ + unsigned long * array_num_fragments, total1, total2; + int length = 0, i; + + array_num_fragments = (unsigned long *) vmalloc(HIST_COUNT * sizeof(unsigned long)); + + if (!array_num_fragments) { + printk("couldn't allocate data structures for free space histogram\n"); + goto out; + } + + length = sprintf(page, + "compressed cache - free space histogram (free space x number of fragments)\n" + " total 0f 1f 2f 3f 4f 5f 6f more\n"); + + memset((void *) array_num_fragments, 0, HIST_COUNT * sizeof(unsigned long)); + + spin_lock(&comp_cache_lock); + + total1 = free_space_count(0, array_num_fragments); + length += sprintf(page + length, + " %4d: %7lu %5lu %5lu %5lu %5lu %5lu %5lu %5lu %5lu\n", + 0, + total1, + HIST_PRINTK); + + for (i = 1; i < free_space_hash_size; i += 2) { + memset((void *) array_num_fragments, 0, HIST_COUNT * sizeof(unsigned long)); + total1 = free_space_count(i, array_num_fragments); + total2 = 0; + if (i + 1 < free_space_hash_size) + total2 = free_space_count(i + 1, array_num_fragments); + + length += sprintf(page + length, + "%4d - %4d: %7lu %5lu %5lu %5lu %5lu %5lu %5lu %5lu %5lu\n", + (i-1)*free_space_interval+1, (i+1)*free_space_interval> 10, + failed_comp_page_allocs); + + print_comp_cache_stats(page, &length); + return proc_calc_metrics(page, start, off, count, eof, length); +} + +static int __init comp_cache_alg(char *str) +{ + char * endp; + + algorithm_idx = simple_strtoul(str, &endp, 0); + return 1; +} + +__setup("compalg=", comp_cache_alg); + + +/* + * Overrides for Emacs so that we follow Linus's tabbing style. + * Emacs will notice this stuff at the end of the file and automatically + * adjust the settings for this buffer only. This must remain at the end + * of the file. + * --------------------------------------------------------------------------- + * Local variables: + * c-file-style: "linux" + * End: + */ diff -urN linux-2.4.26-rc1/mm/comp_cache/swapin.c linux-2.4.26-rc1-ccache/mm/comp_cache/swapin.c --- linux-2.4.26-rc1/mm/comp_cache/swapin.c Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/mm/comp_cache/swapin.c Sun Mar 28 17:12:21 2004 @@ -0,0 +1,346 @@ +/* + * linux/mm/comp_cache/swapin.c + * + * Time-stamp: <2002-11-21 15:23:30 rcastro> + * + * Linux Virtual Memory Compressed Cache + * - Swap In and Decompression Routines + * + * Author: Rodrigo S. de Castro + * Licensed under the GPL - 2001 + */ + +#include +#include +#include +#include + +#include + +#define ACTIVE_FRAGMENT 1 +#define INACTIVE_FRAGMENT 0 + +int last_state_accessed = 0; + +int +invalidate_comp_cache(struct address_space * mapping, unsigned long offset) +{ + struct comp_cache_fragment * fragment; + int err; + + spin_lock(&comp_cache_lock); + err = find_comp_page(mapping, offset, &fragment); + if (!err) + drop_fragment(fragment); + spin_unlock(&comp_cache_lock); + return err; +} + +EXPORT_SYMBOL(flush_comp_cache); + +int FASTCALL(flush_comp_cache(struct page *)); +int flush_comp_cache(struct page * page) +{ + struct comp_cache_fragment * fragment; + int err = -ENOENT; + + spin_lock(&comp_cache_lock); + + if (likely(!PageCompCache(page))) + goto out_unlock; + + /* we may have a null page->mapping if the page have been + * removed from swap cache before flushing its compressed + * cache entry, what may happen in do_swap_page() */ + err = find_comp_page(page->mapping?:&swapper_space, page->index, &fragment); + if (err) { + PageClearCompCache(page); + goto out_unlock; + } + + if (CompFragmentTestandClearDirty(fragment)) { + num_clean_fragments++; + list_del(&fragment->mapping_list); + list_add(&fragment->mapping_list, &fragment->mapping->clean_comp_pages); + + /* it may be already dirty if called from set_page_dirty() */ + __set_page_dirty(page); + } + + if (drop_fragment(fragment)) + PageClearCompCache(page); + out_unlock: + spin_unlock(&comp_cache_lock); + return err; +} + +extern inline void comp_cache_update_read_stats(struct comp_cache_fragment *); + +/* caller may hold pagecache_lock (__find_lock_page()) */ +int +__read_comp_cache(struct address_space *mapping, unsigned long offset, struct page * page, int state) +{ + static int adapt_ratio = 2, last_accessed = 0; + struct comp_cache_fragment * fragment; + int err; + + if (!PageLocked(page)) + BUG(); + + spin_lock(&comp_cache_lock); + err = find_comp_page(mapping, offset, &fragment); + + /* it will happen with vswap address only if the vswap address + * had a real address assigned */ + if (err) + goto out_unlock; + + if (!fragment_count(fragment)) + BUG(); + + get_fragment(fragment); + +#ifndef CONFIG_COMP_DIS_CLEAN + /* *** clean fragment policy *** + * + * All clean fragments read must account as +1 to + * last_state_accessed variable. These fragments are only + * accounted when we are not compressing clean pages + * (clean_page_compress_lock == 1). + */ + if (!CompFragmentDirty(fragment) && !clean_page_compress_lock) + last_state_accessed++; +#endif + +#if 0 +#ifndef CONFIG_COMP_DIS_ADAPT + /* -- version 4 -- */ + if (CompFragmentActive(fragment)) { + /* fragments from compcache active list */ + last_accessed++; + if (last_accessed >= adapt_ratio) + growth_lock = 1; + if (last_accessed >= 2 * adapt_ratio) { + compact_comp_cache(); + + growth_lock = 0; + last_accessed = 0; + } + } + else { + /* fragments from compcache inactive list */ + last_accessed--; + if (last_accessed <= (-1 * adapt_ratio)) { + growth_lock = 0; + last_accessed = 0; + } + } +#endif +#endif + +#if 1 +#ifndef CONFIG_COMP_DIS_ADAPT + /* -- version 2 -- */ + if (CompFragmentActive(fragment)) { + /* fragments from compcache active list */ + if (last_accessed == ACTIVE_FRAGMENT) { + if (growth_lock) { + compact_comp_cache(); + growth_lock = 0; + last_accessed = INACTIVE_FRAGMENT; + goto read; + } + growth_lock = 1; + goto read; + } + last_accessed = ACTIVE_FRAGMENT; + + /* Ver alair1 */ +/* compact_comp_cache(); */ + } + else { + /* fragments from compcache inactive list */ + growth_lock = 0; + last_accessed = INACTIVE_FRAGMENT; + } + read: +#endif +#endif + /* If only dirty fragmenst should be returned (when reading + * the page for writing it), free the fragment and return. A + * scenario where that happens is when writing a page: there + * is no point decompressing a clean fragment. */ + if (CompFragmentDirty(fragment) && state == DIRTY_PAGE) { + put_fragment(fragment); + drop_fragment(fragment); + goto out_unlock; + } + + spin_unlock(&comp_cache_lock); + lock_page(fragment->comp_page->page); + + decompress_fragment_to_page(fragment, page); + comp_cache_update_read_stats(fragment); + + spin_lock(&comp_cache_lock); + + if (CompFragmentTestandClearDirty(fragment)) { + num_clean_fragments++; + __set_page_dirty(page); + } + + UnlockPage(fragment->comp_page->page); + + put_fragment(fragment); + if (!drop_fragment(fragment)) + PageSetCompCache(page); + + out_unlock: + spin_unlock(&comp_cache_lock); + return err; +} + +#ifdef CONFIG_COMP_PAGE_CACHE +static void +truncate_list_comp_pages(struct list_head * list, unsigned long start, unsigned partial) +{ + struct list_head * fragment_lh, * tmp_lh; + struct comp_cache_fragment * fragment; + + spin_lock(&comp_cache_lock); + + list_for_each_safe(fragment_lh, tmp_lh, list) { + fragment = list_entry(fragment_lh, struct comp_cache_fragment, mapping_list); + + if ((fragment->index >= start) || (partial && (fragment->index + 1) == start)) + drop_fragment(fragment); + } + + spin_unlock(&comp_cache_lock); +} + +/* caller must hold pagecache_lock */ +void +truncate_comp_pages(struct address_space * mapping, unsigned long start, unsigned partial) +{ + truncate_list_comp_pages(&mapping->clean_comp_pages, start, partial); + truncate_list_comp_pages(&mapping->dirty_comp_pages, start, partial); +} + +/* caller must hold pagecache_lock */ +void +invalidate_comp_pages(struct address_space * mapping) +{ + truncate_list_comp_pages(&mapping->clean_comp_pages, 0, 0); +} + +/* caller must hold pagecache_lock */ +void +wait_comp_pages(struct address_space * mapping) +{ + struct page * page; + + while (!list_empty(&mapping->locked_comp_pages)) { + page = list_entry(mapping->locked_comp_pages.next, struct page, list); + + list_del_init(&page->list); + + spin_unlock(&pagecache_lock); + wait_on_page(page); + spin_lock(&pagecache_lock); + } +} + +extern struct page * find_or_add_page(struct page * new_page, struct address_space *mapping, unsigned long offset); + +/* caller must hold pagecache_lock */ +void +lookup_comp_pages(struct address_space * mapping) +{ + struct page * page, * old_page; + struct comp_cache_fragment * fragment; + + /*** + * This function may be called from the following code path: + * + * __sync_one() -> filemap_fdatasync() + * + * This code path tries to sync an inode (and keeps it locked + * while it is syncing). However, that inode can be also in + * the clear path (clear_inode() function, called in the exit + * process path) which will lock the super block and then wait + * for the inode, if locked (what happens when syncing it like + * here). + * + * Since the allocation path may write pages, which may need + * to lock the same super block, it will deadlock, because the + * super block is locked by the exit path explained above. So, + * we end up not being able to allocate the page (in order to + * finish this function and unlock the inode) _and_ the super + * block won't be unlocked since the inode doesn't get + * unlocked either. + * + * That's why the page must be allocated with GFP_NOFS mask. + */ + spin_unlock(&pagecache_lock); + page = alloc_page(GFP_NOFS); + if (!page) + return; + + spin_lock(&pagecache_lock); + if (list_empty(&mapping->dirty_comp_pages)) { + spin_unlock(&pagecache_lock); + goto out_release; + } + + spin_lock(&comp_cache_lock); + + fragment = list_entry(mapping->dirty_comp_pages.next, struct comp_cache_fragment, mapping_list); + + list_del(&fragment->mapping_list); + list_add(&fragment->mapping_list, &fragment->mapping->clean_comp_pages); + + get_fragment(fragment); + spin_unlock(&comp_cache_lock); + spin_unlock(&pagecache_lock); + + lock_page(fragment->comp_page->page); + + /* Checks if the page has been added to the page cache in the + * meanwhile and dirty the page in this case. Otherwise, add + * the new page to the page cache. */ + old_page = find_or_add_page(page, mapping, fragment->index); + if (!old_page) { + decompress_fragment_to_page(fragment, page); + goto free_and_dirty; + } + page_cache_release(page); + page = old_page; + + free_and_dirty: + UnlockPage(fragment->comp_page->page); + spin_lock(&comp_cache_lock); + + put_fragment(fragment); + + /* effectively free it */ + if (drop_fragment(fragment)) + PageClearCompCache(page); + spin_unlock(&comp_cache_lock); + __set_page_dirty(page); + UnlockPage(page); + out_release: + page_cache_release(page); +} +#endif + +/* + * Overrides for Emacs so that we follow Linus's tabbing style. + * Emacs will notice this stuff at the end of the file and automatically + * adjust the settings for this buffer only. This must remain at the end + * of the file. + * --------------------------------------------------------------------------- + * Local variables: + * c-file-style: "linux" + * End: + */ diff -urN linux-2.4.26-rc1/mm/comp_cache/swapout.c linux-2.4.26-rc1-ccache/mm/comp_cache/swapout.c --- linux-2.4.26-rc1/mm/comp_cache/swapout.c Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/mm/comp_cache/swapout.c Sun Mar 28 17:12:21 2004 @@ -0,0 +1,743 @@ +/* + * /mm/comp_cache/swapout.c + * + * Time-stamp: <2002-10-25 11:26:59 rcastro> + * + * Linux Virtual Memory Compressed Cache + * - Get Comp Cache Entries Routines + * + * Author: Rodrigo S. de Castro + * Licensed under the GPL - 2001 + */ + +#include +#include +#include +#include +#include +#include + +/* swap buffer */ +struct list_head swp_free_buffer_head, swp_used_buffer_head; +#ifdef CONFIG_COMP_SWAP +static struct { + unsigned short size; + unsigned short offset; + unsigned long index; +} grouped_fragments[255]; +#endif + +static spinlock_t swap_buffer_lock __cacheline_aligned = SPIN_LOCK_UNLOCKED; + +#define SWP_BUFFER_PRIORITY 6 + +static int +refill_swp_buffer(unsigned int gfp_mask, int priority) +{ + struct list_head * swp_buffer_lh; + struct page * buffer_page; + struct swp_buffer * swp_buffer; + struct comp_cache_fragment * fragment; + int wait, maxscan; + + maxscan = NUM_SWP_BUFFERS/priority; + wait = 0; + + try_again: + while(--maxscan >= 0 && (swp_buffer_lh = swp_used_buffer_head.prev) != &swp_used_buffer_head) { + swp_buffer = list_entry(swp_buffer_lh, struct swp_buffer, list); + buffer_page = swp_buffer->page; + list_del(swp_buffer_lh); + + if (TryLockPage(buffer_page)) { + if (!wait) + goto add_to_used; + spin_unlock(&swap_buffer_lock); + lock_page(buffer_page); + spin_lock(&swap_buffer_lock); + } + + /* remove from buffer_page->mapping->locked_comp_pages */ + list_del_init(&buffer_page->list); + + if (buffer_page->buffers) { + spin_unlock(&swap_buffer_lock); + if (!try_to_free_buffers(buffer_page, gfp_mask)) { + UnlockPage(buffer_page); + spin_lock(&swap_buffer_lock); + goto add_to_used; + } + spin_lock(&swap_buffer_lock); + } + + fragment = swp_buffer->fragment; + + /* A swap buffer page that has been set to dirty means + * that the writepage() function failed, so we cannot + * free the fragment and should simply backout. */ + if (PageDirty(buffer_page)) { + spin_lock(&comp_cache_lock); + if (fragment) { + fragment->swp_buffer = NULL; + spin_lock(&pagecache_lock); + list_del(&fragment->mapping_list); + list_add(&fragment->mapping_list, &fragment->mapping->dirty_comp_pages); + spin_unlock(&pagecache_lock); + + CompFragmentSetDirty(fragment); + } + ClearPageDirty(buffer_page); + spin_unlock(&comp_cache_lock); + goto add_to_free; + } + + /* A clean swap buffer page means that the writepage() + * didn't failed, so we can go on freeing the fragment + * (if still needed). */ + spin_lock(&comp_cache_lock); + if (fragment) { + fragment->swp_buffer = NULL; + drop_fragment(fragment); + } + spin_unlock(&comp_cache_lock); + add_to_free: + UnlockPage(buffer_page); + list_add_tail(swp_buffer_lh, &swp_free_buffer_head); + return 1; + add_to_used: + list_add(swp_buffer_lh, &swp_used_buffer_head); + } + + /* couldn't free any swap buffer? so let's start waiting for + * the lock from the locked pages */ + if (!wait) { + wait = 1; + maxscan = NUM_SWP_BUFFERS >> 3; + if (unlikely(current->need_resched)) { + __set_current_state(TASK_RUNNING); + spin_unlock(&swap_buffer_lock); + schedule(); + spin_lock(&swap_buffer_lock); + } + goto try_again; + } + return 0; +} + + +/** + * find_free_swp_buffer - gets a swap buffer page + * @fragment: the fragment which will be decompressed to this buffer + * page. + * + * If there's a free buffer page, it will lock the page and + * return. Otherwise we may sleep to get the lock. + */ +static struct swp_buffer * +find_free_swp_buffer(struct comp_cache_fragment * fragment, unsigned int gfp_mask) +{ + struct page * buffer_page; + struct list_head * swp_buffer_lh; + struct swp_buffer * swp_buffer = NULL; + int priority = SWP_BUFFER_PRIORITY; + + if (!fragment) + BUG(); + + spin_lock(&swap_buffer_lock); + + if (!list_empty(&swp_free_buffer_head)) + goto get_free_buffer; + + while (list_empty(&swp_free_buffer_head) && priority) + refill_swp_buffer(gfp_mask, priority--); + + /* Failed to get a free swap buffer. Probably gfp_mask does + * not allow buffer sync in refill_swp_buffer() function. */ + if (list_empty(&swp_free_buffer_head)) + goto out_unlock; + + get_free_buffer: + swp_buffer = list_entry(swp_buffer_lh = swp_free_buffer_head.prev, struct swp_buffer, list); + + buffer_page = swp_buffer->page; + + if (TryLockPage(buffer_page)) + BUG(); + if (page_count(buffer_page) != 1) + BUG(); + + list_del(swp_buffer_lh); + + spin_lock(&comp_cache_lock); + swp_buffer->fragment = fragment; + fragment->swp_buffer = swp_buffer; + + buffer_page->index = fragment->index; + buffer_page->mapping = fragment->mapping; + spin_unlock(&comp_cache_lock); + + spin_lock(&pagecache_lock); + if (!fragment->mapping) + BUG(); + list_add(&buffer_page->list, &fragment->mapping->locked_comp_pages); + spin_unlock(&pagecache_lock); + + list_add(swp_buffer_lh, &swp_used_buffer_head); + out_unlock: + spin_unlock(&swap_buffer_lock); + return swp_buffer; +} + +extern inline void comp_cache_update_written_stats(struct comp_cache_fragment *); +extern void set_swap_compressed(swp_entry_t, int); + +#ifdef CONFIG_COMP_SWAP +static void +group_fragments(struct comp_cache_fragment * fragment, struct page * page) { + struct list_head * fragment_lh; + struct comp_cache_fragment * aux_fragment; + swp_entry_t entry, real_entry; + unsigned short counter, next_offset, metadata_size; + + entry.val = fragment->index; + real_entry = get_real_swap_page(entry); + + if (!real_entry.val) + BUG(); + + /*** + * Metadata: for each swap block + * + * Header: + * 4 bytes -> + * number of fragments (unsigned short) + * offset for fragment metadata (unsigned short) + * + * Tail: + * - for every fragment - + * 8 bytes -> + * offset (unsigned short) + * compressed size (unsigned short) + * index (unsigned long) + */ + metadata_size = 8; + next_offset = 4; + + /* cannot store the fragment in compressed format */ + if (next_offset + fragment->compressed_size + metadata_size > PAGE_SIZE) { + set_swap_compressed(entry, 0); + decompress_fragment_to_page(fragment, page); + return; + } + + /* prepare header with data from the 1st fragment */ + set_swap_compressed(entry, 1); + + counter = 1; + grouped_fragments[0].size = fragment->compressed_size; + grouped_fragments[0].offset = next_offset; + grouped_fragments[0].index = fragment->index; + + memcpy(page_address(page) + next_offset, page_address(fragment->comp_page->page) + fragment->offset, fragment->compressed_size); + + next_offset += fragment->compressed_size; + + /* try to group other fragments */ + for_each_fragment(fragment_lh, fragment->comp_page) { + aux_fragment = list_entry(fragment_lh, struct comp_cache_fragment, list); + + if (aux_fragment == fragment) + continue; + if (!PageSwapCache(aux_fragment)) + continue; + if (!CompFragmentDirty(aux_fragment)) + continue; + entry.val = aux_fragment->index; + if (vswap_address(entry)) + continue; + if (next_offset + aux_fragment->compressed_size + metadata_size + 8 > PAGE_SIZE) + continue; + + CompFragmentClearDirty(aux_fragment); + num_clean_fragments++; + + set_swap_compressed(entry, 1); + map_swap(entry, real_entry); + + grouped_fragments[counter].size = aux_fragment->compressed_size; + grouped_fragments[counter].offset = next_offset; + grouped_fragments[counter].index = aux_fragment->index; + + memcpy(page_address(page) + next_offset, page_address(fragment->comp_page->page) + aux_fragment->offset, aux_fragment->compressed_size); + + next_offset += aux_fragment->compressed_size; + metadata_size += 8; + counter++; + } + + memcpy(page_address(page), &counter, 2); + memcpy(page_address(page) + 2, &next_offset, 2); + + while (counter--) { + memcpy(page_address(page) + next_offset, &(grouped_fragments[counter].size), 2); + next_offset += 2; + memcpy(page_address(page) + next_offset, &(grouped_fragments[counter].offset), 2); + next_offset += 2; + memcpy(page_address(page) + next_offset, &(grouped_fragments[counter].index), 4); + next_offset += 4; + } +} +#else +static void +group_fragments(struct comp_cache_fragment * fragment, struct page * page) +{ + swp_entry_t entry; + + /* uncompressed fragments or fragments that cannot have the + * metadata written together must be decompressed */ + entry.val = fragment->index; + if (fragment->compressed_size + sizeof(unsigned short) > PAGE_SIZE) { + set_swap_compressed(entry, 0); + decompress_fragment_to_page(fragment, page); + return; + } + + /* copy the compressed data and metadata */ + memcpy(page_address(page), &(fragment->compressed_size), sizeof(unsigned short)); + memcpy(page_address(page) + sizeof(unsigned short), page_address(fragment->comp_page->page) + fragment->offset, fragment->compressed_size); + set_swap_compressed(entry, 1); +} +#endif + +static struct swp_buffer * +prepare_swp_buffer(struct comp_cache_fragment * fragment, unsigned int gfp_mask) { + struct page * buffer_page; + struct swp_buffer * swp_buffer; + + swp_buffer = find_free_swp_buffer(fragment, gfp_mask); + if (!swp_buffer) + return NULL; + + buffer_page = swp_buffer->page; + + if (!buffer_page) + BUG(); + + lock_page(fragment->comp_page->page); + + /* pages from page cache need to have its data decompressed */ + if (!PageSwapCache(fragment)) { + decompress_fragment_to_page(fragment, buffer_page); + goto out_unlock; + } + + group_fragments(fragment, buffer_page); + out_unlock: + UnlockPage(fragment->comp_page->page); + comp_cache_update_written_stats(fragment); + + buffer_page->flags &= (1 << PG_locked); + return swp_buffer; +} + +extern struct address_space swapper_space; +extern int last_state_accessed; + +/* writeout_fragments() - write out some pages in the lru order until + * we have a comp_page where we can store the new page */ +int +writeout_fragments(unsigned int gfp_mask, int nrpages, int priority) { + int (*writepage)(struct page *); + struct list_head * fragment_lh, * list; + int maxscan, swap_cache_page, ratio; + struct comp_cache_fragment * fragment; + struct swp_buffer * swp_buffer; + swp_entry_t entry; + + balance_lru_queues(); + + list = &inactive_lru_queue; + maxscan = max((int) ((num_fragments - num_active_fragments)/priority), (int) (nrpages * 2)); + + if (list_empty(&inactive_lru_queue)) { + list = &active_lru_queue; + maxscan = max((int) (num_active_fragments/priority), (int) (nrpages * 2)); + } + + while (!list_empty(list) && maxscan--) { + fragment = list_entry(fragment_lh = list->prev, struct comp_cache_fragment, lru_queue); + + if (!fragment_count(fragment)) + BUG(); + + /* move it to the back of the list */ + list_del(fragment_lh); + list_add(fragment_lh, list); + + /* clean page, let's free it */ + if (!CompFragmentDirty(fragment)) { +#ifndef CONFIG_COMP_DIS_CLEAN + /* *** clean fragment policy *** + * + * All clean fragments to be freed accounts as + * -1 to last_state_accessed variable. These + * fragments are only accounted while we are + * compressing clean pages + * (clean_page_compress_lock == 1). + */ + if (!clean_page_compress_lock) { + last_state_accessed--; + ratio = -((num_clean_fragments * 40)/num_fragments); + if (ratio > -5) + ratio = -5; + if (last_state_accessed < ratio) { + clean_page_compress_lock = 1; + last_state_accessed = 0; + } + } +#endif + + drop_fragment(fragment); + + if (!--nrpages) + break; + goto try_again; + } + + /* we can't perform IO, so we can't go on */ + if (!(gfp_mask & __GFP_FS)) + goto try_again; + + if ((swap_cache_page = PageSwapCache(fragment))) { + entry.val = fragment->index; + if (vswap_address(entry)) + BUG(); + } + + /* let's remove it from lru queue now (and not only + * when free the fragment) in order to avoid races */ + remove_fragment_from_lru_queue(fragment); + + /* avoid to free this entry if we sleep below */ + if (swap_cache_page && !swap_duplicate(entry)) + BUG(); + + get_fragment(fragment); + spin_unlock(&comp_cache_lock); + + swp_buffer = prepare_swp_buffer(fragment, gfp_mask); + if (!swp_buffer) + goto out; + + spin_lock(&pagecache_lock); + list_del(&fragment->mapping_list); + list_add(&fragment->mapping_list, &fragment->mapping->clean_comp_pages); + spin_unlock(&pagecache_lock); + + CompFragmentClearDirty(fragment); + num_clean_fragments++; + + writepage = fragment->mapping->a_ops->writepage; + + if (!writepage) + BUG(); + + writepage(swp_buffer->page); + nrpages--; + out: + if (swap_cache_page) + swap_free(entry); + spin_lock(&comp_cache_lock); + + if (!swp_buffer) { + if (likely(list == &inactive_lru_queue)) + add_fragment_to_inactive_lru_queue(fragment); + else + add_fragment_to_active_lru_queue(fragment); + put_fragment(fragment); + goto try_again; + } + + put_fragment(fragment); + + if (!nrpages) + break; + + try_again: + if (unlikely(current->need_resched)) { + spin_unlock(&comp_cache_lock); + __set_current_state(TASK_RUNNING); + schedule(); + spin_lock(&comp_cache_lock); + } + } + + return (!nrpages); +} + +/*** + * get_comp_cache_page - gets a Compressed Cache entry, freeing one if + * necessary. This is done writing out the data the chosen (to be + * freed) entry stores. + * + * return value: pointer to the comp cache entry to be used. The comp + * cache entry and its page will be _both_ locked. + * + * @swap_cache_page: the swap cache page for which we are getting a + * comp cache entry + * + * @compressed_size: size of swap cache page in compressed state + * + * @fragment: variable that will store the fragment to store the + * compressed data + * + * @gfp_mask: we need to know if we can perform IO */ +struct comp_cache_page * +get_comp_cache_page(struct page * page, unsigned short compressed_size, struct comp_cache_fragment ** fragment_out, unsigned int gfp_mask, int priority) +{ + struct comp_cache_page * comp_page = NULL, ** hash_table; + struct comp_cache_fragment * fragment = NULL; + struct page * new_page; + unsigned short aux_comp_size; + int maxscan, maxtry; + + if (!page) + BUG(); + if (!PageLocked(page)) + BUG(); + if (compressed_size > PAGE_SIZE) + BUG(); + + page_cache_get(page); +#ifndef CONFIG_COMP_DIS_ADAPT + maxtry = 5; +#else + maxtry = 4; +#endif + hash_table = free_space_hash; + + while (maxtry--) { + aux_comp_size = compressed_size; + maxscan = 3; + + while (maxscan--) { + comp_page = search_comp_page(hash_table, aux_comp_size); + + /* no comp_page, that comp_page->free_space > compressed_size */ + if (!comp_page) + break; + + if (!comp_page->page) { + if (comp_page->free_space != COMP_PAGE_SIZE) + BUG(); + goto alloc_new_page; + } + + aux_comp_size = 0; + + while (comp_page && TryLockPage(comp_page->page)) { + if (hash_table == free_space_hash) { + if (aux_comp_size < comp_page->free_space) + aux_comp_size = comp_page->free_space + 1; + + do { + comp_page = comp_page->next_hash_fs; + } while (comp_page && comp_page->free_space < compressed_size); + } + else { + if (aux_comp_size < comp_page->total_free_space) + aux_comp_size = comp_page->total_free_space + 1; + + do { + comp_page = comp_page->next_hash_tfs; + } while (comp_page && comp_page->total_free_space < compressed_size); + } + + } + + if (!comp_page) + continue; + + /* remove from free space hash table before update */ + remove_comp_page_from_hash_table(comp_page); + + /* found a page whose total free space (ie, + * free space + fragment space) is enough? so + * we need to compact it */ + if (hash_table == total_free_space_hash) + compact_fragments(comp_page); + + if (comp_page->free_space < compressed_size) + BUG(); + + goto check_references; + } + + if (hash_table == free_space_hash) { + hash_table = total_free_space_hash; + continue; + } + + hash_table = free_space_hash; + +#ifndef CONFIG_COMP_DIS_ADAPT + /* *** adaptability policy *** + * + * We couldn't find a comp page with enough free space + * to store the new fragment. Let's then check if we + * are able to grow the compressed cache on demand. + */ + if (grow_on_demand()) + continue; +#endif + + if (!writeout_fragments(gfp_mask, SWAP_CLUSTER_MAX, priority)) + goto out_unlock; + + if (unlikely(current->need_resched)) { + spin_unlock(&comp_cache_lock); + __set_current_state(TASK_RUNNING); + schedule(); + spin_lock(&comp_cache_lock); + } + + /* that's going to be rechecked in shrink_cache() with + * pagecache lock held */ + if (page_count(page) - !!page->buffers == 3) + continue; + + break; + } + + out_unlock: + if (comp_page) + BUG(); + out_release: + page_cache_release(page); + return comp_page; + + alloc_new_page: + /* remove from free space hash table before update */ + remove_comp_page_from_hash_table(comp_page); + + if (comp_page->page) + BUG(); + + spin_unlock(&comp_cache_lock); + new_page = alloc_page(gfp_mask); + spin_lock(&comp_cache_lock); + + if (!new_page) + goto failed; + + set_comp_page(comp_page, new_page); + + if (TryLockPage(comp_page->page)) + BUG(); + + check_references: + /* 3 = shrink_cache() + page cache + us. That's going to be + * rechecked in shrink_cache() with pagecache lock held */ + if (page_count(page) - !!page->buffers != 3) + goto failed; + + if (!comp_page->free_space) + BUG(); + + /* allocate the new fragment */ + fragment = alloc_fragment(); + + if (!fragment) + goto failed; + + fragment->index = page->index; + fragment->mapping = page->mapping; + fragment->offset = comp_page->free_offset; + fragment->swp_buffer = NULL; + fragment->compressed_size = compressed_size; + fragment->flags = 0; + fragment->comp_page = comp_page; + set_fragment_count(fragment, 1); + + /* let's update some important fields */ + comp_page->free_space -= compressed_size; + comp_page->total_free_space -= compressed_size; + comp_page->free_offset += compressed_size; + + last_page_size[last_page] = compressed_size; + last_page = (last_page++)%NUM_MEAN_PAGES; + + comp_cache_free_space -= compressed_size; + + add_to_comp_page_list(comp_page, fragment); + add_fragment_vswap(fragment); + add_fragment_to_hash_table(fragment); + add_fragment_to_active_lru_queue(fragment); + + /* we need to account the number of fragments that are from + * swapper_space to correctly count the cached, swapcached + * memory in /proc/meminfo */ + if (PageSwapCache(page)) + num_swapper_fragments++; + num_fragments++; + + balance_lru_queues(); + + if (comp_page->free_space < 0) + BUG(); + + *fragment_out = fragment; + + add_comp_page_to_hash_table(comp_page); + + if ((*fragment_out)->compressed_size != compressed_size) + BUG(); + + goto out_release; + + failed: + add_comp_page_to_hash_table(comp_page); + if (comp_page->page) + UnlockPage(comp_page->page); + comp_page = NULL; + goto out_unlock; +} + +extern kmem_cache_t * comp_cachep; + +void __init +comp_cache_swp_buffer_init(void) { + struct page * buffer_page; + struct swp_buffer * swp_buffer; + int i; + + INIT_LIST_HEAD(&swp_free_buffer_head); + INIT_LIST_HEAD(&swp_used_buffer_head); + + for (i = 0; i < NUM_SWP_BUFFERS; i++) { + swp_buffer = (struct swp_buffer *) kmalloc(sizeof(struct swp_buffer), GFP_KERNEL); + + buffer_page = swp_buffer->page = alloc_page(GFP_KERNEL); + + if (!buffer_page) + panic("comp_cache_swp_buffer_init(): cannot allocate page"); + + list_add(&(swp_buffer->list), &swp_free_buffer_head); + SetPageDirty(buffer_page); + INIT_LIST_HEAD(&buffer_page->list); + } +} + +/* + * Overrides for Emacs so that we follow Linus's tabbing style. + * Emacs will notice this stuff at the end of the file and automatically + * adjust the settings for this buffer only. This must remain at the end + * of the file. + * --------------------------------------------------------------------------- + * Local variables: + * c-file-style: "linux" + * End: + */ + + diff -urN linux-2.4.26-rc1/mm/comp_cache/vswap.c linux-2.4.26-rc1-ccache/mm/comp_cache/vswap.c --- linux-2.4.26-rc1/mm/comp_cache/vswap.c Thu Jan 1 01:00:00 1970 +++ linux-2.4.26-rc1-ccache/mm/comp_cache/vswap.c Sun Mar 28 17:12:21 2004 @@ -0,0 +1,817 @@ +/* + * linux/mm/comp_cache/vswap.c + * + * Time-stamp: <2002-07-27 11:15:46 rcastro> + * + * Linux Virtual Memory Compressed Cache + * + * Author: Rodrigo S. de Castro + * Licensed under the GPL - 2001 + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +kmem_cache_t * vswap_cachep; + +struct vswap_address ** vswap_address = NULL; +struct list_head vswap_address_used_head; +struct list_head vswap_address_free_head; + +static struct pte_list * pte_list_freelist = NULL; + +/* comp_cache_freeable_space = this is the estimated freeable space in + * the compressed cache, ie, the space in compressed cache minus the + * space used by virtual swap entries */ +unsigned long comp_cache_freeable_space; + +/* vswap total size */ +unsigned long vswap_current_num_entries; + +/* it counts the number of reserved vswap entries, ie the number of + * vswap entries that have not yet been compressed */ +unsigned long vswap_num_reserved_entries; + +/* total number of vswap entries that are used, no matter they are + * only reserved or compressed */ +unsigned long vswap_num_used_entries; + +/* number of vswap entries that have swap cache pages */ +unsigned long vswap_num_swap_cache; + +/* index to the last used vswap entry - used when shrinking vswap */ +unsigned int vswap_last_used; + +/* last vswap index which has been allocated contiguously (the + * following position is empty). This index will be used to try to + * allocate in case the vswap addresses are over) */ +int last_vswap_allocated; + +unsigned short * last_page_size; +unsigned short last_page = 0; + +unsigned long nr_free_vswap = 0, nr_used_vswap = 0; + +spinlock_t virtual_swap_list __cacheline_aligned = SPIN_LOCK_UNLOCKED; + +/* the caller must hold the virtual_swap_list lock */ +static int +comp_cache_vswap_alloc(void) +{ + unsigned long i; + unsigned int failed_alloc = 0; + + vswap_cachep = kmem_cache_create("comp_cache_vswap", sizeof(struct vswap_address), 0, SLAB_HWCACHE_ALIGN, NULL, NULL); + + vswap_address = (struct vswap_address **) vmalloc(NUM_VSWAP_ENTRIES * sizeof(struct vswap_address*)); + + if (!vswap_address) + return 0; + + vswap_current_num_entries = NUM_VSWAP_ENTRIES; + vswap_last_used = NUM_VSWAP_ENTRIES - 1; + vswap_num_used_entries = 0; + vswap_num_swap_cache = 0; + + last_vswap_allocated = NUM_VSWAP_ENTRIES - 1; + for (i = 0; i < NUM_VSWAP_ENTRIES; i++) { + if (!vswap_alloc_and_init(vswap_address, i) && !failed_alloc) { + failed_alloc = 1; + last_vswap_allocated = i - 1; + } + } + return 1; +} + +static inline int +comp_cache_mean_size(void) { + unsigned long total = 0; + int i, mean_size; + + for (i = 0; i < NUM_MEAN_PAGES; total += last_page_size[i], i++); + + mean_size = total/NUM_MEAN_PAGES; + + if (mean_size < 0 || mean_size > COMP_PAGE_SIZE) + BUG(); + + return mean_size; +} + +/*** + * comp_cache_available_vswap - this function returns 1 if we have any + * available vswap entry and also if we can assign any vswap entry. + * + * the caller must hold virtual_swap_list lock + */ +static int +comp_cache_available_vswap(void) { + unsigned short available_mean_size; + unsigned long i; + + /* that should avoid problems when looking for a place in + * compressed cache, even if we have to move fragments in + * order to make room for any vswap entry */ + if (vswap_num_reserved_entries > comp_page_to_page(num_comp_pages)) + return 0; + + /* no more free vswap address? */ + if (list_empty(&vswap_address_free_head)) { + /* have all vswap addresses already been allocated? */ + if (last_vswap_allocated == vswap_current_num_entries - 1) + return 0; + + if (vswap_address[last_vswap_allocated + 1]) + BUG(); + + /* allocate an index that has failed to allocate */ + if (!vswap_alloc_and_init(vswap_address, last_vswap_allocated + 1)) + return 0; + + /* update the last_vswap_allocated variable to the + * actual last vswap allocated entry */ + for (i = last_vswap_allocated + 1; i < vswap_current_num_entries && vswap_address[i]; i++); + last_vswap_allocated = i - 1; + + return 1; + } + + /* or too many used entries for the current compressed cache + * size? so no available space */ + if (vswap_num_used_entries >= NUM_VSWAP_ENTRIES) + return 0; + + available_mean_size = (unsigned short) (comp_cache_freeable_space/num_comp_pages); + + if (available_mean_size > COMP_PAGE_SIZE) + BUG(); + + /* it the available mean size in the comp cache bigger than + * the mean size of the last pages? if it's not, return 0 */ + if (available_mean_size < comp_cache_mean_size()) + return 0; + + return 1; +} + +/*** + * comp_cache_available_space - broader function which will check also + * if we have virtual swap entries to be compressed. + */ +int +comp_cache_available_space(void) { + int ret = 1; + + spin_lock(&virtual_swap_list); + + if (comp_cache_available_vswap()) + goto out_unlock; + + /* can we still compress all these entries? */ + if (vswap_num_reserved_entries > 0) + goto out_unlock; + + ret = 0; + out_unlock: + spin_unlock(&virtual_swap_list); + return ret; +} + +/** + * get_virtual_swap_page - returns a vswap entry, if available, and + * initializes it. + * + * This function checks if we have available virtual swap space. If we + * do, it removes a virtual swap entry from the free list, zeroing the + * data struct, but the count, which will be set to 1. Updates the + * control variables: number of used entries and number of reserved + * entries. + * + */ +swp_entry_t +get_virtual_swap_page(void) +{ + swp_entry_t entry; + int type; + unsigned long offset; + struct vswap_address * vswap; + + entry.val = 0; + + spin_lock(&virtual_swap_list); + + if (!vswap_address && !comp_cache_vswap_alloc()) + goto out_unlock; + + if (!comp_cache_available_vswap()) + goto out_unlock; + + vswap = list_entry(vswap_address_free_head.next, struct vswap_address, list); + list_del_init(vswap_address_free_head.next); + nr_free_vswap--; + + type = COMP_CACHE_SWP_TYPE; + offset = vswap->offset; + + if (offset >= vswap_current_num_entries) + BUG(); + if (!vswap_address[offset]) + BUG(); + if (vswap_address[offset]->fragment) + BUG(); + if (vswap_address[offset]->swap_count) + BUG(); + + vswap_address[offset]->swap_count = 1; + vswap_address[offset]->pte_list = NULL; + vswap_address[offset]->swap_cache_page = NULL; + vswap_address[offset]->fragment = VSWAP_RESERVED; + + vswap_num_used_entries++; + vswap_num_reserved_entries++; + + entry = SWP_ENTRY(type, offset); + + if (!vswap_address(entry)) + BUG(); + + out_unlock: + spin_unlock(&virtual_swap_list); + return entry; +} + +/** + * virtual_swap_duplicate - swap_duplicate for virtual swap addresses. + * + * @entry: the virtual swap entry which will have its count + * incremented + */ +int +virtual_swap_duplicate(swp_entry_t entry) +{ + unsigned long offset = SWP_OFFSET(entry); + + if (!vswap_address(entry)) + BUG(); + if (!vswap_address[offset]) + BUG(); + if (!vswap_address[offset]->swap_count) + return 0; + if (offset >= vswap_current_num_entries) + return 0; + spin_lock(&virtual_swap_list); + vswap_address[offset]->swap_count++; + spin_unlock(&virtual_swap_list); + return 1; +} + +/** + * virtual_swap_free - swap_free() for virtual swap addresses. + * + * @entry: the virtual swap entry which will have its count + * decremented and possibly the vswap entry freed. + * + * This function will decrement the vswap entry counter. If we get to + * swap_count == 0, the entry will have its struct initalized and be + * added to the free list. In the case we have a fragment (recall that + * fragments don't hold references on swap addresses), we will free it + * too. + * + * the caller must hold virtual_swap_list lock + */ +int +__virtual_swap_free(unsigned long offset) +{ + unsigned int swap_count; + struct comp_cache_fragment * fragment; + struct vswap_address * vswap; + + if (offset >= vswap_current_num_entries) + BUG(); + + vswap = vswap_address[offset]; + fragment = vswap->fragment; + + if (!vswap) + BUG(); + + if (!vswap->swap_count) + BUG(); + + swap_count = vswap->swap_count; + if (--swap_count) { + vswap->swap_count = swap_count; + return swap_count; + } + + vswap_num_used_entries--; + + if (vswap->pte_list) + BUG(); + + if (vswap->swap_cache_page) + BUG(); + + vswap->swap_count = 0; + vswap->pte_list = NULL; + vswap->swap_cache_page = NULL; + vswap->fragment = NULL; + + /* if this entry is reserved, it's not on any list (either + * because it has never had a fragment or the fragment has + * already been remove in remove_fragment_vswap()), so we can + * add it right away to the free list */ + if (fragment == VSWAP_RESERVED) { + vswap_num_reserved_entries--; + goto out; + } + + if (!fragment) + BUG(); + + /* remove from used list */ + list_del(&(vswap_address[offset]->list)); + nr_used_vswap--; + + /* global freeable space */ + comp_cache_freeable_space += fragment->compressed_size; + out: + /* add to to the free list */ + list_add(&(vswap->list), &vswap_address_free_head); + nr_free_vswap++; + return 0; +} + +/* caller must hold vswap_list_lock + * retuns virtual_swap_list unlocked */ +int +virtual_swap_free(unsigned long offset) +{ + struct comp_cache_fragment * fragment; + int ret; + + fragment = vswap_address[offset]->fragment; + ret = __virtual_swap_free(offset); + + if (ret) + goto out_unlock; + + if (fragment == VSWAP_RESERVED) + goto out_unlock; + + spin_unlock(&virtual_swap_list); + + spin_lock(&comp_cache_lock); + drop_fragment(fragment); + spin_unlock(&comp_cache_lock); + out: + return ret; + out_unlock: + spin_unlock(&virtual_swap_list); + goto out; +} + +/*** + * remove_fragment_vswap - this function tells the vswap entry that it + * doesn't have a compressed fragment any longer. + * @fragment: fragment which will removed from the vswap entry struct. + * + * Based on the fragment->index, this function sets the fragment field + * of vswap struct to the VSWAP_RESERVED, removes the vswap entry from + * the used list and update some control variables: the number of + * reserved entries (entries w/o fragment) and also the freeable + * space. Once the fragment is removed, this vswap entry will be taken + * as reserved, since it is used but doesn't have a compressed + * fragment. The number of reserved entries will be used to know if we + * can or not give away new vswap entries. + * + */ +inline void +remove_fragment_vswap(struct comp_cache_fragment * fragment) +{ + swp_entry_t entry; + unsigned long offset; + + if (!PageSwapCache(fragment)) + return; + + entry.val = fragment->index; + if (!vswap_address(entry)) + return; + + spin_lock(&virtual_swap_list); + + offset = SWP_OFFSET(entry); + if (!vswap_address[offset]->swap_count) + goto out_unlock; + + if (reserved(offset) || !vswap_address[offset]->fragment) + BUG(); + + vswap_address[offset]->fragment = VSWAP_RESERVED; + + /* remove the virtual swap entry from the used list, since + * there will be no longer a compressed page set to this + * address */ + list_del_init(&(vswap_address[offset]->list)); + nr_used_vswap--; + + comp_cache_freeable_space += fragment->compressed_size; + vswap_num_reserved_entries++; + out_unlock: + spin_unlock(&virtual_swap_list); +} + +/*** + * add_fragment_vswap - this function tells the vswap entry that it + * has a compressed fragment. + * @fragment: fragment which will added to the vswap entry struct. + * + * Based on the fragment->index, this function sets the fragment field + * of vswap struct to the fragment address, adds the vswap entry to + * the used list and update some control variables: the number of + * reserved entries (entries w/o fragment) and also the freeable space + * (since this fragment does not have backing storage). + * + */ +inline void +add_fragment_vswap(struct comp_cache_fragment * fragment) +{ + swp_entry_t entry; + unsigned long offset; + + if (!PageSwapCache(fragment)) + return; + + entry.val = fragment->index; + if (!vswap_address(entry)) + return; + + spin_lock(&virtual_swap_list); + + offset = SWP_OFFSET(entry); + + if (!reserved(offset)) + BUG(); + if (!vswap_address[offset]->swap_count) + BUG(); + + vswap_address[offset]->fragment = fragment; + + if (!list_empty(&(vswap_address[offset]->list))) + BUG(); + + list_add(&(vswap_address[offset]->list), &vswap_address_used_head); + nr_used_vswap++; + + comp_cache_freeable_space -= fragment->compressed_size; + vswap_num_reserved_entries--; + + spin_unlock(&virtual_swap_list); +} + +/** + * pte_list_free - free pte_list structure + * @pte_list: pte_list struct to free + * @prev_pte_list: previous pte_list on the list (may be NULL) + * @page: page this pte_list hangs off (may be NULL) + * + * This function unlinks pte_list from the singly linked list it + * may be on and adds the pte_list to the free list. May also be + * called for new pte_list structures which aren't on any list yet. + * + * caller needs to hold the pagemap_lru_list. + * + * (adapted from Rik van Riel's rmap patch) + */ +static inline void +pte_list_free(struct pte_list * pte_list, struct pte_list * prev_pte_list, unsigned long offset) +{ + if (prev_pte_list) + prev_pte_list->next = pte_list->next; + else + vswap_address[offset]->pte_list = pte_list->next; + + pte_list->ptep = NULL; + pte_list->next = pte_list_freelist; + pte_list_freelist = pte_list; +} + +/** + * alloc_new_pte_lists - convert a free page to pte_list structures + * + * Grabs a free page and converts it to pte_list structures. We really + * should pre-allocate these earlier in the pagefault path or come up + * with some other trick. + * + * Note that we cannot use the slab cache because the pte_list structure + * is way smaller than the minimum size of a slab cache allocation. + */ +static void alloc_new_pte_lists(void) +{ + struct pte_list * pte_list = (void *) get_zeroed_page(GFP_ATOMIC); + int i = PAGE_SIZE / sizeof(struct pte_list); + + if (pte_list) { + for (; i-- > 0; pte_list++) { + pte_list->ptep = NULL; + pte_list->next = pte_list_freelist; + pte_list_freelist = pte_list; + } + } else + panic("Fix pte_list allocation, you lazy bastard!\n"); +} + +/** + * pte_list_alloc - allocate a pte_list struct + * + * Returns a pointer to a fresh pte_list structure. Allocates new + * pte_list structures as required. + * + * caller needs to hold the pagemap_lru_lock. + * + * (adapted from Rik van Riel's rmap patch) + */ +static inline struct pte_list * pte_list_alloc(void) +{ + struct pte_list * pte_list; + + /* Allocate new pte_list structs as needed. */ + if (!pte_list_freelist) + alloc_new_pte_lists(); + + /* Grab the first pte_list from the freelist. */ + pte_list = pte_list_freelist; + pte_list_freelist = pte_list->next; + pte_list->next = NULL; + + return pte_list; +} + +/** + * free_pte_list - free all the ptes listed on pte_list and returns + * the number of ptes freed + * @pte_list: pointer to the pte list we are going to free + * @offset: vswap offset + */ +int FASTCALL(free_pte_list(struct pte_list *, unsigned long)); +int free_pte_list(struct pte_list * pte_list, unsigned long offset) +{ + struct pte_list * next_pte_list; + int counter = 0; + + next_pte_list = pte_list; + while ((pte_list = next_pte_list)) { + next_pte_list = pte_list->next; + + pte_list_free(pte_list, NULL, offset); + + counter++; + } + + return counter; +} + +/** + * remove_pte_vswap - remove a pte from a vswap entry + * @ptep: pointer to the pte to be removed + * + * This function searchs for the pte in the pte_list from its vswap + * address entry, freeing it. Returns for null, present and real swap + * adressed ptes. + * + * caller must hold the mm->page_table_lock. + * + * (adapted from Rik van Riel's rmap patch) + */ +void FASTCALL(remove_pte_vswap(pte_t *)); +void remove_pte_vswap(pte_t * ptep) +{ + struct pte_list * pte_list, * prev_pte_list = NULL; + unsigned long offset; + + if (pte_present(*ptep)) + return; + + if (!vswap_address(pte_to_swp_entry(*ptep))) + return; + + spin_lock(&virtual_swap_list); + + offset = SWP_OFFSET(pte_to_swp_entry(*ptep)); + pte_list = vswap_address[offset]->pte_list; + while (pte_list) { + if (pte_list->ptep == ptep) { + pte_list_free(pte_list, prev_pte_list, offset); + spin_unlock(&virtual_swap_list); + return; + } + + prev_pte_list = pte_list; + pte_list = pte_list->next; + } + + /* not found, should never happen. */ + printk("remove_pte_vswap: pte_list %p not present...\n", ptep); + printk("remove_pte_vswap: only found: "); + pte_list = vswap_address[offset]->pte_list; + while (pte_list) { + printk("%p ", pte_list->ptep); + pte_list = pte_list->next; + } + printk("\n"); + panic("remove_pte_vswap: giving up.\n"); +} + +/** + * add_pte_vswap - add a pte to a vswap entry pte list + * @ptep: pointer to the pte to be added + * @entry: vswap entry to whose list will be added the ptep + * + * This function allocates a new pte_list struct, adding to the + * corresponding virtual swap entry struct. Returns null for real swap + * entries. + * + * caller must hold the mm->page_table_lock. + * + * (adapted from Rik van Riel's rmap patch) + */ +void FASTCALL(add_pte_vswap(pte_t *, swp_entry_t)); +void add_pte_vswap(pte_t * ptep, swp_entry_t entry) { + unsigned long offset; + struct pte_list * pte_list; + + if (!vswap_address(entry)) + return; + + spin_lock(&virtual_swap_list); + offset = SWP_OFFSET(entry); + + pte_list = pte_list_alloc(); + pte_list->next = vswap_address[offset]->pte_list; + pte_list->ptep = ptep; + + vswap_address[offset]->pte_list = pte_list; + spin_unlock(&virtual_swap_list); +} + + + +/** + * add_swap_cache_page_vswap - adds the swap cache page to the + * corresponding vswap entry. + * @page: page to be added to the struct + * @entry: vswap address + * + * This function adds the virtual swap address entry struct to the + * page passed as parameter. It's useful to avoid looking for the page + * whenever some change to the vswap adress is needed. + * + */ +void FASTCALL(add_swap_cache_page_vswap(struct page *, swp_entry_t)); +void add_swap_cache_page_vswap(struct page * page, swp_entry_t entry) +{ + unsigned long offset; + + if (!vswap_address(entry)) + return; + + spin_lock(&virtual_swap_list); + + offset = SWP_OFFSET(entry); + + if (vswap_address[offset]->swap_cache_page && + vswap_address[offset]->swap_cache_page != VSWAP_ALLOCATING) + BUG(); + + vswap_address[offset]->swap_cache_page = page; + vswap_num_swap_cache++; + + spin_unlock(&virtual_swap_list); +} + + +/** + * del_swap_cache_page_vswap - deletes the swap cache page from the + * corresponding vswap entry. + * @page: page to be deleted to the struct + * + * This function removes the page from the corresponding virtual swap + * address entry (we can get the entry from page->index). It will be + * called when removing a page from swap cache or before assigning a + * real entry. + * + */ +void FASTCALL(del_swap_cache_page_vswap(struct page *)); +void del_swap_cache_page_vswap(struct page * page) +{ + unsigned long offset; + swp_entry_t entry; + + entry.val = page->index; + + if (!vswap_address(entry)) + return; + + spin_lock(&virtual_swap_list); + + offset = SWP_OFFSET(entry); + + if (vswap_address[offset]->swap_cache_page != page) + BUG(); + + vswap_address[offset]->swap_cache_page = NULL; + vswap_num_swap_cache--; + + spin_unlock(&virtual_swap_list); +} + +void FASTCALL(get_vswap(swp_entry_t)); +void get_vswap(swp_entry_t entry) { + if (!vswap_address(entry)) + return; + spin_lock(&virtual_swap_list); + vswap_address[SWP_OFFSET(entry)]->fault_count++; + spin_unlock(&virtual_swap_list); +} + +void FASTCALL(put_vswap(swp_entry_t)); +void put_vswap(swp_entry_t entry) { + if (!vswap_address(entry)) + return; + spin_lock(&virtual_swap_list); + vswap_address[SWP_OFFSET(entry)]->fault_count--; + spin_unlock(&virtual_swap_list); +} + + +/** + * vswap_alloc_and_init - allocates a new virtual swap entry and inits + * it. + * @vswap_address: main virtual swap array of pointers to + * vswap_address structs. + * @offset: offset within the vswap_address. + * + * This function allocates a new virtual swap entry and initializes + * its struct, adding it to the list of free vswap entries. + * + * the caller must hold virtual_swap_list lock + */ +int +vswap_alloc_and_init(struct vswap_address ** vswap_address, unsigned long offset) { + vswap_address[offset] = alloc_vswap(); + + if (!vswap_address[offset]) + return 0; + + vswap_address[offset]->offset = offset; + vswap_address[offset]->swap_count = 0; + vswap_address[offset]->pte_list = NULL; + vswap_address[offset]->swap_cache_page = NULL; + vswap_address[offset]->fragment = NULL; + vswap_address[offset]->fault_count = 0; + + list_add(&(vswap_address[offset]->list), &vswap_address_free_head); + nr_free_vswap++; + return 1; +} + +void __init +comp_cache_vswap_init(void) +{ + unsigned long i; + + INIT_LIST_HEAD(&(vswap_address_free_head)); + INIT_LIST_HEAD(&(vswap_address_used_head)); + + comp_cache_freeable_space = COMP_PAGE_SIZE * num_comp_pages; + + last_page_size = (unsigned short *) vmalloc(NUM_MEAN_PAGES * sizeof(unsigned short)); + + for (i = 0; i < NUM_MEAN_PAGES; i++) + last_page_size[i] = COMP_PAGE_SIZE/2; + + /* alloc only one page right now to avoid problems when + * starting using virtual swap address (usually under high + * memory pressure) */ + alloc_new_pte_lists(); +} + +/* + * Overrides for Emacs so that we follow Linus's tabbing style. + * Emacs will notice this stuff at the end of the file and automatically + * adjust the settings for this buffer only. This must remain at the end + * of the file. + * --------------------------------------------------------------------------- + * Local variables: + * c-file-style: "linux" + * End: + */ diff -urN linux-2.4.26-rc1/mm/filemap.c linux-2.4.26-rc1-ccache/mm/filemap.c --- linux-2.4.26-rc1/mm/filemap.c Sun Mar 28 16:12:41 2004 +++ linux-2.4.26-rc1-ccache/mm/filemap.c Sun Mar 28 17:16:36 2004 @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -167,6 +168,9 @@ if (mapping && mapping->host) mark_inode_dirty_pages(mapping->host); +#ifdef CONFIG_COMP_CACHE + flush_comp_cache(page); +#endif if (block_dump) printk(KERN_DEBUG "%s: dirtied page\n", current->comm); } @@ -192,6 +196,10 @@ spin_lock(&pagecache_lock); curr = head->next; +#ifdef CONFIG_COMP_PAGE_CACHE + invalidate_comp_pages(inode->i_mapping); +#endif + while (curr != head) { page = list_entry(curr, struct page, list); curr = curr->next; @@ -340,6 +348,9 @@ unlocked |= truncate_list_pages(&mapping->dirty_pages, start, &partial); unlocked |= truncate_list_pages(&mapping->locked_pages, start, &partial); } while (unlocked); +#ifdef CONFIG_COMP_PAGE_CACHE + truncate_comp_pages(mapping, start, partial); +#endif /* Traversed all three lists without dropping the lock */ spin_unlock(&pagecache_lock); } @@ -435,12 +446,24 @@ { int unlocked; + goto try_again; + + try_again: spin_lock(&pagecache_lock); do { unlocked = invalidate_list_pages2(&mapping->clean_pages); unlocked |= invalidate_list_pages2(&mapping->dirty_pages); unlocked |= invalidate_list_pages2(&mapping->locked_pages); } while (unlocked); + +#ifdef CONFIG_COMP_PAGE_CACHE + if (there_are_dirty_comp_pages(mapping)) { + lookup_comp_pages(mapping); + goto try_again; + } + + truncate_comp_pages(mapping, 0, 0); +#endif spin_unlock(&pagecache_lock); } @@ -604,6 +627,9 @@ int ret = 0; int (*writepage)(struct page *) = mapping->a_ops->writepage; + goto try_again; + + try_again: spin_lock(&pagecache_lock); while (!list_empty(&mapping->dirty_pages)) { @@ -632,6 +658,12 @@ page_cache_release(page); spin_lock(&pagecache_lock); } +#ifdef CONFIG_COMP_PAGE_CACHE + if (there_are_dirty_comp_pages(mapping)) { + lookup_comp_pages(mapping); + goto try_again; + } +#endif spin_unlock(&pagecache_lock); return ret; } @@ -649,6 +681,11 @@ spin_lock(&pagecache_lock); +#ifdef CONFIG_COMP_PAGE_CACHE + try_again: + wait_comp_pages(mapping); +#endif + while (!list_empty(&mapping->locked_pages)) { struct page *page = list_entry(mapping->locked_pages.next, struct page, list); @@ -668,6 +705,10 @@ page_cache_release(page); spin_lock(&pagecache_lock); } +#ifdef CONFIG_COMP_PAGE_CACHE + if (there_are_locked_comp_pages(mapping)) + goto try_again; +#endif spin_unlock(&pagecache_lock); return ret; } @@ -754,8 +795,11 @@ * This adds the requested page to the page cache if it isn't already there, * and schedules an I/O to read in its contents from disk. */ -static int FASTCALL(page_cache_read(struct file * file, unsigned long offset)); -static int page_cache_read(struct file * file, unsigned long offset) +#define page_cache_read(file, offset) __page_cache_read(file, offset, 0) +#define page_cache_readahead(file, offset) __page_cache_read(file, offset, 1) + +static int FASTCALL(__page_cache_read(struct file * file, unsigned long offset, int readahead)); +static int __page_cache_read(struct file * file, unsigned long offset, int readahead) { struct address_space *mapping = file->f_dentry->d_inode->i_mapping; struct page **hash = page_hash(mapping, offset); @@ -771,8 +815,32 @@ if (!page) return -ENOMEM; + if (readahead) { + struct page * tmp_page; + spin_lock(&pagecache_lock); + tmp_page = __find_page_nolock(mapping, offset, *hash); + spin_unlock(&pagecache_lock); + + if (!tmp_page && in_comp_cache(mapping, offset)) { + page_cache_release(page); + return 0; + } + } + if (!add_to_page_cache_unique(page, mapping, offset, hash)) { - int error = mapping->a_ops->readpage(file, page); + int error = 0; +#ifdef CONFIG_COMP_PAGE_CACHE + if (!readahead) { + if (!read_comp_cache(mapping, offset, page)) { + UnlockPage(page); + page_cache_release(page); + return error; + } + } + if (clean_page_compress_lock) + hit_clean_page(page); +#endif + error = mapping->a_ops->readpage(file, page); page_cache_release(page); return error; } @@ -797,7 +865,7 @@ offset = CLUSTER_OFFSET(offset); while ((pages-- > 0) && (offset < filesize)) { - int error = page_cache_read(file, offset); + int error = page_cache_readahead(file, offset); if (error < 0) return error; offset ++; @@ -1054,6 +1122,80 @@ return page; } +#ifdef CONFIG_COMP_CACHE +/* + * Change the swap cache page index in the page cache data structure, + * specifically the hash queue. It's used by shrink_vswap() when * + * compacting the vswap entries. + */ +void change_index_hash_queue(struct page * page, unsigned long new_index) { + struct buffer_head * buffers; + struct page ** p; + + if (!PageLocked(page)) + BUG(); + + spin_lock(&pagecache_lock); + + /* hack to avoid problems in the function to + * add pages to the hash queue, since it does + * not like pages with buffers */ + buffers = page->buffers; + page->buffers = NULL; + + remove_page_from_hash_queue(page); + page->index = new_index; + p = page_hash(page->mapping, new_index); + add_page_to_hash_queue(page, p); + + page->buffers = buffers; + + spin_unlock(&pagecache_lock); +} + +/* + * The same function as below, but doesn't invalidate the comp cache + */ +void __set_page_dirty(struct page *page) +{ + if (!test_and_set_bit(PG_dirty, &page->flags)) { + struct address_space *mapping = page->mapping; + + if (mapping) { + spin_lock(&pagecache_lock); + list_del(&page->list); + list_add(&page->list, &mapping->dirty_pages); + spin_unlock(&pagecache_lock); + + if (mapping->host) + mark_inode_dirty_pages(mapping->host); + } + } +} + +struct page * +find_or_add_page(struct page * new_page, struct address_space *mapping, unsigned long offset) +{ + struct page *page = NULL; + struct page **hash = page_hash(mapping, offset); + + /* + * We scan the hash list read-only. Addition to and removal from + * the hash-list needs a held write-lock. + */ + spin_lock(&pagecache_lock); + page = __find_lock_page_helper(mapping, offset, *hash); + if (page) { + spin_unlock(&pagecache_lock); + return page; + } + __add_to_page_cache(new_page, mapping, offset, hash); + spin_unlock(&pagecache_lock); + lru_cache_add(new_page); + return NULL; +} +#endif + /* * Same as the above, but lock the page too, verifying that * it's still valid once we own it. @@ -1063,6 +1205,9 @@ { struct page *page; + goto repeat; + + repeat: spin_lock(&pagecache_lock); page = __find_lock_page_helper(mapping, offset, *hash); spin_unlock(&pagecache_lock); @@ -1091,12 +1236,20 @@ newpage = NULL; } spin_unlock(&pagecache_lock); - if (newpage == NULL) + if (newpage == NULL) { lru_cache_add(page); +#ifdef CONFIG_COMP_PAGE_CACHE + read_comp_cache(mapping, index, page); +#endif + } else page_cache_release(newpage); } } +#ifdef CONFIG_COMP_PAGE_CACHE + if (page) + flush_comp_cache(page); +#endif return page; } @@ -1348,7 +1501,7 @@ if (ra_index >= end_index) break; - if (page_cache_read(filp, ra_index) < 0) + if (page_cache_readahead(filp, ra_index) < 0) break; ahead++; @@ -1552,6 +1705,10 @@ } readpage: +#ifdef CONFIG_COMP_PAGE_CACHE + if (clean_page_compress_lock) + hit_clean_page(page); +#endif /* ... and start the actual read. The read will unlock the page. */ error = mapping->a_ops->readpage(filp, page); @@ -1606,6 +1763,12 @@ lru_cache_add(page); cached_page = NULL; +#ifdef CONFIG_COMP_PAGE_CACHE + if (!read_comp_cache(mapping, index, page)) { + UnlockPage(page); + goto page_ok; + } +#endif goto readpage; } @@ -1970,7 +2133,7 @@ nr = max; while (nr) { - page_cache_read(file, index); + page_cache_readahead(file, index); index++; nr--; } @@ -2122,7 +2285,7 @@ * Otherwise, we're off the end of a privately mapped file, * so we need to map a zero page. */ - if ((pgoff < size) && !VM_RandomReadHint(area)) + if ((pgoff < size) && !VM_RandomReadHint(area) && !in_comp_cache(mapping, pgoff)) error = read_cluster_nonblocking(file, pgoff, size); else error = page_cache_read(file, pgoff); @@ -2160,6 +2323,10 @@ goto success; } +#ifdef CONFIG_COMP_PAGE_CACHE + if (clean_page_compress_lock) + hit_clean_page(page); +#endif if (!mapping->a_ops->readpage(file, page)) { wait_on_page(page); if (Page_Uptodate(page)) @@ -2187,6 +2354,10 @@ goto success; } ClearPageError(page); +#ifdef CONFIG_COMP_PAGE_CACHE + if (clean_page_compress_lock) + hit_clean_page(page); +#endif if (!mapping->a_ops->readpage(file, page)) { wait_on_page(page); if (Page_Uptodate(page)) @@ -2952,7 +3123,7 @@ { struct page **hash = page_hash(mapping, index); struct page *page, *cached_page = NULL; - int err; + int err = 0; repeat: page = __find_get_page(mapping, index, hash); if (!page) { @@ -2965,6 +3136,13 @@ if (add_to_page_cache_unique(page, mapping, index, hash)) goto repeat; cached_page = NULL; + +#ifdef CONFIG_COMP_PAGE_CACHE + if (!read_comp_cache(mapping, index, page)) { + UnlockPage(page); + goto out; + } +#endif err = filler(data, page); if (err < 0) { page_cache_release(page); @@ -2973,6 +3151,10 @@ } if (cached_page) page_cache_release(cached_page); +#ifdef CONFIG_COMP_PAGE_CACHE + out: + flush_comp_cache(page); +#endif return page; } @@ -3031,7 +3213,13 @@ if (add_to_page_cache_unique(page, mapping, index, hash)) goto repeat; *cached_page = NULL; +#ifdef CONFIG_COMP_PAGE_CACHE + read_dirty_comp_cache(mapping, index, page); +#endif } +#ifdef CONFIG_COMP_PAGE_CACHE + flush_comp_cache(page); +#endif return page; } diff -urN linux-2.4.26-rc1/mm/memory.c linux-2.4.26-rc1-ccache/mm/memory.c --- linux-2.4.26-rc1/mm/memory.c Sat Dec 6 08:14:51 2003 +++ linux-2.4.26-rc1-ccache/mm/memory.c Sun Mar 28 17:12:21 2004 @@ -45,6 +45,7 @@ #include #include #include +#include #include #include @@ -103,6 +104,7 @@ } pte = pte_offset(dir, 0); pmd_clear(dir); + pgtable_remove_rmap(pte); pte_free(pte); } @@ -238,6 +240,7 @@ if (pte_none(pte)) goto cont_copy_pte_range_noset; if (!pte_present(pte)) { + add_pte_vswap(dst_pte, pte_to_swp_entry(pte)); swap_duplicate(pte_to_swp_entry(pte)); goto cont_copy_pte_range; } @@ -320,6 +323,7 @@ /* This will eventually call __free_pte on the pte. */ tlb_remove_page(tlb, ptep, address + offset); } else { + remove_pte_vswap(ptep); free_swap_and_cache(pte_to_swp_entry(pte)); pte_clear(ptep); } @@ -759,7 +763,10 @@ end = PMD_SIZE; do { pte_t zero_pte = pte_wrprotect(mk_pte(ZERO_PAGE(address), prot)); - pte_t oldpage = ptep_get_and_clear(pte); + pte_t oldpage; + + remove_pte_vswap(pte); + oldpage = ptep_get_and_clear(pte); set_pte(pte, zero_pte); forget_pte(oldpage); address += PAGE_SIZE; @@ -834,6 +841,8 @@ do { struct page *page; pte_t oldpage; + + remove_pte_vswap(pte); oldpage = ptep_get_and_clear(pte); page = virt_to_page(__va(phys_addr)); @@ -1096,13 +1105,16 @@ struct page *new_page; unsigned long offset; + if (vswap_address(entry)) + return; + /* * Get the number of handles we should do readahead io to. */ num = valid_swaphandles(entry, &offset); for (i = 0; i < num; offset++, i++) { /* Ok, do the async read-ahead now */ - new_page = read_swap_cache_async(SWP_ENTRY(SWP_TYPE(entry), offset)); + new_page = read_swap_cache_async_ahead(SWP_ENTRY(SWP_TYPE(entry), offset)); if (!new_page) break; page_cache_release(new_page); @@ -1123,10 +1135,18 @@ pte_t pte; int ret = 1; + get_vswap(entry); spin_unlock(&mm->page_table_lock); page = lookup_swap_cache(entry); if (!page) { - swapin_readahead(entry); + /* perform readahead only if the page is on disk */ + if (!in_comp_cache(&swapper_space, entry.val)) { + if (vswap_address(entry)) + BUG(); + swapin_readahead(entry); + /* major fault */ + ret = 2; + } page = read_swap_cache_async(entry); if (!page) { /* @@ -1137,11 +1157,9 @@ spin_lock(&mm->page_table_lock); retval = pte_same(*page_table, orig_pte) ? -1 : 1; spin_unlock(&mm->page_table_lock); + put_vswap(entry); return retval; } - - /* Had to read the page from swap area: Major fault */ - ret = 2; } mark_page_accessed(page); @@ -1157,19 +1175,25 @@ spin_unlock(&mm->page_table_lock); unlock_page(page); page_cache_release(page); + put_vswap(entry); return 1; } + remove_pte_vswap(page_table); + /* The page isn't present yet, go ahead with the fault. */ - + if (PageCompressed(page)) + decompress_swap_cache_page(page); swap_free(entry); if (vm_swap_full()) remove_exclusive_swap_page(page); mm->rss++; pte = mk_pte(page, vma->vm_page_prot); - if (write_access && can_share_swap_page(page)) + if (write_access && can_share_swap_page(page)) { pte = pte_mkdirty(pte_mkwrite(pte)); + flush_comp_cache(page); + } unlock_page(page); flush_page_to_ram(page); @@ -1179,6 +1203,7 @@ /* No need to invalidate - it was non-present before */ update_mmu_cache(vma, address, pte); spin_unlock(&mm->page_table_lock); + put_vswap(entry); return ret; } @@ -1247,6 +1272,7 @@ { struct page * new_page; pte_t entry; + int ret = 2; /* usually major fault */ if (!vma->vm_ops || !vma->vm_ops->nopage) return do_anonymous_page(mm, vma, page_table, write_access, address); @@ -1292,8 +1318,14 @@ flush_page_to_ram(new_page); flush_icache_page(vma, new_page); entry = mk_pte(new_page, vma->vm_page_prot); - if (write_access) + if (PageCompCache(new_page)) + ret = 1; + if (write_access) { entry = pte_mkwrite(pte_mkdirty(entry)); +#ifdef CONFIG_COMP_PAGE_CACHE + flush_comp_cache(new_page); +#endif + } set_pte(page_table, entry); } else { /* One of our sibling threads was faster, back out. */ @@ -1305,7 +1337,7 @@ /* no need to invalidate: a not-present page shouldn't be cached */ update_mmu_cache(vma, address, entry); spin_unlock(&mm->page_table_lock); - return 2; /* Major fault */ + return ret; } /* @@ -1454,6 +1486,7 @@ goto out; } } + pgtable_add_rmap(new, mm, address); pmd_populate(mm, pmd, new); } out: diff -urN linux-2.4.26-rc1/mm/mmap.c linux-2.4.26-rc1-ccache/mm/mmap.c --- linux-2.4.26-rc1/mm/mmap.c Fri Feb 20 07:38:35 2004 +++ linux-2.4.26-rc1-ccache/mm/mmap.c Sun Mar 28 17:19:23 2004 @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -81,6 +82,11 @@ * too small. */ free += swapper_space.nrpages; + +#ifdef CONFIG_COMP_CACHE + /* Let's count the free space left in compressed cache */ + free += comp_cache_free_space; +#endif /* * The code below doesn't account for free space in the inode diff -urN linux-2.4.26-rc1/mm/mremap.c linux-2.4.26-rc1-ccache/mm/mremap.c --- linux-2.4.26-rc1/mm/mremap.c Sat Mar 20 10:08:21 2004 +++ linux-2.4.26-rc1-ccache/mm/mremap.c Sun Mar 28 17:12:21 2004 @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -63,6 +64,7 @@ pte_t pte; if (!pte_none(*src)) { + remove_pte_vswap(src); pte = ptep_get_and_clear(src); if (!dst) { /* No dest? We must put it back. */ @@ -70,6 +72,7 @@ error++; } set_pte(dst, pte); + add_pte_vswap(dst, pte_to_swp_entry(pte)); } return error; } diff -urN linux-2.4.26-rc1/mm/oom_kill.c linux-2.4.26-rc1-ccache/mm/oom_kill.c --- linux-2.4.26-rc1/mm/oom_kill.c Fri Feb 20 07:38:35 2004 +++ linux-2.4.26-rc1-ccache/mm/oom_kill.c Sun Mar 28 17:20:20 2004 @@ -20,6 +20,7 @@ #include #include #include +#include /* #define DEBUG */ @@ -213,6 +214,12 @@ * Enough swap space left? Not OOM. */ if (nr_swap_pages > 0) + return; + + /* + * Enough space in compressed cache left? Not OOM. + */ + if (comp_cache_available_space()) return; spin_lock(&oom_lock); diff -urN linux-2.4.26-rc1/mm/page_alloc.c linux-2.4.26-rc1-ccache/mm/page_alloc.c --- linux-2.4.26-rc1/mm/page_alloc.c Fri Feb 20 07:38:35 2004 +++ linux-2.4.26-rc1-ccache/mm/page_alloc.c Sun Mar 28 17:30:48 2004 @@ -21,6 +21,7 @@ #include #include #include +#include int nr_swap_pages; int nr_active_pages; @@ -39,9 +40,15 @@ EXPORT_SYMBOL(zone_table); static char *zone_names[MAX_NR_ZONES] = { "DMA", "Normal", "HighMem" }; +#ifdef CONFIG_COMP_CACHE +static int zone_balance_ratio[MAX_NR_ZONES] = { 128, 128, 128, }; +static int zone_balance_min[MAX_NR_ZONES] = { 20 , 20, 20, }; +static int zone_balance_max[MAX_NR_ZONES] = { 255 , 255, 255, }; +#else static int zone_balance_ratio[MAX_NR_ZONES] __initdata = { 128, 128, 128, }; static int zone_balance_min[MAX_NR_ZONES] __initdata = { 20 , 20, 20, }; static int zone_balance_max[MAX_NR_ZONES] __initdata = { 255 , 255, 255, }; +#endif static int lower_zone_reserve_ratio[MAX_NR_ZONES-1] = { 256, 32 }; int vm_gfp_debug = 0; @@ -111,6 +118,8 @@ BUG(); ClearPageReferenced(page); ClearPageDirty(page); + ClearPageCompCache(page); + ClearPageCompressed(page); if (current->flags & PF_FREE_PAGES) goto local_freelist; @@ -712,6 +721,42 @@ } #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1)) + +#ifdef CONFIG_COMP_CACHE +void +comp_cache_fix_watermarks(int num_comp_pages) +{ + unsigned long mask; + zone_t *zone; + int num_memory_pages; + + /* We don't have to worry if we have so much memory that it + * will always be above the maximum value. As of 2.4.18, this + * happens when we have 256M, since it always have a + * (zone->size - num_memory_pages) greater than 128M */ + //if (num_physpages >= 2 * zone_balance_ratio[ZONE_NORMAL] * zone_balance_max[ZONE_NORMAL]) + //return; + + /* the real number of memory pages used by compressed cache */ + zone_num_comp_pages = num_memory_pages = comp_page_to_page(num_comp_pages); + + zone = contig_page_data.node_zones + ZONE_NORMAL; + + /* whoops: that should be zone->size minus zholes. Since + * zholes is always 0 when calling free_area_init_core(), I + * guess we don't have to worry about that now */ + mask = ((zone->size - num_memory_pages)/zone_balance_ratio[ZONE_NORMAL]); + + if (mask < zone_balance_min[ZONE_NORMAL]) + mask = zone_balance_min[ZONE_NORMAL]; + else if (mask > zone_balance_max[ZONE_NORMAL]) + mask = zone_balance_max[ZONE_NORMAL]; + + zone->pages_min = mask; + zone->pages_low = mask*2; + zone->pages_high = mask*3; +} +#endif /* * Set up the zone data structures: diff -urN linux-2.4.26-rc1/mm/page_io.c linux-2.4.26-rc1-ccache/mm/page_io.c --- linux-2.4.26-rc1/mm/page_io.c Sun Jan 26 10:12:50 2003 +++ linux-2.4.26-rc1-ccache/mm/page_io.c Sun Mar 28 17:12:21 2004 @@ -87,7 +87,9 @@ swp_entry_t entry; entry.val = page->index; - +#ifdef CONFIG_COMP_SWAP + entry = get_map(entry); +#endif if (!PageLocked(page)) PAGE_BUG(page); if (!PageSwapCache(page)) diff -urN linux-2.4.26-rc1/mm/shmem.c linux-2.4.26-rc1-ccache/mm/shmem.c --- linux-2.4.26-rc1/mm/shmem.c Sat Dec 6 08:14:51 2003 +++ linux-2.4.26-rc1-ccache/mm/shmem.c Sun Mar 28 17:33:34 2004 @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -498,6 +499,8 @@ spin_unlock(&info->lock); return 0; found: + if (PageCompressed(page)) + decompress_swap_cache_page(page); idx += offset; inode = info->inode; mapping = inode->i_mapping; @@ -519,6 +522,11 @@ return 1; } +#ifdef CONFIG_COMP_CACHE +int shmem_page(struct page * page) { + return (page->mapping->a_ops == &shmem_aops); +} +#endif /* * shmem_unuse() search for an eventually swapped out shmem page. */ @@ -690,6 +698,8 @@ goto failed; } + if (PageCompressed(page)) + decompress_swap_cache_page(page); delete_from_swap_cache(swappage); if (filepage) { entry->val = 0; @@ -1004,6 +1014,9 @@ err = shmem_getpage(inode, index, &page, SGP_WRITE); if (err) break; +#ifdef CONFIG_COMP_PAGE_CACHE + flush_comp_cache(page); +#endif kaddr = kmap(page); left = __copy_from_user(kaddr + offset, buf, bytes); diff -urN linux-2.4.26-rc1/mm/swap_state.c linux-2.4.26-rc1-ccache/mm/swap_state.c --- linux-2.4.26-rc1/mm/swap_state.c Sun Jan 26 10:12:50 2003 +++ linux-2.4.26-rc1-ccache/mm/swap_state.c Sun Mar 28 17:12:21 2004 @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include @@ -40,6 +42,11 @@ LIST_HEAD_INIT(swapper_space.clean_pages), LIST_HEAD_INIT(swapper_space.dirty_pages), LIST_HEAD_INIT(swapper_space.locked_pages), +#ifdef CONFIG_COMP_CACHE + LIST_HEAD_INIT(swapper_space.clean_comp_pages), + LIST_HEAD_INIT(swapper_space.dirty_comp_pages), + LIST_HEAD_INIT(swapper_space.locked_comp_pages), +#endif 0, /* nrpages */ &swap_aops, }; @@ -81,6 +88,7 @@ INC_CACHE_INFO(exist_race); return -EEXIST; } + add_swap_cache_page_vswap(page, entry); if (!PageLocked(page)) BUG(); if (!PageSwapCache(page)) @@ -100,6 +108,7 @@ if (!PageSwapCache(page)) BUG(); ClearPageDirty(page); + del_swap_cache_page_vswap(page); __remove_inode_page(page); INC_CACHE_INFO(del_total); } @@ -181,7 +190,7 @@ * A failure return means that either the page allocation failed or that * the swap entry is no longer in use. */ -struct page * read_swap_cache_async(swp_entry_t entry) +struct page * __read_swap_cache_async(swp_entry_t entry, int readahead) { struct page *found_page, *new_page = NULL; int err; @@ -206,6 +215,14 @@ break; /* Out of memory */ } + if (readahead) { + found_page = find_get_page(&swapper_space, entry.val); + if (found_page) + break; + if (in_comp_cache(&swapper_space, entry.val)) + return new_page; + } + /* * Associate the page with swap entry in the swap cache. * May fail (-ENOENT) if swap entry has been freed since @@ -217,9 +234,20 @@ */ err = add_to_swap_cache(new_page, entry); if (!err) { - /* - * Initiate read into locked page and return. - */ + if (!readahead) { + if (!read_comp_cache(&swapper_space, entry.val, new_page)) { + UnlockPage(new_page); + return new_page; + } + } + if (vswap_address(entry)) + BUG(); + if (get_swap_compressed(entry)) + PageSetCompressed(new_page); +#ifdef CONFIG_COMP_PAGE_CACHE + if (clean_page_compress_lock) + hit_clean_page(new_page); +#endif rw_swap_page(READ, new_page); return new_page; } diff -urN linux-2.4.26-rc1/mm/swapfile.c linux-2.4.26-rc1-ccache/mm/swapfile.c --- linux-2.4.26-rc1/mm/swapfile.c Sat Sep 13 07:57:33 2003 +++ linux-2.4.26-rc1-ccache/mm/swapfile.c Sun Mar 28 17:50:26 2004 @@ -14,6 +14,7 @@ #include #include #include +#include #include @@ -103,6 +104,7 @@ swp_entry_t entry; int type, wrapped = 0; + try_again: entry.val = 0; /* Out of memory */ swap_list_lock(); type = swap_list.next; @@ -126,6 +128,11 @@ } else { swap_list.next = type; } + if (comp_cache_use_address(entry)) { + p->swap_map[offset]--; + swap_list_unlock(); + goto try_again; + } goto out; } } @@ -152,6 +159,8 @@ if (!entry.val) goto out; type = SWP_TYPE(entry); + if (vswap_address(entry)) + goto virtual_swap; if (type >= nr_swapfiles) goto bad_nofile; p = & swap_info[type]; @@ -181,30 +190,270 @@ printk(KERN_ERR "swap_free: %s%08lx\n", Bad_file, entry.val); out: return NULL; +virtual_swap: + spin_lock(&virtual_swap_list); + /* it returns a bogus value (not allocated). FIX IT */ + return &swap_info[type]; } static void swap_info_put(struct swap_info_struct * p) { + if (vswap_info_struct(p)) + return; swap_device_unlock(p); swap_list_unlock(); } +#ifdef CONFIG_COMP_SWAP +void +real_swap_free(swp_entry_t entry, int count) +{ + struct swap_info_struct * p; + unsigned long type, offset = SWP_OFFSET(entry); + + type = SWP_TYPE(entry); + p = type + swap_info; + + if (!p->real_swap_map[offset]) + BUG(); + + if (p->real_swap_map[offset] < COMP_SWAP_MAP_MAX) { + p->real_swap_map[offset] -= count; + if (!p->real_swap_map[offset]) { + if (offset < p->real_lowest_bit) + p->real_lowest_bit = offset; + if (offset > p->real_highest_bit) + p->real_highest_bit = offset; + } + } +} + +void +real_swap_duplicate(swp_entry_t entry, int count) +{ + struct swap_info_struct * p; + unsigned long offset, type; + + type = SWP_TYPE(entry); + p = type + swap_info; + offset = SWP_OFFSET(entry); + + if (!p->real_swap_map[offset]) + BUG(); + if (count >= COMP_SWAP_MAP_MAX) + BUG(); + + if (p->real_swap_map[offset] < COMP_SWAP_MAP_MAX - count) + p->real_swap_map[offset] += count; + else if (p->real_swap_map[offset] <= COMP_SWAP_MAP_MAX) + p->real_swap_map[offset] = COMP_SWAP_MAP_MAX; +} + +swp_entry_t +get_real_swap_page(swp_entry_t entry) +{ + unsigned long offset, real_offset; + struct swap_info_struct * p; + swp_entry_t real_entry; + + offset = SWP_OFFSET(entry); + + p = swap_info_get(entry); + + if (!p) + BUG(); + + if (p->real_cluster_nr) { + while (p->real_cluster_next <= p->real_highest_bit) { + real_offset = p->real_cluster_next++; + if (p->real_swap_map[real_offset]) + continue; + p->real_cluster_nr--; + goto got_page; + } + } + p->real_cluster_nr = SWAPFILE_CLUSTER; + + /* try to find an empty (even not aligned) cluster. */ + real_offset = p->real_lowest_bit; + check_next_cluster: + if (real_offset+SWAPFILE_CLUSTER-1 <= p->real_highest_bit) + { + int nr; + for (nr = real_offset; nr < real_offset+SWAPFILE_CLUSTER; nr++) + if (p->real_swap_map[nr]) + { + real_offset = nr+1; + goto check_next_cluster; + } + /* We found a completly empty cluster, so start + * using it. + */ + goto got_page; + } + /* No luck, so now go finegrined as usual. -Andrea */ + for (real_offset = p->real_lowest_bit; real_offset <= p->real_highest_bit ; real_offset++) { + if (p->real_swap_map[real_offset]) + continue; + p->real_lowest_bit = real_offset+1; + got_page: + if (real_offset == p->real_lowest_bit) + p->real_lowest_bit++; + if (real_offset == p->real_highest_bit) + p->real_highest_bit--; + if (p->real_lowest_bit > p->real_highest_bit) { + p->real_lowest_bit = p->max; + p->real_highest_bit = 0; + } + real_entry.val = p->real_swap[offset]; + if (real_entry.val) + real_swap_free(real_entry, swap_map_count(p->swap_map[offset])); + real_entry = SWP_ENTRY(SWP_TYPE(entry), real_offset); + p->real_swap[offset] = real_entry.val; + p->real_swap_map[real_offset] = swap_map_count(p->swap_map[offset]); + p->real_cluster_next = real_offset+1; + swap_info_put(p); + return real_entry; + } + p->real_lowest_bit = p->max; + p->real_highest_bit = 0; + swap_info_put(p); + real_entry.val = 0; + return real_entry; +} + +swp_entry_t +get_map(swp_entry_t entry) +{ + struct swap_info_struct * p; + unsigned long offset; + swp_entry_t real_entry; + + p = swap_info_get(entry); + + if (!p) + BUG(); + + offset = SWP_OFFSET(entry); + if (offset >= p->max) + BUG(); + real_entry.val = p->real_swap[offset]; + + if (!real_entry.val) + BUG(); + swap_info_put(p); + + return real_entry; +} + +void +map_swap(swp_entry_t entry, swp_entry_t real_entry) +{ + struct swap_info_struct * p; + unsigned long offset; + swp_entry_t old_entry; + + p = swap_info_get(entry); + + if (!p) + BUG(); + + offset = SWP_OFFSET(entry); + if (offset >= p->max) + BUG(); + old_entry.val = p->real_swap[offset]; + if (old_entry.val) + real_swap_free(old_entry, swap_map_count(p->swap_map[offset])); + + p->real_swap[offset] = real_entry.val; + + real_swap_duplicate(real_entry, swap_map_count(p->swap_map[offset])); + swap_info_put(p); +} +#endif + +#ifdef CONFIG_COMP_CACHE +void +set_swap_compressed(swp_entry_t entry, int compressed) +{ + struct swap_info_struct * p; + unsigned long offset; + + p = swap_info_get(entry); + + if (!p) + BUG(); + + offset = SWP_OFFSET(entry); + if (offset >= p->max) + BUG(); + if (compressed) + p->swap_map[offset] |= SWAP_MAP_COMP_BIT; + else + p->swap_map[offset] &= SWAP_MAP_COMP_BIT_MASK; + + swap_info_put(p); +} + +int +get_swap_compressed(swp_entry_t entry) +{ + struct swap_info_struct * p; + unsigned long offset; + int ret = 0; + + p = swap_info_get(entry); + + if (!p) + BUG(); + + offset = SWP_OFFSET(entry); + if (offset >= p->max) + BUG(); + if (p->swap_map[offset] & SWAP_MAP_COMP_BIT) + ret = 1; + swap_info_put(p); + + return ret; +} + +#endif + static int swap_entry_free(struct swap_info_struct *p, unsigned long offset) { - int count = p->swap_map[offset]; + int count; + swp_entry_t entry; + + if (vswap_info_struct(p)) + return virtual_swap_free(offset); + count = p->swap_map[offset]; - if (count < SWAP_MAP_MAX) { + if (swap_map_count(count) < SWAP_MAP_MAX) { count--; - p->swap_map[offset] = count; - if (!count) { +#ifdef CONFIG_COMP_SWAP + if (p->real_swap[offset]) { + swp_entry_t real_entry; + real_entry.val = p->real_swap[offset]; + real_swap_free(real_entry, 1); + } +#endif + if (!swap_map_count(count)) { +#ifdef CONFIG_COMP_SWAP + if (p->real_swap[offset]) + p->real_swap[offset] = 0; +#endif + entry = SWP_ENTRY(p - swap_info, offset); + invalidate_comp_cache(&swapper_space, entry.val); if (offset < p->lowest_bit) p->lowest_bit = offset; if (offset > p->highest_bit) p->highest_bit = offset; nr_swap_pages++; + count = 0; } + p->swap_map[offset] = count; } - return count; + return swap_map_count(count); } /* @@ -228,7 +477,7 @@ */ static int exclusive_swap_page(struct page *page) { - int retval = 0; + int retval = 0, exclusive = 0; struct swap_info_struct * p; swp_entry_t entry; @@ -236,7 +485,16 @@ p = swap_info_get(entry); if (p) { /* Is the only swap cache user the cache itself? */ - if (p->swap_map[SWP_OFFSET(entry)] == 1) { + if (vswap_address(entry)) { + if (vswap_address[SWP_OFFSET(entry)]->swap_count == 1) + exclusive = 1; + spin_unlock(&virtual_swap_list); + goto check_exclusive; + } + if (swap_map_count(p->swap_map[SWP_OFFSET(entry)]) == 1) + exclusive = 1; + check_exclusive: + if (exclusive) { /* Recheck the page count with the pagecache lock held.. */ spin_lock(&pagecache_lock); if (page_count(page) - !!page->buffers == 2) @@ -286,7 +544,7 @@ */ int remove_exclusive_swap_page(struct page *page) { - int retval; + int retval, exclusive = 0; struct swap_info_struct * p; swp_entry_t entry; @@ -304,11 +562,22 @@ /* Is the only swap cache user the cache itself? */ retval = 0; - if (p->swap_map[SWP_OFFSET(entry)] == 1) { + if (vswap_address(entry)) { + if (vswap_address[SWP_OFFSET(entry)]->swap_count == 1) + exclusive = 1; + spin_unlock(&virtual_swap_list); + goto check_exclusive; + } + if (swap_map_count(p->swap_map[SWP_OFFSET(entry)]) == 1) + exclusive = 1; + check_exclusive: + if (exclusive) { /* Recheck the page count with the pagecache lock held.. */ spin_lock(&pagecache_lock); if (page_count(page) - !!page->buffers == 2) { __delete_from_swap_cache(page); + if (PageCompressed(page)) + decompress_swap_cache_page(page); SetPageDirty(page); retval = 1; } @@ -498,7 +767,7 @@ prev = 0; i = 1; } - count = si->swap_map[i]; + count = swap_map_count(si->swap_map[i]); if (count && count != SWAP_MAP_BAD) break; } @@ -601,7 +870,7 @@ * to search, but use it as a reminder to search shmem. */ shmem = 0; - swcount = *swap_map; + swcount = swap_map_count(*swap_map); if (swcount > 1) { flush_page_to_ram(page); if (start_mm == &init_mm) @@ -609,17 +878,17 @@ else unuse_process(start_mm, entry, page); } - if (*swap_map > 1) { - int set_start_mm = (*swap_map >= swcount); + if (swap_map_count(*swap_map) > 1) { + int set_start_mm = (swap_map_count(*swap_map) >= swcount); struct list_head *p = &start_mm->mmlist; struct mm_struct *new_start_mm = start_mm; struct mm_struct *mm; spin_lock(&mmlist_lock); - while (*swap_map > 1 && + while (swap_map_count(*swap_map) > 1 && (p = p->next) != &start_mm->mmlist) { mm = list_entry(p, struct mm_struct, mmlist); - swcount = *swap_map; + swcount = swap_map_count(*swap_map); if (mm == &init_mm) { set_start_mm = 1; spin_unlock(&mmlist_lock); @@ -627,7 +896,7 @@ spin_lock(&mmlist_lock); } else unuse_process(mm, entry, page); - if (set_start_mm && *swap_map < swcount) { + if (set_start_mm && swap_map_count(*swap_map) < swcount) { new_start_mm = mm; set_start_mm = 0; } @@ -651,11 +920,11 @@ * We know "Undead"s can happen, they're okay, so don't * report them; but do report if we reset SWAP_MAP_MAX. */ - if (*swap_map == SWAP_MAP_MAX) { + if (swap_map_count(*swap_map) == SWAP_MAP_MAX) { swap_list_lock(); swap_device_lock(si); nr_swap_pages++; - *swap_map = 1; + *swap_map = 1 | (*swap_map & SWAP_MAP_COMP_BIT); swap_device_unlock(si); swap_list_unlock(); reset_overflow = 1; @@ -680,10 +949,12 @@ * swap count to pass quickly through the loops above, * and now we must reincrement count to try again later. */ - if ((*swap_map > 1) && PageDirty(page) && PageSwapCache(page)) { + if (swap_map_count(*swap_map) > 1 && PageDirty(page) && PageSwapCache(page)) { rw_swap_page(WRITE, page); lock_page(page); } + if (PageCompressed(page)) + decompress_swap_cache_page(page); if (PageSwapCache(page)) { if (shmem) swap_duplicate(entry); @@ -724,6 +995,10 @@ struct nameidata nd; int i, type, prev; int err; +#ifdef CONFIG_COMP_SWAP + unsigned long * real_swap; + unsigned short * real_swap_map; +#endif if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -796,10 +1071,20 @@ p->max = 0; swap_map = p->swap_map; p->swap_map = NULL; +#ifdef CONFIG_COMP_SWAP + real_swap = p->real_swap; + p->real_swap = NULL; + real_swap_map = p->real_swap_map; + p->real_swap_map = NULL; +#endif p->flags = 0; swap_device_unlock(p); swap_list_unlock(); vfree(swap_map); +#ifdef CONFIG_COMP_SWAP + vfree(real_swap); + vfree(real_swap_map); +#endif err = 0; out_dput: @@ -833,7 +1118,7 @@ usedswap = 0; for (j = 0; j < ptr->max; ++j) - switch (ptr->swap_map[j]) { + switch (swap_map_count(ptr->swap_map[j])) { case SWAP_MAP_BAD: case 0: continue; @@ -881,6 +1166,10 @@ int swapfilesize; struct block_device *bdev = NULL; unsigned short *swap_map; +#ifdef CONFIG_COMP_SWAP + unsigned long * real_swap; + unsigned short * real_swap_map; +#endif if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -905,6 +1194,13 @@ p->lowest_bit = 0; p->highest_bit = 0; p->cluster_nr = 0; +#ifdef CONFIG_COMP_SWAP + p->real_swap = NULL; + p->real_swap_map = NULL; + p->real_lowest_bit = 0; + p->real_highest_bit = 0; + p->real_cluster_nr = 0; +#endif p->sdev_lock = SPIN_LOCK_UNLOCKED; p->next = -1; if (swap_flags & SWAP_FLAG_PREFER) { @@ -1010,12 +1306,32 @@ error = -ENOMEM; goto bad_swap; } +#ifdef CONFIG_COMP_SWAP + p->real_lowest_bit = p->lowest_bit; + p->real_highest_bit = p->highest_bit; + + p->real_swap = vmalloc(maxpages * sizeof(long)); + p->real_swap_map = vmalloc(maxpages * sizeof(short)); + if (!p->real_swap || !p->real_swap_map) { + error = -ENOMEM; + goto bad_swap; + } + memset(p->real_swap, 0, maxpages * sizeof(long)); +#endif for (i = 1 ; i < maxpages ; i++) { if (test_bit(i,(char *) swap_header)) p->swap_map[i] = 0; else p->swap_map[i] = SWAP_MAP_BAD; } +#ifdef CONFIG_COMP_SWAP + for (i = 1 ; i < maxpages ; i++) { + if (test_bit(i,(char *) swap_header)) + p->real_swap_map[i] = 0; + else + p->real_swap_map[i] = COMP_SWAP_MAP_BAD; + } +#endif break; case 2: @@ -1047,12 +1363,29 @@ error = 0; memset(p->swap_map, 0, maxpages * sizeof(short)); +#ifdef CONFIG_COMP_SWAP + p->real_lowest_bit = p->lowest_bit; + p->real_highest_bit = p->highest_bit; + + p->real_swap = vmalloc(maxpages * sizeof(long)); + p->real_swap_map = vmalloc(maxpages * sizeof(short)); + if (!p->real_swap || !p->real_swap_map) { + error = -ENOMEM; + goto bad_swap; + } + memset(p->real_swap, 0, maxpages * sizeof(long)); + memset(p->real_swap_map, 0, maxpages * sizeof(short)); +#endif for (i=0; iinfo.nr_badpages; i++) { int page = swap_header->info.badpages[i]; if (page <= 0 || page >= swap_header->info.last_page) error = -EINVAL; - else + else { p->swap_map[page] = SWAP_MAP_BAD; +#ifdef CONFIG_COMP_SWAP + p->real_swap_map[page] = COMP_SWAP_MAP_BAD; +#endif + } } nr_good_pages = swap_header->info.last_page - swap_header->info.nr_badpages - @@ -1073,6 +1406,9 @@ goto bad_swap; } p->swap_map[0] = SWAP_MAP_BAD; +#ifdef CONFIG_COMP_SWAP + p->real_swap_map[0] = COMP_SWAP_MAP_BAD; +#endif swap_list_lock(); swap_device_lock(p); p->max = maxpages; @@ -1107,6 +1443,10 @@ bad_swap_2: swap_list_lock(); swap_map = p->swap_map; +#ifdef CONFIG_COMP_SWAP + real_swap = p->real_swap; + real_swap_map = p->real_swap_map; +#endif nd.mnt = p->swap_vfsmnt; nd.dentry = p->swap_file; p->swap_device = 0; @@ -1119,6 +1459,12 @@ swap_list_unlock(); if (swap_map) vfree(swap_map); +#ifdef CONFIG_COMP_SWAP + if (real_swap) + vfree(real_swap); + if (real_swap_map) + vfree(real_swap_map); +#endif path_release(&nd); out: if (swap_header) @@ -1138,7 +1484,7 @@ if (swap_info[i].flags != SWP_USED) continue; for (j = 0; j < swap_info[i].max; ++j) { - switch (swap_info[i].swap_map[j]) { + switch (swap_map_count(swap_info[i].swap_map[j])) { case 0: case SWAP_MAP_BAD: continue; @@ -1164,6 +1510,8 @@ unsigned long offset, type; int result = 0; + if (vswap_address(entry)) + return virtual_swap_duplicate(entry); type = SWP_TYPE(entry); if (type >= nr_swapfiles) goto bad_file; @@ -1172,16 +1520,23 @@ swap_device_lock(p); if (offset < p->max && p->swap_map[offset]) { - if (p->swap_map[offset] < SWAP_MAP_MAX - 1) { + if (swap_map_count(p->swap_map[offset]) < SWAP_MAP_MAX - 1) { p->swap_map[offset]++; result = 1; - } else if (p->swap_map[offset] <= SWAP_MAP_MAX) { + } else if (swap_map_count(p->swap_map[offset]) <= SWAP_MAP_MAX) { if (swap_overflow++ < 5) printk(KERN_WARNING "swap_dup: swap entry overflow\n"); - p->swap_map[offset] = SWAP_MAP_MAX; + p->swap_map[offset] = SWAP_MAP_MAX | (p->swap_map[offset] & SWAP_MAP_COMP_BIT); result = 1; } } +#ifdef CONFIG_COMP_SWAP + if (p->real_swap[offset]) { + swp_entry_t real_entry; + real_entry.val = p->real_swap[offset]; + real_swap_duplicate(real_entry, 1); + } +#endif swap_device_unlock(p); out: return result; @@ -1212,7 +1567,11 @@ printk(KERN_ERR "rw_swap_page: %s%08lx\n", Bad_offset, entry.val); return; } +#ifdef CONFIG_COMP_SWAP + if (p->real_swap_map && !p->real_swap_map[*offset]) { +#else if (p->swap_map && !p->swap_map[*offset]) { +#endif printk(KERN_ERR "rw_swap_page: %s%08lx\n", Unused_offset, entry.val); return; } @@ -1256,7 +1615,7 @@ /* Don't read in free or bad pages */ if (!swapdev->swap_map[toff]) break; - if (swapdev->swap_map[toff] == SWAP_MAP_BAD) + if (swap_map_count(swapdev->swap_map[toff]) == SWAP_MAP_BAD) break; toff++; ret++; @@ -1264,3 +1623,4 @@ swap_device_unlock(swapdev); return ret; } + diff -urN linux-2.4.26-rc1/mm/vmscan.c linux-2.4.26-rc1-ccache/mm/vmscan.c --- linux-2.4.26-rc1/mm/vmscan.c Fri Feb 20 07:38:35 2004 +++ linux-2.4.26-rc1-ccache/mm/vmscan.c Sun Mar 28 17:12:21 2004 @@ -23,6 +23,7 @@ #include #include #include +#include #include @@ -118,6 +119,7 @@ swap_duplicate(entry); set_swap_pte: set_pte(page_table, swp_entry_to_pte(entry)); + add_pte_vswap(page_table, entry); drop_pte: mm->rss--; UnlockPage(page); @@ -162,8 +164,12 @@ */ for (;;) { entry = get_swap_page(); - if (!entry.val) - break; + if (!entry.val) { + entry = get_virtual_swap_page(); + if (!entry.val) + break; + } + /* Add it to the swap cache and mark it dirty * (adding to the page cache will clear the dirty * and uptodate bits, so we need to do it again) @@ -434,17 +440,26 @@ int (*writepage)(struct page *); writepage = page->mapping->a_ops->writepage; - if ((gfp_mask & __GFP_FS) && writepage) { +#ifdef CONFIG_COMP_CACHE + if (writepage) +#else + if ((gfp_mask & __GFP_FS) && writepage) +#endif + { + int compressed; ClearPageDirty(page); SetPageLaunder(page); page_cache_get(page); spin_unlock(&pagemap_lru_lock); - writepage(page); - page_cache_release(page); + compressed = compress_dirty_page(page, writepage, gfp_mask, priority); + page_cache_release(page); + spin_lock(&pagemap_lru_lock); - continue; + + if (!compressed) + continue; } } @@ -543,6 +558,47 @@ __lru_cache_del(page); +#ifdef CONFIG_COMP_CACHE + /*** + * compress the page if it's a clean page that has not + * been compressed in a previous iteration + */ + if (!PageCompCache(page)) { + int compressed; + +#ifndef CONFIG_COMP_DIS_CLEAN + /* enable this #if 0 to enable policy that + * stop STORING clean page in compressed + * cache */ + if (clean_page_compress_lock) { + add_clean_page(page); + goto check_freeable; + } +#endif + page_cache_get(page); + spin_unlock(&pagecache_lock); + spin_unlock(&pagemap_lru_lock); + + compressed = compress_clean_page(page, gfp_mask, priority); + + page_cache_release(page); + spin_lock(&pagemap_lru_lock); + + if (!compressed) { + UnlockPage(page); + continue; + } + + spin_lock(&pagecache_lock); + check_freeable: + if (!is_page_cache_freeable(page)) { + spin_unlock(&pagecache_lock); + UnlockPage(page); + continue; + } + } +#endif + /* point of no return */ if (likely(!PageSwapCache(page))) { __remove_inode_page(page); @@ -830,6 +886,7 @@ { printk("Starting kswapd\n"); swap_setup(); + comp_cache_init(); kernel_thread(kswapd, NULL, CLONE_FS | CLONE_FILES | CLONE_SIGNAL); return 0; }