ia64/linux-2.6.18-xen.hg

changeset 907:cad6f60f0506

Transcendent memory ("tmem") for Linux

Tmem, when called from a tmem-capable (paravirtualized) guest, makes
use of otherwise unutilized ("fallow") memory to create and manage
pools of pages that can be accessed from the guest either as
"ephemeral" pages or as "persistent" pages. In either case, the pages
are not directly addressible by the guest, only copied to and fro via
the tmem interface. Ephemeral pages are a nice place for a guest to
put recently evicted clean pages that it might need again; these pages
can be reclaimed synchronously by Xen for other guests or other uses.
Persistent pages are a nice place for a guest to put "swap" pages to
avoid sending them to disk. These pages retain data as long as the
guest lives, but count against the guest memory allocation.

This patch contains the Linux paravirtualization changes to
complement the tmem Xen patch (xen-unstable c/s 19646). It
implements "precache" (ext3 only as of now), "preswap",
and limited "shared precache" (ocfs2 only as of now) support.
CONFIG options are required to turn on
the support (but in this patch they default to "y"). If
the underlying Xen does not have tmem support or has it
turned off, this is sensed early to avoid nearly all
hypercalls.

Lots of useful prose about tmem can be found at
http://oss.oracle.com/projects/tmem

Signed-off-by: Dan Magenheimer <dan.magenheimer@oracle.com>
author Keir Fraser <keir.fraser@citrix.com>
date Thu Jun 18 10:24:18 2009 +0100 (2009-06-18)
parents 3776d277956f
children baeb818cd2dc
files fs/buffer.c fs/ext3/super.c fs/mpage.c fs/ocfs2/super.c fs/super.c include/asm-i386/mach-xen/asm/hypercall.h include/asm-i386/mach-xen/asm/hypervisor.h include/asm-x86_64/mach-xen/asm/hypercall.h include/linux/fs.h include/linux/precache.h include/linux/swap.h include/linux/sysctl.h kernel/sysctl.c mm/Kconfig mm/Makefile mm/filemap.c mm/page_io.c mm/precache.c mm/preswap.c mm/swapfile.c mm/tmem.c mm/tmem.h mm/truncate.c
line diff
     1.1 --- a/fs/buffer.c	Wed Jun 17 09:07:23 2009 +0100
     1.2 +++ b/fs/buffer.c	Thu Jun 18 10:24:18 2009 +0100
     1.3 @@ -41,6 +41,7 @@
     1.4  #include <linux/bitops.h>
     1.5  #include <linux/mpage.h>
     1.6  #include <linux/bit_spinlock.h>
     1.7 +#include <linux/precache.h>
     1.8  
     1.9  static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
    1.10  static void invalidate_bh_lrus(void);
    1.11 @@ -482,6 +483,11 @@ void invalidate_bdev(struct block_device
    1.12  	 * that, but not until that's cleaned up.
    1.13  	 */
    1.14  	invalidate_inode_pages(mapping);
    1.15 +
    1.16 +	/* 99% of the time, we don't need to flush the precache on the bdev.
    1.17 +	 * But, for the strange corners, lets be cautious
    1.18 +	 */
    1.19 +	precache_flush_inode(mapping);
    1.20  }
    1.21  
    1.22  /*
     2.1 --- a/fs/ext3/super.c	Wed Jun 17 09:07:23 2009 +0100
     2.2 +++ b/fs/ext3/super.c	Thu Jun 18 10:24:18 2009 +0100
     2.3 @@ -35,6 +35,7 @@
     2.4  #include <linux/namei.h>
     2.5  #include <linux/quotaops.h>
     2.6  #include <linux/seq_file.h>
     2.7 +#include <linux/precache.h>
     2.8  
     2.9  #include <asm/uaccess.h>
    2.10  
    2.11 @@ -1167,6 +1168,7 @@ static int ext3_setup_super(struct super
    2.12  	} else {
    2.13  		printk("internal journal\n");
    2.14  	}
    2.15 +	precache_init(sb);
    2.16  	return res;
    2.17  }
    2.18  
     3.1 --- a/fs/mpage.c	Wed Jun 17 09:07:23 2009 +0100
     3.2 +++ b/fs/mpage.c	Thu Jun 18 10:24:18 2009 +0100
     3.3 @@ -26,6 +26,7 @@
     3.4  #include <linux/writeback.h>
     3.5  #include <linux/backing-dev.h>
     3.6  #include <linux/pagevec.h>
     3.7 +#include <linux/precache.h>
     3.8  
     3.9  /*
    3.10   * I/O completion handler for multipage BIOs.
    3.11 @@ -298,6 +299,13 @@ do_mpage_readpage(struct bio *bio, struc
    3.12  		SetPageMappedToDisk(page);
    3.13  	}
    3.14  
    3.15 +	if (fully_mapped &&
    3.16 +	    blocks_per_page == 1 && !PageUptodate(page) &&
    3.17 +	    precache_get(page->mapping, page->index, page) == 1) {
    3.18 +		SetPageUptodate(page);
    3.19 +		goto confused;
    3.20 +	}
    3.21 +
    3.22  	/*
    3.23  	 * This page will go to BIO.  Do we need to send this BIO off first?
    3.24  	 */
     4.1 --- a/fs/ocfs2/super.c	Wed Jun 17 09:07:23 2009 +0100
     4.2 +++ b/fs/ocfs2/super.c	Thu Jun 18 10:24:18 2009 +0100
     4.3 @@ -39,6 +39,7 @@
     4.4  #include <linux/parser.h>
     4.5  #include <linux/crc32.h>
     4.6  #include <linux/debugfs.h>
     4.7 +#include <linux/precache.h>
     4.8  
     4.9  #include <cluster/nodemanager.h>
    4.10  
    4.11 @@ -1457,16 +1458,18 @@ static int ocfs2_initialize_super(struct
    4.12  
    4.13  	di = (struct ocfs2_dinode *) bitmap_bh->b_data;
    4.14  	osb->bitmap_cpg = le16_to_cpu(di->id2.i_chain.cl_cpg);
    4.15 -	brelse(bitmap_bh);
    4.16  	mlog(0, "cluster bitmap inode: %llu, clusters per group: %u\n",
    4.17  	     (unsigned long long)osb->bitmap_blkno, osb->bitmap_cpg);
    4.18  
    4.19  	status = ocfs2_init_slot_info(osb);
    4.20  	if (status < 0) {
    4.21  		mlog_errno(status);
    4.22 -		goto bail;
    4.23 +		goto bail_rel;
    4.24  	}
    4.25 +	shared_precache_init(sb, &di->id2.i_super.s_uuid[0]);
    4.26  
    4.27 +bail_rel:
    4.28 +	brelse(bitmap_bh);
    4.29  bail:
    4.30  	mlog_exit(status);
    4.31  	return status;
     5.1 --- a/fs/super.c	Wed Jun 17 09:07:23 2009 +0100
     5.2 +++ b/fs/super.c	Thu Jun 18 10:24:18 2009 +0100
     5.3 @@ -37,6 +37,7 @@
     5.4  #include <linux/idr.h>
     5.5  #include <linux/kobject.h>
     5.6  #include <linux/mutex.h>
     5.7 +#include <linux/precache.h>
     5.8  #include <asm/uaccess.h>
     5.9  
    5.10  
    5.11 @@ -93,6 +94,9 @@ static struct super_block *alloc_super(s
    5.12  		s->s_qcop = sb_quotactl_ops;
    5.13  		s->s_op = &default_op;
    5.14  		s->s_time_gran = 1000000000;
    5.15 +#ifdef CONFIG_PRECACHE
    5.16 +		s->precache_poolid = -1;
    5.17 +#endif
    5.18  	}
    5.19  out:
    5.20  	return s;
    5.21 @@ -181,6 +185,7 @@ void deactivate_super(struct super_block
    5.22  		DQUOT_OFF(s);
    5.23  		down_write(&s->s_umount);
    5.24  		fs->kill_sb(s);
    5.25 +		precache_flush_filesystem(s);
    5.26  		put_filesystem(fs);
    5.27  		put_super(s);
    5.28  	}
    5.29 @@ -777,6 +782,9 @@ int get_sb_nodev(struct file_system_type
    5.30  		return error;
    5.31  	}
    5.32  	s->s_flags |= MS_ACTIVE;
    5.33 +#ifdef CONFIG_PRECACHE
    5.34 +	s->precache_poolid = -2;
    5.35 +#endif
    5.36  	return simple_set_mnt(mnt, s);
    5.37  }
    5.38  
     6.1 --- a/include/asm-i386/mach-xen/asm/hypercall.h	Wed Jun 17 09:07:23 2009 +0100
     6.2 +++ b/include/asm-i386/mach-xen/asm/hypercall.h	Thu Jun 18 10:24:18 2009 +0100
     6.3 @@ -404,6 +404,12 @@ HYPERVISOR_kexec_op(
     6.4  	return _hypercall2(int, kexec_op, op, args);
     6.5  }
     6.6  
     6.7 +static inline int __must_check
     6.8 +HYPERVISOR_tmem_op(
     6.9 +	struct tmem_op *op)
    6.10 +{
    6.11 +	return _hypercall1(int, tmem_op, op);
    6.12 +}
    6.13  
    6.14  
    6.15  #endif /* __HYPERCALL_H__ */
     7.1 --- a/include/asm-i386/mach-xen/asm/hypervisor.h	Wed Jun 17 09:07:23 2009 +0100
     7.2 +++ b/include/asm-i386/mach-xen/asm/hypervisor.h	Thu Jun 18 10:24:18 2009 +0100
     7.3 @@ -43,6 +43,7 @@
     7.4  #include <xen/interface/physdev.h>
     7.5  #include <xen/interface/sched.h>
     7.6  #include <xen/interface/nmi.h>
     7.7 +#include <xen/interface/tmem.h>
     7.8  #include <asm/ptrace.h>
     7.9  #include <asm/page.h>
    7.10  #if defined(__i386__)
     8.1 --- a/include/asm-x86_64/mach-xen/asm/hypercall.h	Wed Jun 17 09:07:23 2009 +0100
     8.2 +++ b/include/asm-x86_64/mach-xen/asm/hypercall.h	Thu Jun 18 10:24:18 2009 +0100
     8.3 @@ -412,4 +412,11 @@ HYPERVISOR_kexec_op(
     8.4  	return _hypercall2(int, kexec_op, op, args);
     8.5  }
     8.6  
     8.7 +static inline int __must_check
     8.8 +HYPERVISOR_tmem_op(
     8.9 +	struct tmem_op *op)
    8.10 +{
    8.11 +	return _hypercall1(int, tmem_op, op);
    8.12 +}
    8.13 +
    8.14  #endif /* __HYPERCALL_H__ */
     9.1 --- a/include/linux/fs.h	Wed Jun 17 09:07:23 2009 +0100
     9.2 +++ b/include/linux/fs.h	Thu Jun 18 10:24:18 2009 +0100
     9.3 @@ -907,6 +907,9 @@ struct super_block {
     9.4  	/* Granularity of c/m/atime in ns.
     9.5  	   Cannot be worse than a second */
     9.6  	u32		   s_time_gran;
     9.7 +#ifdef CONFIG_PRECACHE
     9.8 +	u32                precache_poolid;
     9.9 +#endif
    9.10  };
    9.11  
    9.12  extern struct timespec current_fs_time(struct super_block *sb);
    10.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    10.2 +++ b/include/linux/precache.h	Thu Jun 18 10:24:18 2009 +0100
    10.3 @@ -0,0 +1,55 @@
    10.4 +#ifndef _LINUX_PRECACHE_H
    10.5 +
    10.6 +#include <linux/fs.h>
    10.7 +#include <linux/mm.h>
    10.8 +
    10.9 +#ifdef CONFIG_PRECACHE
   10.10 +extern void precache_init(struct super_block *sb);
   10.11 +extern void shared_precache_init(struct super_block *sb, char *uuid);
   10.12 +extern int precache_get(struct address_space *mapping, unsigned long index,
   10.13 +	       struct page *empty_page);
   10.14 +extern int precache_put(struct address_space *mapping, unsigned long index,
   10.15 +		struct page *page);
   10.16 +extern int precache_flush(struct address_space *mapping, unsigned long index);
   10.17 +extern int precache_flush_inode(struct address_space *mapping);
   10.18 +extern int precache_flush_filesystem(struct super_block *s);
   10.19 +#else
   10.20 +static inline void precache_init(struct super_block *sb)
   10.21 +{
   10.22 +}
   10.23 +
   10.24 +static inline void shared_precache_init(struct super_block *sb, char *uuid)
   10.25 +{
   10.26 +}
   10.27 +
   10.28 +static inline int precache_get(struct address_space *mapping,
   10.29 +		unsigned long index, struct page *empty_page)
   10.30 +{
   10.31 +	return 0;
   10.32 +}
   10.33 +
   10.34 +static inline int precache_put(struct address_space *mapping,
   10.35 +		unsigned long index, struct page *page)
   10.36 +{
   10.37 +	return 0;
   10.38 +}
   10.39 +
   10.40 +static inline int precache_flush(struct address_space *mapping,
   10.41 +		unsigned long index)
   10.42 +{
   10.43 +	return 0;
   10.44 +}
   10.45 +
   10.46 +static inline int precache_flush_inode(struct address_space *mapping)
   10.47 +{
   10.48 +	return 0;
   10.49 +}
   10.50 +
   10.51 +static inline int precache_flush_filesystem(struct super_block *s)
   10.52 +{
   10.53 +	return 0;
   10.54 +}
   10.55 +#endif
   10.56 +
   10.57 +#define _LINUX_PRECACHE_H
   10.58 +#endif /* _LINUX_PRECACHE_H */
    11.1 --- a/include/linux/swap.h	Wed Jun 17 09:07:23 2009 +0100
    11.2 +++ b/include/linux/swap.h	Thu Jun 18 10:24:18 2009 +0100
    11.3 @@ -6,6 +6,7 @@
    11.4  #include <linux/mmzone.h>
    11.5  #include <linux/list.h>
    11.6  #include <linux/sched.h>
    11.7 +#include <linux/vmalloc.h>
    11.8  
    11.9  #include <asm/atomic.h>
   11.10  #include <asm/page.h>
   11.11 @@ -143,9 +144,60 @@ struct swap_info_struct {
   11.12  	unsigned int pages;
   11.13  	unsigned int max;
   11.14  	unsigned int inuse_pages;
   11.15 +#ifdef CONFIG_PRESWAP
   11.16 +	unsigned long *preswap_map;
   11.17 +	unsigned int preswap_pages;
   11.18 +#endif
   11.19  	int next;			/* next entry on swap list */
   11.20  };
   11.21  
   11.22 +#ifdef CONFIG_PRESWAP
   11.23 +
   11.24 +#include <linux/sysctl.h>
   11.25 +extern int preswap_sysctl_handler(struct ctl_table *, int, struct file *,
   11.26 +	void __user *, size_t *, loff_t *);
   11.27 +extern const unsigned long preswap_zero, preswap_infinity;
   11.28 +
   11.29 +extern void preswap_shrink(unsigned long);
   11.30 +extern int preswap_test(struct swap_info_struct *, unsigned long);
   11.31 +extern void preswap_init(unsigned);
   11.32 +extern int preswap_put(struct page *);
   11.33 +extern int preswap_get(struct page *);
   11.34 +extern void preswap_flush(unsigned, unsigned long);
   11.35 +extern void preswap_flush_area(unsigned);
   11.36 +#else
   11.37 +static inline void preswap_shrink(unsigned long target_pages)
   11.38 +{
   11.39 +}
   11.40 +
   11.41 +static inline int preswap_test(struct swap_info_struct *sis, unsigned long offset)
   11.42 +{
   11.43 +	return 0;
   11.44 +}
   11.45 +
   11.46 +static inline void preswap_init(unsigned type)
   11.47 +{
   11.48 +}
   11.49 +
   11.50 +static inline int preswap_put(struct page *page)
   11.51 +{
   11.52 +	return 0;
   11.53 +}
   11.54 +
   11.55 +static inline int preswap_get(struct page *get)
   11.56 +{
   11.57 +	return 0;
   11.58 +}
   11.59 +
   11.60 +static inline void preswap_flush(unsigned type, unsigned long offset)
   11.61 +{
   11.62 +}
   11.63 +
   11.64 +static inline void preswap_flush_area(unsigned type)
   11.65 +{
   11.66 +}
   11.67 +#endif /* CONFIG_PRESWAP */
   11.68 +
   11.69  struct swap_list_t {
   11.70  	int head;	/* head of priority-ordered swapfile list */
   11.71  	int next;	/* swapfile to be used next */
   11.72 @@ -247,7 +299,6 @@ extern struct swap_info_struct *get_swap
   11.73  extern int can_share_swap_page(struct page *);
   11.74  extern int remove_exclusive_swap_page(struct page *);
   11.75  struct backing_dev_info;
   11.76 -
   11.77  extern spinlock_t swap_lock;
   11.78  
   11.79  /* linux/mm/thrash.c */
    12.1 --- a/include/linux/sysctl.h	Wed Jun 17 09:07:23 2009 +0100
    12.2 +++ b/include/linux/sysctl.h	Thu Jun 18 10:24:18 2009 +0100
    12.3 @@ -200,6 +200,7 @@ enum
    12.4  	VM_PANIC_ON_OOM=33,	/* panic at out-of-memory */
    12.5  	VM_VDSO_ENABLED=34,	/* map VDSO into new processes? */
    12.6  	VM_MIN_SLAB=35,		 /* Percent pages ignored by zone reclaim */
    12.7 +	VM_PRESWAP_PAGES=36,	/* pages/target_pages in preswap */
    12.8  };
    12.9  
   12.10  
    13.1 --- a/kernel/sysctl.c	Wed Jun 17 09:07:23 2009 +0100
    13.2 +++ b/kernel/sysctl.c	Thu Jun 18 10:24:18 2009 +0100
    13.3 @@ -967,6 +967,18 @@ static ctl_table vm_table[] = {
    13.4  		.extra1		= &zero,
    13.5  	},
    13.6  #endif
    13.7 +#ifdef CONFIG_PRESWAP
    13.8 +	{
    13.9 +		.ctl_name	= VM_PRESWAP_PAGES,
   13.10 +		.procname	= "preswap",
   13.11 +		.data		= NULL,
   13.12 +		.maxlen		= sizeof(unsigned long),
   13.13 +		.mode		= 0644,
   13.14 +		.proc_handler	= &preswap_sysctl_handler,
   13.15 +		.extra1		= (void *)&preswap_zero,
   13.16 +		.extra2		= (void *)&preswap_infinity,
   13.17 +	},
   13.18 +#endif
   13.19  	{ .ctl_name = 0 }
   13.20  };
   13.21  
    14.1 --- a/mm/Kconfig	Wed Jun 17 09:07:23 2009 +0100
    14.2 +++ b/mm/Kconfig	Thu Jun 18 10:24:18 2009 +0100
    14.3 @@ -152,3 +152,33 @@ config RESOURCES_64BIT
    14.4  	default 64BIT
    14.5  	help
    14.6  	  This option allows memory and IO resources to be 64 bit.
    14.7 +
    14.8 +#
    14.9 +# support for transcendent memory
   14.10 +#
   14.11 +config TMEM
   14.12 +	bool
   14.13 +	depends on XEN
   14.14 +	help
   14.15 +	  In a virtualized environment, allows unused and underutilized
   14.16 +	  system physical memory to be made accessible through a narrow
   14.17 +	  well-defined page-copy-based API.  If unsure, say Y.
   14.18 +
   14.19 +config PRECACHE
   14.20 +	bool "Cache clean pages in transcendent memory"
   14.21 +	depends on XEN
   14.22 +	select TMEM
   14.23 +	help
   14.24 +	  Allows the transcendent memory pool to be used to store clean
   14.25 +	  page-cache pages which, under some circumstances, will greatly
   14.26 +	  reduce paging and thus improve performance.  If unsure, say Y.
   14.27 +
   14.28 +config PRESWAP
   14.29 +	bool "Swap pages to transcendent memory"
   14.30 +	depends on XEN
   14.31 +	select TMEM
   14.32 +	help
   14.33 +	  Allows the transcendent memory pool to be used as a pseudo-swap
   14.34 +	  device which, under some circumstances, will greatly reduce
   14.35 +	  swapping and thus improve performance.  If unsure, say Y.
   14.36 +
    15.1 --- a/mm/Makefile	Wed Jun 17 09:07:23 2009 +0100
    15.2 +++ b/mm/Makefile	Thu Jun 18 10:24:18 2009 +0100
    15.3 @@ -13,6 +13,9 @@ obj-y			:= bootmem.o filemap.o mempool.o
    15.4  			   prio_tree.o util.o mmzone.o vmstat.o $(mmu-y)
    15.5  
    15.6  obj-$(CONFIG_SWAP)	+= page_io.o swap_state.o swapfile.o thrash.o
    15.7 +obj-$(CONFIG_TMEM)	+= tmem.o
    15.8 +obj-$(CONFIG_PRESWAP)	+= preswap.o
    15.9 +obj-$(CONFIG_PRECACHE)	+= precache.o
   15.10  obj-$(CONFIG_HUGETLBFS)	+= hugetlb.o
   15.11  obj-$(CONFIG_NUMA) 	+= mempolicy.o
   15.12  obj-$(CONFIG_SPARSEMEM)	+= sparse.o
    16.1 --- a/mm/filemap.c	Wed Jun 17 09:07:23 2009 +0100
    16.2 +++ b/mm/filemap.c	Thu Jun 18 10:24:18 2009 +0100
    16.3 @@ -30,6 +30,7 @@
    16.4  #include <linux/security.h>
    16.5  #include <linux/syscalls.h>
    16.6  #include <linux/cpuset.h>
    16.7 +#include <linux/precache.h>
    16.8  #include "filemap.h"
    16.9  #include "internal.h"
   16.10  
   16.11 @@ -116,6 +117,16 @@ void __remove_from_page_cache(struct pag
   16.12  {
   16.13  	struct address_space *mapping = page->mapping;
   16.14  
   16.15 +	/*
   16.16 +	 * if we're uptodate, flush out into the precache, otherwise
   16.17 +	 * invalidate any existing precache entries.  We can't leave
   16.18 +	 * stale data around in the precache once our page is gone
   16.19 +	 */
   16.20 +	if (PageUptodate(page))
   16.21 +		precache_put(page->mapping, page->index, page);
   16.22 +	else
   16.23 +		precache_flush(page->mapping, page->index);
   16.24 +
   16.25  	radix_tree_delete(&mapping->page_tree, page->index);
   16.26  	page->mapping = NULL;
   16.27  	mapping->nrpages--;
    17.1 --- a/mm/page_io.c	Wed Jun 17 09:07:23 2009 +0100
    17.2 +++ b/mm/page_io.c	Thu Jun 18 10:24:18 2009 +0100
    17.3 @@ -99,10 +99,18 @@ int swap_writepage(struct page *page, st
    17.4  		ret = -ENOMEM;
    17.5  		goto out;
    17.6  	}
    17.7 +
    17.8 +	set_page_writeback(page);
    17.9 +	if (preswap_put(page) == 1) {
   17.10 +		unlock_page(page);
   17.11 +		end_page_writeback(page);
   17.12 +		bio_put(bio);
   17.13 +		goto out;
   17.14 +	}
   17.15 +
   17.16  	if (wbc->sync_mode == WB_SYNC_ALL)
   17.17  		rw |= (1 << BIO_RW_SYNC);
   17.18  	count_vm_event(PSWPOUT);
   17.19 -	set_page_writeback(page);
   17.20  	unlock_page(page);
   17.21  	submit_bio(rw, bio);
   17.22  out:
   17.23 @@ -116,6 +124,13 @@ int swap_readpage(struct file *file, str
   17.24  
   17.25  	BUG_ON(!PageLocked(page));
   17.26  	ClearPageUptodate(page);
   17.27 +
   17.28 +	if (preswap_get(page) == 1) {
   17.29 +		SetPageUptodate(page);
   17.30 +		unlock_page(page);
   17.31 +		goto out;
   17.32 +	}
   17.33 +
   17.34  	bio = get_swap_bio(GFP_KERNEL, page_private(page), page,
   17.35  				end_swap_bio_read);
   17.36  	if (bio == NULL) {
    18.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    18.2 +++ b/mm/precache.c	Thu Jun 18 10:24:18 2009 +0100
    18.3 @@ -0,0 +1,140 @@
    18.4 +/*
    18.5 + * linux/mm/precache.c
    18.6 + *
    18.7 + * Implements "precache" for filesystems/pagecache on top of transcendent
    18.8 + * memory ("tmem") API.  A filesystem creates an "ephemeral tmem pool"
    18.9 + * and retains the returned pool_id in its superblock.  Clean pages evicted
   18.10 + * from pagecache may be "put" into the pool and associated with a "handle"
   18.11 + * consisting of the pool_id, an object (inode) id, and an index (page offset).
   18.12 + * Note that the page is copied to tmem; no kernel mappings are changed.
   18.13 + * If the page is later needed, the filesystem (or VFS) issues a "get", passing
   18.14 + * the same handle and an empty pageframe.  If successful, the page is copied
   18.15 + * into the pageframe and a disk read is avoided.  But since the tmem pool
   18.16 + * is of indeterminate size, a "put" page has indeterminate longevity
   18.17 + * ("ephemeral"), and the "get" may fail, in which case the filesystem must
   18.18 + * read the page from disk as before.  Note that the filesystem/pagecache are
   18.19 + * responsible for maintaining coherency between the pagecache, precache,
   18.20 + * and the disk, for which "flush page" and "flush object" actions are
   18.21 + * provided.  And when a filesystem is unmounted, it must "destroy" the pool.
   18.22 + *
   18.23 + * Two types of pools may be created for a precache: "private" or "shared".
   18.24 + * For a private pool, a successful "get" always flushes, implementing
   18.25 + * exclusive semantics; for a "shared" pool (which is intended for use by
   18.26 + * co-resident nodes of a cluster filesystem), the "flush" is not guaranteed.
   18.27 + * In either case, a failed "duplicate" put (overwrite) always guarantee
   18.28 + * the old data is flushed.
   18.29 + *
   18.30 + * Note also that multiple accesses to a tmem pool may be concurrent and any
   18.31 + * ordering must be guaranteed by the caller.
   18.32 + *
   18.33 + * Copyright (C) 2008,2009 Dan Magenheimer, Oracle Corp.
   18.34 + */
   18.35 +
   18.36 +#include <linux/precache.h>
   18.37 +#include <linux/module.h>
   18.38 +#include "tmem.h"
   18.39 +
   18.40 +static int precache_auto_allocate; /* set to 1 to auto_allocate */
   18.41 +
   18.42 +int precache_put(struct address_space *mapping, unsigned long index,
   18.43 + struct page *page)
   18.44 +{
   18.45 +	u32 tmem_pool = mapping->host->i_sb->precache_poolid;
   18.46 +	u64 obj = (unsigned long) mapping->host->i_ino;
   18.47 +	u32 ind = (u32) index;
   18.48 +	unsigned long mfn = pfn_to_mfn(page_to_pfn(page));
   18.49 +	int ret;
   18.50 +
   18.51 +	if ((s32)tmem_pool < 0) {
   18.52 +		if (!precache_auto_allocate)
   18.53 +			return 0;
   18.54 +		/* a put on a non-existent precache may auto-allocate one */
   18.55 +		ret = tmem_new_pool(0, 0, 0);
   18.56 +		if (ret < 0)
   18.57 +			return 0;
   18.58 +		printk(KERN_INFO
   18.59 +			"Mapping superblock for s_id=%s to precache_id=%d\n",
   18.60 +			mapping->host->i_sb->s_id, tmem_pool);
   18.61 +		mapping->host->i_sb->precache_poolid = tmem_pool;
   18.62 +	}
   18.63 +	if (ind != index)
   18.64 +		return 0;
   18.65 +	mb(); /* ensure page is quiescent; tmem may address it with an alias */
   18.66 +	return tmem_put_page(tmem_pool, obj, ind, mfn);
   18.67 +}
   18.68 +
   18.69 +int precache_get(struct address_space *mapping, unsigned long index,
   18.70 + struct page *empty_page)
   18.71 +{
   18.72 +	u32 tmem_pool = mapping->host->i_sb->precache_poolid;
   18.73 +	u64 obj = (unsigned long) mapping->host->i_ino;
   18.74 +	u32 ind = (u32) index;
   18.75 +	unsigned long mfn = pfn_to_mfn(page_to_pfn(empty_page));
   18.76 +
   18.77 +	if ((s32)tmem_pool < 0)
   18.78 +		return 0;
   18.79 +	if (ind != index)
   18.80 +		return 0;
   18.81 +
   18.82 +	return tmem_get_page(tmem_pool, obj, ind, mfn);
   18.83 +}
   18.84 +EXPORT_SYMBOL(precache_get);
   18.85 +
   18.86 +int precache_flush(struct address_space *mapping, unsigned long index)
   18.87 +{
   18.88 +	u32 tmem_pool = mapping->host->i_sb->precache_poolid;
   18.89 +	u64 obj = (unsigned long) mapping->host->i_ino;
   18.90 +	u32 ind = (u32) index;
   18.91 +
   18.92 +	if ((s32)tmem_pool < 0)
   18.93 +		return 0;
   18.94 +	if (ind != index)
   18.95 +		return 0;
   18.96 +
   18.97 +	return tmem_flush_page(tmem_pool, obj, ind);
   18.98 +}
   18.99 +EXPORT_SYMBOL(precache_flush);
  18.100 +
  18.101 +int precache_flush_inode(struct address_space *mapping)
  18.102 +{
  18.103 +	u32 tmem_pool = mapping->host->i_sb->precache_poolid;
  18.104 +	u64 obj = (unsigned long) mapping->host->i_ino;
  18.105 +
  18.106 +	if ((s32)tmem_pool < 0)
  18.107 +		return 0;
  18.108 +
  18.109 +	return tmem_flush_object(tmem_pool, obj);
  18.110 +}
  18.111 +EXPORT_SYMBOL(precache_flush_inode);
  18.112 +
  18.113 +int precache_flush_filesystem(struct super_block *sb)
  18.114 +{
  18.115 +	u32 tmem_pool = sb->precache_poolid;
  18.116 +	int ret;
  18.117 +
  18.118 +	if ((s32)tmem_pool < 0)
  18.119 +		return 0;
  18.120 +	ret = tmem_destroy_pool(tmem_pool);
  18.121 +	if (!ret)
  18.122 +		return 0;
  18.123 +	printk(KERN_INFO
  18.124 +		"Unmapping superblock for s_id=%s from precache_id=%d\n",
  18.125 +		sb->s_id, ret);
  18.126 +	sb->precache_poolid = 0;
  18.127 +	return 1;
  18.128 +}
  18.129 +EXPORT_SYMBOL(precache_flush_filesystem);
  18.130 +
  18.131 +void precache_init(struct super_block *sb)
  18.132 +{
  18.133 +	sb->precache_poolid = tmem_new_pool(0, 0, 0);
  18.134 +}
  18.135 +EXPORT_SYMBOL(precache_init);
  18.136 +
  18.137 +void shared_precache_init(struct super_block *sb, char *uuid)
  18.138 +{
  18.139 +	u64 uuid_lo = *(u64 *)uuid;
  18.140 +	u64 uuid_hi = *(u64 *)(&uuid[8]);
  18.141 +	sb->precache_poolid = tmem_new_pool(uuid_lo, uuid_hi, TMEM_POOL_SHARED);
  18.142 +}
  18.143 +EXPORT_SYMBOL(shared_precache_init);
    19.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    19.2 +++ b/mm/preswap.c	Thu Jun 18 10:24:18 2009 +0100
    19.3 @@ -0,0 +1,184 @@
    19.4 +/*
    19.5 + * linux/mm/preswap.c
    19.6 + *
    19.7 + * Implements a fast "preswap" on top of the transcendent memory ("tmem") API.
    19.8 + * When a swapdisk is enabled (with swapon), a "private persistent tmem pool"
    19.9 + * is created along with a bit-per-page preswap_map.  When swapping occurs
   19.10 + * and a page is about to be written to disk, a "put" into the pool may first
   19.11 + * be attempted by passing the pageframe to be swapped, along with a "handle"
   19.12 + * consisting of a pool_id, an object id, and an index.  Since the pool is of
   19.13 + * indeterminate size, the "put" may be rejected, in which case the page
   19.14 + * is swapped to disk as normal.  If the "put" is successful, the page is
   19.15 + * copied to tmem and the preswap_map records the success.  Later, when
   19.16 + * the page needs to be swapped in, the preswap_map is checked and, if set,
   19.17 + * the page may be obtained with a "get" operation.  Note that the swap
   19.18 + * subsystem is responsible for: maintaining coherency between the swapcache,
   19.19 + * preswap, and the swapdisk; for evicting stale pages from preswap; and for
   19.20 + * emptying preswap when swapoff is performed. The "flush page" and "flush
   19.21 + * object" actions are provided for this.
   19.22 + *
   19.23 + * Note that if a "duplicate put" is performed to overwrite a page and
   19.24 + * the "put" operation fails, the page (and old data) is flushed and lost.
   19.25 + * Also note that multiple accesses to a tmem pool may be concurrent and
   19.26 + * any ordering must be guaranteed by the caller.
   19.27 + *
   19.28 + * Copyright (C) 2008,2009 Dan Magenheimer, Oracle Corp.
   19.29 + */
   19.30 +
   19.31 +#include <linux/mm.h>
   19.32 +#include <linux/mman.h>
   19.33 +#include <linux/sysctl.h>
   19.34 +#include <linux/swap.h>
   19.35 +#include <linux/swapops.h>
   19.36 +#include <linux/proc_fs.h>
   19.37 +#include <linux/security.h>
   19.38 +#include <linux/capability.h>
   19.39 +#include <linux/uaccess.h>
   19.40 +#include "tmem.h"
   19.41 +
   19.42 +static u32 preswap_poolid = -1; /* if negative, preswap will never call tmem */
   19.43 +
   19.44 +const unsigned long preswap_zero = 0, preswap_infinity = ~0UL; /* for sysctl */
   19.45 +
   19.46 +/*
   19.47 + * Swizzling increases objects per swaptype, increasing tmem concurrency
   19.48 + * for heavy swaploads.  Later, larger nr_cpus -> larger SWIZ_BITS
   19.49 + */
   19.50 +#define SWIZ_BITS		4
   19.51 +#define SWIZ_MASK		((1 << SWIZ_BITS) - 1)
   19.52 +#define oswiz(_type, _ind)	((_type << SWIZ_BITS) | (_ind & SWIZ_MASK))
   19.53 +#define iswiz(_ind)		(_ind >> SWIZ_BITS)
   19.54 +
   19.55 +/*
   19.56 + * preswap_map test/set/clear operations (must be atomic)
   19.57 + */
   19.58 +
   19.59 +int preswap_test(struct swap_info_struct *sis, unsigned long offset)
   19.60 +{
   19.61 +	if (!sis->preswap_map)
   19.62 +		return 0;
   19.63 +	return test_bit(offset % BITS_PER_LONG,
   19.64 +		&sis->preswap_map[offset/BITS_PER_LONG]);
   19.65 +}
   19.66 +
   19.67 +static inline void preswap_set(struct swap_info_struct *sis,
   19.68 +				unsigned long offset)
   19.69 +{
   19.70 +	if (!sis->preswap_map)
   19.71 +		return;
   19.72 +	set_bit(offset % BITS_PER_LONG,
   19.73 +		&sis->preswap_map[offset/BITS_PER_LONG]);
   19.74 +}
   19.75 +
   19.76 +static inline void preswap_clear(struct swap_info_struct *sis,
   19.77 +				unsigned long offset)
   19.78 +{
   19.79 +	if (!sis->preswap_map)
   19.80 +		return;
   19.81 +	clear_bit(offset % BITS_PER_LONG,
   19.82 +		&sis->preswap_map[offset/BITS_PER_LONG]);
   19.83 +}
   19.84 +
   19.85 +/*
   19.86 + * preswap tmem operations
   19.87 + */
   19.88 +
   19.89 +/* returns 1 if the page was successfully put into preswap, 0 if the page
   19.90 + * was declined, and -ERRNO for a specific error */
   19.91 +int preswap_put(struct page *page)
   19.92 +{
   19.93 +	swp_entry_t entry = { .val = page_private(page), };
   19.94 +	unsigned type = swp_type(entry);
   19.95 +	pgoff_t offset = swp_offset(entry);
   19.96 +	u64 ind64 = (u64)offset;
   19.97 +	u32 ind = (u32)offset;
   19.98 +	unsigned long mfn = pfn_to_mfn(page_to_pfn(page));
   19.99 +	struct swap_info_struct *sis = get_swap_info_struct(type);
  19.100 +	int dup = 0, ret;
  19.101 +
  19.102 +	if ((s32)preswap_poolid < 0)
  19.103 +		return 0;
  19.104 +	if (ind64 != ind)
  19.105 +		return 0;
  19.106 +	if (preswap_test(sis, offset))
  19.107 +		dup = 1;
  19.108 +	mb(); /* ensure page is quiescent; tmem may address it with an alias */
  19.109 +	ret = tmem_put_page(preswap_poolid, oswiz(type, ind), iswiz(ind), mfn);
  19.110 +	if (ret == 1) {
  19.111 +		preswap_set(sis, offset);
  19.112 +		if (!dup)
  19.113 +			sis->preswap_pages++;
  19.114 +	} else if (dup) {
  19.115 +		/* failed dup put always results in an automatic flush of
  19.116 +		 * the (older) page from preswap */
  19.117 +		preswap_clear(sis, offset);
  19.118 +		sis->preswap_pages--;
  19.119 +	}
  19.120 +	return ret;
  19.121 +}
  19.122 +
  19.123 +/* returns 1 if the page was successfully gotten from preswap, 0 if the page
  19.124 + * was not present (should never happen!), and -ERRNO for a specific error */
  19.125 +int preswap_get(struct page *page)
  19.126 +{
  19.127 +	swp_entry_t entry = { .val = page_private(page), };
  19.128 +	unsigned type = swp_type(entry);
  19.129 +	pgoff_t offset = swp_offset(entry);
  19.130 +	u64 ind64 = (u64)offset;
  19.131 +	u32 ind = (u32)offset;
  19.132 +	unsigned long mfn = pfn_to_mfn(page_to_pfn(page));
  19.133 +	struct swap_info_struct *sis = get_swap_info_struct(type);
  19.134 +	int ret;
  19.135 +
  19.136 +	if ((s32)preswap_poolid < 0)
  19.137 +		return 0;
  19.138 +	if (ind64 != ind)
  19.139 +		return 0;
  19.140 +	if (!preswap_test(sis, offset))
  19.141 +		return 0;
  19.142 +	ret = tmem_get_page(preswap_poolid, oswiz(type, ind), iswiz(ind), mfn);
  19.143 +	return ret;
  19.144 +}
  19.145 +
  19.146 +/* flush a single page from preswap */
  19.147 +void preswap_flush(unsigned type, unsigned long offset)
  19.148 +{
  19.149 +	u64 ind64 = (u64)offset;
  19.150 +	u32 ind = (u32)offset;
  19.151 +	struct swap_info_struct *sis = get_swap_info_struct(type);
  19.152 +	int ret = 1;
  19.153 +
  19.154 +	if ((s32)preswap_poolid < 0)
  19.155 +		return;
  19.156 +	if (ind64 != ind)
  19.157 +		return;
  19.158 +	if (preswap_test(sis, offset)) {
  19.159 +		ret = tmem_flush_page(preswap_poolid,
  19.160 +					oswiz(type, ind), iswiz(ind));
  19.161 +		sis->preswap_pages--;
  19.162 +		preswap_clear(sis, offset);
  19.163 +	}
  19.164 +}
  19.165 +
  19.166 +/* flush all pages from the passed swaptype */
  19.167 +void preswap_flush_area(unsigned type)
  19.168 +{
  19.169 +	struct swap_info_struct *sis = get_swap_info_struct(type);
  19.170 +	int ind;
  19.171 +
  19.172 +	if ((s32)preswap_poolid < 0)
  19.173 +		return;
  19.174 +	for (ind = SWIZ_MASK; ind >= 0; ind--)
  19.175 +		(void)tmem_flush_object(preswap_poolid, oswiz(type, ind));
  19.176 +	sis->preswap_pages = 0;
  19.177 +}
  19.178 +
  19.179 +void preswap_init(unsigned type)
  19.180 +{
  19.181 +	/* only need one tmem pool for all swap types */
  19.182 +	if ((s32)preswap_poolid >= 0)
  19.183 +		return;
  19.184 +	preswap_poolid = tmem_new_pool(0, 0, TMEM_POOL_PERSIST);
  19.185 +	if (preswap_poolid < 0)
  19.186 +		return;
  19.187 +}
    20.1 --- a/mm/swapfile.c	Wed Jun 17 09:07:23 2009 +0100
    20.2 +++ b/mm/swapfile.c	Thu Jun 18 10:24:18 2009 +0100
    20.3 @@ -284,6 +284,7 @@ static int swap_entry_free(struct swap_i
    20.4  				swap_list.next = p - swap_info;
    20.5  			nr_swap_pages++;
    20.6  			p->inuse_pages--;
    20.7 +			preswap_flush(p - swap_info, offset);
    20.8  		}
    20.9  	}
   20.10  	return count;
   20.11 @@ -623,7 +624,7 @@ static int unuse_mm(struct mm_struct *mm
   20.12   * Recycle to start on reaching the end, returning 0 when empty.
   20.13   */
   20.14  static unsigned int find_next_to_unuse(struct swap_info_struct *si,
   20.15 -					unsigned int prev)
   20.16 +				unsigned int prev, unsigned int preswap)
   20.17  {
   20.18  	unsigned int max = si->max;
   20.19  	unsigned int i = prev;
   20.20 @@ -649,6 +650,12 @@ static unsigned int find_next_to_unuse(s
   20.21  			prev = 0;
   20.22  			i = 1;
   20.23  		}
   20.24 +		if (preswap) {
   20.25 +			if (preswap_test(si, i))
   20.26 +				break;
   20.27 +			else
   20.28 +				continue;
   20.29 +		}
   20.30  		count = si->swap_map[i];
   20.31  		if (count && count != SWAP_MAP_BAD)
   20.32  			break;
   20.33 @@ -660,8 +667,12 @@ static unsigned int find_next_to_unuse(s
   20.34   * We completely avoid races by reading each swap page in advance,
   20.35   * and then search for the process using it.  All the necessary
   20.36   * page table adjustments can then be made atomically.
   20.37 + *
   20.38 + * if the boolean preswap is true, only unuse pages_to_unuse pages;
   20.39 + * pages_to_unuse==0 means all pages
   20.40   */
   20.41 -static int try_to_unuse(unsigned int type)
   20.42 +static int try_to_unuse(unsigned int type, unsigned int preswap,
   20.43 +		unsigned long pages_to_unuse)
   20.44  {
   20.45  	struct swap_info_struct * si = &swap_info[type];
   20.46  	struct mm_struct *start_mm;
   20.47 @@ -697,7 +708,7 @@ static int try_to_unuse(unsigned int typ
   20.48  	 * one pass through swap_map is enough, but not necessarily:
   20.49  	 * there are races when an instance of an entry might be missed.
   20.50  	 */
   20.51 -	while ((i = find_next_to_unuse(si, i)) != 0) {
   20.52 +	while ((i = find_next_to_unuse(si, i, preswap)) != 0) {
   20.53  		if (signal_pending(current)) {
   20.54  			retval = -EINTR;
   20.55  			break;
   20.56 @@ -876,6 +887,8 @@ static int try_to_unuse(unsigned int typ
   20.57  		 * interactive performance.
   20.58  		 */
   20.59  		cond_resched();
   20.60 +		if (preswap && pages_to_unuse && !--pages_to_unuse)
   20.61 +			break;
   20.62  	}
   20.63  
   20.64  	mmput(start_mm);
   20.65 @@ -1198,7 +1211,7 @@ asmlinkage long sys_swapoff(const char _
   20.66  	spin_unlock(&swap_lock);
   20.67  
   20.68  	current->flags |= PF_SWAPOFF;
   20.69 -	err = try_to_unuse(type);
   20.70 +	err = try_to_unuse(type, 0, 0);
   20.71  	current->flags &= ~PF_SWAPOFF;
   20.72  
   20.73  	if (err) {
   20.74 @@ -1242,9 +1255,14 @@ asmlinkage long sys_swapoff(const char _
   20.75  	swap_map = p->swap_map;
   20.76  	p->swap_map = NULL;
   20.77  	p->flags = 0;
   20.78 +	preswap_flush_area(p - swap_info);
   20.79  	spin_unlock(&swap_lock);
   20.80  	mutex_unlock(&swapon_mutex);
   20.81  	vfree(swap_map);
   20.82 +#ifdef CONFIG_PRESWAP
   20.83 +	if (p->preswap_map)
   20.84 +		vfree(p->preswap_map);
   20.85 +#endif
   20.86  	inode = mapping->host;
   20.87  	if (S_ISBLK(inode->i_mode)) {
   20.88  		struct block_device *bdev = I_BDEV(inode);
   20.89 @@ -1553,6 +1571,11 @@ asmlinkage long sys_swapon(const char __
   20.90  
   20.91  		error = 0;
   20.92  		memset(p->swap_map, 0, maxpages * sizeof(short));
   20.93 +#ifdef CONFIG_PRESWAP
   20.94 +		p->preswap_map = vmalloc(maxpages / sizeof(long));
   20.95 +		if (p->preswap_map)
   20.96 +			memset(p->preswap_map, 0, maxpages / sizeof(long));
   20.97 +#endif
   20.98  		for (i = 0; i < swap_header->info.nr_badpages; i++) {
   20.99  			int page_nr = swap_header->info.badpages[i];
  20.100  			if (page_nr <= 0 || page_nr >= swap_header->info.last_page)
  20.101 @@ -1615,6 +1638,7 @@ asmlinkage long sys_swapon(const char __
  20.102  	} else {
  20.103  		swap_info[prev].next = p - swap_info;
  20.104  	}
  20.105 +	preswap_init(p - swap_info);
  20.106  	spin_unlock(&swap_lock);
  20.107  	mutex_unlock(&swapon_mutex);
  20.108  	error = 0;
  20.109 @@ -1729,6 +1753,8 @@ int valid_swaphandles(swp_entry_t entry,
  20.110  
  20.111  	if (!page_cluster)	/* no readahead */
  20.112  		return 0;
  20.113 +	if (preswap_test(swapdev, swp_offset(entry)))
  20.114 +		return 0;
  20.115  	toff = (swp_offset(entry) >> page_cluster) << page_cluster;
  20.116  	if (!toff)		/* first page is swap header */
  20.117  		toff++, i--;
  20.118 @@ -1744,9 +1770,101 @@ int valid_swaphandles(swp_entry_t entry,
  20.119  			break;
  20.120  		if (swapdev->swap_map[toff] == SWAP_MAP_BAD)
  20.121  			break;
  20.122 +		/* Don't read in preswap pages */
  20.123 +		if (preswap_test(swapdev, toff))
  20.124 +			break;
  20.125  		toff++;
  20.126  		ret++;
  20.127  	} while (--i);
  20.128  	spin_unlock(&swap_lock);
  20.129  	return ret;
  20.130  }
  20.131 +
  20.132 +#ifdef CONFIG_PRESWAP
  20.133 +/*
  20.134 + * preswap infrastructure functions
  20.135 + */
  20.136 +
  20.137 +/* code structure leveraged from sys_swapoff */
  20.138 +void preswap_shrink(unsigned long target_pages)
  20.139 +{
  20.140 +	struct swap_info_struct *si = NULL;
  20.141 +	unsigned long total_pages = 0, total_pages_to_unuse;
  20.142 +	unsigned long pages = 0, unuse_pages = 0;
  20.143 +	int type;
  20.144 +	int wrapped = 0;
  20.145 +
  20.146 +	do {
  20.147 +		/*
  20.148 +		 * we don't want to hold swap_lock while doing a very
  20.149 +		 * lengthy try_to_unuse, but swap_list may change
  20.150 +		 * so restart scan from swap_list.head each time
  20.151 +		 */
  20.152 +		spin_lock(&swap_lock);
  20.153 +		total_pages = 0;
  20.154 +		for (type = swap_list.head; type >= 0; type = si->next) {
  20.155 +			si = get_swap_info_struct(type);
  20.156 +			total_pages += si->preswap_pages;
  20.157 +		}
  20.158 +		if (total_pages <= target_pages) {
  20.159 +			spin_unlock(&swap_lock);
  20.160 +			return;
  20.161 +		}
  20.162 +		total_pages_to_unuse = total_pages - target_pages;
  20.163 +		for (type = swap_list.head; type >= 0; type = si->next) {
  20.164 +			si = get_swap_info_struct(type);
  20.165 +			if (total_pages_to_unuse < si->preswap_pages)
  20.166 +				pages = unuse_pages = total_pages_to_unuse;
  20.167 +			else {
  20.168 +				pages = si->preswap_pages;
  20.169 +				unuse_pages = 0; /* unuse all */
  20.170 +			}
  20.171 +			if (security_vm_enough_memory(pages))
  20.172 +				continue;
  20.173 +			vm_unacct_memory(pages);
  20.174 +			break;
  20.175 +		}
  20.176 +		spin_unlock(&swap_lock);
  20.177 +		if (type < 0)
  20.178 +			return;
  20.179 +		current->flags |= PF_SWAPOFF;
  20.180 +		(void)try_to_unuse(type, 1, unuse_pages);
  20.181 +		current->flags &= ~PF_SWAPOFF;
  20.182 +		wrapped++;
  20.183 +	} while (wrapped <= 3);
  20.184 +}
  20.185 +
  20.186 +
  20.187 +#ifdef CONFIG_SYSCTL
  20.188 +/* cat /sys/proc/vm/preswap provides total number of pages in preswap
  20.189 + * across all swaptypes.  echo N > /sys/proc/vm/preswap attempts to shrink
  20.190 + * preswap page usage to N (usually 0) */
  20.191 +int preswap_sysctl_handler(ctl_table *table, int write,
  20.192 +	struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
  20.193 +{
  20.194 +	unsigned long npages;
  20.195 +	int type;
  20.196 +	unsigned long totalpages = 0;
  20.197 +	struct swap_info_struct *si = NULL;
  20.198 +
  20.199 +	/* modeled after hugetlb_sysctl_handler in mm/hugetlb.c */
  20.200 +	if (!write) {
  20.201 +		spin_lock(&swap_lock);
  20.202 +		for (type = swap_list.head; type >= 0; type = si->next) {
  20.203 +			si = get_swap_info_struct(type);
  20.204 +			totalpages += si->preswap_pages;
  20.205 +		}
  20.206 +		spin_unlock(&swap_lock);
  20.207 +		npages = totalpages;
  20.208 +	}
  20.209 +	table->data = &npages;
  20.210 +	table->maxlen = sizeof(unsigned long);
  20.211 +	proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
  20.212 +
  20.213 +	if (write)
  20.214 +		preswap_shrink(npages);
  20.215 +
  20.216 +	return 0;
  20.217 +}
  20.218 +#endif
  20.219 +#endif /* CONFIG_PRESWAP */
    21.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    21.2 +++ b/mm/tmem.c	Thu Jun 18 10:24:18 2009 +0100
    21.3 @@ -0,0 +1,41 @@
    21.4 +/*
    21.5 + * Xen implementation for transcendent memory (tmem)
    21.6 + *
    21.7 + * Dan Magenheimer <dan.magenheimer@oracle.com> 2009
    21.8 + */
    21.9 +
   21.10 +#include <linux/types.h>
   21.11 +#include <xen/interface/xen.h>
   21.12 +#include <asm/hypervisor.h>
   21.13 +
   21.14 +int xen_tmem_op(u32 tmem_cmd, u32 tmem_pool, u64 object, u32 index,
   21.15 +	unsigned long gmfn, u32 tmem_offset, u32 pfn_offset, u32 len)
   21.16 +{
   21.17 +	struct tmem_op op;
   21.18 +	int rc = 0;
   21.19 +
   21.20 +	op.cmd = tmem_cmd;
   21.21 +	op.pool_id = tmem_pool;
   21.22 +	op.u.gen.object = object;
   21.23 +	op.u.gen.index = index;
   21.24 +	op.u.gen.tmem_offset = tmem_offset;
   21.25 +	op.u.gen.pfn_offset = pfn_offset;
   21.26 +	op.u.gen.len = len;
   21.27 +	op.u.gen.cmfn = gmfn;
   21.28 +	rc = HYPERVISOR_tmem_op(&op);
   21.29 +	return rc;
   21.30 +}
   21.31 +
   21.32 +int xen_tmem_new_pool(uint32_t tmem_cmd, uint64_t uuid_lo,
   21.33 +	uint64_t uuid_hi, uint32_t flags)
   21.34 +{
   21.35 +	struct tmem_op op;
   21.36 +	int rc = 0;
   21.37 +
   21.38 +	op.cmd = tmem_cmd;
   21.39 +	op.u.new.uuid[0] = uuid_lo;
   21.40 +	op.u.new.uuid[1] = uuid_hi;
   21.41 +	op.u.new.flags = flags;
   21.42 +	rc = HYPERVISOR_tmem_op(&op);
   21.43 +	return rc;
   21.44 +}
    22.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    22.2 +++ b/mm/tmem.h	Thu Jun 18 10:24:18 2009 +0100
    22.3 @@ -0,0 +1,84 @@
    22.4 +/*
    22.5 + * linux/mm/tmem.h
    22.6 + *
    22.7 + * Interface to transcendent memory, used by mm/precache.c and mm/preswap.c
    22.8 + * Currently implemented on XEN, but may be implemented elsewhere in future.
    22.9 + *
   22.10 + * Copyright (C) 2008,2009 Dan Magenheimer, Oracle Corp.
   22.11 + */
   22.12 +
   22.13 +#ifdef CONFIG_XEN
   22.14 +#include <xen/interface/xen.h>
   22.15 +
   22.16 +/* Bits for HYPERVISOR_tmem_op(TMEM_NEW_POOL) */
   22.17 +#define TMEM_POOL_MIN_PAGESHIFT   12
   22.18 +#define TMEM_POOL_PAGEORDER       (PAGE_SHIFT - TMEM_POOL_MIN_PAGESHIFT)
   22.19 +
   22.20 +extern int xen_tmem_op(u32 tmem_cmd, u32 tmem_pool, u64 object, u32 index,
   22.21 +	unsigned long gmfn, u32 tmem_offset, u32 pfn_offset, u32 len);
   22.22 +extern int xen_tmem_new_pool(u32 tmem_cmd, u64 uuid_lo, u64 uuid_hi, u32 flags);
   22.23 +
   22.24 +static inline int tmem_put_page(u32 pool_id, u64 object, u32 index,
   22.25 +	unsigned long gmfn)
   22.26 +{
   22.27 +	return xen_tmem_op(TMEM_PUT_PAGE, pool_id, object, index,
   22.28 +		gmfn, 0, 0, 0);
   22.29 +}
   22.30 +
   22.31 +static inline int tmem_get_page(u32 pool_id, u64 object, u32 index,
   22.32 +	unsigned long gmfn)
   22.33 +{
   22.34 +	return xen_tmem_op(TMEM_GET_PAGE, pool_id, object, index,
   22.35 +		gmfn, 0, 0, 0);
   22.36 +}
   22.37 +
   22.38 +static inline int tmem_flush_page(u32 pool_id, u64 object, u32 index)
   22.39 +{
   22.40 +	return xen_tmem_op(TMEM_FLUSH_PAGE, pool_id, object, index,
   22.41 +		0, 0, 0, 0);
   22.42 +}
   22.43 +
   22.44 +static inline int tmem_flush_object(u32 pool_id, u64 object)
   22.45 +{
   22.46 +	return xen_tmem_op(TMEM_FLUSH_OBJECT, pool_id, object, 0, 0, 0, 0, 0);
   22.47 +}
   22.48 +
   22.49 +static inline int tmem_new_pool(u64 uuid_lo, u64 uuid_hi, u32 flags)
   22.50 +{
   22.51 +	BUILD_BUG_ON((TMEM_POOL_PAGEORDER < 0) ||
   22.52 +		(TMEM_POOL_PAGEORDER >= TMEM_POOL_PAGESIZE_MASK));
   22.53 +	flags |= TMEM_POOL_PAGEORDER << TMEM_POOL_PAGESIZE_SHIFT;
   22.54 +	return xen_tmem_new_pool(TMEM_NEW_POOL, uuid_lo, uuid_hi, flags);
   22.55 +}
   22.56 +
   22.57 +static inline int tmem_destroy_pool(u32 pool_id)
   22.58 +{
   22.59 +	return xen_tmem_op(TMEM_DESTROY_POOL, pool_id, 0, 0, 0, 0, 0, 0);
   22.60 +}
   22.61 +#else
   22.62 +struct tmem_op {
   22.63 +	u32 cmd;
   22.64 +	s32 pool_id; /* private > 0; shared < 0; 0 is invalid */
   22.65 +	union {
   22.66 +		struct {  /* for cmd == TMEM_NEW_POOL */
   22.67 +			u64 uuid[2];
   22.68 +			u32 flags;
   22.69 +		} new;
   22.70 +		struct {  /* for cmd == TMEM_CONTROL */
   22.71 +			u32 subop;
   22.72 +			u32 cli_id;
   22.73 +			u32 arg1;
   22.74 +			u32 arg2;
   22.75 +			void *buf;
   22.76 +		} ctrl;
   22.77 +		struct {
   22.78 +			u64 object;
   22.79 +			u32 index;
   22.80 +			u32 tmem_offset;
   22.81 +			u32 pfn_offset;
   22.82 +			u32 len;
   22.83 +			unsigned long pfn;  /* page frame */
   22.84 +		} gen;
   22.85 +	} u;
   22.86 +};
   22.87 +#endif
    23.1 --- a/mm/truncate.c	Wed Jun 17 09:07:23 2009 +0100
    23.2 +++ b/mm/truncate.c	Thu Jun 18 10:24:18 2009 +0100
    23.3 @@ -12,13 +12,14 @@
    23.4  #include <linux/module.h>
    23.5  #include <linux/pagemap.h>
    23.6  #include <linux/pagevec.h>
    23.7 +#include <linux/precache.h>
    23.8  #include <linux/buffer_head.h>	/* grr. try_to_release_page,
    23.9  				   do_invalidatepage */
   23.10  
   23.11 -
   23.12  static inline void truncate_partial_page(struct page *page, unsigned partial)
   23.13  {
   23.14  	memclear_highpage_flush(page, partial, PAGE_CACHE_SIZE-partial);
   23.15 +	precache_flush(page->mapping, page->index);
   23.16  	if (PagePrivate(page))
   23.17  		do_invalidatepage(page, partial);
   23.18  }
   23.19 @@ -46,6 +47,10 @@ truncate_complete_page(struct address_sp
   23.20  	ClearPageUptodate(page);
   23.21  	ClearPageMappedToDisk(page);
   23.22  	remove_from_page_cache(page);
   23.23 +	/* this must be after the remove_from_page_cache which
   23.24 +	 * calls precache_put
   23.25 +	 */
   23.26 +	precache_flush(mapping, page->index);
   23.27  	page_cache_release(page);	/* pagecache ref */
   23.28  }
   23.29  
   23.30 @@ -118,6 +123,7 @@ void truncate_inode_pages_range(struct a
   23.31  	pgoff_t next;
   23.32  	int i;
   23.33  
   23.34 +	precache_flush_inode(mapping);
   23.35  	if (mapping->nrpages == 0)
   23.36  		return;
   23.37  
   23.38 @@ -191,6 +197,7 @@ void truncate_inode_pages_range(struct a
   23.39  		}
   23.40  		pagevec_release(&pvec);
   23.41  	}
   23.42 +	precache_flush_inode(mapping);
   23.43  }
   23.44  EXPORT_SYMBOL(truncate_inode_pages_range);
   23.45  
   23.46 @@ -324,6 +331,7 @@ int invalidate_inode_pages2_range(struct
   23.47  	int did_range_unmap = 0;
   23.48  	int wrapped = 0;
   23.49  
   23.50 +	precache_flush_inode(mapping);
   23.51  	pagevec_init(&pvec, 0);
   23.52  	next = start;
   23.53  	while (next <= end && !ret && !wrapped &&
   23.54 @@ -379,6 +387,7 @@ int invalidate_inode_pages2_range(struct
   23.55  		pagevec_release(&pvec);
   23.56  		cond_resched();
   23.57  	}
   23.58 +	precache_flush_inode(mapping);
   23.59  	return ret;
   23.60  }
   23.61  EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);