ia64/xen-unstable

changeset 19748:8373971089a7

tools/xenballoond: add tmem capability to directed/self-ballooning
and monitor tool

This patch adds tmem support to the largely unknown/unused
xenballoond scripts that implement both self-ballooning and
a foundation for directed-ballooning. Tmem and automated
ballooning are highly complementary in that, when ballooning
is over-aggressive, paging and swapping can increase noticably.
Precache preserves evicted pages that may be needed again soon
(thus eliminating disk reads) and preswap provides memory-based
swapping that occurs if ballooning is insufficiently responsive
to a sudden increase in activity and memory demand (thus
eliminating disk writes and reads).

There are two changes in this patch:

1) The xenballoond service is a convenient place to implement
userland "preswap shrinking".**
2) The xenballoon-monitor script is a convenient place to
report (and view with "watch -d") the frequent memory
rebalancing that results from tmem usage on a busy system.

Note that for best results (and for the monitor script to
work), the xenstore-* tools should be installed on each guest.

(** In a disk-based swap device, stale pages are often left
on-disk even after they are no longer needed or valid; they
are simply overwritten if/when the disk blocks are needed
again, which may be a very long time. Preswap behaves
much like a disk, but uses precious pages of memory that
count against a guest's memory allocation; thus stale
pages are very undesirable. Preswap shrinking periodically
attempts to remove stale pages from preswap by using a
sysfs interface created by the linux-side tmem patch.)

Signed-off-by: Dan Magenheimer <dan.magenheimer@oracle.com>
author Keir Fraser <keir.fraser@citrix.com>
date Tue Jun 16 11:17:28 2009 +0100 (2009-06-16)
parents 7f9286d21706
children f649dd4454af
files tools/xenballoon/xenballoon-monitor tools/xenballoon/xenballoon.conf tools/xenballoon/xenballoond tools/xenballoon/xenballoond.init
line diff
     1.1 --- a/tools/xenballoon/xenballoon-monitor	Tue Jun 16 11:15:48 2009 +0100
     1.2 +++ b/tools/xenballoon/xenballoon-monitor	Tue Jun 16 11:17:28 2009 +0100
     1.3 @@ -2,6 +2,7 @@
     1.4  #
     1.5  # xenballoon-monitor - monitor certain stats from xenballoond
     1.6  #   (run in dom0 with "watch -d xenballoon-monitor" for xentop-like output)
     1.7 +# updated 090610 to include tmem stats
     1.8  #
     1.9  # Copyright (C) 2009 Oracle Corporation and/or its affiliates.
    1.10  # All rights reserved
    1.11 @@ -9,10 +10,12 @@
    1.12  #
    1.13  # Hint: Use "xm sched-credit -d 0 -w 2000" to watch on heavily loaded machines
    1.14  #
    1.15 -echo "id   mem-kb  tgt-kb  commit   swapin  swapout      pgin     pgout active(sec)"
    1.16 +TMEMTMP=$(/bin/mktemp -q /tmp/xenballoon-monitor.XXXXXX)
    1.17 +echo "id   mem-kb  tgt-kb  commit  swapin swapout    pgin    pgout  preswap  precache"
    1.18  for i in `xenstore-list /local/domain`; do
    1.19   if [ "$i" -ne 0 ]; then
    1.20 - tot=0; tgt=0; sin=0; sout=0; pgin=0; pgout=0; cmt=0; up=0; idle=0; act=0;
    1.21 + tot=0; tgt=0; sin=0; sout=0; pgin=0; pgout=0; cmt=0; up=0; idle=0;
    1.22 + act=0; preswap=0; precache=0
    1.23   if xenstore-exists /local/domain/$i/memory/meminfo; then
    1.24    tot=`xenstore-read /local/domain/$i/memory/meminfo | grep MemTotal \
    1.25     | sed 's/[^1-9]*\([1-9][0-9]*\).*/\1/'`
    1.26 @@ -23,21 +26,34 @@ for i in `xenstore-list /local/domain`; 
    1.27    tgt=`xenstore-read /local/domain/$i/memory/selftarget`
    1.28   fi
    1.29   if xenstore-exists /local/domain/$i/memory/vmstat; then
    1.30 -  sin=`xenstore-read /local/domain/$i/memory/vmstat | grep pswpin \
    1.31 - 	| cut -d" " -f2`
    1.32 -  sout=`xenstore-read /local/domain/$i/memory/vmstat | grep pswpout \
    1.33 - 	| cut -d" " -f2`
    1.34 -  pgin=`xenstore-read /local/domain/$i/memory/vmstat | grep pgpgin \
    1.35 - 	| cut -d" " -f2`
    1.36 -  pgout=`xenstore-read /local/domain/$i/memory/vmstat | grep pgout \
    1.37 -  	| cut -d" " -f2`
    1.38 +  sin=$(xenstore-read /local/domain/$i/memory/vmstat | tr '\\\n' '\n' \
    1.39 +	| grep pswpin | cut -d" " -f2)
    1.40 +  sout=$(xenstore-read /local/domain/$i/memory/vmstat | tr '\\\n' '\n' \
    1.41 +	| grep pswpout | cut -d" " -f2)
    1.42 +  pgin=$(xenstore-read /local/domain/$i/memory/vmstat | tr '\\\n' '\n' \
    1.43 +	| grep pgpgin | cut -d" " -f2)
    1.44 +  pgout=$(xenstore-read /local/domain/$i/memory/vmstat | tr '\\\n' '\n' \
    1.45 +	| grep pgout | cut -d" " -f2)
    1.46   fi
    1.47 - if xenstore-exists /local/domain/$i/memory/uptime; then
    1.48 -  up=`xenstore-read /local/domain/$i/memory/uptime | cut -d" " -f1`
    1.49 -  idle=`xenstore-read /local/domain/$i/memory/uptime | cut -d" " -f2`
    1.50 -  act=`echo $up - $idle | bc -iq`
    1.51 + xm tmem-list --all --long > $TMEMTMP
    1.52 + precache=`grep "C=CI:$i" $TMEMTMP | sed 's/C=CI.*Ec:\([0-9][0-9]*\).*/\1/'`
    1.53 + if xenstore-exists /local/domain/$i/memory/preswap; then
    1.54 +  preswap=`xenstore-read /local/domain/$i/memory/preswap`
    1.55 +  printf "%2d %8d%8d%8d%7d%8d%9d%9d%9d%9d\n" $i $tot $tgt $cmt $sin $sout $pgin $pgout $preswap $precache
    1.56 + else
    1.57 +  printf "%2d %8d%8d%8d%9d%9d%10d%10d\n" $i $tot $tgt $cmt $sin $sout $pgin $pgout
    1.58   fi
    1.59 - printf "%2d %8d%8d%8d%9d%9d%10d%10d%10.2f\n" $i $tot $tgt $cmt $sin $sout $pgin $pgout $act
    1.60   fi
    1.61  done
    1.62 -echo Free memory: `xm info | grep free | sed 's/[^1-9]*\([1-9][0-9]*\).*/\1/'` MB
    1.63 +echo -n Free memory: `xm info | grep free | sed 's/[^1-9]*\([1-9][0-9]*\).*/\1/'` MiB "  "
    1.64 +tmem_free_pages=`grep "G=" $TMEMTMP | sed 's/G=.*Ta:\([0-9][0-9]*\).*/\1/'`
    1.65 +if [ ! -z "$tmem_free_pages" ]; then
    1.66 + let "tmem_free_mb=$tmem_free_pages/256"
    1.67 + echo -n Idle tmem: $tmem_free_mb MiB "  "
    1.68 +fi
    1.69 +tmem_eph_pages=`grep "G=" $TMEMTMP | sed 's/G=.*Ec:\([0-9][0-9]*\).*/\1/'`
    1.70 +if [ ! -z "$tmem_eph_pages" ]; then
    1.71 + let "tmem_eph_mb=$tmem_eph_pages/256"
    1.72 + echo -n Ephemeral tmem: $tmem_eph_mb MiB
    1.73 +fi
    1.74 +echo ""
     2.1 --- a/tools/xenballoon/xenballoon.conf	Tue Jun 16 11:15:48 2009 +0100
     2.2 +++ b/tools/xenballoon/xenballoon.conf	Tue Jun 16 11:17:28 2009 +0100
     2.3 @@ -89,3 +89,48 @@ XENBALLOON_SEND_VMSTAT=1
     2.4  # If xenbus is enabled, whether selfballooning or directed ballooning,
     2.5  # place the result of 'cat /proc/uptime" on xenbus at memory/uptime
     2.6  XENBALLOON_SEND_UPTIME=1
     2.7 +
     2.8 +## Type: boolean
     2.9 +## Default: false
    2.10 +#
    2.11 +# If tmem is running, pages swapped to a swap disk may instead go
    2.12 +# into preswap.  These pages may become stale (i.e. no longer need
    2.13 +# to be saved because, e.g., the process owning them has gone away)
    2.14 +# and stale pages use precious precious domain-allocated memory.
    2.15 +# Periodically try to reduce preswap to squeeze out stale pages
    2.16 +XENBALLOON_PRESWAP_SHRINK=false
    2.17 +
    2.18 +## Type: string
    2.19 +## Default: "/proc/sys/vm/preswap"
    2.20 +## (change to /sys/proc/vm/preswap in later kernels)
    2.21 +#
    2.22 +# If tmem and preswap are running, reading this file gives the number of
    2.23 +# pages currently in preswap.  Writing it with N invokes the preswap_shrink
    2.24 +# routine to reduce preswap to N pages.
    2.25 +XENBALLOON_PRESWAP_SYSFILE=/proc/sys/vm/preswap
    2.26 +
    2.27 +## Type: integer (must be > 0)
    2.28 +## Default: 20
    2.29 +#
    2.30 +# If tmem and preswap are running, attempts to reduce number of pages
    2.31 +# currently in preswap.  For a value of n, 1/n of the pages will
    2.32 +# be attempted to be shrunk.
    2.33 +# If xenbus is enabled, may be overridden by {memory/preswaphysteresis}
    2.34 +XENBALLOON_PRESWAP_HYSTERESIS=20
    2.35 +
    2.36 +## Type: integer (must be > 0)
    2.37 +## Default: 10
    2.38 +#
    2.39 +# Number of ballooning intervals where preswap size remains at N pages
    2.40 +# before preswap shrinking is attempted.  Also if shrinking fails to
    2.41 +# shrink to the target, counter resets to this value.
    2.42 +# If xenbus is enabled, may be overridden by {memory/preswapinertia}
    2.43 +XENBALLOON_PRESWAP_INERTIA=10
    2.44 +
    2.45 +## Type: integer (0 or 1)
    2.46 +## Default: 1
    2.47 +#
    2.48 +# If xenbus is enabled, whether selfballooning or directed ballooning,
    2.49 +# place the size of preswap (in pages) on xenbus at memory/preswap
    2.50 +XENBALLOON_SEND_PRESWAP=1
    2.51 +
     3.1 --- a/tools/xenballoon/xenballoond	Tue Jun 16 11:15:48 2009 +0100
     3.2 +++ b/tools/xenballoon/xenballoond	Tue Jun 16 11:17:28 2009 +0100
     3.3 @@ -5,9 +5,10 @@
     3.4  # Written by: Dan Magenheimer <dan.magenheimer@oracle.com>
     3.5  #
     3.6  # xenballoond - In-guest engine for Xen memory ballooning
     3.7 -# Version: 080630
     3.8 +# Original version: 080630
     3.9 +# Updated 0906XX: add tmem preswap auto-shrinking
    3.10  #
    3.11 -# Two "policies" are implemented:
    3.12 +# Two self-ballooning "policies" are implemented:
    3.13  # - Selfballooning: Adjust memory periodically, with no (or little) input
    3.14  #     from domain0.  Target memory is determined solely by the
    3.15  #     Committed_AS line in /proc/meminfo, but parameters may adjust
    3.16 @@ -149,6 +150,75 @@ send_memory_stats() {
    3.17  	fi
    3.18  }
    3.19  
    3.20 +
    3.21 +curpreswappages() {
    3.22 +	pages=$(cat $XENBALLOON_PRESWAP_SYSFILE)
    3.23 +	RETVAL=$pages
    3.24 +	return  # value returned in RETVAL in pages
    3.25 +}
    3.26 +
    3.27 +preswaphysteresis() {
    3.28 +	RETVAL=$XENBALLOON_PRESWAP_HYSTERESIS
    3.29 +	if [ $xenstore_enabled = "true" ]; then
    3.30 +		if xenstore-exists memory/preswaphysteresis ; then
    3.31 +			RETVAL=`xenstore-read memory/preswaphysteresis`
    3.32 +		fi
    3.33 +	fi
    3.34 +	return
    3.35 +}
    3.36 +
    3.37 +preswapinertia() {
    3.38 +	RETVAL=$XENBALLOON_PRESWAP_INERTIA
    3.39 +	if [ $xenstore_enabled = "true" ]; then
    3.40 +		if xenstore-exists memory/preswapinertia ; then
    3.41 +			RETVAL=`xenstore-read memory/preswapinertia`
    3.42 +		fi
    3.43 +	fi
    3.44 +	return
    3.45 +}
    3.46 +
    3.47 +send_preswap_stats() {
    3.48 +	if [ ! $xenstore_enabled = "true" ]; then
    3.49 +		return
    3.50 +	fi
    3.51 +	curpreswappages
    3.52 +	preswap_pgs=$RETVAL
    3.53 +	if [ $XENBALLOON_SEND_PRESWAP ]; then
    3.54 +		xenstore-write memory/preswap "$preswap_pgs"
    3.55 +	fi
    3.56 +}
    3.57 +
    3.58 +shrink_preswap() {
    3.59 +	if [ "$XENBALLOON_PRESWAP_SHRINK" = "false" ]; then
    3.60 +		return
    3.61 +	fi
    3.62 +	if [ ! -f "$XENBALLOON_PRESWAP_SYSFILE" ]; then
    3.63 +		return
    3.64 +	fi
    3.65 +	curpreswappages
    3.66 +	preswaplast=$preswapnow
    3.67 +	preswapnow=$RETVAL
    3.68 +	if [ $preswapnow -eq 0 -o $preswapnow -ne $preswaplast ]; then
    3.69 +		preswapinertia
    3.70 +		preswapinertiacounter=$RETVAL
    3.71 +		return
    3.72 +	fi
    3.73 +	let "preswapinertiacounter=$preswapinertiacounter-1"
    3.74 +	if [ $preswapinertiacounter -ne 0 ]; then
    3.75 +		return
    3.76 +	fi
    3.77 +	preswaphysteresis
    3.78 +	preswaphys=$RETVAL
    3.79 +	if [ $preswaphys -eq 0 ]; then
    3.80 +		return
    3.81 +	fi
    3.82 +	let "tgtpreswappages=$(( $preswapnow - \
    3.83 +				( $preswapnow / $preswaphys ) ))"
    3.84 +	preswapinertia
    3.85 +	preswapinertiacounter=$RETVAL
    3.86 +	echo $tgtpreswappages > "$XENBALLOON_PRESWAP_SYSFILE"
    3.87 +}
    3.88 +
    3.89  if [ ! -f /proc/xen/balloon ]; then
    3.90  	echo "$0: no balloon driver installed"
    3.91  	exit 0
    3.92 @@ -165,6 +235,7 @@ else
    3.93  	echo "$0: missing /usr/bin/xenstore-* tools, disabling directed ballooning"
    3.94  	xenstore_enabled=false
    3.95  fi
    3.96 +preswapnow=0
    3.97  
    3.98  . /etc/sysconfig/xenballoon.conf
    3.99  
   3.100 @@ -194,7 +265,9 @@ do
   3.101  		fi
   3.102  		interval=$XENBALLOON_INTERVAL
   3.103  	fi
   3.104 +	shrink_preswap
   3.105  	send_memory_stats
   3.106 +	send_preswap_stats
   3.107  	if [ $xenstore_enabled = "true" ]; then
   3.108  		if xenstore-exists memory/interval ; then
   3.109  			interval=`xenstore-read memory/interval`
     4.1 --- a/tools/xenballoon/xenballoond.init	Tue Jun 16 11:15:48 2009 +0100
     4.2 +++ b/tools/xenballoon/xenballoond.init	Tue Jun 16 11:17:28 2009 +0100
     4.3 @@ -2,7 +2,7 @@
     4.4  #
     4.5  # xenballoond	Script to start and stop Xen ballooning daemon.
     4.6  #
     4.7 -# Copyright (C) 2008 Oracle Corporation and/or its affiliates.
     4.8 +# Copyright (C) 2009 Oracle Corporation and/or its affiliates.
     4.9  # All rights reserved.
    4.10  # Written by: Dan Magenheimer <dan.magenheimer@oracle.com>
    4.11  #