ia64/xen-unstable

changeset 5988:76794dad0aaf

Fix checkpointing - add save/restore support to the xenbus driver.
- Add xenbus driver suspend/resume functions
- Change xenbus irq to not be shared
- Translate store mfn in suspend record to pfn and back
- Make tools re-introduce the domain to the store
Signed-off-by: Rusty Russel <rusty@rustcorp.com.au>
Signed-off-by: Christian Limpach <Christian.Limpach@cl.cam.ac.uk>
author cl349@firebug.cl.cam.ac.uk
date Tue Aug 02 17:12:36 2005 +0000 (2005-08-02)
parents a75db157d12f
children 361d31028129
files linux-2.6-xen-sparse/arch/xen/kernel/reboot.c linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.c linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.h linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c linux-2.6-xen-sparse/include/asm-xen/xenbus.h linux-2.6-xen-sparse/kernel/irq/manage.c tools/libxc/xc.h tools/libxc/xc_linux_restore.c tools/libxc/xc_linux_save.c tools/python/xen/xend/XendCheckpoint.py tools/xcutils/xc_restore.c
line diff
     1.1 --- a/linux-2.6-xen-sparse/arch/xen/kernel/reboot.c	Tue Aug 02 09:26:30 2005 +0000
     1.2 +++ b/linux-2.6-xen-sparse/arch/xen/kernel/reboot.c	Tue Aug 02 17:12:36 2005 +0000
     1.3 @@ -16,6 +16,7 @@ static int errno;
     1.4  #include <asm-xen/xen-public/dom0_ops.h>
     1.5  #include <asm-xen/linux-public/suspend.h>
     1.6  #include <asm-xen/queues.h>
     1.7 +#include <asm-xen/xenbus.h>
     1.8  
     1.9  void machine_restart(char * __unused)
    1.10  {
    1.11 @@ -114,6 +115,8 @@ static void __do_suspend(void)
    1.12  
    1.13      time_suspend();
    1.14  
    1.15 +    xenbus_suspend();
    1.16 +
    1.17      ctrl_if_suspend();
    1.18  
    1.19      irq_suspend();
    1.20 @@ -153,6 +156,8 @@ static void __do_suspend(void)
    1.21  
    1.22      ctrl_if_resume();
    1.23  
    1.24 +    xenbus_resume();
    1.25 +
    1.26      time_resume();
    1.27  
    1.28      blkdev_resume();
     2.1 --- a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.c	Tue Aug 02 09:26:30 2005 +0000
     2.2 +++ b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.c	Tue Aug 02 17:12:36 2005 +0000
     2.3 @@ -36,6 +36,8 @@
     2.4  #include <linux/err.h>
     2.5  #include "xenbus_comms.h"
     2.6  
     2.7 +static unsigned int xb_irq;
     2.8 +
     2.9  #define RINGBUF_DATASIZE ((PAGE_SIZE / 2) - sizeof(struct ringbuf_head))
    2.10  struct ringbuf_head
    2.11  {
    2.12 @@ -202,14 +204,17 @@ int xb_read(void *data, unsigned len)
    2.13  	return 0;
    2.14  }
    2.15  
    2.16 -/* Set up interrpt handler off store event channel. */
    2.17 +/* Set up interrupt handler off store event channel. */
    2.18  int xb_init_comms(void)
    2.19  {
    2.20 -	int err, irq;
    2.21 +	int err;
    2.22  
    2.23 -	irq = bind_evtchn_to_irq(xen_start_info.store_evtchn);
    2.24 +	if (!xen_start_info.store_evtchn)
    2.25 +		return 0;
    2.26  
    2.27 -	err = request_irq(irq, wake_waiting, SA_SHIRQ, "xenbus", &xb_waitq);
    2.28 +	xb_irq = bind_evtchn_to_irq(xen_start_info.store_evtchn);
    2.29 +
    2.30 +	err = request_irq(xb_irq, wake_waiting, 0, "xenbus", &xb_waitq);
    2.31  	if (err) {
    2.32  		printk(KERN_ERR "XENBUS request irq failed %i\n", err);
    2.33  		unbind_evtchn_from_irq(xen_start_info.store_evtchn);
    2.34 @@ -222,3 +227,13 @@ int xb_init_comms(void)
    2.35  
    2.36  	return 0;
    2.37  }
    2.38 +
    2.39 +void xb_suspend_comms(void)
    2.40 +{
    2.41 +
    2.42 +	if (!xen_start_info.store_evtchn)
    2.43 +		return;
    2.44 +
    2.45 +	free_irq(xb_irq, &xb_waitq);
    2.46 +	unbind_evtchn_from_irq(xen_start_info.store_evtchn);
    2.47 +}
     3.1 --- a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.h	Tue Aug 02 09:26:30 2005 +0000
     3.2 +++ b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.h	Tue Aug 02 17:12:36 2005 +0000
     3.3 @@ -3,6 +3,7 @@
     3.4  #define _XENBUS_COMMS_H
     3.5  int xs_init(void);
     3.6  int xb_init_comms(void);
     3.7 +void xb_suspend_comms(void);
     3.8  
     3.9  /* Low level routines. */
    3.10  int xb_write(const void *data, unsigned len);
     4.1 --- a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c	Tue Aug 02 09:26:30 2005 +0000
     4.2 +++ b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c	Tue Aug 02 17:12:36 2005 +0000
     4.3 @@ -295,6 +295,19 @@ static struct xenbus_watch dev_watch = {
     4.4  	.callback = dev_changed,
     4.5  };
     4.6  
     4.7 +void xenbus_suspend(void)
     4.8 +{
     4.9 +	/* We keep lock, so no comms can happen as page moves. */
    4.10 +	down(&xenbus_lock);
    4.11 +	xb_suspend_comms();
    4.12 +}
    4.13 +
    4.14 +void xenbus_resume(void)
    4.15 +{
    4.16 +	xb_init_comms();
    4.17 +	up(&xenbus_lock);
    4.18 +}
    4.19 +
    4.20  /* called from a thread in privcmd/privcmd.c */
    4.21  int do_xenbus_probe(void *unused)
    4.22  {
     5.1 --- a/linux-2.6-xen-sparse/include/asm-xen/xenbus.h	Tue Aug 02 09:26:30 2005 +0000
     5.2 +++ b/linux-2.6-xen-sparse/include/asm-xen/xenbus.h	Tue Aug 02 17:12:36 2005 +0000
     5.3 @@ -115,4 +115,8 @@ struct xenbus_watch
     5.4  int register_xenbus_watch(struct xenbus_watch *watch);
     5.5  void unregister_xenbus_watch(struct xenbus_watch *watch);
     5.6  
     5.7 +/* Called from xen core code. */
     5.8 +void xenbus_suspend(void);
     5.9 +void xenbus_resume(void);
    5.10 +
    5.11  #endif /* _ASM_XEN_XENBUS_H */
     7.1 --- a/tools/libxc/xc.h	Tue Aug 02 09:26:30 2005 +0000
     7.2 +++ b/tools/libxc/xc.h	Tue Aug 02 17:12:36 2005 +0000
     7.3 @@ -279,9 +279,12 @@ int xc_linux_save(int xc_handle, int fd,
     7.4   * @parm fd the file descriptor to restore a domain from
     7.5   * @parm dom the id of the domain
     7.6   * @parm nr_pfns the number of pages
     7.7 + * @parm store_evtchn the store event channel for this domain to use
     7.8 + * @parm store_mfn returned with the mfn of the store page
     7.9   * @return 0 on success, -1 on failure
    7.10   */
    7.11 -int xc_linux_restore(int xc_handle, int io_fd, u32 dom, unsigned long nr_pfns);
    7.12 +int xc_linux_restore(int xc_handle, int io_fd, u32 dom, unsigned long nr_pfns,
    7.13 +		     unsigned int store_evtchn, unsigned long *store_mfn);
    7.14  
    7.15  int xc_linux_build(int xc_handle,
    7.16                     u32 domid,
     8.1 --- a/tools/libxc/xc_linux_restore.c	Tue Aug 02 09:26:30 2005 +0000
     8.2 +++ b/tools/libxc/xc_linux_restore.c	Tue Aug 02 17:12:36 2005 +0000
     8.3 @@ -48,7 +48,8 @@ read_exact(int fd, void *buf, size_t cou
     8.4      return r;
     8.5  }
     8.6  
     8.7 -int xc_linux_restore(int xc_handle, int io_fd, u32 dom, unsigned long nr_pfns)
     8.8 +int xc_linux_restore(int xc_handle, int io_fd, u32 dom, unsigned long nr_pfns,
     8.9 +		     unsigned int store_evtchn, unsigned long *store_mfn)
    8.10  {
    8.11      dom0_op_t op;
    8.12      int rc = 1, i, n, k;
    8.13 @@ -464,10 +465,13 @@ int xc_linux_restore(int xc_handle, int 
    8.14      }
    8.15      ctxt.user_regs.esi = mfn = pfn_to_mfn_table[pfn];
    8.16      p_srec = xc_map_foreign_range(
    8.17 -        xc_handle, dom, PAGE_SIZE, PROT_WRITE, mfn);
    8.18 +        xc_handle, dom, PAGE_SIZE, PROT_READ | PROT_WRITE, mfn);
    8.19      p_srec->resume_info.nr_pages    = nr_pfns;
    8.20      p_srec->resume_info.shared_info = shared_info_frame << PAGE_SHIFT;
    8.21      p_srec->resume_info.flags       = 0;
    8.22 +    *store_mfn = p_srec->resume_info.store_mfn   =
    8.23 +	pfn_to_mfn_table[p_srec->resume_info.store_mfn];
    8.24 +    p_srec->resume_info.store_evtchn = store_evtchn;
    8.25      munmap(p_srec, PAGE_SIZE);
    8.26  
    8.27      /* Uncanonicalise each GDT frame number. */
     9.1 --- a/tools/libxc/xc_linux_save.c	Tue Aug 02 09:26:30 2005 +0000
     9.2 +++ b/tools/libxc/xc_linux_save.c	Tue Aug 02 17:12:36 2005 +0000
     9.3 @@ -20,7 +20,7 @@
     9.4  #define DEBUG 0
     9.5  
     9.6  #if 1
     9.7 -#define ERR(_f, _a...) fprintf ( stderr, _f , ## _a )
     9.8 +#define ERR(_f, _a...) do { fprintf(stderr, _f , ## _a); fflush(stderr); } while (0)
     9.9  #else
    9.10  #define ERR(_f, _a...) ((void)0)
    9.11  #endif
    9.12 @@ -643,6 +643,22 @@ int xc_linux_save(int xc_handle, int io_
    9.13          goto out;
    9.14      }
    9.15  
    9.16 +    /* Map the suspend-record MFN to pin it. The page must be owned by 
    9.17 +       dom for this to succeed. */
    9.18 +    p_srec = xc_map_foreign_range(xc_handle, dom,
    9.19 +                                   sizeof(*p_srec), PROT_READ | PROT_WRITE, 
    9.20 +                                   ctxt.user_regs.esi);
    9.21 +    if (!p_srec){
    9.22 +        ERR("Couldn't map suspend record");
    9.23 +        goto out;
    9.24 +    }
    9.25 +
    9.26 +    /* Canonicalize store mfn. */
    9.27 +    if ( !translate_mfn_to_pfn(&p_srec->resume_info.store_mfn) ) {
    9.28 +	ERR("Store frame is not in range of pseudophys map");
    9.29 +	goto out;
    9.30 +    }
    9.31 +
    9.32      print_stats( xc_handle, dom, 0, &stats, 0 );
    9.33  
    9.34      /* Now write out each data page, canonicalising page tables as we go... */
    9.35 @@ -983,16 +999,6 @@ int xc_linux_save(int xc_handle, int io_
    9.36  	}
    9.37      }
    9.38  
    9.39 -    /* Map the suspend-record MFN to pin it. The page must be owned by 
    9.40 -       dom for this to succeed. */
    9.41 -    p_srec = xc_map_foreign_range(xc_handle, dom,
    9.42 -                                   sizeof(*p_srec), PROT_READ, 
    9.43 -                                   ctxt.user_regs.esi);
    9.44 -    if (!p_srec){
    9.45 -        ERR("Couldn't map suspend record");
    9.46 -        goto out;
    9.47 -    }
    9.48 -
    9.49      if (nr_pfns != p_srec->nr_pfns )
    9.50      {
    9.51  	ERR("Suspend record nr_pfns unexpected (%ld != %ld)",
    10.1 --- a/tools/python/xen/xend/XendCheckpoint.py	Tue Aug 02 09:26:30 2005 +0000
    10.2 +++ b/tools/python/xen/xend/XendCheckpoint.py	Tue Aug 02 17:12:36 2005 +0000
    10.3 @@ -6,6 +6,7 @@
    10.4  
    10.5  import errno
    10.6  import os
    10.7 +import re
    10.8  import select
    10.9  import sxp
   10.10  from string import join
   10.11 @@ -64,6 +65,13 @@ def save(xd, fd, dominfo):
   10.12                  if l.rstrip() == "suspend":
   10.13                      log.info("suspending %d" % dominfo.id)
   10.14                      xd.domain_shutdown(dominfo.id, reason='suspend')
   10.15 +                    if dominfo.store_channel:
   10.16 +                        try:
   10.17 +                            dominfo.db.releaseDomain(dominfo.id)
   10.18 +                        except Exception, ex:
   10.19 +                            log.warning("error in domain release on xenstore: %s",
   10.20 +                                        ex)
   10.21 +                            pass
   10.22                      dominfo.state_wait("suspended")
   10.23                      log.info("suspend %d done" % dominfo.id)
   10.24                      child.tochild.write("done\n")
   10.25 @@ -76,6 +84,11 @@ def save(xd, fd, dominfo):
   10.26      if child.wait() != 0:
   10.27          raise XendError("xc_save failed: %s" % lasterr)
   10.28  
   10.29 +    if dominfo.store_channel:
   10.30 +        dominfo.store_channel.close()
   10.31 +        dominfo.db['store_channel'].delete()
   10.32 +        dominfo.db.saveDB(save=True)
   10.33 +        dominfo.store_channel = None
   10.34      xd.domain_destroy(dominfo.id)
   10.35      return None
   10.36  
   10.37 @@ -107,8 +120,13 @@ def restore(xd, fd):
   10.38          raise XendError(
   10.39              "not a valid guest state file: pfn count out of range")
   10.40  
   10.41 +    if dominfo.store_channel:
   10.42 +        evtchn = dominfo.store_channel.port2
   10.43 +    else:
   10.44 +        evtchn = 0
   10.45 +
   10.46      cmd = [PATH_XC_RESTORE, str(xc.handle()), str(fd),
   10.47 -           str(dominfo.id), str(nr_pfns)]
   10.48 +           str(dominfo.id), str(nr_pfns), str(evtchn)]
   10.49      log.info("[xc_restore] " + join(cmd))
   10.50      child = xPopen3(cmd, True, -1, [fd, xc.handle()])
   10.51      child.tochild.close()
   10.52 @@ -128,7 +146,21 @@ def restore(xd, fd):
   10.53                  lasterr = l.rstrip()
   10.54              if fd == child.fromchild.fileno():
   10.55                  l = child.fromchild.readline()
   10.56 -                log.info(l.rstrip())
   10.57 +                while l:
   10.58 +                    m = re.match(r"^(store-mfn) (\d+)\n$", l)
   10.59 +                    if m:
   10.60 +                        if dominfo.store_channel:
   10.61 +                            dominfo.store_mfn = int(m.group(2))
   10.62 +                            if dominfo.store_mfn >= 0:
   10.63 +                                dominfo.db.introduceDomain(dominfo.id,
   10.64 +                                                           dominfo.store_mfn,
   10.65 +                                                           dominfo.store_channel)
   10.66 +                            dominfo.exportToDB(save=True, sync=True))
   10.67 +                    log.info(l.rstrip())
   10.68 +                    try:
   10.69 +                        l = child.fromchild.readline()
   10.70 +                    except:
   10.71 +                        l = None
   10.72          if filter(lambda (fd, event): event & select.POLLHUP, r):
   10.73              break
   10.74  
    11.1 --- a/tools/xcutils/xc_restore.c	Tue Aug 02 09:26:30 2005 +0000
    11.2 +++ b/tools/xcutils/xc_restore.c	Tue Aug 02 17:12:36 2005 +0000
    11.3 @@ -16,15 +16,23 @@
    11.4  int
    11.5  main(int argc, char **argv)
    11.6  {
    11.7 -    unsigned int xc_fd, io_fd, domid, nr_pfns;
    11.8 +    unsigned int xc_fd, io_fd, domid, nr_pfns, evtchn;
    11.9 +    int ret;
   11.10 +    unsigned long mfn;
   11.11  
   11.12 -    if (argc != 5)
   11.13 -	errx(1, "usage: %s xcfd iofd domid nr_pfns", argv[0]);
   11.14 +    if (argc != 6)
   11.15 +	errx(1, "usage: %s xcfd iofd domid nr_pfns evtchn", argv[0]);
   11.16  
   11.17      xc_fd = atoi(argv[1]);
   11.18      io_fd = atoi(argv[2]);
   11.19      domid = atoi(argv[3]);
   11.20      nr_pfns = atoi(argv[4]);
   11.21 +    evtchn = atoi(argv[5]);
   11.22  
   11.23 -    return xc_linux_restore(xc_fd, io_fd, domid, nr_pfns);
   11.24 +    ret = xc_linux_restore(xc_fd, io_fd, domid, nr_pfns, evtchn, &mfn);
   11.25 +    if (ret == 0) {
   11.26 +	printf("store-mfn %li\n", mfn);
   11.27 +	fflush(stdout);
   11.28 +    }
   11.29 +    return ret;
   11.30  }