ia64/xen-unstable

changeset 14313:eedbddf55e51

[HVM] Save/restore: make sure all ioreqs are gone when a domain is saved
by pausing the domain and pulling them through.
Signed-off-by: Tim Deegan <Tim.Deegan@xensource.com>
author Tim Deegan <Tim.Deegan@xensource.com>
date Thu Mar 08 16:38:15 2007 +0000 (2007-03-08)
parents 0e08d98dfd57
children 560183e6452a
files tools/ioemu/hw/ide.c tools/ioemu/target-i386-dm/helper2.c tools/ioemu/vl.h tools/libxc/xc_hvm_restore.c tools/libxc/xc_hvm_save.c tools/libxc/xenguest.h tools/libxc/xg_private.c xen/arch/x86/hvm/hvm.c xen/include/public/hvm/hvm_op.h
line diff
     1.1 --- a/tools/ioemu/hw/ide.c	Thu Mar 08 16:20:01 2007 +0000
     1.2 +++ b/tools/ioemu/hw/ide.c	Thu Mar 08 16:38:15 2007 +0000
     1.3 @@ -396,17 +396,41 @@ typedef struct PCIIDEState {
     1.4  
     1.5  #ifdef DMA_MULTI_THREAD
     1.6  
     1.7 +static pthread_t ide_dma_thread;
     1.8  static int file_pipes[2];
     1.9  
    1.10  static void ide_dma_loop(BMDMAState *bm);
    1.11  static void dma_thread_loop(BMDMAState *bm);
    1.12  
    1.13 +extern int suspend_requested;
    1.14  static void *dma_thread_func(void* opaque)
    1.15  {
    1.16      BMDMAState* req;
    1.17 +    fd_set fds;
    1.18 +    int rv, nfds = file_pipes[0] + 1;
    1.19 +    struct timeval tm;
    1.20  
    1.21 -    while (read(file_pipes[0], &req, sizeof(req))) {
    1.22 -        dma_thread_loop(req);
    1.23 +    while (1) {
    1.24 +
    1.25 +        /* Wait at most a second for the pipe to become readable */
    1.26 +        FD_ZERO(&fds);
    1.27 +        FD_SET(file_pipes[0], &fds);
    1.28 +        tm.tv_sec = 1;
    1.29 +        tm.tv_usec = 0;
    1.30 +        rv = select(nfds, &fds, NULL, NULL, &tm);
    1.31 +        
    1.32 +        if (rv != 0) {
    1.33 +            if (read(file_pipes[0], &req, sizeof(req)) == 0)
    1.34 +                return NULL;
    1.35 +            dma_thread_loop(req);
    1.36 +        } else {
    1.37 +            if (suspend_requested)  {
    1.38 +                /* Need to tidy up the DMA thread so that we don't end up 
    1.39 +                 * finishing operations after the domain's ioreqs are 
    1.40 +                 * drained and its state saved */
    1.41 +                return NULL;
    1.42 +            }
    1.43 +        }
    1.44      }
    1.45  
    1.46      return NULL;
    1.47 @@ -414,24 +438,41 @@ static void *dma_thread_func(void* opaqu
    1.48  
    1.49  static void dma_create_thread(void)
    1.50  {
    1.51 -    pthread_t tid;
    1.52      int rt;
    1.53 +    pthread_attr_t a;
    1.54  
    1.55      if (pipe(file_pipes) != 0) {
    1.56          fprintf(stderr, "create pipe failed\n");
    1.57          exit(1);
    1.58      }
    1.59  
    1.60 -    if ((rt = pthread_create(&tid, NULL, dma_thread_func, NULL))) {
    1.61 +    if ((rt = pthread_attr_init(&a))
    1.62 +        || (rt = pthread_attr_setdetachstate(&a, PTHREAD_CREATE_JOINABLE))) {
    1.63 +        fprintf(stderr, "Oops, dma thread attr setup failed, errno=%d\n", rt);
    1.64 +        exit(1);
    1.65 +    }    
    1.66 +    
    1.67 +    if ((rt = pthread_create(&ide_dma_thread, &a, dma_thread_func, NULL))) {
    1.68          fprintf(stderr, "Oops, dma thread creation failed, errno=%d\n", rt);
    1.69          exit(1);
    1.70      }
    1.71 +}
    1.72  
    1.73 -    if ((rt = pthread_detach(tid))) {
    1.74 -        fprintf(stderr, "Oops, dma thread detachment failed, errno=%d\n", rt);
    1.75 -        exit(1);
    1.76 +void ide_stop_dma_thread(void)
    1.77 +{
    1.78 +    int rc;
    1.79 +    /* Make sure the IDE DMA thread is stopped */
    1.80 +    if ( (rc = pthread_join(ide_dma_thread, NULL)) != 0 )
    1.81 +    {
    1.82 +        fprintf(stderr, "Oops, error collecting IDE DMA thread (%s)\n", 
    1.83 +                strerror(rc));
    1.84      }
    1.85  }
    1.86 +
    1.87 +#else
    1.88 +void ide_stop_dma_thread(void)
    1.89 +{
    1.90 +}
    1.91  #endif /* DMA_MULTI_THREAD */
    1.92  
    1.93  #if defined(__ia64__)
     2.1 --- a/tools/ioemu/target-i386-dm/helper2.c	Thu Mar 08 16:20:01 2007 +0000
     2.2 +++ b/tools/ioemu/target-i386-dm/helper2.c	Thu Mar 08 16:38:15 2007 +0000
     2.3 @@ -577,7 +577,28 @@ int main_loop(void)
     2.4          destroy_hvm_domain();
     2.5      else {
     2.6          char qemu_file[20];
     2.7 +        ioreq_t *req;
     2.8 +        int rc;
     2.9 +
    2.10          sprintf(qemu_file, "/tmp/xen.qemu-dm.%d", domid);
    2.11 +        xc_domain_pause(xc_handle, domid);
    2.12 +
    2.13 +        /* Pull all outstanding ioreqs through the system */
    2.14 +        handle_buffered_io(env);
    2.15 +        main_loop_wait(1); /* For the select() on events */
    2.16 +        
    2.17 +        /* Stop the IDE thread */
    2.18 +        ide_stop_dma_thread();
    2.19 +
    2.20 +        /* Make sure that all outstanding IO responses are handled too */ 
    2.21 +        if ( xc_hvm_drain_io(xc_handle, domid) != 0 )
    2.22 +        {
    2.23 +            fprintf(stderr, "error clearing ioreq rings (%s)\n", 
    2.24 +                    strerror(errno));
    2.25 +            return -1;
    2.26 +        }
    2.27 +
    2.28 +        /* Save the device state */
    2.29          if (qemu_savevm(qemu_file) < 0)
    2.30              fprintf(stderr, "qemu save fail.\n");
    2.31      }
     3.1 --- a/tools/ioemu/vl.h	Thu Mar 08 16:20:01 2007 +0000
     3.2 +++ b/tools/ioemu/vl.h	Thu Mar 08 16:38:15 2007 +0000
     3.3 @@ -843,6 +843,7 @@ void pci_cmd646_ide_init(PCIBus *bus, Bl
     3.4  void pci_piix3_ide_init(PCIBus *bus, BlockDriverState **hd_table, int devfn);
     3.5  int pmac_ide_init (BlockDriverState **hd_table,
     3.6                     SetIRQFunc *set_irq, void *irq_opaque, int irq);
     3.7 +void ide_stop_dma_thread(void);
     3.8  
     3.9  /* cdrom.c */
    3.10  int cdrom_read_toc(int nb_sectors, uint8_t *buf, int msf, int start_track);
     4.1 --- a/tools/libxc/xc_hvm_restore.c	Thu Mar 08 16:20:01 2007 +0000
     4.2 +++ b/tools/libxc/xc_hvm_restore.c	Thu Mar 08 16:38:15 2007 +0000
     4.3 @@ -281,6 +281,14 @@ int xc_hvm_restore(int xc_handle, int io
     4.4      else
     4.5          shared_page_nr = (v_end >> PAGE_SHIFT) - 1;
     4.6  
     4.7 +    /* Paranoia: clean pages. */
     4.8 +    if ( xc_clear_domain_page(xc_handle, dom, shared_page_nr) ||
     4.9 +         xc_clear_domain_page(xc_handle, dom, shared_page_nr-1) ||
    4.10 +         xc_clear_domain_page(xc_handle, dom, shared_page_nr-2) ) {
    4.11 +        ERROR("error clearing comms frames!\n");
    4.12 +        goto out;
    4.13 +    }
    4.14 +
    4.15      xc_set_hvm_param(xc_handle, dom, HVM_PARAM_STORE_PFN, shared_page_nr-1);
    4.16      xc_set_hvm_param(xc_handle, dom, HVM_PARAM_BUFIOREQ_PFN, shared_page_nr-2);
    4.17      xc_set_hvm_param(xc_handle, dom, HVM_PARAM_IOREQ_PFN, shared_page_nr);
     5.1 --- a/tools/libxc/xc_hvm_save.c	Thu Mar 08 16:20:01 2007 +0000
     5.2 +++ b/tools/libxc/xc_hvm_save.c	Thu Mar 08 16:38:15 2007 +0000
     5.3 @@ -27,12 +27,14 @@
     5.4  #include <stdlib.h>
     5.5  #include <unistd.h>
     5.6  #include <sys/time.h>
     5.7 -#include <xen/hvm/e820.h>
     5.8  
     5.9  #include "xc_private.h"
    5.10  #include "xg_private.h"
    5.11  #include "xg_save_restore.h"
    5.12  
    5.13 +#include <xen/hvm/e820.h>
    5.14 +#include <xen/hvm/params.h>
    5.15 +
    5.16  /*
    5.17  ** Default values for important tuning parameters. Can override by passing
    5.18  ** non-zero replacement values to xc_hvm_save().
    5.19 @@ -49,12 +51,29 @@ static unsigned long max_mfn;
    5.20  /* virtual starting address of the hypervisor */
    5.21  static unsigned long hvirt_start;
    5.22  
    5.23 -/* #levels of page tables used by the currrent guest */
    5.24 +/* #levels of page tables used by the current guest */
    5.25  static unsigned int pt_levels;
    5.26  
    5.27  /* total number of pages used by the current guest */
    5.28  static unsigned long max_pfn;
    5.29  
    5.30 +int xc_hvm_drain_io(int handle, domid_t dom)
    5.31 +{
    5.32 +    DECLARE_HYPERCALL;
    5.33 +    xen_hvm_drain_io_t arg;
    5.34 +    int rc;
    5.35 +
    5.36 +    hypercall.op     = __HYPERVISOR_hvm_op;
    5.37 +    hypercall.arg[0] = HVMOP_drain_io;
    5.38 +    hypercall.arg[1] = (unsigned long)&arg;
    5.39 +    arg.domid = dom;
    5.40 +    if ( lock_pages(&arg, sizeof(arg)) != 0 )
    5.41 +        return -1;
    5.42 +    rc = do_xen_hypercall(handle, &hypercall);
    5.43 +    unlock_pages(&arg, sizeof(arg));
    5.44 +    return rc;
    5.45 +}
    5.46 +
    5.47  /*
    5.48  ** During (live) save/migrate, we maintain a number of bitmaps to track
    5.49  ** which pages we have to send, to fixup, and to skip.
     6.1 --- a/tools/libxc/xenguest.h	Thu Mar 08 16:20:01 2007 +0000
     6.2 +++ b/tools/libxc/xenguest.h	Thu Mar 08 16:38:15 2007 +0000
     6.3 @@ -155,6 +155,8 @@ int xc_set_hvm_param(
     6.4  int xc_get_hvm_param(
     6.5      int handle, domid_t dom, int param, unsigned long *value);
     6.6  
     6.7 +int xc_hvm_drain_io(int handle, domid_t dom);
     6.8 +
     6.9  /* PowerPC specific. */
    6.10  int xc_prose_build(int xc_handle,
    6.11                     uint32_t domid,
     7.1 --- a/tools/libxc/xg_private.c	Thu Mar 08 16:20:01 2007 +0000
     7.2 +++ b/tools/libxc/xg_private.c	Thu Mar 08 16:38:15 2007 +0000
     7.3 @@ -229,6 +229,11 @@ unsigned long csum_page(void *page)
     7.4      return -ENOSYS;
     7.5  }
     7.6  
     7.7 +__attribute__((weak)) int xc_hvm_drain_io(int handle, domid_t dom)
     7.8 +{
     7.9 +    return -ENOSYS;
    7.10 +}
    7.11 +
    7.12  /*
    7.13   * Local variables:
    7.14   * mode: C
     8.1 --- a/xen/arch/x86/hvm/hvm.c	Thu Mar 08 16:20:01 2007 +0000
     8.2 +++ b/xen/arch/x86/hvm/hvm.c	Thu Mar 08 16:38:15 2007 +0000
     8.3 @@ -146,6 +146,48 @@ void hvm_do_resume(struct vcpu *v)
     8.4      }
     8.5  }
     8.6  
     8.7 +/* Called from the tools when saving a domain to make sure the io
     8.8 + * request-response ring is entirely empty. */
     8.9 +static int hvmop_drain_io(
    8.10 +    XEN_GUEST_HANDLE(xen_hvm_drain_io_t) uop)
    8.11 +{
    8.12 +    struct xen_hvm_drain_io op;
    8.13 +    struct domain *d;
    8.14 +    struct vcpu *v;
    8.15 +    ioreq_t *p;
    8.16 +    int rc;
    8.17 +
    8.18 +    if ( copy_from_guest(&op, uop, 1) )
    8.19 +        return -EFAULT;
    8.20 +
    8.21 +    if ( !IS_PRIV(current->domain) )
    8.22 +        return -EPERM;
    8.23 +
    8.24 +    d = rcu_lock_domain_by_id(op.domid);
    8.25 +    if ( d == NULL )
    8.26 +        return -ESRCH;
    8.27 +
    8.28 +    rc = -EINVAL;
    8.29 +    /* Can't do this to yourself, or to a domain without an ioreq ring */
    8.30 +    if ( d == current->domain || !is_hvm_domain(d) || get_sp(d) == NULL )
    8.31 +        goto out;
    8.32 +
    8.33 +    rc = 0;
    8.34 +
    8.35 +    domain_pause(d);  /* It's not safe to do this to running vcpus */
    8.36 +    for_each_vcpu(d, v)
    8.37 +    {
    8.38 +        p = &get_vio(v->domain, v->vcpu_id)->vp_ioreq;
    8.39 +        if ( p->state == STATE_IORESP_READY )
    8.40 +            hvm_io_assist(v);
    8.41 +    }
    8.42 +    domain_unpause(d);
    8.43 +
    8.44 + out:
    8.45 +    rcu_unlock_domain(d);
    8.46 +    return rc;
    8.47 +}
    8.48 +
    8.49  int hvm_domain_initialise(struct domain *d)
    8.50  {
    8.51      int rc;
    8.52 @@ -916,6 +958,12 @@ long do_hvm_op(unsigned long op, XEN_GUE
    8.53              guest_handle_cast(arg, xen_hvm_set_pci_link_route_t));
    8.54          break;
    8.55  
    8.56 +    case HVMOP_drain_io:
    8.57 +        rc = hvmop_drain_io(
    8.58 +            guest_handle_cast(arg, xen_hvm_drain_io_t));
    8.59 +        break;
    8.60 +
    8.61 +
    8.62      default:
    8.63      {
    8.64          gdprintk(XENLOG_WARNING, "Bad HVM op %ld.\n", op);
     9.1 --- a/xen/include/public/hvm/hvm_op.h	Thu Mar 08 16:20:01 2007 +0000
     9.2 +++ b/xen/include/public/hvm/hvm_op.h	Thu Mar 08 16:38:15 2007 +0000
     9.3 @@ -70,4 +70,12 @@ struct xen_hvm_set_pci_link_route {
     9.4  typedef struct xen_hvm_set_pci_link_route xen_hvm_set_pci_link_route_t;
     9.5  DEFINE_XEN_GUEST_HANDLE(xen_hvm_set_pci_link_route_t);
     9.6  
     9.7 +/* Drain all outstanding qemu-dm IO responses from a domain's ioreq ring. */
     9.8 +#define HVMOP_drain_io            5
     9.9 +struct xen_hvm_drain_io {
    9.10 +    domid_t  domid;
    9.11 +};
    9.12 +typedef struct xen_hvm_drain_io xen_hvm_drain_io_t;
    9.13 +DEFINE_XEN_GUEST_HANDLE(xen_hvm_drain_io_t);
    9.14 +
    9.15  #endif /* __XEN_PUBLIC_HVM_HVM_OP_H__ */