ia64/xen-unstable

changeset 1411:fa3cfd1bdd96

bitkeeper revision 1.911.1.5 (40acee35tmC494sW3OcQEIN4ymTS1w)

Nasty temporary mechanism that allows dom0 to block after issuing a stop on another domain.
author iap10@labyrinth.cl.cam.ac.uk
date Thu May 20 17:43:17 2004 +0000 (2004-05-20)
parents 27b5376a7ec5
children 7843b8fd800f
files tools/xc/lib/xc_domain.c tools/xc/lib/xc_linux_save.c tools/xc/lib/xc_private.c tools/xc/lib/xc_private.h xen/common/dom0_ops.c xen/common/domain.c xen/common/schedule.c xen/include/hypervisor-ifs/dom0_ops.h
line diff
     1.1 --- a/tools/xc/lib/xc_domain.c	Thu May 20 11:01:03 2004 +0000
     1.2 +++ b/tools/xc/lib/xc_domain.c	Thu May 20 17:43:17 2004 +0000
     1.3 @@ -46,6 +46,7 @@ int xc_domain_stop(int xc_handle,
     1.4      dom0_op_t op;
     1.5      op.cmd = DOM0_STOPDOMAIN;
     1.6      op.u.stopdomain.domain = (domid_t)domid;
     1.7 +    op.u.stopdomain.sync = 0; // async
     1.8      return do_dom0_op(xc_handle, &op);
     1.9  }    
    1.10  
     2.1 --- a/tools/xc/lib/xc_linux_save.c	Thu May 20 11:01:03 2004 +0000
     2.2 +++ b/tools/xc/lib/xc_linux_save.c	Thu May 20 17:43:17 2004 +0000
     2.3 @@ -82,6 +82,67 @@ inline void set_bit ( int nr, volatile v
     2.4  	(1 << (nr % (sizeof(unsigned long)*8) ) );
     2.5  }
     2.6  
     2.7 +long long tv_to_us( struct timeval *new )
     2.8 +{
     2.9 +    return (new->tv_sec * 1000000) + new->tv_usec;
    2.10 +}
    2.11 +
    2.12 +long long tvdelta( struct timeval *new, struct timeval *old )
    2.13 +{
    2.14 +    return ((new->tv_sec - old->tv_sec)*1000000 ) + 
    2.15 +	(new->tv_usec - old->tv_usec);
    2.16 +}
    2.17 +
    2.18 +int track_cpu_usage_dom0( int xc_handle, int print )
    2.19 +{
    2.20 +    static struct timeval wall_last;
    2.21 +    static long long      cpu_last;
    2.22 +
    2.23 +    struct timeval        wall_now;
    2.24 +    long long             cpu_now, wall_delta, cpu_delta;
    2.25 +
    2.26 +    gettimeofday(&wall_now, NULL);
    2.27 +
    2.28 +    cpu_now = xc_domain_get_cpu_usage( xc_handle, 0 )/1000;
    2.29 +
    2.30 +    wall_delta = tvdelta(&wall_now,&wall_last)/1000;
    2.31 +    cpu_delta  = (cpu_now - cpu_last)/1000;
    2.32 +
    2.33 +    if(print)
    2.34 +	printf("Dom0  : wall delta %lldms, cpu delta %lldms    : %d%%\n",
    2.35 +	   wall_delta, cpu_delta, (cpu_delta*100)/wall_delta);
    2.36 +
    2.37 +    cpu_last  = cpu_now;
    2.38 +    wall_last = wall_now;	
    2.39 +
    2.40 +    return 0;
    2.41 +}
    2.42 +
    2.43 +int track_cpu_usage_target( int xc_handle, u64 domid, int print )
    2.44 +{
    2.45 +    static struct timeval wall_last;
    2.46 +    static long long      cpu_last;
    2.47 +
    2.48 +    struct timeval        wall_now;
    2.49 +    long long             cpu_now, wall_delta, cpu_delta;
    2.50 +
    2.51 +    gettimeofday(&wall_now, NULL);
    2.52 +
    2.53 +    cpu_now = xc_domain_get_cpu_usage( xc_handle, domid )/1000;
    2.54 +
    2.55 +    wall_delta = tvdelta(&wall_now,&wall_last)/1000;
    2.56 +    cpu_delta  = (cpu_now - cpu_last)/1000;
    2.57 +
    2.58 +    if(print)
    2.59 +	printf("Target: wall delta %lldms, cpu delta %lldms    : %d%%\n",
    2.60 +	   wall_delta, cpu_delta, (cpu_delta*100)/wall_delta);
    2.61 +
    2.62 +    cpu_last  = cpu_now;
    2.63 +    wall_last = wall_now;	
    2.64 +
    2.65 +    return 0;
    2.66 +}
    2.67 +
    2.68  
    2.69  int xc_linux_save(int xc_handle,
    2.70                    u64 domid, 
    2.71 @@ -95,10 +156,11 @@ int xc_linux_save(int xc_handle,
    2.72      int verbose = flags & XCFLAGS_VERBOSE;
    2.73      int live = flags & XCFLAGS_LIVE;
    2.74      int debug = flags & XCFLAGS_DEBUG;
    2.75 -    int sent_last_iter, sent_this_iter, skip_this_iter, max_iters;
    2.76 -
    2.77 -    /* Remember if we stopped the guest, so we can restart it on exit. */
    2.78 -    int we_stopped_it = 0;
    2.79 +    int sent_last_iter, sent_this_iter, skip_this_iter;
    2.80 +    
    2.81 +    /* Important tuning parameters */
    2.82 +    int max_iters  = 29; // limit us to 30 times round loop
    2.83 +    int max_factor = 3;  // never send more than 3x nr_pfns 
    2.84  
    2.85      /* The new domain's shared-info frame number. */
    2.86      unsigned long shared_info_frame;
    2.87 @@ -153,38 +215,15 @@ int xc_linux_save(int xc_handle,
    2.88      }
    2.89  
    2.90      /* Ensure that the domain exists, and that it is stopped. */
    2.91 -    for ( ; ; )
    2.92 -    {
    2.93 -        op.cmd = DOM0_GETDOMAININFO;
    2.94 -        op.u.getdomaininfo.domain = (domid_t)domid;
    2.95 -        op.u.getdomaininfo.ctxt = &ctxt;
    2.96 -        if ( (do_dom0_op(xc_handle, &op) < 0) || 
    2.97 -             ((u64)op.u.getdomaininfo.domain != domid) )
    2.98 -        {
    2.99 -            PERROR("Could not get info on domain");
   2.100 -            goto out;
   2.101 -        }
   2.102 -
   2.103 -        memcpy(name, op.u.getdomaininfo.name, sizeof(name));
   2.104 -        shared_info_frame = op.u.getdomaininfo.shared_info_frame;
   2.105  
   2.106 -        if ( op.u.getdomaininfo.state == DOMSTATE_STOPPED )
   2.107 -            break;
   2.108 -
   2.109 -        we_stopped_it = 1;
   2.110 +    if ( xc_domain_stop_sync( xc_handle, domid, &op, &ctxt ) )
   2.111 +    {
   2.112 +	PERROR("Could not sync stop domain");
   2.113 +	goto out;
   2.114 +    }
   2.115  
   2.116 -        op.cmd = DOM0_STOPDOMAIN;
   2.117 -        op.u.stopdomain.domain = (domid_t)domid;
   2.118 -        if ( do_dom0_op(xc_handle, &op) != 0 )
   2.119 -        {
   2.120 -            we_stopped_it = 0;
   2.121 -            PERROR("Stopping target domain failed");
   2.122 -            goto out;
   2.123 -        }
   2.124 -
   2.125 -        usleep(1000); // 1ms
   2.126 -	printf("Sleep for 1ms\n");
   2.127 -    }
   2.128 +    memcpy(name, op.u.getdomaininfo.name, sizeof(name));
   2.129 +    shared_info_frame = op.u.getdomaininfo.shared_info_frame;
   2.130  
   2.131      /* A cheesy test to see whether the domain contains valid state. */
   2.132      if ( ctxt.pt_base == 0 )
   2.133 @@ -292,7 +331,6 @@ int xc_linux_save(int xc_handle,
   2.134  
   2.135  	last_iter = 0;
   2.136  	sent_last_iter = 1<<20; // 4GB's worth of pages
   2.137 -	max_iters = 29; // limit us to 30 times round loop
   2.138      }
   2.139      else
   2.140  	last_iter = 1;
   2.141 @@ -384,8 +422,11 @@ int xc_linux_save(int xc_handle,
   2.142          goto out;
   2.143      }
   2.144  
   2.145 +    track_cpu_usage_dom0(xc_handle, 0);
   2.146 +    track_cpu_usage_target( xc_handle, domid, 0);
   2.147 +
   2.148      /* Now write out each data page, canonicalising page tables as we go... */
   2.149 -
   2.150 +    
   2.151      while(1)
   2.152      {
   2.153  	unsigned int prev_pc, batch, sent_this_iter;
   2.154 @@ -615,6 +656,10 @@ int xc_linux_save(int xc_handle,
   2.155  
   2.156  	verbose_printf("\b\b\b\b100%% (pages sent= %d, skipped= %d )\n", 
   2.157  		       sent_this_iter, skip_this_iter );
   2.158 +
   2.159 +	track_cpu_usage_dom0(xc_handle, 1);
   2.160 +	track_cpu_usage_target( xc_handle, domid, 1);
   2.161 +
   2.162  	
   2.163  	if ( last_iter )
   2.164  	{
   2.165 @@ -649,12 +694,12 @@ int xc_linux_save(int xc_handle,
   2.166  		 // ( sent_this_iter > (sent_last_iter * 0.95) ) ||		 
   2.167  		 (iter >= max_iters) || 
   2.168  		 (sent_this_iter+skip_this_iter < 10) || 
   2.169 -		 (total_sent > nr_pfns*2) )
   2.170 +		 (total_sent > nr_pfns*max_factor) )
   2.171  	    {
   2.172  		DPRINTF("Start last iteration\n");
   2.173  		last_iter = 1;
   2.174  
   2.175 -		xc_domain_stop_sync( xc_handle, domid );
   2.176 +		xc_domain_stop_sync( xc_handle, domid, &op, NULL );
   2.177  
   2.178  	    } 
   2.179  
     3.1 --- a/tools/xc/lib/xc_private.c	Thu May 20 11:01:03 2004 +0000
     3.2 +++ b/tools/xc/lib/xc_private.c	Thu May 20 17:43:17 2004 +0000
     3.3 @@ -376,51 +376,67 @@ int finish_mmu_updates(int xc_handle, mm
     3.4  
     3.5  /* this function is a hack until we get proper synchronous domain stop */
     3.6  
     3.7 -int xc_domain_stop_sync( int xc_handle, domid_t domid )
     3.8 +int xc_domain_stop_sync( int xc_handle, domid_t domid,
     3.9 +			 dom0_op_t *op, full_execution_context_t *ctxt)
    3.10  {
    3.11 -    dom0_op_t op;
    3.12      int i;
    3.13 -    
    3.14  
    3.15 -    op.cmd = DOM0_STOPDOMAIN;
    3.16 -    op.u.stopdomain.domain = (domid_t)domid;
    3.17 -    if ( do_dom0_op(xc_handle, &op) != 0 )
    3.18 -    {
    3.19 -	PERROR("Stopping target domain failed");
    3.20 -	goto out;
    3.21 -    }
    3.22 -
    3.23 -    usleep(100); // 100us
    3.24 +    printf("Sleep:");
    3.25  
    3.26      for(i=0;;i++)
    3.27 -    {
    3.28 -	if (i>0)
    3.29 -	    if (i==1) printf("Sleep.");
    3.30 -	    else printf(".");
    3.31 +    {    
    3.32  
    3.33 -        op.cmd = DOM0_GETDOMAININFO;
    3.34 -        op.u.getdomaininfo.domain = (domid_t)domid;
    3.35 -        op.u.getdomaininfo.ctxt = NULL;
    3.36 -        if ( (do_dom0_op(xc_handle, &op) < 0) || 
    3.37 -             ((u64)op.u.getdomaininfo.domain != domid) )
    3.38 +	op->cmd = DOM0_STOPDOMAIN;
    3.39 +	op->u.stopdomain.domain = (domid_t)domid;
    3.40 +	op->u.stopdomain.sync = 1;
    3.41 +	do_dom0_op(xc_handle, op);
    3.42 +	/* can't trust return code due to sync stop hack :-(( */
    3.43 +
    3.44 +       
    3.45 +        op->cmd = DOM0_GETDOMAININFO;
    3.46 +        op->u.getdomaininfo.domain = (domid_t)domid;
    3.47 +        op->u.getdomaininfo.ctxt = ctxt;
    3.48 +        if ( (do_dom0_op(xc_handle, op) < 0) || 
    3.49 +             ((u64)op->u.getdomaininfo.domain != domid) )
    3.50          {
    3.51              PERROR("Could not get info on domain");
    3.52              goto out;
    3.53          }
    3.54  
    3.55 -        if ( op.u.getdomaininfo.state == DOMSTATE_STOPPED )
    3.56 +        if ( op->u.getdomaininfo.state == DOMSTATE_STOPPED )
    3.57  	{
    3.58  	    printf("Domain %lld stopped\n",domid);
    3.59              return 0;
    3.60  	}
    3.61  	
    3.62 -	usleep(1000);
    3.63 +	printf(".");
    3.64 +
    3.65 +	//usleep(1000);
    3.66      }
    3.67  
    3.68 +    printf("\n");
    3.69 +
    3.70  out:
    3.71      return -1;    
    3.72  }
    3.73  
    3.74 +long long  xc_domain_get_cpu_usage( int xc_handle, domid_t domid )
    3.75 +{
    3.76 +    dom0_op_t op;
    3.77 +
    3.78 +    op.cmd = DOM0_GETDOMAININFO;
    3.79 +    op.u.getdomaininfo.domain = (domid_t)domid;
    3.80 +    op.u.getdomaininfo.ctxt = NULL;
    3.81 +    if ( (do_dom0_op(xc_handle, &op) < 0) || 
    3.82 +	 ((u64)op.u.getdomaininfo.domain != domid) )
    3.83 +    {
    3.84 +	PERROR("Could not get info on domain");
    3.85 +	return -1;
    3.86 +    }
    3.87 +    return op.u.getdomaininfo.cpu_time;
    3.88 +}
    3.89 +
    3.90 +
    3.91  /**********************************************************************/
    3.92  
    3.93  // this is shared between save and restore, and may be useful.
     4.1 --- a/tools/xc/lib/xc_private.h	Thu May 20 11:01:03 2004 +0000
     4.2 +++ b/tools/xc/lib/xc_private.h	Thu May 20 17:43:17 2004 +0000
     4.3 @@ -248,6 +248,9 @@ void * mfn_mapper_queue_entry(mfn_mapper
     4.4  
     4.5  /*********************/
     4.6  
     4.7 -int xc_domain_stop_sync( int xc_handle, domid_t dom );
     4.8 +int xc_domain_stop_sync( int xc_handle, domid_t dom, 
     4.9 +			 dom0_op_t *op, full_execution_context_t *ctxt );
    4.10 +
    4.11 +long long  xc_domain_get_cpu_usage( int xc_handle, domid_t domid );
    4.12  
    4.13  #endif /* __XC_PRIVATE_H__ */
     5.1 --- a/xen/common/dom0_ops.c	Thu May 20 11:01:03 2004 +0000
     5.2 +++ b/xen/common/dom0_ops.c	Thu May 20 17:43:17 2004 +0000
     5.3 @@ -102,6 +102,20 @@ long do_dom0_op(dom0_op_t *u_dom0_op)
     5.4      case DOM0_STOPDOMAIN:
     5.5      {
     5.6          ret = stop_other_domain(op->u.stopdomain.domain);
     5.7 +	
     5.8 +	/* This is grim, but helps for live migrate. It's also unsafe
     5.9 +	   in the strict sense as we're not explicitly setting a
    5.10 +	   timeout, but dom0 is bound to have other timers going off to
    5.11 +	   wake us back up. 
    5.12 +	   We go to sleep so that the other domain can stop quicker, hence
    5.13 +	   we have less total down time in a migrate.
    5.14 +	 */
    5.15 +	if( ret == 0 && op->u.stopdomain.sync == 1 )
    5.16 +	{
    5.17 +	    extern long do_block( void );
    5.18 +	    printk("T\n");
    5.19 +	    do_block(); // Yuk...
    5.20 +	}
    5.21      }
    5.22      break;
    5.23  
     6.1 --- a/xen/common/domain.c	Thu May 20 11:01:03 2004 +0000
     6.2 +++ b/xen/common/domain.c	Thu May 20 17:43:17 2004 +0000
     6.3 @@ -266,6 +266,16 @@ void stop_domain(void)
     6.4      unlazy_fpu(current);
     6.5      wmb(); /* All CPUs must see saved info in state TASK_STOPPED. */
     6.6      set_current_state(TASK_STOPPED);
     6.7 +
     6.8 +    /* OK, this is grim, but helps speed up live migrate. When a domain stops,
     6.9 +       kick Dom0 */
    6.10 +    {
    6.11 +	struct task_struct *p;
    6.12 +	printk("S\n");
    6.13 +	guest_schedule_to_run( p = find_domain_by_id(0ULL) );
    6.14 +	put_task_struct(p);
    6.15 +    }
    6.16 +
    6.17      __enter_scheduler();
    6.18  }
    6.19  
     7.1 --- a/xen/common/schedule.c	Thu May 20 11:01:03 2004 +0000
     7.2 +++ b/xen/common/schedule.c	Thu May 20 17:43:17 2004 +0000
     7.3 @@ -27,6 +27,9 @@
     7.4  #include <xen/perfc.h>
     7.5  #include <xen/sched-if.h>
     7.6  #include <hypervisor-ifs/sched_ctl.h>
     7.7 +
     7.8 +#undef  TRACE_BUFFER
     7.9 +
    7.10  #include <xen/trace.h>
    7.11  
    7.12  /*#define WAKEUP_HISTO*/
    7.13 @@ -216,7 +219,7 @@ void wake_up(struct task_struct *p)
    7.14  /* 
    7.15   * Block the currently-executing domain until a pertinent event occurs.
    7.16   */
    7.17 -static long do_block(void)
    7.18 +long do_block(void)
    7.19  {
    7.20      ASSERT(current->domain != IDLE_DOMAIN_ID);
    7.21      current->shared_info->vcpu_data[0].evtchn_upcall_mask = 0;
     8.1 --- a/xen/include/hypervisor-ifs/dom0_ops.h	Thu May 20 11:01:03 2004 +0000
     8.2 +++ b/xen/include/hypervisor-ifs/dom0_ops.h	Thu May 20 17:43:17 2004 +0000
     8.3 @@ -74,6 +74,9 @@ typedef struct dom0_stopdomain_st
     8.4  {
     8.5      /* IN parameters. */
     8.6      domid_t domain;
     8.7 +    /* hack to indicate that you want to wait for other domain -- replace
     8.8 +       with proper sychronous stop soon! */
     8.9 +    int     sync;  
    8.10  } dom0_stopdomain_t;
    8.11  
    8.12  #define DOM0_GETDOMAININFO    12