ia64/xen-unstable

changeset 4796:4ce7020ef1ba

bitkeeper revision 1.1389.5.33 (427dd05a9r9PEjJTAexFhIB1u7f7Sw)

[PATCH] [PATCH] AP boot support

# This is a BitKeeper generated diff -Nru style patch.
#
# ChangeSet
# 2005/05/07 21:32:49-07:00 kmacy@curly.lab.netapp.com
# get AP booting working
# currently crashing in init_secondary - will fix after adding SMP debug support
# Signed-off-by: Kip Macy <kmacy@fsmware.com>
#
# freebsd-5.3-xen-sparse/i386-xen/include/xenfunc.h
# 2005/05/07 21:32:47-07:00 kmacy@curly.lab.netapp.com +4 -0
# add declaration for per-cpu clock init
#
# freebsd-5.3-xen-sparse/i386-xen/include/pmap.h
# 2005/05/07 21:32:47-07:00 kmacy@curly.lab.netapp.com +1 -0
# make pmap_lazyfix_action global
#
# freebsd-5.3-xen-sparse/i386-xen/include/pcpu.h
# 2005/05/07 21:32:47-07:00 kmacy@curly.lab.netapp.com +6 -1
# add IPI fields
#
# freebsd-5.3-xen-sparse/i386-xen/include/hypervisor.h
# 2005/05/07 21:32:47-07:00 kmacy@curly.lab.netapp.com +16 -0
# add boot_vcpu call
#
# freebsd-5.3-xen-sparse/i386-xen/i386-xen/xen_machdep.c
# 2005/05/07 21:32:47-07:00 kmacy@curly.lab.netapp.com +0 -2
# make PANIC_IF declaration global
#
# freebsd-5.3-xen-sparse/i386-xen/i386-xen/pmap.c
# 2005/05/07 21:32:47-07:00 kmacy@curly.lab.netapp.com +0 -1
# make pmap_lazyfix_action global
#
# freebsd-5.3-xen-sparse/i386-xen/i386-xen/mp_machdep.c
# 2005/05/07 21:32:47-07:00 kmacy@curly.lab.netapp.com +229 -55
# add support for booting APs
#
# freebsd-5.3-xen-sparse/i386-xen/i386-xen/machdep.c
# 2005/05/07 21:32:47-07:00 kmacy@curly.lab.netapp.com +78 -46
# do per-cpu GDT initialization up-front
#
# freebsd-5.3-xen-sparse/i386-xen/i386-xen/evtchn.c
# 2005/05/07 21:32:47-07:00 kmacy@curly.lab.netapp.com +15 -8
# special case AST IPI
#
# freebsd-5.3-xen-sparse/i386-xen/i386-xen/clock.c
# 2005/05/07 21:32:46-07:00 kmacy@curly.lab.netapp.com +60 -14
# add per-cpu clock support
#
author kmacy@netapp.com[kaf24]
date Sun May 08 08:39:54 2005 +0000 (2005-05-08)
parents c44b77da583e
children c0463989fca8
files freebsd-5.3-xen-sparse/i386-xen/i386-xen/clock.c freebsd-5.3-xen-sparse/i386-xen/i386-xen/evtchn.c freebsd-5.3-xen-sparse/i386-xen/i386-xen/machdep.c freebsd-5.3-xen-sparse/i386-xen/i386-xen/mp_machdep.c freebsd-5.3-xen-sparse/i386-xen/i386-xen/pmap.c freebsd-5.3-xen-sparse/i386-xen/i386-xen/xen_machdep.c freebsd-5.3-xen-sparse/i386-xen/include/hypervisor.h freebsd-5.3-xen-sparse/i386-xen/include/pcpu.h freebsd-5.3-xen-sparse/i386-xen/include/pmap.h freebsd-5.3-xen-sparse/i386-xen/include/xenfunc.h
line diff
     1.1 --- a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/clock.c	Sun May 08 08:39:44 2005 +0000
     1.2 +++ b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/clock.c	Sun May 08 08:39:54 2005 +0000
     1.3 @@ -87,6 +87,12 @@
     1.4  
     1.5  /* XEN specific defines */
     1.6  #include <machine/xen_intr.h>
     1.7 +#include <vm/vm.h>   /* needed by machine/pmap.h */
     1.8 +#include <vm/pmap.h> /* needed by machine/pmap.h */
     1.9 +#include <machine/pmap.h> /* needed by xen-os.h */
    1.10 +#include <machine/hypervisor-ifs.h>
    1.11 +#include <machine/xen-os.h> /* needed by xenfunc.h */
    1.12 +#include <machine/xenfunc.h>
    1.13  
    1.14  /*
    1.15   * 32-bit time_t's can't reach leap years before 1904 or after 2036, so we
    1.16 @@ -129,7 +135,15 @@ static uint64_t shadow_system_time;
    1.17  static uint32_t shadow_time_version;
    1.18  static struct timeval shadow_tv;
    1.19  
    1.20 +#define DEFINE_PER_CPU(type, name) \
    1.21 +    __typeof__(type) per_cpu__##name
    1.22 +
    1.23 +#define per_cpu(var, cpu)           (*((void)cpu, &per_cpu__##var))
    1.24 +
    1.25 +
    1.26  static uint64_t processed_system_time;/* System time (ns) at last processing. */
    1.27 +static DEFINE_PER_CPU(uint64_t, processed_system_time);
    1.28 +
    1.29  
    1.30  #define NS_PER_TICK (1000000000ULL/hz)
    1.31  
    1.32 @@ -202,18 +216,19 @@ static struct timecounter xen_timecounte
    1.33  static void 
    1.34  clkintr(struct clockframe *frame)
    1.35  {
    1.36 -    int64_t delta;
    1.37 +    int64_t cpu_delta, delta;
    1.38 +    int cpu = smp_processor_id();
    1.39      long ticks = 0;
    1.40  
    1.41 -
    1.42      do {
    1.43      	__get_time_values_from_xen();
    1.44 -    	delta = (int64_t)(shadow_system_time + 
    1.45 -			  xen_get_offset() * 1000 - 
    1.46 -			  processed_system_time);
    1.47 +    	delta = cpu_delta = (int64_t)shadow_system_time + 
    1.48 +		(int64_t)xen_get_offset() * 1000;
    1.49 +	delta -= processed_system_time;
    1.50 +	cpu_delta -= per_cpu(processed_system_time, cpu);
    1.51      } while (!TIME_VALUES_UP_TO_DATE);
    1.52  
    1.53 -    if (unlikely(delta < 0)) {
    1.54 +    if (unlikely(delta < 0) || unlikely(cpu_delta < 0)) {
    1.55          printk("Timer ISR: Time went backwards: %lld\n", delta);
    1.56          return;
    1.57      }
    1.58 @@ -225,15 +240,28 @@ clkintr(struct clockframe *frame)
    1.59          delta -= NS_PER_TICK;
    1.60          processed_system_time += NS_PER_TICK;
    1.61      }
    1.62 -
    1.63 -    if (ticks > 0) {
    1.64 -	if (frame)
    1.65 -		timer_func(frame);
    1.66 -#ifdef SMP
    1.67 -	if (timer_func == hardclock && frame)
    1.68 -		forward_hardclock();
    1.69 +    /* Local CPU jiffy work. */
    1.70 +    while (cpu_delta >= NS_PER_TICK) {
    1.71 +	    cpu_delta -= NS_PER_TICK;
    1.72 +	    per_cpu(processed_system_time, cpu) += NS_PER_TICK;
    1.73 +#if 0
    1.74 +	    update_process_times(user_mode(regs));
    1.75 +	    profile_tick(CPU_PROFILING, regs);
    1.76  #endif
    1.77      }
    1.78 +    if (ticks > 0) {
    1.79 +	if (frame) timer_func(frame);
    1.80 +    }
    1.81 +    
    1.82 +    if (cpu != 0)
    1.83 +	    return;
    1.84 +    /*
    1.85 +     * Take synchronised time from Xen once a minute if we're not
    1.86 +     * synchronised ourselves, and we haven't chosen to keep an independent
    1.87 +     * time base.
    1.88 +     */
    1.89 +    
    1.90 +    /* XXX TODO */
    1.91  }
    1.92  
    1.93  #include "opt_ddb.h"
    1.94 @@ -429,7 +457,7 @@ resettodr()
    1.95   * Start clocks running.
    1.96   */
    1.97  void
    1.98 -cpu_initclocks()
    1.99 +cpu_initclocks(void)
   1.100  {
   1.101  	int diag;
   1.102  	int time_irq = bind_virq_to_irq(VIRQ_TIMER);
   1.103 @@ -445,8 +473,26 @@ cpu_initclocks()
   1.104  	/* initialize xen values */
   1.105  	__get_time_values_from_xen();
   1.106  	processed_system_time = shadow_system_time;
   1.107 +	per_cpu(processed_system_time, 0) = processed_system_time;
   1.108 +
   1.109  }
   1.110  
   1.111 +#ifdef SMP 
   1.112 +void
   1.113 +ap_cpu_initclocks(void)
   1.114 +{
   1.115 +	int irq;
   1.116 +	int cpu = smp_processor_id();
   1.117 +
   1.118 +	per_cpu(processed_system_time, cpu) = shadow_system_time;
   1.119 +	
   1.120 +	irq = bind_virq_to_irq(VIRQ_TIMER);
   1.121 +	PCPU_SET(time_irq, irq);
   1.122 +	PANIC_IF(intr_add_handler("clk", irq, (driver_intr_t *)clkintr, 
   1.123 +				  NULL, INTR_TYPE_CLK | INTR_FAST, NULL));
   1.124 +}
   1.125 +#endif
   1.126 +
   1.127  void
   1.128  cpu_startprofclock(void)
   1.129  {
     2.1 --- a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/evtchn.c	Sun May 08 08:39:44 2005 +0000
     2.2 +++ b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/evtchn.c	Sun May 08 08:39:54 2005 +0000
     2.3 @@ -79,9 +79,14 @@ evtchn_do_upcall(struct intrframe *frame
     2.4                  l2 &= ~(1 << l2i);
     2.5              
     2.6                  port = (l1i << 5) + l2i;
     2.7 +		irq = evtchn_to_irq[port];
     2.8 +#ifdef SMP		
     2.9 +		if (irq == PCPU_GET(cpuast)) 
    2.10 +			continue;
    2.11 +#endif
    2.12                  if ( (owned = mtx_owned(&sched_lock)) != 0 )
    2.13                      mtx_unlock_spin_flags(&sched_lock, MTX_QUIET);
    2.14 -                if ( (irq = evtchn_to_irq[port]) != -1 ) {
    2.15 +                if ( irq != -1 ) {
    2.16  		    struct intsrc *isrc = intr_lookup_source(irq);
    2.17  		    intr_execute_handlers(isrc, frame);
    2.18  		} else {
    2.19 @@ -584,6 +589,7 @@ ap_evtchn_init(int cpu)
    2.20          PCPU_GET(virq_to_irq)[i] = -1;
    2.21  }
    2.22  
    2.23 +
    2.24  static void 
    2.25  evtchn_init(void *dummy __unused)
    2.26  {
    2.27 @@ -591,13 +597,6 @@ evtchn_init(void *dummy __unused)
    2.28      struct xenpic *xp;
    2.29      struct xenpic_intsrc *pin;
    2.30  
    2.31 -    /*
    2.32 -     * xenpic_lock: in order to allow an interrupt to occur in a critical
    2.33 -     * 	        section, to set pcpu->ipending (etc...) properly, we
    2.34 -     *	        must be able to get the icu lock, so it can't be
    2.35 -     *	        under witness.
    2.36 -     */
    2.37 -    mtx_init(&irq_mapping_update_lock, "xp", NULL, MTX_DEF);
    2.38  
    2.39      /* XXX -- expedience hack */
    2.40      PCPU_SET(virq_to_irq, (int *)&virq_to_irq[0]);
    2.41 @@ -657,3 +656,11 @@ evtchn_init(void *dummy __unused)
    2.42  }
    2.43  
    2.44  SYSINIT(evtchn_init, SI_SUB_INTR, SI_ORDER_ANY, evtchn_init, NULL);
    2.45 +    /*
    2.46 +     * xenpic_lock: in order to allow an interrupt to occur in a critical
    2.47 +     * 	        section, to set pcpu->ipending (etc...) properly, we
    2.48 +     *	        must be able to get the icu lock, so it can't be
    2.49 +     *	        under witness.
    2.50 +     */
    2.51 +
    2.52 +MTX_SYSINIT(irq_mapping_update_lock, &irq_mapping_update_lock, "xp", MTX_DEF|MTX_NOWITNESS);
     3.1 --- a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/machdep.c	Sun May 08 08:39:44 2005 +0000
     3.2 +++ b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/machdep.c	Sun May 08 08:39:54 2005 +0000
     3.3 @@ -78,6 +78,7 @@
     3.4  #include <sys/sched.h>
     3.5  #include <sys/sysent.h>
     3.6  #include <sys/sysctl.h>
     3.7 +#include <sys/smp.h>
     3.8  #include <sys/ucontext.h>
     3.9  #include <sys/vmmeter.h>
    3.10  #include <sys/bus.h>
    3.11 @@ -883,14 +884,6 @@ SYSCTL_INT(_machdep, OID_AUTO, cpu_idle_
    3.12  static void
    3.13  cpu_idle_default(void)
    3.14  {
    3.15 -#if 0
    3.16 -	/*
    3.17 -	 * we must absolutely guarentee that hlt is the
    3.18 -	 * absolute next instruction after sti or we
    3.19 -	 * introduce a timing window.
    3.20 -	 */
    3.21 -	__asm __volatile("sti; hlt");
    3.22 -#endif
    3.23  	idle_block();
    3.24  	enable_intr();
    3.25  }
    3.26 @@ -1376,6 +1369,7 @@ pteinfo_t *pteinfo_list;
    3.27  unsigned long *xen_machine_phys = ((unsigned long *)VADDR(1008, 0));
    3.28  int preemptable;
    3.29  int gdt_set;
    3.30 +static int ncpus;
    3.31  
    3.32  /* Linux infection */
    3.33  #define PAGE_OFFSET  KERNBASE
    3.34 @@ -1387,6 +1381,10 @@ initvalues(start_info_t *startinfo)
    3.35      int i;
    3.36      vm_paddr_t pdir_shadow_ma, KPTphys;
    3.37      vm_offset_t *pdir_shadow;
    3.38 +#ifdef SMP
    3.39 +    int j;
    3.40 +#endif
    3.41 +
    3.42  #ifdef WRITABLE_PAGETABLES
    3.43      printk("using writable pagetables\n");
    3.44      HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_writable_pagetables);
    3.45 @@ -1447,18 +1445,19 @@ initvalues(start_info_t *startinfo)
    3.46  
    3.47  
    3.48  #ifdef SMP
    3.49 +#if 0
    3.50      /* allocate cpu0 private page */
    3.51      cpu0prvpage = (KERNBASE + (tmpindex << PAGE_SHIFT));
    3.52      tmpindex++; 
    3.53 -
    3.54 +#endif
    3.55      /* allocate SMP page table */
    3.56      SMPpt = (unsigned long *)(KERNBASE + (tmpindex << PAGE_SHIFT));
    3.57 -
    3.58 +#if 0
    3.59      /* Map the private page into the SMP page table */
    3.60      SMPpt[0] = vtomach(cpu0prvpage) | PG_RW | PG_M | PG_V | PG_A;
    3.61 -
    3.62 +#endif
    3.63      /* map SMP page table RO */
    3.64 -    PT_SET_MA(SMPpt, vtomach(SMPpt) & ~PG_RW);
    3.65 +    PT_SET_MA(SMPpt, *vtopte((vm_offset_t)SMPpt) & ~PG_RW);
    3.66  
    3.67      /* put the page table into the page directory */
    3.68      xen_queue_pt_update((vm_paddr_t)(IdlePTD + MPPTDI), 
    3.69 @@ -1496,44 +1495,61 @@ initvalues(start_info_t *startinfo)
    3.70      tmpindex++;
    3.71  
    3.72      HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list = (unsigned long)xen_phys_machine;
    3.73 +    ncpus = HYPERVISOR_shared_info->n_vcpu; 
    3.74 +#ifdef SMP
    3.75 +    for (i = 0; i < ncpus; i++) {
    3.76 +	    int npages = (sizeof(struct privatespace) + 1)/PAGE_SIZE;
    3.77 +	    for (j = 0; j < npages; j++) {
    3.78 +		    vm_paddr_t ma = xpmap_ptom(tmpindex << PAGE_SHIFT);
    3.79 +		    tmpindex++;
    3.80 +		    PT_SET_VA_MA(SMPpt + i*npages + j, ma | PG_A | PG_V | PG_RW | PG_M, FALSE);
    3.81 +	    }
    3.82 +    }
    3.83 +    xen_flush_queue();
    3.84 +#endif
    3.85      
    3.86      init_first = tmpindex;
    3.87      
    3.88  }
    3.89  
    3.90 +
    3.91 +trap_info_t trap_table[] = {
    3.92 +	{ 0,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(div)},
    3.93 +	{ 1,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(dbg)},
    3.94 +	{ 3,   3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(bpt)},
    3.95 +	{ 4,   3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(ofl)},
    3.96 +	/* This is UPL on Linux and KPL on BSD */
    3.97 +	{ 5,   3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(bnd)},
    3.98 +	{ 6,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(ill)},
    3.99 +	{ 7,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(dna)},
   3.100 +	/*
   3.101 +	 * { 8,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(XXX)},
   3.102 +	 *   no handler for double fault
   3.103 +	 */
   3.104 +	{ 9,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(fpusegm)},
   3.105 +	{10,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(tss)},
   3.106 +	{11,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(missing)},
   3.107 +	{12,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(stk)},
   3.108 +	{13,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(prot)},
   3.109 +	{14,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(page)},
   3.110 +	{15,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(rsvd)},
   3.111 +	{16,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(fpu)},
   3.112 +	{17,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(align)},
   3.113 +	{18,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(mchk)},
   3.114 +	{19,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(xmm)},
   3.115 +	{0x80, 3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(int0x80_syscall)},
   3.116 +	{  0, 0,           0, 0 }
   3.117 +};
   3.118 +
   3.119  void
   3.120  init386(void)
   3.121  {
   3.122  	int gsel_tss, metadata_missing, off, x, error;
   3.123  	struct pcpu *pc;
   3.124  	unsigned long gdtmachpfn;
   3.125 -	trap_info_t trap_table[] = {
   3.126 -	    { 0,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(div)},
   3.127 -	    { 1,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(dbg)},
   3.128 -	    { 3,   3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(bpt)},
   3.129 -	    { 4,   3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(ofl)},
   3.130 -	    /* This is UPL on Linux and KPL on BSD */
   3.131 -	    { 5,   3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(bnd)},
   3.132 -	    { 6,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(ill)},
   3.133 -	    { 7,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(dna)},
   3.134 -	    /*
   3.135 -	     * { 8,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(XXX)},
   3.136 -	     *   no handler for double fault
   3.137 -	     */
   3.138 -	    { 9,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(fpusegm)},
   3.139 -	    {10,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(tss)},
   3.140 -	    {11,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(missing)},
   3.141 -	    {12,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(stk)},
   3.142 -	    {13,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(prot)},
   3.143 -	    {14,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(page)},
   3.144 -	    {15,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(rsvd)},
   3.145 -	    {16,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(fpu)},
   3.146 -	    {17,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(align)},
   3.147 -	    {18,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(mchk)},
   3.148 -	    {19,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(xmm)},
   3.149 -	    {0x80, 3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(int0x80_syscall)},
   3.150 -	    {  0, 0,           0, 0 }
   3.151 -        };
   3.152 +#ifdef SMP
   3.153 +	int i;
   3.154 +#endif
   3.155  	proc0.p_uarea = proc0uarea;
   3.156  	thread0.td_kstack = proc0kstack;
   3.157  	thread0.td_pcb = (struct pcb *)
   3.158 @@ -1583,26 +1599,42 @@ init386(void)
   3.159  	gdt_segs[GDATA_SEL].ssd_limit = atop(0 - ((1 << 26) - (1 << 22) + (1 << 16))); 
   3.160  #endif
   3.161  #ifdef SMP
   3.162 -	/* this correspond to the cpu private page as mapped into the SMP page 
   3.163 -	 * table in initvalues
   3.164 +	/* XXX this will blow up if there are more than 512/NGDT vcpus - will never 
   3.165 +	 * be an issue in the real world but should add an assert on general principles
   3.166 +	 * we'll likely blow up when we hit LAST_RESERVED_GDT_ENTRY, at which point we
   3.167 +	 * would need to start allocating more pages for the GDT
   3.168  	 */
   3.169  	pc = &SMP_prvspace[0].pcpu;
   3.170 -	gdt_segs[GPRIV_SEL].ssd_limit =
   3.171 -		atop(sizeof(struct privatespace) - 1);
   3.172 +	for (i = 0; i < ncpus; i++) {
   3.173 +		cpu_add(i, (i == 0));
   3.174 +
   3.175 +		gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[i];
   3.176 +		gdt_segs[GPRIV_SEL].ssd_limit =
   3.177 +			atop(sizeof(struct privatespace) - 1);
   3.178 +		gdt_segs[GPROC0_SEL].ssd_base =
   3.179 +			(int) &SMP_prvspace[i].pcpu.pc_common_tss;
   3.180 +		SMP_prvspace[i].pcpu.pc_prvspace =
   3.181 +			&SMP_prvspace[i].pcpu;
   3.182 +		
   3.183 +		for (x = 0; x < NGDT; x++) {
   3.184 +			ssdtosd(&gdt_segs[x], &gdt[i * NGDT + x].sd);
   3.185 +		}
   3.186 +	}
   3.187  #else
   3.188  	pc = &__pcpu;
   3.189  	gdt_segs[GPRIV_SEL].ssd_limit =
   3.190  		atop(sizeof(struct pcpu) - 1);
   3.191 -#endif
   3.192  	gdt_segs[GPRIV_SEL].ssd_base = (int) pc;
   3.193  	gdt_segs[GPROC0_SEL].ssd_base = (int) &pc->pc_common_tss;
   3.194  	for (x = 0; x < NGDT; x++)
   3.195  	    ssdtosd(&gdt_segs[x], &gdt[x].sd);
   3.196 +#endif
   3.197 +
   3.198  
   3.199  	PT_SET_MA(gdt, *vtopte((unsigned long)gdt) & ~PG_RW);
   3.200  	gdtmachpfn = vtomach(gdt) >> PAGE_SHIFT;
   3.201 -	if ((error = HYPERVISOR_set_gdt(&gdtmachpfn, LAST_RESERVED_GDT_ENTRY + 1))) 
   3.202 -	    panic("set_gdt failed");
   3.203 +	PANIC_IF(HYPERVISOR_set_gdt(&gdtmachpfn, LAST_RESERVED_GDT_ENTRY + 1) != 0);
   3.204 +
   3.205  	
   3.206  	lgdt_finish();
   3.207  	gdt_set = 1;
     4.1 --- a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/mp_machdep.c	Sun May 08 08:39:44 2005 +0000
     4.2 +++ b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/mp_machdep.c	Sun May 08 08:39:54 2005 +0000
     4.3 @@ -83,7 +83,16 @@
     4.4  #include <machine/specialreg.h>
     4.5  #include <machine/privatespace.h>
     4.6  
     4.7 +
     4.8 +/* XEN includes */
     4.9  #include <machine/xenfunc.h>
    4.10 +#include <machine/xen_intr.h>
    4.11 +
    4.12 +void Xhypervisor_callback(void);
    4.13 +void failsafe_callback(void);
    4.14 +
    4.15 +/***************/
    4.16 +
    4.17  
    4.18  #define WARMBOOT_TARGET		0
    4.19  #define WARMBOOT_OFF		(KERNBASE + 0x0467)
    4.20 @@ -94,6 +103,10 @@
    4.21  #define BIOS_RESET		(0x0f)
    4.22  #define BIOS_WARM		(0x0a)
    4.23  
    4.24 +
    4.25 +#undef POSTCODE
    4.26 +#define POSTCODE(x)
    4.27 +
    4.28  /*
    4.29   * this code MUST be enabled here and in mpboot.s.
    4.30   * it follows the very early stages of AP boot by placing values in CMOS ram.
    4.31 @@ -175,6 +188,8 @@ extern pt_entry_t *KPTphys;
    4.32  /* SMP page table page */
    4.33  extern pt_entry_t *SMPpt;
    4.34  
    4.35 +extern trap_info_t trap_table[];
    4.36 +
    4.37  struct pcb stoppcbs[MAXCPU];
    4.38  
    4.39  /* Variables needed for SMP tlb shootdown. */
    4.40 @@ -208,7 +223,9 @@ static u_int boot_address;
    4.41  
    4.42  static void	set_logical_apic_ids(void);
    4.43  static int	start_all_aps(void);
    4.44 +#if 0
    4.45  static void	install_ap_tramp(void);
    4.46 +#endif
    4.47  static int	start_ap(int apic_id);
    4.48  static void	release_aps(void *dummy);
    4.49  
    4.50 @@ -314,6 +331,7 @@ int
    4.51  cpu_mp_probe(void)
    4.52  {
    4.53  
    4.54 +	mp_ncpus = HYPERVISOR_shared_info->n_vcpu;
    4.55  	/*
    4.56  	 * Always record BSP in CPU map so that the mbuf init code works
    4.57  	 * correctly.
    4.58 @@ -342,20 +360,24 @@ cpu_mp_probe(void)
    4.59  	return (1);
    4.60  }
    4.61  
    4.62 -/*
    4.63 - * Initialize the IPI handlers and start up the AP's.
    4.64 - */
    4.65 -void
    4.66 -cpu_mp_start(void)
    4.67 +static void
    4.68 +cpu_mp_ipi_init(void)
    4.69  {
    4.70 -	int i;
    4.71 -
    4.72 -	POSTCODE(MP_START_POST);
    4.73 -
    4.74 -	/* Initialize the logical ID to APIC ID table. */
    4.75 -	for (i = 0; i < MAXCPU; i++)
    4.76 -		cpu_apic_ids[i] = -1;
    4.77 -
    4.78 +	int irq;
    4.79 +	int cpu = smp_processor_id();
    4.80 +	/* 
    4.81 +	 * these are not needed by XenFreeBSD - from Keir:
    4.82 +	 * For TLB-flush related IPIs, Xen has hypercalls 
    4.83 +	 * you should use instead. You can pass a pointer 
    4.84 +	 * to a vcpu bitmap to update_va_mapping(), and to
    4.85 +	 * MMUEXT_flush_tlb_multi and MMEXT_invlpg_multi. 
    4.86 +	 * Xen will then make sure that those vcpus get 
    4.87 +	 * flushed appropriately before returning to the
    4.88 +	 * caller.
    4.89 +	 * There is also no indication that we need to forward
    4.90 +	 * clock interrupts.
    4.91 +	 */
    4.92 +#if 0 
    4.93  	/* Install an inter-CPU IPI for TLB invalidation */
    4.94  	setidt(IPI_INVLTLB, IDTVEC(invltlb),
    4.95  	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
    4.96 @@ -371,22 +393,69 @@ cpu_mp_start(void)
    4.97  	/* Install an inter-CPU IPI for forwarding statclock() */
    4.98  	setidt(IPI_STATCLOCK, IDTVEC(statclock),
    4.99  	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
   4.100 -	
   4.101 +#endif
   4.102 +
   4.103 +	/* 
   4.104 +	 * These can all be consolidated. For now leaving 
   4.105 +	 * as individual IPIs.
   4.106 +	 *
   4.107 +	 */
   4.108 +#if 0
   4.109  	/* Install an inter-CPU IPI for lazy pmap release */
   4.110  	setidt(IPI_LAZYPMAP, IDTVEC(lazypmap),
   4.111  	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
   4.112 +#else
   4.113 +	irq = bind_ipi_on_cpu_to_irq(cpu, IPI_LAZYPMAP);
   4.114 +	PCPU_SET(lazypmap, irq);
   4.115 +	PANIC_IF(intr_add_handler("pmap_lazyfix", irq, 
   4.116 +				  (driver_intr_t *)pmap_lazyfix_action, 
   4.117 +				  NULL, INTR_TYPE_CLK | INTR_FAST, NULL));
   4.118 +#endif
   4.119  
   4.120 +#if 0
   4.121  	/* Install an inter-CPU IPI for all-CPU rendezvous */
   4.122  	setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous),
   4.123  	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
   4.124 +#else 
   4.125 +	irq = bind_ipi_on_cpu_to_irq(cpu, IPI_RENDEZVOUS);
   4.126 +	PCPU_SET(rendezvous, irq);
   4.127 +	PANIC_IF(intr_add_handler("smp_rendezvous", irq, 
   4.128 +				  (driver_intr_t *)smp_rendezvous_action, 
   4.129 +				  NULL, INTR_TYPE_CLK | INTR_FAST, NULL));
   4.130 +#endif
   4.131  
   4.132 +#if 0
   4.133  	/* Install an inter-CPU IPI for forcing an additional software trap */
   4.134  	setidt(IPI_AST, IDTVEC(cpuast),
   4.135  	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
   4.136 -
   4.137 +#else
   4.138 +	irq = bind_ipi_on_cpu_to_irq(cpu, IPI_AST);
   4.139 +	PCPU_SET(cpuast, irq);
   4.140 +#endif
   4.141 +	/* XXX ignore for now */
   4.142 +#if 0 
   4.143  	/* Install an inter-CPU IPI for CPU stop/restart */
   4.144  	setidt(IPI_STOP, IDTVEC(cpustop),
   4.145  	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
   4.146 +#endif
   4.147 +
   4.148 +}
   4.149 +
   4.150 +SYSINIT(ipi_setup, SI_SUB_INTR, SI_ORDER_ANY, cpu_mp_ipi_init, NULL);
   4.151 +
   4.152 +/*
   4.153 + * Initialize the IPI handlers and start up the AP's.
   4.154 + */
   4.155 +void
   4.156 +cpu_mp_start(void) /* --- Start here --- */
   4.157 +{
   4.158 +	int i;
   4.159 +
   4.160 +	POSTCODE(MP_START_POST);
   4.161 +
   4.162 +	/* Initialize the logical ID to APIC ID table. */
   4.163 +	for (i = 0; i < MAXCPU; i++)
   4.164 +		cpu_apic_ids[i] = -1;
   4.165  
   4.166  
   4.167  	/* Set boot_cpu_id if needed. */
   4.168 @@ -437,35 +506,44 @@ cpu_mp_announce(void)
   4.169  void
   4.170  init_secondary(void)
   4.171  {
   4.172 -	int	gsel_tss;
   4.173 -	int	x, myid;
   4.174 +	int	myid;
   4.175 +	unsigned long gdtmachpfn;
   4.176 +	printk("MADE IT!!");
   4.177 +
   4.178  #if 0
   4.179  	u_int	cr0;
   4.180  #endif
   4.181 +	/* Steps to booting SMP on xen as gleaned from XenLinux:
   4.182 +	 * - cpu_init() - processor specific initialization
   4.183 +	 * - smp_callin() 
   4.184 +	 *    - wait 2s for BP to finish its startup sequence
   4.185 +	 *    - map_cpu_to_logical_apicid()
   4.186 +	 *    - save cpuid info
   4.187 +	 *    - set bit in callin map to let master (BP?) continue
   4.188 +	 * - local setup timer() - per cpu timer initialization
   4.189 +	 * - ldebug_setup() - bind debug IRQ to local CPU.
   4.190 +	 * - smp_intr_init() - IPI setup that we do in cpu_mp_start
   4.191 +	 * - local_irq_enable() - enable interrupts locally
   4.192 +	 * - cpu_set(id, map) - announce that we're up
   4.193 +	 * - cpu_idle() - make us schedulable
   4.194 +	 */
   4.195 +
   4.196 +
   4.197  	/* bootAP is set in start_ap() to our ID. */
   4.198  	myid = bootAP;
   4.199 -	gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid];
   4.200 -	gdt_segs[GPROC0_SEL].ssd_base =
   4.201 -		(int) &SMP_prvspace[myid].pcpu.pc_common_tss;
   4.202 -	SMP_prvspace[myid].pcpu.pc_prvspace =
   4.203 -		&SMP_prvspace[myid].pcpu;
   4.204 +
   4.205 +	gdtmachpfn = vtomach(gdt) >> PAGE_SHIFT;
   4.206 +	PANIC_IF(HYPERVISOR_set_gdt(&gdtmachpfn, LAST_RESERVED_GDT_ENTRY + 1) != 0); 
   4.207  
   4.208 -	for (x = 0; x < NGDT; x++) {
   4.209 -		ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd);
   4.210 -	}
   4.211 +	
   4.212 +	lgdt_finish();
   4.213  
   4.214 -#if 0
   4.215 -	r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
   4.216 -	r_gdt.rd_base = (int) &gdt[myid * NGDT];
   4.217 -	lgdt(&r_gdt);			/* does magic intra-segment return */
   4.218 +	PCPU_SET(cpuid, myid);
   4.219  
   4.220 -	lidt(&r_idt);
   4.221 -	lldt(_default_ldt);
   4.222 -#endif
   4.223 +
   4.224 +	set_user_ldt((struct mdproc *)_default_ldt);
   4.225  	PCPU_SET(currentldt, _default_ldt);
   4.226  
   4.227 -	gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
   4.228 -	gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS;
   4.229  	PCPU_SET(common_tss.tss_esp0, 0); /* not used until after switch */
   4.230  	PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL));
   4.231  	PCPU_SET(common_tss.tss_ioopt, (sizeof (struct i386tss)) << 16);
   4.232 @@ -557,6 +635,13 @@ init_secondary(void)
   4.233  	while (smp_started == 0)
   4.234  		ia32_pause();
   4.235  
   4.236 +	/* need to wait until now to setup the IPIs as SI_SUB_CPU is
   4.237 +	 * much earlier than SI_SUB_INTR
   4.238 +	 */  
   4.239 +	ap_evtchn_init(myid);
   4.240 +	ap_cpu_initclocks();
   4.241 +	cpu_mp_ipi_init();
   4.242 +
   4.243  	/* ok, now grab sched_lock and enter the scheduler */
   4.244  	mtx_lock_spin(&sched_lock);
   4.245  
   4.246 @@ -610,28 +695,35 @@ set_logical_apic_ids(void)
   4.247  static int
   4.248  start_all_aps(void)
   4.249  {
   4.250 -#ifndef PC98
   4.251 -	u_char mpbiosreason;
   4.252 -#endif
   4.253 -	u_long mpbioswarmvec;
   4.254  	struct pcpu *pc;
   4.255  	char *stack;
   4.256 -	uintptr_t kptbase;
   4.257 -	int i, pg, apic_id, cpu;
   4.258 +	int i, apic_id, cpu;
   4.259 +
   4.260 +	/* 
   4.261 +	 * This function corresponds most closely to 
   4.262 +	 * smp_boot_cpus in XenLinux - the sequence there 
   4.263 +	 * is:
   4.264 +	 * - check if SMP config is found - if not:
   4.265 +	 *     - clear the I/O APIC IRQs
   4.266 +	 *     - map cpu to logical apicid
   4.267 +	 *     - exit
   4.268 +	 * - smp_intr_init - IPI initialization
   4.269 +	 * - map cpu to logical apicid
   4.270 +	 * - boot each of the vcpus
   4.271 +	 * - clear and then construct the cpu sibling [logical CPUs] map.
   4.272 +	 *
   4.273 +	 */
   4.274  
   4.275  	POSTCODE(START_ALL_APS_POST);
   4.276  
   4.277  	mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
   4.278 -
   4.279 +#if 0
   4.280  	/* install the AP 1st level boot code */
   4.281  	install_ap_tramp();
   4.282  
   4.283  	/* save the current value of the warm-start vector */
   4.284  	mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
   4.285 -#ifndef PC98
   4.286 -	outb(CMOS_REG, BIOS_RESET);
   4.287 -	mpbiosreason = inb(CMOS_DATA);
   4.288 -#endif
   4.289 +
   4.290  
   4.291  	/* set up temporary P==V mapping for AP boot */
   4.292  	/* XXX this is a hack, we should boot the AP on its own stack/PTD */
   4.293 @@ -640,7 +732,7 @@ start_all_aps(void)
   4.294  		PTD[i] = (pd_entry_t)(PG_V | PG_RW |
   4.295  		    ((kptbase + i * PAGE_SIZE) & PG_FRAME));
   4.296  	invltlb();
   4.297 -
   4.298 +#endif
   4.299  	/* start each AP */
   4.300  	for (cpu = 0, apic_id = 0; apic_id < MAXCPU; apic_id++) {
   4.301  		if (!cpu_info[apic_id].cpu_present ||
   4.302 @@ -650,7 +742,7 @@ start_all_aps(void)
   4.303  
   4.304  		/* save APIC ID for this logical ID */
   4.305  		cpu_apic_ids[cpu] = apic_id;
   4.306 -
   4.307 +#if 0
   4.308  		/* first page of AP's private space */
   4.309  		pg = cpu * i386_btop(sizeof(struct privatespace));
   4.310  
   4.311 @@ -665,11 +757,14 @@ start_all_aps(void)
   4.312  		for (i = 0; i < KSTACK_PAGES; i++)
   4.313  			SMPpt[pg + 1 + i] = (pt_entry_t)
   4.314  			    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
   4.315 +#endif
   4.316 +		pc = &SMP_prvspace[cpu].pcpu;
   4.317  
   4.318  		/* prime data page for it to use */
   4.319  		pcpu_init(pc, cpu, sizeof(struct pcpu));
   4.320  		pc->pc_apic_id = apic_id;
   4.321  
   4.322 +#if 0
   4.323  		/* setup a vector to our boot code */
   4.324  		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
   4.325  		*((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4);
   4.326 @@ -677,7 +772,7 @@ start_all_aps(void)
   4.327  		outb(CMOS_REG, BIOS_RESET);
   4.328  		outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
   4.329  #endif
   4.330 -
   4.331 +#endif
   4.332  		bootSTK = &SMP_prvspace[cpu].idlekstack[KSTACK_PAGES *
   4.333  		    PAGE_SIZE];
   4.334  		bootAP = cpu;
   4.335 @@ -700,13 +795,10 @@ start_all_aps(void)
   4.336  	/* build our map of 'other' CPUs */
   4.337  	PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
   4.338  
   4.339 +#if 0
   4.340  	/* restore the warmstart vector */
   4.341  	*(u_long *) WARMBOOT_OFF = mpbioswarmvec;
   4.342 -#ifndef PC98
   4.343 -	outb(CMOS_REG, BIOS_RESET);
   4.344 -	outb(CMOS_DATA, mpbiosreason);
   4.345  #endif
   4.346 -
   4.347  	/*
   4.348  	 * Set up the idle context for the BSP.  Similar to above except
   4.349  	 * that some was done by locore, some by pmap.c and some is implicit
   4.350 @@ -739,7 +831,7 @@ extern void bootDataSeg(void);
   4.351  extern void MPentry(void);
   4.352  extern u_int MP_GDT;
   4.353  extern u_int mp_gdtbase;
   4.354 -
   4.355 +#if 0
   4.356  static void
   4.357  install_ap_tramp(void)
   4.358  {
   4.359 @@ -791,6 +883,21 @@ install_ap_tramp(void)
   4.360  	*dst16 = (u_int) boot_address & 0xffff;
   4.361  	*dst8 = ((u_int) boot_address >> 16) & 0xff;
   4.362  }
   4.363 +#endif
   4.364 +
   4.365 +static int 
   4.366 +cpu_mp_trap_init(trap_info_t *trap_ctxt)
   4.367 +{
   4.368 +
   4.369 +        trap_info_t *t = trap_table;
   4.370 +
   4.371 +        for (t = trap_table; t->address; t++) {
   4.372 +                trap_ctxt[t->vector].flags = t->flags;
   4.373 +                trap_ctxt[t->vector].cs = t->cs;
   4.374 +                trap_ctxt[t->vector].address = t->address;
   4.375 +        }
   4.376 +        return 0x80 /*SYSCALL_VECTOR*/;
   4.377 +}
   4.378  
   4.379  /*
   4.380   * This function starts the AP (application processor) identified
   4.381 @@ -802,8 +909,25 @@ install_ap_tramp(void)
   4.382  static int
   4.383  start_ap(int apic_id)
   4.384  {
   4.385 -	int vector, ms;
   4.386 -	int cpus;
   4.387 +	int vector, ms, i;
   4.388 +	int cpus, boot_error;
   4.389 +	vcpu_guest_context_t ctxt;
   4.390 +
   4.391 +	/* 
   4.392 +	 * This is the FreeBSD equivalent to do_boot_cpu(apicid) in
   4.393 +	 * smpboot.c. 
   4.394 +	 * its initialization sequence consists of:
   4.395 +	 * - fork_idle(cpu) to create separate idle context
   4.396 +	 * - initialization of idle's context to start_secondary
   4.397 +	 * - initialization of cpu ctxt to start in startup_32_smp
   4.398 +	 * - then we call HYPERVISOR_boot_vcpu with the cpu index and
   4.399 +	 *   a pointer to the context.
   4.400 +	 * - on boot success we:
   4.401 +	 *   - set ourselves in the callout_map
   4.402 +	 *   - wait up to 5 seconds for us to be set in the callin map
   4.403 +	 * - set x86_cpu_to_apicid[cpu] = apicid;
   4.404 +	 *
   4.405 +	 */
   4.406  
   4.407  	POSTCODE(START_AP_POST);
   4.408  
   4.409 @@ -813,6 +937,55 @@ start_ap(int apic_id)
   4.410  	/* used as a watchpoint to signal AP startup */
   4.411  	cpus = mp_naps;
   4.412  
   4.413 +	memset(&ctxt, 0, sizeof(ctxt));
   4.414 +
   4.415 +	ctxt.user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
   4.416 +	ctxt.user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
   4.417 +	ctxt.user_regs.fs = 0;
   4.418 +	ctxt.user_regs.gs = 0;
   4.419 +	ctxt.user_regs.ss = __KERNEL_DS;
   4.420 +	ctxt.user_regs.cs = __KERNEL_CS;
   4.421 +	ctxt.user_regs.eip = (unsigned long)init_secondary;
   4.422 +	ctxt.user_regs.esp = (unsigned long)bootSTK;
   4.423 +#ifdef notyet
   4.424 +	ctxt.user_regs.eflags = (1<<9) | (1<<2) | (idle->thread.io_pl<<12);
   4.425 +#else
   4.426 +	ctxt.user_regs.eflags = (1<<9) | (1<<2);
   4.427 +#endif
   4.428 +	/* FPU is set up to default initial state. */
   4.429 +	memset(&ctxt.fpu_ctxt, 0, sizeof(ctxt.fpu_ctxt));
   4.430 +
   4.431 +	/* Virtual IDT is empty at start-of-day. */
   4.432 +	for ( i = 0; i < 256; i++ )
   4.433 +	{
   4.434 +		ctxt.trap_ctxt[i].vector = i;
   4.435 +		ctxt.trap_ctxt[i].cs     = FLAT_KERNEL_CS;
   4.436 +	}
   4.437 +	ctxt.fast_trap_idx = cpu_mp_trap_init(ctxt.trap_ctxt);
   4.438 +
   4.439 +	/* No LDT. */
   4.440 +	ctxt.ldt_ents = 0;
   4.441 +
   4.442 +	/* Ring 1 stack is the initial stack. */
   4.443 +	ctxt.kernel_ss = __KERNEL_DS;
   4.444 +	ctxt.kernel_sp = (unsigned long)bootSTK;
   4.445 +
   4.446 +	/* Callback handlers. */
   4.447 +	ctxt.event_callback_cs     = __KERNEL_CS;
   4.448 +	ctxt.event_callback_eip    = (unsigned long)Xhypervisor_callback;
   4.449 +	ctxt.failsafe_callback_cs  = __KERNEL_CS;
   4.450 +	ctxt.failsafe_callback_eip = (unsigned long)failsafe_callback;
   4.451 +
   4.452 +	ctxt.pt_base = (vm_paddr_t)IdlePTD;
   4.453 +
   4.454 +	boot_error = HYPERVISOR_boot_vcpu(bootAP, &ctxt);
   4.455 +
   4.456 +	
   4.457 +	if (boot_error) 
   4.458 +		printk("Houston we have a problem\n");
   4.459 +	else
   4.460 +		printk("boot_vcpu succeeded\n");
   4.461 +#if 0
   4.462  	/*
   4.463  	 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
   4.464  	 * and running the target CPU. OR this INIT IPI might be latched (P5
   4.465 @@ -862,6 +1035,7 @@ start_ap(int apic_id)
   4.466  	    APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP |
   4.467  	    vector, apic_id);
   4.468  	lapic_ipi_wait(-1);
   4.469 +#endif
   4.470  	DELAY(200);		/* wait ~200uS */
   4.471  
   4.472  	/* Wait up to 5 seconds for it to start. */
     5.1 --- a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/pmap.c	Sun May 08 08:39:44 2005 +0000
     5.2 +++ b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/pmap.c	Sun May 08 08:39:54 2005 +0000
     5.3 @@ -1374,7 +1374,6 @@ static u_int *lazymask;
     5.4  static u_int lazyptd;
     5.5  static volatile u_int lazywait;
     5.6  
     5.7 -void pmap_lazyfix_action(void);
     5.8  
     5.9  void
    5.10  pmap_lazyfix_action(void)
     6.1 --- a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/xen_machdep.c	Sun May 08 08:39:44 2005 +0000
     6.2 +++ b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/xen_machdep.c	Sun May 08 08:39:54 2005 +0000
     6.3 @@ -380,8 +380,6 @@ printk(const char *fmt, ...)
     6.4          (void)HYPERVISOR_console_write(buf, ret);
     6.5  }
     6.6  
     6.7 -#define PANIC_IF(exp) if (unlikely(exp)) {printk("%s failed\n",#exp); panic("%s: %s:%d", #exp, __FILE__, __LINE__);} 
     6.8 -
     6.9  
    6.10  #define XPQUEUE_SIZE 128
    6.11  #ifdef SMP
     7.1 --- a/freebsd-5.3-xen-sparse/i386-xen/include/hypervisor.h	Sun May 08 08:39:44 2005 +0000
     7.2 +++ b/freebsd-5.3-xen-sparse/i386-xen/include/hypervisor.h	Sun May 08 08:39:54 2005 +0000
     7.3 @@ -441,4 +441,20 @@ HYPERVISOR_vm_assist(unsigned int cmd, u
     7.4      return ret;
     7.5  }
     7.6  
     7.7 +static inline int
     7.8 +HYPERVISOR_boot_vcpu(
     7.9 +    unsigned long vcpu, vcpu_guest_context_t *ctxt)
    7.10 +{
    7.11 +    int ret;
    7.12 +    unsigned long ign1, ign2;
    7.13 +
    7.14 +    __asm__ __volatile__ (
    7.15 +        TRAP_INSTR
    7.16 +        : "=a" (ret), "=b" (ign1), "=c" (ign2)
    7.17 +	: "0" (__HYPERVISOR_boot_vcpu), "1" (vcpu), "2" (ctxt)
    7.18 +	: "memory");
    7.19 +
    7.20 +    return ret;
    7.21 +}
    7.22 +
    7.23  #endif /* __HYPERVISOR_H__ */
     8.1 --- a/freebsd-5.3-xen-sparse/i386-xen/include/pcpu.h	Sun May 08 08:39:44 2005 +0000
     8.2 +++ b/freebsd-5.3-xen-sparse/i386-xen/include/pcpu.h	Sun May 08 08:39:54 2005 +0000
     8.3 @@ -53,7 +53,12 @@
     8.4          int     *pc_ipi_to_evtchn;                                      \
     8.5          int     *pc_virq_to_irq;                                        \
     8.6          u_int   pc_cr2;                                                 \
     8.7 -        u_int   pc_pdir                                        
     8.8 +        u_int   pc_pdir;                                                \
     8.9 +        u_int   pc_lazypmap;                                            \
    8.10 +        u_int   pc_rendezvous;                                          \
    8.11 +        u_int   pc_cpuast;                                              \
    8.12 +        u_int   pc_time_irq;                                              \
    8.13 +        uint64_t pc_processed_system_time;  
    8.14  
    8.15  #if defined(lint)
    8.16   
     9.1 --- a/freebsd-5.3-xen-sparse/i386-xen/include/pmap.h	Sun May 08 08:39:44 2005 +0000
     9.2 +++ b/freebsd-5.3-xen-sparse/i386-xen/include/pmap.h	Sun May 08 08:39:54 2005 +0000
     9.3 @@ -343,6 +343,7 @@ void	pmap_set_pg(void);
     9.4  void	pmap_invalidate_page(pmap_t, vm_offset_t);
     9.5  void	pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t);
     9.6  void	pmap_invalidate_all(pmap_t);
     9.7 +void    pmap_lazyfix_action(void);
     9.8  
     9.9  void pmap_map_readonly(pmap_t pmap, vm_offset_t va, int len);
    9.10  void pmap_map_readwrite(pmap_t pmap, vm_offset_t va, int len);
    10.1 --- a/freebsd-5.3-xen-sparse/i386-xen/include/xenfunc.h	Sun May 08 08:39:44 2005 +0000
    10.2 +++ b/freebsd-5.3-xen-sparse/i386-xen/include/xenfunc.h	Sun May 08 08:39:54 2005 +0000
    10.3 @@ -61,6 +61,9 @@ void load_cr3(uint32_t val);
    10.4  void xen_machphys_update(unsigned long, unsigned long);
    10.5  void xen_update_descriptor(union descriptor *, union descriptor *);
    10.6  void lldt(u_short sel);
    10.7 +void ap_cpu_initclocks(void);
    10.8 +
    10.9 +
   10.10  /*
   10.11   * Invalidate a patricular VA on all cpus
   10.12   *
   10.13 @@ -79,5 +82,6 @@ invltlb(void)
   10.14  	
   10.15  }
   10.16  
   10.17 +#define PANIC_IF(exp) if (unlikely(exp)) {printk("%s failed\n",#exp); panic("%s: %s:%d", #exp, __FILE__, __LINE__);} 
   10.18  
   10.19  #endif /* _XEN_XENFUNC_H_ */