AP startup on PVH follows the PV method, so we need to add a hook in
order to diverge from bare metal.
---
 sys/amd64/amd64/mp_machdep.c |   16 ++++---
 sys/amd64/include/cpu.h      |    1 +
 sys/x86/xen/hvm.c            |   17 +++++++-
 sys/x86/xen/pv.c             |   90 ++++++++++++++++++++++++++++++++++++++++++
 sys/xen/pv.h                 |    1 +
 5 files changed, 117 insertions(+), 8 deletions(-)

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 4ef4b3d..e302886 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -90,7 +90,7 @@ extern  struct pcpu __pcpu[];
 
 /* AP uses this during bootstrap.  Do not staticize.  */
 char *bootSTK;
-static int bootAP;
+int bootAP;
 
 /* Free these after use */
 void *bootstacks[MAXCPU];
@@ -122,9 +122,12 @@ u_long *ipi_rendezvous_counts[MAXCPU];
 static u_long *ipi_hardclock_counts[MAXCPU];
 #endif
 
+int native_start_all_aps(void);
+
 /* Default cpu_ops implementation. */
 struct cpu_ops cpu_ops = {
-       .ipi_vectored = lapic_ipi_vectored
+       .ipi_vectored = lapic_ipi_vectored,
+       .start_all_aps = native_start_all_aps,
 };
 
 extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32);
@@ -138,7 +141,7 @@ extern int pmap_pcid_enabled;
 static volatile cpuset_t ipi_nmi_pending;
 
 /* used to hold the AP's until we are ready to release them */
-static struct mtx ap_boot_mtx;
+struct mtx ap_boot_mtx;
 
 /* Set to 1 once we're ready to let the APs out of the pen. */
 static volatile int aps_ready = 0;
@@ -165,7 +168,6 @@ static int cpu_cores;                       /* cores per 
package */
 
 static void    assign_cpu_ids(void);
 static void    set_interrupt_apic_ids(void);
-static int     start_all_aps(void);
 static int     start_ap(int apic_id);
 static void    release_aps(void *dummy);
 
@@ -569,7 +571,7 @@ cpu_mp_start(void)
        assign_cpu_ids();
 
        /* Start each Application Processor */
-       start_all_aps();
+       cpu_ops.start_all_aps();
 
        set_interrupt_apic_ids();
 }
@@ -908,8 +910,8 @@ assign_cpu_ids(void)
 /*
  * start each AP in our list
  */
-static int
-start_all_aps(void)
+int
+native_start_all_aps(void)
 {
        vm_offset_t va = boot_address + KERNBASE;
        u_int64_t *pt4, *pt3, *pt2;
diff --git a/sys/amd64/include/cpu.h b/sys/amd64/include/cpu.h
index 3d9ff531..ed9f1db 100644
--- a/sys/amd64/include/cpu.h
+++ b/sys/amd64/include/cpu.h
@@ -64,6 +64,7 @@ struct cpu_ops {
        void (*cpu_init)(void);
        void (*cpu_resume)(void);
        void (*ipi_vectored)(u_int, int);
+       int  (*start_all_aps)(void);
 };
 
 extern struct  cpu_ops cpu_ops;
diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c
index fb1ed79..5ec9f3a 100644
--- a/sys/x86/xen/hvm.c
+++ b/sys/x86/xen/hvm.c
@@ -53,6 +53,9 @@ __FBSDID("$FreeBSD$");
 #include <xen/hypervisor.h>
 #include <xen/hvm.h>
 #include <xen/xen_intr.h>
+#ifdef __amd64__
+#include <xen/pv.h>
+#endif
 
 #include <xen/interface/hvm/params.h>
 #include <xen/interface/vcpu.h>
@@ -97,6 +100,11 @@ extern void pmap_lazyfix_action(void);
 /* Variables used by mp_machdep to perform the bitmap IPI */
 extern volatile u_int cpu_ipi_pending[MAXCPU];
 
+#ifdef __amd64__
+/* Native AP start used on PVHVM */
+extern int native_start_all_aps(void);
+#endif
+
 /*---------------------------------- Macros 
----------------------------------*/
 #define        IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS)
 
@@ -119,7 +127,10 @@ enum xen_domain_type xen_domain_type = XEN_NATIVE;
 struct cpu_ops xen_hvm_cpu_ops = {
        .ipi_vectored   = lapic_ipi_vectored,
        .cpu_init       = xen_hvm_cpu_init,
-       .cpu_resume     = xen_hvm_cpu_resume
+       .cpu_resume     = xen_hvm_cpu_resume,
+#ifdef __amd64__
+       .start_all_aps = native_start_all_aps,
+#endif
 };
 
 static MALLOC_DEFINE(M_XENHVM, "xen_hvm", "Xen HVM PV Support");
@@ -698,6 +709,10 @@ xen_hvm_init(enum xen_hvm_init_type init_type)
                setup_xen_features();
                cpu_ops = xen_hvm_cpu_ops;
                vm_guest = VM_GUEST_XEN;
+#ifdef __amd64__
+               if (xen_pv_domain())
+                       cpu_ops.start_all_aps = xen_pv_start_all_aps;
+#endif
                break;
        case XEN_HVM_INIT_RESUME:
                if (error != 0)
diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c
index db576e0..7e45a83 100644
--- a/sys/x86/xen/pv.c
+++ b/sys/x86/xen/pv.c
@@ -34,21 +34,43 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/reboot.h>
 #include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
 
 #include <machine/sysarch.h>
 #include <machine/clock.h>
 #include <machine/pc/bios.h>
+#include <machine/smp.h>
 
 #include <xen/xen-os.h>
 #include <xen/pv.h>
 #include <xen/hypervisor.h>
 
+#include <xen/interface/vcpu.h>
+
 #define MAX_E820_ENTRIES       128
 
 /*--------------------------- Forward Declarations 
---------------------------*/
 static caddr_t xen_pv_parse_preload_data(u_int64_t);
 static void xen_pv_parse_memmap(caddr_t, vm_paddr_t *, int *);
 
+/*---------------------------- Extern Declarations 
---------------------------*/
+/* Variables used by amd64 mp_machdep to start APs */
+extern struct mtx ap_boot_mtx;
+extern void *bootstacks[];
+extern char *doublefault_stack;
+extern char *nmi_stack;
+extern void *dpcpu;
+extern int bootAP;
+extern char *bootSTK;
+
 /*-------------------------------- Global Data 
-------------------------------*/
 /* Xen init_ops implementation. */
 struct init_ops xen_init_ops = {
@@ -78,6 +100,74 @@ static struct
 
 static struct bios_smap xen_smap[MAX_E820_ENTRIES];
 
+static int
+start_xen_ap(int cpu)
+{
+       struct vcpu_guest_context *ctxt;
+       int ms, cpus = mp_naps;
+
+       ctxt = malloc(sizeof(*ctxt), M_TEMP, M_NOWAIT | M_ZERO);
+       if (ctxt == NULL)
+               panic("unable to allocate memory");
+
+       ctxt->flags = VGCF_IN_KERNEL;
+       ctxt->user_regs.rip = (unsigned long) init_secondary;
+       ctxt->user_regs.rsp = (unsigned long) bootSTK;
+
+       /* Set the AP to use the same page tables */
+       ctxt->ctrlreg[3] = KPML4phys;
+
+       if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
+               panic("unable to initialize AP#%d\n", cpu);
+
+       free(ctxt, M_TEMP);
+
+       /* Launch the vCPU */
+       if (HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL))
+               panic("unable to start AP#%d\n", cpu);
+
+       /* Wait up to 5 seconds for it to start. */
+       for (ms = 0; ms < 5000; ms++) {
+               if (mp_naps > cpus)
+                       return (1);     /* return SUCCESS */
+               DELAY(1000);
+       }
+
+       return (0);
+}
+
+int
+xen_pv_start_all_aps(void)
+{
+       int cpu;
+
+       mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
+
+       for (cpu = 1; cpu < mp_ncpus; cpu++) {
+
+               /* allocate and set up an idle stack data page */
+               bootstacks[cpu] = (void *)kmem_malloc(kernel_arena,
+                   KSTACK_PAGES * PAGE_SIZE, M_WAITOK | M_ZERO);
+               doublefault_stack = (char *)kmem_malloc(kernel_arena,
+                   PAGE_SIZE, M_WAITOK | M_ZERO);
+               nmi_stack = (char *)kmem_malloc(kernel_arena, PAGE_SIZE,
+                   M_WAITOK | M_ZERO);
+               dpcpu = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE,
+                   M_WAITOK | M_ZERO);
+
+               bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 
8;
+               bootAP = cpu;
+
+               /* attempt to start the Application Processor */
+               if (!start_xen_ap(cpu))
+                       panic("AP #%d failed to start!", cpu);
+
+               CPU_SET(cpu, &all_cpus);        /* record AP in CPU map */
+       }
+
+       return (mp_naps);
+}
+
 /*
  * Functions to convert the "extra" parameters passed by Xen
  * into FreeBSD boot options (from the i386 Xen port).
diff --git a/sys/xen/pv.h b/sys/xen/pv.h
index 71b8776..60d9def 100644
--- a/sys/xen/pv.h
+++ b/sys/xen/pv.h
@@ -24,5 +24,6 @@
 #define        __XEN_PV_H__
 
 void   xen_pv_set_init_ops(void);
+int    xen_pv_start_all_aps(void);
 
 #endif /* __XEN_PV_H__ */
-- 
1.7.7.5 (Apple Git-26)

_______________________________________________
freebsd-current@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-current
To unsubscribe, send any mail to "freebsd-current-unsubscr...@freebsd.org"

Reply via email to