--- i386/i386/gdt.c | 63 ++++++++++++++++++++++++++++++++++------- i386/i386/gdt.h | 1 + i386/i386/idt-gen.h | 4 +++ i386/i386/idt.c | 40 ++++++++++++++++++++++++-- i386/i386/ktss.c | 36 +++++++++++++++++++++++ i386/i386/ktss.h | 1 + i386/i386/ldt.c | 39 +++++++++++++++++++++++++ i386/i386/ldt.h | 9 ++++++ i386/i386/locore.S | 4 +++ i386/i386at/idt.h | 4 +++ i386/i386at/int_init.c | 26 ++++++++++++++++- i386/i386at/int_init.h | 1 + i386/i386at/interrupt.S | 6 ++++ 13 files changed, 220 insertions(+), 14 deletions(-)
diff --git a/i386/i386/gdt.c b/i386/i386/gdt.c index fb18360e..44bcd29c 100644 --- a/i386/i386/gdt.c +++ b/i386/i386/gdt.c @@ -39,6 +39,7 @@ #include "vm_param.h" #include "seg.h" #include "gdt.h" +#include "mp_desc.h" #ifdef MACH_PV_DESCRIPTORS /* It is actually defined in xen_boothdr.S */ @@ -46,28 +47,28 @@ extern #endif /* MACH_PV_DESCRIPTORS */ struct real_descriptor gdt[GDTSZ]; -void -gdt_init(void) +static void +gdt_fill(struct real_descriptor *mygdt) { /* Initialize the kernel code and data segment descriptors. */ #ifdef __x86_64__ assert(LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS == 0); - fill_gdt_descriptor(KERNEL_CS, 0, 0, ACC_PL_K|ACC_CODE_R, SZ_64); - fill_gdt_descriptor(KERNEL_DS, 0, 0, ACC_PL_K|ACC_DATA_W, SZ_64); + _fill_gdt_descriptor(mygdt, KERNEL_CS, 0, 0, ACC_PL_K|ACC_CODE_R, SZ_64); + _fill_gdt_descriptor(mygdt, KERNEL_DS, 0, 0, ACC_PL_K|ACC_DATA_W, SZ_64); #ifndef MACH_PV_DESCRIPTORS - fill_gdt_descriptor(LINEAR_DS, 0, 0, ACC_PL_K|ACC_DATA_W, SZ_64); + _fill_gdt_descriptor(mygdt, LINEAR_DS, 0, 0, ACC_PL_K|ACC_DATA_W, SZ_64); #endif /* MACH_PV_DESCRIPTORS */ #else - fill_gdt_descriptor(KERNEL_CS, + _fill_gdt_descriptor(mygdt, KERNEL_CS, LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS, LINEAR_MAX_KERNEL_ADDRESS - (LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) - 1, ACC_PL_K|ACC_CODE_R, SZ_32); - fill_gdt_descriptor(KERNEL_DS, + _fill_gdt_descriptor(mygdt, KERNEL_DS, LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS, LINEAR_MAX_KERNEL_ADDRESS - (LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) - 1, ACC_PL_K|ACC_DATA_W, SZ_32); #ifndef MACH_PV_DESCRIPTORS - fill_gdt_descriptor(LINEAR_DS, + _fill_gdt_descriptor(mygdt, LINEAR_DS, 0, 0xffffffff, ACC_PL_K|ACC_DATA_W, SZ_32); @@ -75,8 +76,8 @@ gdt_init(void) #endif #ifdef MACH_PV_DESCRIPTORS - unsigned long frame = kv_to_mfn(gdt); - pmap_set_page_readonly(gdt); + unsigned long frame = kv_to_mfn(mygdt); + pmap_set_page_readonly(mygdt); if (hyp_set_gdt(kv_to_la(&frame), GDTSZ)) panic("couldn't set gdt\n"); #endif @@ -88,6 +89,12 @@ gdt_init(void) panic("couldn't set 4gb segments vm assist notify"); #endif #endif /* MACH_PV_PAGETABLES */ +} + +void +gdt_init(void) +{ + gdt_fill(gdt); #ifndef MACH_PV_DESCRIPTORS /* Load the new GDT. */ @@ -128,3 +135,39 @@ gdt_init(void) #endif /* MACH_PV_PAGETABLES */ } +#if NCPUS > 1 +void +ap_gdt_init(int cpu) +{ + gdt_fill(mp_gdt[cpu]); + +#ifndef MACH_PV_DESCRIPTORS + /* Load the new GDT. */ + { + struct pseudo_descriptor pdesc; + + pdesc.limit = sizeof(gdt)-1; + pdesc.linear_base = kvtolin(mp_gdt[cpu]); + lgdt(&pdesc); + } +#endif /* MACH_PV_DESCRIPTORS */ + + /* Reload all the segment registers from the new GDT. + We must load ds and es with 0 before loading them with KERNEL_DS + because some processors will "optimize out" the loads + if the previous selector values happen to be the same. */ +#ifndef __x86_64__ + asm volatile("ljmp %0,$1f\n" + "1:\n" + "movw %w2,%%ds\n" + "movw %w2,%%es\n" + "movw %w2,%%fs\n" + "movw %w2,%%gs\n" + + "movw %w1,%%ds\n" + "movw %w1,%%es\n" + "movw %w1,%%ss\n" + : : "i" (KERNEL_CS), "r" (KERNEL_DS), "r" (0)); +#endif +} +#endif diff --git a/i386/i386/gdt.h b/i386/i386/gdt.h index 9879ad3e..5def73cb 100644 --- a/i386/i386/gdt.h +++ b/i386/i386/gdt.h @@ -115,5 +115,6 @@ extern struct real_descriptor gdt[GDTSZ]; #endif extern void gdt_init(void); +extern void ap_gdt_init(int cpu); #endif /* _I386_GDT_ */ diff --git a/i386/i386/idt-gen.h b/i386/i386/idt-gen.h index f86afb41..a94b39c0 100644 --- a/i386/i386/idt-gen.h +++ b/i386/i386/idt-gen.h @@ -44,4 +44,8 @@ extern struct real_gate idt[IDTSZ]; #define fill_idt_gate(int_num, entry, selector, access, dword_count) \ fill_gate(&idt[int_num], entry, selector, access, dword_count) +/* Fill a gate in a custom IDT. */ +#define _fill_idt_gate(_idt, int_num, entry, selector, access, dword_count) \ + fill_gate(&_idt[int_num], entry, selector, access, dword_count) + #endif /* _I386_IDT_ */ diff --git a/i386/i386/idt.c b/i386/i386/idt.c index c6a778f1..8513e158 100644 --- a/i386/i386/idt.c +++ b/i386/i386/idt.c @@ -25,6 +25,7 @@ #include <i386/seg.h> #include <i386at/idt.h> #include <i386/gdt.h> +#include <i386/mp_desc.h> struct real_gate idt[IDTSZ]; @@ -36,7 +37,7 @@ struct idt_init_entry }; extern struct idt_init_entry idt_inittab[]; -void idt_init(void) +static void idt_fill(void) { #ifdef MACH_PV_DESCRIPTORS if (hyp_set_trap_table(kvtolin(idt_inittab))) @@ -50,8 +51,14 @@ void idt_init(void) fill_idt_gate(iie->vector, iie->entrypoint, KERNEL_CS, iie->type, 0); iie++; } +#endif /* MACH_PV_DESCRIPTORS */ +} - /* Load the IDT pointer into the processor. */ +void idt_init(void) +{ + idt_fill(); + + /* Load the IDT pointer into the BSP */ { struct pseudo_descriptor pdesc; @@ -59,6 +66,33 @@ void idt_init(void) pdesc.linear_base = kvtolin(&idt); lidt(&pdesc); } -#endif /* MACH_PV_DESCRIPTORS */ } +#if NCPUS > 1 +void ap_idt_init(int cpu) +{ +#ifdef MACH_PV_DESCRIPTORS + if (hyp_set_trap_table(kvtolin(idt_inittab))) + panic("couldn't set trap table\n"); +#else /* MACH_PV_DESCRIPTORS */ + struct idt_init_entry *iie = idt_inittab; + + /* Initialize the exception vectors from the idt_inittab. */ + while (iie->entrypoint) + { + _fill_idt_gate(mp_desc_table[cpu]->idt, iie->vector, iie->entrypoint, + KERNEL_CS, iie->type, 0); + iie++; + } +#endif /* MACH_PV_DESCRIPTORS */ + + /* Load the IDT pointer into the AP */ + { + struct pseudo_descriptor pdesc; + + pdesc.limit = sizeof(mp_desc_table[cpu]->idt)-1; + pdesc.linear_base = kvtolin(&mp_desc_table[cpu]->idt); + lidt(&pdesc); + } +} +#endif diff --git a/i386/i386/ktss.c b/i386/i386/ktss.c index 0d21d3eb..33b084b7 100644 --- a/i386/i386/ktss.c +++ b/i386/i386/ktss.c @@ -35,6 +35,7 @@ #include "seg.h" #include "gdt.h" #include "ktss.h" +#include "mp_desc.h" /* A kernel TSS with a complete I/O bitmap. */ struct task_tss ktss; @@ -73,3 +74,38 @@ ktss_init(void) #endif /* MACH_RING1 */ } +#if NCPUS > 1 +void +ap_ktss_init(int cpu) +{ + /* XXX temporary exception stack */ + static int exception_stack[1024]; + +#ifdef MACH_RING1 + /* Xen won't allow us to do any I/O by default anyway, just register + * exception stack */ + if (hyp_stack_switch(KERNEL_DS, (unsigned long)(exception_stack+1024))) + panic("couldn't register exception stack\n"); +#else /* MACH_RING1 */ + /* Initialize the master TSS descriptor. */ + _fill_gdt_sys_descriptor(mp_gdt[cpu], KERNEL_TSS, + kvtolin(&mp_desc_table[cpu]->ktss), sizeof(struct task_tss) - 1, + ACC_PL_K|ACC_TSS, 0); + + /* Initialize the master TSS. */ +#ifdef __x86_64__ + ktss.tss.rsp0 = (unsigned long)(exception_stack+1024); +#else /* ! __x86_64__ */ + ktss.tss.ss0 = KERNEL_DS; + ktss.tss.esp0 = (unsigned long)(exception_stack+1024); +#endif /* __x86_64__ */ + + ktss.tss.io_bit_map_offset = IOPB_INVAL; + /* Set the last byte in the I/O bitmap to all 1's. */ + ktss.barrier = 0xff; + + /* Load the TSS. */ + ltr(KERNEL_TSS); +#endif /* MACH_RING1 */ +} +#endif diff --git a/i386/i386/ktss.h b/i386/i386/ktss.h index 304a877a..171332da 100644 --- a/i386/i386/ktss.h +++ b/i386/i386/ktss.h @@ -28,5 +28,6 @@ extern struct task_tss ktss; extern void ktss_init(void); +extern void ap_ktss_init(int cpu); #endif /* _I386_KTSS_ */ diff --git a/i386/i386/ldt.c b/i386/i386/ldt.c index 261df93a..4bbc8e80 100644 --- a/i386/i386/ldt.c +++ b/i386/i386/ldt.c @@ -37,6 +37,7 @@ #include "gdt.h" #include "ldt.h" #include "locore.h" +#include "mp_desc.h" #ifdef MACH_PV_DESCRIPTORS /* It is actually defined in xen_boothdr.S */ @@ -79,3 +80,41 @@ ldt_init(void) lldt(KERNEL_LDT); #endif /* MACH_PV_DESCRIPTORS */ } + +#if NCPUS > 1 +void +ap_ldt_init(int cpu) +{ +#ifdef MACH_PV_DESCRIPTORS +#ifdef MACH_PV_PAGETABLES + pmap_set_page_readwrite(mp_desc_table[cpu]->ldt); +#endif /* MACH_PV_PAGETABLES */ +#else /* MACH_PV_DESCRIPTORS */ + /* Initialize the master LDT descriptor in the GDT. */ + _fill_gdt_sys_descriptor(mp_gdt[cpu], KERNEL_LDT, + kvtolin(&mp_desc_table[cpu]->ldt), sizeof(mp_desc_table[cpu]->ldt)-1, + ACC_PL_K|ACC_LDT, 0); +#endif /* MACH_PV_DESCRIPTORS */ + + /* Initialize the 32bit LDT descriptors. */ + _fill_ldt_gate(mp_desc_table[cpu]->ldt, USER_SCALL, + (vm_offset_t)&syscall, KERNEL_CS, + ACC_PL_U|ACC_CALL_GATE, 0); + _fill_ldt_descriptor(mp_desc_table[cpu]->ldt, USER_CS, + VM_MIN_ADDRESS, + VM_MAX_ADDRESS-VM_MIN_ADDRESS-4096, + /* XXX LINEAR_... */ + ACC_PL_U|ACC_CODE_R, SZ_32); + _fill_ldt_descriptor(mp_desc_table[cpu]->ldt, USER_DS, + VM_MIN_ADDRESS, + VM_MAX_ADDRESS-VM_MIN_ADDRESS-4096, + ACC_PL_U|ACC_DATA_W, SZ_32); + + /* Activate the LDT. */ +#ifdef MACH_PV_DESCRIPTORS + hyp_set_ldt(&mp_desc_table[cpu]->ldt, LDTSZ); +#else /* MACH_PV_DESCRIPTORS */ + lldt(KERNEL_LDT); +#endif /* MACH_PV_DESCRIPTORS */ +} +#endif diff --git a/i386/i386/ldt.h b/i386/i386/ldt.h index 1f0d7014..55edc396 100644 --- a/i386/i386/ldt.h +++ b/i386/i386/ldt.h @@ -64,7 +64,16 @@ extern struct real_descriptor ldt[LDTSZ]; fill_gate((struct real_gate*)&ldt[sel_idx(selector)], \ offset, dest_selector, access, word_count) +/* Fill a 32bit segment descriptor in a custom LDT. */ +#define _fill_ldt_descriptor(_ldt, selector, base, limit, access, sizebits) \ + fill_descriptor(&_ldt[sel_idx(selector)], base, limit, access, sizebits) + +#define _fill_ldt_gate(_ldt, selector, offset, dest_selector, access, word_count) \ + fill_gate((struct real_gate*)&_ldt[sel_idx(selector)], \ + offset, dest_selector, access, word_count) + void ldt_init(void); +void ap_ldt_init(int cpu); #endif /* !__ASSEMBLER__ */ diff --git a/i386/i386/locore.S b/i386/i386/locore.S index 162bb13a..8c2f57ea 100644 --- a/i386/i386/locore.S +++ b/i386/i386/locore.S @@ -649,6 +649,10 @@ INTERRUPT(20) INTERRUPT(21) INTERRUPT(22) INTERRUPT(23) +#endif +/* Invalidate TLB IPI to call pmap_update_interrupt() on a specific cpu */ +INTERRUPT(251) +#ifdef APIC /* Spurious interrupt, set irq number to vect number */ INTERRUPT(255) #endif diff --git a/i386/i386at/idt.h b/i386/i386at/idt.h index ac065aef..f080bb12 100644 --- a/i386/i386at/idt.h +++ b/i386/i386at/idt.h @@ -37,10 +37,14 @@ /* IOAPIC spurious interrupt vector set to 0xff */ #define IOAPIC_SPURIOUS_BASE 0xff +/* Currently for TLB shootdowns */ +#define CALL_SINGLE_FUNCTION_BASE 0xfb + #include <i386/idt-gen.h> #ifndef __ASSEMBLER__ extern void idt_init (void); +extern void ap_idt_init (int cpu); #endif /* __ASSEMBLER__ */ #endif /* _I386AT_IDT_ */ diff --git a/i386/i386at/int_init.c b/i386/i386at/int_init.c index 6da627dd..6c242557 100644 --- a/i386/i386at/int_init.c +++ b/i386/i386at/int_init.c @@ -23,6 +23,7 @@ #include <i386at/idt.h> #include <i386/gdt.h> +#include <i386/mp_desc.h> /* defined in locore.S */ extern vm_offset_t int_entry_table[]; @@ -36,15 +37,38 @@ void int_init(void) int_entry_table[i], KERNEL_CS, ACC_PL_K|ACC_INTR_GATE, 0); } + fill_idt_gate(CALL_SINGLE_FUNCTION_BASE, + int_entry_table[16], KERNEL_CS, + ACC_PL_K|ACC_INTR_GATE, 0); #else for (i = 0; i < 24; i++) { fill_idt_gate(IOAPIC_INT_BASE + i, int_entry_table[i], KERNEL_CS, ACC_PL_K|ACC_INTR_GATE, 0); } - fill_idt_gate(IOAPIC_SPURIOUS_BASE, + fill_idt_gate(CALL_SINGLE_FUNCTION_BASE, int_entry_table[24], KERNEL_CS, ACC_PL_K|ACC_INTR_GATE, 0); + fill_idt_gate(IOAPIC_SPURIOUS_BASE, + int_entry_table[25], KERNEL_CS, + ACC_PL_K|ACC_INTR_GATE, 0); #endif } +#if NCPUS > 1 +void ap_int_init(int cpu) +{ +#ifndef APIC + _fill_idt_gate(mp_desc_table[cpu]->idt, CALL_SINGLE_FUNCTION_BASE, + int_entry_table[16], KERNEL_CS, + ACC_PL_K|ACC_INTR_GATE, 0); +#else + _fill_idt_gate(mp_desc_table[cpu]->idt, CALL_SINGLE_FUNCTION_BASE, + int_entry_table[24], KERNEL_CS, + ACC_PL_K|ACC_INTR_GATE, 0); + _fill_idt_gate(mp_desc_table[cpu]->idt, IOAPIC_SPURIOUS_BASE, + int_entry_table[25], KERNEL_CS, + ACC_PL_K|ACC_INTR_GATE, 0); +#endif +} +#endif diff --git a/i386/i386at/int_init.h b/i386/i386at/int_init.h index f9b03b74..3c11ebce 100644 --- a/i386/i386at/int_init.h +++ b/i386/i386at/int_init.h @@ -29,6 +29,7 @@ #ifndef __ASSEMBLER__ extern void int_init (void); +extern void ap_int_init (int cpu); #endif /* __ASSEMBLER__ */ #endif /* _INT_INIT_H_ */ diff --git a/i386/i386at/interrupt.S b/i386/i386at/interrupt.S index 8fd18392..a5aac966 100644 --- a/i386/i386at/interrupt.S +++ b/i386/i386at/interrupt.S @@ -41,6 +41,9 @@ ENTRY(interrupt) cmpl $255,%eax /* was this a spurious intr? */ je _no_eoi /* if so, just return */ #endif + cmpl $251,%eax /* was this a SMP call single function request? */ + je _call_single + subl $28,%esp /* Two local variables + 5 parameters */ movl %eax,S_IRQ /* save irq number */ call spl7 /* set ipl */ @@ -118,4 +121,7 @@ _isa_eoi: addl $28,%esp /* pop local variables */ _no_eoi: ret +_call_single: + call EXT(pmap_update_interrupt) /* TODO: Allow other functions */ + ret END(interrupt) -- 2.34.1