Okay, enough cleanup; time to simplify the TSS handling by moving from
TSS-per-process to TSS-per-CPU. This eliminates the need to ever change
GDT entries after startup and the associated ~4k process limit.
Where do we place that TSS? Before, it was part of the PCB; here I've put
the primary CPU's TSS between the IDT and the primary CPU's GDT. For
secondary CPU's I place it right after the CPU's GDT. The exact location
isn't important, they just need to be in unpageable, unmanaged memory like
the GDT, so putting it next to the GDT is convenient.
With the TSS being per-CPU, we no longer need to change the 'task busy'
flag or load the task register when doing a context switch. Instead, we
just poke into the TSS the pointer to the process's kernel stack.
The TSS also contains the pointer to the stack used for double faults.
Previously, this was placed one page above each process's PCB. Rather
than change that on each context switch, I've just left it set to one page
above each CPU's idle process's PCB. I think we can reduce the size of
normal processes' USPACE by a page, or maybe even two, what with the PCB
shrinking, but that can wait.
To increase speed and paranoia slightly, I've inlined pmap_activate() and
pmap_deactivate() into the asm cpu_switchto routine. That let me add a
'free' check for the new pmap already being marked as active on the CPU;
that check earlier caught some sloppy handling of that bitmap.
Garbage collect the hasn't-been-used-in-years GDT update IPI.
As before, I would like to hear tests for both real AMD and Intel CPUs;
thank you to everyone that sent results before!
Philip Guenther
Index: include/cpu.h
===================================================================
RCS file: /cvs/src/sys/arch/amd64/include/cpu.h,v
retrieving revision 1.58
diff -u -p -r1.58 cpu.h
--- include/cpu.h 2 Oct 2010 23:13:27 -0000 1.58
+++ include/cpu.h 31 Oct 2010 23:03:59 -0000
@@ -43,7 +43,6 @@
*/
#include <machine/frame.h>
#include <machine/segments.h>
-#include <machine/tss.h>
#include <machine/intrdefs.h>
#include <machine/cacheinfo.h>
@@ -59,6 +58,7 @@
#ifdef _KERNEL
+struct x86_64_tss;
struct cpu_info {
struct device *ci_dev;
struct cpu_info *ci_self;
@@ -79,7 +79,6 @@ struct cpu_info {
struct pcb *ci_curpcb;
struct pcb *ci_idle_pcb;
- int ci_idle_tss_sel;
struct intrsource *ci_isources[MAX_INTR_SOURCES];
u_int32_t ci_ipending;
@@ -110,6 +109,7 @@ struct cpu_info {
struct x86_cache_info ci_cinfo[CAI_COUNT];
+ struct x86_64_tss *ci_tss;
char *ci_gdt;
volatile int ci_ddb_paused;
@@ -125,10 +125,6 @@ struct cpu_info {
#define CI_SETPERF_INTRANSIT 2
#define CI_SETPERF_DONE 3
- struct x86_64_tss ci_doubleflt_tss;
-
- char *ci_doubleflt_stack;
-
struct ksensordev ci_sensordev;
struct ksensor ci_sensor;
};
@@ -272,7 +268,6 @@ void cpu_probe_features(struct cpu_info
/* machdep.c */
void dumpconf(void);
-int cpu_maxproc(void);
void cpu_reset(void);
void x86_64_proc0_tss_ldt_init(void);
void x86_64_bufinit(void);
Index: include/gdt.h
===================================================================
RCS file: /cvs/src/sys/arch/amd64/include/gdt.h,v
retrieving revision 1.4
diff -u -p -r1.4 gdt.h
--- include/gdt.h 3 Feb 2009 11:24:19 -0000 1.4
+++ include/gdt.h 31 Oct 2010 23:03:59 -0000
@@ -30,29 +30,5 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#ifndef _LOCORE
-struct proc;
-struct pmap;
-
-void gdt_init(void);
void gdt_init_cpu(struct cpu_info *);
-void gdt_reload_cpu(struct cpu_info *);
void gdt_alloc_cpu(struct cpu_info *);
-
-int tss_alloc(struct pcb *);
-void tss_free(int);
-
-void ldt_alloc(struct pmap *, char *, size_t);
-void ldt_free(struct pmap *);
-
-void set_mem_gdt(struct mem_segment_descriptor *, void *, size_t,
- int, int, int, int, int);
-void set_sys_gdt(struct sys_segment_descriptor *, void *, size_t, int, int,
- int);
-#endif
-
-/*
- * Maximum GDT size. It cannot exceed 65536 since the selector field of
- * a descriptor is just 16 bits, and used as free list link.
- */
-#define MAXGDTSIZ 65536
Index: include/intrdefs.h
===================================================================
RCS file: /cvs/src/sys/arch/amd64/include/intrdefs.h,v
retrieving revision 1.7
diff -u -p -r1.7 intrdefs.h
--- include/intrdefs.h 22 May 2010 21:31:05 -0000 1.7
+++ include/intrdefs.h 31 Oct 2010 23:03:59 -0000
@@ -75,16 +75,14 @@
#define X86_IPI_SYNCH_FPU 0x00000008
#define X86_IPI_TLB 0x00000010
#define X86_IPI_MTRR 0x00000020
-#define X86_IPI_GDT 0x00000040
+#define X86_IPI_SETPERF 0x00000040
#define X86_IPI_DDB 0x00000080
-#define X86_IPI_SETPERF 0x00000100
-#define X86_NIPI 9
+#define X86_NIPI 8
#define X86_IPI_NAMES { "halt IPI", "nop IPI", "FPU flush IPI", \
"FPU synch IPI", "TLB shootdown IPI", \
- "MTRR update IPI", "GDT update IPI", "ddb IPI", \
- "setperf IPI"}
+ "MTRR update IPI", "setperf IPI", "ddb IPI" }
#define IREENT_MAGIC 0x18041969
Index: include/pcb.h
===================================================================
RCS file: /cvs/src/sys/arch/amd64/include/pcb.h,v
retrieving revision 1.6
diff -u -p -r1.6 pcb.h
--- include/pcb.h 26 Oct 2010 05:49:10 -0000 1.6
+++ include/pcb.h 31 Oct 2010 23:04:00 -0000
@@ -73,36 +73,23 @@
#include <sys/signal.h>
-#include <machine/segments.h>
#include <machine/tss.h>
#include <machine/fpu.h>
-#include <machine/sysarch.h>
-
-#define NIOPORTS 1024 /* # of ports we allow to be
mapped */
/*
* Please note that the pcb_savefpu field in struct below must be
* on a 16-byte boundary.
*/
struct pcb {
- /*
- * XXXfvdl
- * It's overkill to have a TSS here, as it's only needed
- * for compatibility processes who use an I/O permission map.
- * The pcb fields below are not in the TSS anymore (and there's
- * not enough room in the TSS to store them all)
- * Should just make this a pointer and allocate.
- */
- struct x86_64_tss pcb_tss;
- u_int64_t pcb_cr3;
- u_int64_t pcb_rsp;
- u_int64_t pcb_rbp;
struct savefpu pcb_savefpu; /* floating point state */
- int pcb_cr0; /* saved image of CR0 */
+ u_int64_t pcb_cr3;
+ u_int64_t pcb_rsp;
+ u_int64_t pcb_rbp;
+ u_int64_t pcb_kstack; /* kernel stack address */
caddr_t pcb_onfault; /* copyin/out fault recovery */
- struct cpu_info *pcb_fpcpu; /* cpu holding our fp state. */
- unsigned pcb_iomap[NIOPORTS/32]; /* I/O bitmap */
- struct pmap *pcb_pmap; /* back pointer to our pmap */
+ struct cpu_info *pcb_fpcpu; /* cpu holding our fp state. */
+ struct pmap *pcb_pmap; /* back pointer to our pmap */
+ int pcb_cr0; /* saved image of CR0 */
};
/*
Index: include/proc.h
===================================================================
RCS file: /cvs/src/sys/arch/amd64/include/proc.h,v
retrieving revision 1.3
diff -u -p -r1.3 proc.h
--- include/proc.h 13 Dec 2005 00:18:19 -0000 1.3
+++ include/proc.h 31 Oct 2010 23:04:00 -0000
@@ -42,7 +42,6 @@
*/
struct mdproc {
struct trapframe *md_regs; /* registers on current frame */
- int md_tss_sel; /* TSS selector */
int md_flags;
__volatile int md_astpending;
};
Index: include/segments.h
===================================================================
RCS file: /cvs/src/sys/arch/amd64/include/segments.h,v
retrieving revision 1.7
diff -u -p -r1.7 segments.h
--- include/segments.h 26 Oct 2010 05:49:10 -0000 1.7
+++ include/segments.h 31 Oct 2010 23:04:03 -0000
@@ -61,16 +61,14 @@
#define ISLDT(s) ((s) & SEL_LDT) /* is it local or global */
#define SEL_LDT 4 /* local descriptor table */
-/* Dynamically allocated TSSs start (byte offset) */
#define SYSSEL_START (NGDT_MEM << 3)
-#define DYNSEL_START (SYSSEL_START + (NGDT_SYS << 4))
+#define GDT_SIZE (SYSSEL_START + (NGDT_SYS << 4))
/*
* These define the index not from the start of the GDT, but from
* the part of the GDT that they're allocated from.
* First NGDT_MEM entries are 8-byte descriptors for CS and DS.
- *
- * The rest is 16-byte descriptors for TSSs
+ * Next NGDT_SYS entries are 16-byte descriptors defining TSSs.
*/
#define IDXSEL(s) (((s) >> 3) & 0x1fff)
@@ -249,7 +247,7 @@ void cpu_init_idt(void);
* The code and data descriptors must come first. There
* are NGDT_MEM of them.
*
- * Then come the predefined LDT (and possibly TSS) descriptors.
+ * Then comes the predefined TSS descriptor.
* There are NGDT_SYS of them.
*
* The particular order of the UCODE32, UDATA, and UCODE descriptors is
@@ -263,7 +261,8 @@ void cpu_init_idt(void);
#define GUCODE_SEL 5 /* User code descriptor */
#define NGDT_MEM 6
-#define NGDT_SYS 0
+#define GPROC0_SEL 0 /* common TSS */
+#define NGDT_SYS 1
#define GDT_SYS_OFFSET (NGDT_MEM << 3)
Index: include/tss.h
===================================================================
RCS file: /cvs/src/sys/arch/amd64/include/tss.h,v
retrieving revision 1.1
diff -u -p -r1.1 tss.h
--- include/tss.h 28 Jan 2004 01:39:39 -0000 1.1
+++ include/tss.h 31 Oct 2010 23:04:03 -0000
@@ -49,7 +49,7 @@ struct x86_64_tss {
u_int32_t tss_reserved1;
u_int64_t tss_rsp0;
u_int64_t tss_rsp1;
- u_int64_t tss_rsp3;
+ u_int64_t tss_rsp2;
u_int32_t tss_reserved2;
u_int32_t tss_reserved3;
u_int64_t tss_ist[7];
Index: amd64/acpi_machdep.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/acpi_machdep.c,v
retrieving revision 1.46
diff -u -p -r1.46 acpi_machdep.c
--- amd64/acpi_machdep.c 6 Oct 2010 16:37:29 -0000 1.46
+++ amd64/acpi_machdep.c 31 Oct 2010 23:04:03 -0000
@@ -303,7 +303,7 @@ acpi_resume_machdep(void)
p = ci->ci_schedstate.spc_idleproc;
pcb = &p->p_addr->u_pcb;
- tf = (struct trapframe *)pcb->pcb_tss.tss_rsp0 - 1;
+ tf = (struct trapframe *)pcb->pcb_kstack - 1;
sf = (struct switchframe *)tf - 1;
sf->sf_r12 = (u_int64_t)sched_idle;
sf->sf_r13 = (u_int64_t)ci;
Index: amd64/cpu.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/cpu.c,v
retrieving revision 1.37
diff -u -p -r1.37 cpu.c
--- amd64/cpu.c 26 Oct 2010 05:49:10 -0000 1.37
+++ amd64/cpu.c 31 Oct 2010 23:04:08 -0000
@@ -280,9 +280,8 @@ cpu_attach(struct device *parent, struct
pcb = ci->ci_idle_pcb = (struct pcb *) kstack;
memset(pcb, 0, USPACE);
- pcb->pcb_tss.tss_rsp0 = kstack + USPACE - 16;
+ pcb->pcb_kstack = kstack + USPACE - 16;
pcb->pcb_rbp = pcb->pcb_rsp = kstack + USPACE - 16;
- pcb->pcb_tss.tss_ist[0] = kstack + PAGE_SIZE - 16;
pcb->pcb_pmap = pmap_kernel();
pcb->pcb_cr0 = rcr0();
pcb->pcb_cr3 = pcb->pcb_pmap->pm_pdirpa;
Index: amd64/gdt.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/gdt.c,v
retrieving revision 1.17
diff -u -p -r1.17 gdt.c
--- amd64/gdt.c 26 Oct 2010 05:49:10 -0000 1.17
+++ amd64/gdt.c 31 Oct 2010 23:04:10 -0000
@@ -33,8 +33,6 @@
/*
* Modified to deal with variable-length entries for amd64 by
* [email protected], may 2001
- * XXX this file should be shared with the i386 code, the difference
- * can be hidden in macros.
*/
#include <sys/param.h>
@@ -45,90 +43,9 @@
#include <uvm/uvm.h>
#include <machine/gdt.h>
+#include <machine/tss.h>
#include <machine/pcb.h>
-int gdt_next; /* next available slot for sweeping */
-int gdt_free; /* next free slot; terminated with GNULL_SEL */
-
-struct mutex gdt_lock_store = MUTEX_INITIALIZER(IPL_HIGH);
-
-void gdt_init(void);
-int gdt_get_slot(void);
-void gdt_put_slot(int);
-
-/*
- * Lock and unlock the GDT.
- */
-#define gdt_lock() (mtx_enter(&gdt_lock_store))
-#define gdt_unlock() (mtx_leave(&gdt_lock_store))
-
-void
-set_mem_gdt(struct mem_segment_descriptor *sd, void *base, size_t limit,
- int type, int dpl, int gran, int def32, int is64)
-{
- CPU_INFO_ITERATOR cii;
- struct cpu_info *ci;
- int off;
-
- set_mem_segment(sd, base, limit, type, dpl, gran, def32, is64);
- off = (char *)sd - gdtstore;
- CPU_INFO_FOREACH(cii, ci) {
- if (ci->ci_gdt != NULL)
- *(struct mem_segment_descriptor *)(ci->ci_gdt + off) =
- *sd;
- }
-}
-
-void
-set_sys_gdt(struct sys_segment_descriptor *sd, void *base, size_t limit,
- int type, int dpl, int gran)
-{
- CPU_INFO_ITERATOR cii;
- struct cpu_info *ci;
- int off;
-
- set_sys_segment(sd, base, limit, type, dpl, gran);
- off = (char *)sd - gdtstore;
- CPU_INFO_FOREACH(cii, ci) {
- if (ci->ci_gdt != NULL)
- *(struct sys_segment_descriptor *)(ci->ci_gdt + off) =
- *sd;
- }
-}
-
-
-/*
- * Initialize the GDT.
- */
-void
-gdt_init(void)
-{
- char *old_gdt;
- struct vm_page *pg;
- vaddr_t va;
- struct cpu_info *ci = &cpu_info_primary;
-
- gdt_next = 0;
- gdt_free = GNULL_SEL;
-
- old_gdt = gdtstore;
- gdtstore = (char *)uvm_km_valloc(kernel_map, MAXGDTSIZ);
- for (va = (vaddr_t)gdtstore; va < (vaddr_t)gdtstore + MAXGDTSIZ;
- va += PAGE_SIZE) {
- pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
- if (pg == NULL) {
- panic("gdt_init: no pages");
- }
- pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
- VM_PROT_READ | VM_PROT_WRITE);
- }
- bcopy(old_gdt, gdtstore, DYNSEL_START);
- ci->ci_gdt = gdtstore;
-
- gdt_init_cpu(ci);
-}
-
-#ifdef MULTIPROCESSOR
/*
* Allocate shadow GDT for a slave cpu.
*/
@@ -138,10 +55,13 @@ gdt_alloc_cpu(struct cpu_info *ci)
struct vm_page *pg;
vaddr_t va;
- ci->ci_gdt = (char *)uvm_km_valloc(kernel_map, MAXGDTSIZ);
+ ci->ci_gdt = (char *)uvm_km_valloc(kernel_map,
+ GDT_SIZE + sizeof(*ci->ci_tss));
+ ci->ci_tss = (void *)(ci->ci_gdt + GDT_SIZE);
uvm_map_pageable(kernel_map, (vaddr_t)ci->ci_gdt,
- (vaddr_t)ci->ci_gdt + MAXGDTSIZ, FALSE, FALSE);
- for (va = (vaddr_t)ci->ci_gdt; va < (vaddr_t)ci->ci_gdt + MAXGDTSIZ;
+ (vaddr_t)ci->ci_gdt + GDT_SIZE, FALSE, FALSE);
+ for (va = (vaddr_t)ci->ci_gdt;
+ va < (vaddr_t)ci->ci_gdt + GDT_SIZE + sizeof(*ci->ci_tss);
va += PAGE_SIZE) {
pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
if (pg == NULL)
@@ -149,10 +69,9 @@ gdt_alloc_cpu(struct cpu_info *ci)
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
}
- bzero(ci->ci_gdt, MAXGDTSIZ);
- bcopy(gdtstore, ci->ci_gdt, MAXGDTSIZ);
+ bcopy(gdtstore, ci->ci_gdt, GDT_SIZE);
+ bzero(ci->ci_tss, sizeof(*ci->ci_tss));
}
-#endif /* MULTIPROCESSOR */
/*
@@ -164,107 +83,10 @@ gdt_init_cpu(struct cpu_info *ci)
{
struct region_descriptor region;
- setregion(®ion, ci->ci_gdt, (u_int16_t)(MAXGDTSIZ - 1));
- lgdt(®ion);
-}
-
-#ifdef MULTIPROCESSOR
+ set_sys_segment(GDT_ADDR_SYS(ci->ci_gdt, GPROC0_SEL), ci->ci_tss,
+ sizeof (struct x86_64_tss)-1, SDT_SYS386TSS, SEL_KPL, 0);
-void
-gdt_reload_cpu(struct cpu_info *ci)
-{
- struct region_descriptor region;
-
- setregion(®ion, ci->ci_gdt, MAXGDTSIZ - 1);
+ setregion(®ion, ci->ci_gdt, GDT_SIZE - 1);
lgdt(®ion);
+ ltr(GSYSSEL(GPROC0_SEL, SEL_KPL));
}
-#endif
-
-/*
- * Allocate a GDT slot as follows:
- * 1) If there are entries on the free list, use those.
- * 2) If there are fewer than MAXGDTSIZ entries in use, there are free slots
- * near the end that we can sweep through.
- */
-int
-gdt_get_slot(void)
-{
- int slot;
- struct sys_segment_descriptor *gdt;
-
- gdt = (struct sys_segment_descriptor *)&gdtstore[DYNSEL_START];
-
- gdt_lock();
-
- if (gdt_free != GNULL_SEL) {
- slot = gdt_free;
- gdt_free = gdt[slot].sd_xx3; /* XXXfvdl res. field abuse */
- } else {
- if (gdt_next >= MAXGDTSIZ)
- panic("gdt_get_slot: out of GDT descriptors");
- slot = gdt_next++;
- }
-
- gdt_unlock();
- return (slot);
-}
-
-/*
- * Deallocate a GDT slot, putting it on the free list.
- */
-void
-gdt_put_slot(int slot)
-{
- struct sys_segment_descriptor *gdt;
-
- gdt = (struct sys_segment_descriptor *)&gdtstore[DYNSEL_START];
-
- gdt_lock();
-
- gdt[slot].sd_type = SDT_SYSNULL;
- gdt[slot].sd_xx3 = gdt_free;
- gdt_free = slot;
-
- gdt_unlock();
-}
-
-int
-tss_alloc(struct pcb *pcb)
-{
- int slot;
- struct sys_segment_descriptor *gdt;
-
- gdt = (struct sys_segment_descriptor *)&gdtstore[DYNSEL_START];
-
- slot = gdt_get_slot();
-#if 0
- printf("tss_alloc: slot %d addr %p\n", slot, &gdt[slot]);
-#endif
- set_sys_gdt(&gdt[slot], &pcb->pcb_tss, sizeof (struct x86_64_tss)-1,
- SDT_SYS386TSS, SEL_KPL, 0);
-#if 0
- printf("lolimit %lx lobase %lx type %lx dpl %lx p %lx hilimit %lx\n"
- "xx1 %lx gran %lx hibase %lx xx2 %lx zero %lx xx3 %lx pad %lx\n",
- (unsigned long)gdt[slot].sd_lolimit,
- (unsigned long)gdt[slot].sd_lobase,
- (unsigned long)gdt[slot].sd_type,
- (unsigned long)gdt[slot].sd_dpl,
- (unsigned long)gdt[slot].sd_p,
- (unsigned long)gdt[slot].sd_hilimit,
- (unsigned long)gdt[slot].sd_xx1,
- (unsigned long)gdt[slot].sd_gran,
- (unsigned long)gdt[slot].sd_hibase,
- (unsigned long)gdt[slot].sd_xx2,
- (unsigned long)gdt[slot].sd_zero,
- (unsigned long)gdt[slot].sd_xx3);
-#endif
- return GDYNSEL(slot, SEL_KPL);
-}
-
-void
-tss_free(int sel)
-{
-
- gdt_put_slot(IDXDYNSEL(sel));
-}
-
Index: amd64/genassym.cf
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/genassym.cf,v
retrieving revision 1.24
diff -u -p -r1.24 genassym.cf
--- amd64/genassym.cf 26 Oct 2010 05:49:10 -0000 1.24
+++ amd64/genassym.cf 31 Oct 2010 23:04:10 -0000
@@ -15,6 +15,7 @@ include <machine/pte.h>
include <machine/vmparam.h>
include <machine/intr.h>
include <machine/pic.h>
+include <machine/tss.h>
include <machine/i82093var.h>
include <machine/mutex.h>
@@ -36,7 +37,6 @@ member p_addr
member p_priority
member p_stat
member p_wchan
-member P_MD_TSS_SEL p_md.md_tss_sel
member P_MD_REGS p_md.md_regs
member P_MD_FLAGS p_md.md_flags
member P_MD_ASTPENDING p_md.md_astpending
@@ -82,22 +82,29 @@ define FRAMESIZE sizeof(s
struct pcb
member pcb_cr3
-member pcb_rbp
member pcb_rsp
-member PCB_RSP0 pcb_tss.tss_rsp0
-member pcb_cr0
+member pcb_rbp
+member pcb_kstack
member pcb_onfault
member pcb_fpcpu
+member pcb_pmap
+member pcb_cr0
+
+struct pmap
+member pm_cpus
+
+struct x86_64_tss
+member tss_rsp0
struct cpu_info
member CPU_INFO_SCRATCH ci_scratch
member CPU_INFO_SELF ci_self
+member CPU_INFO_CPUID ci_cpuid
member CPU_INFO_APICID ci_apicid
member CPU_INFO_RESCHED ci_want_resched
member CPU_INFO_CURPROC ci_curproc
member CPU_INFO_CURPCB ci_curpcb
member CPU_INFO_IDLE_PCB ci_idle_pcb
-member CPU_INFO_IDLE_TSS_SEL ci_idle_tss_sel
member CPU_INFO_ILEVEL ci_ilevel
member CPU_INFO_IDEPTH ci_idepth
member CPU_INFO_ISOURCES ci_isources
@@ -107,6 +114,7 @@ ifdef DIAGNOSTIC
member CPU_INFO_MUTEX_LEVEL ci_mutex_level
endif
member CPU_INFO_GDT ci_gdt
+member CPU_INFO_TSS ci_tss
struct intrsource
member is_recurse
Index: amd64/ipifuncs.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/ipifuncs.c,v
retrieving revision 1.14
diff -u -p -r1.14 ipifuncs.c
--- amd64/ipifuncs.c 2 Oct 2010 23:14:32 -0000 1.14
+++ amd64/ipifuncs.c 31 Oct 2010 23:04:10 -0000
@@ -54,7 +54,6 @@
#include <machine/i82489reg.h>
#include <machine/i82489var.h>
#include <machine/mtrr.h>
-#include <machine/gdt.h>
#include <machine/fpu.h>
#include <ddb/db_output.h>
@@ -80,13 +79,12 @@ void (*ipifunc[X86_NIPI])(struct cpu_inf
x86_64_ipi_synch_fpu,
NULL,
x86_64_ipi_reload_mtrr,
- gdt_reload_cpu,
+ x86_setperf_ipi,
#ifdef DDB
x86_ipi_db,
#else
NULL,
#endif
- x86_setperf_ipi,
};
void
Index: amd64/locore.S
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/locore.S,v
retrieving revision 1.42
diff -u -p -r1.42 locore.S
--- amd64/locore.S 26 Oct 2010 05:49:10 -0000 1.42
+++ amd64/locore.S 31 Oct 2010 23:04:12 -0000
@@ -727,6 +727,7 @@ ENTRY(cpu_switchto)
pushq %r15
movq %rdi, %r13
+ movq %rdi, %r15
movq %rsi, %r12
#ifdef DIAGNOSTIC
@@ -741,6 +742,8 @@ ENTRY(cpu_switchto)
movb $SONPROC,P_STAT(%r12) # p->p_stat = SONPROC
SET_CURPROC(%r12,%rcx)
+ movl CPUVAR(CPUID),%edi
+
/* If old proc exited, don't bother. */
testq %r13,%r13
jz switch_exited
@@ -752,13 +755,16 @@ ENTRY(cpu_switchto)
* %rax, %rcx - scratch
* %r13 - old proc, then old pcb
* %r12 - new proc
+ * %edi - cpuid
*/
- movq %r13,%rdi
- call pmap_deactivate
-
movq P_ADDR(%r13),%r13
+ /* clear the old pmap's bit for the cpu */
+ movq PCB_PMAP(%r13),%rcx
+ lock
+ btcl %edi,PM_CPUS(%rcx)
+
/* Save stack pointers. */
movq %rsp,PCB_RSP(%r13)
movq %rbp,PCB_RBP(%r13)
@@ -781,30 +787,29 @@ switch_exited:
movq PCB_RSP(%r13),%rsp
movq PCB_RBP(%r13),%rbp
-#if 0
+ movq CPUVAR(TSS),%rcx
+ movq PCB_KSTACK(%r13),%rdx
+ movq %rdx,TSS_RSP0(%rcx)
+
+ movq PCB_CR3(%r13),%rax
+ movq %rax,%cr3
+
/* Don't bother with the rest if switching to a system process. */
testl $P_SYSTEM,P_FLAG(%r12)
jnz switch_restored
-#endif
- /* Load TSS info. */
-#ifdef MULTIPROCESSOR
- movq CPUVAR(GDT),%rax
-#else
- movq _C_LABEL(gdtstore)(%rip),%rax
+ /* set the new pmap's bit for the cpu */
+ movl CPUVAR(CPUID),%edi
+ movq PCB_PMAP(%r13),%rcx
+ movl PM_CPUS(%rcx),%eax
+ movq %rax,%r14
+ lock
+ btsl %edi,PM_CPUS(%rcx)
+#ifdef DIAGNOSTIC
+ jc _C_LABEL(switch_pmcpu_set)
#endif
- movl P_MD_TSS_SEL(%r12),%edx
-
- /* Switch TSS. Reset "task busy" flag before */
- andl $~0x0200,4(%rax,%rdx, 1)
- ltr %dx
- movq %r12,%rdi
- call _C_LABEL(pmap_activate)
-
-#if 0
switch_restored:
-#endif
/* Restore cr0 (including FPU state). */
movl PCB_CR0(%r13),%ecx
#ifdef MULTIPROCESSOR
@@ -876,6 +881,11 @@ NENTRY(switch_error3)
call _C_LABEL(panic)
/* NOTREACHED */
1: .asciz "cpu_switch 3"
+NENTRY(switch_pmcpu_set)
+ movabsq $1f,%rdi
+ call _C_LABEL(panic)
+ /* NOTREACHED */
+1: .asciz "activate already active pmap"
#endif /* DIAGNOSTIC */
/*
@@ -900,7 +910,7 @@ IDTVEC(syscall)
swapgs
movq %r15,CPUVAR(SCRATCH)
movq CPUVAR(CURPCB),%r15
- movq PCB_RSP0(%r15),%r15
+ movq PCB_KSTACK(%r15),%r15
xchgq %r15,%rsp
sti
Index: amd64/machdep.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/machdep.c,v
retrieving revision 1.126
diff -u -p -r1.126 machdep.c
--- amd64/machdep.c 26 Oct 2010 05:49:10 -0000 1.126
+++ amd64/machdep.c 31 Oct 2010 23:04:13 -0000
@@ -111,6 +111,7 @@
#include <machine/mpbiosvar.h>
#include <machine/reg.h>
#include <machine/kcore.h>
+#include <machine/tss.h>
#include <dev/isa/isareg.h>
#include <machine/isa_machdep.h>
@@ -360,30 +361,25 @@ cpu_startup(void)
}
/*
- * Set up proc0's TSS
+ * Set up proc0's PCB and the cpu's TSS.
*/
void
x86_64_proc0_tss_ldt_init(void)
{
struct pcb *pcb;
- int x;
-
- gdt_init();
cpu_info_primary.ci_curpcb = pcb = &proc0.p_addr->u_pcb;
-
- pcb->pcb_tss.tss_iobase =
- (u_int16_t)((caddr_t)pcb->pcb_iomap - (caddr_t)&pcb->pcb_tss);
- for (x = 0; x < sizeof(pcb->pcb_iomap) / 4; x++)
- pcb->pcb_iomap[x] = 0xffffffff;
-
pcb->pcb_cr0 = rcr0();
- pcb->pcb_tss.tss_rsp0 = (u_int64_t)proc0.p_addr + USPACE - 16;
- pcb->pcb_tss.tss_ist[0] = (u_int64_t)proc0.p_addr + PAGE_SIZE;
- proc0.p_md.md_regs = (struct trapframe *)pcb->pcb_tss.tss_rsp0 - 1;
- proc0.p_md.md_tss_sel = tss_alloc(pcb);
+ pcb->pcb_kstack = (u_int64_t)proc0.p_addr + USPACE - 16;
+ proc0.p_md.md_regs = (struct trapframe *)pcb->pcb_kstack - 1;
- ltr(proc0.p_md.md_tss_sel);
+ /* an empty iomap, by setting its offset to the TSS limit */
+ cpu_info_primary.ci_tss->tss_iobase = sizeof(struct x86_64_tss);
+ cpu_info_primary.ci_tss->tss_rsp0 = pcb->pcb_kstack;
+ cpu_info_primary.ci_tss->tss_ist[0] =
+ (u_int64_t)proc0.p_addr + PAGE_SIZE - 16;
+
+ ltr(GSYSSEL(GPROC0_SEL, SEL_KPL));
lldt(0);
}
@@ -395,17 +391,13 @@ x86_64_proc0_tss_ldt_init(void)
void
x86_64_init_pcb_tss_ldt(struct cpu_info *ci)
{
- int x;
struct pcb *pcb = ci->ci_idle_pcb;
- pcb->pcb_tss.tss_iobase =
- (u_int16_t)((caddr_t)pcb->pcb_iomap - (caddr_t)&pcb->pcb_tss);
- for (x = 0; x < sizeof(pcb->pcb_iomap) / 4; x++)
- pcb->pcb_iomap[x] = 0xffffffff;
+ ci->ci_tss->tss_iobase = sizeof(*ci->ci_tss);
+ ci->ci_tss->tss_rsp0 = pcb->pcb_kstack;
+ ci->ci_tss->tss_ist[0] = pcb->pcb_kstack - USPACE + PAGE_SIZE;
pcb->pcb_cr0 = rcr0();
-
- ci->ci_idle_tss_sel = tss_alloc(pcb);
}
#endif /* MULTIPROCESSOR */
@@ -1471,7 +1463,8 @@ init_x86_64(paddr_t first_avail)
#endif
idt = (struct gate_descriptor *)idt_vaddr;
- gdtstore = (char *)(idt + NIDT);
+ cpu_info_primary.ci_tss = (void *)(idt + NIDT);
+ gdtstore = (void *)(cpu_info_primary.ci_tss + 1);
/* make gdt gates and memory segments */
set_mem_segment(GDT_ADDR_MEM(gdtstore, GCODE_SEL), 0,
@@ -1489,9 +1482,14 @@ init_x86_64(paddr_t first_avail)
set_mem_segment(GDT_ADDR_MEM(gdtstore, GUCODE_SEL), 0,
atop(VM_MAXUSER_ADDRESS) - 1, SDT_MEMERA, SEL_UPL, 1, 0, 1);
+ set_sys_segment(GDT_ADDR_SYS(gdtstore, GPROC0_SEL),
+ cpu_info_primary.ci_tss, sizeof (struct x86_64_tss)-1,
+ SDT_SYS386TSS, SEL_KPL, 0);
+
/* exceptions */
for (x = 0; x < 32; x++) {
ist = (x == 8) ? 1 : 0;
+ // ist = (x == 8) ? 1 : (x == 2) ? 2 : 0;
setgate(&idt[x], IDTVEC(exceptions)[x], ist, SDT_SYS386IGT,
(x == 3 || x == 4) ? SEL_UPL : SEL_KPL,
GSEL(GCODE_SEL, SEL_KPL));
@@ -1503,7 +1501,7 @@ init_x86_64(paddr_t first_avail)
GSEL(GCODE_SEL, SEL_KPL));
idt_allocmap[128] = 1;
- setregion(®ion, gdtstore, DYNSEL_START - 1);
+ setregion(®ion, gdtstore, GDT_SIZE - 1);
lgdt(®ion);
cpu_init_idt();
@@ -1528,9 +1526,6 @@ init_x86_64(paddr_t first_avail)
}
#endif
- /* Make sure maxproc is sane */
- if (maxproc > cpu_maxproc())
- maxproc = cpu_maxproc();
}
#ifdef KGDB
@@ -1702,15 +1697,6 @@ idt_vec_free(int vec)
unsetgate(&idt[vec]);
idt_allocmap[vec] = 0;
simple_unlock(&idt_lock);
-}
-
-/*
- * Number of processes is limited by number of available GDT slots.
- */
-int
-cpu_maxproc(void)
-{
- return (MAXGDTSIZ - DYNSEL_START) / 16;
}
#ifdef DIAGNOSTIC
Index: amd64/mptramp.S
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/mptramp.S,v
retrieving revision 1.6
diff -u -p -r1.6 mptramp.S
--- amd64/mptramp.S 1 Apr 2010 19:47:59 -0000 1.6
+++ amd64/mptramp.S 31 Oct 2010 23:04:14 -0000
@@ -83,7 +83,6 @@
#include <machine/segments.h>
#include <machine/mpbiosvar.h>
#include <machine/i82489reg.h>
-#include <machine/gdt.h>
#define _RELOC(x) ((x) - KERNBASE)
#define RELOC(x) _RELOC(_C_LABEL(x))
@@ -224,7 +223,7 @@ _C_LABEL(cpu_spinup_trampoline_end): #en
movq PCB_RBP(%rsi),%rbp
movq CPU_INFO_GDT(%rdi),%rax
- movw $(MAXGDTSIZ-1),-10(%rsp)
+ movw $(GDT_SIZE-1),-10(%rsp)
movq %rax,-8(%rsp)
lgdt -10(%rsp)
Index: amd64/pmap.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/pmap.c,v
retrieving revision 1.57
diff -u -p -r1.57 pmap.c
--- amd64/pmap.c 26 Oct 2010 05:49:10 -0000 1.57
+++ amd64/pmap.c 31 Oct 2010 23:04:16 -0000
@@ -1118,7 +1118,7 @@ pmap_reference(struct pmap *pmap)
/*
* pmap_activate: activate a process' pmap (fill in %cr3)
*
- * => called from cpu_switch()
+ * => called from cpu_fork() and when switching pmaps during exec
* => if p is the curproc, then load it into the MMU
*/
Index: amd64/vm_machdep.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/vm_machdep.c,v
retrieving revision 1.23
diff -u -p -r1.23 vm_machdep.c
--- amd64/vm_machdep.c 26 Oct 2010 05:49:10 -0000 1.23
+++ amd64/vm_machdep.c 31 Oct 2010 23:04:16 -0000
@@ -111,15 +111,13 @@ cpu_fork(struct proc *p1, struct proc *p
*/
pmap_activate(p2);
- /* Fix up the TSS. */
- pcb->pcb_tss.tss_rsp0 = (u_int64_t)p2->p_addr + USPACE - 16;
- pcb->pcb_tss.tss_ist[0] = (u_int64_t)p2->p_addr + PAGE_SIZE - 16;
- p2->p_md.md_tss_sel = tss_alloc(pcb);
+ /* Record where this process's kernel stack is */
+ pcb->pcb_kstack = (u_int64_t)p2->p_addr + USPACE - 16;
/*
* Copy the trapframe.
*/
- p2->p_md.md_regs = tf = (struct trapframe *)pcb->pcb_tss.tss_rsp0 - 1;
+ p2->p_md.md_regs = tf = (struct trapframe *)pcb->pcb_kstack - 1;
*tf = *p1->p_md.md_regs;
setredzone(p2);
@@ -160,7 +158,6 @@ cpu_exit(struct proc *p)
mtrr_clean(p);
pmap_deactivate(p);
- tss_free(p->p_md.md_tss_sel);
sched_exit(p);
}
Index: conf/files.amd64
===================================================================
RCS file: /cvs/src/sys/arch/amd64/conf/files.amd64,v
retrieving revision 1.58
diff -u -p -r1.58 files.amd64
--- conf/files.amd64 7 Sep 2010 16:22:48 -0000 1.58
+++ conf/files.amd64 31 Oct 2010 23:30:27 -0000
@@ -6,7 +6,7 @@ maxusers 2 16 128
file arch/amd64/amd64/autoconf.c
file arch/amd64/amd64/conf.c
file arch/amd64/amd64/disksubr.c disk
-file arch/amd64/amd64/gdt.c
+file arch/amd64/amd64/gdt.c multiprocessor
file arch/amd64/amd64/machdep.c
file arch/amd64/amd64/identcpu.c
file arch/amd64/amd64/via.c
