Author: kib
Date: Thu Jul 19 19:09:12 2012
New Revision: 238623
URL: http://svn.freebsd.org/changeset/base/238623
Log:
Introduce curpcb magic variable, similar to curthread, which is MD
amd64. It is implemented as __pure2 inline with non-volatile asm read
from pcpu, which allows a compiler to cache its results.
Convert most PCPU_GET(pcb) and curthread->td_pcb accesses into curpcb.
Note that __curthread() uses magic value 0 as an offsetof(struct pcpu,
pc_curthread). It seems to be done this way due to machine/pcpu.h
needs to be processed before sys/pcpu.h, because machine/pcpu.h
contributes machine-depended fields to the struct pcpu definition. As
result, machine/pcpu.h cannot use struct pcpu yet.
The __curpcb() also uses a magic constant instead of offsetof(struct
pcpu, pc_curpcb) for the same reason. The constants are now defined as
symbols and CTASSERTs are added to ensure that future KBI changes do
not break the code.
Requested and reviewed by: bde
MFC after: 3 weeks
Modified:
head/sys/amd64/amd64/fpu.c
head/sys/amd64/amd64/machdep.c
head/sys/amd64/amd64/trap.c
head/sys/amd64/amd64/vm_machdep.c
head/sys/amd64/include/pcpu.h
Modified: head/sys/amd64/amd64/fpu.c
==============================================================================
--- head/sys/amd64/amd64/fpu.c Thu Jul 19 15:36:36 2012 (r238622)
+++ head/sys/amd64/amd64/fpu.c Thu Jul 19 19:09:12 2012 (r238623)
@@ -327,7 +327,7 @@ fpuexit(struct thread *td)
critical_enter();
if (curthread == PCPU_GET(fpcurthread)) {
stop_emulating();
- fpusave(PCPU_GET(curpcb)->pcb_save);
+ fpusave(curpcb->pcb_save);
start_emulating();
PCPU_SET(fpcurthread, 0);
}
@@ -547,7 +547,7 @@ fputrap_x87(void)
* wherever they are.
*/
if (PCPU_GET(fpcurthread) != curthread) {
- pcb_save = PCPU_GET(curpcb)->pcb_save;
+ pcb_save = curpcb->pcb_save;
control = pcb_save->sv_env.en_cw;
status = pcb_save->sv_env.en_sw;
} else {
@@ -567,7 +567,7 @@ fputrap_sse(void)
critical_enter();
if (PCPU_GET(fpcurthread) != curthread)
- mxcsr = PCPU_GET(curpcb)->pcb_save->sv_env.en_mxcsr;
+ mxcsr = curpcb->pcb_save->sv_env.en_mxcsr;
else
stmxcsr(&mxcsr);
critical_exit();
@@ -609,7 +609,7 @@ fpudna(void)
* Record new context early in case frstor causes a trap.
*/
PCPU_SET(fpcurthread, curthread);
- pcb = PCPU_GET(curpcb);
+ pcb = curpcb;
fpu_clean_state();
@@ -970,7 +970,7 @@ fpu_kern_thread(u_int flags)
{
struct pcb *pcb;
- pcb = PCPU_GET(curpcb);
+ pcb = curpcb;
KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0,
("Only kthread may use fpu_kern_thread"));
KASSERT(pcb->pcb_save == get_pcb_user_save_pcb(pcb),
@@ -987,5 +987,5 @@ is_fpu_kern_thread(u_int flags)
if ((curthread->td_pflags & TDP_KTHREAD) == 0)
return (0);
- return ((PCPU_GET(curpcb)->pcb_flags & PCB_KERNFPU) != 0);
+ return ((curpcb->pcb_flags & PCB_KERNFPU) != 0);
}
Modified: head/sys/amd64/amd64/machdep.c
==============================================================================
--- head/sys/amd64/amd64/machdep.c Thu Jul 19 15:36:36 2012
(r238622)
+++ head/sys/amd64/amd64/machdep.c Thu Jul 19 19:09:12 2012
(r238623)
@@ -996,7 +996,7 @@ exec_setregs(struct thread *td, struct i
pcb->pcb_dr3 = 0;
pcb->pcb_dr6 = 0;
pcb->pcb_dr7 = 0;
- if (pcb == PCPU_GET(curpcb)) {
+ if (pcb == curpcb) {
/*
* Clear the debug registers on the running
* CPU, otherwise they will end up affecting
Modified: head/sys/amd64/amd64/trap.c
==============================================================================
--- head/sys/amd64/amd64/trap.c Thu Jul 19 15:36:36 2012 (r238622)
+++ head/sys/amd64/amd64/trap.c Thu Jul 19 19:09:12 2012 (r238623)
@@ -520,9 +520,8 @@ trap(struct trapframe *frame)
frame->tf_rip = (long)fsbase_load_fault;
goto out;
}
- if (PCPU_GET(curpcb)->pcb_onfault != NULL) {
- frame->tf_rip =
- (long)PCPU_GET(curpcb)->pcb_onfault;
+ if (curpcb->pcb_onfault != NULL) {
+ frame->tf_rip = (long)curpcb->pcb_onfault;
goto out;
}
break;
@@ -708,7 +707,7 @@ trap_pfault(frame, usermode)
* it normally, and panic immediately.
*/
if (!usermode && (td->td_intr_nesting_level != 0 ||
- PCPU_GET(curpcb)->pcb_onfault == NULL)) {
+ curpcb->pcb_onfault == NULL)) {
trap_fatal(frame, eva);
return (-1);
}
@@ -764,8 +763,8 @@ trap_pfault(frame, usermode)
nogo:
if (!usermode) {
if (td->td_intr_nesting_level == 0 &&
- PCPU_GET(curpcb)->pcb_onfault != NULL) {
- frame->tf_rip = (long)PCPU_GET(curpcb)->pcb_onfault;
+ curpcb->pcb_onfault != NULL) {
+ frame->tf_rip = (long)curpcb->pcb_onfault;
return (0);
}
trap_fatal(frame, eva);
Modified: head/sys/amd64/amd64/vm_machdep.c
==============================================================================
--- head/sys/amd64/amd64/vm_machdep.c Thu Jul 19 15:36:36 2012
(r238622)
+++ head/sys/amd64/amd64/vm_machdep.c Thu Jul 19 19:09:12 2012
(r238623)
@@ -90,6 +90,10 @@ static u_int cpu_reset_proxyid;
static volatile u_int cpu_reset_proxy_active;
#endif
+CTASSERT((struct thread **)OFFSETOF_CURTHREAD ==
+ &((struct pcpu *)NULL)->pc_curthread);
+CTASSERT((struct pcb **)OFFSETOF_CURPCB == &((struct pcpu *)NULL)->pc_curpcb);
+
struct savefpu *
get_pcb_user_save_td(struct thread *td)
{
Modified: head/sys/amd64/include/pcpu.h
==============================================================================
--- head/sys/amd64/include/pcpu.h Thu Jul 19 15:36:36 2012
(r238622)
+++ head/sys/amd64/include/pcpu.h Thu Jul 19 19:09:12 2012
(r238623)
@@ -216,16 +216,29 @@ extern struct pcpu *pcpup;
#define PCPU_PTR(member) __PCPU_PTR(pc_ ## member)
#define PCPU_SET(member, val) __PCPU_SET(pc_ ## member, val)
+#define OFFSETOF_CURTHREAD 0
static __inline __pure2 struct thread *
__curthread(void)
{
struct thread *td;
- __asm("movq %%gs:0,%0" : "=r" (td));
+ __asm("movq %%gs:%1,%0" : "=r" (td)
+ : "m" (*(char *)OFFSETOF_CURTHREAD));
return (td);
}
#define curthread (__curthread())
+#define OFFSETOF_CURPCB 32
+static __inline __pure2 struct pcb *
+__curpcb(void)
+{
+ struct pcb *pcb;
+
+ __asm("movq %%gs:%1,%0" : "=r" (pcb) : "m" (*(char *)OFFSETOF_CURPCB));
+ return (pcb);
+}
+#define curpcb (__curpcb())
+
#define IS_BSP() (PCPU_GET(cpuid) == 0)
#else /* !lint || defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) */
_______________________________________________
svn-src-head@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
