Module Name: src
Committed By: rmind
Date: Mon Apr 26 04:48:50 UTC 2010
Modified Files:
src/sys/arch/amd64/amd64 [rmind-uvmplock]: genassym.cf vector.S
src/sys/arch/x86/include [rmind-uvmplock]: pmap.h
src/sys/arch/x86/x86 [rmind-uvmplock]: pmap.c
Log Message:
Partly rewrite amd64 TLB shutdown handler for the changes in x86 pmap.
At this point, branch seems to pass preliminar stress tests on amd64.
To generate a diff of this commit:
cvs rdiff -u -r1.43 -r1.43.4.1 src/sys/arch/amd64/amd64/genassym.cf
cvs rdiff -u -r1.33 -r1.33.2.1 src/sys/arch/amd64/amd64/vector.S
cvs rdiff -u -r1.29.2.2 -r1.29.2.3 src/sys/arch/x86/include/pmap.h
cvs rdiff -u -r1.105.2.4 -r1.105.2.5 src/sys/arch/x86/x86/pmap.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/genassym.cf
diff -u src/sys/arch/amd64/amd64/genassym.cf:1.43 src/sys/arch/amd64/amd64/genassym.cf:1.43.4.1
--- src/sys/arch/amd64/amd64/genassym.cf:1.43 Fri Nov 27 03:23:04 2009
+++ src/sys/arch/amd64/amd64/genassym.cf Mon Apr 26 04:48:49 2010
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.43 2009/11/27 03:23:04 rmind Exp $
+# $NetBSD: genassym.cf,v 1.43.4.1 2010/04/26 04:48:49 rmind Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -136,6 +136,8 @@
define KERNTEXTOFF_LO KERNTEXTOFF_LO
define KERNTEXTOFF KERNTEXTOFF
+define PG_G PG_G
+
define NBPG NBPG
define L4_SLOT_KERNBASE L4_SLOT_KERNBASE
@@ -225,7 +227,6 @@
define CPU_INFO_SELF offsetof(struct cpu_info, ci_self)
define CPU_INFO_RESCHED offsetof(struct cpu_info, ci_want_resched)
define CPU_INFO_WANT_PMAPLOAD offsetof(struct cpu_info, ci_want_pmapload)
-define CPU_INFO_PMAP_CPU offsetof(struct cpu_info, ci_pmap_cpu)
define CPU_INFO_TLBSTATE offsetof(struct cpu_info, ci_tlbstate)
define TLBSTATE_VALID TLBSTATE_VALID
define TLBSTATE_LAZY TLBSTATE_LAZY
@@ -349,12 +350,11 @@
define RW_READER RW_READER
define RW_WRITER RW_WRITER
-define MB_POINTER offsetof(struct pmap_mbox, mb_pointer)
-define MB_GLOBAL offsetof(struct pmap_mbox, mb_global)
-define MB_ADDR1 offsetof(struct pmap_mbox, mb_addr1)
-define MB_ADDR2 offsetof(struct pmap_mbox, mb_addr2)
-define MB_HEAD offsetof(struct pmap_mbox, mb_head)
-define MB_TAIL offsetof(struct pmap_mbox, mb_tail)
+define TM_PENDING offsetof(struct pmap_tlb_mailbox, tm_pending)
+define TP_COUNT offsetof(struct pmap_tlb_packet, tp_count)
+define TP_VA offsetof(struct pmap_tlb_packet, tp_va)
+define TP_USERMASK offsetof(struct pmap_tlb_packet, tp_usermask)
+define TP_PTE offsetof(struct pmap_tlb_packet, tp_pte)
define PM_CPUS offsetof(struct pmap, pm_cpus)
Index: src/sys/arch/amd64/amd64/vector.S
diff -u src/sys/arch/amd64/amd64/vector.S:1.33 src/sys/arch/amd64/amd64/vector.S:1.33.2.1
--- src/sys/arch/amd64/amd64/vector.S:1.33 Tue Feb 23 06:27:40 2010
+++ src/sys/arch/amd64/amd64/vector.S Mon Apr 26 04:48:49 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: vector.S,v 1.33 2010/02/23 06:27:40 cegger Exp $ */
+/* $NetBSD: vector.S,v 1.33.2.1 2010/04/26 04:48:49 rmind Exp $ */
/*-
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -480,13 +480,10 @@
#ifndef XEN
/*
- * Multicast TLB shootdown handler for !kernel_pmap.
+ * TLB shootdown handler.
*/
-IDTVEC(intr_lapic_tlb_mcast)
- /*
- * Save state, ack interrupt and count it. Count locally
- * so other CPUs don't have to touch the counter's line.
- */
+IDTVEC(intr_lapic_tlb)
+ /* Save state and ack the interrupt. */
testq $SEL_UPL,8(%rsp)
jz 0f
swapgs
@@ -499,41 +496,42 @@
pushq %rcx
pushq %r8
pushq %r9
- incq CPUVAR(TLB_EVCNT)+EV_COUNT
- /* Find out what needs to be invalidated and unlock the mailbox. */
- movq CPUVAR(PMAP_CPU),%rcx
- movq MB_ADDR1(%rcx), %rax
- movq MB_ADDR2(%rcx), %rdx
- xorq %rbx, %rbx
- xchgq MB_POINTER(%rcx), %rbx
movl $0, _C_LABEL(local_apic)+LAPIC_EOI
- cmpq $-1, %rax
+
+ /* Find out what we need to invalidate. */
+ leaq _C_LABEL(pmap_tlb_packet)(%rip), %rbx
+ movswq TP_COUNT(%rbx), %rcx
+ cmpq $-1, %rcx
je 5f
+ leaq TP_VA(%rbx), %rdx
1:
/* Invalidate a single page or a range of pages. */
+ movq (%rdx), %rax
invlpg (%rax)
- addq $PAGE_SIZE, %rax
- cmpq %rdx, %rax
- jb 1b
+ addq $8, %rdx
+ decq %rcx
+ jg 1b
2:
- /* Ack the request. */
- lock
- incq (%rbx)
/*
- * Check the current TLB state. If we don't want further
+ * Check the current TLB state. If we do not want further
* invalidations for this pmap, then take the CPU out of
* the pmap's bitmask.
*/
+ movl CPUVAR(CPUMASK), %eax
cmpl $TLBSTATE_LAZY, CPUVAR(TLBSTATE)
jne 3f
- movq CPUVAR(PMAP), %rdx
- movl CPUVAR(CPUMASK), %ecx
+ testl %eax, TP_USERMASK(%rbx)
+ jz 3f
+ movl CPUVAR(PMAP), %edx
+ movl %eax, %ecx
notl %ecx
lock
- andl %ecx, PM_CPUS(%rdx)
+ andl %ecx, PM_CPUS(%edx)
movl $TLBSTATE_STALE, CPUVAR(TLBSTATE)
3:
- /* Restore state and return. */
+ /* Ack the request, restore state & return. */
+ lock
+ xorl %eax, _C_LABEL(pmap_tlb_mailbox)+TM_PENDING
popq %r9
popq %r8
popq %rcx
@@ -548,11 +546,18 @@
4:
iretq
5:
+ /* Invalidate whole address space: */
+ testw $PG_G, TP_PTE(%rbx)
+ jnz 6f
/*
- * Get the emap generation number. Invalidate user TLB entries.
- * Perform emap update, pass the generation number. Note that
- * caller-save registers might be modified (all saved in the
- * beginning). Only %rbx value is used by 2b context.
+ * a) Invalidating user TLB entries only.
+ *
+ * - Get the emap generation number.
+ * - Invalidate TLB entries.
+ * - Perform emap update, pass the generation number.
+ *
+ * Note that caller-save registers might be modified (all saved in the
+ * beginning). Only %rbx value must be preserved for the 2b context.
*/
callq _C_LABEL(uvm_emap_gen_return)
movq %rax, %rdi
@@ -560,60 +565,11 @@
movq %rax, %cr3
callq _C_LABEL(uvm_emap_update)
jmp 2b
-
-/*
- * Broadcast TLB shootdown handler for kernel_pmap.
- */
-IDTVEC(intr_lapic_tlb_bcast)
- testq $SEL_UPL,8(%rsp)
- jz 0f
- swapgs
-0:
- /* Save state. */
- pushq %rax
- pushq %rdi
- pushq %rsi
- pushq %rdx
- pushq %rcx
- pushq %r8
- pushq %r9
- /* Find out what needs to be invalidated. */
- movq _C_LABEL(pmap_mbox)+MB_ADDR1, %rax
- movq _C_LABEL(pmap_mbox)+MB_ADDR2, %rdx
- movq _C_LABEL(pmap_mbox)+MB_GLOBAL, %rdi
- movl $0, _C_LABEL(local_apic)+LAPIC_EOI
- cmpq $-1, %rax
- je,pn 3f
-1:
- /* Invalidate a single page or a range of pages. */
- invlpg (%rax)
- addq $PAGE_SIZE, %rax
- cmpq %rdx, %rax
- jb 1b
-2:
- /* Notify waiter of completion, restore state & return */
- lock
- incq _C_LABEL(pmap_mbox)+MB_TAIL
- popq %r9
- popq %r8
- popq %rcx
- popq %rdx
- popq %rsi
- popq %rdi
- popq %rax
- testq $SEL_UPL, 8(%rsp)
- jz 5f
- swapgs
-5:
- iretq
-3:
- testq %rdi, %rdi
- jz 4f
+6:
/*
- * If we have been asked to invalidate the entire TLB we arrive here.
- * Get the emap generation before flush, and use it after for update.
- * Note that caller-save registers might be modified, though no
- * registers need to be preserved for 2b context.
+ * b) Invalidating user and kernel TLB entries.
+ *
+ * See notes above.
*/
callq _C_LABEL(uvm_emap_gen_return)
movq %rax, %rdi
@@ -624,19 +580,6 @@
movq %rax, %cr4
callq _C_LABEL(uvm_emap_update)
jmp 2b
-4:
- /*
- * Get the emap generation number. Invalidate user TLB entries.
- * Perform emap update, pass the generation number. Note that
- * caller-save registers might be modified, though no registers
- * need to be preserved for 2b context.
- */
- callq _C_LABEL(uvm_emap_gen_return)
- movq %rax, %rdi
- movq %cr3, %rax
- movq %rax, %cr3
- callq _C_LABEL(uvm_emap_update)
- jmp 2b
#endif /* !XEN */
Index: src/sys/arch/x86/include/pmap.h
diff -u src/sys/arch/x86/include/pmap.h:1.29.2.2 src/sys/arch/x86/include/pmap.h:1.29.2.3
--- src/sys/arch/x86/include/pmap.h:1.29.2.2 Mon Apr 26 02:43:34 2010
+++ src/sys/arch/x86/include/pmap.h Mon Apr 26 04:48:49 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.29.2.2 2010/04/26 02:43:34 rmind Exp $ */
+/* $NetBSD: pmap.h,v 1.29.2.3 2010/04/26 04:48:49 rmind Exp $ */
/*
*
@@ -454,10 +454,10 @@
#define TP_MAXVA 6 /* no more than N seperate invlpg */
struct pmap_tlb_mailbox {
- uintptr_t tm_pending;
- uintptr_t tm_gen;
- uintptr_t tm_usergen;
- uintptr_t tm_globalgen;
+ uint32_t tm_pending;
+ uint32_t tm_gen;
+ uint32_t tm_usergen;
+ uint32_t tm_globalgen;
char tm_pad[64 - sizeof(uintptr_t) * 4];
};
Index: src/sys/arch/x86/x86/pmap.c
diff -u src/sys/arch/x86/x86/pmap.c:1.105.2.4 src/sys/arch/x86/x86/pmap.c:1.105.2.5
--- src/sys/arch/x86/x86/pmap.c:1.105.2.4 Mon Apr 26 02:43:35 2010
+++ src/sys/arch/x86/x86/pmap.c Mon Apr 26 04:48:49 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.105.2.4 2010/04/26 02:43:35 rmind Exp $ */
+/* $NetBSD: pmap.c,v 1.105.2.5 2010/04/26 04:48:49 rmind Exp $ */
/*-
* Copyright (c) 2008, 2010 The NetBSD Foundation, Inc.
@@ -177,7 +177,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.4 2010/04/26 02:43:35 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.5 2010/04/26 04:48:49 rmind Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -1779,14 +1779,14 @@
evcnt_attach_dynamic(&tlbstat_single_issue, EVCNT_TYPE_MISC,
NULL, "tlbshoot single page", "issues");
#endif
-
+#if 0 /* XXXrmind */
evcnt_attach_dynamic(&pmap_tlb_evcnt, EVCNT_TYPE_INTR,
NULL, "TLB", "shootdown");
evcnt_attach_dynamic(&pmap_iobmp_evcnt, EVCNT_TYPE_MISC,
NULL, "x86", "io bitmap copy");
evcnt_attach_dynamic(&pmap_ldt_evcnt, EVCNT_TYPE_MISC,
NULL, "x86", "ldt sync");
-
+#endif
/*
* done: pmap module is up (and ready for business)
*/
@@ -4785,13 +4785,13 @@
*/
if ((tp->tp_cpumask & ci->ci_cpumask) != 0) {
if (tp->tp_count == (uint16_t)-1) {
- u_int gen = uvm_emap_gen_return();
+ u_int egen = uvm_emap_gen_return();
if ((tp->tp_pte & PG_G) != 0) {
tlbflushg();
} else {
tlbflush();
}
- uvm_emap_update(gen);
+ uvm_emap_update(egen);
} else {
for (i = tp->tp_count - 1; i >= 0; i--) {
pmap_update_pg(tp->tp_va[i]);
