Module Name: src
Committed By: skrll
Date: Sun Sep 3 08:48:20 UTC 2023
Modified Files:
src/sys/arch/riscv/conf: GENERIC.common files.riscv
src/sys/arch/riscv/fdt: cpu_fdt.c
src/sys/arch/riscv/include: cpu.h intr.h pmap.h
src/sys/arch/riscv/riscv: cpu.c cpu_subr.c db_interface.c ipifuncs.c
locore.S pmap_machdep.c riscv_machdep.c riscv_tlb.c
Log Message:
Fix and enable MULTIPROCESSOR
To generate a diff of this commit:
cvs rdiff -u -r1.6 -r1.7 src/sys/arch/riscv/conf/GENERIC.common
cvs rdiff -u -r1.13 -r1.14 src/sys/arch/riscv/conf/files.riscv
cvs rdiff -u -r1.2 -r1.3 src/sys/arch/riscv/fdt/cpu_fdt.c
cvs rdiff -u -r1.13 -r1.14 src/sys/arch/riscv/include/cpu.h
cvs rdiff -u -r1.4 -r1.5 src/sys/arch/riscv/include/intr.h
cvs rdiff -u -r1.20 -r1.21 src/sys/arch/riscv/include/pmap.h
cvs rdiff -u -r1.4 -r1.5 src/sys/arch/riscv/riscv/cpu.c
cvs rdiff -u -r1.3 -r1.4 src/sys/arch/riscv/riscv/cpu_subr.c \
src/sys/arch/riscv/riscv/db_interface.c
cvs rdiff -u -r1.1 -r1.2 src/sys/arch/riscv/riscv/ipifuncs.c \
src/sys/arch/riscv/riscv/riscv_tlb.c
cvs rdiff -u -r1.42 -r1.43 src/sys/arch/riscv/riscv/locore.S
cvs rdiff -u -r1.18 -r1.19 src/sys/arch/riscv/riscv/pmap_machdep.c
cvs rdiff -u -r1.33 -r1.34 src/sys/arch/riscv/riscv/riscv_machdep.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/riscv/conf/GENERIC.common
diff -u src/sys/arch/riscv/conf/GENERIC.common:1.6 src/sys/arch/riscv/conf/GENERIC.common:1.7
--- src/sys/arch/riscv/conf/GENERIC.common:1.6 Tue Aug 1 22:06:23 2023
+++ src/sys/arch/riscv/conf/GENERIC.common Sun Sep 3 08:48:19 2023
@@ -1,5 +1,5 @@
#
-# $NetBSD: GENERIC.common,v 1.6 2023/08/01 22:06:23 rjs Exp $
+# $NetBSD: GENERIC.common,v 1.7 2023/09/03 08:48:19 skrll Exp $
#
# GENERIC common RISC-V kernel config items shared between 32 and 64
# kernels
@@ -8,7 +8,7 @@
maxusers 64 # estimated number of users
options FDT
-#options MULTIPROCESSOR
+options MULTIPROCESSOR
# Standard system options
options FPE # Floating-point extension support
Index: src/sys/arch/riscv/conf/files.riscv
diff -u src/sys/arch/riscv/conf/files.riscv:1.13 src/sys/arch/riscv/conf/files.riscv:1.14
--- src/sys/arch/riscv/conf/files.riscv:1.13 Mon Jun 12 19:04:13 2023
+++ src/sys/arch/riscv/conf/files.riscv Sun Sep 3 08:48:19 2023
@@ -1,4 +1,4 @@
-# $NetBSD: files.riscv,v 1.13 2023/06/12 19:04:13 skrll Exp $
+# $NetBSD: files.riscv,v 1.14 2023/09/03 08:48:19 skrll Exp $
#
maxpartitions 16
@@ -64,7 +64,6 @@ file kern/subr_disk_mbr.c disk
file uvm/pmap/pmap.c
file uvm/pmap/pmap_devmap.c
file uvm/pmap/pmap_segtab.c
-file uvm/pmap/pmap_synci.c
file uvm/pmap/pmap_tlb.c
device plic
Index: src/sys/arch/riscv/fdt/cpu_fdt.c
diff -u src/sys/arch/riscv/fdt/cpu_fdt.c:1.2 src/sys/arch/riscv/fdt/cpu_fdt.c:1.3
--- src/sys/arch/riscv/fdt/cpu_fdt.c:1.2 Mon Jun 12 19:04:13 2023
+++ src/sys/arch/riscv/fdt/cpu_fdt.c Sun Sep 3 08:48:19 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_fdt.c,v 1.2 2023/06/12 19:04:13 skrll Exp $ */
+/* $NetBSD: cpu_fdt.c,v 1.3 2023/09/03 08:48:19 skrll Exp $ */
/*-
* Copyright (c) 2017 Jared McNeill <[email protected]>
@@ -29,7 +29,7 @@
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu_fdt.c,v 1.2 2023/06/12 19:04:13 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu_fdt.c,v 1.3 2023/09/03 08:48:19 skrll Exp $");
#include <sys/param.h>
#include <sys/cpu.h>
@@ -44,7 +44,6 @@ __KERNEL_RCSID(0, "$NetBSD: cpu_fdt.c,v
#include <riscv/fdt/riscv_fdtvar.h>
-#ifdef MULTIPROCESSOR
static bool
riscv_fdt_cpu_okay(const int child)
{
@@ -65,33 +64,32 @@ riscv_fdt_cpu_okay(const int child)
return true;
}
}
-#endif /* MULTIPROCESSOR */
void
riscv_fdt_cpu_bootstrap(void)
{
-#ifdef MULTIPROCESSOR
-
const int cpus = OF_finddevice("/cpus");
if (cpus == -1) {
aprint_error("%s: no /cpus node found\n", __func__);
- riscv_cpu_max = 1;
return;
}
- /* Count harts and add hart IDs to to cpu_hartid array */
- size_t cpuindex = 1;
+ /* Count harts and add hart index numbers to the cpu_hartindex array */
+ u_int cpuindex = 1;
for (int child = OF_child(cpus); child; child = OF_peer(child)) {
if (!riscv_fdt_cpu_okay(child))
continue;
- riscv_cpu_max++;
-
uint64_t reg;
if (fdtbus_get_reg64(child, 0, ®, NULL) != 0)
continue;
const cpuid_t hartid = reg;
+ if (hartid > MAXCPUS) {
+ aprint_error("hart id too big %lu (%u)", hartid,
+ MAXCPUS);
+ continue;
+ }
struct sbiret sbiret = sbi_hart_get_status(hartid);
switch (sbiret.error) {
@@ -107,22 +105,19 @@ riscv_fdt_cpu_bootstrap(void)
/* Assume the BP is the only one started. */
if (sbiret.value == SBI_HART_STARTED) {
- if (cpu_hartid[0] != -1) {
+ if (cpu_bphartid != ~0UL) {
panic("more than 1 hart started");
}
- cpu_hartid[0] = hartid;
+ cpu_bphartid = hartid;
+ cpu_hartindex[hartid] = 0;
continue;
}
KASSERT(cpuindex < MAXCPUS);
- cpu_hartid[cpuindex] = hartid;
- cpu_dcache_wb_range((vaddr_t)&cpu_hartid[cpuindex],
- sizeof(cpu_hartid[cpuindex]));
-
- cpuindex++;
+ cpu_hartindex[hartid] = cpuindex++;
}
-#endif
}
+
int
riscv_fdt_cpu_mpstart(void)
{
@@ -134,11 +129,8 @@ riscv_fdt_cpu_mpstart(void)
return 0;
}
- // riscv_fdt_cpu_bootstrap put the boot hart id in cpu_hartid[0]
- const cpuid_t bp_hartid = cpu_hartid[0];
-
/* BootAPs */
- size_t cpuindex = 1;
+ u_int cpuindex = 1;
for (int child = OF_child(cpus); child; child = OF_peer(child)) {
if (!riscv_fdt_cpu_okay(child))
continue;
@@ -147,13 +139,12 @@ riscv_fdt_cpu_mpstart(void)
if (fdtbus_get_reg64(child, 0, ®, NULL) != 0)
continue;
- cpuid_t hartid = reg;
-
- if (hartid == bp_hartid)
+ const cpuid_t hartid = reg;
+ if (hartid == cpu_bphartid)
continue; /* BP already started */
const paddr_t entry = KERN_VTOPHYS(cpu_mpstart);
- struct sbiret sbiret = sbi_hart_start(hartid, entry, 0);
+ struct sbiret sbiret = sbi_hart_start(hartid, entry, cpuindex);
switch (sbiret.error) {
case SBI_SUCCESS:
break;
@@ -179,8 +170,8 @@ riscv_fdt_cpu_mpstart(void)
if (i == 0) {
ret++;
- aprint_error("cpu%zu: WARNING: AP failed to start\n",
- cpuindex);
+ aprint_error("hart%ld: WARNING: AP %u failed to start\n",
+ hartid, cpuindex);
}
cpuindex++;
@@ -207,13 +198,14 @@ cpu_fdt_attach(device_t parent, device_t
{
struct fdt_attach_args * const faa = aux;
const int phandle = faa->faa_phandle;
- bus_addr_t cpuid;
+ bus_addr_t hartid;
+
- if (fdtbus_get_reg(phandle, 0, &cpuid, NULL) != 0)
- cpuid = 0;
+ if (fdtbus_get_reg(phandle, 0, &hartid, NULL) != 0)
+ hartid = 0;
/* Attach the CPU */
- cpu_attach(self, cpuid);
+ cpu_attach(self, hartid);
fdt_add_bus(self, phandle, faa);
}
Index: src/sys/arch/riscv/include/cpu.h
diff -u src/sys/arch/riscv/include/cpu.h:1.13 src/sys/arch/riscv/include/cpu.h:1.14
--- src/sys/arch/riscv/include/cpu.h:1.13 Sat Jul 29 06:59:47 2023
+++ src/sys/arch/riscv/include/cpu.h Sun Sep 3 08:48:20 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.13 2023/07/29 06:59:47 skrll Exp $ */
+/* $NetBSD: cpu.h,v 1.14 2023/09/03 08:48:20 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -91,7 +91,6 @@ struct cpu_info {
#define CPUF_PRESENT __BIT(1) /* CPU is present */
#define CPUF_RUNNING __BIT(2) /* CPU is running */
#define CPUF_PAUSED __BIT(3) /* CPU is paused */
-#define CPUF_USERPMAP __BIT(4) /* CPU has a user pmap activated */
volatile u_long ci_request_ipis;
/* bitmask of IPIs requested */
@@ -118,23 +117,21 @@ struct cpu_info {
#ifdef _KERNEL
-extern struct cpu_info *cpu_info[];
extern struct cpu_info cpu_info_store[];
-
+extern cpuid_t cpu_bphartid;
+extern u_int cpu_hartindex[];
#ifdef MULTIPROCESSOR
-extern u_int riscv_cpu_max;
-extern cpuid_t cpu_hartid[];
-void cpu_hatch(struct cpu_info *);
+void cpu_hatch(struct cpu_info *, unsigned long);
-void cpu_init_secondary_processor(int);
+void cpu_init_secondary_processor(u_int);
void cpu_boot_secondary_processors(void);
void cpu_mpstart(void);
bool cpu_hatched_p(u_int);
-void cpu_clr_mbox(int);
-void cpu_set_hatched(int);
+void cpu_clr_mbox(u_int);
+void cpu_set_hatched(u_int);
void cpu_halt(void);
@@ -201,9 +198,9 @@ void cpu_boot_secondary_processors(void)
#ifdef MULTIPROCESSOR
#define CPU_IS_PRIMARY(ci) ((ci)->ci_flags & CPUF_PRIMARY)
#define CPU_INFO_FOREACH(cii, ci) \
- cii = 0, ci = &cpu_info_store[0]; \
- ci != NULL; \
- cii++, ncpu ? (ci = cpu_infos[cii]) \
+ cii = 0, ci = &cpu_info_store[0]; \
+ ci != NULL; \
+ cii++, ncpu ? (ci = cpu_infos[cii]) \
: (ci = NULL)
#else
#define CPU_IS_PRIMARY(ci) true
Index: src/sys/arch/riscv/include/intr.h
diff -u src/sys/arch/riscv/include/intr.h:1.4 src/sys/arch/riscv/include/intr.h:1.5
--- src/sys/arch/riscv/include/intr.h:1.4 Mon Jun 12 19:04:14 2023
+++ src/sys/arch/riscv/include/intr.h Sun Sep 3 08:48:20 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: intr.h,v 1.4 2023/06/12 19:04:14 skrll Exp $ */
+/* $NetBSD: intr.h,v 1.5 2023/09/03 08:48:20 skrll Exp $ */
/*-
* Copyright (c) 2009, 2010 The NetBSD Foundation, Inc.
@@ -70,14 +70,12 @@
#define IPI_NOP 0 /* do nothing, interrupt only */
#define IPI_AST 1 /* force ast */
-#define IPI_SHOOTDOWN 2 /* do a tlb shootdown */
-#define IPI_SYNCICACHE 3 /* sync icache for pages */
-#define IPI_KPREEMPT 4 /* schedule a kernel preemption */
-#define IPI_SUSPEND 5 /* DDB suspend signaling */
-#define IPI_HALT 6 /* halt cpu */
-#define IPI_XCALL 7 /* xcall */
-#define IPI_GENERIC 8 /* generic IPI */
-#define NIPIS 9
+#define IPI_KPREEMPT 2 /* schedule a kernel preemption */
+#define IPI_SUSPEND 3 /* DDB suspend signaling */
+#define IPI_HALT 4 /* halt cpu */
+#define IPI_XCALL 5 /* xcall */
+#define IPI_GENERIC 6 /* generic IPI */
+#define NIPIS 7
#ifdef __INTR_PRIVATE
#if 0
Index: src/sys/arch/riscv/include/pmap.h
diff -u src/sys/arch/riscv/include/pmap.h:1.20 src/sys/arch/riscv/include/pmap.h:1.21
--- src/sys/arch/riscv/include/pmap.h:1.20 Sat Aug 12 07:48:11 2023
+++ src/sys/arch/riscv/include/pmap.h Sun Sep 3 08:48:20 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.20 2023/08/12 07:48:11 skrll Exp $ */
+/* $NetBSD: pmap.h,v 1.21 2023/09/03 08:48:20 skrll Exp $ */
/*
* Copyright (c) 2014, 2019, 2021 The NetBSD Foundation, Inc.
@@ -80,7 +80,7 @@
#define KERNEL_PID 0
#define PMAP_HWPAGEWALKER 1
-#define PMAP_TLB_MAX MAXCPUS
+#define PMAP_TLB_MAX 1
#ifdef _LP64
#define PMAP_INVALID_PDETAB_ADDRESS ((pmap_pdetab_t *)(VM_MIN_KERNEL_ADDRESS - PAGE_SIZE))
#define PMAP_INVALID_SEGTAB_ADDRESS ((pmap_segtab_t *)(VM_MIN_KERNEL_ADDRESS - PAGE_SIZE))
@@ -90,9 +90,7 @@
#endif
#define PMAP_TLB_NUM_PIDS (__SHIFTOUT_MASK(SATP_ASID) + 1)
#define PMAP_TLB_BITMAP_LENGTH PMAP_TLB_NUM_PIDS
-// Should use SBI TLB ops
-#define PMAP_TLB_NEED_SHOOTDOWN 1
-#define PMAP_TLB_FLUSH_ASID_ON_RESET false
+#define PMAP_TLB_FLUSH_ASID_ON_RESET true
#define pmap_phys_address(x) (x)
Index: src/sys/arch/riscv/riscv/cpu.c
diff -u src/sys/arch/riscv/riscv/cpu.c:1.4 src/sys/arch/riscv/riscv/cpu.c:1.5
--- src/sys/arch/riscv/riscv/cpu.c:1.4 Mon Aug 28 11:12:42 2023
+++ src/sys/arch/riscv/riscv/cpu.c Sun Sep 3 08:48:20 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.4 2023/08/28 11:12:42 skrll Exp $ */
+/* $NetBSD: cpu.c,v 1.5 2023/09/03 08:48:20 skrll Exp $ */
/*-
* Copyright (c) 2023 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.4 2023/08/28 11:12:42 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.5 2023/09/03 08:48:20 skrll Exp $");
#include <sys/param.h>
@@ -95,8 +95,7 @@ struct cpu_vendor {
};
/*
- * Our exported cpu_info structs; these will be first used by the
- * secondary cpus as part of cpu_mpstart and the hatching process.
+ * Our exported cpu_info structs; indexed by BP as 0 and APs [1, ncpu - 1]
*/
struct cpu_info cpu_info_store[NCPUINFO] = {
[0] = {
@@ -109,7 +108,6 @@ struct cpu_info cpu_info_store[NCPUINFO]
}
};
-
/*
* setup the per-cpu sysctl tree.
*/
@@ -177,14 +175,15 @@ cpu_identify(device_t self, struct cpu_i
void
-cpu_attach(device_t dv, cpuid_t id)
+cpu_attach(device_t dv, cpuid_t hartid)
{
- const int unit = device_unit(dv);
struct cpu_info *ci;
- if (unit == 0) {
+ /* Check for the BP */
+ if (hartid == cpu_bphartid) {
ci = curcpu();
- ci->ci_cpuid = id;
+ KASSERTMSG(ci == &cpu_info_store[0], "ci %p", ci);
+ ci->ci_cpuid = hartid;
} else {
#ifdef MULTIPROCESSOR
if ((boothowto & RB_MD1) != 0) {
@@ -193,14 +192,15 @@ cpu_attach(device_t dv, cpuid_t id)
return;
}
- KASSERT(unit < MAXCPUS);
- ci = &cpu_info_store[unit];
+ KASSERT(hartid < MAXCPUS);
+ KASSERT(cpu_hartindex[hartid] < MAXCPUS);
+
+ ci = &cpu_info_store[cpu_hartindex[hartid]];
ci->ci_cpl = IPL_HIGH;
- ci->ci_cpuid = id;
- /* ci_cpuid is stored by own cpus when hatching */
+ ci->ci_cpuid = hartid;
- if (cpu_hatched_p(unit) == 0) {
+ if (!cpu_hatched_p(cpu_hartindex[hartid])) {
ci->ci_dev = dv;
device_set_private(dv, ci);
ci->ci_index = -1;
@@ -230,18 +230,11 @@ cpu_attach(device_t dv, cpuid_t id)
kcpuset_create(&ci->ci_watchcpus, true);
kcpuset_create(&ci->ci_ddbcpus, true);
- if (unit != 0) {
+ if (hartid != cpu_bphartid) {
mi_cpu_attach(ci);
- struct pmap_tlb_info *ti = kmem_zalloc(sizeof(*ti), KM_SLEEP);
- pmap_tlb_info_init(ti);
- pmap_tlb_info_attach(ti, ci);
}
#endif /* MULTIPROCESSOR */
cpu_setup_sysctl(dv, ci);
-
- if (unit != 0) {
- return;
- }
}
#ifdef MULTIPROCESSOR
@@ -252,7 +245,7 @@ cpu_attach(device_t dv, cpuid_t id)
*
*/
void __noasan
-cpu_init_secondary_processor(int cpuindex)
+cpu_init_secondary_processor(u_int cpuindex)
{
cpu_set_hatched(cpuindex);
@@ -267,14 +260,20 @@ cpu_init_secondary_processor(int cpuinde
* of the idlelwp for this cpu.
*/
void
-cpu_hatch(struct cpu_info *ci)
+cpu_hatch(struct cpu_info *ci, unsigned long cpuindex)
{
KASSERT(curcpu() == ci);
+ // Show this CPU as present.
+ atomic_or_ulong(&ci->ci_flags, CPUF_PRESENT);
+
ci->ci_cpu_freq = riscv_timer_frequency_get();
riscv_timer_init();
+ kcpuset_set(cpus_hatched, cpu_index(ci));
+ kcpuset_set(cpus_running, cpu_index(ci));
+
/*
* clear my bit of the mailbox to tell cpu_boot_secondary_processors().
* Consider that if there are cpu0, 1, 2, 3, and cpu2 is unresponsive,
@@ -282,6 +281,6 @@ cpu_hatch(struct cpu_info *ci)
* therefore we have to use device_unit instead of ci_index for mbox.
*/
- cpu_clr_mbox(device_unit(ci->ci_dev));
+ cpu_clr_mbox(cpuindex);
}
#endif /* MULTIPROCESSOR */
Index: src/sys/arch/riscv/riscv/cpu_subr.c
diff -u src/sys/arch/riscv/riscv/cpu_subr.c:1.3 src/sys/arch/riscv/riscv/cpu_subr.c:1.4
--- src/sys/arch/riscv/riscv/cpu_subr.c:1.3 Mon Jun 12 19:04:14 2023
+++ src/sys/arch/riscv/riscv/cpu_subr.c Sun Sep 3 08:48:20 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_subr.c,v 1.3 2023/06/12 19:04:14 skrll Exp $ */
+/* $NetBSD: cpu_subr.c,v 1.4 2023/09/03 08:48:20 skrll Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
#include "opt_riscv_debug.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.3 2023/06/12 19:04:14 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.4 2023/09/03 08:48:20 skrll Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -56,16 +56,12 @@ __KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v
#define VPRINTF(...) __nothing
#endif
-#ifdef MULTIPROCESSOR
-#define NCPUINFO MAXCPUS
-#else
-#define NCPUINFO 1
-#endif /* MULTIPROCESSOR */
-
-unsigned long cpu_hartid[NCPUINFO] = {
- [0 ... NCPUINFO - 1] = ~0,
+unsigned int cpu_hartindex[MAXCPUS] = {
+ [0 ... MAXCPUS - 1] = ~0U,
};
+cpuid_t cpu_bphartid = ~0UL;
+
#ifdef MULTIPROCESSOR
kcpuset_t *cpus_halted;
@@ -82,16 +78,17 @@ kcpuset_t *cpus_running;
CTASSERT(N == 1);
volatile u_long riscv_cpu_hatched[N] __cacheline_aligned = { };
volatile u_long riscv_cpu_mbox[N] __cacheline_aligned = { };
-u_int riscv_cpu_max = 1;
/* IPI all APs to GO! */
static void
cpu_ipi_aps(void)
{
unsigned long hartmask = 0;
+
// BP is index 0
for (size_t i = 1; i < ncpu; i++) {
- const cpuid_t hartid = cpu_hartid[i];
+ const struct cpu_info * const ci = &cpu_info_store[i];
+ const cpuid_t hartid = ci->ci_cpuid;
KASSERT(hartid < sizeof(unsigned long) * NBBY);
hartmask |= __BIT(hartid);
}
@@ -103,8 +100,6 @@ cpu_ipi_aps(void)
void
cpu_boot_secondary_processors(void)
{
- u_int cpuno;
-
if ((boothowto & RB_MD1) != 0)
return;
@@ -120,21 +115,25 @@ cpu_boot_secondary_processors(void)
cpu_ipi_aps();
/* wait for all cpus to have done cpu_hatch() */
- for (cpuno = 1; cpuno < ncpu; cpuno++) {
- if (!cpu_hatched_p(cpuno))
+ for (u_int cpuindex = 1; cpuindex < ncpu; cpuindex++) {
+ if (!cpu_hatched_p(cpuindex))
continue;
- const size_t off = cpuno / CPUINDEX_DIVISOR;
- const u_long bit = __BIT(cpuno % CPUINDEX_DIVISOR);
+ const size_t off = cpuindex / CPUINDEX_DIVISOR;
+ const u_long bit = __BIT(cpuindex % CPUINDEX_DIVISOR);
/* load-acquire matches cpu_clr_mbox */
while (atomic_load_acquire(&riscv_cpu_mbox[off]) & bit) {
/* spin - it shouldn't be long */
;
}
+ struct cpu_info *ci = &cpu_info_store[cpuindex];
+ atomic_or_ulong(&ci->ci_flags, CPUF_RUNNING);
+ kcpuset_set(cpus_running, cpu_index(ci));
}
- VPRINTF("%s: secondary processors hatched\n", __func__);
+ VPRINTF("%s: secondary processors hatched. %d running\n", __func__,
+ kcpuset_countset(cpus_running));
}
bool
@@ -149,7 +148,7 @@ cpu_hatched_p(u_int cpuindex)
void
-cpu_set_hatched(int cpuindex)
+cpu_set_hatched(u_int cpuindex)
{
const size_t off = cpuindex / CPUINDEX_DIVISOR;
@@ -163,7 +162,7 @@ cpu_set_hatched(int cpuindex)
}
void
-cpu_clr_mbox(int cpuindex)
+cpu_clr_mbox(u_int cpuindex)
{
const size_t off = cpuindex / CPUINDEX_DIVISOR;
@@ -213,8 +212,7 @@ cpu_ipi_wait(const char *s, const kcpuse
kcpuset_t *kcp = ci->ci_watchcpus;
/* some finite amount of time */
-
- for (u_long limit = curcpu()->ci_cpu_freq / 10; !done && limit--; ) {
+ for (u_long limit = curcpu()->ci_cpu_freq /* / 10 */; !done && limit--; ) {
kcpuset_copy(kcp, watchset);
kcpuset_intersect(kcp, wanted);
done = kcpuset_match(kcp, wanted);
@@ -288,7 +286,7 @@ cpu_halt_others(void)
* Pause this cpu
*/
void
-cpu_pause(void )
+cpu_pause(void)
{
const int s = splhigh();
cpuid_t cii = cpu_index(curcpu());
@@ -349,6 +347,7 @@ cpu_resume(cpuid_t cii)
struct cpu_info * const ci = curcpu();
kcpuset_t *kcp = ci->ci_ddbcpus;
+ kcpuset_zero(kcp);
kcpuset_set(kcp, cii);
kcpuset_atomicly_remove(cpus_resumed, cpus_resumed);
kcpuset_atomic_clear(cpus_paused, cii);
@@ -391,20 +390,21 @@ cpu_debug_dump(void)
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
char running, hatched, paused, resumed, halted;
- db_printf("CPU CPUID STATE CPUINFO CPL INT MTX IPIS(A/R)\n");
+ db_printf("CPU CPUID STATE CPUINFO CPL INT MTX IPIS(A/R)\n");
for (CPU_INFO_FOREACH(cii, ci)) {
hatched = (kcpuset_isset(cpus_hatched, cpu_index(ci)) ? 'H' : '-');
running = (kcpuset_isset(cpus_running, cpu_index(ci)) ? 'R' : '-');
paused = (kcpuset_isset(cpus_paused, cpu_index(ci)) ? 'P' : '-');
resumed = (kcpuset_isset(cpus_resumed, cpu_index(ci)) ? 'r' : '-');
halted = (kcpuset_isset(cpus_halted, cpu_index(ci)) ? 'h' : '-');
- db_printf("%3d 0x%03lx %c%c%c%c%c %p "
- "%3d %3d %3d "
- "0x%02lx/0x%02lx\n",
- cpu_index(ci), ci->ci_cpuid,
- running, hatched, paused, resumed, halted,
- ci, ci->ci_cpl, ci->ci_intr_depth, ci->ci_mtx_count,
- ci->ci_active_ipis, ci->ci_request_ipis);
+ db_printf("%3d 0x%03lx%c%c%c%c%c%c%c %p "
+ "%3d %3d %3d 0x%02lx/0x%02lx\n",
+ cpu_index(ci), ci->ci_cpuid,
+ ci == curcpu() ? '<' : ' ',
+ CPU_IS_PRIMARY(ci) ? '*' : ' ',
+ hatched, running, paused, resumed, halted,
+ ci, ci->ci_cpl, ci->ci_intr_depth, ci->ci_mtx_count,
+ ci->ci_active_ipis, ci->ci_request_ipis);
}
}
#endif
Index: src/sys/arch/riscv/riscv/db_interface.c
diff -u src/sys/arch/riscv/riscv/db_interface.c:1.3 src/sys/arch/riscv/riscv/db_interface.c:1.4
--- src/sys/arch/riscv/riscv/db_interface.c:1.3 Mon Jun 12 19:04:14 2023
+++ src/sys/arch/riscv/riscv/db_interface.c Sun Sep 3 08:48:20 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: db_interface.c,v 1.3 2023/06/12 19:04:14 skrll Exp $ */
+/* $NetBSD: db_interface.c,v 1.4 2023/09/03 08:48:20 skrll Exp $ */
/*
* Mach Operating System
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: db_interface.c,v 1.3 2023/06/12 19:04:14 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: db_interface.c,v 1.4 2023/09/03 08:48:20 skrll Exp $");
#ifdef _KERNEL_OPT
#include "opt_multiprocessor.h"
@@ -107,10 +107,11 @@ kdb_trap(int type, db_regs_t *regs)
}
s = splhigh();
+ struct cpu_info * const ci = curcpu();
#if defined(MULTIPROCESSOR)
bool first_in_ddb = false;
- const u_int cpu_me = cpu_number();
+ const u_int cpu_me = cpu_index(ci);
const u_int old_ddb_cpu = atomic_cas_uint(&ddb_cpu, NOCPU, cpu_me);
if (old_ddb_cpu == NOCPU) {
first_in_ddb = true;
@@ -125,7 +126,6 @@ kdb_trap(int type, db_regs_t *regs)
}
KASSERT(!cpu_is_paused(cpu_me));
#endif
- struct cpu_info * const ci = curcpu();
ddb_regs = *regs;
ci->ci_ddb_regs = &ddb_regs;
@@ -201,7 +201,7 @@ const struct db_command db_machine_comma
bool
ddb_running_on_this_cpu_p(void)
{
- return ddb_cpu == cpu_number();
+ return ddb_cpu == cpu_index(curcpu());
}
bool
@@ -213,7 +213,7 @@ ddb_running_on_any_cpu_p(void)
void
db_resume_others(void)
{
- u_int cpu_me = cpu_number();
+ u_int cpu_me = cpu_index(curcpu());
if (atomic_cas_uint(&ddb_cpu, cpu_me, NOCPU) == cpu_me)
cpu_resume_others();
@@ -242,7 +242,7 @@ db_mach_cpu_cmd(db_expr_t addr, bool hav
db_printf("CPU %ld not paused\n", (long)addr);
return;
}
- (void)atomic_cas_uint(&ddb_cpu, cpu_number(), cpu_index(ci));
+ (void)atomic_cas_uint(&ddb_cpu, cpu_index(curcpu()), cpu_index(ci));
db_continue_cmd(0, false, 0, "");
}
}
Index: src/sys/arch/riscv/riscv/ipifuncs.c
diff -u src/sys/arch/riscv/riscv/ipifuncs.c:1.1 src/sys/arch/riscv/riscv/ipifuncs.c:1.2
--- src/sys/arch/riscv/riscv/ipifuncs.c:1.1 Mon Jun 12 19:04:14 2023
+++ src/sys/arch/riscv/riscv/ipifuncs.c Sun Sep 3 08:48:20 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: ipifuncs.c,v 1.1 2023/06/12 19:04:14 skrll Exp $ */
+/* $NetBSD: ipifuncs.c,v 1.2 2023/09/03 08:48:20 skrll Exp $ */
/*-
* Copyright (c) 2010 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.1 2023/06/12 19:04:14 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.2 2023/09/03 08:48:20 skrll Exp $");
#include <sys/param.h>
@@ -49,8 +49,6 @@ static void ipi_halt(void) __dead;
static const char * const ipi_names[] = {
[IPI_NOP] = "ipi nop",
[IPI_AST] = "ipi ast",
- [IPI_SHOOTDOWN] = "ipi shootdown",
- [IPI_SYNCICACHE]= "ipi isync",
[IPI_KPREEMPT] = "ipi kpreempt",
[IPI_SUSPEND] = "ipi suspend",
[IPI_HALT] = "ipi halt",
@@ -73,18 +71,6 @@ ipi_ast(struct cpu_info *ci)
ci->ci_onproc->l_md.md_astpending = 1;
}
-static void
-ipi_shootdown(struct cpu_info *ci)
-{
- pmap_tlb_shootdown_process();
-}
-
-static inline void
-ipi_syncicache(struct cpu_info *ci)
-{
- pmap_tlb_syncicache_wanted(ci);
-}
-
#ifdef __HAVE_PREEMPTION
static inline void
ipi_kpreempt(struct cpu_info *ci)
@@ -100,7 +86,7 @@ ipi_kpreempt(struct cpu_info *ci)
static void
ipi_halt(void)
{
- const u_int my_cpu = cpu_number();
+ const u_int my_cpu = cpu_index(curcpu());
printf("cpu%u: shutting down\n", my_cpu);
kcpuset_set(cpus_halted, my_cpu);
splhigh();
@@ -122,14 +108,6 @@ ipi_process(struct cpu_info *ci, unsigne
ci->ci_evcnt_per_ipi[IPI_AST].ev_count++;
ipi_ast(ci);
}
- if (ipi_mask & __BIT(IPI_SHOOTDOWN)) {
- ci->ci_evcnt_per_ipi[IPI_SHOOTDOWN].ev_count++;
- ipi_shootdown(ci);
- }
- if (ipi_mask & __BIT(IPI_SYNCICACHE)) {
- ci->ci_evcnt_per_ipi[IPI_SYNCICACHE].ev_count++;
- ipi_syncicache(ci);
- }
if (ipi_mask & __BIT(IPI_SUSPEND)) {
ci->ci_evcnt_per_ipi[IPI_SUSPEND].ev_count++;
cpu_pause();
Index: src/sys/arch/riscv/riscv/riscv_tlb.c
diff -u src/sys/arch/riscv/riscv/riscv_tlb.c:1.1 src/sys/arch/riscv/riscv/riscv_tlb.c:1.2
--- src/sys/arch/riscv/riscv/riscv_tlb.c:1.1 Mon Jun 12 19:04:14 2023
+++ src/sys/arch/riscv/riscv/riscv_tlb.c Sun Sep 3 08:48:20 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: riscv_tlb.c,v 1.1 2023/06/12 19:04:14 skrll Exp $ */
+/* $NetBSD: riscv_tlb.c,v 1.2 2023/09/03 08:48:20 skrll Exp $ */
/*
* Copyright (c) 2014, 2019, 2021 The NetBSD Foundation, Inc.
@@ -34,12 +34,16 @@
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__RCSID("$NetBSD: riscv_tlb.c,v 1.1 2023/06/12 19:04:14 skrll Exp $");
+__RCSID("$NetBSD: riscv_tlb.c,v 1.2 2023/09/03 08:48:20 skrll Exp $");
#include <sys/param.h>
#include <sys/types.h>
-#include <uvm/uvm.h>
+#include <sys/cpu.h>
+
+#include <uvm/uvm_extern.h>
+
+#include <riscv/sbi.h>
tlb_asid_t
tlb_get_asid(void)
@@ -71,13 +75,39 @@ tlb_invalidate_globals(void)
void
tlb_invalidate_asids(tlb_asid_t lo, tlb_asid_t hi)
{
- for (; lo <= hi; lo++) {
+ tlb_asid_t asid;
+ for (asid = lo; asid <= hi; asid++) {
asm volatile("sfence.vma zero, %[asid]"
: /* output operands */
- : [asid] "r" (lo)
+ : [asid] "r" (asid)
: "memory");
}
+#ifdef MULTIPROCESSOR
+#if PMAP_TLB_MAX == 1
+ const cpuid_t myhartid = curcpu()->ci_cpuid;
+ unsigned long hartmask = 0;
+ struct cpu_info *ci;
+ CPU_INFO_ITERATOR cii;
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ const cpuid_t hartid = ci->ci_cpuid;
+
+ if (hartid == myhartid)
+ continue;
+
+ KASSERT(hartid < sizeof(unsigned long) * NBBY);
+ hartmask |= __BIT(hartid);
+ }
+ for (asid = lo; asid <= hi; asid++) {
+ struct sbiret sbiret = sbi_remote_sfence_vma_asid(hartmask,
+ 0 /* hartmask_base */, 0, ~0UL, asid);
+
+ KASSERTMSG(sbiret.error == SBI_SUCCESS, "error %ld",
+ sbiret.error);
+ }
+#endif
+#endif
}
+
void
tlb_invalidate_addr(vaddr_t va, tlb_asid_t asid)
{
@@ -92,22 +122,36 @@ tlb_invalidate_addr(vaddr_t va, tlb_asid
: [va] "r" (va), [asid] "r" (asid)
: "memory");
}
+#ifdef MULTIPROCESSOR
+#if PMAP_TLB_MAX == 1
+ const cpuid_t myhartid = curcpu()->ci_cpuid;
+ unsigned long hartmask = 0;
+ struct cpu_info *ci;
+ CPU_INFO_ITERATOR cii;
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ const cpuid_t hartid = ci->ci_cpuid;
+ if (hartid == myhartid)
+ continue;
+
+ KASSERT(hartid < sizeof(unsigned long) * NBBY);
+ hartmask |= __BIT(hartid);
+ }
+ struct sbiret sbiret = sbi_remote_sfence_vma(hartmask,
+ 0 /* hartmask_base */, va, PAGE_SIZE);
+
+ KASSERTMSG(sbiret.error == SBI_SUCCESS, "error %ld",
+ sbiret.error);
+#endif
+#endif
}
bool
tlb_update_addr(vaddr_t va, tlb_asid_t asid, pt_entry_t pte, bool insert_p)
{
- if (asid == KERNEL_PID) {
- asm volatile("sfence.vma %[va]"
- : /* output operands */
- : [va] "r" (va)
- : "memory");
- } else {
- asm volatile("sfence.vma %[va], %[asid]"
- : /* output operands */
- : [va] "r" (va), [asid] "r" (asid)
- : "memory");
- }
+ KASSERT((va & PAGE_MASK) == 0);
+
+ tlb_invalidate_addr(va, asid);
+
return true;
}
Index: src/sys/arch/riscv/riscv/locore.S
diff -u src/sys/arch/riscv/riscv/locore.S:1.42 src/sys/arch/riscv/riscv/locore.S:1.43
--- src/sys/arch/riscv/riscv/locore.S:1.42 Mon Jun 12 19:04:14 2023
+++ src/sys/arch/riscv/riscv/locore.S Sun Sep 3 08:48:20 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.42 2023/06/12 19:04:14 skrll Exp $ */
+/* $NetBSD: locore.S,v 1.43 2023/09/03 08:48:20 skrll Exp $ */
/*-
* Copyright (c) 2014, 2022 The NetBSD Foundation, Inc.
@@ -316,7 +316,6 @@ vstart:
PTR_LA tp, _C_LABEL(lwp0) // put curlwp in tp
-
/* Set supervisor trap vector base register */
PTR_LA a0, _C_LABEL(cpu_exception_handler)
csrw stvec, a0
@@ -379,30 +378,12 @@ ENTRY(cpu_mpstart)
mv s11, a1 // copy sbi_hart_start cookie
/*
- * resolve our cpuindex. each hartid is stored in
- * extern unsigned long cpu_hartid[MAXCPUS]
- */
- PTR_LA t0, _C_LABEL(cpu_hartid)
- li t6, MAXCPUS
- li t1, 0
-1:
- addi t1, t1, 1
- bgeu t1, t6, toomanyharts /* cpuindex >= MAXCPUS ? */
-
- slli t2, t1, LONG_SCALESHIFT
- add t2, t0, t2
- LONG_L t3, 0(t2) /* cpu_hartid[cpuindex] */
- bne t3, s10, 1b
-
- mv s9, t1 /* s9 = cpuindex */
-
- /*
- * s9 = cpuindex
+ * s11 = cpuindex
*/
/* set stack pointer for boot */
li t1, BOOT_AP_STACKSIZE // XXXNH do a shift
- mul t1, s9, t1
+ mul t1, s11, t1
PTR_LA t0, _C_LABEL(bootstk)
/* sp = bootstk + (BOOT_AP_STACKSIZE * cpuindex) */
add sp, t0, t1
@@ -465,17 +446,17 @@ vmpstart:
csrsi sstatus, SR_SIE // enable interrupts
li tp, 0
- mv a0, s9
+ mv a0, s11
call _C_LABEL(cpu_init_secondary_processor)
/* t3 = __BIT(cpuindex % (sizeof(u_long) * NBBY)) */
li t3, 1
- andi t0, s9, (1U << (LONG_SCALESHIFT + NBBY_SHIFT)) - 1
+ andi t0, s11, (1U << (LONG_SCALESHIFT + NBBY_SHIFT)) - 1
sll t3, t3, t0
/* t4 = &riscv_cpu_mbox[cpuindex / (sizeof(u_long) * NBBY)] */
PTR_LA t0, _C_LABEL(riscv_cpu_mbox)
- srli t1, s9, LONG_SCALESHIFT + NBBY_SHIFT
+ srli t1, s11, LONG_SCALESHIFT + NBBY_SHIFT
slli t1, t1, LONG_SCALESHIFT
add t4, t0, t1
@@ -494,7 +475,7 @@ vmpstart:
csrw stvec, a0
li t0, CI_SIZE
- mul t0, s9, t0
+ mul t0, s11, t0
PTR_LA t1, _C_LABEL(cpu_info_store)
add a0, t0, t1 /* a0 = &cpu_info_store[cpuindex] */
@@ -513,6 +494,7 @@ vmpstart:
li s0, 0 /* trace back starts here (fp = 0) */
PTR_L a0, L_CPU(tp) /* curlwp->l_cpu */
+ mv a1, s11 /* cpuindex */
call _C_LABEL(cpu_hatch)
li s0, 0 // zero frame pointer
@@ -618,7 +600,6 @@ cpu_Debugger_ret:
ret
END(cpu_Debugger)
-#if defined(VERBOSE_INIT_RISCV)
ENTRY_NP(locore_prints)
addi sp, sp, -(SZREG * 2)
REG_S s0, (0 * SZREG)(sp)
@@ -642,6 +623,7 @@ ENTRY_NP(locore_prints)
END(locore_prints)
+#if defined(VERBOSE_INIT_RISCV)
ENTRY_NP(locore_printx)
addi sp, sp, -(SZREG * 4)
REG_S s0, (0 * SZREG)(sp)
Index: src/sys/arch/riscv/riscv/pmap_machdep.c
diff -u src/sys/arch/riscv/riscv/pmap_machdep.c:1.18 src/sys/arch/riscv/riscv/pmap_machdep.c:1.19
--- src/sys/arch/riscv/riscv/pmap_machdep.c:1.18 Mon Jun 12 19:04:14 2023
+++ src/sys/arch/riscv/riscv/pmap_machdep.c Sun Sep 3 08:48:20 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap_machdep.c,v 1.18 2023/06/12 19:04:14 skrll Exp $ */
+/* $NetBSD: pmap_machdep.c,v 1.19 2023/09/03 08:48:20 skrll Exp $ */
/*
* Copyright (c) 2014, 2019, 2021 The NetBSD Foundation, Inc.
@@ -36,10 +36,11 @@
#define __PMAP_PRIVATE
#include <sys/cdefs.h>
-__RCSID("$NetBSD: pmap_machdep.c,v 1.18 2023/06/12 19:04:14 skrll Exp $");
+__RCSID("$NetBSD: pmap_machdep.c,v 1.19 2023/09/03 08:48:20 skrll Exp $");
#include <sys/param.h>
#include <sys/buf.h>
+#include <sys/cpu.h>
#include <uvm/uvm.h>
@@ -175,7 +176,7 @@ pmap_md_xtab_activate(struct pmap *pmap,
{
// UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
- struct cpu_info * const ci = curcpu();
+// struct cpu_info * const ci = curcpu();
struct pmap_asid_info * const pai = PMAP_PAI(pmap, cpu_tlb_info(ci));
uint64_t satp =
@@ -335,8 +336,9 @@ pmap_bootstrap(vaddr_t vstart, vaddr_t v
kcpuset_create(&pm->pm_active, true);
KASSERT(pm->pm_onproc != NULL);
KASSERT(pm->pm_active != NULL);
- kcpuset_set(pm->pm_onproc, cpu_number());
- kcpuset_set(pm->pm_active, cpu_number());
+
+ kcpuset_set(pm->pm_onproc, cpu_index(ci));
+ kcpuset_set(pm->pm_active, cpu_index(ci));
#endif
VPRINTF("nkmempages ");
Index: src/sys/arch/riscv/riscv/riscv_machdep.c
diff -u src/sys/arch/riscv/riscv/riscv_machdep.c:1.33 src/sys/arch/riscv/riscv/riscv_machdep.c:1.34
--- src/sys/arch/riscv/riscv/riscv_machdep.c:1.33 Thu Aug 24 05:46:55 2023
+++ src/sys/arch/riscv/riscv/riscv_machdep.c Sun Sep 3 08:48:20 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: riscv_machdep.c,v 1.33 2023/08/24 05:46:55 rin Exp $ */
+/* $NetBSD: riscv_machdep.c,v 1.34 2023/09/03 08:48:20 skrll Exp $ */
/*-
* Copyright (c) 2014, 2019, 2022 The NetBSD Foundation, Inc.
@@ -35,7 +35,7 @@
#include "opt_riscv_debug.h"
#include <sys/cdefs.h>
-__RCSID("$NetBSD: riscv_machdep.c,v 1.33 2023/08/24 05:46:55 rin Exp $");
+__RCSID("$NetBSD: riscv_machdep.c,v 1.34 2023/09/03 08:48:20 skrll Exp $");
#include <sys/param.h>
@@ -536,8 +536,8 @@ cpu_startup(void)
kcpuset_create(&cpus_running, true);
KASSERT(cpus_running != NULL);
- kcpuset_set(cpus_hatched, cpu_number());
- kcpuset_set(cpus_running, cpu_number());
+ kcpuset_set(cpus_hatched, cpu_index(curcpu()));
+ kcpuset_set(cpus_running, cpu_index(curcpu()));
#endif
fdtbus_intr_init();
