Module Name: src
Committed By: maxv
Date: Sun Apr 12 07:49:58 UTC 2020
Modified Files:
src/sys/arch/aarch64/aarch64: aarch64_machdep.c cpufunc.c cpuswitch.S
db_trace.c genassym.cf locore.S vectors.S vm_machdep.c
src/sys/arch/aarch64/conf: Makefile.aarch64
src/sys/arch/aarch64/include: armreg.h asm.h cpufunc.h proc.h
src/sys/arch/arm/conf: files.arm
src/sys/arch/evbarm/conf: GENERIC64
Log Message:
Add support for Pointer Authentication (PAC).
We use the "pac-ret" option, to sign the return instruction pointer on
function entry, and authenticate it on function exit. This acts as a
mitigation against ROP.
The authentication uses a per-lwp (secret) I-A key stored in the 128bit
APIAKey register and part of the lwp context. During lwp creation, the
kernel generates a random key, and during context switches, it installs
the key of the target lwp on the CPU.
Userland cannot read the APIAKey register directly. However, it can sign
its pointers with it, because the register is architecturally shared
between userland and the kernel. Although part of the CPU design, it is
a bit of an undesired behavior, because it allows to forge valid kernel
pointers from userland. To avoid that, we don't share the key with
userland, and rather switch it in EL0<->EL1 transitions. This means that
when userland executes, a different key is loaded in APIAKey than the one
the kernel uses. For now the userland key is a fixed 128bit zero value.
The DDB stack unwinder is changed to strip the authentication code from
the pointers in lr.
Two problems are known:
* Currently the idlelwps' keys are not really secret. This is because
the RNG is not yet available when we spawn these lwps. Not overly
important, but would be nice to fix with UEFI RNG.
* The key switching in EL0<->EL1 transitions is not the most optimized
code on the planet. Instead of checking aarch64_pac_enabled, it would
be better to hot-patch the code at boot time, but there currently is
no hot-patch support on aarch64.
Tested on Qemu.
To generate a diff of this commit:
cvs rdiff -u -r1.40 -r1.41 src/sys/arch/aarch64/aarch64/aarch64_machdep.c
cvs rdiff -u -r1.16 -r1.17 src/sys/arch/aarch64/aarch64/cpufunc.c
cvs rdiff -u -r1.15 -r1.16 src/sys/arch/aarch64/aarch64/cpuswitch.S
cvs rdiff -u -r1.8 -r1.9 src/sys/arch/aarch64/aarch64/db_trace.c
cvs rdiff -u -r1.22 -r1.23 src/sys/arch/aarch64/aarch64/genassym.cf
cvs rdiff -u -r1.58 -r1.59 src/sys/arch/aarch64/aarch64/locore.S
cvs rdiff -u -r1.12 -r1.13 src/sys/arch/aarch64/aarch64/vectors.S
cvs rdiff -u -r1.5 -r1.6 src/sys/arch/aarch64/aarch64/vm_machdep.c
cvs rdiff -u -r1.17 -r1.18 src/sys/arch/aarch64/conf/Makefile.aarch64
cvs rdiff -u -r1.39 -r1.40 src/sys/arch/aarch64/include/armreg.h
cvs rdiff -u -r1.6 -r1.7 src/sys/arch/aarch64/include/asm.h
cvs rdiff -u -r1.11 -r1.12 src/sys/arch/aarch64/include/cpufunc.h
cvs rdiff -u -r1.5 -r1.6 src/sys/arch/aarch64/include/proc.h
cvs rdiff -u -r1.153 -r1.154 src/sys/arch/arm/conf/files.arm
cvs rdiff -u -r1.149 -r1.150 src/sys/arch/evbarm/conf/GENERIC64
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/aarch64/aarch64/aarch64_machdep.c
diff -u src/sys/arch/aarch64/aarch64/aarch64_machdep.c:1.40 src/sys/arch/aarch64/aarch64/aarch64_machdep.c:1.41
--- src/sys/arch/aarch64/aarch64/aarch64_machdep.c:1.40 Sat Feb 29 21:36:03 2020
+++ src/sys/arch/aarch64/aarch64/aarch64_machdep.c Sun Apr 12 07:49:58 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: aarch64_machdep.c,v 1.40 2020/02/29 21:36:03 ryo Exp $ */
+/* $NetBSD: aarch64_machdep.c,v 1.41 2020/04/12 07:49:58 maxv Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aarch64_machdep.c,v 1.40 2020/02/29 21:36:03 ryo Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aarch64_machdep.c,v 1.41 2020/04/12 07:49:58 maxv Exp $");
#include "opt_arm_debug.h"
#include "opt_ddb.h"
@@ -456,6 +456,14 @@ SYSCTL_SETUP(sysctl_machdep_setup, "sysc
SYSCTL_DESCR("top byte ignored in the address calculation"),
sysctl_machdep_tagged_address, 0, NULL, 0,
CTL_MACHDEP, CTL_CREATE, CTL_EOL);
+
+ sysctl_createv(clog, 0, NULL, NULL,
+ CTLFLAG_PERMANENT,
+ CTLTYPE_INT, "pac",
+ SYSCTL_DESCR("Whether Pointer Authentication is enabled"),
+ NULL, 0,
+ &aarch64_pac_enabled, 0,
+ CTL_MACHDEP, CTL_CREATE, CTL_EOL);
}
void
Index: src/sys/arch/aarch64/aarch64/cpufunc.c
diff -u src/sys/arch/aarch64/aarch64/cpufunc.c:1.16 src/sys/arch/aarch64/aarch64/cpufunc.c:1.17
--- src/sys/arch/aarch64/aarch64/cpufunc.c:1.16 Sun Apr 5 22:54:51 2020
+++ src/sys/arch/aarch64/aarch64/cpufunc.c Sun Apr 12 07:49:58 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.c,v 1.16 2020/04/05 22:54:51 jmcneill Exp $ */
+/* $NetBSD: cpufunc.c,v 1.17 2020/04/12 07:49:58 maxv Exp $ */
/*
* Copyright (c) 2017 Ryo Shimizu <r...@nerv.org>
@@ -26,10 +26,11 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
+#include "opt_cpuoptions.h"
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.16 2020/04/05 22:54:51 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.17 2020/04/12 07:49:58 maxv Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -46,6 +47,8 @@ u_int arm_dcache_maxline;
u_int aarch64_cache_vindexsize;
u_int aarch64_cache_prefer_mask;
+int aarch64_pac_enabled __read_mostly;
+
/* cache info per cluster. the same cluster has the same cache configuration? */
#define MAXCPUPACKAGES MAXCPUS /* maximum of ci->ci_package_id */
static struct aarch64_cache_info *aarch64_cacheinfo[MAXCPUPACKAGES];
@@ -438,3 +441,44 @@ set_cpufuncs(void)
return 0;
}
+
+/*
+ * TODO: this function should have a "no-pac" attribute. Right now it
+ * doesn't use PAC so that's fine.
+ */
+void
+aarch64_pac_init(int primary)
+{
+#ifdef ARMV83_PAC
+ uint64_t reg, sctlr;
+
+ /* CPU0 does the detection. */
+ if (primary) {
+ reg = reg_id_aa64isar1_el1_read();
+ if (__SHIFTOUT(reg, ID_AA64ISAR1_EL1_APA) !=
+ ID_AA64ISAR1_EL1_APA_NONE)
+ aarch64_pac_enabled = 1;
+ if (__SHIFTOUT(reg, ID_AA64ISAR1_EL1_API) !=
+ ID_AA64ISAR1_EL1_API_NONE)
+ aarch64_pac_enabled = 1;
+ if (__SHIFTOUT(reg, ID_AA64ISAR1_EL1_GPA) !=
+ ID_AA64ISAR1_EL1_GPA_NONE)
+ aarch64_pac_enabled = 1;
+ if (__SHIFTOUT(reg, ID_AA64ISAR1_EL1_GPI) !=
+ ID_AA64ISAR1_EL1_GPI_NONE)
+ aarch64_pac_enabled = 1;
+ }
+
+ if (!aarch64_pac_enabled)
+ return;
+
+ /* Enable PAC on the CPU. */
+ sctlr = reg_sctlr_el1_read();
+ sctlr |= SCTLR_EnIA;
+ reg_sctlr_el1_write(sctlr);
+
+ /* Set the key. Curlwp here is the CPU's idlelwp. */
+ reg_APIAKeyLo_EL1_write(curlwp->l_md.md_ia_kern_lo);
+ reg_APIAKeyHi_EL1_write(curlwp->l_md.md_ia_kern_hi);
+#endif
+}
Index: src/sys/arch/aarch64/aarch64/cpuswitch.S
diff -u src/sys/arch/aarch64/aarch64/cpuswitch.S:1.15 src/sys/arch/aarch64/aarch64/cpuswitch.S:1.16
--- src/sys/arch/aarch64/aarch64/cpuswitch.S:1.15 Wed Jan 8 20:59:18 2020
+++ src/sys/arch/aarch64/aarch64/cpuswitch.S Sun Apr 12 07:49:58 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpuswitch.S,v 1.15 2020/01/08 20:59:18 skrll Exp $ */
+/* $NetBSD: cpuswitch.S,v 1.16 2020/04/12 07:49:58 maxv Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -34,10 +34,13 @@
#include "assym.h"
#include "opt_compat_netbsd32.h"
+#include "opt_cpuoptions.h"
#include "opt_ddb.h"
#include "opt_kasan.h"
-RCSID("$NetBSD: cpuswitch.S,v 1.15 2020/01/08 20:59:18 skrll Exp $")
+RCSID("$NetBSD: cpuswitch.S,v 1.16 2020/04/12 07:49:58 maxv Exp $")
+
+ ARMV8_DEFINE_OPTIONS
/*
* At IPL_SCHED:
@@ -77,6 +80,18 @@ ENTRY_NP(cpu_switchto)
mov sp, x4 /* restore stack pointer */
msr cpacr_el1, x5 /* restore cpacr_el1 */
+#ifdef ARMV83_PAC
+ /* Switch the PAC key. */
+ adrl x4, _C_LABEL(aarch64_pac_enabled)
+ ldr w4, [x4]
+ cbz w4, 1f
+ ldr x5, [x1, #L_MD_IA_KERN_LO]
+ ldr x6, [x1, #L_MD_IA_KERN_HI]
+ msr APIAKeyLo_EL1, x5
+ msr APIAKeyHi_EL1, x6
+1:
+#endif
+
mrs x3, tpidr_el1
str x1, [x3, #CI_CURLWP] /* switch curlwp to new lwp */
ENABLE_INTERRUPT
@@ -132,6 +147,18 @@ ENTRY_NP(cpu_switchto_softint)
str x5, [x19, #L_MD_CPACR]
str x0, [x3, #CI_CURLWP] /* curcpu()->ci_curlwp = softlwp; */
+#ifdef ARMV83_PAC
+ /* Switch the PAC key. */
+ adrl x4, _C_LABEL(aarch64_pac_enabled)
+ ldr w4, [x4]
+ cbz w4, 1f
+ ldr x5, [x0, #L_MD_IA_KERN_LO]
+ ldr x6, [x0, #L_MD_IA_KERN_HI]
+ msr APIAKeyLo_EL1, x5
+ msr APIAKeyHi_EL1, x6
+1:
+#endif
+
#ifdef KASAN
/* clear the new stack */
stp x0, x1, [sp, #-16]!
@@ -159,6 +186,18 @@ ENTRY_NP(cpu_switchto_softint)
mov sp, x4 /* restore pinned_lwp sp */
msr cpacr_el1, x5 /* restore pinned_lwp cpacr */
+#ifdef ARMV83_PAC
+ /* Restore the PAC key. */
+ adrl x4, _C_LABEL(aarch64_pac_enabled)
+ ldr w4, [x4]
+ cbz w4, 1f
+ ldr x5, [x19, #L_MD_IA_KERN_LO]
+ ldr x6, [x19, #L_MD_IA_KERN_HI]
+ msr APIAKeyLo_EL1, x5
+ msr APIAKeyHi_EL1, x6
+1:
+#endif
+
ldp x19, x20, [sp, #TF_X19]
msr daif, x19 /* restore interrupt mask */
mov lr, x20 /* restore pinned_lwp lr */
@@ -199,7 +238,24 @@ END(softint_cleanup)
* x28 = arg
*/
ENTRY_NP(lwp_trampoline)
+ stp x0, x1, [sp, #-16]! /* save x0, x1 */
bl _C_LABEL(lwp_startup)
+ ldp x0, x1, [sp], #16 /* restore x0, x1 */
+
+#ifdef ARMV83_PAC
+ /* Generate a new PAC key. */
+ adrl x4, _C_LABEL(aarch64_pac_enabled)
+ ldr w4, [x4]
+ cbz w4, 1f
+ mov x26, x1
+ bl _C_LABEL(cprng_strong64)
+ ldr x0, [x26, #L_MD_IA_KERN_LO]
+ msr APIAKeyLo_EL1, x0
+ bl _C_LABEL(cprng_strong64)
+ ldr x0, [x26, #L_MD_IA_KERN_HI]
+ msr APIAKeyHi_EL1, x0
+1:
+#endif
/*
* When the x27 function returns, it will jump to el0_trap_exit.
@@ -311,6 +367,18 @@ ENTRY_NP(el0_trap_exit)
msr tpidrro_el0, x0
#endif
+#ifdef ARMV83_PAC
+ /* Switch to the user PAC key. */
+ adrl x4, _C_LABEL(aarch64_pac_enabled)
+ ldr w4, [x4]
+ cbz w4, 1f
+ ldr x5, [x9, #L_MD_IA_USER_LO]
+ ldr x6, [x9, #L_MD_IA_USER_HI]
+ msr APIAKeyLo_EL1, x5
+ msr APIAKeyHi_EL1, x6
+1:
+#endif
+
unwind_x3_x30
#if TF_PC + 8 == TF_SPSR
Index: src/sys/arch/aarch64/aarch64/db_trace.c
diff -u src/sys/arch/aarch64/aarch64/db_trace.c:1.8 src/sys/arch/aarch64/aarch64/db_trace.c:1.9
--- src/sys/arch/aarch64/aarch64/db_trace.c:1.8 Sun Jan 27 02:08:36 2019
+++ src/sys/arch/aarch64/aarch64/db_trace.c Sun Apr 12 07:49:58 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: db_trace.c,v 1.8 2019/01/27 02:08:36 pgoyette Exp $ */
+/* $NetBSD: db_trace.c,v 1.9 2020/04/12 07:49:58 maxv Exp $ */
/*
* Copyright (c) 2017 Ryo Shimizu <r...@nerv.org>
@@ -28,7 +28,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: db_trace.c,v 1.8 2019/01/27 02:08:36 pgoyette Exp $");
+__KERNEL_RCSID(0, "$NetBSD: db_trace.c,v 1.9 2020/04/12 07:49:58 maxv Exp $");
#include <sys/param.h>
#include <sys/proc.h>
@@ -236,6 +236,7 @@ db_stack_trace_print(db_expr_t addr, boo
lastfp = lastlr = lr = fp = 0;
db_read_bytes((db_addr_t)&tf->tf_pc, sizeof(lr), (char *)&lr);
db_read_bytes((db_addr_t)&tf->tf_reg[29], sizeof(fp), (char *)&fp);
+ lr = ptr_strip_pac(lr);
pr_traceaddr("fp", fp, lr - 4, flags, pr);
}
@@ -251,6 +252,7 @@ db_stack_trace_print(db_expr_t addr, boo
*/
db_read_bytes(lastfp + 0, sizeof(fp), (char *)&fp);
db_read_bytes(lastfp + 8, sizeof(lr), (char *)&lr);
+ lr = ptr_strip_pac(lr);
if (!trace_user && IN_USER_VM_ADDRESS(lr))
break;
@@ -268,6 +270,7 @@ db_stack_trace_print(db_expr_t addr, boo
lr = fp = 0;
db_read_bytes((db_addr_t)&tf->tf_pc, sizeof(lr), (char *)&lr);
db_read_bytes((db_addr_t)&tf->tf_reg[29], sizeof(fp), (char *)&fp);
+ lr = ptr_strip_pac(lr);
/*
* no need to display the frame of el0_trap
Index: src/sys/arch/aarch64/aarch64/genassym.cf
diff -u src/sys/arch/aarch64/aarch64/genassym.cf:1.22 src/sys/arch/aarch64/aarch64/genassym.cf:1.23
--- src/sys/arch/aarch64/aarch64/genassym.cf:1.22 Thu Feb 20 08:27:38 2020
+++ src/sys/arch/aarch64/aarch64/genassym.cf Sun Apr 12 07:49:58 2020
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.22 2020/02/20 08:27:38 skrll Exp $
+# $NetBSD: genassym.cf,v 1.23 2020/04/12 07:49:58 maxv Exp $
#-
# Copyright (c) 2014 The NetBSD Foundation, Inc.
# All rights reserved.
@@ -150,6 +150,10 @@ define L_MD_FLAGS offsetof(struct lwp,
define L_MD_UTF offsetof(struct lwp, l_md.md_utf)
define L_MD_CPACR offsetof(struct lwp, l_md.md_cpacr)
define L_MD_ONFAULT offsetof(struct lwp, l_md.md_onfault)
+define L_MD_IA_KERN_LO offsetof(struct lwp, l_md.md_ia_kern_lo)
+define L_MD_IA_KERN_HI offsetof(struct lwp, l_md.md_ia_kern_hi)
+define L_MD_IA_USER_LO offsetof(struct lwp, l_md.md_ia_user_lo)
+define L_MD_IA_USER_HI offsetof(struct lwp, l_md.md_ia_user_hi)
define FB_X19 FB_X19
define FB_X20 FB_X20
Index: src/sys/arch/aarch64/aarch64/locore.S
diff -u src/sys/arch/aarch64/aarch64/locore.S:1.58 src/sys/arch/aarch64/aarch64/locore.S:1.59
--- src/sys/arch/aarch64/aarch64/locore.S:1.58 Thu Feb 20 07:52:55 2020
+++ src/sys/arch/aarch64/aarch64/locore.S Sun Apr 12 07:49:58 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.58 2020/02/20 07:52:55 skrll Exp $ */
+/* $NetBSD: locore.S,v 1.59 2020/04/12 07:49:58 maxv Exp $ */
/*
* Copyright (c) 2017 Ryo Shimizu <r...@nerv.org>
@@ -38,7 +38,7 @@
#include <aarch64/hypervisor.h>
#include "assym.h"
-RCSID("$NetBSD: locore.S,v 1.58 2020/02/20 07:52:55 skrll Exp $")
+RCSID("$NetBSD: locore.S,v 1.59 2020/04/12 07:49:58 maxv Exp $")
#ifdef AARCH64_DEVICE_MEM_STRONGLY_ORDERED
#define MAIR_DEVICE_MEM MAIR_DEVICE_nGnRnE
@@ -178,6 +178,10 @@ vstart:
msr tpidr_el1, x0 /* curcpu is cpu_info[0] */
DPRINTREG("curcpu = ", x0);
+ /* init PAC if supported */
+ mov x0, #1
+ bl aarch64_pac_init
+
/* set topology information */
mrs x0, tpidr_el1 /* curcpu */
mrs x1, mpidr_el1
@@ -502,8 +506,12 @@ mp_vstart:
add x2, x2, #(UPAGES * PAGE_SIZE)
sub sp, x2, #TF_SIZE /* sp = pcb + USPACE - TF_SIZE */
+ /* init PAC if supported */
+ mov x0, #0
+ bl aarch64_pac_init
mov fp, xzr /* trace back starts here */
+ mrs x0, tpidr_el1 /* curcpu() */
bl _C_LABEL(cpu_hatch)
mov x0, xzr
b _C_LABEL(idle_loop) /* never to return */
Index: src/sys/arch/aarch64/aarch64/vectors.S
diff -u src/sys/arch/aarch64/aarch64/vectors.S:1.12 src/sys/arch/aarch64/aarch64/vectors.S:1.13
--- src/sys/arch/aarch64/aarch64/vectors.S:1.12 Sat Apr 11 09:02:04 2020
+++ src/sys/arch/aarch64/aarch64/vectors.S Sun Apr 12 07:49:58 2020
@@ -1,10 +1,13 @@
-/* $NetBSD: vectors.S,v 1.12 2020/04/11 09:02:04 maxv Exp $ */
+/* $NetBSD: vectors.S,v 1.13 2020/04/12 07:49:58 maxv Exp $ */
#include <aarch64/asm.h>
#include "assym.h"
+#include "opt_cpuoptions.h"
#include "opt_ddb.h"
#include "opt_dtrace.h"
+ ARMV8_DEFINE_OPTIONS
+
#ifdef KDTRACE_HOOKS
/*
* dtrace needs to emulate stp x29,x30,[sp,#-FRAMESIZE]! where
@@ -81,6 +84,18 @@ ENTRY_NP(\func)
ldr x1, [x1, #CI_CURLWP] /* x1 = curcpu()->ci_curlwp */
mrs x0, tpidr\tpidr\()_el0
str x0, [x1, #L_PRIVATE] /* curlwp->l_private = tpidr{,ro}_el0 */
+
+#ifdef ARMV83_PAC
+ /* Switch to the kern PAC key. */
+ adrl x4, _C_LABEL(aarch64_pac_enabled)
+ ldr w4, [x4]
+ cbz w4, 1f
+ ldr x5, [x1, #L_MD_IA_KERN_LO]
+ ldr x6, [x1, #L_MD_IA_KERN_HI]
+ msr APIAKeyLo_EL1, x5
+ msr APIAKeyHi_EL1, x6
+1:
+#endif
.endif
adr x30, el\el\()_trap_exit /* el[01]_trap_exit */
Index: src/sys/arch/aarch64/aarch64/vm_machdep.c
diff -u src/sys/arch/aarch64/aarch64/vm_machdep.c:1.5 src/sys/arch/aarch64/aarch64/vm_machdep.c:1.6
--- src/sys/arch/aarch64/aarch64/vm_machdep.c:1.5 Thu Dec 27 09:55:27 2018
+++ src/sys/arch/aarch64/aarch64/vm_machdep.c Sun Apr 12 07:49:58 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vm_machdep.c,v 1.5 2018/12/27 09:55:27 mrg Exp $ */
+/* $NetBSD: vm_machdep.c,v 1.6 2020/04/12 07:49:58 maxv Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vm_machdep.c,v 1.5 2018/12/27 09:55:27 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vm_machdep.c,v 1.6 2020/04/12 07:49:58 maxv Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -96,6 +96,17 @@ cpu_lwp_fork(struct lwp *l1, struct lwp
l2->l_md.md_cpacr = CPACR_FPEN_NONE;
/*
+ * Temporary kern PAC key, not really strong, but better than zero.
+ * The real key gets generated by the new thread in lwp_trampoline().
+ *
+ * The reason we defer the generation, is because cpu_lwp_fork() can
+ * be called early at boot time, when the CPRNG subsystem hasn't yet
+ * been initialized.
+ */
+ l2->l_md.md_ia_kern_lo = (uintptr_t)l2;
+ l2->l_md.md_ia_kern_hi = (uintptr_t)stack;
+
+ /*
* Set up the kernel stack for the process.
* Note: this stack is not in use if we are forking from p1
*/
Index: src/sys/arch/aarch64/conf/Makefile.aarch64
diff -u src/sys/arch/aarch64/conf/Makefile.aarch64:1.17 src/sys/arch/aarch64/conf/Makefile.aarch64:1.18
--- src/sys/arch/aarch64/conf/Makefile.aarch64:1.17 Wed Mar 4 19:28:04 2020
+++ src/sys/arch/aarch64/conf/Makefile.aarch64 Sun Apr 12 07:49:58 2020
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile.aarch64,v 1.17 2020/03/04 19:28:04 ryo Exp $
+# $NetBSD: Makefile.aarch64,v 1.18 2020/04/12 07:49:58 maxv Exp $
# Makefile for NetBSD
#
@@ -39,6 +39,10 @@ CFLAGS+= -fno-omit-frame-pointer
CFLAGS+= -mno-omit-leaf-frame-pointer
#CFLAGS+= -mno-unaligned-access
+.if ${ARMV83_PAC:U0} > 0 && ${HAVE_LLVM:Uno} == "yes"
+CFLAGS+= -mbranch-protection=pac-ret
+.endif
+
.if ${KASAN:U0} > 0 && ${HAVE_GCC:U0} > 0
KASANFLAGS= -fsanitize=kernel-address \
--param asan-globals=1 --param asan-stack=1 \
Index: src/sys/arch/aarch64/include/armreg.h
diff -u src/sys/arch/aarch64/include/armreg.h:1.39 src/sys/arch/aarch64/include/armreg.h:1.40
--- src/sys/arch/aarch64/include/armreg.h:1.39 Mon Mar 30 11:38:29 2020
+++ src/sys/arch/aarch64/include/armreg.h Sun Apr 12 07:49:58 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: armreg.h,v 1.39 2020/03/30 11:38:29 jmcneill Exp $ */
+/* $NetBSD: armreg.h,v 1.40 2020/04/12 07:49:58 maxv Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -786,6 +786,31 @@ AARCH64REG_WRITE_INLINE(sctlr_el1)
#define SCTLR_ATA __BIT(43)
#define SCTLR_DSSBS __BIT(44)
+static __inline void
+reg_APIAKeyLo_EL1_write(uint64_t __val)
+{
+ __asm __volatile(".arch armv8.3-a+pac\n"
+ "msr APIAKeyLo_EL1, %0" :: "r"(__val));
+}
+
+static __inline void
+reg_APIAKeyHi_EL1_write(uint64_t __val)
+{
+ __asm __volatile(".arch armv8.3-a+pac\n"
+ "msr APIAKeyHi_EL1, %0" :: "r"(__val));
+}
+
+#define PTR_VA_RANGE_SELECT __BIT(55)
+#define PTR_PAC_MASK (__BITS(63,56) | __BITS(54, 48))
+
+static __inline uint64_t
+ptr_strip_pac(uint64_t __val)
+{
+ if (__val & PTR_VA_RANGE_SELECT)
+ return __val | PTR_PAC_MASK;
+ return __val & ~PTR_PAC_MASK;
+}
+
// current EL stack pointer
static __inline uint64_t
reg_sp_read(void)
Index: src/sys/arch/aarch64/include/asm.h
diff -u src/sys/arch/aarch64/include/asm.h:1.6 src/sys/arch/aarch64/include/asm.h:1.7
--- src/sys/arch/aarch64/include/asm.h:1.6 Sun Jan 19 16:12:56 2020
+++ src/sys/arch/aarch64/include/asm.h Sun Apr 12 07:49:58 2020
@@ -1,8 +1,12 @@
-/* $NetBSD: asm.h,v 1.6 2020/01/19 16:12:56 skrll Exp $ */
+/* $NetBSD: asm.h,v 1.7 2020/04/12 07:49:58 maxv Exp $ */
#ifndef _AARCH64_ASM_H_
#define _AARCH64_ASM_H_
+#if defined(_KERNEL_OPT)
+#include "opt_cpuoptions.h"
+#endif
+
#include <arm/asm.h>
#ifdef __aarch64__
@@ -25,6 +29,20 @@
#define ERET \
eret; dsb sy; isb
+/*
+ * ARMv8 options to be made available for the compiler to use. Should be
+ * inserted at the beginning of the ASM files that need them.
+ *
+ * For now the only option is PAC, needed for the compiler to recognize
+ * the key registers.
+ */
+#ifdef ARMV83_PAC
+#define ARMV8_DEFINE_OPTIONS \
+ .arch armv8.3-a+pac
+#else
+#define ARMV8_DEFINE_OPTIONS /* nothing */
+#endif
+
#endif
#endif /* !_AARCH64_ASM_H_ */
Index: src/sys/arch/aarch64/include/cpufunc.h
diff -u src/sys/arch/aarch64/include/cpufunc.h:1.11 src/sys/arch/aarch64/include/cpufunc.h:1.12
--- src/sys/arch/aarch64/include/cpufunc.h:1.11 Wed Jan 15 08:34:04 2020
+++ src/sys/arch/aarch64/include/cpufunc.h Sun Apr 12 07:49:58 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.h,v 1.11 2020/01/15 08:34:04 mrg Exp $ */
+/* $NetBSD: cpufunc.h,v 1.12 2020/04/12 07:49:58 maxv Exp $ */
/*
* Copyright (c) 2017 Ryo Shimizu <r...@nerv.org>
@@ -68,6 +68,10 @@ extern u_int aarch64_cache_vindexsize; /
extern u_int aarch64_cache_prefer_mask;
extern u_int cputype; /* compat arm */
+extern int aarch64_pac_enabled;
+
+void aarch64_pac_init(int);
+
int set_cpufuncs(void);
void aarch64_getcacheinfo(int);
void aarch64_printcacheinfo(device_t);
Index: src/sys/arch/aarch64/include/proc.h
diff -u src/sys/arch/aarch64/include/proc.h:1.5 src/sys/arch/aarch64/include/proc.h:1.6
--- src/sys/arch/aarch64/include/proc.h:1.5 Sun Nov 24 11:28:40 2019
+++ src/sys/arch/aarch64/include/proc.h Sun Apr 12 07:49:58 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: proc.h,v 1.5 2019/11/24 11:28:40 rin Exp $ */
+/* $NetBSD: proc.h,v 1.6 2020/04/12 07:49:58 maxv Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -43,6 +43,12 @@ struct mdlwp {
struct trapframe *md_utf;
uint64_t md_cpacr;
uint32_t md_flags;
+
+ uint64_t md_ia_kern_lo; /* APIAKeyLo_EL1 */
+ uint64_t md_ia_kern_hi; /* APIAKeyHi_EL1 */
+
+ uint64_t md_ia_user_lo; /* APIAKeyLo_EL1 */
+ uint64_t md_ia_user_hi; /* APIAKeyHi_EL1 */
};
struct mdproc {
Index: src/sys/arch/arm/conf/files.arm
diff -u src/sys/arch/arm/conf/files.arm:1.153 src/sys/arch/arm/conf/files.arm:1.154
--- src/sys/arch/arm/conf/files.arm:1.153 Sat Feb 15 08:16:11 2020
+++ src/sys/arch/arm/conf/files.arm Sun Apr 12 07:49:58 2020
@@ -1,4 +1,4 @@
-# $NetBSD: files.arm,v 1.153 2020/02/15 08:16:11 skrll Exp $
+# $NetBSD: files.arm,v 1.154 2020/04/12 07:49:58 maxv Exp $
# temporary define to allow easy moving to ../arch/arm/arm32
defflag ARM32
@@ -78,6 +78,9 @@ defflag opt_arm_debug.h ARM_LOCK_CAS_DE
defflag opt_arm_debug.h VERBOSE_INIT_ARM
defparam opt_arm_debug.h EARLYCONS
+# ARMv8-specific options
+defflag opt_cpuoptions.h ARMV83_PAC
+
# Board-specific bus_space(9)/bus_dma(9) definitions
defflag opt_arm_bus_space.h __BUS_SPACE_HAS_STREAM_METHODS
__BUS_SPACE_HAS_PROBING_METHODS
Index: src/sys/arch/evbarm/conf/GENERIC64
diff -u src/sys/arch/evbarm/conf/GENERIC64:1.149 src/sys/arch/evbarm/conf/GENERIC64:1.150
--- src/sys/arch/evbarm/conf/GENERIC64:1.149 Tue Mar 31 12:23:17 2020
+++ src/sys/arch/evbarm/conf/GENERIC64 Sun Apr 12 07:49:58 2020
@@ -1,5 +1,5 @@
#
-# $NetBSD: GENERIC64,v 1.149 2020/03/31 12:23:17 jmcneill Exp $
+# $NetBSD: GENERIC64,v 1.150 2020/04/12 07:49:58 maxv Exp $
#
# GENERIC ARM (aarch64) kernel
#
@@ -164,6 +164,10 @@ options INCLUDE_CONFIG_FILE
#options EARLYCONS=thunderx, CONSADDR=0x87e024000000
#options EARLYCONS=virt
+# Pointer Authentication (PAC).
+#makeoptions ARMV83_PAC=1
+#options ARMV83_PAC
+
# Kernel Address Sanitizer (kASan). The quarantine is optional and can
# help KASAN find more use-after-frees. Use KASAN_PANIC if you want panics
# instead of warnings.
