Module Name: src
Committed By: tls
Date: Wed Apr 1 03:56:54 UTC 2009
Modified Files:
src/sys/arch/i386/i386: machdep.c
src/sys/arch/x86/include: via_padlock.h
src/sys/arch/x86/x86: identcpu.c via_padlock.c
Log Message:
Fix probe for VIA C3 and successors -- these are CPU family 6, not 5.
The broken probe was causing the VIA padlock driver to never attach!
Now we can see that its AES appears to be broken -- it makes FAST_IPSEC
ESP not work, on systems where it works fine with cryptosoft.
Rework code to detect and (if necessary) enable VIA crypto and RNG.
Add RNG support to VIA padlock driver. In the process, have a quick
go at debugging the AES support but no luck thus far.
To generate a diff of this commit:
cvs rdiff -u -r1.667 -r1.668 src/sys/arch/i386/i386/machdep.c
cvs rdiff -u -r1.3 -r1.4 src/sys/arch/x86/include/via_padlock.h
cvs rdiff -u -r1.14 -r1.15 src/sys/arch/x86/x86/identcpu.c
cvs rdiff -u -r1.10 -r1.11 src/sys/arch/x86/x86/via_padlock.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/i386/i386/machdep.c
diff -u src/sys/arch/i386/i386/machdep.c:1.667 src/sys/arch/i386/i386/machdep.c:1.668
--- src/sys/arch/i386/i386/machdep.c:1.667 Sun Mar 29 10:58:54 2009
+++ src/sys/arch/i386/i386/machdep.c Wed Apr 1 03:56:54 2009
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.667 2009/03/29 10:58:54 ad Exp $ */
+/* $NetBSD: machdep.c,v 1.668 2009/04/01 03:56:54 tls Exp $ */
/*-
* Copyright (c) 1996, 1997, 1998, 2000, 2004, 2006, 2008, 2009
@@ -67,7 +67,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.667 2009/03/29 10:58:54 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.668 2009/04/01 03:56:54 tls Exp $");
#include "opt_beep.h"
#include "opt_compat_ibcs2.h"
@@ -250,6 +250,7 @@
unsigned int cpu_feature;
unsigned int cpu_feature2;
unsigned int cpu_feature_padlock;
+
int cpu_class;
int i386_fpu_present;
int i386_fpu_exception;
Index: src/sys/arch/x86/include/via_padlock.h
diff -u src/sys/arch/x86/include/via_padlock.h:1.3 src/sys/arch/x86/include/via_padlock.h:1.4
--- src/sys/arch/x86/include/via_padlock.h:1.3 Sat Mar 7 21:59:25 2009
+++ src/sys/arch/x86/include/via_padlock.h Wed Apr 1 03:56:54 2009
@@ -1,4 +1,4 @@
-/* $NetBSD: via_padlock.h,v 1.3 2009/03/07 21:59:25 ad Exp $ */
+/* $NetBSD: via_padlock.h,v 1.4 2009/04/01 03:56:54 tls Exp $ */
/*-
* Copyright (c) 2003 Jason Wright
@@ -23,6 +23,8 @@
#if defined(_KERNEL) || defined(_KMEMUSER)
+#include <sys/rnd.h>
+#include <sys/callout.h>
#include <crypto/rijndael/rijndael.h>
/* VIA C3 xcrypt-* instruction context control options */
@@ -55,6 +57,10 @@
uint8_t op_iv[16]; /* 128 bit aligned */
void *op_buf;
+ int sc_rnd_hz;
+ struct callout sc_rnd_co;
+ rndsource_element_t sc_rnd_source;
+
/* normal softc stuff */
int32_t sc_cid;
int sc_nsessions;
@@ -64,6 +70,8 @@
#define VIAC3_SESSION(sid) ((sid) & 0x0fffffff)
#define VIAC3_SID(crd,ses) (((crd) << 28) | ((ses) & 0x0fffffff))
+#define VIAC3_RNG_BUFSIZ 16
+
struct cpu_info;
struct via_padlock {
Index: src/sys/arch/x86/x86/identcpu.c
diff -u src/sys/arch/x86/x86/identcpu.c:1.14 src/sys/arch/x86/x86/identcpu.c:1.15
--- src/sys/arch/x86/x86/identcpu.c:1.14 Wed Mar 25 22:53:51 2009
+++ src/sys/arch/x86/x86/identcpu.c Wed Apr 1 03:56:54 2009
@@ -1,4 +1,4 @@
-/* $NetBSD: identcpu.c,v 1.14 2009/03/25 22:53:51 dyoung Exp $ */
+/* $NetBSD: identcpu.c,v 1.15 2009/04/01 03:56:54 tls Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -56,7 +56,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: identcpu.c,v 1.14 2009/03/25 22:53:51 dyoung Exp $");
+__KERNEL_RCSID(0, "$NetBSD: identcpu.c,v 1.15 2009/04/01 03:56:54 tls Exp $");
#include "opt_enhanced_speedstep.h"
#include "opt_intel_odcm.h"
@@ -480,7 +480,7 @@
struct x86_cache_info *cai;
if (cpu_vendor != CPUVENDOR_IDT ||
- CPUID2FAMILY(ci->ci_signature) != 5)
+ CPUID2FAMILY(ci->ci_signature) < 6)
return;
family = CPUID2FAMILY(ci->ci_signature);
@@ -497,25 +497,61 @@
ci->ci_feature_flags |= descs[3];
}
- if (model >= 0x9) {
+ if (family > 6 || model > 0x9 || (model == 0x9 && stepping >= 3)) {
/* Nehemiah or Esther */
x86_cpuid(0xc0000000, descs);
lfunc = descs[0];
if (lfunc >= 0xc0000001) { /* has ACE, RNG */
- x86_cpuid(0xc0000001, descs);
- lfunc = descs[3];
- if (model > 0x9 || stepping >= 8) { /* ACE */
- if (lfunc & CPUID_VIA_HAS_ACE) {
- ci->ci_padlock_flags = lfunc;
- if ((lfunc & CPUID_VIA_DO_ACE) == 0) {
- msr = rdmsr(MSR_VIA_ACE);
- wrmsr(MSR_VIA_ACE, msr |
- MSR_VIA_ACE_ENABLE);
- ci->ci_padlock_flags |=
- CPUID_VIA_DO_ACE;
- }
- }
+ int rng_enable = 0, ace_enable = 0;
+ x86_cpuid(0xc0000001, descs);
+ lfunc = descs[3];
+ ci->ci_padlock_flags = lfunc;
+ /* Check for and enable RNG */
+ if (lfunc & CPUID_VIA_HAS_RNG) {
+ if (!(lfunc & CPUID_VIA_DO_RNG)) {
+ rng_enable++;
+ ci->ci_padlock_flags |= CPUID_VIA_HAS_RNG;
+ }
+ }
+ /* Check for and enable ACE (AES-CBC) */
+ if (lfunc & CPUID_VIA_HAS_ACE) {
+ if (!(lfunc & CPUID_VIA_DO_ACE)) {
+ ace_enable++;
+ ci->ci_padlock_flags |= CPUID_VIA_DO_ACE;
}
+ }
+ /* Check for and enable SHA */
+ if (lfunc & CPUID_VIA_HAS_PHE) {
+ if (!(lfunc & CPUID_VIA_DO_PHE)) {
+ ace_enable++;
+ ci->ci_padlock_flags |= CPUID_VIA_DO_PHE;
+ }
+ }
+ /* Check for and enable ACE2 (AES-CTR) */
+ if (lfunc & CPUID_VIA_HAS_ACE2) {
+ if (!(lfunc & CPUID_VIA_DO_ACE2)) {
+ ace_enable++;
+ ci->ci_padlock_flags |= CPUID_VIA_DO_ACE2;
+ }
+ }
+ /* Check for and enable PMM (modmult engine) */
+ if (lfunc & CPUID_VIA_HAS_PMM) {
+ if (!(lfunc & CPUID_VIA_DO_PMM)) {
+ ace_enable++;
+ ci->ci_padlock_flags |= CPUID_VIA_DO_PMM;
+ }
+ }
+
+ /* Actually do the enables. */
+ if (rng_enable) {
+ msr = rdmsr(MSR_VIA_RNG);
+ wrmsr(MSR_VIA_RNG, msr | MSR_VIA_RNG_ENABLE);
+ }
+ if (ace_enable) {
+ msr = rdmsr(MSR_VIA_ACE);
+ wrmsr(MSR_VIA_ACE, msr | MSR_VIA_ACE_ENABLE);
+ }
+
}
}
@@ -543,7 +579,7 @@
cai->cai_totalsize = VIA_L1_ECX_DC_SIZE(descs[2]);
cai->cai_associativity = VIA_L1_ECX_DC_ASSOC(descs[2]);
cai->cai_linesize = VIA_L1_EDX_IC_LS(descs[2]);
- if (model == 9 && stepping == 8) {
+ if (family == 6 && model == 9 && stepping == 8) {
/* Erratum: stepping 8 reports 4 when it should be 2 */
cai->cai_associativity = 2;
}
@@ -552,11 +588,11 @@
cai->cai_totalsize = VIA_L1_EDX_IC_SIZE(descs[3]);
cai->cai_associativity = VIA_L1_EDX_IC_ASSOC(descs[3]);
cai->cai_linesize = VIA_L1_EDX_IC_LS(descs[3]);
- if (model == 9 && stepping == 8) {
+ if (family == 6 && model == 9 && stepping == 8) {
/* Erratum: stepping 8 reports 4 when it should be 2 */
cai->cai_associativity = 2;
}
-
+
/*
* Determine L2 cache/TLB info.
*/
@@ -568,7 +604,7 @@
x86_cpuid(0x80000006, descs);
cai = &ci->ci_cinfo[CAI_L2CACHE];
- if (model >= 9) {
+ if (family > 6 || model >= 9) {
cai->cai_totalsize = VIA_L2N_ECX_C_SIZE(descs[2]);
cai->cai_associativity = VIA_L2N_ECX_C_ASSOC(descs[2]);
cai->cai_linesize = VIA_L2N_ECX_C_LS(descs[2]);
Index: src/sys/arch/x86/x86/via_padlock.c
diff -u src/sys/arch/x86/x86/via_padlock.c:1.10 src/sys/arch/x86/x86/via_padlock.c:1.11
--- src/sys/arch/x86/x86/via_padlock.c:1.10 Wed Dec 17 20:51:33 2008
+++ src/sys/arch/x86/x86/via_padlock.c Wed Apr 1 03:56:54 2009
@@ -1,5 +1,5 @@
/* $OpenBSD: via.c,v 1.8 2006/11/17 07:47:56 tom Exp $ */
-/* $NetBSD: via_padlock.c,v 1.10 2008/12/17 20:51:33 cegger Exp $ */
+/* $NetBSD: via_padlock.c,v 1.11 2009/04/01 03:56:54 tls Exp $ */
/*-
* Copyright (c) 2003 Jason Wright
@@ -20,7 +20,13 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: via_padlock.c,v 1.10 2008/12/17 20:51:33 cegger Exp $");
+__KERNEL_RCSID(0, "$NetBSD: via_padlock.c,v 1.11 2009/04/01 03:56:54 tls Exp $");
+
+#include "rnd.h"
+
+#if NRND == 0
+#error padlock requires rnd pseudo-devices
+#endif
#include "opt_viapadlock.h"
@@ -32,6 +38,7 @@
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/cpu.h>
+#include <sys/rnd.h>
#include <x86/specialreg.h>
@@ -46,6 +53,8 @@
#ifdef VIA_PADLOCK
+char xxx_via_buffer[1024];
+
int via_padlock_crypto_newsession(void *, uint32_t *, struct cryptoini *);
int via_padlock_crypto_process(void *, struct cryptop *, int);
int via_padlock_crypto_swauth(struct cryptop *, struct cryptodesc *,
@@ -56,19 +65,74 @@
static __inline void via_padlock_cbc(void *, void *, void *, void *, int,
void *);
-void
-via_padlock_attach(void)
+static void
+via_c3_rnd(void *arg)
{
-#define VIA_ACE (CPUID_VIA_HAS_ACE|CPUID_VIA_DO_ACE)
- if ((cpu_feature_padlock & VIA_ACE) != VIA_ACE)
- return;
+ struct via_padlock_softc *vp_sc = arg;
- struct via_padlock_softc *vp_sc;
- if ((vp_sc = malloc(sizeof(*vp_sc), M_DEVBUF, M_NOWAIT)) == NULL)
- return;
- memset(vp_sc, 0, sizeof(*vp_sc));
+ unsigned int rv, creg0, len = VIAC3_RNG_BUFSIZ;
+ static uint32_t buffer[VIAC3_RNG_BUFSIZ + 2]; /* XXX 2? */
+
+ /*
+ * Sadly, we have to monkey with the coprocessor enable and fault
+ * registers, which are really for the FPU, in order to read
+ * from the RNG.
+ *
+ * Don't remove CR0_TS from the call below -- comments in the Linux
+ * driver indicate that the xstorerng instruction can generate
+ * spurious DNA faults though no FPU or SIMD state is changed
+ * even if such a fault is generated.
+ *
+ */
+ kpreempt_disable();
+ x86_disable_intr();
+ creg0 = rcr0();
+ lcr0(creg0 & ~(CR0_EM|CR0_TS)); /* Permit access to SIMD/FPU path */
+ /*
+ * Collect the random data from the C3 RNG into our buffer.
+ * We turn on maximum whitening (is this actually desirable
+ * if we will feed the data to SHA1?) (%edx[0,1] = "11").
+ */
+ __asm __volatile("rep xstorerng"
+ : "=a" (rv) : "d" (3), "D" (buffer),
+ "c" (len * sizeof(int)) : "memory", "cc");
+ /* Put CR0 back how it was */
+ lcr0(creg0);
+ x86_enable_intr();
+ kpreempt_enable();
+ rnd_add_data(&vp_sc->sc_rnd_source, buffer, len * sizeof(int),
+ len * sizeof(int));
+ callout_reset(&vp_sc->sc_rnd_co, vp_sc->sc_rnd_hz, via_c3_rnd, vp_sc);
+}
+
+static void
+via_c3_rnd_init(struct via_padlock_softc *const vp_sc)
+{
+ if (hz >= 100) {
+ vp_sc->sc_rnd_hz = 10 * hz / 100;
+ } else {
+ vp_sc->sc_rnd_hz = 10;
+ }
+ /* See hifn7751.c re use of RND_FLAG_NO_ESTIMATE */
+ rnd_attach_source(&vp_sc->sc_rnd_source, "padlock",
+ RND_TYPE_RNG, RND_FLAG_NO_ESTIMATE);
+ callout_init(&vp_sc->sc_rnd_co, 0);
+ /* Call once to prime the pool early and set callout. */
+ via_c3_rnd(vp_sc);
+}
- vp_sc->sc_cid = crypto_get_driverid(0);
+static void
+via_c3_ace_init(struct via_padlock_softc *const vp_sc)
+{
+ /*
+ * There is no reason to call into the kernel to use this
+ * driver from userspace, because the crypto instructions can
+ * be directly accessed there. Setting CRYPTOCAP_F_SOFTWARE
+ * has approximately the right semantics though the name is
+ * confusing (however, consider that crypto via unprivileged
+ * instructions _is_ "just software" in some sense).
+ */
+ vp_sc->sc_cid = crypto_get_driverid(CRYPTOCAP_F_SOFTWARE);
if (vp_sc->sc_cid < 0) {
printf("PadLock: Could not get a crypto driver ID\n");
free(vp_sc, M_DEVBUF);
@@ -80,6 +144,19 @@
* we don't support hardware offloading for various HMAC algorithms,
* we will handle them, because opencrypto prefers drivers that
* support all requested algorithms.
+ *
+ *
+ * XXX We should actually implement the HMAC modes this hardware
+ * XXX can accellerate (wrap its plain SHA1/SHA2 as HMAC) and
+ * XXX strongly consider removing those passed through to cryptosoft.
+ * XXX As it stands, we can "steal" sessions from drivers which could
+ * XXX better accellerate them.
+ *
+ * XXX Note the ordering dependency between when this (or any
+ * XXX crypto driver) attaches and when cryptosoft does. We are
+ * XXX basically counting on the swcrypto pseudo-device to just
+ * XXX happen to attach last, or _it_ will steal every session
+ * XXX from _us_!
*/
#define REGISTER(alg) \
crypto_register(vp_sc->sc_cid, alg, 0, 0, \
@@ -94,8 +171,36 @@
REGISTER(CRYPTO_RIPEMD160_HMAC_96);
REGISTER(CRYPTO_RIPEMD160_HMAC);
REGISTER(CRYPTO_SHA2_HMAC);
+}
+
+void
+via_padlock_attach(void)
+{
+ struct via_padlock_softc *vp_sc;
- printf("PadLock: registered support for AES_CBC\n");
+ printf("%s", xxx_via_buffer);
+
+ if (!((cpu_feature_padlock & CPUID_VIA_HAS_ACE) ||
+ (cpu_feature_padlock & CPUID_VIA_HAS_RNG))) {
+ printf("PadLock: Nothing (%08x ! %08X ! %08X)\n",
+ cpu_feature_padlock, CPUID_VIA_HAS_ACE,
+ CPUID_VIA_HAS_RNG);
+ return; /* Nothing to see here, move along. */
+ }
+
+ if ((vp_sc = malloc(sizeof(*vp_sc), M_DEVBUF, M_NOWAIT)) == NULL)
+ return;
+ memset(vp_sc, 0, sizeof(*vp_sc));
+
+ if (cpu_feature_padlock & CPUID_VIA_HAS_RNG) {
+ via_c3_rnd_init(vp_sc);
+ printf("PadLock: RNG attached\n");
+ }
+
+ if (cpu_feature_padlock & CPUID_VIA_HAS_ACE) {
+ via_c3_ace_init(vp_sc);
+ printf("PadLock: AES-CBC attached\n");
+ }
}
int
