Vendor extensions are maintained in riscv_isa_vendor (separate from
standard extensions which live in riscv_isa). Create vendor variants for
the existing extension helpers to interface with the riscv_isa_vendor
bitmap. There is a good amount of overlap between these functions, so
the alternative checking code can be factored out.
Signed-off-by: Charlie Jenkins <char...@rivosinc.com>
---
arch/riscv/errata/sifive/errata.c | 2 +
arch/riscv/errata/thead/errata.c | 2 +
arch/riscv/include/asm/cpufeature.h | 142 +++++++++++++++++++++++++++---------
arch/riscv/include/asm/hwprobe.h | 3 +
arch/riscv/kernel/cpufeature.c | 53 ++++++++++++--
arch/riscv/kernel/sys_hwprobe.c | 4 +-
6 files changed, 161 insertions(+), 45 deletions(-)
diff --git a/arch/riscv/errata/sifive/errata.c
b/arch/riscv/errata/sifive/errata.c
index 3d9a32d791f7..847ff85cc911 100644
--- a/arch/riscv/errata/sifive/errata.c
+++ b/arch/riscv/errata/sifive/errata.c
@@ -99,6 +99,8 @@ void sifive_errata_patch_func(struct alt_entry *begin, struct
alt_entry *end,
for (alt = begin; alt < end; alt++) {
if (alt->vendor_id != SIFIVE_VENDOR_ID)
continue;
+ if (alt->patch_id >= RISCV_ISA_VENDOR_EXT_BASE)
+ continue;
if (alt->patch_id >= ERRATA_SIFIVE_NUMBER) {
WARN(1, "This errata id:%d is not in kernel errata
list", alt->patch_id);
continue;
diff --git a/arch/riscv/errata/thead/errata.c b/arch/riscv/errata/thead/errata.c
index b1c410bbc1ae..6e3eabfe92af 100644
--- a/arch/riscv/errata/thead/errata.c
+++ b/arch/riscv/errata/thead/errata.c
@@ -163,6 +163,8 @@ void thead_errata_patch_func(struct alt_entry *begin,
struct alt_entry *end,
for (alt = begin; alt < end; alt++) {
if (alt->vendor_id != THEAD_VENDOR_ID)
continue;
+ if (alt->patch_id >= RISCV_ISA_VENDOR_EXT_BASE)
+ continue;
if (alt->patch_id >= ERRATA_THEAD_NUMBER)
continue;
diff --git a/arch/riscv/include/asm/cpufeature.h
b/arch/riscv/include/asm/cpufeature.h
index 50fa174cccb9..12dd36bafa2a 100644
--- a/arch/riscv/include/asm/cpufeature.h
+++ b/arch/riscv/include/asm/cpufeature.h
@@ -110,23 +110,19 @@ bool __riscv_isa_extension_available(const unsigned long
*isa_bitmap, unsigned i
#define riscv_isa_extension_available(isa_bitmap, ext) \
__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_##ext)
+bool __riscv_isa_vendor_extension_available(const unsigned long
*vendor_isa_bitmap,
+ unsigned int bit);
+#define riscv_isa_vendor_extension_available(isa_bitmap, ext) \
+ __riscv_isa_vendor_extension_available(isa_bitmap,
RISCV_ISA_VENDOR_EXT_##ext)
+
static __always_inline bool
-riscv_has_extension_likely(const unsigned long ext)
+__riscv_has_extension_likely_alternatives(const unsigned long vendor, const
unsigned long ext)
{
- compiletime_assert(ext < RISCV_ISA_EXT_MAX,
- "ext must be < RISCV_ISA_EXT_MAX");
-
- if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
- asm goto(
- ALTERNATIVE("j %l[l_no]", "nop", 0, %[ext], 1)
- :
- : [ext] "i" (ext)
- :
- : l_no);
- } else {
- if (!__riscv_isa_extension_available(NULL, ext))
- goto l_no;
- }
+ asm goto(ALTERNATIVE("j %l[l_no]", "nop", %[vendor], %[ext], 1)
+ :
+ : [vendor] "i" (vendor), [ext] "i" (ext)
+ :
+ : l_no);
return true;
l_no:
@@ -134,42 +130,118 @@ riscv_has_extension_likely(const unsigned long ext)
}
static __always_inline bool
-riscv_has_extension_unlikely(const unsigned long ext)
+__riscv_has_extension_unlikely_alternatives(const unsigned long vendor, const
unsigned long ext)
{
- compiletime_assert(ext < RISCV_ISA_EXT_MAX,
- "ext must be < RISCV_ISA_EXT_MAX");
-
- if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
- asm goto(
- ALTERNATIVE("nop", "j %l[l_yes]", 0, %[ext], 1)
- :
- : [ext] "i" (ext)
- :
- : l_yes);
- } else {
- if (__riscv_isa_extension_available(NULL, ext))
- goto l_yes;
- }
+ asm goto(ALTERNATIVE("nop", "j %l[l_yes]", %[vendor], %[ext], 1)
+ :
+ : [vendor] "i" (vendor), [ext] "i" (ext)
+ :
+ : l_yes);
return false;
l_yes:
return true;
}
+/* Standard extension helpers */
+
+static __always_inline bool
+riscv_has_extension_likely(const unsigned long ext)
+{
+ compiletime_assert(ext < RISCV_ISA_EXT_MAX,
+ "ext must be < RISCV_ISA_EXT_MAX");
+
+ if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
+ return __riscv_has_extension_likely_alternatives(0, ext);
+ else
+ return __riscv_isa_extension_available(NULL, ext);
+}
+
+static __always_inline bool
+riscv_has_extension_unlikely(const unsigned long ext)
+{
+ compiletime_assert(ext < RISCV_ISA_EXT_MAX,
+ "ext must be < RISCV_ISA_EXT_MAX");
+
+ if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
+ return __riscv_has_extension_unlikely_alternatives(0, ext);
+ else
+ return __riscv_isa_extension_available(NULL, ext);
+}
+
static __always_inline bool riscv_cpu_has_extension_likely(int cpu, const
unsigned long ext)
{
- if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) &&
riscv_has_extension_likely(ext))
- return true;
+ compiletime_assert(ext < RISCV_ISA_EXT_MAX,
+ "ext must be < RISCV_ISA_EXT_MAX");
- return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);
+ if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) &&
+ __riscv_has_extension_likely_alternatives(0, ext))
+ return true;
+ else
+ return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);
}
static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const
unsigned long ext)
{
- if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) &&
riscv_has_extension_unlikely(ext))
+ compiletime_assert(ext < RISCV_ISA_EXT_MAX,
+ "ext must be < RISCV_ISA_EXT_MAX");
+
+ if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) &&
+ __riscv_has_extension_unlikely_alternatives(0, ext))
return true;
+ else
+ return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);
+}
+
+/* Vendor extension helpers */
+
+static __always_inline bool
+riscv_has_vendor_extension_likely(const unsigned long vendor, const unsigned
long ext)
+{
+ compiletime_assert(ext < RISCV_ISA_VENDOR_EXT_MAX,
+ "ext must be < RISCV_ISA_VENDOR_EXT_MAX");
+
+ if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
+ return __riscv_has_extension_likely_alternatives(vendor, ext);
+ else
+ return __riscv_isa_vendor_extension_available(NULL, ext);
+}
- return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);
+static __always_inline bool
+riscv_has_vendor_extension_unlikely(const unsigned long vendor, const unsigned
long ext)
+{
+ compiletime_assert(ext < RISCV_ISA_VENDOR_EXT_MAX,
+ "ext must be < RISCV_ISA_VENDOR_EXT_MAX");
+
+ if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
+ return __riscv_has_extension_unlikely_alternatives(vendor, ext);
+ else
+ return __riscv_isa_vendor_extension_available(NULL, ext);
+}
+
+static __always_inline bool riscv_cpu_has_vendor_extension_likely(const
unsigned long vendor,
+ int cpu,
const unsigned long ext)
+{
+ compiletime_assert(ext < RISCV_ISA_VENDOR_EXT_MAX,
+ "ext must be < RISCV_ISA_VENDOR_EXT_MAX");
+
+ if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
+ return __riscv_has_extension_likely_alternatives(vendor, ext);
+ else
+ return
__riscv_isa_vendor_extension_available(hart_isa_vendor[cpu].isa, ext);
+}
+
+static __always_inline bool riscv_cpu_has_vendor_extension_unlikely(const
unsigned long vendor,
+ int cpu,
+ const
unsigned long ext)
+{
+ compiletime_assert(ext < RISCV_ISA_VENDOR_EXT_MAX,
+ "ext must be < RISCV_ISA_VENDOR_EXT_MAX");
+
+ if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
+ return __riscv_has_extension_unlikely_alternatives(vendor, ext);
+ else
+ return
__riscv_isa_vendor_extension_available(hart_isa_vendor[cpu].isa, ext);
}
#endif
diff --git a/arch/riscv/include/asm/hwprobe.h b/arch/riscv/include/asm/hwprobe.h
index 630507dff5ea..1378c3c9401a 100644
--- a/arch/riscv/include/asm/hwprobe.h
+++ b/arch/riscv/include/asm/hwprobe.h
@@ -6,6 +6,7 @@
#ifndef _ASM_HWPROBE_H
#define _ASM_HWPROBE_H
+#include <linux/cpumask.h>
#include <uapi/asm/hwprobe.h>
#define RISCV_HWPROBE_MAX_KEY 6
@@ -39,4 +40,6 @@ static inline bool riscv_hwprobe_pair_cmp(struct
riscv_hwprobe *pair,
return pair->value == other_pair->value;
}
+void hwprobe_arch_id(struct riscv_hwprobe *pair, const struct cpumask *cpus);
+
#endif
diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c
index d7a33e017a15..799ec2d2e9e0 100644
--- a/arch/riscv/kernel/cpufeature.c
+++ b/arch/riscv/kernel/cpufeature.c
@@ -20,6 +20,7 @@
#include <asm/cacheflush.h>
#include <asm/cpufeature.h>
#include <asm/hwcap.h>
+#include <asm/hwprobe.h>
#include <asm/patch.h>
#include <asm/processor.h>
#include <asm/sbi.h>
@@ -82,6 +83,30 @@ bool __riscv_isa_extension_available(const unsigned long
*isa_bitmap, unsigned i
}
EXPORT_SYMBOL_GPL(__riscv_isa_extension_available);
+/**
+ * __riscv_isa_vendor_extension_available() - Check whether given vendor
+ * extension is available or not. The vendor extension must be associated
+ * with the same vendor that was used to populate isa_bitmap.
+ *
+ * @isa_bitmap: ISA bitmap to use
+ * @bit: bit position of the desired extension
+ * Return: true or false
+ *
+ * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
+ */
+bool __riscv_isa_vendor_extension_available(const unsigned long *isa_bitmap,
unsigned int bit)
+{
+ const unsigned long *bmap = (isa_bitmap) ? isa_bitmap :
riscv_isa_vendor;
+
+ bit -= RISCV_ISA_VENDOR_EXT_BASE;
+
+ if (bit < 0 || bit >= RISCV_ISA_VENDOR_EXT_MAX)
+ return false;
+
+ return test_bit(bit, bmap) ? true : false;
+}
+EXPORT_SYMBOL_GPL(__riscv_isa_vendor_extension_available);
+
static bool riscv_isa_extension_check(int id)
{
switch (id) {
@@ -832,25 +857,37 @@ void __init_or_module riscv_cpufeature_patch_func(struct
alt_entry *begin,
{
struct alt_entry *alt;
void *oldptr, *altptr;
- u16 id, value;
+ u16 id, value, vendor;
if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
return;
- for (alt = begin; alt < end; alt++) {
- if (alt->vendor_id != 0)
- continue;
+ struct riscv_hwprobe mvendorid = {
+ .key = RISCV_HWPROBE_KEY_MVENDORID,
+ .value = 0
+ };
+ hwprobe_arch_id(&mvendorid, cpu_possible_mask);
+
+ for (alt = begin; alt < end; alt++) {
id = PATCH_ID_CPUFEATURE_ID(alt->patch_id);
+ vendor = PATCH_ID_CPUFEATURE_ID(alt->vendor_id);
- if (id >= RISCV_ISA_EXT_MAX) {
+ if (id >= RISCV_ISA_VENDOR_EXT_BASE) {
+ if (vendor != mvendorid.value ||
+ !__riscv_isa_vendor_extension_available(NULL, id))
+ continue;
+ } else if (id < RISCV_ISA_EXT_MAX) {
+ if (alt->vendor_id != 0)
+ continue;
+
+ if (!__riscv_isa_extension_available(NULL, id))
+ continue;
+ } else {
WARN(1, "This extension id:%d is not in ISA extension
list", id);
continue;
}
- if (!__riscv_isa_extension_available(NULL, id))
- continue;
-
value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id);
if (!riscv_cpufeature_patch_check(id, value))
continue;
diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c
index 8cae41a502dd..394f1343490c 100644
--- a/arch/riscv/kernel/sys_hwprobe.c
+++ b/arch/riscv/kernel/sys_hwprobe.c
@@ -13,11 +13,11 @@
#include <asm/uaccess.h>
#include <asm/unistd.h>
#include <asm/vector.h>
+#include <asm/vendor/thead.h>
#include <vdso/vsyscall.h>
-static void hwprobe_arch_id(struct riscv_hwprobe *pair,
- const struct cpumask *cpus)
+void hwprobe_arch_id(struct riscv_hwprobe *pair, const struct cpumask *cpus)
{
u64 id = -1ULL;
bool first = true;
--
2.44.0
