[PULL 06/61] util: add util function buffer_zero_avx512()
From: Robert Hoo
And intialize buffer_is_zero() with it, when Intel AVX512F is
available on host.
This function utilizes Intel AVX512 fundamental instructions which
is faster than its implementation with AVX2 (in my unit test, with
4K buffer, on CascadeLake SP, ~36% faster, buffer_zero_avx512() V.S.
buffer_zero_avx2()).
Signed-off-by: Robert Hoo
Reviewed-by: Richard Henderson
Signed-off-by: Paolo Bonzini
---
include/qemu/cpuid.h | 3 +++
util/bufferiszero.c | 71
2 files changed, 64 insertions(+), 10 deletions(-)
diff --git a/include/qemu/cpuid.h b/include/qemu/cpuid.h
index 6930170..09fc245 100644
--- a/include/qemu/cpuid.h
+++ b/include/qemu/cpuid.h
@@ -45,6 +45,9 @@
#ifndef bit_AVX2
#define bit_AVX2(1 << 5)
#endif
+#ifndef bit_AVX512F
+#define bit_AVX512F(1 << 16)
+#endif
#ifndef bit_BMI2
#define bit_BMI2(1 << 8)
#endif
diff --git a/util/bufferiszero.c b/util/bufferiszero.c
index bfb2605..6639035 100644
--- a/util/bufferiszero.c
+++ b/util/bufferiszero.c
@@ -63,11 +63,11 @@ buffer_zero_int(const void *buf, size_t len)
}
}
-#if defined(CONFIG_AVX2_OPT) || defined(__SSE2__)
+#if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT) ||
defined(__SSE2__)
/* Do not use push_options pragmas unnecessarily, because clang
* does not support them.
*/
-#ifdef CONFIG_AVX2_OPT
+#if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT)
#pragma GCC push_options
#pragma GCC target("sse2")
#endif
@@ -104,7 +104,7 @@ buffer_zero_sse2(const void *buf, size_t len)
return _mm_movemask_epi8(_mm_cmpeq_epi8(t, zero)) == 0x;
}
-#ifdef CONFIG_AVX2_OPT
+#if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT)
#pragma GCC pop_options
#endif
@@ -187,18 +187,54 @@ buffer_zero_avx2(const void *buf, size_t len)
#pragma GCC pop_options
#endif /* CONFIG_AVX2_OPT */
+#ifdef CONFIG_AVX512F_OPT
+#pragma GCC push_options
+#pragma GCC target("avx512f")
+#include
+
+static bool
+buffer_zero_avx512(const void *buf, size_t len)
+{
+/* Begin with an unaligned head of 64 bytes. */
+__m512i t = _mm512_loadu_si512(buf);
+__m512i *p = (__m512i *)(((uintptr_t)buf + 5 * 64) & -64);
+__m512i *e = (__m512i *)(((uintptr_t)buf + len) & -64);
+
+/* Loop over 64-byte aligned blocks of 256. */
+while (p <= e) {
+__builtin_prefetch(p);
+if (unlikely(_mm512_test_epi64_mask(t, t))) {
+return false;
+}
+t = p[-4] | p[-3] | p[-2] | p[-1];
+p += 4;
+}
+
+t |= _mm512_loadu_si512(buf + len - 4 * 64);
+t |= _mm512_loadu_si512(buf + len - 3 * 64);
+t |= _mm512_loadu_si512(buf + len - 2 * 64);
+t |= _mm512_loadu_si512(buf + len - 1 * 64);
+
+return !_mm512_test_epi64_mask(t, t);
+
+}
+#pragma GCC pop_options
+#endif
+
+
/* Note that for test_buffer_is_zero_next_accel, the most preferred
* ISA must have the least significant bit.
*/
-#define CACHE_AVX21
-#define CACHE_SSE42
-#define CACHE_SSE24
+#define CACHE_AVX512F 1
+#define CACHE_AVX22
+#define CACHE_SSE44
+#define CACHE_SSE28
/* Make sure that these variables are appropriately initialized when
* SSE2 is enabled on the compiler command-line, but the compiler is
* too old to support CONFIG_AVX2_OPT.
*/
-#ifdef CONFIG_AVX2_OPT
+#if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT)
# define INIT_CACHE 0
# define INIT_ACCEL buffer_zero_int
#else
@@ -211,6 +247,7 @@ buffer_zero_avx2(const void *buf, size_t len)
static unsigned cpuid_cache = INIT_CACHE;
static bool (*buffer_accel)(const void *, size_t) = INIT_ACCEL;
+static int length_to_accel = 64;
static void init_accel(unsigned cache)
{
@@ -226,10 +263,16 @@ static void init_accel(unsigned cache)
fn = buffer_zero_avx2;
}
#endif
+#ifdef CONFIG_AVX512F_OPT
+if (cache & CACHE_AVX512F) {
+fn = buffer_zero_avx512;
+length_to_accel = 256;
+}
+#endif
buffer_accel = fn;
}
-#ifdef CONFIG_AVX2_OPT
+#if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT)
#include "qemu/cpuid.h"
static void __attribute__((constructor)) init_cpuid_cache(void)
@@ -252,9 +295,17 @@ static void __attribute__((constructor))
init_cpuid_cache(void)
int bv;
__asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0));
__cpuid_count(7, 0, a, b, c, d);
-if ((bv & 6) == 6 && (b & bit_AVX2)) {
+if ((bv & 0x6) == 0x6 && (b & bit_AVX2)) {
cache |= CACHE_AVX2;
}
+/* 0xe6:
+* XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15
+*and ZMM16-ZMM31 state are enabled by OS)
+* XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS)
+*/
+if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512F)) {
+cache |= CACHE_AVX512F;
+}
}
}
cpuid_cache = cache;
@@ -277,7
Re: [PULL 06/61] util: add util function buffer_zero_avx512()
This breaks if AVX2 and AVX512 are both disabled but SSE2 isn't, because
it calls buffer_is_zero_sse2 even with length < 64. Fix:
diff --git a/util/bufferiszero.c b/util/bufferiszero.c
index b6eab0c..6639035 100644
--- a/util/bufferiszero.c
+++ b/util/bufferiszero.c
@@ -247,23 +247,20 @@ buffer_zero_avx512(const void *buf, size_t len)
static unsigned cpuid_cache = INIT_CACHE;
static bool (*buffer_accel)(const void *, size_t) = INIT_ACCEL;
-static int length_to_accel;
+static int length_to_accel = 64;
static void init_accel(unsigned cache)
{
bool (*fn)(const void *, size_t) = buffer_zero_int;
if (cache & CACHE_SSE2) {
fn = buffer_zero_sse2;
-length_to_accel = 64;
}
#ifdef CONFIG_AVX2_OPT
if (cache & CACHE_SSE4) {
fn = buffer_zero_sse4;
-length_to_accel = 64;
}
if (cache & CACHE_AVX2) {
fn = buffer_zero_avx2;
-length_to_accel = 64;
}
#endif
#ifdef CONFIG_AVX512F_OPT
Paolo
[PULL 06/61] util: add util function buffer_zero_avx512()
From: Robert Hoo
And intialize buffer_is_zero() with it, when Intel AVX512F is
available on host.
This function utilizes Intel AVX512 fundamental instructions which
is faster than its implementation with AVX2 (in my unit test, with
4K buffer, on CascadeLake SP, ~36% faster, buffer_zero_avx512() V.S.
buffer_zero_avx2()).
Signed-off-by: Robert Hoo
Reviewed-by: Richard Henderson
Signed-off-by: Paolo Bonzini
---
include/qemu/cpuid.h | 3 +++
util/bufferiszero.c | 74 +---
2 files changed, 67 insertions(+), 10 deletions(-)
diff --git a/include/qemu/cpuid.h b/include/qemu/cpuid.h
index 6930170..09fc245 100644
--- a/include/qemu/cpuid.h
+++ b/include/qemu/cpuid.h
@@ -45,6 +45,9 @@
#ifndef bit_AVX2
#define bit_AVX2(1 << 5)
#endif
+#ifndef bit_AVX512F
+#define bit_AVX512F(1 << 16)
+#endif
#ifndef bit_BMI2
#define bit_BMI2(1 << 8)
#endif
diff --git a/util/bufferiszero.c b/util/bufferiszero.c
index bfb2605..b6eab0c 100644
--- a/util/bufferiszero.c
+++ b/util/bufferiszero.c
@@ -63,11 +63,11 @@ buffer_zero_int(const void *buf, size_t len)
}
}
-#if defined(CONFIG_AVX2_OPT) || defined(__SSE2__)
+#if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT) ||
defined(__SSE2__)
/* Do not use push_options pragmas unnecessarily, because clang
* does not support them.
*/
-#ifdef CONFIG_AVX2_OPT
+#if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT)
#pragma GCC push_options
#pragma GCC target("sse2")
#endif
@@ -104,7 +104,7 @@ buffer_zero_sse2(const void *buf, size_t len)
return _mm_movemask_epi8(_mm_cmpeq_epi8(t, zero)) == 0x;
}
-#ifdef CONFIG_AVX2_OPT
+#if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT)
#pragma GCC pop_options
#endif
@@ -187,18 +187,54 @@ buffer_zero_avx2(const void *buf, size_t len)
#pragma GCC pop_options
#endif /* CONFIG_AVX2_OPT */
+#ifdef CONFIG_AVX512F_OPT
+#pragma GCC push_options
+#pragma GCC target("avx512f")
+#include
+
+static bool
+buffer_zero_avx512(const void *buf, size_t len)
+{
+/* Begin with an unaligned head of 64 bytes. */
+__m512i t = _mm512_loadu_si512(buf);
+__m512i *p = (__m512i *)(((uintptr_t)buf + 5 * 64) & -64);
+__m512i *e = (__m512i *)(((uintptr_t)buf + len) & -64);
+
+/* Loop over 64-byte aligned blocks of 256. */
+while (p <= e) {
+__builtin_prefetch(p);
+if (unlikely(_mm512_test_epi64_mask(t, t))) {
+return false;
+}
+t = p[-4] | p[-3] | p[-2] | p[-1];
+p += 4;
+}
+
+t |= _mm512_loadu_si512(buf + len - 4 * 64);
+t |= _mm512_loadu_si512(buf + len - 3 * 64);
+t |= _mm512_loadu_si512(buf + len - 2 * 64);
+t |= _mm512_loadu_si512(buf + len - 1 * 64);
+
+return !_mm512_test_epi64_mask(t, t);
+
+}
+#pragma GCC pop_options
+#endif
+
+
/* Note that for test_buffer_is_zero_next_accel, the most preferred
* ISA must have the least significant bit.
*/
-#define CACHE_AVX21
-#define CACHE_SSE42
-#define CACHE_SSE24
+#define CACHE_AVX512F 1
+#define CACHE_AVX22
+#define CACHE_SSE44
+#define CACHE_SSE28
/* Make sure that these variables are appropriately initialized when
* SSE2 is enabled on the compiler command-line, but the compiler is
* too old to support CONFIG_AVX2_OPT.
*/
-#ifdef CONFIG_AVX2_OPT
+#if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT)
# define INIT_CACHE 0
# define INIT_ACCEL buffer_zero_int
#else
@@ -211,25 +247,35 @@ buffer_zero_avx2(const void *buf, size_t len)
static unsigned cpuid_cache = INIT_CACHE;
static bool (*buffer_accel)(const void *, size_t) = INIT_ACCEL;
+static int length_to_accel;
static void init_accel(unsigned cache)
{
bool (*fn)(const void *, size_t) = buffer_zero_int;
if (cache & CACHE_SSE2) {
fn = buffer_zero_sse2;
+length_to_accel = 64;
}
#ifdef CONFIG_AVX2_OPT
if (cache & CACHE_SSE4) {
fn = buffer_zero_sse4;
+length_to_accel = 64;
}
if (cache & CACHE_AVX2) {
fn = buffer_zero_avx2;
+length_to_accel = 64;
+}
+#endif
+#ifdef CONFIG_AVX512F_OPT
+if (cache & CACHE_AVX512F) {
+fn = buffer_zero_avx512;
+length_to_accel = 256;
}
#endif
buffer_accel = fn;
}
-#ifdef CONFIG_AVX2_OPT
+#if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT)
#include "qemu/cpuid.h"
static void __attribute__((constructor)) init_cpuid_cache(void)
@@ -252,9 +298,17 @@ static void __attribute__((constructor))
init_cpuid_cache(void)
int bv;
__asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0));
__cpuid_count(7, 0, a, b, c, d);
-if ((bv & 6) == 6 && (b & bit_AVX2)) {
+if ((bv & 0x6) == 0x6 && (b & bit_AVX2)) {
cache |= CACHE_AVX2;
}
+/* 0xe6:
+* XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15
+*and ZMM16-ZMM31 state
