Introduce a generic memcpy_streaming() interface for write-once copy sites that can fall back to memcpy() when no architecture-specific optimization is available, or when an architecture-specific backend cannot safely handle a given transfer.
Add memcpy_streaming_drain() alongside it so callers can separate the copy primitive from any required ordering point. On x86, use memcpy_flushcache() and sfence only for aligned transfers that can stay entirely on the non-temporal store path; otherwise fall back to memcpy() so the generic API does not expose flushcache semantics on cached head/tail fragments. Callers are responsible for invoking memcpy_streaming_drain() before later normal stores that must be ordered after the streaming copy. Signed-off-by: Li Zhe <[email protected]> --- arch/x86/include/asm/string_64.h | 32 ++++++++++++++++++++++++++++++++ include/linux/string.h | 20 ++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h index 4635616863f5..aee63108577f 100644 --- a/arch/x86/include/asm/string_64.h +++ b/arch/x86/include/asm/string_64.h @@ -4,6 +4,7 @@ #ifdef __KERNEL__ #include <linux/jump_label.h> +#include <linux/align.h> /* Written 2002 by Andi Kleen */ @@ -100,6 +101,37 @@ static __always_inline void memcpy_flushcache(void *dst, const void *src, size_t } __memcpy_flushcache(dst, src, cnt); } + +/* + * Only map memcpy_streaming() to memcpy_flushcache() when the destination + * is already 8-byte aligned and the size can be handled without cached + * head/tail fragments in __memcpy_flushcache(). + */ +static __always_inline bool memcpy_flushcache_nt_safe(const void *dst, + size_t cnt) +{ + unsigned long d = (unsigned long)dst; + + return cnt && IS_ALIGNED(d, 8) && IS_ALIGNED(cnt, 4); +} + +#define __HAVE_ARCH_MEMCPY_STREAMING 1 +static __always_inline void memcpy_streaming(void *dst, const void *src, + size_t cnt) +{ + if (!cnt) + return; + + if (memcpy_flushcache_nt_safe(dst, cnt)) + memcpy_flushcache(dst, src, cnt); + else + memcpy(dst, src, cnt); +} + +static __always_inline void memcpy_streaming_drain(void) +{ + asm volatile("sfence" : : : "memory"); +} #endif #endif /* __KERNEL__ */ diff --git a/include/linux/string.h b/include/linux/string.h index b850bd91b3d8..a4c2d4347f58 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -281,6 +281,26 @@ static inline void memcpy_flushcache(void *dst, const void *src, size_t cnt) } #endif +#ifndef __HAVE_ARCH_MEMCPY_STREAMING +/* + * memcpy_streaming() is for write-once copy sites that may use + * non-temporal stores on some architectures. Callers must follow it + * with memcpy_streaming_drain() before later normal stores that need to + * be ordered after the streaming copy. Implementations may fall back to + * memcpy() when a specialized backend cannot safely handle the given + * transfer, and backends that use regular cached stores can make the + * drain a no-op. + */ +static inline void memcpy_streaming(void *dst, const void *src, size_t cnt) +{ + memcpy(dst, src, cnt); +} + +static inline void memcpy_streaming_drain(void) +{ +} +#endif + void *memchr_inv(const void *s, int c, size_t n); char *strreplace(char *str, char old, char new); -- 2.20.1
