By querying previously-defined file-identifier macros, `host-config.h'
is able to get information about its environment and, based on this
information, select more appropriate function-specific ifunc
selectors. This reduces the number of unnecessary feature tests that
need to be carried out in order to find the best atomic implementation
for a function at run-time.
An immediate benefit of this is that we can further fine-tune the
architectural requirements for each atomic function without risk of
incurring the maintenance and runtime-performance penalties of having
to maintain an ifunc selector with a huge number of alternatives, most
of which are irrelevant for any particular function. Consequently,
for AArch64 targets, we relax the architectural requirements of
`compare_exchange_16', which now requires only LSE as opposed to the
newer LSE2.
The new flexibility provided by this approach also means that certain
functions can now be called directly, doing away with ifunc selectors
altogether when only a single implementation is available for it on a
given target. As per the macro expansion framework laid out in
`libatomic_i.h', such functions should have their names prefixed with
`__atomic_' as opposed to `libat_'. This is the same prefix applied
to function names when Libatomic is configured with
`--disable-gnu-indirect-function'.
To achieve this, these functions unconditionally apply the aliasing
rule that at present is conditionally applied only when libatomic is
built without ifunc support, which ensures that the default
`libat_##NAME' is accessible via the equivalent `__atomic_##NAME' too.
This is ensured by using the new `ENTRY_ALIASED' macro.
libatomic/ChangeLog:
* config/linux/aarch64/atomic_16.S (LSE): New.
(ENTRY_ALIASED): Likewise.
* config/linux/aarch64/host-config.h (LSE_ATOP): New.
(LSE2_ATOP): Likewise.
(LSE128_ATOP): Likewise.
(IFUNC_COND_1): Make its definition conditional on above 3
macros.
(IFUNC_NCOND): Likewise.
---
libatomic/config/linux/aarch64/atomic_16.S | 31 +++++++++--------
libatomic/config/linux/aarch64/host-config.h | 35 ++++++++++++++++----
2 files changed, 45 insertions(+), 21 deletions(-)
diff --git a/libatomic/config/linux/aarch64/atomic_16.S
b/libatomic/config/linux/aarch64/atomic_16.S
index b63e97ac5a2..1517e9e78df 100644
--- a/libatomic/config/linux/aarch64/atomic_16.S
+++ b/libatomic/config/linux/aarch64/atomic_16.S
@@ -54,17 +54,20 @@
#endif
#define LSE128(NAME) libat_##NAME##_i1
-#define LSE2(NAME) libat_##NAME##_i2
+#define LSE(NAME) libat_##NAME##_i1
+#define LSE2(NAME) libat_##NAME##_i1
#define CORE(NAME) libat_##NAME
#define ATOMIC(NAME) __atomic_##NAME
+/* Emit __atomic_* entrypoints if no ifuncs. */
+#define ENTRY_ALIASED(NAME) ENTRY2 (CORE (NAME), ALIAS (NAME, ATOMIC, CORE))
+
#if HAVE_IFUNC
# define ENTRY(NAME) ENTRY2 (CORE (NAME), )
# define ENTRY_FEAT(NAME, FEAT) ENTRY2 (FEAT (NAME), )
# define END_FEAT(NAME, FEAT) END2 (FEAT (NAME))
#else
-/* Emit __atomic_* entrypoints if no ifuncs. */
-# define ENTRY(NAME) ENTRY2 (CORE (NAME), ALIAS (NAME, ATOMIC, CORE))
+# define ENTRY(NAME) ENTRY_ALIASED (NAME)
#endif
#define END(NAME) END2 (CORE (NAME))
@@ -299,7 +302,7 @@ END (compare_exchange_16)
#if HAVE_FEAT_LSE2
-ENTRY_FEAT (compare_exchange_16, LSE2)
+ENTRY_FEAT (compare_exchange_16, LSE)
ldp exp0, exp1, [x1]
mov tmp0, exp0
mov tmp1, exp1
@@ -332,11 +335,11 @@ ENTRY_FEAT (compare_exchange_16, LSE2)
/* ACQ_REL/SEQ_CST. */
4: caspal exp0, exp1, in0, in1, [x0]
b 0b
-END_FEAT (compare_exchange_16, LSE2)
+END_FEAT (compare_exchange_16, LSE)
#endif
-ENTRY (fetch_add_16)
+ENTRY_ALIASED (fetch_add_16)
mov x5, x0
cbnz w4, 2f
@@ -358,7 +361,7 @@ ENTRY (fetch_add_16)
END (fetch_add_16)
-ENTRY (add_fetch_16)
+ENTRY_ALIASED (add_fetch_16)
mov x5, x0
cbnz w4, 2f
@@ -380,7 +383,7 @@ ENTRY (add_fetch_16)
END (add_fetch_16)
-ENTRY (fetch_sub_16)
+ENTRY_ALIASED (fetch_sub_16)
mov x5, x0
cbnz w4, 2f
@@ -402,7 +405,7 @@ ENTRY (fetch_sub_16)
END (fetch_sub_16)
-ENTRY (sub_fetch_16)
+ENTRY_ALIASED (sub_fetch_16)
mov x5, x0
cbnz w4, 2f
@@ -624,7 +627,7 @@ END_FEAT (and_fetch_16, LSE128)
#endif
-ENTRY (fetch_xor_16)
+ENTRY_ALIASED (fetch_xor_16)
mov x5, x0
cbnz w4, 2f
@@ -646,7 +649,7 @@ ENTRY (fetch_xor_16)
END (fetch_xor_16)
-ENTRY (xor_fetch_16)
+ENTRY_ALIASED (xor_fetch_16)
mov x5, x0
cbnz w4, 2f
@@ -668,7 +671,7 @@ ENTRY (xor_fetch_16)
END (xor_fetch_16)
-ENTRY (fetch_nand_16)
+ENTRY_ALIASED (fetch_nand_16)
mov x5, x0
mvn in0, in0
mvn in1, in1
@@ -692,7 +695,7 @@ ENTRY (fetch_nand_16)
END (fetch_nand_16)
-ENTRY (nand_fetch_16)
+ENTRY_ALIASED (nand_fetch_16)
mov x5, x0
mvn in0, in0
mvn in1, in1
@@ -718,7 +721,7 @@ END (nand_fetch_16)
/* __atomic_test_and_set is always inlined, so this entry is unused and
only required for completeness. */
-ENTRY (test_and_set_16)
+ENTRY_ALIASED (test_and_set_16)
/* RELAXED/ACQUIRE/CONSUME/RELEASE/ACQ_REL/SEQ_CST. */
mov x5, x0
diff --git a/libatomic/config/linux/aarch64/host-config.h
b/libatomic/config/linux/aarch64/host-config.h
index e1a699948f4..6e010594a6c 100644
--- a/libatomic/config/linux/aarch64/host-config.h
+++ b/libatomic/config/linux/aarch64/host-config.h
@@ -48,15 +48,36 @@ typedef struct __ifunc_arg_t {
# define _IFUNC_ARG_HWCAP (1ULL << 62)
#endif
-#if N == 16
-# define IFUNC_COND_1 (has_lse128 (hwcap, features))
-# define IFUNC_COND_2 (has_lse2 (hwcap, features))
-# define IFUNC_NCOND(N) 2
-#else
-# define IFUNC_COND_1 (hwcap & HWCAP_ATOMICS)
-# define IFUNC_NCOND(N) 1
+/* From the file which imported `host-config.h' we can ascertain which
+ architectural extension provides relevant atomic support. From this,
+ we can proceed to tweak the ifunc selector behavior. */
+#if defined (LAT_CAS_N)
+# define LSE_ATOP
+#elif defined (LAT_LOAD_N) || defined (LAT_STORE_N)
+# define LSE2_ATOP
+#elif defined (LAT_EXCH_N) || defined (LAT_FIOR_N) || defined (LAT_FAND_N)
+# define LSE128_ATOP
#endif
+# if N == 16
+# if defined (LSE_ATOP)
+# define IFUNC_NCOND(N) 1
+# define IFUNC_COND_1 (hwcap & HWCAP_ATOMICS)
+# elif defined (LSE2_ATOP)
+# define IFUNC_NCOND(N) 1
+# define IFUNC_COND_1 (has_lse2 (hwcap, features))
+# elif HAVE_FEAT_LSE128 && defined (LSE128_ATOP)
+# define IFUNC_NCOND(N) 1
+# define IFUNC_COND_1 (has_lse128 (hwcap, features))
+# else
+# define IFUNC_NCOND(N) 0
+# define IFUNC_ALT 1
+# endif
+# else
+# define IFUNC_COND_1 (hwcap & HWCAP_ATOMICS)
+# define IFUNC_NCOND(N) 1
+# endif
+
#define MIDR_IMPLEMENTOR(midr) (((midr) >> 24) & 255)
#define MIDR_PARTNUM(midr) (((midr) >> 4) & 0xfff)
--
2.34.1
