Module Name: src
Committed By: nakayama
Date: Sat Dec 5 07:58:57 UTC 2009
Modified Files:
src/sys/arch/sparc64/include: ctlreg.h
Log Message:
Bring together similar inline asm codes of ld*a and st*a definitions
using macro, also remove unnecessary membar #Sync and %asi restore in
the case of 32-bit kernel.
While there, introduce an optimization case that asi is constant if
we are using gcc.
To generate a diff of this commit:
cvs rdiff -u -r1.46 -r1.47 src/sys/arch/sparc64/include/ctlreg.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/sparc64/include/ctlreg.h
diff -u src/sys/arch/sparc64/include/ctlreg.h:1.46 src/sys/arch/sparc64/include/ctlreg.h:1.47
--- src/sys/arch/sparc64/include/ctlreg.h:1.46 Sat Nov 28 21:07:02 2009
+++ src/sys/arch/sparc64/include/ctlreg.h Sat Dec 5 07:58:57 2009
@@ -1,4 +1,4 @@
-/* $NetBSD: ctlreg.h,v 1.46 2009/11/28 21:07:02 mrg Exp $ */
+/* $NetBSD: ctlreg.h,v 1.47 2009/12/05 07:58:57 nakayama Exp $ */
/*
* Copyright (c) 1996-2002 Eduardo Horvath
@@ -477,490 +477,372 @@
*/
#ifdef __arch64__
-static __inline u_char
-lduba(paddr_t loc, int asi)
-{
- register unsigned int _lduba_v;
-
- __asm volatile(
- "wr %2, %%g0, %%asi; "
- "lduba [%1]%%asi, %0 "
- : "=r" (_lduba_v)
- : "r" ((unsigned long)(loc)), "r" (asi));
- return (_lduba_v);
-}
-#else
-static __inline u_char
-lduba(paddr_t loc, int asi)
-{
- register unsigned int _lduba_v, _loc_hi, _pstate;
- _loc_hi = (((uint64_t)loc)>>32);
- if (PHYS_ASI(asi)) {
- __asm volatile(
- "wr %4,%%g0,%%asi; "
- "sllx %3,32,%0; "
- "rdpr %%pstate,%1; "
- "or %0,%2,%0; "
- "wrpr %1,8,%%pstate; "
- "membar #Sync; "
- "lduba [%0]%%asi,%0; "
- "wrpr %1,0,%%pstate; "
- "membar #Sync; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_lduba_v), "=&r" (_pstate)
- : "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- } else {
- __asm volatile(
- "wr %3,%%g0,%%asi; "
- "sllx %2,32,%0; "
- "or %0,%1,%0; "
- "lduba [%0]%%asi,%0; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_lduba_v)
- : "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- }
- return (_lduba_v);
+/* 64-bit kernel, non-constant */
+#define SPARC64_LD_NONCONST(ld) \
+ __asm volatile( \
+ "wr %2,%%g0,%%asi; " \
+ #ld " [%1]%%asi,%0 " \
+ : "=r" (_v) \
+ : "r" ((__uintptr_t)(loc)), "r" (asi))
+
+#if defined(__GNUC__) && defined(__OPTIMIZE__)
+#define SPARC64_LD_DEF(ld, type, vtype) \
+static __inline type ld(paddr_t loc, int asi) \
+{ \
+ vtype _v; \
+ if (__builtin_constant_p(asi)) \
+ __asm volatile( \
+ #ld " [%1]%2,%0 " \
+ : "=r" (_v) \
+ : "r" ((__uintptr_t)(loc)), "n" (asi)); \
+ else \
+ SPARC64_LD_NONCONST(ld); \
+ return _v; \
+}
+#else
+#define SPARC64_LD_DEF(ld, type, vtype) \
+static __inline type ld(paddr_t loc, int asi) \
+{ \
+ vtype _v; \
+ SPARC64_LD_NONCONST(ld); \
+ return _v; \
}
#endif
+#define SPARC64_LD_DEF64(ld, type) SPARC64_LD_DEF(ld, type, uint64_t)
-#ifdef __arch64__
-/* load half-word from alternate address space */
-static __inline u_short
-lduha(paddr_t loc, int asi)
-{
- register unsigned int _lduha_v;
-
- __asm volatile(
- "wr %2, %%g0, %%asi; "
- "lduha [%1]%%asi, %0 "
- : "=r" (_lduha_v)
- : "r" ((unsigned long)(loc)), "r" (asi));
- return (_lduha_v);
-}
-#else
-/* load half-word from alternate address space */
-static __inline u_short
-lduha(paddr_t loc, int asi) {
- register unsigned int _lduha_v, _loc_hi, _pstate;
-
- _loc_hi = (((uint64_t)loc)>>32);
+#else /* __arch64__ */
- if (PHYS_ASI(asi)) {
- __asm volatile(
- "wr %4,%%g0,%%asi; "
- "sllx %3,32,%0; "
- "rdpr %%pstate,%1; "
- "wrpr %1,8,%%pstate; "
- "or %0,%2,%0; "
- "membar #Sync; "
- "lduha [%0]%%asi,%0; "
- "wrpr %1,0,%%pstate; "
- "membar #Sync; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_lduha_v), "=&r" (_pstate)
- : "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- } else {
- __asm volatile(
- "wr %3,%%g0,%%asi; "
- "sllx %2,32,%0; "
- "or %0,%1,%0; "
- "lduha [%0]%%asi,%0; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_lduha_v)
- : "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- }
- return (_lduha_v);
+/* 32-bit kernel, MMU bypass, non-constant */
+#define SPARC64_LD_PHYS_NONCONST(ld) \
+ __asm volatile( \
+ "rdpr %%pstate,%1; " \
+ "sllx %3,32,%0; " \
+ "wrpr %1,8,%%pstate; " \
+ "or %0,%2,%0; " \
+ "wr %4,%%g0,%%asi; " \
+ #ld " [%0]%%asi,%0; " \
+ "wrpr %1,0,%%pstate " \
+ : "=&r" (_v), "=&r" (_pstate) \
+ : "r" ((uint32_t)(loc)), "r" (_hi), "r" (asi))
+/* 32-bit kernel, non-constant */
+#define SPARC64_LD_NONCONST(ld) \
+ __asm volatile( \
+ "wr %2,%%g0,%%asi; " \
+ #ld " [%1]%%asi,%0 " \
+ : "=&r" (_v) \
+ : "r" ((uint32_t)(loc)), "r" (asi))
+/* 32-bit kernel, MMU bypass, non-constant, 64-bit value */
+#define SPARC64_LD_PHYS_NONCONST64(ld) \
+ __asm volatile( \
+ "rdpr %%pstate,%1; " \
+ "sllx %3,32,%0; " \
+ "wrpr %1,8,%%pstate; " \
+ "or %0,%2,%0; " \
+ "wr %4,%%g0,%%asi; " \
+ #ld " [%0]%%asi,%0; " \
+ "wrpr %1,0,%%pstate; " \
+ "srlx %0,32,%1; " \
+ "srl %0,0,%0 " \
+ : "=&r" (_vlo), "=&r" (_vhi) \
+ : "r" ((uint32_t)(loc)), "r" (_hi), "r" (asi))
+/* 32-bit kernel, non-constant, 64-bit value */
+#define SPARC64_LD_NONCONST64(ld) \
+ __asm volatile( \
+ "wr %3,%%g0,%%asi; " \
+ #ld " [%2]%%asi,%0; " \
+ "srlx %0,32,%1; " \
+ "srl %0,0,%0 " \
+ : "=&r" (_vlo), "=&r" (_vhi) \
+ : "r" ((uint32_t)(loc)), "r" (asi))
+
+#if defined(__GNUC__) && defined(__OPTIMIZE__)
+#define SPARC64_LD_DEF(ld, type, vtype) \
+static __inline type ld(paddr_t loc, int asi) \
+{ \
+ vtype _v; \
+ uint32_t _hi, _pstate; \
+ if (PHYS_ASI(asi)) { \
+ _hi = (uint64_t)(loc) >> 32; \
+ if (__builtin_constant_p(asi)) \
+ __asm volatile( \
+ "rdpr %%pstate,%1; " \
+ "sllx %3,32,%0; " \
+ "wrpr %1,8,%%pstate; " \
+ "or %0,%2,%0; " \
+ #ld " [%0]%4,%0; " \
+ "wrpr %1,0,%%pstate; " \
+ : "=&r" (_v), "=&r" (_pstate) \
+ : "r" ((uint32_t)(loc)), "r" (_hi), \
+ "n" (asi)); \
+ else \
+ SPARC64_LD_PHYS_NONCONST(ld); \
+ } else { \
+ if (__builtin_constant_p(asi)) \
+ __asm volatile( \
+ #ld " [%1]%2,%0 " \
+ : "=&r" (_v) \
+ : "r" ((uint32_t)(loc)), "n" (asi)); \
+ else \
+ SPARC64_LD_NONCONST(ld); \
+ } \
+ return _v; \
+}
+#define SPARC64_LD_DEF64(ld, type) \
+static __inline type ld(paddr_t loc, int asi) \
+{ \
+ uint32_t _vlo, _vhi, _hi; \
+ if (PHYS_ASI(asi)) { \
+ _hi = (uint64_t)(loc) >> 32; \
+ if (__builtin_constant_p(asi)) \
+ __asm volatile( \
+ "rdpr %%pstate,%1; " \
+ "sllx %3,32,%0; " \
+ "wrpr %1,8,%%pstate; " \
+ "or %0,%2,%0; " \
+ #ld " [%0]%4,%0; " \
+ "wrpr %1,0,%%pstate; " \
+ "srlx %0,32,%1; " \
+ "srl %0,0,%0 " \
+ : "=&r" (_vlo), "=&r" (_vhi) \
+ : "r" ((uint32_t)(loc)), "r" (_hi), \
+ "n" (asi)); \
+ else \
+ SPARC64_LD_PHYS_NONCONST64(ld); \
+ } else { \
+ if (__builtin_constant_p(asi)) \
+ __asm volatile( \
+ #ld " [%2]%3,%0; " \
+ "srlx %0,32,%1; " \
+ "srl %0,0,%0 " \
+ : "=&r" (_vlo), "=&r" (_vhi) \
+ : "r" ((uint32_t)(loc)), "n" (asi)); \
+ else \
+ SPARC64_LD_NONCONST64(ld); \
+ } \
+ return ((uint64_t)_vhi << 32) | _vlo; \
+}
+#else
+#define SPARC64_LD_DEF(ld, type, vtype) \
+static __inline type ld(paddr_t loc, int asi) \
+{ \
+ vtype _v; \
+ uint32_t _hi, _pstate; \
+ if (PHYS_ASI(asi)) { \
+ _hi = (uint64_t)(loc) >> 32; \
+ SPARC64_LD_PHYS_NONCONST(ld); \
+ } else \
+ SPARC64_LD_NONCONST(ld); \
+ return _v; \
+}
+#define SPARC64_LD_DEF64(ld, type) \
+static __inline type ld(paddr_t loc, int asi) \
+{ \
+ uint32_t _vlo, _vhi, _hi; \
+ if (PHYS_ASI(asi)) { \
+ _hi = (uint64_t)(loc) >> 32; \
+ SPARC64_LD_PHYS_NONCONST64(ld); \
+ } else \
+ SPARC64_LD_NONCONST64(ld); \
+ return ((uint64_t)_vhi << 32) | _vlo; \
}
#endif
+#endif /* __arch64__ */
-#ifdef __arch64__
-/* load unsigned int from alternate address space */
-static __inline u_int
-lda(paddr_t loc, int asi)
-{
- register unsigned int _lda_v;
-
- __asm volatile(
- "wr %2,%%g0,%%asi; "
- "lda [%1]%%asi,%0 "
- : "=r" (_lda_v)
- : "r" ((unsigned long)(loc)), "r" (asi));
- return (_lda_v);
-}
-
-/* load signed int from alternate address space */
-static __inline int
-ldswa(paddr_t loc, int asi)
-{
- register int _lda_v;
-
- __asm volatile(
- "wr %2,%%g0,%%asi; "
- "ldswa [%1]%%asi,%0; "
- : "=r" (_lda_v)
- : "r" ((unsigned long)(loc)), "r" (asi));
- return (_lda_v);
-}
-#else /* __arch64__ */
+/* load byte from alternate address space */
+SPARC64_LD_DEF(lduba, uint8_t, uint32_t)
+/* load half-word from alternate address space */
+SPARC64_LD_DEF(lduha, uint16_t, uint32_t)
/* load unsigned int from alternate address space */
-static __inline u_int
-lda(paddr_t loc, int asi)
-{
- register unsigned int _lda_v, _loc_hi, _pstate;
-
- _loc_hi = (((uint64_t)loc)>>32);
- if (PHYS_ASI(asi)) {
- __asm volatile(
- "wr %4,%%g0,%%asi; "
- "rdpr %%pstate,%1; "
- "sllx %3,32,%0; "
- "wrpr %1,8,%%pstate; "
- "or %0,%2,%0; "
- "membar #Sync; "
- "lda [%0]%%asi,%0; "
- "wrpr %1,0,%%pstate; "
- "membar #Sync; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_lda_v), "=&r" (_pstate)
- : "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- } else {
- __asm volatile(
- "wr %3,%%g0,%%asi; "
- "sllx %2,32,%0; "
- "or %0,%1,%0; "
- "lda [%0]%%asi,%0; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_lda_v)
- : "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- }
- return (_lda_v);
-}
-
+SPARC64_LD_DEF(lda, uint32_t, uint32_t)
/* load signed int from alternate address space */
-static __inline int
-ldswa(paddr_t loc, int asi)
-{
- register int _lda_v, _loc_hi, _pstate;
-
- _loc_hi = (((uint64_t)loc)>>32);
- if (PHYS_ASI(asi)) {
- __asm volatile(
- "wr %4,%%g0,%%asi; "
- "rdpr %%pstate,%1; "
- "wrpr %1,8,%%pstate; "
- "sllx %3,32,%0; "
- " or %0,%2,%0; "
- "membar #Sync; "
- "ldswa [%0]%%asi,%0; "
- "wrpr %1,0,%%pstate; "
- "membar #Sync; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_lda_v), "=&r" (_pstate)
- : "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- } else {
- __asm volatile(
- "wr %3,%%g0,%%asi; "
- "sllx %2,32,%0; "
- "or %0,%1,%0; "
- "ldswa [%0]%%asi,%0; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_lda_v)
- : "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- }
- return (_lda_v);
-}
-#endif /* __arch64__ */
-
-#ifdef __arch64__
-/* load 64-bit int from alternate address space -- these should never be used */
-static __inline uint64_t
-ldda(paddr_t loc, int asi)
-{
- register long long _lda_v;
-
- __asm volatile(
- "wr %2,%%g0,%%asi; "
- "ldda [%1]%%asi,%0 "
- : "=r" (_lda_v)
- : "r" ((unsigned long)(loc)), "r" (asi));
- return (_lda_v);
-}
-#else
-/* load 64-bit int from alternate address space */
-static __inline uint64_t
-ldda(paddr_t loc, int asi)
-{
- register long long _lda_v, _loc_hi, _pstate;
-
- _loc_hi = (((uint64_t)loc)>>32);
- if (PHYS_ASI(asi)) {
- __asm volatile(
- "wr %4,%%g0,%%asi; "
- "rdpr %%pstate,%1; "
- "wrpr %1,8,%%pstate; "
- "sllx %3,32,%0; "
- "or %0,%2,%0; "
- "membar #Sync; "
- "ldda [%0]%%asi,%0; "
- "wrpr %1,0,%%pstate; "
- "membar #Sync; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_lda_v), "=&r" (_pstate)
- : "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- } else {
- __asm volatile(
- "wr %3,%%g0,%%asi; "
- "sllx %2,32,%0; "
- " or %0,%1,%0; "
- "ldda [%0]%%asi,%0; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_lda_v)
- : "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- }
- return (_lda_v);
-}
-#endif
+SPARC64_LD_DEF(ldswa, int, int)
+/* load 64-bit unsigned int from alternate address space */
+SPARC64_LD_DEF64(ldxa, uint64_t)
#ifdef __arch64__
-/* native load 64-bit int from alternate address space w/64-bit compiler*/
-static __inline uint64_t
-ldxa(paddr_t loc, int asi)
-{
- register unsigned long _lda_v;
-
- __asm volatile(
- "wr %2,%%g0,%%asi; "
- "ldxa [%1]%%asi,%0 "
- : "=r" (_lda_v)
- : "r" ((unsigned long)(loc)), "r" (asi));
- return (_lda_v);
-}
-#else
-/* native load 64-bit int from alternate address space w/32-bit compiler*/
-static __inline uint64_t
-ldxa(paddr_t loc, int asi)
-{
- register unsigned long _ldxa_lo, _ldxa_hi, _loc_hi;
- _loc_hi = (((uint64_t)loc)>>32);
- if (PHYS_ASI(asi)) {
- __asm volatile(
- "wr %4,%%g0,%%asi; "
- "rdpr %%pstate,%1; "
- "sllx %3,32,%0; "
- "wrpr %1,8,%%pstate; "
- "or %0, %2, %0; "
- "membar #Sync; "
- "ldxa [%0]%%asi,%0; "
- "wrpr %1,0,%%pstate; "
- "membar #Sync; "
- "srlx %0, 32, %1; "
- "srl %0, 0, %0; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_ldxa_lo), "=&r" (_ldxa_hi)
- : "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- } else {
- __asm volatile(
- "wr %4,%%g0,%%asi; "
- "sllx %3,32,%0; "
- "or %0,%2,%0; "
- "ldxa [%0]%%asi,%0; "
- "srlx %0,32,%1; "
- "srl %0, 0, %0; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_ldxa_lo), "=&r" (_ldxa_hi)
- : "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- }
- return ((((int64_t)_ldxa_hi)<<32)|_ldxa_lo);
+/* 64-bit kernel, non-constant */
+#define SPARC64_ST_NONCONST(st) \
+ __asm volatile( \
+ "wr %2,%%g0,%%asi; " \
+ #st " %0,[%1]%%asi " \
+ : : "r" (value), "r" ((__uintptr_t)(loc)), \
+ "r" (asi))
+
+#if defined(__GNUC__) && defined(__OPTIMIZE__)
+#define SPARC64_ST_DEF(st, type) \
+static __inline void st(paddr_t loc, int asi, type value) \
+{ \
+ if (__builtin_constant_p(asi)) \
+ __asm volatile( \
+ #st " %0,[%1]%2 " \
+ : : "r" (value), "r" ((__uintptr_t)(loc)), \
+ "n" (asi)); \
+ else \
+ SPARC64_ST_NONCONST(st); \
+}
+#else
+#define SPARC64_ST_DEF(st, type) \
+static __inline void st(paddr_t loc, int asi, type value) \
+{ \
+ SPARC64_ST_NONCONST(st); \
}
#endif
+#define SPARC64_ST_DEF64(st, type) SPARC64_ST_DEF(st, type)
-/* store byte to alternate address space */
-#ifdef __arch64__
-static __inline void
-stba(paddr_t loc, int asi, u_char value)
-{
- __asm volatile(
- "wr %2, %%g0, %%asi; "
- "stba %0, [%1]%%asi "
- : : "r" ((int)(value)), "r" ((unsigned long)(loc)), "r" (asi));
-}
-#else
-static __inline void
-stba(paddr_t loc, int asi, u_char value)
-{
- register int _loc_hi, _pstate;
-
- _loc_hi = (((uint64_t)loc)>>32);
- if (PHYS_ASI(asi)) {
- __asm volatile(
- "wr %5,%%g0,%%asi; "
- "sllx %4,32,%0; "
- "rdpr %%pstate,%1; "
- "or %3,%0,%0; "
- "wrpr %1,8,%%pstate; "
- "stba %2,[%0]%%asi; "
- "wrpr %1,0,%%pstate; "
- "membar #Sync; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_loc_hi), "=&r" (_pstate)
- : "r" ((int)(value)), "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- } else {
- __asm volatile(
- "wr %4,%%g0,%%asi; "
- "sllx %3,32,%0; "
- "or %2,%0,%0; "
- "stba %1,[%0]%%asi; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_loc_hi)
- : "r" ((int)(value)), "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi));
- }
-}
-#endif
-
-/* store half-word to alternate address space */
-#ifdef __arch64__
-static __inline void
-stha(paddr_t loc, int asi, u_short value)
-{
- __asm volatile(
- "wr %2,%%g0,%%asi; "
- "stha %0,[%1]%%asi "
- : : "r" ((int)(value)), "r" ((unsigned long)(loc)),
- "r" (asi) : "memory");
-}
-#else
-static __inline void
-stha(paddr_t loc, int asi, u_short value)
-{
- register int _loc_hi, _pstate;
+#else /* __arch64__ */
- _loc_hi = (((uint64_t)loc)>>32);
- if (PHYS_ASI(asi)) {
- __asm volatile(
- "wr %5,%%g0,%%asi; "
- "sllx %4,32,%0; "
- "rdpr %%pstate,%1; "
- "or %3,%0,%0; "
- "wrpr %1,8,%%pstate; "
- "stha %2,[%0]%%asi; "
- "wrpr %1,0,%%pstate; "
- "membar #Sync; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_loc_hi), "=&r" (_pstate)
- : "r" ((int)(value)), "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi)
- : "memory");
- } else {
- __asm volatile(
- "wr %4,%%g0,%%asi; "
- "sllx %3,32,%0; "
- "or %2,%0,%0; "
- "stha %1,[%0]%%asi; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_loc_hi)
- : "r" ((int)(value)), "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi)
- : "memory");
- }
+/* 32-bit kernel, MMU bypass, non-constant */
+#define SPARC64_ST_PHYS_NONCONST(st) \
+ __asm volatile( \
+ "rdpr %%pstate,%1; " \
+ "sllx %4,32,%0; " \
+ "wrpr %1,8,%%pstate; " \
+ "or %0,%3,%0; " \
+ "wr %5,%%g0,%%asi; " \
+ #st " %2,[%0]%%asi; " \
+ "wrpr %1,0,%%pstate " \
+ : "=&r" (_hi), "=&r" (_pstate) \
+ : "r" (value), "r" ((uint32_t)(loc)), \
+ "r" (_hi), "r" (asi))
+/* 32-bit kernel, non-constant */
+#define SPARC64_ST_NONCONST(st) \
+ __asm volatile( \
+ "wr %2,%%g0,%%asi; " \
+ #st " %0,[%1]%%asi " \
+ : : "r" (value), "r" ((uint32_t)(loc)), "r" (asi))
+/* 32-bit kernel, MMU bypass, non-constant, 64-bit value */
+#define SPARC64_ST_PHYS_NONCONST64(st) \
+ __asm volatile( \
+ "sllx %4,32,%1; " \
+ "sllx %6,32,%0; " \
+ "rdpr %%pstate,%2; " \
+ "or %1,%3,%1; " \
+ "wrpr %2,8,%%pstate; " \
+ "or %0,%5,%0; " \
+ "wr %7,%%g0,%%asi; " \
+ #st " %1,[%0]%%asi; " \
+ "wrpr %2,0,%%pstate " \
+ : "=&r" (_hi), "=&r" (_vhi), "=&r" (_vlo) \
+ : "r" (_vlo), "r" (_vhi), \
+ "r" ((uint32_t)(loc)), "r" (_hi), "r" (asi))
+/* 32-bit kernel, non-constant, 64-bit value */
+#define SPARC64_ST_NONCONST64(st) \
+ __asm volatile( \
+ "sllx %2,32,%0; " \
+ "or %0,%1,%0; " \
+ "wr %4,%%g0,%%asi; " \
+ #st " %0,[%3]%%asi " \
+ : "=&r" (_vhi) \
+ : "r" (_vlo), "r" (_vhi), \
+ "r" ((uint32_t)(loc)), "r" (asi))
+
+#if defined(__GNUC__) && defined(__OPTIMIZE__)
+#define SPARC64_ST_DEF(st, type) \
+static __inline void st(paddr_t loc, int asi, type value) \
+{ \
+ uint32_t _hi, _pstate; \
+ if (PHYS_ASI(asi)) { \
+ _hi = (uint64_t)(loc) >> 32; \
+ if (__builtin_constant_p(asi)) \
+ __asm volatile( \
+ "sllx %4,32,%0; " \
+ "rdpr %%pstate,%1; " \
+ "or %0,%3,%0; " \
+ "wrpr %1,8,%%pstate; " \
+ #st " %2,[%0]%5; " \
+ "wrpr %1,0,%%pstate " \
+ : "=&r" (_hi), "=&r" (_pstate) \
+ : "r" (value), "r" ((uint32_t)(loc)), \
+ "r" (_hi), "n" (asi)); \
+ else \
+ SPARC64_ST_PHYS_NONCONST(st); \
+ } else { \
+ if (__builtin_constant_p(asi)) \
+ __asm volatile( \
+ #st " %0,[%1]%2 " \
+ : : "r" (value), "r" ((uint32_t)(loc)), \
+ "n" (asi)); \
+ else \
+ SPARC64_ST_NONCONST(st); \
+ } \
+}
+#define SPARC64_ST_DEF64(st, type) \
+static __inline void st(paddr_t loc, int asi, type value) \
+{ \
+ uint32_t _vlo, _vhi, _hi; \
+ _vlo = value; \
+ _vhi = (uint64_t)(value) >> 32; \
+ if (PHYS_ASI(asi)) { \
+ _hi = (uint64_t)(loc) >> 32; \
+ if (__builtin_constant_p(asi)) \
+ __asm volatile( \
+ "sllx %4,32,%1; " \
+ "sllx %6,32,%0; " \
+ "rdpr %%pstate,%2; " \
+ "or %1,%3,%1; " \
+ "or %0,%5,%0; " \
+ "wrpr %2,8,%%pstate; " \
+ #st " %1,[%0]%7; " \
+ "wrpr %2,0,%%pstate " \
+ : "=&r" (_hi), "=&r" (_vhi), "=&r" (_vlo) \
+ : "r" (_vlo), "r" (_vhi), \
+ "r" ((uint32_t)(loc)), "r" (_hi), \
+ "n" (asi)); \
+ else \
+ SPARC64_ST_PHYS_NONCONST64(st); \
+ } else { \
+ if (__builtin_constant_p(asi)) \
+ __asm volatile( \
+ "sllx %2,32,%0; " \
+ "or %0,%1,%0; " \
+ #st " %0,[%3]%4 " \
+ : "=&r" (_vhi) \
+ : "r" (_vlo), "r" (_vhi), \
+ "r" ((uint32_t)(loc)), "n" (asi)); \
+ else \
+ SPARC64_ST_NONCONST64(st); \
+ } \
+}
+#else
+#define SPARC64_ST_DEF(st, type) \
+static __inline void st(paddr_t loc, int asi, type value) \
+{ \
+ uint32_t _hi, _pstate; \
+ if (PHYS_ASI(asi)) { \
+ _hi = (uint64_t)(loc) >> 32; \
+ SPARC64_ST_PHYS_NONCONST(st); \
+ } else \
+ SPARC64_ST_NONCONST(st); \
+}
+#define SPARC64_ST_DEF64(st, type) \
+static __inline void st(paddr_t loc, int asi, type value) \
+{ \
+ uint32_t _vlo, _vhi, _hi; \
+ _vlo = value; \
+ _vhi = (uint64_t)(value) >> 32; \
+ if (PHYS_ASI(asi)) { \
+ _hi = (uint64_t)(loc) >> 32; \
+ SPARC64_ST_PHYS_NONCONST64(st); \
+ } else \
+ SPARC64_ST_NONCONST64(st); \
}
#endif
+#endif /* __arch64__ */
-/* store int to alternate address space */
-#ifdef __arch64__
-static __inline void
-sta(paddr_t loc, int asi, u_int value)
-{
- __asm volatile(
- "wr %2,%%g0,%%asi; "
- "sta %0,[%1]%%asi "
- : : "r" ((int)(value)), "r" ((unsigned long)(loc)),
- "r" (asi) : "memory");
-}
-#else
-static __inline void
-sta(paddr_t loc, int asi, u_int value)
-{
- register int _loc_hi, _pstate;
-
- _loc_hi = (((uint64_t)loc)>>32);
- if (PHYS_ASI(asi)) {
- __asm volatile(
- "wr %5,%%g0,%%asi; "
- "sllx %4,32,%0; "
- "rdpr %%pstate,%1; "
- "or %3,%0,%0; "
- "wrpr %1,8,%%pstate; "
- "sta %2,[%0]%%asi; "
- "wrpr %1,0,%%pstate; "
- "membar #Sync; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_loc_hi), "=&r" (_pstate)
- : "r" ((int)(value)), "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi)
- : "memory");
- } else {
- __asm volatile(
- "wr %4,%%g0,%%asi; "
- "sllx %3,32,%0; "
- "or %2,%0,%0; "
- "sta %1,[%0]%%asi; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_loc_hi)
- : "r" ((int)(value)), "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi)
- : "memory");
- }
-}
-#endif
-
-/* store 64-bit int to alternate address space */
-#ifdef __arch64__
-static __inline void
-stda(paddr_t loc, int asi, uint64_t value)
-{
- __asm volatile(
- "wr %2,%%g0,%%asi; "
- "stda %0,[%1]%%asi "
- : : "r" ((long long)(value)), "r" ((unsigned long)(loc)), "r" (asi)
- : "memory");
-}
-#else
-static __inline void
-stda(paddr_t loc, int asi, uint64_t value)
-{
- register int _loc_hi, _pstate;
+/* store byte to alternate address space */
+SPARC64_ST_DEF(stba, uint8_t)
+/* store half-word to alternate address space */
+SPARC64_ST_DEF(stha, uint16_t)
+/* store unsigned int to alternate address space */
+SPARC64_ST_DEF(sta, uint32_t)
+/* store 64-bit unsigned int to alternate address space */
+SPARC64_ST_DEF64(stxa, uint64_t)
- _loc_hi = (((uint64_t)loc)>>32);
- if (PHYS_ASI(asi)) {
- __asm volatile(
- "wr %5,%%g0,%%asi; "
- "sllx %4,32,%0; "
- "rdpr %%pstate,%1; "
- "or %3,%0,%0; "
- "wrpr %1,8,%%pstate; "
- "stda %2,[%0]%%asi; "
- "wrpr %1,0,%%pstate; "
- "membar #Sync; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_loc_hi), "=&r" (_pstate)
- : "r" ((long long)(value)), "r" ((unsigned long)(loc)),
- "r" (_loc_hi), "r" (asi)
- : "memory");
- } else {
- __asm volatile(
- "wr %4,%%g0,%%asi; "
- "sllx %3,32,%0; "
- "or %2,%0,%0; "
- "stda %1,[%0]%%asi; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_loc_hi)
- : "r" ((long long)(value)), "r" ((unsigned long)(loc)),
- "r" (_loc_hi), "r" (asi)
- : "memory");
- }
-}
-#endif
/* set dmmu secondary context */
static __inline void
@@ -969,69 +851,11 @@
__asm volatile(
"stxa %0,[%1]%2; "
"membar #Sync "
- : : "r" (ctx),
- "r" (CTX_SECONDARY), "n" (ASI_DMMU)
+ : : "r" (ctx), "r" (CTX_SECONDARY), "n" (ASI_DMMU)
: "memory");
}
#ifdef __arch64__
-/* native store 64-bit int to alternate address space w/64-bit compiler*/
-static __inline void
-stxa(paddr_t loc, int asi, uint64_t value)
-{
- __asm volatile(
- "wr %2,%%g0,%%asi; "
- "stxa %0,[%1]%%asi "
- : : "r" ((unsigned long)(value)),
- "r" ((unsigned long)(loc)), "r" (asi)
- : "memory");
-}
-#else
-/* native store 64-bit int to alternate address space w/32-bit compiler*/
-static __inline void
-stxa(paddr_t loc, int asi, uint64_t value)
-{
- int _stxa_lo, _stxa_hi, _loc_hi;
-
- _stxa_lo = value;
- _stxa_hi = ((uint64_t)value)>>32;
- _loc_hi = (((uint64_t)loc)>>32);
-
- if (PHYS_ASI(asi)) {
- __asm volatile(
- "wr %7,%%g0,%%asi; "
- "sllx %4,32,%1; "
- "sllx %6,32,%0; "
- "or %1,%3,%1; "
- "rdpr %%pstate,%2; "
- "or %0,%5,%0; "
- "wrpr %2,8,%%pstate; "
- "stxa %1,[%0]%%asi; "
- "wrpr %2,0,%%pstate; "
- "membar #Sync; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_loc_hi), "=&r" (_stxa_hi), "=&r" ((int)(_stxa_lo))
- : "r" ((int)(_stxa_lo)), "r" ((int)(_stxa_hi)),
- "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi)
- : "memory");
- } else {
- __asm volatile(
- "wr %6,%%g0,%%asi; "
- "sllx %3,32,%1; "
- "sllx %5,32,%0; "
- "or %1,%2,%1; "
- "or %0,%4,%0; "
- "stxa %1,[%0]%%asi; "
- "wr %%g0, 0x82, %%asi "
- : "=&r" (_loc_hi), "=&r" (_stxa_hi)
- : "r" ((int)(_stxa_lo)), "r" ((int)(_stxa_hi)),
- "r" ((unsigned long)(loc)), "r" (_loc_hi), "r" (asi)
- : "memory");
- }
-}
-#endif
-
-#ifdef __arch64__
/* native store 32-bit int to alternate address space w/64-bit compiler*/
static __inline uint32_t
casa(paddr_t loc, int asi, uint32_t value, uint32_t oldvalue)
