[PATCH v4 15/26] arm64: irqflags: Use ICC_PMR_EL1 for interrupt masking

[Julien Thierry]

Instead disabling interrupts by setting the PSR.I bit, use a priority
higher than the one used for interrupts to mask them via PMR.

The value chosen for PMR to enable/disable interrupts encodes the status
of interrupts on a single bit. This information is stored in the irqflags
values used when saving/restoring IRQ status.

Signed-off-by: Julien Thierry 
Suggested-by: Daniel Thompson 
Cc: Catalin Marinas 
Cc: Will Deacon 
Cc: Ard Biesheuvel 
Cc: Oleg Nesterov 
---
 arch/arm64/include/asm/assembler.h | 17 ++-
 arch/arm64/include/asm/efi.h   |  3 +-
 arch/arm64/include/asm/irqflags.h  | 97 ++
 arch/arm64/include/asm/ptrace.h| 10 ++--
 arch/arm64/kernel/entry.S  |  6 +--
 5 files changed, 104 insertions(+), 29 deletions(-)

diff --git a/arch/arm64/include/asm/assembler.h 
b/arch/arm64/include/asm/assembler.h
index 0bcc98d..0b2dcfd 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -23,6 +23,7 @@
 #ifndef __ASM_ASSEMBLER_H
 #define __ASM_ASSEMBLER_H
 
+#include 
 #include 
 #include 
 #include 
@@ -62,12 +63,24 @@
 /*
  * Enable and disable interrupts.
  */
-   .macro  disable_irq
+   .macro  disable_irq, tmp
+   mov \tmp, #ICC_PMR_EL1_MASKED
+alternative_if_not ARM64_HAS_IRQ_PRIO_MASKING
msr daifset, #2
+alternative_else
+   msr_s   SYS_ICC_PMR_EL1, \tmp
+alternative_endif
.endm
 
-   .macro  enable_irq
+   .macro  enable_irq, tmp
+   mov \tmp, #ICC_PMR_EL1_UNMASKED
+alternative_if_not ARM64_HAS_IRQ_PRIO_MASKING
msr daifclr, #2
+   nop
+alternative_else
+   msr_s   SYS_ICC_PMR_EL1, \tmp
+   dsb sy
+alternative_endif
.endm
 
.macro  save_and_disable_irq, flags
diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h
index 192d791..a4e0730 100644
--- a/arch/arm64/include/asm/efi.h
+++ b/arch/arm64/include/asm/efi.h
@@ -42,7 +42,8 @@
 
 efi_status_t __efi_rt_asm_wrapper(void *, const char *, ...);
 
-#define ARCH_EFI_IRQ_FLAGS_MASK (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
+#define ARCH_EFI_IRQ_FLAGS_MASK \
+   (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | ARCH_FLAG_PMR_EN)
 
 /* arch specific definitions used by the stub code */
 
diff --git a/arch/arm64/include/asm/irqflags.h 
b/arch/arm64/include/asm/irqflags.h
index 24692ed..193cfd0 100644
--- a/arch/arm64/include/asm/irqflags.h
+++ b/arch/arm64/include/asm/irqflags.h
@@ -18,7 +18,27 @@
 
 #ifdef __KERNEL__
 
+#include 
+#include 
 #include 
+#include 
+
+
+/*
+ * When ICC_PMR_EL1 is used for interrupt masking, only the bit indicating
+ * whether the normal interrupts are masked is kept along with the daif
+ * flags.
+ */
+#define ARCH_FLAG_PMR_EN 0x1
+
+#define MAKE_ARCH_FLAGS(daif, pmr) \
+   ((daif) | (((pmr) >> ICC_PMR_EL1_EN_SHIFT) & ARCH_FLAG_PMR_EN))
+
+#define ARCH_FLAGS_GET_PMR(flags)  \
+   flags) & ARCH_FLAG_PMR_EN) << ICC_PMR_EL1_EN_SHIFT) \
+   | ICC_PMR_EL1_MASKED)
+
+#define ARCH_FLAGS_GET_DAIF(flags) ((flags) & ~ARCH_FLAG_PMR_EN)
 
 /*
  * Aarch64 has flags for masking: Debug, Asynchronous (serror), Interrupts and
@@ -38,31 +58,50 @@
  */
 static inline unsigned long arch_local_irq_save(void)
 {
-   unsigned long flags;
-   asm volatile(
+   unsigned long flags, masked = ICC_PMR_EL1_MASKED;
+   unsigned long pmr = 0;
+
+   asm volatile(ALTERNATIVE(
"mrs%0, daif// arch_local_irq_save\n"
-   "msrdaifset, #2"
-   : "=r" (flags)
-   :
+   "msrdaifset, #2\n"
+   "mov%1, #" __stringify(ICC_PMR_EL1_UNMASKED),
+   /* --- */
+   "mrs%0, daif\n"
+   "mrs_s  %1, " __stringify(SYS_ICC_PMR_EL1) "\n"
+   "msr_s  " __stringify(SYS_ICC_PMR_EL1) ", %2",
+   ARM64_HAS_IRQ_PRIO_MASKING)
+   : "=" (flags), "=" (pmr)
+   : "r" (masked)
: "memory");
-   return flags;
+
+   return MAKE_ARCH_FLAGS(flags, pmr);
 }
 
 static inline void arch_local_irq_enable(void)
 {
-   asm volatile(
-   "msrdaifclr, #2 // arch_local_irq_enable"
-   :
+   unsigned long unmasked = ICC_PMR_EL1_UNMASKED;
+
+   asm volatile(ALTERNATIVE(
+   "msrdaifclr, #2 // arch_local_irq_enable\n"
+   "nop",
+   "msr_s  " __stringify(SYS_ICC_PMR_EL1) ",%0\n"
+   "dsbsy",
+   ARM64_HAS_IRQ_PRIO_MASKING)
:
+   : "r" (unmasked)
: "memory");
 }
 
 static inline void arch_local_irq_disable(void)
 {
-   asm volatile(
-   "msr

[PATCH v4 15/26] arm64: irqflags: Use ICC_PMR_EL1 for interrupt masking

[Julien Thierry]

Instead disabling interrupts by setting the PSR.I bit, use a priority
higher than the one used for interrupts to mask them via PMR.

The value chosen for PMR to enable/disable interrupts encodes the status
of interrupts on a single bit. This information is stored in the irqflags
values used when saving/restoring IRQ status.

Signed-off-by: Julien Thierry 
Suggested-by: Daniel Thompson 
Cc: Catalin Marinas 
Cc: Will Deacon 
Cc: Ard Biesheuvel 
Cc: Oleg Nesterov 
---
 arch/arm64/include/asm/assembler.h | 17 ++-
 arch/arm64/include/asm/efi.h   |  3 +-
 arch/arm64/include/asm/irqflags.h  | 97 ++
 arch/arm64/include/asm/ptrace.h| 10 ++--
 arch/arm64/kernel/entry.S  |  6 +--
 5 files changed, 104 insertions(+), 29 deletions(-)

diff --git a/arch/arm64/include/asm/assembler.h 
b/arch/arm64/include/asm/assembler.h
index 0bcc98d..0b2dcfd 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -23,6 +23,7 @@
 #ifndef __ASM_ASSEMBLER_H
 #define __ASM_ASSEMBLER_H
 
+#include 
 #include 
 #include 
 #include 
@@ -62,12 +63,24 @@
 /*
  * Enable and disable interrupts.
  */
-   .macro  disable_irq
+   .macro  disable_irq, tmp
+   mov \tmp, #ICC_PMR_EL1_MASKED
+alternative_if_not ARM64_HAS_IRQ_PRIO_MASKING
msr daifset, #2
+alternative_else
+   msr_s   SYS_ICC_PMR_EL1, \tmp
+alternative_endif
.endm
 
-   .macro  enable_irq
+   .macro  enable_irq, tmp
+   mov \tmp, #ICC_PMR_EL1_UNMASKED
+alternative_if_not ARM64_HAS_IRQ_PRIO_MASKING
msr daifclr, #2
+   nop
+alternative_else
+   msr_s   SYS_ICC_PMR_EL1, \tmp
+   dsb sy
+alternative_endif
.endm
 
.macro  save_and_disable_irq, flags
diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h
index 192d791..a4e0730 100644
--- a/arch/arm64/include/asm/efi.h
+++ b/arch/arm64/include/asm/efi.h
@@ -42,7 +42,8 @@
 
 efi_status_t __efi_rt_asm_wrapper(void *, const char *, ...);
 
-#define ARCH_EFI_IRQ_FLAGS_MASK (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
+#define ARCH_EFI_IRQ_FLAGS_MASK \
+   (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | ARCH_FLAG_PMR_EN)
 
 /* arch specific definitions used by the stub code */
 
diff --git a/arch/arm64/include/asm/irqflags.h 
b/arch/arm64/include/asm/irqflags.h
index 24692ed..193cfd0 100644
--- a/arch/arm64/include/asm/irqflags.h
+++ b/arch/arm64/include/asm/irqflags.h
@@ -18,7 +18,27 @@
 
 #ifdef __KERNEL__
 
+#include 
+#include 
 #include 
+#include 
+
+
+/*
+ * When ICC_PMR_EL1 is used for interrupt masking, only the bit indicating
+ * whether the normal interrupts are masked is kept along with the daif
+ * flags.
+ */
+#define ARCH_FLAG_PMR_EN 0x1
+
+#define MAKE_ARCH_FLAGS(daif, pmr) \
+   ((daif) | (((pmr) >> ICC_PMR_EL1_EN_SHIFT) & ARCH_FLAG_PMR_EN))
+
+#define ARCH_FLAGS_GET_PMR(flags)  \
+   flags) & ARCH_FLAG_PMR_EN) << ICC_PMR_EL1_EN_SHIFT) \
+   | ICC_PMR_EL1_MASKED)
+
+#define ARCH_FLAGS_GET_DAIF(flags) ((flags) & ~ARCH_FLAG_PMR_EN)
 
 /*
  * Aarch64 has flags for masking: Debug, Asynchronous (serror), Interrupts and
@@ -38,31 +58,50 @@
  */
 static inline unsigned long arch_local_irq_save(void)
 {
-   unsigned long flags;
-   asm volatile(
+   unsigned long flags, masked = ICC_PMR_EL1_MASKED;
+   unsigned long pmr = 0;
+
+   asm volatile(ALTERNATIVE(
"mrs%0, daif// arch_local_irq_save\n"
-   "msrdaifset, #2"
-   : "=r" (flags)
-   :
+   "msrdaifset, #2\n"
+   "mov%1, #" __stringify(ICC_PMR_EL1_UNMASKED),
+   /* --- */
+   "mrs%0, daif\n"
+   "mrs_s  %1, " __stringify(SYS_ICC_PMR_EL1) "\n"
+   "msr_s  " __stringify(SYS_ICC_PMR_EL1) ", %2",
+   ARM64_HAS_IRQ_PRIO_MASKING)
+   : "=" (flags), "=" (pmr)
+   : "r" (masked)
: "memory");
-   return flags;
+
+   return MAKE_ARCH_FLAGS(flags, pmr);
 }
 
 static inline void arch_local_irq_enable(void)
 {
-   asm volatile(
-   "msrdaifclr, #2 // arch_local_irq_enable"
-   :
+   unsigned long unmasked = ICC_PMR_EL1_UNMASKED;
+
+   asm volatile(ALTERNATIVE(
+   "msrdaifclr, #2 // arch_local_irq_enable\n"
+   "nop",
+   "msr_s  " __stringify(SYS_ICC_PMR_EL1) ",%0\n"
+   "dsbsy",
+   ARM64_HAS_IRQ_PRIO_MASKING)
:
+   : "r" (unmasked)
: "memory");
 }
 
 static inline void arch_local_irq_disable(void)
 {
-   asm volatile(
-   "msrdaifset, #2 // arch_local_irq_disable"
-   :
+   unsigned long masked = ICC_PMR_EL1_MASKED;
+
+   asm