Re: [PATCH v6 10/18] arm64/mm: New ptep layer to manage contig bit
On Thu, Feb 15, 2024 at 10:31:57AM +, Ryan Roberts wrote: > Create a new layer for the in-table PTE manipulation APIs. For now, The > existing API is prefixed with double underscore to become the > arch-private API and the public API is just a simple wrapper that calls > the private API. > > The public API implementation will subsequently be used to transparently > manipulate the contiguous bit where appropriate. But since there are > already some contig-aware users (e.g. hugetlb, kernel mapper), we must > first ensure those users use the private API directly so that the future > contig-bit manipulations in the public API do not interfere with those > existing uses. > > The following APIs are treated this way: > > - ptep_get > - set_pte > - set_ptes > - pte_clear > - ptep_get_and_clear > - ptep_test_and_clear_young > - ptep_clear_flush_young > - ptep_set_wrprotect > - ptep_set_access_flags > > Tested-by: John Hubbard > Signed-off-by: Ryan Roberts Acked-by: Catalin Marinas
Re: [PATCH v6 10/18] arm64/mm: New ptep layer to manage contig bit
On Thu, Feb 15, 2024 at 10:31:57AM +, Ryan Roberts wrote:
> Create a new layer for the in-table PTE manipulation APIs. For now, The
> existing API is prefixed with double underscore to become the
> arch-private API and the public API is just a simple wrapper that calls
> the private API.
>
> The public API implementation will subsequently be used to transparently
> manipulate the contiguous bit where appropriate. But since there are
> already some contig-aware users (e.g. hugetlb, kernel mapper), we must
> first ensure those users use the private API directly so that the future
> contig-bit manipulations in the public API do not interfere with those
> existing uses.
>
> The following APIs are treated this way:
>
> - ptep_get
> - set_pte
> - set_ptes
> - pte_clear
> - ptep_get_and_clear
> - ptep_test_and_clear_young
> - ptep_clear_flush_young
> - ptep_set_wrprotect
> - ptep_set_access_flags
>
> Tested-by: John Hubbard
> Signed-off-by: Ryan Roberts
Acked-by: Mark Rutland
Mark.
> ---
> arch/arm64/include/asm/pgtable.h | 83 +---
> arch/arm64/kernel/efi.c | 4 +-
> arch/arm64/kernel/mte.c | 2 +-
> arch/arm64/kvm/guest.c | 2 +-
> arch/arm64/mm/fault.c| 12 ++---
> arch/arm64/mm/fixmap.c | 4 +-
> arch/arm64/mm/hugetlbpage.c | 40 +++
> arch/arm64/mm/kasan_init.c | 6 +--
> arch/arm64/mm/mmu.c | 14 +++---
> arch/arm64/mm/pageattr.c | 6 +--
> arch/arm64/mm/trans_pgd.c| 6 +--
> 11 files changed, 93 insertions(+), 86 deletions(-)
>
> diff --git a/arch/arm64/include/asm/pgtable.h
> b/arch/arm64/include/asm/pgtable.h
> index 9a2df85eb493..7336d40a893a 100644
> --- a/arch/arm64/include/asm/pgtable.h
> +++ b/arch/arm64/include/asm/pgtable.h
> @@ -93,7 +93,8 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
> __pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) |
> pgprot_val(prot))
>
> #define pte_none(pte)(!pte_val(pte))
> -#define pte_clear(mm,addr,ptep) set_pte(ptep, __pte(0))
> +#define __pte_clear(mm, addr, ptep) \
> + __set_pte(ptep, __pte(0))
> #define pte_page(pte)(pfn_to_page(pte_pfn(pte)))
>
> /*
> @@ -137,7 +138,7 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
> * so that we don't erroneously return false for pages that have been
> * remapped as PROT_NONE but are yet to be flushed from the TLB.
> * Note that we can't make any assumptions based on the state of the access
> - * flag, since ptep_clear_flush_young() elides a DSB when invalidating the
> + * flag, since __ptep_clear_flush_young() elides a DSB when invalidating the
> * TLB.
> */
> #define pte_accessible(mm, pte) \
> @@ -261,7 +262,7 @@ static inline pte_t pte_mkdevmap(pte_t pte)
> return set_pte_bit(pte, __pgprot(PTE_DEVMAP | PTE_SPECIAL));
> }
>
> -static inline void set_pte(pte_t *ptep, pte_t pte)
> +static inline void __set_pte(pte_t *ptep, pte_t pte)
> {
> WRITE_ONCE(*ptep, pte);
>
> @@ -275,8 +276,7 @@ static inline void set_pte(pte_t *ptep, pte_t pte)
> }
> }
>
> -#define ptep_get ptep_get
> -static inline pte_t ptep_get(pte_t *ptep)
> +static inline pte_t __ptep_get(pte_t *ptep)
> {
> return READ_ONCE(*ptep);
> }
> @@ -308,7 +308,7 @@ static inline void __check_safe_pte_update(struct
> mm_struct *mm, pte_t *ptep,
> if (!IS_ENABLED(CONFIG_DEBUG_VM))
> return;
>
> - old_pte = ptep_get(ptep);
> + old_pte = __ptep_get(ptep);
>
> if (!pte_valid(old_pte) || !pte_valid(pte))
> return;
> @@ -317,7 +317,7 @@ static inline void __check_safe_pte_update(struct
> mm_struct *mm, pte_t *ptep,
>
> /*
>* Check for potential race with hardware updates of the pte
> - * (ptep_set_access_flags safely changes valid ptes without going
> + * (__ptep_set_access_flags safely changes valid ptes without going
>* through an invalid entry).
>*/
> VM_WARN_ONCE(!pte_young(pte),
> @@ -363,23 +363,22 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned
> long nr)
> return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte));
> }
>
> -static inline void set_ptes(struct mm_struct *mm,
> - unsigned long __always_unused addr,
> - pte_t *ptep, pte_t pte, unsigned int nr)
> +static inline void __set_ptes(struct mm_struct *mm,
> + unsigned long __always_unused addr,
> + pte_t *ptep, pte_t pte, unsigned int nr)
> {
> page_table_check_ptes_set(mm, ptep, pte, nr);
> __sync_cache_and_tags(pte, nr);
>
> for (;;) {
> __check_safe_pte_update(mm, ptep, pte);
> - set_pte(ptep, pte);
> + __set_pte(ptep, pte);
> if (--nr == 0)
> break;
> ptep++;
>
[PATCH v6 10/18] arm64/mm: New ptep layer to manage contig bit
Create a new layer for the in-table PTE manipulation APIs. For now, The
existing API is prefixed with double underscore to become the
arch-private API and the public API is just a simple wrapper that calls
the private API.
The public API implementation will subsequently be used to transparently
manipulate the contiguous bit where appropriate. But since there are
already some contig-aware users (e.g. hugetlb, kernel mapper), we must
first ensure those users use the private API directly so that the future
contig-bit manipulations in the public API do not interfere with those
existing uses.
The following APIs are treated this way:
- ptep_get
- set_pte
- set_ptes
- pte_clear
- ptep_get_and_clear
- ptep_test_and_clear_young
- ptep_clear_flush_young
- ptep_set_wrprotect
- ptep_set_access_flags
Tested-by: John Hubbard
Signed-off-by: Ryan Roberts
---
arch/arm64/include/asm/pgtable.h | 83 +---
arch/arm64/kernel/efi.c | 4 +-
arch/arm64/kernel/mte.c | 2 +-
arch/arm64/kvm/guest.c | 2 +-
arch/arm64/mm/fault.c| 12 ++---
arch/arm64/mm/fixmap.c | 4 +-
arch/arm64/mm/hugetlbpage.c | 40 +++
arch/arm64/mm/kasan_init.c | 6 +--
arch/arm64/mm/mmu.c | 14 +++---
arch/arm64/mm/pageattr.c | 6 +--
arch/arm64/mm/trans_pgd.c| 6 +--
11 files changed, 93 insertions(+), 86 deletions(-)
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 9a2df85eb493..7336d40a893a 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -93,7 +93,8 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
__pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) |
pgprot_val(prot))
#define pte_none(pte) (!pte_val(pte))
-#define pte_clear(mm,addr,ptep)set_pte(ptep, __pte(0))
+#define __pte_clear(mm, addr, ptep) \
+ __set_pte(ptep, __pte(0))
#define pte_page(pte) (pfn_to_page(pte_pfn(pte)))
/*
@@ -137,7 +138,7 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
* so that we don't erroneously return false for pages that have been
* remapped as PROT_NONE but are yet to be flushed from the TLB.
* Note that we can't make any assumptions based on the state of the access
- * flag, since ptep_clear_flush_young() elides a DSB when invalidating the
+ * flag, since __ptep_clear_flush_young() elides a DSB when invalidating the
* TLB.
*/
#define pte_accessible(mm, pte)\
@@ -261,7 +262,7 @@ static inline pte_t pte_mkdevmap(pte_t pte)
return set_pte_bit(pte, __pgprot(PTE_DEVMAP | PTE_SPECIAL));
}
-static inline void set_pte(pte_t *ptep, pte_t pte)
+static inline void __set_pte(pte_t *ptep, pte_t pte)
{
WRITE_ONCE(*ptep, pte);
@@ -275,8 +276,7 @@ static inline void set_pte(pte_t *ptep, pte_t pte)
}
}
-#define ptep_get ptep_get
-static inline pte_t ptep_get(pte_t *ptep)
+static inline pte_t __ptep_get(pte_t *ptep)
{
return READ_ONCE(*ptep);
}
@@ -308,7 +308,7 @@ static inline void __check_safe_pte_update(struct mm_struct
*mm, pte_t *ptep,
if (!IS_ENABLED(CONFIG_DEBUG_VM))
return;
- old_pte = ptep_get(ptep);
+ old_pte = __ptep_get(ptep);
if (!pte_valid(old_pte) || !pte_valid(pte))
return;
@@ -317,7 +317,7 @@ static inline void __check_safe_pte_update(struct mm_struct
*mm, pte_t *ptep,
/*
* Check for potential race with hardware updates of the pte
-* (ptep_set_access_flags safely changes valid ptes without going
+* (__ptep_set_access_flags safely changes valid ptes without going
* through an invalid entry).
*/
VM_WARN_ONCE(!pte_young(pte),
@@ -363,23 +363,22 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned
long nr)
return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte));
}
-static inline void set_ptes(struct mm_struct *mm,
- unsigned long __always_unused addr,
- pte_t *ptep, pte_t pte, unsigned int nr)
+static inline void __set_ptes(struct mm_struct *mm,
+ unsigned long __always_unused addr,
+ pte_t *ptep, pte_t pte, unsigned int nr)
{
page_table_check_ptes_set(mm, ptep, pte, nr);
__sync_cache_and_tags(pte, nr);
for (;;) {
__check_safe_pte_update(mm, ptep, pte);
- set_pte(ptep, pte);
+ __set_pte(ptep, pte);
if (--nr == 0)
break;
ptep++;
pte = pte_advance_pfn(pte, 1);
}
}
-#define set_ptes set_ptes
/*
* Huge pte definitions.
@@ -546,7 +545,7 @@ static inline void __set_pte_at(struct mm_struct *mm,
{
__sync_cache_and_tags(pte, nr);
__check_safe_pte_update(mm, ptep, pte);
-
