Re: [PATCH v3 12/32] KVM: arm64: Introduce a Hyp buddy page allocator
On Thursday 04 Mar 2021 at 15:30:36 (+), Will Deacon wrote:
> On Tue, Mar 02, 2021 at 02:59:42PM +, Quentin Perret wrote:
> > +void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order)
> > +{
> > + unsigned int i = order;
> > + struct hyp_page *p;
> > +
> > + hyp_spin_lock(&pool->lock);
> > +
> > + /* Look for a high-enough-order page */
> > + while (i < pool->max_order && list_empty(&pool->free_area[i]))
> > + i++;
> > + if (i >= pool->max_order) {
> > + hyp_spin_unlock(&pool->lock);
> > + return NULL;
> > + }
> > +
> > + /* Extract it from the tree at the right order */
> > + p = list_first_entry(&pool->free_area[i], struct hyp_page, node);
> > + p = __hyp_extract_page(pool, p, order);
> > +
> > + hyp_spin_unlock(&pool->lock);
> > + hyp_page_ref_inc(p);
>
> I find this a little scary, as we momentarily drop the lock. It think
> it's ok because the reference count on the page must be 0 at this point,
Yep, @p shouldn't be visible to the caller yet so this should be fine.
> but actually then I think it would be clearer to have a
> hyp_page_ref_init() function which could take the lock, check that the
> refcount is indeed 0 and then set it to 1.
Works for me. Maybe I'll use another name for the API to stay consistent
with the kernel gfp code (hyp_page_ref_inc() and friends are inspired
from their kernel counterpart). And I guess I can hyp_panic() if the
refcount is not 0 at this point to match the VM_BUG_ON_PAGE() in
set_page_refcounted().
Thanks!
Quentin
___
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Re: [PATCH v3 12/32] KVM: arm64: Introduce a Hyp buddy page allocator
On Tue, Mar 02, 2021 at 02:59:42PM +, Quentin Perret wrote:
> When memory protection is enabled, the hyp code will require a basic
> form of memory management in order to allocate and free memory pages at
> EL2. This is needed for various use-cases, including the creation of hyp
> mappings or the allocation of stage 2 page tables.
>
> To address these use-case, introduce a simple memory allocator in the
> hyp code. The allocator is designed as a conventional 'buddy allocator',
> working with a page granularity. It allows to allocate and free
> physically contiguous pages from memory 'pools', with a guaranteed order
> alignment in the PA space. Each page in a memory pool is associated
> with a struct hyp_page which holds the page's metadata, including its
> refcount, as well as its current order, hence mimicking the kernel's
> buddy system in the GFP infrastructure. The hyp_page metadata are made
> accessible through a hyp_vmemmap, following the concept of
> SPARSE_VMEMMAP in the kernel.
>
> Signed-off-by: Quentin Perret
> ---
> arch/arm64/kvm/hyp/include/nvhe/gfp.h| 55 +++
> arch/arm64/kvm/hyp/include/nvhe/memory.h | 28
> arch/arm64/kvm/hyp/nvhe/Makefile | 2 +-
> arch/arm64/kvm/hyp/nvhe/page_alloc.c | 195 +++
> 4 files changed, 279 insertions(+), 1 deletion(-)
> create mode 100644 arch/arm64/kvm/hyp/include/nvhe/gfp.h
> create mode 100644 arch/arm64/kvm/hyp/nvhe/page_alloc.c
[...]
> +static void __hyp_attach_page(struct hyp_pool *pool,
> + struct hyp_page *p)
> +{
> + unsigned int order = p->order;
> + struct hyp_page *buddy;
> +
> + memset(hyp_page_to_virt(p), 0, PAGE_SIZE << p->order);
> +
> + /*
> + * Only the first struct hyp_page of a high-order page (otherwise known
> + * as the 'head') should have p->order set. The non-head pages should
> + * have p->order = HYP_NO_ORDER. Here @p may no longer be the head
> + * after coallescing, so make sure to mark it HYP_NO_ORDER proactively.
> + */
> + p->order = HYP_NO_ORDER;
> + for (; (order + 1) < pool->max_order; order++) {
> + buddy = __find_buddy_avail(pool, p, order);
> + if (!buddy)
> + break;
> +
> + /* Take the buddy out of its list, and coallesce with @p */
> + list_del_init(&buddy->node);
> + buddy->order = HYP_NO_ORDER;
> + p = (p < buddy) ? p : buddy;
nit: this is min()
> + }
> +
> + /* Mark the new head, and insert it */
> + p->order = order;
> + list_add_tail(&p->node, &pool->free_area[order]);
> +}
> +
> +static void hyp_attach_page(struct hyp_page *p)
> +{
> + struct hyp_pool *pool = hyp_page_to_pool(p);
> +
> + hyp_spin_lock(&pool->lock);
> + __hyp_attach_page(pool, p);
> + hyp_spin_unlock(&pool->lock);
> +}
> +
> +static struct hyp_page *__hyp_extract_page(struct hyp_pool *pool,
> +struct hyp_page *p,
> +unsigned int order)
> +{
> + struct hyp_page *buddy;
> +
> + list_del_init(&p->node);
> + while (p->order > order) {
> + /*
> + * The buddy of order n - 1 currently has HYP_NO_ORDER as it
> + * is covered by a higher-level page (whose head is @p). Use
> + * __find_buddy_nocheck() to find it and inject it in the
> + * free_list[n - 1], effectively splitting @p in half.
> + */
> + p->order--;
> + buddy = __find_buddy_nocheck(pool, p, p->order);
> + buddy->order = p->order;
> + list_add_tail(&buddy->node, &pool->free_area[buddy->order]);
> + }
> +
> + return p;
> +}
> +
> +void hyp_put_page(void *addr)
> +{
> + struct hyp_page *p = hyp_virt_to_page(addr);
> +
> + if (hyp_page_ref_dec_and_test(p))
> + hyp_attach_page(p);
> +}
> +
> +void hyp_get_page(void *addr)
> +{
> + struct hyp_page *p = hyp_virt_to_page(addr);
> +
> + hyp_page_ref_inc(p);
> +}
> +
> +void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order)
> +{
> + unsigned int i = order;
> + struct hyp_page *p;
> +
> + hyp_spin_lock(&pool->lock);
> +
> + /* Look for a high-enough-order page */
> + while (i < pool->max_order && list_empty(&pool->free_area[i]))
> + i++;
> + if (i >= pool->max_order) {
> + hyp_spin_unlock(&pool->lock);
> + return NULL;
> + }
> +
> + /* Extract it from the tree at the right order */
> + p = list_first_entry(&pool->free_area[i], struct hyp_page, node);
> + p = __hyp_extract_page(pool, p, order);
> +
> + hyp_spin_unlock(&pool->lock);
> + hyp_page_ref_inc(p);
I find this a little scary, as we momentarily drop the lock. It think
it's ok because the reference count on the page must be 0 at this point,
but actually then I think it would be clearer to have a
hyp_page
[PATCH v3 12/32] KVM: arm64: Introduce a Hyp buddy page allocator
When memory protection is enabled, the hyp code will require a basic
form of memory management in order to allocate and free memory pages at
EL2. This is needed for various use-cases, including the creation of hyp
mappings or the allocation of stage 2 page tables.
To address these use-case, introduce a simple memory allocator in the
hyp code. The allocator is designed as a conventional 'buddy allocator',
working with a page granularity. It allows to allocate and free
physically contiguous pages from memory 'pools', with a guaranteed order
alignment in the PA space. Each page in a memory pool is associated
with a struct hyp_page which holds the page's metadata, including its
refcount, as well as its current order, hence mimicking the kernel's
buddy system in the GFP infrastructure. The hyp_page metadata are made
accessible through a hyp_vmemmap, following the concept of
SPARSE_VMEMMAP in the kernel.
Signed-off-by: Quentin Perret
---
arch/arm64/kvm/hyp/include/nvhe/gfp.h| 55 +++
arch/arm64/kvm/hyp/include/nvhe/memory.h | 28
arch/arm64/kvm/hyp/nvhe/Makefile | 2 +-
arch/arm64/kvm/hyp/nvhe/page_alloc.c | 195 +++
4 files changed, 279 insertions(+), 1 deletion(-)
create mode 100644 arch/arm64/kvm/hyp/include/nvhe/gfp.h
create mode 100644 arch/arm64/kvm/hyp/nvhe/page_alloc.c
diff --git a/arch/arm64/kvm/hyp/include/nvhe/gfp.h
b/arch/arm64/kvm/hyp/include/nvhe/gfp.h
new file mode 100644
index ..d039086d86b5
--- /dev/null
+++ b/arch/arm64/kvm/hyp/include/nvhe/gfp.h
@@ -0,0 +1,55 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __KVM_HYP_GFP_H
+#define __KVM_HYP_GFP_H
+
+#include
+
+#include
+#include
+
+#define HYP_NO_ORDER UINT_MAX
+
+struct hyp_pool {
+ /*
+* Spinlock protecting concurrent changes to the memory pool as well as
+* the struct hyp_page of the pool's pages until we have a proper atomic
+* API at EL2.
+*/
+ hyp_spinlock_t lock;
+ struct list_head free_area[MAX_ORDER];
+ phys_addr_t range_start;
+ phys_addr_t range_end;
+ unsigned int max_order;
+};
+
+static inline void hyp_page_ref_inc(struct hyp_page *p)
+{
+ struct hyp_pool *pool = hyp_page_to_pool(p);
+
+ hyp_spin_lock(&pool->lock);
+ p->refcount++;
+ hyp_spin_unlock(&pool->lock);
+}
+
+static inline int hyp_page_ref_dec_and_test(struct hyp_page *p)
+{
+ struct hyp_pool *pool = hyp_page_to_pool(p);
+ int ret;
+
+ hyp_spin_lock(&pool->lock);
+ p->refcount--;
+ ret = (p->refcount == 0);
+ hyp_spin_unlock(&pool->lock);
+
+ return ret;
+}
+
+/* Allocation */
+void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order);
+void hyp_get_page(void *addr);
+void hyp_put_page(void *addr);
+
+/* Used pages cannot be freed */
+int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages,
+ unsigned int reserved_pages);
+#endif /* __KVM_HYP_GFP_H */
diff --git a/arch/arm64/kvm/hyp/include/nvhe/memory.h
b/arch/arm64/kvm/hyp/include/nvhe/memory.h
index 3e49eaa7e682..d2fb307c5952 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/memory.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/memory.h
@@ -6,7 +6,17 @@
#include
+struct hyp_pool;
+struct hyp_page {
+ unsigned int refcount;
+ unsigned int order;
+ struct hyp_pool *pool;
+ struct list_head node;
+};
+
extern s64 hyp_physvirt_offset;
+extern u64 __hyp_vmemmap;
+#define hyp_vmemmap ((struct hyp_page *)__hyp_vmemmap)
#define __hyp_pa(virt) ((phys_addr_t)(virt) + hyp_physvirt_offset)
#define __hyp_va(phys) ((void *)((phys_addr_t)(phys) - hyp_physvirt_offset))
@@ -21,4 +31,22 @@ static inline phys_addr_t hyp_virt_to_phys(void *addr)
return __hyp_pa(addr);
}
+#define hyp_phys_to_pfn(phys) ((phys) >> PAGE_SHIFT)
+#define hyp_pfn_to_phys(pfn) ((phys_addr_t)((pfn) << PAGE_SHIFT))
+#define hyp_phys_to_page(phys) (&hyp_vmemmap[hyp_phys_to_pfn(phys)])
+#define hyp_virt_to_page(virt) hyp_phys_to_page(__hyp_pa(virt))
+#define hyp_virt_to_pfn(virt) hyp_phys_to_pfn(__hyp_pa(virt))
+
+#define hyp_page_to_pfn(page) ((struct hyp_page *)(page) - hyp_vmemmap)
+#define hyp_page_to_phys(page) hyp_pfn_to_phys((hyp_page_to_pfn(page)))
+#define hyp_page_to_virt(page) __hyp_va(hyp_page_to_phys(page))
+#define hyp_page_to_pool(page) (((struct hyp_page *)page)->pool)
+
+static inline int hyp_page_count(void *addr)
+{
+ struct hyp_page *p = hyp_virt_to_page(addr);
+
+ return p->refcount;
+}
+
#endif /* __KVM_HYP_MEMORY_H */
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index 144da72ad510..6894a917f290 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -13,7 +13,7 @@ lib-objs := clear_page.o copy_page.o memcpy.o memset.o
lib-objs := $(addprefix ../../../lib/, $(lib-objs))
obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \
-hyp-main.o h
