Currently, we store class:fullness into page->mapping.
The number of class we can support is 255 and fullness is 4 so
(8 + 2 = 10bit) is enough to represent them.
Meanwhile, the bits we need to store in-use objects in zspage
is that 11bit is enough.
For example, If we assume that 64K PAGE_SIZE, class_size 32
which is worst case, class->pages_per_zspage become 1 so
the number of objects in zspage is 2048 so 11bit is enough.
The next class is 32 + 256(i.e., ZS_SIZE_CLASS_DELTA).
With worst case that ZS_MAX_PAGES_PER_ZSPAGE, 64K * 4 /
(32 + 256) = 910 so 11bit is still enough.
So, we could squeeze inuse object count to page->mapping.
Signed-off-by: Minchan Kim
---
mm/zsmalloc.c | 103 --
1 file changed, 71 insertions(+), 32 deletions(-)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 8649d0243e6c..4dd72a803568 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -34,8 +34,7 @@
* metadata.
* page->lru: links together first pages of various zspages.
* Basically forming list of zspages in a fullness group.
- * page->mapping: class index and fullness group of the zspage
- * page->inuse: the number of objects that are used in this zspage
+ * page->mapping: override by struct zs_meta
*
* Usage of struct page flags:
* PG_private: identifies the first component page
@@ -132,6 +131,13 @@
/* each chunk includes extra space to keep handle */
#define ZS_MAX_ALLOC_SIZE PAGE_SIZE
+#define CLASS_BITS 8
+#define CLASS_MASK ((1 << CLASS_BITS) - 1)
+#define FULLNESS_BITS 2
+#define FULLNESS_MASK ((1 << FULLNESS_BITS) - 1)
+#define INUSE_BITS 11
+#define INUSE_MASK ((1 << INUSE_BITS) - 1)
+
/*
* On systems with 4K page size, this gives 255 size classes! There is a
* trader-off here:
@@ -145,7 +151,7 @@
* ZS_MIN_ALLOC_SIZE and ZS_SIZE_CLASS_DELTA must be multiple of ZS_ALIGN
* (reason above)
*/
-#define ZS_SIZE_CLASS_DELTA(PAGE_SIZE >> 8)
+#define ZS_SIZE_CLASS_DELTA(PAGE_SIZE >> CLASS_BITS)
/*
* We do not maintain any list for completely empty or full pages
@@ -155,7 +161,7 @@ enum fullness_group {
ZS_ALMOST_EMPTY,
_ZS_NR_FULLNESS_GROUPS,
- ZS_EMPTY,
+ ZS_EMPTY = _ZS_NR_FULLNESS_GROUPS,
ZS_FULL
};
@@ -263,14 +269,11 @@ struct zs_pool {
#endif
};
-/*
- * A zspage's class index and fullness group
- * are encoded in its (first)page->mapping
- */
-#define CLASS_IDX_BITS 28
-#define FULLNESS_BITS 4
-#define CLASS_IDX_MASK ((1 << CLASS_IDX_BITS) - 1)
-#define FULLNESS_MASK ((1 << FULLNESS_BITS) - 1)
+struct zs_meta {
+ unsigned long class:CLASS_BITS;
+ unsigned long fullness:FULLNESS_BITS;
+ unsigned long inuse:INUSE_BITS;
+};
struct mapping_area {
#ifdef CONFIG_PGTABLE_MAPPING
@@ -412,28 +415,61 @@ static int is_last_page(struct page *page)
return PagePrivate2(page);
}
+static int get_zspage_inuse(struct page *first_page)
+{
+ struct zs_meta *m;
+
+ VM_BUG_ON_PAGE(!is_first_page(first_page), first_page);
+
+ m = (struct zs_meta *)&first_page->mapping;
+
+ return m->inuse;
+}
+
+static void set_zspage_inuse(struct page *first_page, int val)
+{
+ struct zs_meta *m;
+
+ VM_BUG_ON_PAGE(!is_first_page(first_page), first_page);
+
+ m = (struct zs_meta *)&first_page->mapping;
+ m->inuse = val;
+}
+
+static void mod_zspage_inuse(struct page *first_page, int val)
+{
+ struct zs_meta *m;
+
+ VM_BUG_ON_PAGE(!is_first_page(first_page), first_page);
+
+ m = (struct zs_meta *)&first_page->mapping;
+ m->inuse += val;
+}
+
static void get_zspage_mapping(struct page *first_page,
unsigned int *class_idx,
enum fullness_group *fullness)
{
- unsigned long m;
+ struct zs_meta *m;
+
VM_BUG_ON_PAGE(!is_first_page(first_page), first_page);
- m = (unsigned long)first_page->mapping;
- *fullness = m & FULLNESS_MASK;
- *class_idx = (m >> FULLNESS_BITS) & CLASS_IDX_MASK;
+ m = (struct zs_meta *)&first_page->mapping;
+ *fullness = m->fullness;
+ *class_idx = m->class;
}
static void set_zspage_mapping(struct page *first_page,
unsigned int class_idx,
enum fullness_group fullness)
{
- unsigned long m;
+ struct zs_meta *m;
+
VM_BUG_ON_PAGE(!is_first_page(first_page), first_page);
- m = ((class_idx & CLASS_IDX_MASK) << FULLNESS_BITS) |
- (fullness & FULLNESS_MASK);
- first_page->mapping = (struct address_space *)m;
+ m = (struct zs_meta *)&first_page->mapping;
+ m->fullness = fullness;
+ m->class = class_idx;
}
/*
@@ -632,9 +668,7 @@ static enum fullness_group get_fullness_group(struct
size_class *class,
int inuse, objs_per_zspage;
enum fullness_group fg;
- VM_BU
