[PATCH v2 5/5] mm: migrate zbud pages
Add migration support for zbud. This allows adding __GFP_MOVABLE flag
when allocating zbud pages and effectively CMA pool can be used for
zswap.
zbud pages are not movable and are not stored under any LRU (except
zbud's LRU). PageZbud flag is used in isolate_migratepages_range() to
grab zbud pages and pass them later for migration.
page->private field is used for storing pointer to zbud_pool.
This pointer to zbud_pool is needed during migration for locking the
pool and accessing radix tree.
The zbud migration code utilizes mapping so many exceptions to migrate
code was added. It can be replaced for example with pin page control
subsystem:
http://article.gmane.org/gmane.linux.kernel.mm/105308
In such case the zbud migration code (zbud_migrate_page()) can be safely
re-used.
Signed-off-by: Krzysztof Kozlowski
---
include/linux/zbud.h |1 +
mm/compaction.c |7 +++
mm/migrate.c | 17 +-
mm/zbud.c| 164 +++---
mm/zswap.c |4 +-
5 files changed, 179 insertions(+), 14 deletions(-)
diff --git a/include/linux/zbud.h b/include/linux/zbud.h
index 12d72df..3bc2e38 100644
--- a/include/linux/zbud.h
+++ b/include/linux/zbud.h
@@ -11,6 +11,7 @@ struct zbud_ops {
struct zbud_pool *zbud_create_pool(gfp_t gfp, struct zbud_ops *ops);
void zbud_destroy_pool(struct zbud_pool *pool);
+int zbud_put_page(struct page *page);
int zbud_alloc(struct zbud_pool *pool, int size, gfp_t gfp,
unsigned long *handle);
void zbud_free(struct zbud_pool *pool, unsigned long handle);
diff --git a/mm/compaction.c b/mm/compaction.c
index 05ccb4c..8acd198 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -534,6 +534,12 @@ isolate_migratepages_range(struct zone *zone, struct
compact_control *cc,
goto next_pageblock;
}
+ if (PageZbud(page)) {
+ BUG_ON(PageLRU(page));
+ get_page(page);
+ goto isolated;
+ }
+
/*
* Check may be lockless but that's ok as we recheck later.
* It's possible to migrate LRU pages and balloon pages
@@ -601,6 +607,7 @@ isolate_migratepages_range(struct zone *zone, struct
compact_control *cc,
/* Successfully isolated */
cc->finished_update_migrate = true;
del_page_from_lru_list(page, lruvec, page_lru(page));
+isolated:
list_add(>lru, migratelist);
cc->nr_migratepages++;
nr_isolated++;
diff --git a/mm/migrate.c b/mm/migrate.c
index 6f0c244..5254eb2 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -36,6 +36,7 @@
#include
#include
#include
+#include
#include
@@ -105,6 +106,8 @@ void putback_movable_pages(struct list_head *l)
page_is_file_cache(page));
if (unlikely(balloon_page_movable(page)))
balloon_page_putback(page);
+ else if (unlikely(PageZbud(page)))
+ zbud_put_page(page);
else
putback_lru_page(page);
}
@@ -832,6 +835,10 @@ static int __unmap_and_move(struct page *page, struct page
*newpage,
goto skip_unmap;
}
+ if (unlikely(PageZbud(page))) {
+ remap_swapcache = 0;
+ goto skip_unmap;
+ }
/* Establish migration ptes or remove ptes */
try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
@@ -902,13 +909,19 @@ out:
list_del(>lru);
dec_zone_page_state(page, NR_ISOLATED_ANON +
page_is_file_cache(page));
- putback_lru_page(page);
+ if (unlikely(PageZbud(page)))
+ zbud_put_page(page);
+ else
+ putback_lru_page(page);
}
/*
* Move the new page to the LRU. If migration was not successful
* then this will free the page.
*/
- putback_lru_page(newpage);
+ if (unlikely(PageZbud(newpage)))
+ zbud_put_page(newpage);
+ else
+ putback_lru_page(newpage);
if (result) {
if (rc)
*result = rc;
diff --git a/mm/zbud.c b/mm/zbud.c
index 795d56a..de8132d 100644
--- a/mm/zbud.c
+++ b/mm/zbud.c
@@ -51,6 +51,8 @@
#include
#include
#include
+#include
+#include
#include
/*
@@ -211,17 +213,9 @@ static void get_zbud_page(struct zbud_header *zhdr)
*
* Returns 1 if page was freed and 0 otherwise.
*/
-static int put_zbud_page(struct zbud_header *zhdr)
+static inline int put_zbud_page(struct zbud_header *zhdr)
{
- struct page *page = virt_to_page(zhdr);
- VM_BUG_ON(!PageZbud(page));
-
- if (put_page_testzero(page)) {
- ClearPageZbud(page);
-
[PATCH v2 5/5] mm: migrate zbud pages
Add migration support for zbud. This allows adding __GFP_MOVABLE flag
when allocating zbud pages and effectively CMA pool can be used for
zswap.
zbud pages are not movable and are not stored under any LRU (except
zbud's LRU). PageZbud flag is used in isolate_migratepages_range() to
grab zbud pages and pass them later for migration.
page-private field is used for storing pointer to zbud_pool.
This pointer to zbud_pool is needed during migration for locking the
pool and accessing radix tree.
The zbud migration code utilizes mapping so many exceptions to migrate
code was added. It can be replaced for example with pin page control
subsystem:
http://article.gmane.org/gmane.linux.kernel.mm/105308
In such case the zbud migration code (zbud_migrate_page()) can be safely
re-used.
Signed-off-by: Krzysztof Kozlowski k.kozlow...@samsung.com
---
include/linux/zbud.h |1 +
mm/compaction.c |7 +++
mm/migrate.c | 17 +-
mm/zbud.c| 164 +++---
mm/zswap.c |4 +-
5 files changed, 179 insertions(+), 14 deletions(-)
diff --git a/include/linux/zbud.h b/include/linux/zbud.h
index 12d72df..3bc2e38 100644
--- a/include/linux/zbud.h
+++ b/include/linux/zbud.h
@@ -11,6 +11,7 @@ struct zbud_ops {
struct zbud_pool *zbud_create_pool(gfp_t gfp, struct zbud_ops *ops);
void zbud_destroy_pool(struct zbud_pool *pool);
+int zbud_put_page(struct page *page);
int zbud_alloc(struct zbud_pool *pool, int size, gfp_t gfp,
unsigned long *handle);
void zbud_free(struct zbud_pool *pool, unsigned long handle);
diff --git a/mm/compaction.c b/mm/compaction.c
index 05ccb4c..8acd198 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -534,6 +534,12 @@ isolate_migratepages_range(struct zone *zone, struct
compact_control *cc,
goto next_pageblock;
}
+ if (PageZbud(page)) {
+ BUG_ON(PageLRU(page));
+ get_page(page);
+ goto isolated;
+ }
+
/*
* Check may be lockless but that's ok as we recheck later.
* It's possible to migrate LRU pages and balloon pages
@@ -601,6 +607,7 @@ isolate_migratepages_range(struct zone *zone, struct
compact_control *cc,
/* Successfully isolated */
cc-finished_update_migrate = true;
del_page_from_lru_list(page, lruvec, page_lru(page));
+isolated:
list_add(page-lru, migratelist);
cc-nr_migratepages++;
nr_isolated++;
diff --git a/mm/migrate.c b/mm/migrate.c
index 6f0c244..5254eb2 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -36,6 +36,7 @@
#include linux/hugetlb_cgroup.h
#include linux/gfp.h
#include linux/balloon_compaction.h
+#include linux/zbud.h
#include asm/tlbflush.h
@@ -105,6 +106,8 @@ void putback_movable_pages(struct list_head *l)
page_is_file_cache(page));
if (unlikely(balloon_page_movable(page)))
balloon_page_putback(page);
+ else if (unlikely(PageZbud(page)))
+ zbud_put_page(page);
else
putback_lru_page(page);
}
@@ -832,6 +835,10 @@ static int __unmap_and_move(struct page *page, struct page
*newpage,
goto skip_unmap;
}
+ if (unlikely(PageZbud(page))) {
+ remap_swapcache = 0;
+ goto skip_unmap;
+ }
/* Establish migration ptes or remove ptes */
try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
@@ -902,13 +909,19 @@ out:
list_del(page-lru);
dec_zone_page_state(page, NR_ISOLATED_ANON +
page_is_file_cache(page));
- putback_lru_page(page);
+ if (unlikely(PageZbud(page)))
+ zbud_put_page(page);
+ else
+ putback_lru_page(page);
}
/*
* Move the new page to the LRU. If migration was not successful
* then this will free the page.
*/
- putback_lru_page(newpage);
+ if (unlikely(PageZbud(newpage)))
+ zbud_put_page(newpage);
+ else
+ putback_lru_page(newpage);
if (result) {
if (rc)
*result = rc;
diff --git a/mm/zbud.c b/mm/zbud.c
index 795d56a..de8132d 100644
--- a/mm/zbud.c
+++ b/mm/zbud.c
@@ -51,6 +51,8 @@
#include linux/slab.h
#include linux/spinlock.h
#include linux/radix-tree.h
+#include linux/fs.h
+#include linux/pagemap.h
#include linux/zbud.h
/*
@@ -211,17 +213,9 @@ static void get_zbud_page(struct zbud_header *zhdr)
*
* Returns 1 if page was freed and 0 otherwise.
*/
-static int put_zbud_page(struct zbud_header *zhdr)
+static inline int put_zbud_page(struct
