Re: [PATCHv3 3/3] z3fold: add compaction worker

[Dan Streetman]

On Thu, Oct 27, 2016 at 7:13 AM, Vitaly Wool  wrote:
> This patch implements compaction worker thread for z3fold. This
> worker does not free up any pages directly but it allows for a
> denser placement of compressed objects which results in less
> actual pages consumed and higher compression ratio therefore.
>
> This patch has been checked with the latest Linus's tree.
>
> Signed-off-by: Vitaly Wool 
> ---
>  mm/z3fold.c | 166 
> ++--
>  1 file changed, 140 insertions(+), 26 deletions(-)
>
> diff --git a/mm/z3fold.c b/mm/z3fold.c
> index 014d84f..cc26ff5 100644
> --- a/mm/z3fold.c
> +++ b/mm/z3fold.c
> @@ -27,6 +27,7 @@
>  #include 
>  #include 
>  #include 
> +#include 
>  #include 
>  #include 
>  #include 
> @@ -59,6 +60,7 @@ struct z3fold_ops {
>
>  /**
>   * struct z3fold_pool - stores metadata for each z3fold pool
> + * @name:  pool name
>   * @lock:  protects all pool fields and first|last_chunk fields of any
>   * z3fold page in the pool
>   * @unbuddied: array of lists tracking z3fold pages that contain 2- buddies;
> @@ -72,11 +74,14 @@ struct z3fold_ops {
>   * @unbuddied_nr:  number of unbuddied z3fold pages in the pool.
>   * @ops:   pointer to a structure of user defined operations specified at
>   * pool creation time.
> + * @compact_wq:workqueue for page layout background optimization
> + * @work:  compaction work item
>   *
>   * This structure is allocated at pool creation time and maintains metadata
>   * pertaining to a particular z3fold pool.
>   */
>  struct z3fold_pool {
> +   const char *name;
> rwlock_t lock;
> struct list_head unbuddied[NCHUNKS];
> struct list_head buddied;
> @@ -86,6 +91,8 @@ struct z3fold_pool {
> const struct z3fold_ops *ops;
> struct zpool *zpool;
> const struct zpool_ops *zpool_ops;
> +   struct workqueue_struct *compact_wq;
> +   struct delayed_work work;
>  };
>
>  enum buddy {
> @@ -121,6 +128,7 @@ enum z3fold_page_flags {
> UNDER_RECLAIM = 0,
> PAGE_HEADLESS,
> MIDDLE_CHUNK_MAPPED,
> +   COMPACTION_DEFERRED,
>  };
>
>  /*
> @@ -136,6 +144,9 @@ static int size_to_chunks(size_t size)
>  #define for_each_unbuddied_list(_iter, _begin) \
> for ((_iter) = (_begin); (_iter) < NCHUNKS; (_iter)++)
>
> +#define for_each_unbuddied_list_reverse(_iter, _end) \
> +   for ((_iter) = (_end); (_iter) > 0; (_iter)--)
> +
>  /* Initializes the z3fold header of a newly allocated z3fold page */
>  static struct z3fold_header *init_z3fold_page(struct page *page)
>  {
> @@ -145,6 +156,7 @@ static struct z3fold_header *init_z3fold_page(struct page 
> *page)
> clear_bit(UNDER_RECLAIM, >private);
> clear_bit(PAGE_HEADLESS, >private);
> clear_bit(MIDDLE_CHUNK_MAPPED, >private);
> +   clear_bit(COMPACTION_DEFERRED, >private);
>
> zhdr->first_chunks = 0;
> zhdr->middle_chunks = 0;
> @@ -211,6 +223,116 @@ static int num_free_chunks(struct z3fold_header *zhdr)
> return nfree;
>  }
>
> +static inline void *mchunk_memmove(struct z3fold_header *zhdr,
> +   unsigned short dst_chunk)
> +{
> +   void *beg = zhdr;
> +   return memmove(beg + (dst_chunk << CHUNK_SHIFT),
> +  beg + (zhdr->start_middle << CHUNK_SHIFT),
> +  zhdr->middle_chunks << CHUNK_SHIFT);
> +}
> +
> +static int z3fold_compact_page(struct z3fold_header *zhdr, bool sync)
> +{
> +   struct page *page = virt_to_page(zhdr);
> +   int ret = 0;
> +
> +   if (test_bit(MIDDLE_CHUNK_MAPPED, >private)) {
> +   set_bit(COMPACTION_DEFERRED, >private);
> +   ret = -1;
> +   goto out;
> +   }
> +
> +   clear_bit(COMPACTION_DEFERRED, >private);
> +   if (zhdr->middle_chunks != 0) {
> +   if (zhdr->first_chunks == 0 && zhdr->last_chunks == 0) {
> +   /*
> +* If we are here, no one can access this page
> +* except for z3fold_map or z3fold_free. Both
> +* will wait for page_lock to become free.
> +*/
> +   mchunk_memmove(zhdr, 1); /* move to the beginning */
> +   zhdr->first_chunks = zhdr->middle_chunks;
> +   zhdr->middle_chunks = 0;
> +   zhdr->start_middle = 0;
> +   zhdr->first_num++;
> +   ret = 1;
> +   goto out;
> +   }
> +   if (sync)
> +   goto out;
> +
> +   /*
> +* These are more complicated cases: first or last object
> +* may be mapped. Luckily we don't touch these anyway.
> +*
> +* NB: moving data is 

Re: [PATCHv3 3/3] z3fold: add compaction worker

[Dan Streetman]

On Thu, Oct 27, 2016 at 7:13 AM, Vitaly Wool  wrote:
> This patch implements compaction worker thread for z3fold. This
> worker does not free up any pages directly but it allows for a
> denser placement of compressed objects which results in less
> actual pages consumed and higher compression ratio therefore.
>
> This patch has been checked with the latest Linus's tree.
>
> Signed-off-by: Vitaly Wool 
> ---
>  mm/z3fold.c | 166 
> ++--
>  1 file changed, 140 insertions(+), 26 deletions(-)
>
> diff --git a/mm/z3fold.c b/mm/z3fold.c
> index 014d84f..cc26ff5 100644
> --- a/mm/z3fold.c
> +++ b/mm/z3fold.c
> @@ -27,6 +27,7 @@
>  #include 
>  #include 
>  #include 
> +#include 
>  #include 
>  #include 
>  #include 
> @@ -59,6 +60,7 @@ struct z3fold_ops {
>
>  /**
>   * struct z3fold_pool - stores metadata for each z3fold pool
> + * @name:  pool name
>   * @lock:  protects all pool fields and first|last_chunk fields of any
>   * z3fold page in the pool
>   * @unbuddied: array of lists tracking z3fold pages that contain 2- buddies;
> @@ -72,11 +74,14 @@ struct z3fold_ops {
>   * @unbuddied_nr:  number of unbuddied z3fold pages in the pool.
>   * @ops:   pointer to a structure of user defined operations specified at
>   * pool creation time.
> + * @compact_wq:workqueue for page layout background optimization
> + * @work:  compaction work item
>   *
>   * This structure is allocated at pool creation time and maintains metadata
>   * pertaining to a particular z3fold pool.
>   */
>  struct z3fold_pool {
> +   const char *name;
> rwlock_t lock;
> struct list_head unbuddied[NCHUNKS];
> struct list_head buddied;
> @@ -86,6 +91,8 @@ struct z3fold_pool {
> const struct z3fold_ops *ops;
> struct zpool *zpool;
> const struct zpool_ops *zpool_ops;
> +   struct workqueue_struct *compact_wq;
> +   struct delayed_work work;
>  };
>
>  enum buddy {
> @@ -121,6 +128,7 @@ enum z3fold_page_flags {
> UNDER_RECLAIM = 0,
> PAGE_HEADLESS,
> MIDDLE_CHUNK_MAPPED,
> +   COMPACTION_DEFERRED,
>  };
>
>  /*
> @@ -136,6 +144,9 @@ static int size_to_chunks(size_t size)
>  #define for_each_unbuddied_list(_iter, _begin) \
> for ((_iter) = (_begin); (_iter) < NCHUNKS; (_iter)++)
>
> +#define for_each_unbuddied_list_reverse(_iter, _end) \
> +   for ((_iter) = (_end); (_iter) > 0; (_iter)--)
> +
>  /* Initializes the z3fold header of a newly allocated z3fold page */
>  static struct z3fold_header *init_z3fold_page(struct page *page)
>  {
> @@ -145,6 +156,7 @@ static struct z3fold_header *init_z3fold_page(struct page 
> *page)
> clear_bit(UNDER_RECLAIM, >private);
> clear_bit(PAGE_HEADLESS, >private);
> clear_bit(MIDDLE_CHUNK_MAPPED, >private);
> +   clear_bit(COMPACTION_DEFERRED, >private);
>
> zhdr->first_chunks = 0;
> zhdr->middle_chunks = 0;
> @@ -211,6 +223,116 @@ static int num_free_chunks(struct z3fold_header *zhdr)
> return nfree;
>  }
>
> +static inline void *mchunk_memmove(struct z3fold_header *zhdr,
> +   unsigned short dst_chunk)
> +{
> +   void *beg = zhdr;
> +   return memmove(beg + (dst_chunk << CHUNK_SHIFT),
> +  beg + (zhdr->start_middle << CHUNK_SHIFT),
> +  zhdr->middle_chunks << CHUNK_SHIFT);
> +}
> +
> +static int z3fold_compact_page(struct z3fold_header *zhdr, bool sync)
> +{
> +   struct page *page = virt_to_page(zhdr);
> +   int ret = 0;
> +
> +   if (test_bit(MIDDLE_CHUNK_MAPPED, >private)) {
> +   set_bit(COMPACTION_DEFERRED, >private);
> +   ret = -1;
> +   goto out;
> +   }
> +
> +   clear_bit(COMPACTION_DEFERRED, >private);
> +   if (zhdr->middle_chunks != 0) {
> +   if (zhdr->first_chunks == 0 && zhdr->last_chunks == 0) {
> +   /*
> +* If we are here, no one can access this page
> +* except for z3fold_map or z3fold_free. Both
> +* will wait for page_lock to become free.
> +*/
> +   mchunk_memmove(zhdr, 1); /* move to the beginning */
> +   zhdr->first_chunks = zhdr->middle_chunks;
> +   zhdr->middle_chunks = 0;
> +   zhdr->start_middle = 0;
> +   zhdr->first_num++;
> +   ret = 1;
> +   goto out;
> +   }
> +   if (sync)
> +   goto out;
> +
> +   /*
> +* These are more complicated cases: first or last object
> +* may be mapped. Luckily we don't touch these anyway.
> +*
> +* NB: moving data is expensive, so let's only do that if
> + 

[PATCHv3 3/3] z3fold: add compaction worker

[Vitaly Wool]

This patch implements compaction worker thread for z3fold. This
worker does not free up any pages directly but it allows for a
denser placement of compressed objects which results in less
actual pages consumed and higher compression ratio therefore.

This patch has been checked with the latest Linus's tree.

Signed-off-by: Vitaly Wool 
---
 mm/z3fold.c | 166 ++--
 1 file changed, 140 insertions(+), 26 deletions(-)

diff --git a/mm/z3fold.c b/mm/z3fold.c
index 014d84f..cc26ff5 100644
--- a/mm/z3fold.c
+++ b/mm/z3fold.c
@@ -27,6 +27,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -59,6 +60,7 @@ struct z3fold_ops {
 
 /**
  * struct z3fold_pool - stores metadata for each z3fold pool
+ * @name:  pool name
  * @lock:  protects all pool fields and first|last_chunk fields of any
  * z3fold page in the pool
  * @unbuddied: array of lists tracking z3fold pages that contain 2- buddies;
@@ -72,11 +74,14 @@ struct z3fold_ops {
  * @unbuddied_nr:  number of unbuddied z3fold pages in the pool.
  * @ops:   pointer to a structure of user defined operations specified at
  * pool creation time.
+ * @compact_wq:workqueue for page layout background optimization
+ * @work:  compaction work item
  *
  * This structure is allocated at pool creation time and maintains metadata
  * pertaining to a particular z3fold pool.
  */
 struct z3fold_pool {
+   const char *name;
rwlock_t lock;
struct list_head unbuddied[NCHUNKS];
struct list_head buddied;
@@ -86,6 +91,8 @@ struct z3fold_pool {
const struct z3fold_ops *ops;
struct zpool *zpool;
const struct zpool_ops *zpool_ops;
+   struct workqueue_struct *compact_wq;
+   struct delayed_work work;
 };
 
 enum buddy {
@@ -121,6 +128,7 @@ enum z3fold_page_flags {
UNDER_RECLAIM = 0,
PAGE_HEADLESS,
MIDDLE_CHUNK_MAPPED,
+   COMPACTION_DEFERRED,
 };
 
 /*
@@ -136,6 +144,9 @@ static int size_to_chunks(size_t size)
 #define for_each_unbuddied_list(_iter, _begin) \
for ((_iter) = (_begin); (_iter) < NCHUNKS; (_iter)++)
 
+#define for_each_unbuddied_list_reverse(_iter, _end) \
+   for ((_iter) = (_end); (_iter) > 0; (_iter)--)
+
 /* Initializes the z3fold header of a newly allocated z3fold page */
 static struct z3fold_header *init_z3fold_page(struct page *page)
 {
@@ -145,6 +156,7 @@ static struct z3fold_header *init_z3fold_page(struct page 
*page)
clear_bit(UNDER_RECLAIM, >private);
clear_bit(PAGE_HEADLESS, >private);
clear_bit(MIDDLE_CHUNK_MAPPED, >private);
+   clear_bit(COMPACTION_DEFERRED, >private);
 
zhdr->first_chunks = 0;
zhdr->middle_chunks = 0;
@@ -211,6 +223,116 @@ static int num_free_chunks(struct z3fold_header *zhdr)
return nfree;
 }
 
+static inline void *mchunk_memmove(struct z3fold_header *zhdr,
+   unsigned short dst_chunk)
+{
+   void *beg = zhdr;
+   return memmove(beg + (dst_chunk << CHUNK_SHIFT),
+  beg + (zhdr->start_middle << CHUNK_SHIFT),
+  zhdr->middle_chunks << CHUNK_SHIFT);
+}
+
+static int z3fold_compact_page(struct z3fold_header *zhdr, bool sync)
+{
+   struct page *page = virt_to_page(zhdr);
+   int ret = 0;
+
+   if (test_bit(MIDDLE_CHUNK_MAPPED, >private)) {
+   set_bit(COMPACTION_DEFERRED, >private);
+   ret = -1;
+   goto out;
+   }
+
+   clear_bit(COMPACTION_DEFERRED, >private);
+   if (zhdr->middle_chunks != 0) {
+   if (zhdr->first_chunks == 0 && zhdr->last_chunks == 0) {
+   /*
+* If we are here, no one can access this page
+* except for z3fold_map or z3fold_free. Both
+* will wait for page_lock to become free.
+*/
+   mchunk_memmove(zhdr, 1); /* move to the beginning */
+   zhdr->first_chunks = zhdr->middle_chunks;
+   zhdr->middle_chunks = 0;
+   zhdr->start_middle = 0;
+   zhdr->first_num++;
+   ret = 1;
+   goto out;
+   }
+   if (sync)
+   goto out;
+
+   /*
+* These are more complicated cases: first or last object
+* may be mapped. Luckily we don't touch these anyway.
+*
+* NB: moving data is expensive, so let's only do that if
+* there's substantial gain (2+ chunks)
+*/
+   if (zhdr->first_chunks != 0 && zhdr->last_chunks == 0 &&
+   zhdr->start_middle > zhdr->first_chunks + 2) {
+   mchunk_memmove(zhdr, zhdr->first_chunks + 1);
+  

[PATCHv3 3/3] z3fold: add compaction worker

[Vitaly Wool]

This patch implements compaction worker thread for z3fold. This
worker does not free up any pages directly but it allows for a
denser placement of compressed objects which results in less
actual pages consumed and higher compression ratio therefore.

This patch has been checked with the latest Linus's tree.

Signed-off-by: Vitaly Wool 
---
 mm/z3fold.c | 166 ++--
 1 file changed, 140 insertions(+), 26 deletions(-)

diff --git a/mm/z3fold.c b/mm/z3fold.c
index 014d84f..cc26ff5 100644
--- a/mm/z3fold.c
+++ b/mm/z3fold.c
@@ -27,6 +27,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -59,6 +60,7 @@ struct z3fold_ops {
 
 /**
  * struct z3fold_pool - stores metadata for each z3fold pool
+ * @name:  pool name
  * @lock:  protects all pool fields and first|last_chunk fields of any
  * z3fold page in the pool
  * @unbuddied: array of lists tracking z3fold pages that contain 2- buddies;
@@ -72,11 +74,14 @@ struct z3fold_ops {
  * @unbuddied_nr:  number of unbuddied z3fold pages in the pool.
  * @ops:   pointer to a structure of user defined operations specified at
  * pool creation time.
+ * @compact_wq:workqueue for page layout background optimization
+ * @work:  compaction work item
  *
  * This structure is allocated at pool creation time and maintains metadata
  * pertaining to a particular z3fold pool.
  */
 struct z3fold_pool {
+   const char *name;
rwlock_t lock;
struct list_head unbuddied[NCHUNKS];
struct list_head buddied;
@@ -86,6 +91,8 @@ struct z3fold_pool {
const struct z3fold_ops *ops;
struct zpool *zpool;
const struct zpool_ops *zpool_ops;
+   struct workqueue_struct *compact_wq;
+   struct delayed_work work;
 };
 
 enum buddy {
@@ -121,6 +128,7 @@ enum z3fold_page_flags {
UNDER_RECLAIM = 0,
PAGE_HEADLESS,
MIDDLE_CHUNK_MAPPED,
+   COMPACTION_DEFERRED,
 };
 
 /*
@@ -136,6 +144,9 @@ static int size_to_chunks(size_t size)
 #define for_each_unbuddied_list(_iter, _begin) \
for ((_iter) = (_begin); (_iter) < NCHUNKS; (_iter)++)
 
+#define for_each_unbuddied_list_reverse(_iter, _end) \
+   for ((_iter) = (_end); (_iter) > 0; (_iter)--)
+
 /* Initializes the z3fold header of a newly allocated z3fold page */
 static struct z3fold_header *init_z3fold_page(struct page *page)
 {
@@ -145,6 +156,7 @@ static struct z3fold_header *init_z3fold_page(struct page 
*page)
clear_bit(UNDER_RECLAIM, >private);
clear_bit(PAGE_HEADLESS, >private);
clear_bit(MIDDLE_CHUNK_MAPPED, >private);
+   clear_bit(COMPACTION_DEFERRED, >private);
 
zhdr->first_chunks = 0;
zhdr->middle_chunks = 0;
@@ -211,6 +223,116 @@ static int num_free_chunks(struct z3fold_header *zhdr)
return nfree;
 }
 
+static inline void *mchunk_memmove(struct z3fold_header *zhdr,
+   unsigned short dst_chunk)
+{
+   void *beg = zhdr;
+   return memmove(beg + (dst_chunk << CHUNK_SHIFT),
+  beg + (zhdr->start_middle << CHUNK_SHIFT),
+  zhdr->middle_chunks << CHUNK_SHIFT);
+}
+
+static int z3fold_compact_page(struct z3fold_header *zhdr, bool sync)
+{
+   struct page *page = virt_to_page(zhdr);
+   int ret = 0;
+
+   if (test_bit(MIDDLE_CHUNK_MAPPED, >private)) {
+   set_bit(COMPACTION_DEFERRED, >private);
+   ret = -1;
+   goto out;
+   }
+
+   clear_bit(COMPACTION_DEFERRED, >private);
+   if (zhdr->middle_chunks != 0) {
+   if (zhdr->first_chunks == 0 && zhdr->last_chunks == 0) {
+   /*
+* If we are here, no one can access this page
+* except for z3fold_map or z3fold_free. Both
+* will wait for page_lock to become free.
+*/
+   mchunk_memmove(zhdr, 1); /* move to the beginning */
+   zhdr->first_chunks = zhdr->middle_chunks;
+   zhdr->middle_chunks = 0;
+   zhdr->start_middle = 0;
+   zhdr->first_num++;
+   ret = 1;
+   goto out;
+   }
+   if (sync)
+   goto out;
+
+   /*
+* These are more complicated cases: first or last object
+* may be mapped. Luckily we don't touch these anyway.
+*
+* NB: moving data is expensive, so let's only do that if
+* there's substantial gain (2+ chunks)
+*/
+   if (zhdr->first_chunks != 0 && zhdr->last_chunks == 0 &&
+   zhdr->start_middle > zhdr->first_chunks + 2) {
+   mchunk_memmove(zhdr, zhdr->first_chunks + 1);
+