When a folio's refcount drops to zero, the service may need to perform
cleanup before the page returns to the buddy allocator (e.g. zeroing
pages to scrub stale compressed data / release compression ratio).
Add folio_managed_on_free() to wrap both zone_device and private node
semantics for this operation since they are the same.
One difference between zone_device and private node folios:
- private nodes may choose to either take a reference and return true
("handled"), or return false to return it back to the buddy.
- zone_device returns the page to the buddy (always returns true)
Signed-off-by: Gregory Price <[email protected]>
---
include/linux/node_private.h | 6 ++++++
mm/internal.h | 30 ++++++++++++++++++++++++++++++
mm/swap.c | 21 ++++++++++-----------
3 files changed, 46 insertions(+), 11 deletions(-)
diff --git a/include/linux/node_private.h b/include/linux/node_private.h
index 7687a4cf990c..09ea7c4cb13c 100644
--- a/include/linux/node_private.h
+++ b/include/linux/node_private.h
@@ -39,10 +39,16 @@ struct vm_fault;
* callback to prevent node_private from being freed.
* These callbacks MUST NOT sleep.
*
+ * @free_folio: Called when a folio refcount drops to 0
+ * [folio-referenced callback]
+ * Returns: true if handled (skip return to buddy)
+ * false if no op (return to buddy)
+ *
* @flags: Operation exclusion flags (NP_OPS_* constants).
*
*/
struct node_private_ops {
+ bool (*free_folio)(struct folio *folio);
unsigned long flags;
};
diff --git a/mm/internal.h b/mm/internal.h
index 97023748e6a9..658da41cdb8e 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1412,6 +1412,36 @@ int numa_migrate_check(struct folio *folio, struct
vm_fault *vmf,
void free_zone_device_folio(struct folio *folio);
int migrate_device_coherent_folio(struct folio *folio);
+/**
+ * folio_managed_on_free - Notify managed-memory service that folio
+ * refcount reached zero.
+ * @folio: the folio being freed
+ *
+ * Returns true if the folio is fully handled (zone_device -- caller
+ * must return immediately). Returns false if the callback ran but
+ * the folio should continue through the normal free path
+ * (private_node -- pages go back to buddy).
+ *
+ * Returns false for normal folios (no-op).
+ */
+static inline bool folio_managed_on_free(struct folio *folio)
+{
+ if (folio_is_zone_device(folio)) {
+ free_zone_device_folio(folio);
+ return true;
+ }
+ if (folio_is_private_node(folio)) {
+ const struct node_private_ops *ops =
+ folio_node_private_ops(folio);
+
+ if (ops && ops->free_folio) {
+ if (ops->free_folio(folio))
+ return true;
+ }
+ }
+ return false;
+}
+
struct vm_struct *__get_vm_area_node(unsigned long size,
unsigned long align, unsigned long shift,
unsigned long vm_flags, unsigned long
start,
diff --git a/mm/swap.c b/mm/swap.c
index 2260dcd2775e..dca306e1ae6d 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -37,6 +37,7 @@
#include <linux/page_idle.h>
#include <linux/local_lock.h>
#include <linux/buffer_head.h>
+#include <linux/node_private.h>
#include "internal.h"
@@ -96,10 +97,9 @@ static void page_cache_release(struct folio *folio)
void __folio_put(struct folio *folio)
{
- if (unlikely(folio_is_zone_device(folio))) {
- free_zone_device_folio(folio);
- return;
- }
+ if (unlikely(folio_is_private_managed(folio)))
+ if (folio_managed_on_free(folio))
+ return;
if (folio_test_hugetlb(folio)) {
free_huge_folio(folio);
@@ -961,19 +961,18 @@ void folios_put_refs(struct folio_batch *folios, unsigned
int *refs)
if (is_huge_zero_folio(folio))
continue;
- if (folio_is_zone_device(folio)) {
+ if (!folio_ref_sub_and_test(folio, nr_refs))
+ continue;
+
+ if (unlikely(folio_is_private_managed(folio))) {
if (lruvec) {
unlock_page_lruvec_irqrestore(lruvec, flags);
lruvec = NULL;
}
- if (folio_ref_sub_and_test(folio, nr_refs))
- free_zone_device_folio(folio);
- continue;
+ if (folio_managed_on_free(folio))
+ continue;
}
- if (!folio_ref_sub_and_test(folio, nr_refs))
- continue;
-
/* hugetlb has its own memcg */
if (folio_test_hugetlb(folio)) {
if (lruvec) {
--
2.53.0
