Re: [RFC v2 01/43] mm: add PKRAM API stubs and Kconfig
On 3/31/21 11:43 AM, Randy Dunlap wrote: > On 3/30/21 2:35 PM, Anthony Yznaga wrote: >> Preserved-across-kexec memory or PKRAM is a method for saving memory >> pages of the currently executing kernel and restoring them after kexec >> boot into a new one. This can be utilized for preserving guest VM state, >> large in-memory databases, process memory, etc. across reboot. While >> DRAM-as-PMEM or actual persistent memory could be used to accomplish >> these things, PKRAM provides the latency of DRAM with the flexibility >> of dynamically determining the amount of memory to preserve. >> > ... > >> Originally-by: Vladimir Davydov >> Signed-off-by: Anthony Yznaga >> --- >> include/linux/pkram.h | 47 + >> mm/Kconfig| 9 +++ >> mm/Makefile | 1 + >> mm/pkram.c| 179 >> ++ >> 4 files changed, 236 insertions(+) >> create mode 100644 include/linux/pkram.h >> create mode 100644 mm/pkram.c >> >> diff --git a/mm/pkram.c b/mm/pkram.c >> new file mode 100644 >> index ..59e4661b2fb7 >> --- /dev/null >> +++ b/mm/pkram.c >> @@ -0,0 +1,179 @@ >> +// SPDX-License-Identifier: GPL-2.0 >> +#include >> +#include >> +#include >> +#include >> +#include >> +#include >> + > Hi, > > There are several doc blocks that begin with "/**" but that are not > in kernel-doc format (/** means kernel-doc format when inside the kernel > source tree). > > Please either change those to "/*" or convert them to kernel-doc format. > The latter is preferable for exported interfaces. Thank you. I'll fix these up. > >> +/** >> + * Create a preserved memory node with name @name and initialize stream @ps >> + * for saving data to it. >> + * >> + * @gfp_mask specifies the memory allocation mask to be used when saving >> data. >> + * >> + * Returns 0 on success, -errno on failure. >> + * >> + * After the save has finished, pkram_finish_save() (or >> pkram_discard_save() in >> + * case of failure) is to be called. >> + */ > > b) from patch 00/43: > > documentation/core-api/xarray.rst |8 + > > How did "documentation" become lower case (instead of Documentation)? That is odd. The patch (41) has it correct. Anthony > > > thanks.
[RFC v2 39/43] shmem: optimize adding pages to the LRU in shmem_insert_pages()
Reduce LRU lock contention when inserting shmem pages by staging pages
to be added to the same LRU and adding them en masse.
Signed-off-by: Anthony Yznaga
---
mm/shmem.c | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/mm/shmem.c b/mm/shmem.c
index c3fa72061d8a..63299da75166 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -845,6 +845,7 @@ int shmem_insert_pages(struct mm_struct *charge_mm, struct
inode *inode,
struct shmem_inode_info *info = SHMEM_I(inode);
struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
gfp_t gfp = mapping_gfp_mask(mapping);
+ LRU_SPLICE(splice);
int i, err;
int nr = 0;
@@ -908,7 +909,7 @@ int shmem_insert_pages(struct mm_struct *charge_mm, struct
inode *inode,
for (i = 0; i < npages; i++) {
if (!PageLRU(pages[i]))
- lru_cache_add(pages[i]);
+ lru_splice_add(pages[i], );
flush_dcache_page(pages[i]);
SetPageUptodate(pages[i]);
@@ -917,6 +918,8 @@ int shmem_insert_pages(struct mm_struct *charge_mm, struct
inode *inode,
unlock_page(pages[i]);
}
+ add_splice_to_lru_list();
+
return 0;
out_release:
--
1.8.3.1
[RFC v2 31/43] memblock, mm: defer initialization of preserved pages
Preserved pages are represented in the memblock reserved list, but page
structs for pages in the reserved list are initialized early while boot
is single threaded which means that a large number of preserved pages
can impact boot time. To mitigate, defer initialization of preserved
pages by skipping them when other reserved pages are initialized and
initializing them later with a separate kernel thread.
Signed-off-by: Anthony Yznaga
---
arch/x86/mm/init_64.c | 1 -
include/linux/mm.h| 2 +-
mm/memblock.c | 11 +--
mm/page_alloc.c | 55 +++
4 files changed, 57 insertions(+), 12 deletions(-)
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 69bd71996b8b..8efb2fb2a88b 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1294,7 +1294,6 @@ void __init mem_init(void)
after_bootmem = 1;
x86_init.hyper.init_after_bootmem();
- pkram_cleanup();
totalram_pages_add(pkram_reserved_pages);
/*
* Must be done after boot memory is put on freelist, because here we
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 64a71bf20536..2a93b2a6ec8d 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2337,7 +2337,7 @@ extern unsigned long free_reserved_area(void *start, void
*end,
extern void adjust_managed_page_count(struct page *page, long count);
extern void mem_init_print_info(const char *str);
-extern void reserve_bootmem_region(phys_addr_t start, phys_addr_t end);
+extern void reserve_bootmem_region(phys_addr_t start, phys_addr_t end, int
nid);
/* Free the reserved page into the buddy system, so it gets managed. */
static inline void free_reserved_page(struct page *page)
diff --git a/mm/memblock.c b/mm/memblock.c
index afaefa8fc6ab..461ea0f85495 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -2007,11 +2007,18 @@ static unsigned long __init
free_low_memory_core_early(void)
unsigned long count = 0;
phys_addr_t start, end;
u64 i;
+ struct memblock_region *r;
memblock_clear_hotplug(0, -1);
- for_each_reserved_mem_range(i, , )
- reserve_bootmem_region(start, end);
+ for_each_reserved_mem_region(r) {
+ if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT) &&
memblock_is_preserved(r))
+ continue;
+
+ start = r->base;
+ end = r->base + r->size;
+ reserve_bootmem_region(start, end, NUMA_NO_NODE);
+ }
/*
* We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index cfc72873961d..999fcc8fe907 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -72,6 +72,7 @@
#include
#include
#include
+#include
#include
#include
@@ -1475,15 +1476,18 @@ static void __meminit __init_single_page(struct page
*page, unsigned long pfn,
}
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
-static void __meminit init_reserved_page(unsigned long pfn)
+static void __meminit init_reserved_page(unsigned long pfn, int nid)
{
pg_data_t *pgdat;
- int nid, zid;
+ int zid;
- if (!early_page_uninitialised(pfn))
- return;
+ if (nid == NUMA_NO_NODE) {
+ if (!early_page_uninitialised(pfn))
+ return;
+
+ nid = early_pfn_to_nid(pfn);
+ }
- nid = early_pfn_to_nid(pfn);
pgdat = NODE_DATA(nid);
for (zid = 0; zid < MAX_NR_ZONES; zid++) {
@@ -1495,7 +1499,7 @@ static void __meminit init_reserved_page(unsigned long
pfn)
__init_single_page(pfn_to_page(pfn), pfn, zid, nid);
}
#else
-static inline void init_reserved_page(unsigned long pfn)
+static inline void init_reserved_page(unsigned long pfn, int nid)
{
}
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
@@ -1506,7 +1510,7 @@ static inline void init_reserved_page(unsigned long pfn)
* marks the pages PageReserved. The remaining valid pages are later
* sent to the buddy page allocator.
*/
-void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
+void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end, int
nid)
{
unsigned long start_pfn = PFN_DOWN(start);
unsigned long end_pfn = PFN_UP(end);
@@ -1515,7 +1519,7 @@ void __meminit reserve_bootmem_region(phys_addr_t start,
phys_addr_t end)
if (pfn_valid(start_pfn)) {
struct page *page = pfn_to_page(start_pfn);
- init_reserved_page(start_pfn);
+ init_reserved_page(start_pfn, nid);
/* Avoid false-positive PageTail() */
INIT_LIST_HEAD(>lru);
@@ -2008,6 +2012,35 @@ static int __init deferred_init_memmap(void *data)
return 0;
}
+#ifdef CONFIG_PKRAM
+static int __init deferred_init_preserved(void *dummy)
+{
+
[RFC v2 42/43] shmem: reduce time holding xa_lock when inserting pages
Rather than adding one page at a time to the page cache and taking the
page cache xarray lock each time, where possible add pages in bulk by
first populating an xarray node outside of the page cache before taking
the lock to insert it.
When a group of pages to be inserted will fill an xarray node, add them
to a local xarray, export the xarray node, and then take the lock on the
page cache xarray and insert the node.
Signed-off-by: Anthony Yznaga
---
mm/shmem.c | 162 ++---
1 file changed, 156 insertions(+), 6 deletions(-)
diff --git a/mm/shmem.c b/mm/shmem.c
index f495af51042e..a7c23b43b57f 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -827,17 +827,149 @@ static void shmem_delete_from_page_cache(struct page
*page, void *radswap)
BUG_ON(error);
}
+static int shmem_add_aligned_to_page_cache(struct page *pages[], int npages,
+ struct address_space *mapping,
+ pgoff_t index, gfp_t gfp, int order,
+ struct mm_struct *charge_mm)
+{
+ int xa_shift = order + XA_CHUNK_SHIFT - (order % XA_CHUNK_SHIFT);
+ XA_STATE_ORDER(xas, >i_pages, index, xa_shift);
+ struct xarray xa_tmp;
+ /*
+* Specify order so xas_create_range() only needs to be called once
+* to allocate the entire range. This guarantees that xas_store()
+* will not fail due to lack of memory.
+* Specify index == 0 so the minimum necessary nodes are allocated.
+*/
+ XA_STATE_ORDER(xas_tmp, _tmp, 0, xa_shift);
+ unsigned long nr = 1UL << order;
+ struct xa_node *node;
+ int i, error;
+
+ if (npages * nr != 1 << xa_shift) {
+ WARN_ONCE(1, "npages (%d) not aligned to xa_shift\n", npages);
+ return -EINVAL;
+ }
+ if (!IS_ALIGNED(index, 1 << xa_shift)) {
+ WARN_ONCE(1, "index (%lu) not aligned to xa_shift\n", index);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < npages; i++) {
+ bool skipcharge = page_memcg(pages[i]) ? true : false;
+
+ VM_BUG_ON_PAGE(PageTail(pages[i]), pages[i]);
+ VM_BUG_ON_PAGE(!PageLocked(pages[i]), pages[i]);
+ VM_BUG_ON_PAGE(!PageSwapBacked(pages[i]), pages[i]);
+
+ page_ref_add(pages[i], nr);
+ pages[i]->mapping = mapping;
+ pages[i]->index = index + (i * nr);
+
+ if (!skipcharge && !PageSwapCache(pages[i])) {
+ error = mem_cgroup_charge(pages[i], charge_mm, gfp);
+ if (error) {
+ if (PageTransHuge(pages[i])) {
+ count_vm_event(THP_FILE_FALLBACK);
+
count_vm_event(THP_FILE_FALLBACK_CHARGE);
+ }
+ goto error;
+ }
+ }
+ cgroup_throttle_swaprate(pages[i], gfp);
+ }
+
+ xa_init(_tmp);
+ do {
+ xas_lock(_tmp);
+ xas_create_range(_tmp);
+ if (xas_error(_tmp))
+ goto unlock;
+ for (i = 0; i < npages; i++) {
+ int j = 0;
+next:
+ xas_store(_tmp, pages[i]);
+ if (++j < nr) {
+ xas_next(_tmp);
+ goto next;
+ }
+ if (i < npages - 1)
+ xas_next(_tmp);
+ }
+ xas_set_order(_tmp, 0, xa_shift);
+ node = xas_export_node(_tmp);
+unlock:
+ xas_unlock(_tmp);
+ } while (xas_nomem(_tmp, gfp));
+
+ if (xas_error(_tmp)) {
+ error = xas_error(_tmp);
+ i = npages - 1;
+ goto error;
+ }
+
+ do {
+ xas_lock_irq();
+ xas_import_node(, node);
+ if (xas_error())
+ goto unlock1;
+ mapping->nrpages += nr * npages;
+ xas_unlock();
+ for (i = 0; i < npages; i++) {
+ __mod_lruvec_page_state(pages[i], NR_FILE_PAGES, nr);
+ __mod_lruvec_page_state(pages[i], NR_SHMEM, nr);
+ if (PageTransHuge(pages[i])) {
+ count_vm_event(THP_FILE_ALLOC);
+ __inc_node_page_state(pages[i], NR_SHMEM_THPS);
+ }
+ }
+ local_irq_enable();
+ break;
+unlock1:
+ xas_unlock_irq();
+ } while (xas_nomem(, gfp));
+
+ if (xas_error()) {
+ error = xas_error();
+
[RFC v2 41/43] XArray: add xas_export_node() and xas_import_node()
Contention on the xarray lock when multiple threads are adding to the
same xarray can be mitigated by providing a way to add entries in
bulk.
Allow a caller to allocate and populate an xarray node outside of
the target xarray and then only take the xarray lock long enough to
import the node into it.
Signed-off-by: Anthony Yznaga
---
Documentation/core-api/xarray.rst | 8 +++
include/linux/xarray.h| 2 +
lib/test_xarray.c | 45 +
lib/xarray.c | 100 ++
4 files changed, 155 insertions(+)
diff --git a/Documentation/core-api/xarray.rst
b/Documentation/core-api/xarray.rst
index a137a0e6d068..12ec59038fc8 100644
--- a/Documentation/core-api/xarray.rst
+++ b/Documentation/core-api/xarray.rst
@@ -444,6 +444,14 @@ called each time the XArray updates a node. This is used
by the page
cache workingset code to maintain its list of nodes which contain only
shadow entries.
+xas_export_node() is used to remove and return a node from an XArray
+while xas_import_node() is used to add a node to an XArray. Together
+these can be used, for example, to reduce lock contention when multiple
+threads are updating an XArray by allowing a caller to allocate and
+populate a node outside of the target XArray in a local XArray, export
+the node, and then take the target XArray lock just long enough to import
+the node.
+
Multi-Index Entries
---
diff --git a/include/linux/xarray.h b/include/linux/xarray.h
index 92c0160b3352..1eda38cbe020 100644
--- a/include/linux/xarray.h
+++ b/include/linux/xarray.h
@@ -1506,6 +1506,8 @@ static inline bool xas_retry(struct xa_state *xas, const
void *entry)
void xas_pause(struct xa_state *);
void xas_create_range(struct xa_state *);
+struct xa_node *xas_export_node(struct xa_state *xas);
+void xas_import_node(struct xa_state *xas, struct xa_node *node);
#ifdef CONFIG_XARRAY_MULTI
int xa_get_order(struct xarray *, unsigned long index);
diff --git a/lib/test_xarray.c b/lib/test_xarray.c
index 8294f43f4981..9cca0921cf9b 100644
--- a/lib/test_xarray.c
+++ b/lib/test_xarray.c
@@ -1765,6 +1765,50 @@ static noinline void check_destroy(struct xarray *xa)
#endif
}
+static noinline void check_export_import_1(struct xarray *xa,
+ unsigned long index, unsigned int order)
+{
+ int xa_shift = order + XA_CHUNK_SHIFT - (order % XA_CHUNK_SHIFT);
+ XA_STATE(xas, xa, index);
+ struct xa_node *node;
+ unsigned long i;
+
+ xa_store_many_order(xa, index, xa_shift);
+
+ xas_lock();
+ xas_set_order(, index, xa_shift);
+ node = xas_export_node();
+ xas_unlock();
+
+ XA_BUG_ON(xa, !xa_empty(xa));
+
+ do {
+ xas_lock();
+ xas_set_order(, index, xa_shift);
+ xas_import_node(, node);
+ xas_unlock();
+ } while (xas_nomem(, GFP_KERNEL));
+
+ for (i = index; i < index + (1UL << xa_shift); i++)
+ xa_erase_index(xa, i);
+
+ XA_BUG_ON(xa, !xa_empty(xa));
+}
+
+static noinline void check_export_import(struct xarray *xa)
+{
+ unsigned int order;
+ unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 12 : 1;
+
+ for (order = 0; order < max_order; order += XA_CHUNK_SHIFT) {
+ int xa_shift = order + XA_CHUNK_SHIFT;
+ unsigned long j;
+
+ for (j = 0; j < XA_CHUNK_SIZE; j++)
+ check_export_import_1(xa, j << xa_shift, order);
+ }
+}
+
static DEFINE_XARRAY(array);
static int xarray_checks(void)
@@ -1797,6 +1841,7 @@ static int xarray_checks(void)
check_workingset(, 0);
check_workingset(, 64);
check_workingset(, 4096);
+ check_export_import();
printk("XArray: %u of %u tests passed\n", tests_passed, tests_run);
return (tests_run == tests_passed) ? 0 : -EINVAL;
diff --git a/lib/xarray.c b/lib/xarray.c
index 5fa51614802a..58d58333f0d0 100644
--- a/lib/xarray.c
+++ b/lib/xarray.c
@@ -510,6 +510,30 @@ static void xas_delete_node(struct xa_state *xas)
xas_shrink(xas);
}
+static void xas_unlink_node(struct xa_state *xas)
+{
+ struct xa_node *node = xas->xa_node;
+ struct xa_node *parent;
+
+ parent = xa_parent_locked(xas->xa, node);
+ xas->xa_node = parent;
+ xas->xa_offset = node->offset;
+
+ if (!parent) {
+ xas->xa->xa_head = NULL;
+ xas->xa_node = XAS_BOUNDS;
+ return;
+ }
+
+ parent->slots[xas->xa_offset] = NULL;
+ parent->count--;
+ XA_NODE_BUG_ON(parent, parent->count > XA_CHUNK_SIZE);
+
+ xas_update(xas, parent);
+
+ xas_delete_node(xas);
+}
+
/**
* xas_free_nodes() - Free this node and all nodes that it references
* @xas: Array operation state.
@@ -1690,6 +1714,82 @
[RFC v2 43/43] PKRAM: improve index alignment of pkram_link entries
To take advantage of optimizations when adding pages to the page cache
via shmem_insert_pages(), improve the likelihood that the pages array
passed to shmem_insert_pages() starts on an aligned index. Do this
when preserving pages by starting a new pkram_link page when the current
page is aligned and the next aligned page will not fit on the pkram_link
page.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 13 -
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index b63b2a3958e7..3f43809c8a85 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -911,9 +911,20 @@ static int __pkram_save_page(struct pkram_access *pa,
struct page *page,
{
struct pkram_data_stream *pds = >pds;
struct pkram_link *link = pds->link;
+ int align, align_cnt;
+
+ if (PageTransHuge(page)) {
+ align = 1 << (HPAGE_PMD_ORDER + XA_CHUNK_SHIFT -
(HPAGE_PMD_ORDER % XA_CHUNK_SHIFT));
+ align_cnt = align >> HPAGE_PMD_ORDER;
+ } else {
+ align = XA_CHUNK_SIZE;
+ align_cnt = XA_CHUNK_SIZE;
+ }
if (!link || pds->entry_idx >= PKRAM_LINK_ENTRIES_MAX ||
- index != pa->pages.next_index) {
+ index != pa->pages.next_index ||
+ (IS_ALIGNED(index, align) &&
+ (pds->entry_idx + align_cnt > PKRAM_LINK_ENTRIES_MAX))) {
link = pkram_new_link(pds, pa->ps->gfp_mask);
if (!link)
return -ENOMEM;
--
1.8.3.1
[RFC v2 40/43] shmem: initial support for adding multiple pages to pagecache
shmem_insert_pages() currently loops over the array of pages passed
to it and calls shmem_add_to_page_cache() for each one. Prepare
for adding pages to the pagecache in bulk by adding and using a
shmem_add_pages_to_cache() call. For now it just iterates over
an array and adds pages individually, but improvements in performance
when multiple threads are adding to the same pagecache are achieved
by calling a new shmem_add_to_page_cache_fast() function that does
not check for conflicts and drops the xarray lock before updating stats.
Signed-off-by: Anthony Yznaga
---
mm/shmem.c | 123 +
1 file changed, 108 insertions(+), 15 deletions(-)
diff --git a/mm/shmem.c b/mm/shmem.c
index 63299da75166..f495af51042e 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -738,6 +738,74 @@ static int shmem_add_to_page_cache(struct page *page,
return error;
}
+static int shmem_add_to_page_cache_fast(struct page *page,
+ struct address_space *mapping,
+ pgoff_t index, gfp_t gfp,
+ struct mm_struct *charge_mm, bool skipcharge)
+{
+ XA_STATE_ORDER(xas, >i_pages, index, thp_order(page));
+ unsigned long nr = thp_nr_pages(page);
+ unsigned long i = 0;
+ int error;
+
+ VM_BUG_ON_PAGE(PageTail(page), page);
+ VM_BUG_ON_PAGE(index != round_down(index, nr), page);
+ VM_BUG_ON_PAGE(!PageLocked(page), page);
+ VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
+
+ page_ref_add(page, nr);
+ page->mapping = mapping;
+ page->index = index;
+
+ if (!skipcharge && !PageSwapCache(page)) {
+ error = mem_cgroup_charge(page, charge_mm, gfp);
+ if (error) {
+ if (PageTransHuge(page)) {
+ count_vm_event(THP_FILE_FALLBACK);
+ count_vm_event(THP_FILE_FALLBACK_CHARGE);
+ }
+ goto error;
+ }
+ }
+ cgroup_throttle_swaprate(page, gfp);
+
+ do {
+ xas_lock_irq();
+ xas_create_range();
+ if (xas_error())
+ goto unlock;
+next:
+ xas_store(, page);
+ if (++i < nr) {
+ xas_next();
+ goto next;
+ }
+ mapping->nrpages += nr;
+ xas_unlock();
+ if (PageTransHuge(page)) {
+ count_vm_event(THP_FILE_ALLOC);
+ __inc_node_page_state(page, NR_SHMEM_THPS);
+ }
+ __mod_lruvec_page_state(page, NR_FILE_PAGES, nr);
+ __mod_lruvec_page_state(page, NR_SHMEM, nr);
+ local_irq_enable();
+ break;
+unlock:
+ xas_unlock_irq();
+ } while (xas_nomem(, gfp));
+
+ if (xas_error()) {
+ error = xas_error();
+ goto error;
+ }
+
+ return 0;
+error:
+ page->mapping = NULL;
+ page_ref_sub(page, nr);
+ return error;
+}
+
/*
* Like delete_from_page_cache, but substitutes swap for page.
*/
@@ -759,6 +827,41 @@ static void shmem_delete_from_page_cache(struct page
*page, void *radswap)
BUG_ON(error);
}
+static int shmem_add_pages_to_cache(struct page *pages[], int npages,
+ struct address_space *mapping,
+ pgoff_t start, gfp_t gfp,
+ struct mm_struct *charge_mm)
+{
+ pgoff_t index = start;
+ int i, err;
+
+ i = 0;
+ while (i < npages) {
+ if (PageTransHuge(pages[i])) {
+ err = shmem_add_to_page_cache_fast(pages[i], mapping,
index, gfp, charge_mm, page_memcg(pages[i]) ? true : false);
+ if (err)
+ goto out_release;
+ index += thp_nr_pages(pages[i]);
+ i++;
+ continue;
+ }
+
+ err = shmem_add_to_page_cache_fast(pages[i], mapping, index,
gfp, charge_mm, page_memcg(pages[i]) ? true : false);
+ if (err)
+ goto out_release;
+ index++;
+ i++;
+ }
+ return 0;
+
+out_release:
+ while (i < npages) {
+ delete_from_page_cache(pages[i]);
+ i--;
+ }
+ return err;
+}
+
int shmem_insert_page(struct mm_struct *mm, struct inode *inode, pgoff_t index,
struct page *page)
{
@@ -889,17 +992,10 @@ int shmem_insert_pages(struct mm_struct *charge_mm,
struct inode *inode,
__SetPageReferenced(pages[i]);
}
- for (i = 0; i < npages; i++) {
- bool ischarged = page_memcg(pages[i]) ?
[RFC v2 36/43] PKRAM: add support for loading pages in bulk
Implement a new API function, pkram_load_file_pages(), to support
loading pages in bulk. A caller provided buffer not smaller than
PKRAM_PAGES_BUFSIZE is populated with pages pointers that are contiguous
by their original mapping index values. The number of pages in the buffer
and the mapping index of the first page are provided to the caller.
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 4
mm/pkram.c| 46 ++
2 files changed, 50 insertions(+)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index 977cf45a1bcf..ca46e5eafe71 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -96,6 +96,10 @@ int pkram_prepare_save(struct pkram_stream *ps, const char
*name,
int pkram_save_file_page(struct pkram_access *pa, struct page *page);
struct page *pkram_load_file_page(struct pkram_access *pa, unsigned long
*index);
+#define PKRAM_PAGES_BUFSIZEPAGE_SIZE
+
+int pkram_load_file_pages(struct pkram_access *pa, struct page *pages[],
unsigned int *nr_pages, unsigned long *index);
+
ssize_t pkram_write(struct pkram_access *pa, const void *buf, size_t count);
size_t pkram_read(struct pkram_access *pa, void *buf, size_t count);
diff --git a/mm/pkram.c b/mm/pkram.c
index 382ccf6f789f..b63b2a3958e7 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -1099,6 +1099,52 @@ struct page *pkram_load_file_page(struct pkram_access
*pa, unsigned long *index)
}
/**
+ * Load pages from the preserved memory node and object associated with
+ * pkram stream access @pa. The stream must have been initialized with
+ * pkram_prepare_load() and pkram_prepare_load_obj() and access initialized
+ * with PKRAM_ACCESS().
+ * The page entries of a single pkram_link are processed, and @pages is
+ * populated with the page pointers. @nr_pages is set to the number of
+ * pages, and @index is set to the mapping index of the first page.
+ *
+ * Returns 0 if one or more pages are loaded or -ENODATA if there are no
+ * pages to load.
+ *
+ * The pages loaded have an incremented refcount either because the page
+ * was initialized with a refcount of 1 at boot or because the page was
+ * subsequently preserved which increased the refcount.
+ */
+int pkram_load_file_pages(struct pkram_access *pa, struct page *pages[],
unsigned int *nr_pages, unsigned long *index)
+{
+ struct pkram_data_stream *pds = >pds;
+ struct pkram_link *link;
+ int nr_entries = 0;
+ int i, ret;
+
+ ret = pkram_next_link(pds, );
+ if (ret)
+ return ret;
+
+ for (i = 0; i < PKRAM_LINK_ENTRIES_MAX; i++) {
+ unsigned long p = link->entry[i];
+
+ if (!p)
+ break;
+
+ pages[i] = __pkram_prep_load_page(p);
+ nr_entries++;
+ }
+
+ *nr_pages = nr_entries;
+ *index = link->index;
+
+ pkram_free_page(link);
+ pds->link = NULL;
+
+ return 0;
+}
+
+/**
* Copy @count bytes from @buf to the preserved memory node and object
* associated with pkram stream access @pa. The stream must have been
* initialized with pkram_prepare_save() and pkram_prepare_save_obj()
--
1.8.3.1
[RFC v2 38/43] mm: implement splicing a list of pages to the LRU
Considerable contention on the LRU lock happens when multiple threads
are used to insert pages into a shmem file in parallel. To alleviate this
provide a way for pages to be added to the same LRU to be staged so that
they can be added by splicing lists and updating stats once with the lock
held. For now only unevictable pages are supported.
Signed-off-by: Anthony Yznaga
---
include/linux/swap.h | 13
mm/swap.c| 86
2 files changed, 99 insertions(+)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 4cc6ec3bf0ab..254c9c8d71d0 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -351,6 +351,19 @@ extern void lru_note_cost(struct lruvec *lruvec, bool file,
extern void lru_cache_add_inactive_or_unevictable(struct page *page,
struct vm_area_struct *vma);
+struct lru_splice {
+ struct list_headsplice;
+ struct list_head*lru_head;
+ struct lruvec *lruvec;
+ enum lru_list lru;
+ unsigned long nr_pages[MAX_NR_ZONES];
+ unsigned long pgculled;
+};
+#define LRU_SPLICE_INIT(name) { .splice = LIST_HEAD_INIT(name.splice) }
+#define LRU_SPLICE(name) \
+ struct lru_splice name = LRU_SPLICE_INIT(name)
+extern void lru_splice_add(struct page *page, struct lru_splice *splice);
+extern void add_splice_to_lru_list(struct lru_splice *splice);
/* linux/mm/vmscan.c */
extern unsigned long zone_reclaimable_pages(struct zone *zone);
diff --git a/mm/swap.c b/mm/swap.c
index 31b844d4ed94..a1db6a748608 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -200,6 +200,92 @@ int get_kernel_page(unsigned long start, int write, struct
page **pages)
}
EXPORT_SYMBOL_GPL(get_kernel_page);
+/*
+ * Update stats and move accumulated pages from an lru_splice to the lru.
+ */
+void add_splice_to_lru_list(struct lru_splice *splice)
+{
+ struct lruvec *lruvec = splice->lruvec;
+ enum lru_list lru = splice->lru;
+ unsigned long flags = 0;
+ int zid;
+
+ if (list_empty(>splice))
+ return;
+
+ spin_lock_irqsave(>lru_lock, flags);
+ for (zid = 0; zid < MAX_NR_ZONES; zid++) {
+ if (splice->nr_pages[zid])
+ update_lru_size(lruvec, lru, zid,
splice->nr_pages[zid]);
+ }
+ count_vm_events(UNEVICTABLE_PGCULLED, splice->pgculled);
+ list_splice_init(>splice, splice->lru_head);
+ spin_unlock_irqrestore(>lru_lock, flags);
+}
+
+static void add_page_to_lru_splice(struct page *page, struct lru_splice
*splice,
+ struct lruvec *lruvec, enum lru_list lru)
+{
+ if (list_empty(>splice)) {
+ int zid;
+
+ splice->lruvec = lruvec;
+ splice->lru_head = >lists[lru];
+ splice->lru = lru;
+ for (zid = 0; zid < MAX_NR_ZONES; zid++)
+ splice->nr_pages[zid] = 0;
+ splice->pgculled = 0;
+ }
+
+ BUG_ON(splice->lruvec != lruvec);
+ BUG_ON(splice->lru_head != >lists[lru]);
+
+ list_add(>lru, >splice);
+ splice->nr_pages[page_zonenum(page)] += thp_nr_pages(page);
+}
+
+/*
+ * Similar in functionality to __pagevec_lru_add_fn() but here the page is
+ * being added to an lru_splice and the LRU lock is not held.
+ */
+static void page_lru_splice_add(struct page *page, struct lru_splice *splice,
struct lruvec *lruvec)
+{
+ enum lru_list lru;
+ int was_unevictable = TestClearPageUnevictable(page);
+ int nr_pages = thp_nr_pages(page);
+
+ VM_BUG_ON_PAGE(PageLRU(page), page);
+ /* XXX only supports unevictable pages at the moment */
+ VM_BUG_ON_PAGE(was_unevictable, page);
+
+ SetPageLRU(page);
+ smp_mb__after_atomic();
+
+ lru = LRU_UNEVICTABLE;
+ ClearPageActive(page);
+ SetPageUnevictable(page);
+ if (!was_unevictable)
+ splice->pgculled += nr_pages;
+
+ add_page_to_lru_splice(page, splice, lruvec, lru);
+ trace_mm_lru_insertion(page);
+}
+
+void lru_splice_add(struct page *page, struct lru_splice *splice)
+{
+ struct lruvec *lruvec;
+
+ VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page);
+ VM_BUG_ON_PAGE(PageLRU(page), page);
+
+ get_page(page);
+ lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page));
+ if (lruvec != splice->lruvec)
+ add_splice_to_lru_list(splice);
+ page_lru_splice_add(page, splice, lruvec);
+ put_page(page);
+}
+
static void pagevec_lru_move_fn(struct pagevec *pvec,
void (*move_fn)(struct page *page, struct lruvec *lruvec))
{
--
1.8.3.1
[RFC v2 25/43] mm: shmem: prevent swapping of PKRAM-enabled tmpfs pages
Work around the limitation that shmem pages must be in memory in order
to be preserved by preventing them from being swapped out in the first
place. Do this by marking shmem pages associated with a PKRAM node
as unevictable.
Signed-off-by: Anthony Yznaga
---
mm/shmem.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/mm/shmem.c b/mm/shmem.c
index c1c5760465f2..8dfe80aeee97 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -2400,6 +2400,8 @@ static struct inode *shmem_get_inode(struct super_block
*sb, const struct inode
INIT_LIST_HEAD(>swaplist);
simple_xattrs_init(>xattrs);
cache_no_acl(inode);
+ if (sbinfo->pkram)
+ mapping_set_unevictable(inode->i_mapping);
switch (mode & S_IFMT) {
default:
--
1.8.3.1
[RFC v2 34/43] shmem: PKRAM: multithread preserving and restoring shmem pages
Improve performance by multithreading the work to preserve and restore
shmem pages.
When preserving pages each thread saves non-overlapping ranges of a file
to a pkram_obj until all pages are preserved.
When restoring pages each thread loads pages using a local pkram_access.
Signed-off-by: Anthony Yznaga
---
mm/shmem_pkram.c | 94 +---
1 file changed, 89 insertions(+), 5 deletions(-)
diff --git a/mm/shmem_pkram.c b/mm/shmem_pkram.c
index e52722b3a709..354c2b58962c 100644
--- a/mm/shmem_pkram.c
+++ b/mm/shmem_pkram.c
@@ -115,6 +115,7 @@ static int save_file_content_range(struct pkram_access *pa,
}
struct shmem_pkram_arg {
+ int *error;
struct pkram_stream *ps;
struct address_space *mapping;
struct mm_struct *mm;
@@ -137,6 +138,16 @@ static int get_save_range(unsigned long max, atomic64_t
*next, unsigned long *st
return 0;
}
+/* Completion tracking for save_file_content_thr() threads */
+static atomic_t pkram_save_n_undone;
+static DECLARE_COMPLETION(pkram_save_all_done_comp);
+
+static inline void pkram_save_report_one_done(void)
+{
+ if (atomic_dec_and_test(_save_n_undone))
+ complete(_save_all_done_comp);
+}
+
static int do_save_file_content(struct pkram_stream *ps,
struct address_space *mapping,
atomic64_t *next)
@@ -160,11 +171,40 @@ static int do_save_file_content(struct pkram_stream *ps,
return ret;
}
-static int save_file_content(struct pkram_stream *ps, struct address_space
*mapping)
+static int save_file_content_thr(void *data)
{
- struct shmem_pkram_arg arg = { ps, mapping, NULL, ATOMIC64_INIT(0) };
-
- return do_save_file_content(arg.ps, arg.mapping, );
+ struct shmem_pkram_arg *arg = data;
+ int ret;
+
+ ret = do_save_file_content(arg->ps, arg->mapping, >next);
+ if (ret && !*arg->error)
+ *arg->error = ret;
+
+ pkram_save_report_one_done();
+ return 0;
+}
+
+static int shmem_pkram_max_threads = 16;
+
+static int save_file_content(struct pkram_stream *ps, struct address_space
*mapping)
+ {
+ int err = 0;
+ struct shmem_pkram_arg arg = { , ps, mapping, NULL,
ATOMIC64_INIT(0) };
+ unsigned int thr, nr_threads;
+
+ nr_threads = num_online_cpus() - 1;
+ nr_threads = clamp_val(shmem_pkram_max_threads, 1, nr_threads);
+
+ if (nr_threads == 1)
+ return do_save_file_content(arg.ps, arg.mapping, );
+
+ atomic_set(_save_n_undone, nr_threads);
+ for (thr = 0; thr < nr_threads; thr++)
+ kthread_run(save_file_content_thr, , "pkram_save%d", thr);
+
+ wait_for_completion(_save_all_done_comp);
+
+ return err;
}
static int save_file(struct dentry *dentry, struct pkram_stream *ps)
@@ -275,7 +315,17 @@ int shmem_save_pkram(struct super_block *sb)
return err;
}
-static int load_file_content(struct pkram_stream *ps, struct address_space
*mapping, struct mm_struct *mm)
+/* Completion tracking for load_file_content_thr() threads */
+static atomic_t pkram_load_n_undone;
+static DECLARE_COMPLETION(pkram_load_all_done_comp);
+
+static inline void pkram_load_report_one_done(void)
+{
+ if (atomic_dec_and_test(_load_n_undone))
+ complete(_load_all_done_comp);
+}
+
+static int do_load_file_content(struct pkram_stream *ps, struct address_space
*mapping, struct mm_struct *mm)
{
PKRAM_ACCESS(pa, ps, pages);
unsigned long index;
@@ -296,6 +346,40 @@ static int load_file_content(struct pkram_stream *ps,
struct address_space *mapp
return err;
}
+static int load_file_content_thr(void *data)
+{
+ struct shmem_pkram_arg *arg = data;
+ int ret;
+
+ ret = do_load_file_content(arg->ps, arg->mapping, arg->mm);
+ if (ret && !*arg->error)
+ *arg->error = ret;
+
+ pkram_load_report_one_done();
+ return 0;
+}
+
+static int load_file_content(struct pkram_stream *ps, struct address_space
*mapping, struct mm_struct *mm)
+{
+ int err = 0;
+ struct shmem_pkram_arg arg = { , ps, mapping, mm };
+ unsigned int thr, nr_threads;
+
+ nr_threads = num_online_cpus() - 1;
+ nr_threads = clamp_val(shmem_pkram_max_threads, 1, nr_threads);
+
+ if (nr_threads == 1)
+ return do_load_file_content(ps, mapping, mm);
+
+ atomic_set(_load_n_undone, nr_threads);
+ for (thr = 0; thr < nr_threads; thr++)
+ kthread_run(load_file_content_thr, , "pkram_load%d", thr);
+
+ wait_for_completion(_load_all_done_comp);
+
+ return err;
+}
+
static int load_file(struct dentry *parent, struct pkram_stream *ps,
char *buf, size_t bufsize)
{
--
1.8.3.1
[RFC v2 35/43] shmem: introduce shmem_insert_pages()
Calling shmem_insert_page() to insert one page at a time does
not scale well when multiple threads are inserting pages into
the same shmem segment. This is primarily due to the locking needed
when adding to the pagecache and LRU but also due to contention
on the shmem_inode_info lock. To address the shmem_inode_info lock
and prepare for future optimizations, introduce shmem_insert_pages()
which allows a caller to pass an array of pages to be inserted into a
shmem segment.
Signed-off-by: Anthony Yznaga
---
include/linux/shmem_fs.h | 3 +-
mm/shmem.c | 93
2 files changed, 95 insertions(+), 1 deletion(-)
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index 78149d702a62..bc116c4fe145 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -112,7 +112,8 @@ extern int shmem_getpage(struct inode *inode, pgoff_t index,
extern int shmem_insert_page(struct mm_struct *mm, struct inode *inode,
pgoff_t index, struct page *page);
-
+extern int shmem_insert_pages(struct mm_struct *mm, struct inode *inode,
+ pgoff_t index, struct page *pages[], int npages);
#ifdef CONFIG_PKRAM
extern int shmem_parse_pkram(const char *str, struct shmem_pkram_info **pkram);
extern void shmem_show_pkram(struct seq_file *seq, struct shmem_pkram_info
*pkram,
diff --git a/mm/shmem.c b/mm/shmem.c
index 44cc158ab34d..c3fa72061d8a 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -838,6 +838,99 @@ int shmem_insert_page(struct mm_struct *mm, struct inode
*inode, pgoff_t index,
return err;
}
+int shmem_insert_pages(struct mm_struct *charge_mm, struct inode *inode,
+ pgoff_t index, struct page *pages[], int npages)
+{
+ struct address_space *mapping = inode->i_mapping;
+ struct shmem_inode_info *info = SHMEM_I(inode);
+ struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
+ gfp_t gfp = mapping_gfp_mask(mapping);
+ int i, err;
+ int nr = 0;
+
+ for (i = 0; i < npages; i++)
+ nr += thp_nr_pages(pages[i]);
+
+ if (index + nr - 1 > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
+ return -EFBIG;
+
+retry:
+ err = 0;
+ if (!shmem_inode_acct_block(inode, nr))
+ err = -ENOSPC;
+ if (err) {
+ int retry = 5;
+
+ /*
+* Try to reclaim some space by splitting a huge page
+* beyond i_size on the filesystem.
+*/
+ while (retry--) {
+ int ret;
+
+ ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
+ if (ret == SHRINK_STOP)
+ break;
+ if (ret)
+ goto retry;
+ }
+ goto failed;
+ }
+
+ for (i = 0; i < npages; i++) {
+ if (!PageLRU(pages[i])) {
+ __SetPageLocked(pages[i]);
+ __SetPageSwapBacked(pages[i]);
+ } else {
+ lock_page(pages[i]);
+ }
+
+ __SetPageReferenced(pages[i]);
+ }
+
+ for (i = 0; i < npages; i++) {
+ bool ischarged = page_memcg(pages[i]) ? true : false;
+
+ err = shmem_add_to_page_cache(pages[i], mapping, index,
+ NULL, gfp & GFP_RECLAIM_MASK,
+ charge_mm, ischarged);
+ if (err)
+ goto out_release;
+
+ index += thp_nr_pages(pages[i]);
+ }
+
+ spin_lock(>lock);
+ info->alloced += nr;
+ inode->i_blocks += BLOCKS_PER_PAGE * nr;
+ shmem_recalc_inode(inode);
+ spin_unlock(>lock);
+
+ for (i = 0; i < npages; i++) {
+ if (!PageLRU(pages[i]))
+ lru_cache_add(pages[i]);
+
+ flush_dcache_page(pages[i]);
+ SetPageUptodate(pages[i]);
+ set_page_dirty(pages[i]);
+
+ unlock_page(pages[i]);
+ }
+
+ return 0;
+
+out_release:
+ while (--i >= 0)
+ delete_from_page_cache(pages[i]);
+
+ for (i = 0; i < npages; i++)
+ unlock_page(pages[i]);
+
+ shmem_inode_unacct_blocks(inode, nr);
+failed:
+ return err;
+}
+
/*
* Remove swap entry from page cache, free the swap and its page cache.
*/
--
1.8.3.1
[RFC v2 37/43] shmem: PKRAM: enable bulk loading of preserved pages into shmem
Make use of new interfaces for loading and inserting preserved pages
into a shmem file in bulk.
Signed-off-by: Anthony Yznaga
---
mm/shmem_pkram.c | 23 +--
1 file changed, 17 insertions(+), 6 deletions(-)
diff --git a/mm/shmem_pkram.c b/mm/shmem_pkram.c
index 354c2b58962c..24a1ebb4af59 100644
--- a/mm/shmem_pkram.c
+++ b/mm/shmem_pkram.c
@@ -328,20 +328,31 @@ static inline void pkram_load_report_one_done(void)
static int do_load_file_content(struct pkram_stream *ps, struct address_space
*mapping, struct mm_struct *mm)
{
PKRAM_ACCESS(pa, ps, pages);
+ struct page **pages;
+ unsigned int nr_pages;
unsigned long index;
- struct page *page;
- int err = 0;
+ int i, err;
+
+ pages = kzalloc(PKRAM_PAGES_BUFSIZE, GFP_KERNEL);
+ if (!pages)
+ return -ENOMEM;
do {
- page = pkram_load_file_page(, );
- if (!page)
+ err = pkram_load_file_pages(, pages, _pages, );
+ if (err) {
+ if (err == -ENODATA)
+ err = 0;
break;
+ }
+
+ err = shmem_insert_pages(mm, mapping->host, index, pages,
nr_pages);
- err = shmem_insert_page(mm, mapping->host, index, page);
- put_page(page);
+ for (i = 0; i < nr_pages; i++)
+ put_page(pages[i]);
cond_resched();
} while (!err);
+ kfree(pages);
pkram_finish_access(, err == 0);
return err;
}
--
1.8.3.1
[RFC v2 30/43] memblock: PKRAM: mark memblocks that contain preserved pages
To support deferred initialization of page structs for preserved pages,
separate memblocks containing preserved pages by setting a new flag
when adding them to the memblock reserved list.
Signed-off-by: Anthony Yznaga
---
include/linux/memblock.h | 6 ++
mm/pkram.c | 2 ++
2 files changed, 8 insertions(+)
diff --git a/include/linux/memblock.h b/include/linux/memblock.h
index d13e3cd938b4..39c53d08d9f7 100644
--- a/include/linux/memblock.h
+++ b/include/linux/memblock.h
@@ -37,6 +37,7 @@ enum memblock_flags {
MEMBLOCK_HOTPLUG= 0x1, /* hotpluggable region */
MEMBLOCK_MIRROR = 0x2, /* mirrored region */
MEMBLOCK_NOMAP = 0x4, /* don't add to kernel direct mapping */
+ MEMBLOCK_PRESERVED = 0x8, /* preserved pages region */
};
/**
@@ -248,6 +249,11 @@ static inline bool memblock_is_nomap(struct
memblock_region *m)
return m->flags & MEMBLOCK_NOMAP;
}
+static inline bool memblock_is_preserved(struct memblock_region *m)
+{
+ return m->flags & MEMBLOCK_PRESERVED;
+}
+
int memblock_search_pfn_nid(unsigned long pfn, unsigned long *start_pfn,
unsigned long *end_pfn);
void __next_mem_pfn_range(int *idx, int nid, unsigned long *out_start_pfn,
diff --git a/mm/pkram.c b/mm/pkram.c
index b8d6b549fa6c..08144c18d425 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -1607,6 +1607,7 @@ int __init pkram_create_merged_reserved(struct
memblock_type *new)
} else if (pkr->base + pkr->size <= r->base) {
rgn->base = pkr->base;
rgn->size = pkr->size;
+ rgn->flags = MEMBLOCK_PRESERVED;
memblock_set_region_node(rgn, MAX_NUMNODES);
nr_preserved += (rgn->size >> PAGE_SHIFT);
@@ -1636,6 +1637,7 @@ int __init pkram_create_merged_reserved(struct
memblock_type *new)
rgn = >regions[k];
rgn->base = pkr->base;
rgn->size = pkr->size;
+ rgn->flags = MEMBLOCK_PRESERVED;
memblock_set_region_node(rgn, MAX_NUMNODES);
nr_preserved += (rgn->size >> PAGE_SHIFT);
--
1.8.3.1
[RFC v2 33/43] PKRAM: atomically add and remove link pages
Add and remove pkram_link pages from a pkram_obj atomically to prepare
for multithreading.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 39 ---
1 file changed, 24 insertions(+), 15 deletions(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index 08144c18d425..382ccf6f789f 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -535,33 +535,42 @@ static void pkram_truncate(void)
static void pkram_add_link(struct pkram_link *link, struct pkram_data_stream
*pds)
{
__u64 link_pfn = page_to_pfn(virt_to_page(link));
+ __u64 *tail = pds->tail_link_pfnp;
+ __u64 tail_pfn;
- if (!*pds->head_link_pfnp) {
+ do {
+ tail_pfn = *tail;
+ } while (cmpxchg64(tail, tail_pfn, link_pfn) != tail_pfn);
+
+ if (!tail_pfn) {
*pds->head_link_pfnp = link_pfn;
- *pds->tail_link_pfnp = link_pfn;
} else {
- struct pkram_link *tail = pfn_to_kaddr(*pds->tail_link_pfnp);
+ struct pkram_link *prev_tail = pfn_to_kaddr(tail_pfn);
- tail->link_pfn = link_pfn;
- *pds->tail_link_pfnp = link_pfn;
+ prev_tail->link_pfn = link_pfn;
}
}
static struct pkram_link *pkram_remove_link(struct pkram_data_stream *pds)
{
- struct pkram_link *link;
+ __u64 *head = pds->head_link_pfnp;
+ __u64 head_pfn = *head;
- if (!*pds->head_link_pfnp)
- return NULL;
+ while (head_pfn) {
+ struct pkram_link *link = pfn_to_kaddr(head_pfn);
- link = pfn_to_kaddr(*pds->head_link_pfnp);
- *pds->head_link_pfnp = link->link_pfn;
- if (!*pds->head_link_pfnp)
- *pds->tail_link_pfnp = 0;
- else
- link->link_pfn = 0;
+ if (cmpxchg64(head, head_pfn, link->link_pfn) == head_pfn) {
+ if (!*head)
+ *pds->tail_link_pfnp = 0;
+ else
+ link->link_pfn = 0;
+ return link;
+ }
- return link;
+ head_pfn = *head;
+ }
+
+ return NULL;
}
static struct pkram_link *pkram_new_link(struct pkram_data_stream *pds, gfp_t
gfp_mask)
--
1.8.3.1
[RFC v2 20/43] PKRAM: disable feature when running the kdump kernel
The kdump kernel should not preserve or restore pages.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index 8700fd77dc67..aea069cc49be 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -1,4 +1,5 @@
// SPDX-License-Identifier: GPL-2.0
+#include
#include
#include
#include
@@ -189,7 +190,7 @@ void __init pkram_reserve(void)
{
int err = 0;
- if (!pkram_sb_pfn)
+ if (!pkram_sb_pfn || is_kdump_kernel())
return;
pr_info("PKRAM: Examining preserved memory...\n");
@@ -286,6 +287,9 @@ static void pkram_show_banned(void)
int i;
unsigned long n, total = 0;
+ if (is_kdump_kernel())
+ return;
+
pr_info("PKRAM: banned regions:\n");
for (i = 0; i < nr_banned; i++) {
n = banned[i].end - banned[i].start + 1;
@@ -1315,7 +1319,7 @@ static int __init pkram_init_sb(void)
static int __init pkram_init(void)
{
- if (pkram_init_sb()) {
+ if (!is_kdump_kernel() && pkram_init_sb()) {
register_reboot_notifier(_reboot_notifier);
register_shrinker(_pages_shrinker);
sysfs_update_group(kernel_kobj, _attr_group);
--
1.8.3.1
[RFC v2 29/43] PKRAM: ensure memblocks with preserved pages init'd for numa
In order to facilitate fast initialization of page structs for
preserved pages, memblocks with preserved pages must not cross
numa node boundaries and must have a node id assigned to them.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 9 +
1 file changed, 9 insertions(+)
diff --git a/mm/pkram.c b/mm/pkram.c
index aea069cc49be..b8d6b549fa6c 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -21,6 +21,7 @@
#include
#include
+#include
#include "internal.h"
#define PKRAM_MAGIC0x706B726D
@@ -226,6 +227,14 @@ void __init pkram_reserve(void)
return;
}
+ /*
+* Fix up the reserved memblock list to ensure the
+* memblock regions are split along node boundaries
+* and have a node ID set. This will allow the page
+* structs for the preserved pages to be initialized
+* more efficiently.
+*/
+ numa_isolate_memblocks();
done:
pr_info("PKRAM: %lu pages reserved\n", pkram_reserved_pages);
}
--
1.8.3.1
[RFC v2 27/43] mm: shmem: when inserting, handle pages already charged to a memcg
If shmem_insert_page() is called to insert a page that was preserved
using PKRAM on the current boot (i.e. preserved page is restored without
an intervening kexec boot), the page will still be charged to a memory
cgroup because it is never freed. Don't try to charge it again.
Signed-off-by: Anthony Yznaga
---
mm/shmem.c | 14 --
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/mm/shmem.c b/mm/shmem.c
index 8dfe80aeee97..44cc158ab34d 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -671,7 +671,7 @@ static inline bool is_huge_enabled(struct shmem_sb_info
*sbinfo)
static int shmem_add_to_page_cache(struct page *page,
struct address_space *mapping,
pgoff_t index, void *expected, gfp_t gfp,
- struct mm_struct *charge_mm)
+ struct mm_struct *charge_mm, bool skipcharge)
{
XA_STATE_ORDER(xas, >i_pages, index, compound_order(page));
unsigned long i = 0;
@@ -688,7 +688,7 @@ static int shmem_add_to_page_cache(struct page *page,
page->mapping = mapping;
page->index = index;
- if (!PageSwapCache(page)) {
+ if (!skipcharge && !PageSwapCache(page)) {
error = mem_cgroup_charge(page, charge_mm, gfp);
if (error) {
if (PageTransHuge(page)) {
@@ -770,6 +770,7 @@ int shmem_insert_page(struct mm_struct *mm, struct inode
*inode, pgoff_t index,
int nr;
pgoff_t hindex = index;
bool on_lru = PageLRU(page);
+ bool ischarged = page_memcg(page) ? true : false;
if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
return -EFBIG;
@@ -809,7 +810,8 @@ int shmem_insert_page(struct mm_struct *mm, struct inode
*inode, pgoff_t index,
__SetPageReferenced(page);
err = shmem_add_to_page_cache(page, mapping, hindex,
- NULL, gfp & GFP_RECLAIM_MASK, mm);
+ NULL, gfp & GFP_RECLAIM_MASK,
+ mm, ischarged);
if (err)
goto out_unlock;
@@ -1829,7 +1831,7 @@ static int shmem_swapin_page(struct inode *inode, pgoff_t
index,
error = shmem_add_to_page_cache(page, mapping, index,
swp_to_radix_entry(swap), gfp,
- charge_mm);
+ charge_mm, false);
if (error)
goto failed;
@@ -2009,7 +2011,7 @@ static int shmem_getpage_gfp(struct inode *inode, pgoff_t
index,
error = shmem_add_to_page_cache(page, mapping, hindex,
NULL, gfp & GFP_RECLAIM_MASK,
- charge_mm);
+ charge_mm, false);
if (error)
goto unacct;
lru_cache_add(page);
@@ -2500,7 +2502,7 @@ static int shmem_mfill_atomic_pte(struct mm_struct
*dst_mm,
goto out_release;
ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL,
- gfp & GFP_RECLAIM_MASK, dst_mm);
+ gfp & GFP_RECLAIM_MASK, dst_mm, false);
if (ret)
goto out_release;
--
1.8.3.1
[RFC v2 11/43] PKRAM: prepare for adding preserved ranges to memblock reserved
Calling memblock_reserve() repeatedly to add preserved ranges is
inefficient and risks clobbering preserved memory if the memblock
reserved regions array must be resized. Instead, calculate the size
needed to accomodate the preserved ranges, find a suitable range for
a new reserved regions array that does not overlap any preserved range,
and populate it with a new, merged regions array.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 241 +
1 file changed, 241 insertions(+)
diff --git a/mm/pkram.c b/mm/pkram.c
index 4cfa236a4126..b4a14837946a 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -7,6 +7,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -1121,3 +1122,243 @@ static unsigned long pkram_populate_regions_list(void)
return priv.nr_regions;
}
+
+struct pkram_region *pkram_first_region(struct pkram_super_block *sb, struct
pkram_region_list **rlp, int *idx)
+{
+ WARN_ON(!sb);
+ WARN_ON(!sb->region_list_pfn);
+
+ if (!sb || !sb->region_list_pfn)
+ return NULL;
+
+ *rlp = pfn_to_kaddr(sb->region_list_pfn);
+ *idx = 0;
+
+ return &(*rlp)->regions[0];
+}
+
+struct pkram_region *pkram_next_region(struct pkram_region_list **rlp, int
*idx)
+{
+ struct pkram_region_list *rl = *rlp;
+ int i = *idx;
+
+ i++;
+ if (i >= PKRAM_REGIONS_LIST_MAX) {
+ if (!rl->next_pfn) {
+ pr_err("PKRAM: %s: no more pkram_region_list pages\n",
__func__);
+ return NULL;
+ }
+ rl = pfn_to_kaddr(rl->next_pfn);
+ *rlp = rl;
+ i = 0;
+ }
+ *idx = i;
+
+ if (rl->regions[i].size == 0)
+ return NULL;
+
+ return >regions[i];
+}
+
+struct pkram_region *pkram_first_region_topdown(struct pkram_super_block *sb,
struct pkram_region_list **rlp, int *idx)
+{
+ struct pkram_region_list *rl;
+
+ WARN_ON(!sb);
+ WARN_ON(!sb->region_list_pfn);
+
+ if (!sb || !sb->region_list_pfn)
+ return NULL;
+
+ rl = pfn_to_kaddr(sb->region_list_pfn);
+ if (!rl->prev_pfn) {
+ WARN_ON(1);
+ return NULL;
+ }
+ rl = pfn_to_kaddr(rl->prev_pfn);
+
+ *rlp = rl;
+
+ *idx = (sb->nr_regions - 1) % PKRAM_REGIONS_LIST_MAX;
+
+ return >regions[*idx];
+}
+
+struct pkram_region *pkram_next_region_topdown(struct pkram_region_list **rlp,
int *idx)
+{
+ struct pkram_region_list *rl = *rlp;
+ int i = *idx;
+
+ if (i == 0) {
+ if (!rl->prev_pfn)
+ return NULL;
+ rl = pfn_to_kaddr(rl->prev_pfn);
+ *rlp = rl;
+ i = PKRAM_REGIONS_LIST_MAX - 1;
+ } else
+ i--;
+
+ *idx = i;
+
+ return >regions[i];
+}
+
+/*
+ * Use the pkram regions list to find an available block of memory that does
+ * not overlap with preserved pages.
+ */
+phys_addr_t __init find_available_topdown(phys_addr_t size)
+{
+ phys_addr_t hole_start, hole_end, hole_size;
+ struct pkram_region_list *rl;
+ struct pkram_region *r;
+ phys_addr_t addr = 0;
+ int idx;
+
+ hole_end = memblock.current_limit;
+ r = pkram_first_region_topdown(pkram_sb, , );
+
+ while (r) {
+ hole_start = r->base + r->size;
+ hole_size = hole_end - hole_start;
+
+ if (hole_size >= size) {
+ addr = memblock_find_in_range(hole_start, hole_end,
+ size, PAGE_SIZE);
+ if (addr)
+ break;
+ }
+
+ hole_end = r->base;
+ r = pkram_next_region_topdown(, );
+ }
+
+ if (!addr)
+ addr = memblock_find_in_range(0, hole_end, size, PAGE_SIZE);
+
+ return addr;
+}
+
+int __init pkram_create_merged_reserved(struct memblock_type *new)
+{
+ unsigned long cnt_a;
+ unsigned long cnt_b;
+ long i, j, k;
+ struct memblock_region *r;
+ struct memblock_region *rgn;
+ struct pkram_region *pkr;
+ struct pkram_region_list *rl;
+ int idx;
+ unsigned long total_size = 0;
+ unsigned long nr_preserved = 0;
+
+ cnt_a = memblock.reserved.cnt;
+ cnt_b = pkram_sb->nr_regions;
+
+ i = 0;
+ j = 0;
+ k = 0;
+
+ pkr = pkram_first_region(pkram_sb, , );
+ if (!pkr)
+ return -EINVAL;
+ while (i < cnt_a && j < cnt_b && pkr) {
+ r = [i];
+ rgn = >regions[k];
+
+ if (r->base + r->size <= pkr->base) {
+ *rgn = *r;
+
[RFC v2 14/43] PKRAM: prevent inadvertent use of a stale superblock
When pages have been saved to be preserved by the current boot, set
a magic number on the super block to be validated by the next kernel.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 9 +
1 file changed, 9 insertions(+)
diff --git a/mm/pkram.c b/mm/pkram.c
index dab6657080bf..8670d1633a9d 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -22,6 +22,7 @@
#include "internal.h"
+#define PKRAM_MAGIC0x706B726D
/*
* Represents a reference to a data page saved to PKRAM.
@@ -112,6 +113,8 @@ struct pkram_region_list {
* The structure occupies a memory page.
*/
struct pkram_super_block {
+ __u32 magic;
+
__u64 node_pfn; /* first element of the node list */
__u64 region_list_pfn;
__u64 nr_regions;
@@ -180,6 +183,11 @@ void __init pkram_reserve(void)
err = PTR_ERR(pkram_sb);
goto out;
}
+ if (pkram_sb->magic != PKRAM_MAGIC) {
+ pr_err("PKRAM: invalid super block\n");
+ err = -EINVAL;
+ goto out;
+ }
/* An empty pkram_sb is not an error */
if (!pkram_sb->node_pfn) {
pkram_sb = NULL;
@@ -993,6 +1001,7 @@ static void __pkram_reboot(void)
*/
memset(pkram_sb, 0, PAGE_SIZE);
if (!err && node_pfn) {
+ pkram_sb->magic = PKRAM_MAGIC;
pkram_sb->node_pfn = node_pfn;
pkram_sb->region_list_pfn = rl_pfn;
pkram_sb->nr_regions = nr_regions;
--
1.8.3.1
[RFC v2 32/43] shmem: preserve shmem files a chunk at a time
To prepare for multithreading the work to preserve a shmem file,
divide the work into subranges of the total index range of the file.
The chunk size is a rather arbitrary 256k indices.
Signed-off-by: Anthony Yznaga
---
mm/shmem_pkram.c | 64 +---
1 file changed, 57 insertions(+), 7 deletions(-)
diff --git a/mm/shmem_pkram.c b/mm/shmem_pkram.c
index 8682b0c002c0..e52722b3a709 100644
--- a/mm/shmem_pkram.c
+++ b/mm/shmem_pkram.c
@@ -74,16 +74,14 @@ static int save_page(struct page *page, struct pkram_access
*pa)
return err;
}
-static int save_file_content(struct pkram_stream *ps, struct address_space
*mapping)
+static int save_file_content_range(struct pkram_access *pa,
+ struct address_space *mapping,
+ unsigned long start, unsigned long end)
{
- PKRAM_ACCESS(pa, ps, pages);
struct pagevec pvec;
- unsigned long start, end;
int err = 0;
int i;
- start = 0;
- end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
pagevec_init();
for ( ; ; ) {
pvec.nr = find_get_pages_range(mapping, , end,
@@ -95,7 +93,7 @@ static int save_file_content(struct pkram_stream *ps, struct
address_space *mapp
lock_page(page);
BUG_ON(page->mapping != mapping);
- err = save_page(page, );
+ err = save_page(page, pa);
if (PageCompound(page)) {
start = page->index + compound_nr(page);
i += compound_nr(page);
@@ -113,10 +111,62 @@ static int save_file_content(struct pkram_stream *ps,
struct address_space *mapp
cond_resched();
}
- pkram_finish_access(, err == 0);
return err;
}
+struct shmem_pkram_arg {
+ struct pkram_stream *ps;
+ struct address_space *mapping;
+ struct mm_struct *mm;
+ atomic64_t next;
+};
+
+unsigned long shmem_pkram_max_index_range = 512 * 512;
+
+static int get_save_range(unsigned long max, atomic64_t *next, unsigned long
*start, unsigned long *end)
+{
+ unsigned long index;
+
+ index = atomic64_fetch_add(shmem_pkram_max_index_range, next);
+ if (index >= max)
+ return -ENODATA;
+
+ *start = index;
+ *end = index + shmem_pkram_max_index_range - 1;
+
+ return 0;
+}
+
+static int do_save_file_content(struct pkram_stream *ps,
+ struct address_space *mapping,
+ atomic64_t *next)
+{
+ PKRAM_ACCESS(pa, ps, pages);
+ unsigned long start, end, max;
+ int ret;
+
+ max = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
+
+ do {
+ ret = get_save_range(max, next, , );
+ if (!ret)
+ ret = save_file_content_range(, mapping, start, end);
+ } while (!ret);
+
+ if (ret == -ENODATA)
+ ret = 0;
+
+ pkram_finish_access(, ret == 0);
+ return ret;
+}
+
+static int save_file_content(struct pkram_stream *ps, struct address_space
*mapping)
+{
+ struct shmem_pkram_arg arg = { ps, mapping, NULL, ATOMIC64_INIT(0) };
+
+ return do_save_file_content(arg.ps, arg.mapping, );
+}
+
static int save_file(struct dentry *dentry, struct pkram_stream *ps)
{
PKRAM_ACCESS(pa_bytes, ps, bytes);
--
1.8.3.1
[RFC v2 28/43] x86/mm/numa: add numa_isolate_memblocks()
Provide a way for a caller external to numa to ensure memblocks in the
memblock reserved list do not cross node boundaries and have a node id
assigned to them. This will be used by PKRAM to ensure initialization of
page structs for preserved pages can be deferred and multithreaded
efficiently.
Signed-off-by: Anthony Yznaga
---
arch/x86/include/asm/numa.h | 4
arch/x86/mm/numa.c | 32
2 files changed, 24 insertions(+), 12 deletions(-)
diff --git a/arch/x86/include/asm/numa.h b/arch/x86/include/asm/numa.h
index e3bae2b60a0d..632b5b6d8cb3 100644
--- a/arch/x86/include/asm/numa.h
+++ b/arch/x86/include/asm/numa.h
@@ -41,6 +41,7 @@ static inline void set_apicid_to_node(int apicid, s16 node)
}
extern int numa_cpu_node(int cpu);
+extern void __init numa_isolate_memblocks(void);
#else /* CONFIG_NUMA */
static inline void set_apicid_to_node(int apicid, s16 node)
@@ -51,6 +52,9 @@ static inline int numa_cpu_node(int cpu)
{
return NUMA_NO_NODE;
}
+static inline void numa_isolate_memblocks(void)
+{
+}
#endif /* CONFIG_NUMA */
#ifdef CONFIG_X86_32
diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c
index 5eb4dc2b97da..dd85098f9d72 100644
--- a/arch/x86/mm/numa.c
+++ b/arch/x86/mm/numa.c
@@ -473,6 +473,25 @@ static bool __init numa_meminfo_cover_memory(const struct
numa_meminfo *mi)
return true;
}
+void __init numa_isolate_memblocks(void)
+{
+ int i;
+
+ /*
+* Iterate over all memory known to the x86 architecture,
+* and use those ranges to set the nid in memblock.reserved.
+* This will split up the memblock regions along node
+* boundaries and will set the node IDs as well.
+*/
+ for (i = 0; i < numa_meminfo.nr_blks; i++) {
+ struct numa_memblk *mb = numa_meminfo.blk + i;
+ int ret;
+
+ ret = memblock_set_node(mb->start, mb->end - mb->start,
, mb->nid);
+ WARN_ON_ONCE(ret);
+ }
+}
+
/*
* Mark all currently memblock-reserved physical memory (which covers the
* kernel's own memory ranges) as hot-unswappable.
@@ -491,19 +510,8 @@ static void __init numa_clear_kernel_node_hotplug(void)
* used by the kernel, but those regions are not split up
* along node boundaries yet, and don't necessarily have their
* node ID set yet either.
-*
-* So iterate over all memory known to the x86 architecture,
-* and use those ranges to set the nid in memblock.reserved.
-* This will split up the memblock regions along node
-* boundaries and will set the node IDs as well.
*/
- for (i = 0; i < numa_meminfo.nr_blks; i++) {
- struct numa_memblk *mb = numa_meminfo.blk + i;
- int ret;
-
- ret = memblock_set_node(mb->start, mb->end - mb->start,
, mb->nid);
- WARN_ON_ONCE(ret);
- }
+ numa_isolate_memblocks();
/*
* Now go over all reserved memblock regions, to construct a
--
1.8.3.1
[RFC v2 24/43] mm: shmem: enable saving to PKRAM
This patch illustrates how the PKRAM API can be used for preserving tmpfs.
Two options are added to tmpfs:
The 'pkram=' option specifies the PKRAM node to load/save the
filesystem tree from/to.
The 'preserve' option initiates preservation of a read-only
filesystem tree.
If the 'pkram=' options is passed on mount, shmem will look for the
corresponding PKRAM node and load the FS tree from it.
If the 'pkram=' options was passed on mount and the 'preserve' option is
passed on remount and the filesystem is read-only, shmem will save the
FS tree to the PKRAM node.
A typical usage scenario looks like:
# mount -t tmpfs -o pkram=mytmpfs none /mnt
# echo something > /mnt/smth
# mount -o remount ro,preserve /mnt
# mount -t tmpfs -o pkram=mytmpfs none /mnt
# cat /mnt/smth
Each FS tree is saved into a PKRAM node, and each file is saved into a
PKRAM object. A byte stream written to the object is used for saving file
metadata (name, permissions, etc) while the page stream written to
the object accommodates file content pages and their offsets.
This implementation serves as a demonstration and therefore is
simplified: it supports only regular files in the root directory without
multiple hard links, and it does not save swapped out files and aborts if
any are found. However, it can be elaborated to fully support tmpfs.
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
include/linux/shmem_fs.h | 24 +++
mm/Makefile | 2 +-
mm/shmem.c | 64
mm/shmem_pkram.c | 385 +++
4 files changed, 474 insertions(+), 1 deletion(-)
create mode 100644 mm/shmem_pkram.c
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index 3f0dd95efd46..78149d702a62 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -26,6 +26,11 @@ struct shmem_inode_info {
struct inodevfs_inode;
};
+#define SHMEM_PKRAM_NAME_MAX 128
+struct shmem_pkram_info {
+ char name[SHMEM_PKRAM_NAME_MAX];
+};
+
struct shmem_sb_info {
unsigned long max_blocks; /* How many blocks are allowed */
struct percpu_counter used_blocks; /* How many are allocated */
@@ -43,6 +48,8 @@ struct shmem_sb_info {
spinlock_t shrinklist_lock; /* Protects shrinklist */
struct list_head shrinklist; /* List of shinkable inodes */
unsigned long shrinklist_len; /* Length of shrinklist */
+ struct shmem_pkram_info *pkram;
+ bool preserve; /* PKRAM-enabled data is preserved */
};
static inline struct shmem_inode_info *SHMEM_I(struct inode *inode)
@@ -106,6 +113,23 @@ extern int shmem_getpage(struct inode *inode, pgoff_t
index,
extern int shmem_insert_page(struct mm_struct *mm, struct inode *inode,
pgoff_t index, struct page *page);
+#ifdef CONFIG_PKRAM
+extern int shmem_parse_pkram(const char *str, struct shmem_pkram_info **pkram);
+extern void shmem_show_pkram(struct seq_file *seq, struct shmem_pkram_info
*pkram,
+ bool preserve);
+extern int shmem_save_pkram(struct super_block *sb);
+extern void shmem_load_pkram(struct super_block *sb);
+extern int shmem_release_pkram(struct super_block *sb);
+#else
+static inline int shmem_parse_pkram(const char *str,
+ struct shmem_pkram_info **pkram) { return 1; }
+static inline void shmem_show_pkram(struct seq_file *seq,
+ struct shmem_pkram_info *pkram, bool preserve) { }
+static inline int shmem_save_pkram(struct super_block *sb) { return 0; }
+static inline void shmem_load_pkram(struct super_block *sb) { }
+static inline int shmem_release_pkram(struct super_block *sb) { return 0; }
+#endif
+
static inline struct page *shmem_read_mapping_page(
struct address_space *mapping, pgoff_t index)
{
diff --git a/mm/Makefile b/mm/Makefile
index f5c0dd0a3707..a4e9dd5545df 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -120,4 +120,4 @@ obj-$(CONFIG_MEMFD_CREATE) += memfd.o
obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o
obj-$(CONFIG_PTDUMP_CORE) += ptdump.o
obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o
-obj-$(CONFIG_PKRAM) += pkram.o pkram_pagetable.o
+obj-$(CONFIG_PKRAM) += pkram.o pkram_pagetable.o shmem_pkram.o
diff --git a/mm/shmem.c b/mm/shmem.c
index 60e4f0ad23b9..c1c5760465f2 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -111,16 +111,20 @@ struct shmem_options {
unsigned long long blocks;
unsigned long long inodes;
struct mempolicy *mpol;
+ struct shmem_pkram_info *pkram;
kuid_t uid;
kgid_t gid;
umode_t mode;
bool full_inums;
+ bool preserve;
int huge;
int seen;
#define SHMEM_SEEN_BLOCKS 1
#define SHMEM_SEEN_INODES 2
#define SHMEM_SEEN_HUGE 4
#define SHMEM_SEEN_INUMS 8
+#define SHMEM_SEEN_PKRAM 16
+#define SHMEM_SEEN_PRESERVE 32
};
#ifdef CONFIG_TMPFS
@@ -344
[RFC v2 23/43] mm: shmem: introduce shmem_insert_page
The function inserts a page into a shmem file at a specified offset.
The page can be a regular PAGE_SIZE page or a transparent huge page.
If there is something at the offset (page or swap), the function fails.
The function will be used by the next patch.
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
include/linux/shmem_fs.h | 3 ++
mm/shmem.c | 77
2 files changed, 80 insertions(+)
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index d82b6f396588..3f0dd95efd46 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -103,6 +103,9 @@ enum sgp_type {
extern int shmem_getpage(struct inode *inode, pgoff_t index,
struct page **pagep, enum sgp_type sgp);
+extern int shmem_insert_page(struct mm_struct *mm, struct inode *inode,
+ pgoff_t index, struct page *page);
+
static inline struct page *shmem_read_mapping_page(
struct address_space *mapping, pgoff_t index)
{
diff --git a/mm/shmem.c b/mm/shmem.c
index b2db4ed0fbc7..60e4f0ad23b9 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -755,6 +755,83 @@ static void shmem_delete_from_page_cache(struct page
*page, void *radswap)
BUG_ON(error);
}
+int shmem_insert_page(struct mm_struct *mm, struct inode *inode, pgoff_t index,
+ struct page *page)
+{
+ struct address_space *mapping = inode->i_mapping;
+ struct shmem_inode_info *info = SHMEM_I(inode);
+ struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
+ gfp_t gfp = mapping_gfp_mask(mapping);
+ int err;
+ int nr;
+ pgoff_t hindex = index;
+ bool on_lru = PageLRU(page);
+
+ if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
+ return -EFBIG;
+
+ nr = thp_nr_pages(page);
+retry:
+ err = 0;
+ if (!shmem_inode_acct_block(inode, nr))
+ err = -ENOSPC;
+ if (err) {
+ int retry = 5;
+
+ /*
+* Try to reclaim some space by splitting a huge page
+* beyond i_size on the filesystem.
+*/
+ while (retry--) {
+ int ret;
+
+ ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
+ if (ret == SHRINK_STOP)
+ break;
+ if (ret)
+ goto retry;
+ }
+ goto failed;
+ }
+
+ if (!on_lru) {
+ __SetPageLocked(page);
+ __SetPageSwapBacked(page);
+ } else {
+ lock_page(page);
+ }
+
+ hindex = round_down(index, nr);
+ __SetPageReferenced(page);
+
+ err = shmem_add_to_page_cache(page, mapping, hindex,
+ NULL, gfp & GFP_RECLAIM_MASK, mm);
+ if (err)
+ goto out_unlock;
+
+ if (!on_lru)
+ lru_cache_add(page);
+
+ spin_lock(>lock);
+ info->alloced += nr;
+ inode->i_blocks += BLOCKS_PER_PAGE << thp_order(page);
+ shmem_recalc_inode(inode);
+ spin_unlock(>lock);
+
+ flush_dcache_page(page);
+ SetPageUptodate(page);
+ set_page_dirty(page);
+
+ unlock_page(page);
+ return 0;
+
+out_unlock:
+ unlock_page(page);
+ shmem_inode_unacct_blocks(inode, nr);
+failed:
+ return err;
+}
+
/*
* Remove swap entry from page cache, free the swap and its page cache.
*/
--
1.8.3.1
[RFC v2 22/43] x86/boot/compressed/64: use 1GB pages for mappings
pkram kaslr code can incur multiple page faults when it walks its preserved ranges list called via mem_avoid_overlap(). The multiple faults can easily end up using up the small number of pages available to be allocated for page table pages. This patch hacks things so that mappings are 1GB which results in the need for far fewer page table pages. As is this breaks AMD SEV-ES which expects the mappings to be 2M. This could possibly be fixed by updating split code to split 1GB page if the aren't any other issues with using 1GB mappings. Signed-off-by: Anthony Yznaga --- arch/x86/boot/compressed/ident_map_64.c | 9 + 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index f7213d0943b8..6ff02da4cc1a 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -95,8 +95,8 @@ static void add_identity_map(unsigned long start, unsigned long end) int ret; /* Align boundary to 2M. */ - start = round_down(start, PMD_SIZE); - end = round_up(end, PMD_SIZE); + start = round_down(start, PUD_SIZE); + end = round_up(end, PUD_SIZE); if (start >= end) return; @@ -119,6 +119,7 @@ void initialize_identity_maps(void *rmode) mapping_info.context = _data; mapping_info.page_flag = __PAGE_KERNEL_LARGE_EXEC | sme_me_mask; mapping_info.kernpg_flag = _KERNPG_TABLE; + mapping_info.direct_gbpages = true; /* * It should be impossible for this not to already be true, @@ -329,8 +330,8 @@ void do_boot_page_fault(struct pt_regs *regs, unsigned long error_code) ghcb_fault = sev_es_check_ghcb_fault(address); - address &= PMD_MASK; - end= address + PMD_SIZE; + address &= PUD_MASK; + end= address + PUD_SIZE; /* * Check for unexpected error codes. Unexpected are: -- 1.8.3.1
[RFC v2 26/43] mm: shmem: specify the mm to use when inserting pages
Explicitly specify the mm to pass to shmem_insert_page() when
the pkram_stream is initialized rather than use the mm of the
current thread. This will allow for multiple kernel threads to
target the same mm when inserting pages in parallel.
Signed-off-by: Anthony Yznaga
---
mm/shmem_pkram.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/mm/shmem_pkram.c b/mm/shmem_pkram.c
index 904b1b861ce5..8682b0c002c0 100644
--- a/mm/shmem_pkram.c
+++ b/mm/shmem_pkram.c
@@ -225,7 +225,7 @@ int shmem_save_pkram(struct super_block *sb)
return err;
}
-static int load_file_content(struct pkram_stream *ps, struct address_space
*mapping)
+static int load_file_content(struct pkram_stream *ps, struct address_space
*mapping, struct mm_struct *mm)
{
PKRAM_ACCESS(pa, ps, pages);
unsigned long index;
@@ -237,7 +237,7 @@ static int load_file_content(struct pkram_stream *ps,
struct address_space *mapp
if (!page)
break;
- err = shmem_insert_page(current->mm, mapping->host, index,
page);
+ err = shmem_insert_page(mm, mapping->host, index, page);
put_page(page);
cond_resched();
} while (!err);
@@ -291,7 +291,7 @@ static int load_file(struct dentry *parent, struct
pkram_stream *ps,
inode->i_ctime = ns_to_timespec64(hdr.ctime);
i_size_write(inode, hdr.size);
- err = load_file_content(ps, inode->i_mapping);
+ err = load_file_content(ps, inode->i_mapping, current->mm);
out_unlock:
inode_unlock(d_inode(parent));
out:
--
1.8.3.1
[RFC v2 18/43] kexec: PKRAM: avoid clobbering already preserved pages
Ensure destination ranges of the kexec segments do not overlap
with any kernel pages marked to be preserved across kexec.
For kexec_load, return EADDRNOTAVAIL if overlap is detected.
For kexec_file_load, skip ranges containing preserved pages when
seaching for available ranges to use.
Signed-off-by: Anthony Yznaga
---
kernel/kexec_core.c | 3 +++
kernel/kexec_file.c | 5 +
2 files changed, 8 insertions(+)
diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c
index a0b6780740c8..fda4abb865ff 100644
--- a/kernel/kexec_core.c
+++ b/kernel/kexec_core.c
@@ -37,6 +37,7 @@
#include
#include
#include
+#include
#include
#include
@@ -175,6 +176,8 @@ int sanity_check_segment_list(struct kimage *image)
return -EADDRNOTAVAIL;
if (mend >= KEXEC_DESTINATION_MEMORY_LIMIT)
return -EADDRNOTAVAIL;
+ if (pkram_has_preserved_pages(mstart, mend))
+ return -EADDRNOTAVAIL;
}
/* Verify our destination addresses do not overlap.
diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c
index 1ec47a3c60dd..94109bcdbeff 100644
--- a/kernel/kexec_file.c
+++ b/kernel/kexec_file.c
@@ -516,6 +516,11 @@ static int locate_mem_hole_bottom_up(unsigned long start,
unsigned long end,
continue;
}
+ if (pkram_has_preserved_pages(temp_start, temp_end + 1)) {
+ temp_start = temp_start - PAGE_SIZE;
+ continue;
+ }
+
/* We found a suitable memory range */
break;
} while (1);
--
1.8.3.1
[RFC v2 10/43] PKRAM: pass a list of preserved ranges to the next kernel
In order to build a new memblock reserved list during boot that
includes ranges preserved by the previous kernel, a list of preserved
ranges is passed to the next kernel via the pkram superblock. The
ranges are stored in ascending order in a linked list of pages. A more
complete memblock list is not prepared to avoid possible conflicts with
changes in a newer kernel and to avoid having to allocate a contiguous
range larger than a page.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 183 ++---
1 file changed, 176 insertions(+), 7 deletions(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index a9e6cd8ca084..4cfa236a4126 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -86,6 +86,20 @@ struct pkram_node {
#define PKRAM_LOAD 2
#define PKRAM_ACCMODE_MASK 3
+struct pkram_region {
+ phys_addr_t base;
+ phys_addr_t size;
+};
+
+struct pkram_region_list {
+ __u64 prev_pfn;
+ __u64 next_pfn;
+
+ struct pkram_region regions[0];
+};
+
+#define PKRAM_REGIONS_LIST_MAX \
+ ((PAGE_SIZE-sizeof(struct pkram_region_list))/sizeof(struct
pkram_region))
/*
* The PKRAM super block contains data needed to restore the preserved memory
* structure on boot. The pointer to it (pfn) should be passed via the 'pkram'
@@ -98,13 +112,20 @@ struct pkram_node {
*/
struct pkram_super_block {
__u64 node_pfn; /* first element of the node list */
+ __u64 region_list_pfn;
+ __u64 nr_regions;
};
+static struct pkram_region_list *pkram_regions_list;
+static int pkram_init_regions_list(void);
+static unsigned long pkram_populate_regions_list(void);
+
static unsigned long pkram_sb_pfn __initdata;
static struct pkram_super_block *pkram_sb;
extern int pkram_add_identity_map(struct page *page);
extern void pkram_remove_identity_map(struct page *page);
+extern void pkram_find_preserved(unsigned long start, unsigned long end, void
*private, int (*callback)(unsigned long base, unsigned long size, void
*private));
/*
* For convenience sake PKRAM nodes are kept in an auxiliary doubly-linked list
@@ -862,21 +883,48 @@ static void __pkram_reboot(void)
struct page *page;
struct pkram_node *node;
unsigned long node_pfn = 0;
+ unsigned long rl_pfn = 0;
+ unsigned long nr_regions = 0;
+ int err = 0;
- list_for_each_entry_reverse(page, _nodes, lru) {
- node = page_address(page);
- if (WARN_ON(node->flags & PKRAM_ACCMODE_MASK))
- continue;
- node->node_pfn = node_pfn;
- node_pfn = page_to_pfn(page);
+ if (!list_empty(_nodes)) {
+ err = pkram_add_identity_map(virt_to_page(pkram_sb));
+ if (err) {
+ pr_err("PKRAM: failed to add super block to
pagetable\n");
+ goto done;
+ }
+ list_for_each_entry_reverse(page, _nodes, lru) {
+ node = page_address(page);
+ if (WARN_ON(node->flags & PKRAM_ACCMODE_MASK))
+ continue;
+ node->node_pfn = node_pfn;
+ node_pfn = page_to_pfn(page);
+ }
+ err = pkram_init_regions_list();
+ if (err) {
+ pr_err("PKRAM: failed to init regions list\n");
+ goto done;
+ }
+ nr_regions = pkram_populate_regions_list();
+ if (IS_ERR_VALUE(nr_regions)) {
+ err = nr_regions;
+ pr_err("PKRAM: failed to populate regions list\n");
+ goto done;
+ }
+ rl_pfn = page_to_pfn(virt_to_page(pkram_regions_list));
}
+done:
/*
* Zero out pkram_sb completely since it may have been passed from
* the previous boot.
*/
memset(pkram_sb, 0, PAGE_SIZE);
- pkram_sb->node_pfn = node_pfn;
+ if (!err && node_pfn) {
+ pkram_sb->node_pfn = node_pfn;
+ pkram_sb->region_list_pfn = rl_pfn;
+ pkram_sb->nr_regions = nr_regions;
+ }
}
static int pkram_reboot(struct notifier_block *notifier,
@@ -952,3 +1000,124 @@ static int __init pkram_init(void)
return 0;
}
module_init(pkram_init);
+
+static int count_region_cb(unsigned long base, unsigned long size, void
*private)
+{
+ unsigned long *nr_regions = (unsigned long *)private;
+
+ (*nr_regions)++;
+ return 0;
+}
+
+static unsigned long pkram_count_regions(void)
+{
+ unsigned long nr_regions = 0;
+
+ pkram_find_preserved(0, PHYS_ADDR_MAX, _regions, count_region_cb);
+
+ return nr_regions;
+}
+
+/*
+ * To faciliate rapidly building a new memblock reserved list during boot
+
[RFC v2 19/43] mm: PKRAM: allow preserved memory to be freed from userspace
To free all space utilized for preserved memory, one can write 0 to
/sys/kernel/pkram. This will destroy all PKRAM nodes that are not
currently being read or written.
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 39 ++-
1 file changed, 38 insertions(+), 1 deletion(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index dcf84ba785a7..8700fd77dc67 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -493,6 +493,32 @@ static void pkram_truncate_node(struct pkram_node *node)
node->obj_pfn = 0;
}
+/*
+ * Free all nodes that are not under operation.
+ */
+static void pkram_truncate(void)
+{
+ struct page *page, *tmp;
+ struct pkram_node *node;
+ LIST_HEAD(dispose);
+
+ mutex_lock(_mutex);
+ list_for_each_entry_safe(page, tmp, _nodes, lru) {
+ node = page_address(page);
+ if (!(node->flags & PKRAM_ACCMODE_MASK))
+ list_move(>lru, );
+ }
+ mutex_unlock(_mutex);
+
+ while (!list_empty()) {
+ page = list_first_entry(, struct page, lru);
+ list_del(>lru);
+ node = page_address(page);
+ pkram_truncate_node(node);
+ pkram_free_page(node);
+ }
+}
+
static void pkram_add_link(struct pkram_link *link, struct pkram_data_stream
*pds)
{
__u64 link_pfn = page_to_pfn(virt_to_page(link));
@@ -1233,8 +1259,19 @@ static ssize_t show_pkram_sb_pfn(struct kobject *kobj,
return sprintf(buf, "%lx\n", pfn);
}
+static ssize_t store_pkram_sb_pfn(struct kobject *kobj,
+ struct kobj_attribute *attr, const char *buf, size_t count)
+{
+ int val;
+
+ if (kstrtoint(buf, 0, ) || val)
+ return -EINVAL;
+ pkram_truncate();
+ return count;
+}
+
static struct kobj_attribute pkram_sb_pfn_attr =
- __ATTR(pkram, 0444, show_pkram_sb_pfn, NULL);
+ __ATTR(pkram, 0644, show_pkram_sb_pfn, store_pkram_sb_pfn);
static struct attribute *pkram_attrs[] = {
_sb_pfn_attr.attr,
--
1.8.3.1
[RFC v2 21/43] x86/KASLR: PKRAM: support physical kaslr
Avoid regions of memory that contain preserved pages when computing
slots used to select where to put the decompressed kernel.
Signed-off-by: Anthony Yznaga
---
arch/x86/boot/compressed/Makefile | 3 ++
arch/x86/boot/compressed/kaslr.c | 10 +++-
arch/x86/boot/compressed/misc.h | 10
arch/x86/boot/compressed/pkram.c | 109 ++
4 files changed, 130 insertions(+), 2 deletions(-)
create mode 100644 arch/x86/boot/compressed/pkram.c
diff --git a/arch/x86/boot/compressed/Makefile
b/arch/x86/boot/compressed/Makefile
index e0bc3988c3fa..ef27d411b641 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -93,6 +93,9 @@ ifdef CONFIG_X86_64
vmlinux-objs-y += $(obj)/mem_encrypt.o
vmlinux-objs-y += $(obj)/pgtable_64.o
vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev-es.o
+ifdef CONFIG_RANDOMIZE_BASE
+ vmlinux-objs-$(CONFIG_PKRAM) += $(obj)/pkram.o
+endif
endif
vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o
diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c
index b92fffbe761f..a007363a7698 100644
--- a/arch/x86/boot/compressed/kaslr.c
+++ b/arch/x86/boot/compressed/kaslr.c
@@ -440,6 +440,7 @@ static bool mem_avoid_overlap(struct mem_vector *img,
struct setup_data *ptr;
u64 earliest = img->start + img->size;
bool is_overlapping = false;
+ struct mem_vector avoid;
for (i = 0; i < MEM_AVOID_MAX; i++) {
if (mem_overlaps(img, _avoid[i]) &&
@@ -453,8 +454,6 @@ static bool mem_avoid_overlap(struct mem_vector *img,
/* Avoid all entries in the setup_data linked list. */
ptr = (struct setup_data *)(unsigned long)boot_params->hdr.setup_data;
while (ptr) {
- struct mem_vector avoid;
-
avoid.start = (unsigned long)ptr;
avoid.size = sizeof(*ptr) + ptr->len;
@@ -479,6 +478,12 @@ static bool mem_avoid_overlap(struct mem_vector *img,
ptr = (struct setup_data *)(unsigned long)ptr->next;
}
+ if (pkram_has_overlap(img, ) && (avoid.start < earliest)) {
+ *overlap = avoid;
+ earliest = overlap->start;
+ is_overlapping = true;
+ }
+
return is_overlapping;
}
@@ -840,6 +845,7 @@ void choose_random_location(unsigned long input,
return;
}
+ pkram_init();
boot_params->hdr.loadflags |= KASLR_FLAG;
if (IS_ENABLED(CONFIG_X86_32))
diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h
index 901ea5ebec22..f8232ffd8141 100644
--- a/arch/x86/boot/compressed/misc.h
+++ b/arch/x86/boot/compressed/misc.h
@@ -116,6 +116,16 @@ static inline void console_init(void)
{ }
#endif
+#ifdef CONFIG_PKRAM
+void pkram_init(void);
+int pkram_has_overlap(struct mem_vector *entry, struct mem_vector *overlap);
+#else
+static inline void pkram_init(void) { }
+static inline int pkram_has_overlap(struct mem_vector *entry,
+ struct mem_vector *overlap);
+{ return 0; }
+#endif
+
void set_sev_encryption_mask(void);
#ifdef CONFIG_AMD_MEM_ENCRYPT
diff --git a/arch/x86/boot/compressed/pkram.c b/arch/x86/boot/compressed/pkram.c
new file mode 100644
index ..60380f074c3f
--- /dev/null
+++ b/arch/x86/boot/compressed/pkram.c
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "misc.h"
+
+#define PKRAM_MAGIC0x706B726D
+
+struct pkram_super_block {
+ __u32 magic;
+
+ __u64 node_pfn;
+ __u64 region_list_pfn;
+ __u64 nr_regions;
+};
+
+struct pkram_region {
+ phys_addr_t base;
+ phys_addr_t size;
+};
+
+struct pkram_region_list {
+ __u64 prev_pfn;
+ __u64 next_pfn;
+
+ struct pkram_region regions[0];
+};
+
+#define PKRAM_REGIONS_LIST_MAX \
+ ((PAGE_SIZE-sizeof(struct pkram_region_list))/sizeof(struct
pkram_region))
+
+static u64 pkram_sb_pfn;
+static struct pkram_super_block *pkram_sb;
+
+void pkram_init(void)
+{
+ struct pkram_super_block *sb;
+ char arg[32];
+
+ if (cmdline_find_option("pkram", arg, sizeof(arg)) > 0) {
+ if (kstrtoull(arg, 16, _sb_pfn) != 0)
+ return;
+ } else
+ return;
+
+ sb = (struct pkram_super_block *)(pkram_sb_pfn << PAGE_SHIFT);
+ if (sb->magic != PKRAM_MAGIC) {
+ debug_putstr("PKRAM: invalid super block\n");
+ return;
+ }
+
+ pkram_sb = sb;
+}
+
+static struct pkram_region *pkram_first_region(struct pkram_super_block *sb,
struct pkram_region_list **rlp, int *idx)
+{
+ if (!sb || !sb->region_list_pfn)
+ return NULL;
+
+ *rlp = (struct pkram_region_list *)(sb->region_list_pfn << PAGE_SHIFT);
+ *idx = 0;
+
+ return
[RFC v2 09/43] PKRAM: track preserved pages in a physical mapping pagetable
Later patches in this series will need a way to efficiently identify
physically contiguous ranges of preserved pages independent of their
virtual addresses. To facilitate this all pages to be preserved across
kexec are added to a pseudo identity mapping pagetable.
The pagetable makes use of the existing architecture definitions for
building a memory mapping pagetable except that a bitmap is used to
represent the presence or absence of preserved pages at the PTE level.
Signed-off-by: Anthony Yznaga
---
mm/Makefile | 2 +-
mm/pkram.c | 30 +++-
mm/pkram_pagetable.c | 376 +++
3 files changed, 404 insertions(+), 4 deletions(-)
create mode 100644 mm/pkram_pagetable.c
diff --git a/mm/Makefile b/mm/Makefile
index ab3a724769b5..f5c0dd0a3707 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -120,4 +120,4 @@ obj-$(CONFIG_MEMFD_CREATE) += memfd.o
obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o
obj-$(CONFIG_PTDUMP_CORE) += ptdump.o
obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o
-obj-$(CONFIG_PKRAM) += pkram.o
+obj-$(CONFIG_PKRAM) += pkram.o pkram_pagetable.o
diff --git a/mm/pkram.c b/mm/pkram.c
index 2809371a9aec..a9e6cd8ca084 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -103,6 +103,9 @@ struct pkram_super_block {
static unsigned long pkram_sb_pfn __initdata;
static struct pkram_super_block *pkram_sb;
+extern int pkram_add_identity_map(struct page *page);
+extern void pkram_remove_identity_map(struct page *page);
+
/*
* For convenience sake PKRAM nodes are kept in an auxiliary doubly-linked list
* connected through the lru field of the page struct.
@@ -121,11 +124,24 @@ static int __init parse_pkram_sb_pfn(char *arg)
static inline struct page *pkram_alloc_page(gfp_t gfp_mask)
{
- return alloc_page(gfp_mask);
+ struct page *page;
+ int err;
+
+ page = alloc_page(gfp_mask);
+ if (page) {
+ err = pkram_add_identity_map(page);
+ if (err) {
+ __free_page(page);
+ page = NULL;
+ }
+ }
+
+ return page;
}
static inline void pkram_free_page(void *addr)
{
+ pkram_remove_identity_map(virt_to_page(addr));
free_page((unsigned long)addr);
}
@@ -163,6 +179,7 @@ static void pkram_truncate_link(struct pkram_link *link)
if (!p)
continue;
page = pfn_to_page(PHYS_PFN(p));
+ pkram_remove_identity_map(page);
put_page(page);
}
}
@@ -615,10 +632,15 @@ static int __pkram_save_page(struct pkram_access *pa,
struct page *page,
int pkram_save_file_page(struct pkram_access *pa, struct page *page)
{
struct pkram_node *node = pa->ps->node;
+ int err;
BUG_ON((node->flags & PKRAM_ACCMODE_MASK) != PKRAM_SAVE);
- return __pkram_save_page(pa, page, page->index);
+ err = __pkram_save_page(pa, page, page->index);
+ if (!err)
+ err = pkram_add_identity_map(page);
+
+ return err;
}
static int __pkram_bytes_save_page(struct pkram_access *pa, struct page *page)
@@ -652,6 +674,8 @@ static struct page *__pkram_prep_load_page(pkram_entry_t p)
prep_transhuge_page(page);
}
+ pkram_remove_identity_map(page);
+
return page;
}
@@ -898,7 +922,7 @@ static int __init pkram_init_sb(void)
if (!pkram_sb) {
struct page *page;
- page = pkram_alloc_page(GFP_KERNEL | __GFP_ZERO);
+ page = alloc_page(GFP_KERNEL | __GFP_ZERO);
if (!page) {
pr_err("PKRAM: Failed to allocate super block\n");
return 0;
diff --git a/mm/pkram_pagetable.c b/mm/pkram_pagetable.c
new file mode 100644
index ..9c5443bd7686
--- /dev/null
+++ b/mm/pkram_pagetable.c
@@ -0,0 +1,376 @@
+// SPDX-License-Identifier: GPL-2.0
+#include
+//#include
+#include
+
+static pgd_t *pkram_pgd;
+static DEFINE_SPINLOCK(pkram_pgd_lock);
+
+#define set_p4d(p4dp, p4d) WRITE_ONCE(*(p4dp), (p4d))
+
+#define PKRAM_PTE_BM_BYTES (PTRS_PER_PTE / BITS_PER_BYTE)
+#define PKRAM_PTE_BM_MASK (PAGE_SIZE / PKRAM_PTE_BM_BYTES - 1)
+
+static pmd_t make_bitmap_pmd(unsigned long *bitmap)
+{
+ unsigned long val;
+
+ val = __pa(ALIGN_DOWN((unsigned long)bitmap, PAGE_SIZE));
+ val |= (((unsigned long)bitmap & ~PAGE_MASK) / PKRAM_PTE_BM_BYTES);
+
+ return __pmd(val);
+}
+
+static unsigned long *get_bitmap_addr(pmd_t pmd)
+{
+ unsigned long val, off;
+
+ val = pmd_val(pmd);
+ off = (val & PKRAM_PTE_BM_MASK) * PKRAM_PTE_BM_BYTES;
+
+ val = (val & PAGE_MASK) + off;
+
+ return __va(val);
+}
+
+int pkram_add_identity_map(struct page *page)
+{
+ unsigned long paddr;
+ unsigned long *bitmap;
+ unsigned int index;
+ struct page *p
[RFC v2 17/43] PKRAM: provide a way to check if a memory range has preserved pages
When a kernel is loaded for kexec the address ranges where the kexec
segments will be copied to may conflict with pages already set to be
preserved. Provide a way to determine if preserved pages exist in a
specified range.
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 2 ++
mm/pkram.c| 20
2 files changed, 22 insertions(+)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index 97a7c2ac44a9..977cf45a1bcf 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -104,11 +104,13 @@ int pkram_prepare_save(struct pkram_stream *ps, const
char *name,
void pkram_reserve(void);
void pkram_cleanup(void);
void pkram_ban_region(unsigned long start, unsigned long end);
+int pkram_has_preserved_pages(unsigned long start, unsigned long end);
#else
#define pkram_reserved_pages 0UL
static inline void pkram_reserve(void) { }
static inline void pkram_cleanup(void) { }
static inline void pkram_ban_region(unsigned long start, unsigned long end) { }
+static inline int pkram_has_preserved_pages(unsigned long start, unsigned long
end) { return 0; }
#endif
#endif /* _LINUX_PKRAM_H */
diff --git a/mm/pkram.c b/mm/pkram.c
index d15be75c1032..dcf84ba785a7 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -1668,3 +1668,23 @@ void __init pkram_cleanup(void)
pkram_reserved_pages--;
}
}
+
+static int has_preserved_pages_cb(unsigned long base, unsigned long size, void
*private)
+{
+ int *has_preserved = (int *)private;
+
+ *has_preserved = 1;
+ return 1;
+}
+
+/*
+ * Check whether the memory range [start, end) contains preserved pages.
+ */
+int pkram_has_preserved_pages(unsigned long start, unsigned long end)
+{
+ int has_preserved = 0;
+
+ pkram_find_preserved(start, end, _preserved,
has_preserved_pages_cb);
+
+ return has_preserved;
+}
--
1.8.3.1
[RFC v2 16/43] kexec: PKRAM: prevent kexec clobbering preserved pages in some cases
When loading a kernel for kexec, dynamically update the list of physical
ranges that are not to be used for storing preserved pages with the ranges
where kexec segments will be copied to on reboot. This ensures no pages
preserved after the new kernel has been loaded will reside in these ranges
on reboot.
Not yet handled is the case where pages have been preserved before a
kexec kernel is loaded. This will be covered by a later patch.
Signed-off-by: Anthony Yznaga
---
kernel/kexec.c | 9 +
kernel/kexec_file.c | 10 ++
2 files changed, 19 insertions(+)
diff --git a/kernel/kexec.c b/kernel/kexec.c
index c82c6c06f051..826c8fb824d8 100644
--- a/kernel/kexec.c
+++ b/kernel/kexec.c
@@ -16,6 +16,7 @@
#include
#include
#include
+#include
#include "kexec_internal.h"
@@ -163,6 +164,14 @@ static int do_kexec_load(unsigned long entry, unsigned
long nr_segments,
if (ret)
goto out;
+ for (i = 0; i < nr_segments; i++) {
+ unsigned long mem = image->segment[i].mem;
+ size_t memsz = image->segment[i].memsz;
+
+ if (memsz)
+ pkram_ban_region(PFN_DOWN(mem), PFN_UP(mem + memsz) -
1);
+ }
+
/* Install the new kernel and uninstall the old */
image = xchg(dest_image, image);
diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c
index 5c3447cf7ad5..1ec47a3c60dd 100644
--- a/kernel/kexec_file.c
+++ b/kernel/kexec_file.c
@@ -27,6 +27,8 @@
#include
#include
#include
+#include
+
#include "kexec_internal.h"
static int kexec_calculate_store_digests(struct kimage *image);
@@ -429,6 +431,14 @@ void kimage_file_post_load_cleanup(struct kimage *image)
if (ret)
goto out;
+ for (i = 0; i < image->nr_segments; i++) {
+ unsigned long mem = image->segment[i].mem;
+ size_t memsz = image->segment[i].memsz;
+
+ if (memsz)
+ pkram_ban_region(PFN_DOWN(mem), PFN_UP(mem + memsz) -
1);
+ }
+
/*
* Free up any temporary buffers allocated which are not needed
* after image has been loaded
--
1.8.3.1
[RFC v2 13/43] PKRAM: free the preserved ranges list
Free the pages used to pass the preserved ranges to the new boot.
Signed-off-by: Anthony Yznaga
---
arch/x86/mm/init_64.c | 1 +
include/linux/pkram.h | 2 ++
mm/pkram.c| 20
3 files changed, 23 insertions(+)
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 8efb2fb2a88b..69bd71996b8b 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1294,6 +1294,7 @@ void __init mem_init(void)
after_bootmem = 1;
x86_init.hyper.init_after_bootmem();
+ pkram_cleanup();
totalram_pages_add(pkram_reserved_pages);
/*
* Must be done after boot memory is put on freelist, because here we
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index 8d3d780d9fe1..c2099a4f2004 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -102,9 +102,11 @@ int pkram_prepare_save(struct pkram_stream *ps, const char
*name,
#ifdef CONFIG_PKRAM
extern unsigned long pkram_reserved_pages;
void pkram_reserve(void);
+void pkram_cleanup(void);
#else
#define pkram_reserved_pages 0UL
static inline void pkram_reserve(void) { }
+static inline void pkram_cleanup(void) { }
#endif
#endif /* _LINUX_PKRAM_H */
diff --git a/mm/pkram.c b/mm/pkram.c
index 03731bb6af26..dab6657080bf 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -1434,3 +1434,23 @@ int __init pkram_merge_with_reserved(void)
return 0;
}
+
+void __init pkram_cleanup(void)
+{
+ struct pkram_region_list *rl;
+ unsigned long next_pfn;
+
+ if (!pkram_sb || !pkram_reserved_pages)
+ return;
+
+ next_pfn = pkram_sb->region_list_pfn;
+
+ while (next_pfn) {
+ struct page *page = pfn_to_page(next_pfn);
+
+ rl = pfn_to_kaddr(next_pfn);
+ next_pfn = rl->next_pfn;
+ __free_pages_core(page, 0);
+ pkram_reserved_pages--;
+ }
+}
--
1.8.3.1
[RFC v2 07/43] mm: PKRAM: link nodes by pfn before reboot
Since page structs are used for linking PKRAM nodes and cleared
on boot, organize all PKRAM nodes into a list singly-linked by pfns
before reboot to facilitate restoring the node list in the new kernel.
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 50 ++
1 file changed, 50 insertions(+)
diff --git a/mm/pkram.c b/mm/pkram.c
index d81af26c9a66..975f200aef38 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -2,12 +2,16 @@
#include
#include
#include
+#include
#include
#include
#include
#include
+#include
#include
+#include
#include
+#include
#include
#include
#include
@@ -62,11 +66,15 @@ struct pkram_obj {
* singly-linked list of PKRAM link structures (see above), the node has a
* pointer to the head of.
*
+ * To facilitate data restore in the new kernel, before reboot all PKRAM nodes
+ * are organized into a list singly-linked by pfn's (see pkram_reboot()).
+ *
* The structure occupies a memory page.
*/
struct pkram_node {
__u32 flags;
__u64 obj_pfn;/* points to the first obj of the node */
+ __u64 node_pfn; /* points to the next node in the node list */
__u8name[PKRAM_NAME_MAX];
};
@@ -75,6 +83,10 @@ struct pkram_node {
#define PKRAM_LOAD 2
#define PKRAM_ACCMODE_MASK 3
+/*
+ * For convenience sake PKRAM nodes are kept in an auxiliary doubly-linked list
+ * connected through the lru field of the page struct.
+ */
static LIST_HEAD(pkram_nodes); /* linked through page::lru */
static DEFINE_MUTEX(pkram_mutex); /* serializes open/close */
@@ -780,3 +792,41 @@ size_t pkram_read(struct pkram_access *pa, void *buf,
size_t count)
}
return read_count;
}
+
+/*
+ * Build the list of PKRAM nodes.
+ */
+static void __pkram_reboot(void)
+{
+ struct page *page;
+ struct pkram_node *node;
+ unsigned long node_pfn = 0;
+
+ list_for_each_entry_reverse(page, _nodes, lru) {
+ node = page_address(page);
+ if (WARN_ON(node->flags & PKRAM_ACCMODE_MASK))
+ continue;
+ node->node_pfn = node_pfn;
+ node_pfn = page_to_pfn(page);
+ }
+}
+
+static int pkram_reboot(struct notifier_block *notifier,
+ unsigned long val, void *v)
+{
+ if (val != SYS_RESTART)
+ return NOTIFY_DONE;
+ __pkram_reboot();
+ return NOTIFY_OK;
+}
+
+static struct notifier_block pkram_reboot_notifier = {
+ .notifier_call = pkram_reboot,
+};
+
+static int __init pkram_init(void)
+{
+ register_reboot_notifier(_reboot_notifier);
+ return 0;
+}
+module_init(pkram_init);
--
1.8.3.1
[RFC v2 02/43] mm: PKRAM: implement node load and save functions
Preserved memory is divided into nodes which can be saved and loaded
independently of each other. PKRAM nodes are kept on a list and
identified by unique names. Whenever a save operation is initiated by
calling pkram_prepare_save(), a new node is created and linked to the
list. When the save operation has been committed by calling
pkram_finish_save(), the node becomes loadable. A load operation can be
then initiated by calling pkram_prepare_load() which deletes the node
from the list and prepares the corresponding stream for loading data
from it. After the load has been finished, the pkram_finish_load()
function must be called to free the node. Nodes are also deleted when a
save operation is discarded, i.e. pkram_discard_save() is called instead
of pkram_finish_save().
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 8 ++-
mm/pkram.c| 148 --
2 files changed, 150 insertions(+), 6 deletions(-)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index a575da2d6c79..01055a876450 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -6,6 +6,8 @@
#include
#include
+struct pkram_node;
+
/**
* enum pkram_data_flags - definition of data types contained in a pkram obj
* @PKRAM_DATA_none: No data types configured
@@ -14,7 +16,11 @@ enum pkram_data_flags {
PKRAM_DATA_none = 0x0, /* No data types configured */
};
-struct pkram_stream;
+struct pkram_stream {
+ gfp_t gfp_mask;
+ struct pkram_node *node;
+};
+
struct pkram_access;
#define PKRAM_NAME_MAX 256 /* including nul */
diff --git a/mm/pkram.c b/mm/pkram.c
index 59e4661b2fb7..21976df6e0ea 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -2,16 +2,85 @@
#include
#include
#include
+#include
#include
+#include
#include
+#include
#include
+/*
+ * Preserved memory is divided into nodes that can be saved or loaded
+ * independently of each other. The nodes are identified by unique name
+ * strings.
+ *
+ * The structure occupies a memory page.
+ */
+struct pkram_node {
+ __u32 flags;
+
+ __u8name[PKRAM_NAME_MAX];
+};
+
+#define PKRAM_SAVE 1
+#define PKRAM_LOAD 2
+#define PKRAM_ACCMODE_MASK 3
+
+static LIST_HEAD(pkram_nodes); /* linked through page::lru */
+static DEFINE_MUTEX(pkram_mutex); /* serializes open/close */
+
+static inline struct page *pkram_alloc_page(gfp_t gfp_mask)
+{
+ return alloc_page(gfp_mask);
+}
+
+static inline void pkram_free_page(void *addr)
+{
+ free_page((unsigned long)addr);
+}
+
+static inline void pkram_insert_node(struct pkram_node *node)
+{
+ list_add(_to_page(node)->lru, _nodes);
+}
+
+static inline void pkram_delete_node(struct pkram_node *node)
+{
+ list_del(_to_page(node)->lru);
+}
+
+static struct pkram_node *pkram_find_node(const char *name)
+{
+ struct page *page;
+ struct pkram_node *node;
+
+ list_for_each_entry(page, _nodes, lru) {
+ node = page_address(page);
+ if (strcmp(node->name, name) == 0)
+ return node;
+ }
+ return NULL;
+}
+
+static void pkram_stream_init(struct pkram_stream *ps,
+struct pkram_node *node, gfp_t gfp_mask)
+{
+ memset(ps, 0, sizeof(*ps));
+ ps->gfp_mask = gfp_mask;
+ ps->node = node;
+}
+
/**
* Create a preserved memory node with name @name and initialize stream @ps
* for saving data to it.
*
* @gfp_mask specifies the memory allocation mask to be used when saving data.
*
+ * Error values:
+ * %ENAMETOOLONG: name len >= PKRAM_NAME_MAX
+ * %ENOMEM: insufficient memory available
+ * %EEXIST: node with specified name already exists
+ *
* Returns 0 on success, -errno on failure.
*
* After the save has finished, pkram_finish_save() (or pkram_discard_save() in
@@ -19,7 +88,34 @@
*/
int pkram_prepare_save(struct pkram_stream *ps, const char *name, gfp_t
gfp_mask)
{
- return -ENOSYS;
+ struct page *page;
+ struct pkram_node *node;
+ int err = 0;
+
+ if (strlen(name) >= PKRAM_NAME_MAX)
+ return -ENAMETOOLONG;
+
+ page = pkram_alloc_page(gfp_mask | __GFP_ZERO);
+ if (!page)
+ return -ENOMEM;
+ node = page_address(page);
+
+ node->flags = PKRAM_SAVE;
+ strcpy(node->name, name);
+
+ mutex_lock(_mutex);
+ if (!pkram_find_node(name))
+ pkram_insert_node(node);
+ else
+ err = -EEXIST;
+ mutex_unlock(_mutex);
+ if (err) {
+ pkram_free_page(node);
+ return err;
+ }
+
+ pkram_stream_init(ps, node, gfp_mask);
+ return 0;
}
/**
@@ -50,7 +146,12 @@ void pkram_finish_save_obj(struct pkram_stream *ps)
*/
void pkram_finish_save(struct pkram_stream *ps)
{
-
[RFC v2 03/43] mm: PKRAM: implement object load and save functions
PKRAM nodes are further divided into a list of objects. After a save
operation has been initiated for a node, a save operation for an object
associated with the node is initiated by calling pkram_prepare_save_obj().
A new object is created and linked to the node. The save operation for
the object is committed by calling pkram_finish_save_obj(). After a load
operation has been initiated, pkram_prepare_load_obj() is called to
delete the next object from the node and prepare the corresponding
stream for loading data from it. After the load of object has been
finished, pkram_finish_load_obj() is called to free the object. Objects
are also deleted when a save operation is discarded.
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 2 ++
mm/pkram.c| 72 ---
2 files changed, 70 insertions(+), 4 deletions(-)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index 01055a876450..a4d55af392c0 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -7,6 +7,7 @@
#include
struct pkram_node;
+struct pkram_obj;
/**
* enum pkram_data_flags - definition of data types contained in a pkram obj
@@ -19,6 +20,7 @@ enum pkram_data_flags {
struct pkram_stream {
gfp_t gfp_mask;
struct pkram_node *node;
+ struct pkram_obj *obj;
};
struct pkram_access;
diff --git a/mm/pkram.c b/mm/pkram.c
index 21976df6e0ea..7c977c5982f8 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -6,9 +6,14 @@
#include
#include
#include
+#include
#include
#include
+struct pkram_obj {
+ __u64 obj_pfn;/* points to the next object in the list */
+};
+
/*
* Preserved memory is divided into nodes that can be saved or loaded
* independently of each other. The nodes are identified by unique name
@@ -18,6 +23,7 @@
*/
struct pkram_node {
__u32 flags;
+ __u64 obj_pfn;/* points to the first obj of the node */
__u8name[PKRAM_NAME_MAX];
};
@@ -62,6 +68,21 @@ static struct pkram_node *pkram_find_node(const char *name)
return NULL;
}
+static void pkram_truncate_node(struct pkram_node *node)
+{
+ unsigned long obj_pfn;
+ struct pkram_obj *obj;
+
+ obj_pfn = node->obj_pfn;
+ while (obj_pfn) {
+ obj = pfn_to_kaddr(obj_pfn);
+ obj_pfn = obj->obj_pfn;
+ pkram_free_page(obj);
+ cond_resched();
+ }
+ node->obj_pfn = 0;
+}
+
static void pkram_stream_init(struct pkram_stream *ps,
struct pkram_node *node, gfp_t gfp_mask)
{
@@ -124,12 +145,31 @@ int pkram_prepare_save(struct pkram_stream *ps, const
char *name, gfp_t gfp_mask
*
* Returns 0 on success, -errno on failure.
*
+ * Error values:
+ * %ENOMEM: insufficient memory available
+ *
* After the save has finished, pkram_finish_save_obj() (or
pkram_discard_save()
* in case of failure) is to be called.
*/
int pkram_prepare_save_obj(struct pkram_stream *ps, enum pkram_data_flags
flags)
{
- return -ENOSYS;
+ struct pkram_node *node = ps->node;
+ struct pkram_obj *obj;
+ struct page *page;
+
+ BUG_ON((node->flags & PKRAM_ACCMODE_MASK) != PKRAM_SAVE);
+
+ page = pkram_alloc_page(ps->gfp_mask | __GFP_ZERO);
+ if (!page)
+ return -ENOMEM;
+ obj = page_address(page);
+
+ if (node->obj_pfn)
+ obj->obj_pfn = node->obj_pfn;
+ node->obj_pfn = page_to_pfn(page);
+
+ ps->obj = obj;
+ return 0;
}
/**
@@ -137,7 +177,9 @@ int pkram_prepare_save_obj(struct pkram_stream *ps, enum
pkram_data_flags flags)
*/
void pkram_finish_save_obj(struct pkram_stream *ps)
{
- BUG();
+ struct pkram_node *node = ps->node;
+
+ BUG_ON((node->flags & PKRAM_ACCMODE_MASK) != PKRAM_SAVE);
}
/**
@@ -169,6 +211,7 @@ void pkram_discard_save(struct pkram_stream *ps)
pkram_delete_node(node);
mutex_unlock(_mutex);
+ pkram_truncate_node(node);
pkram_free_page(node);
}
@@ -216,11 +259,26 @@ int pkram_prepare_load(struct pkram_stream *ps, const
char *name)
*
* Returns 0 on success, -errno on failure.
*
+ * Error values:
+ * %ENODATA: Stream @ps has no preserved memory objects
+ *
* After the load has finished, pkram_finish_load_obj() is to be called.
*/
int pkram_prepare_load_obj(struct pkram_stream *ps)
{
- return -ENOSYS;
+ struct pkram_node *node = ps->node;
+ struct pkram_obj *obj;
+
+ BUG_ON((node->flags & PKRAM_ACCMODE_MASK) != PKRAM_LOAD);
+
+ if (!node->obj_pfn)
+ return -ENODATA;
+
+ obj = pfn_to_kaddr(node->obj_pfn);
+ node->obj_pfn = obj->obj_pfn;
+
+ ps->obj = obj;
+ return 0;
}
/**
@@ -230,7 +288,12 @@ int pkram_prepare_load_obj(struct pkram_stream *ps)
*/
void pkram_finish_load_obj(struct
[RFC v2 06/43] mm: PKRAM: implement byte stream operations
This patch adds the ability to save an arbitrary byte streams to a
a PKRAM object using pkram_write() to be restored later using pkram_read().
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 11 +
mm/pkram.c| 123 --
2 files changed, 130 insertions(+), 4 deletions(-)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index 9d8a6fd96dd9..4f95d4fb5339 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -14,10 +14,12 @@
* enum pkram_data_flags - definition of data types contained in a pkram obj
* @PKRAM_DATA_none: No data types configured
* @PKRAM_DATA_pages: obj contains file page data
+ * @PKRAM_DATA_bytes: obj contains byte data
*/
enum pkram_data_flags {
PKRAM_DATA_none = 0x0, /* No data types configured */
PKRAM_DATA_pages= 0x1, /* Contains file page data */
+ PKRAM_DATA_bytes= 0x2, /* Contains byte data */
};
struct pkram_data_stream {
@@ -36,18 +38,27 @@ struct pkram_stream {
__u64 *pages_head_link_pfnp;
__u64 *pages_tail_link_pfnp;
+
+ __u64 *bytes_head_link_pfnp;
+ __u64 *bytes_tail_link_pfnp;
};
struct pkram_pages_access {
unsigned long next_index;
};
+struct pkram_bytes_access {
+ struct page *data_page; /* current page */
+ unsigned int data_offset; /* offset into current page */
+};
+
struct pkram_access {
enum pkram_data_flags dtype;
struct pkram_stream *ps;
struct pkram_data_stream pds;
struct pkram_pages_access pages;
+ struct pkram_bytes_access bytes;
};
#define PKRAM_NAME_MAX 256 /* including nul */
diff --git a/mm/pkram.c b/mm/pkram.c
index da44a6060c5f..d81af26c9a66 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
#include
#include
+#include
#include
#include
#include
@@ -46,6 +47,9 @@ struct pkram_link {
struct pkram_obj {
__u64 pages_head_link_pfn;/* the first pages link of the object */
__u64 pages_tail_link_pfn;/* the last pages link of the object */
+ __u64 bytes_head_link_pfn;/* the first bytes link of the object */
+ __u64 bytes_tail_link_pfn;/* the last bytes link of the object */
+ __u64 data_len; /* byte data size */
__u64 obj_pfn;/* points to the next object in the list */
};
@@ -140,6 +144,11 @@ static void pkram_truncate_obj(struct pkram_obj *obj)
pkram_truncate_links(obj->pages_head_link_pfn);
obj->pages_head_link_pfn = 0;
obj->pages_tail_link_pfn = 0;
+
+ pkram_truncate_links(obj->bytes_head_link_pfn);
+ obj->bytes_head_link_pfn = 0;
+ obj->bytes_tail_link_pfn = 0;
+ obj->data_len = 0;
}
static void pkram_truncate_node(struct pkram_node *node)
@@ -315,7 +324,7 @@ int pkram_prepare_save_obj(struct pkram_stream *ps, enum
pkram_data_flags flags)
BUG_ON((node->flags & PKRAM_ACCMODE_MASK) != PKRAM_SAVE);
- if (flags & ~PKRAM_DATA_pages)
+ if (flags & ~(PKRAM_DATA_pages | PKRAM_DATA_bytes))
return -EINVAL;
page = pkram_alloc_page(ps->gfp_mask | __GFP_ZERO);
@@ -331,6 +340,10 @@ int pkram_prepare_save_obj(struct pkram_stream *ps, enum
pkram_data_flags flags)
ps->pages_head_link_pfnp = >pages_head_link_pfn;
ps->pages_tail_link_pfnp = >pages_tail_link_pfn;
}
+ if (flags & PKRAM_DATA_bytes) {
+ ps->bytes_head_link_pfnp = >bytes_head_link_pfn;
+ ps->bytes_tail_link_pfnp = >bytes_tail_link_pfn;
+ }
ps->obj = obj;
return 0;
}
@@ -438,7 +451,7 @@ int pkram_prepare_load_obj(struct pkram_stream *ps)
return -ENODATA;
obj = pfn_to_kaddr(node->obj_pfn);
- if (!obj->pages_head_link_pfn) {
+ if (!obj->pages_head_link_pfn && !obj->bytes_head_link_pfn) {
WARN_ON(1);
return -EINVAL;
}
@@ -449,6 +462,10 @@ int pkram_prepare_load_obj(struct pkram_stream *ps)
ps->pages_head_link_pfnp = >pages_head_link_pfn;
ps->pages_tail_link_pfnp = >pages_tail_link_pfn;
}
+ if (obj->bytes_head_link_pfn) {
+ ps->bytes_head_link_pfnp = >bytes_head_link_pfn;
+ ps->bytes_tail_link_pfnp = >bytes_tail_link_pfn;
+ }
ps->obj = obj;
return 0;
}
@@ -499,6 +516,9 @@ void pkram_finish_access(struct pkram_access *pa, bool
status_ok)
if (pa->pds.link)
pkram_truncate_link(pa->pds.link);
+
+ if ((pa->dtype == PKRAM_DATA_bytes) && (pa->bytes.data_page))
+ pkram_free_page(page_address(pa->bytes.data_page));
}
/*
@@
[RFC v2 00/43] PKRAM: Preserved-over-Kexec RAM
ed. * Pages for PKRAM-enabled files are prevented from swapping out to avoid the performance penalty of swapping in and the possibility of insufficient memory. -- Patches -- The patches are broken down into the following groups: Patches 1-22 implement the API and supporting functionality. Patches 23-27 implement the use of PKRAM within tmpfs The remaining patches implement optimizations to the initialization of preserved pages and to the preservation and restoration of shmem pages. To give an idea of the improvement in performance here is an example comparison with and without these patches when saving and loading a 100G file: Save a 100G file: | No optimizations | Optimized (16 cpus) | -- huge=never | 2265ms | 232ms | -- huge=always | 58ms |22ms | Load a 100G file: | No optimizations | Optimized (16 cpus) | -- huge=never | 8833ms | 516ms | -- huge=always | 752ms | 105ms | Patches 28-31 Defer initialization of page structs for preserved pages Patches 32-34 Implement multi-threading of shmem page preservation and restoration. Patches 35-37 Implement and use an API for inserting shmem pages in bulk Patches 38-39: Reduce contention on the LRU lock by staging and adding pages in bulk to the LRU Patches 40-43: Reduce contention on the pagecache xarray lock by inserting pages in bulk in certain cases [1] https://lkml.org/lkml/2013/7/1/211 [2] https://www.youtube.com/watch?v=pBsHnf93tcQ https://static.sched.com/hosted_files/kvmforum2019/66/VMM-fast-restart_kvmforum2019.pdf [3] https://www.youtube.com/watch?v=pBsHnf93tcQ https://static.sched.com/hosted_files/kvmforum2020/10/Device-Keepalive-State-KVMForum2020.pdf Anthony Yznaga (43): mm: add PKRAM API stubs and Kconfig mm: PKRAM: implement node load and save functions mm: PKRAM: implement object load and save functions mm: PKRAM: implement page stream operations mm: PKRAM: support preserving transparent hugepages mm: PKRAM: implement byte stream operations mm: PKRAM: link nodes by pfn before reboot mm: PKRAM: introduce super block PKRAM: track preserved pages in a physical mapping pagetable PKRAM: pass a list of preserved ranges to the next kernel PKRAM: prepare for adding preserved ranges to memblock reserved mm: PKRAM: reserve preserved memory at boot PKRAM: free the preserved ranges list PKRAM: prevent inadvertent use of a stale superblock PKRAM: provide a way to ban pages from use by PKRAM kexec: PKRAM: prevent kexec clobbering preserved pages in some cases PKRAM: provide a way to check if a memory range has preserved pages kexec: PKRAM: avoid clobbering already preserved pages mm: PKRAM: allow preserved memory to be freed from userspace PKRAM: disable feature when running the kdump kernel x86/KASLR: PKRAM: support physical kaslr x86/boot/compressed/64: use 1GB pages for mappings mm: shmem: introduce shmem_insert_page mm: shmem: enable saving to PKRAM mm: shmem: prevent swapping of PKRAM-enabled tmpfs pages mm: shmem: specify the mm to use when inserting pages mm: shmem: when inserting, handle pages already charged to a memcg x86/mm/numa: add numa_isolate_memblocks() PKRAM: ensure memblocks with preserved pages init'd for numa memblock: PKRAM: mark memblocks that contain preserved pages memblock, mm: defer initialization of preserved pages shmem: preserve shmem files a chunk at a time PKRAM: atomically add and remove link pages shmem: PKRAM: multithread preserving and restoring shmem pages shmem: introduce shmem_insert_pages() PKRAM: add support for loading pages in bulk shmem: PKRAM: enable bulk loading of preserved pages into shmem mm: implement splicing a list of pages to the LRU shmem: optimize adding pages to the LRU in shmem_insert_pages() shmem: initial support for adding multiple pages to pagecache XArray: add xas_export_node() and xas_import_node() shmem: reduce time holding xa_lock when inserting pages PKRAM: improve index alignment of pkram_link entries documentation/core-api/xarray.rst |8 + arch/x86/boot/compressed/Makefile |3 + arch/x86/boot/compressed/ident_map_64.c |9 +- arch/x86/boot/compressed/kaslr.c| 10 +- arch/x86/boot/compressed/misc.h | 10 + arch/x86/boot/compressed/pkram.c| 109 ++ arch/x86/include/asm/numa.h |4 + arch/x86/kernel/setup.c |3 + arch/x86/mm/init_64.c |2 + arch/x86/mm/numa.c | 32 +- include/linux/memblock.h|6 + include/linux/mm.h |2 +- include/linux/pkram.h
[RFC v2 01/43] mm: add PKRAM API stubs and Kconfig
Preserved-across-kexec memory or PKRAM is a method for saving memory
pages of the currently executing kernel and restoring them after kexec
boot into a new one. This can be utilized for preserving guest VM state,
large in-memory databases, process memory, etc. across reboot. While
DRAM-as-PMEM or actual persistent memory could be used to accomplish
these things, PKRAM provides the latency of DRAM with the flexibility
of dynamically determining the amount of memory to preserve.
The proposed API:
* Preserved memory is divided into nodes which can be saved or loaded
independently of each other. The nodes are identified by unique name
strings. A PKRAM node is created when save is initiated by calling
pkram_prepare_save(). A PKRAM node is removed when load is initiated by
calling pkram_prepare_load(). See below
* A node is further divided into objects. An object represents closely
coupled data in the form of a grouping of pages and/or a stream of
byte data. For example, the pages and attributes of a file.
After initiating an operation on a PKRAM node, PKRAM objects are
initialized for saving or loading by calling pkram_prepare_save_obj()
or pkram_prepare_load_obj().
* For saving/loading data from a PKRAM node/object instances of the
pkram_stream and pkram_access structs are used. pkram_stream tracks
the node and object being operated on while pkram_access tracks the
data type and position within an object.
The pkram_stream struct is initialized by calling pkram_prepare_save()
or pkram_prepare_load() and then pkram_prepare_save_obj() or
pkram_prepare_load_obj().
Once a pkram_stream is fully initialized, a pkram_access struct
is initialized for each data type associated with the object.
After save or load of a data type for the object is complete,
pkram_finish_access() is called.
After save or load is complete for the object, pkram_finish_save_obj()
or pkram_finish_load_obj() must be called followed by pkram_finish_save()
or pkram_finish_load() when save or load is completed for the node.
If an error occurred during save, the saved data and the PKRAM node
may be freed by calling pkram_discard_save() instead of
pkram_finish_save().
* Both page data and byte data can separately be streamed to a PKRAM
object. pkram_save_file_page() and pkram_load_file_page() are used
to stream page data while pkram_write() and pkram_read() are used to
stream byte data.
A sequence of operations for saving/loading data from PKRAM would
look like:
* For saving data to PKRAM:
/* create a PKRAM node and do initial stream setup */
pkram_prepare_save()
/* create a PKRAM object associated with the PKRAM node and complete stream
initialization */
pkram_prepare_save_obj()
/* save data to the node/object */
PKRAM_ACCESS(pa_pages,...)
PKRAM_ACCESS(pa_bytes,...)
pkram_save_file_page(pa_pages,...)[,...] /* for file pages */
pkram_write(pa_bytes,...)[,...] /* for a byte stream */
pkram_finish_access(pa_pages)
pkram_finish_access(pa_bytes)
pkram_finish_save_obj()
/* commit the save or discard and delete the node */
pkram_finish_save() /* on success, or
pkram_discard_save() * ... in case of error */
* For loading data from PKRAM:
/* remove a PKRAM node from the list and do initial stream setup */
pkram_prepare_load()
/* Remove a PKRAM object from the node and complete stream initializtion
for loading data from it. */
pkram_prepare_load_obj()
/* load data from the node/object */
PKRAM_ACCESS(pa_pages,...)
PKRAM_ACCESS(pa_bytes,...)
pkram_load_file_page(pa_pages,...)[,...] /* for file pages */
pkram_read(pa_bytes,...)[,...] /* for a byte stream */
*/
pkram_finish_access(pa_pages)
pkram_finish_access(pa_bytes)
/* free the object */
pkram_finish_load_obj()
/* free the node */
pkram_finish_load()
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 47 +
mm/Kconfig| 9 +++
mm/Makefile | 1 +
mm/pkram.c| 179 ++
4 files changed, 236 insertions(+)
create mode 100644 include/linux/pkram.h
create mode 100644 mm/pkram.c
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
new file mode 100644
index ..a575da2d6c79
--- /dev/null
+++ b/include/linux/pkram.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_PKRAM_H
+#define _LINUX_PKRAM_H
+
+#include
+#include
+#include
+
+/**
+ * enum pkram_data_flags - definition of data types contained in a pkram obj
+ * @PKRAM_DATA_none: No data types configured
+ */
+enum pkram_data_flags {
+ PKRAM_DATA_none = 0x0, /* No data types configured */
+};
+
+struct pkram_stream;
+struct pkram_access;
+
+#define PKRAM_NAME_MAX 256 /* including nul
[RFC v2 04/43] mm: PKRAM: implement page stream operations
Using the pkram_save_file_page() function, one can populate PKRAM objects
with in-memory pages which can later be loaded using the pkram_load_file_page()
function. Saving a page to PKRAM is accomplished by recording its pfn and
mapping index and incrementing its refcount so that it will not be freed
after the last user puts it.
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 42 +++-
mm/pkram.c| 282 +-
2 files changed, 317 insertions(+), 7 deletions(-)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index a4d55af392c0..9d8a6fd96dd9 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -8,22 +8,47 @@
struct pkram_node;
struct pkram_obj;
+struct pkram_link;
/**
* enum pkram_data_flags - definition of data types contained in a pkram obj
* @PKRAM_DATA_none: No data types configured
+ * @PKRAM_DATA_pages: obj contains file page data
*/
enum pkram_data_flags {
- PKRAM_DATA_none = 0x0, /* No data types configured */
+ PKRAM_DATA_none = 0x0, /* No data types configured */
+ PKRAM_DATA_pages= 0x1, /* Contains file page data */
+};
+
+struct pkram_data_stream {
+ /* List of link pages to add/remove from */
+ __u64 *head_link_pfnp;
+ __u64 *tail_link_pfnp;
+
+ struct pkram_link *link;/* current link */
+ unsigned int entry_idx; /* next entry in link */
};
struct pkram_stream {
gfp_t gfp_mask;
struct pkram_node *node;
struct pkram_obj *obj;
+
+ __u64 *pages_head_link_pfnp;
+ __u64 *pages_tail_link_pfnp;
+};
+
+struct pkram_pages_access {
+ unsigned long next_index;
};
-struct pkram_access;
+struct pkram_access {
+ enum pkram_data_flags dtype;
+ struct pkram_stream *ps;
+ struct pkram_data_stream pds;
+
+ struct pkram_pages_access pages;
+};
#define PKRAM_NAME_MAX 256 /* including nul */
@@ -41,8 +66,19 @@ int pkram_prepare_save(struct pkram_stream *ps, const char
*name,
void pkram_finish_load(struct pkram_stream *ps);
void pkram_finish_load_obj(struct pkram_stream *ps);
+#define PKRAM_PDS_INIT(name, stream, type) { \
+ .head_link_pfnp=(stream)->type##_head_link_pfnp,\
+ .tail_link_pfnp=(stream)->type##_tail_link_pfnp,\
+ }
+
+#define PKRAM_ACCESS_INIT(name, stream, type) {\
+ .dtype = PKRAM_DATA_##type, \
+ .ps = (stream), \
+ .pds = PKRAM_PDS_INIT(name, stream, type), \
+ }
+
#define PKRAM_ACCESS(name, stream, type) \
- struct pkram_access name
+ struct pkram_access name = PKRAM_ACCESS_INIT(name, stream, type)
void pkram_finish_access(struct pkram_access *pa, bool status_ok);
diff --git a/mm/pkram.c b/mm/pkram.c
index 7c977c5982f8..9c42db66d022 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
#include
#include
+#include
#include
#include
#include
@@ -10,8 +11,39 @@
#include
#include
+#include "internal.h"
+
+
+/*
+ * Represents a reference to a data page saved to PKRAM.
+ */
+typedef __u64 pkram_entry_t;
+
+#define PKRAM_ENTRY_FLAGS_SHIFT0x5
+#define PKRAM_ENTRY_FLAGS_MASK 0x7f
+
+/*
+ * Keeps references to data pages saved to PKRAM.
+ * The structure occupies a memory page.
+ */
+struct pkram_link {
+ __u64 link_pfn; /* points to the next link of the object */
+ __u64 index; /* mapping index of first pkram_entry_t */
+
+ /*
+* the array occupies the rest of the link page; if the link is not
+* full, the rest of the array must be filled with zeros
+*/
+ pkram_entry_t entry[0];
+};
+
+#define PKRAM_LINK_ENTRIES_MAX \
+ ((PAGE_SIZE-sizeof(struct pkram_link))/sizeof(pkram_entry_t))
+
struct pkram_obj {
- __u64 obj_pfn;/* points to the next object in the list */
+ __u64 pages_head_link_pfn;/* the first pages link of the object */
+ __u64 pages_tail_link_pfn;/* the last pages link of the object */
+ __u64 obj_pfn;/* points to the next object in the list */
};
/*
@@ -19,6 +51,10 @@ struct pkram_obj {
* independently of each other. The nodes are identified by unique name
* strings.
*
+ * References to data pages saved to a preserved memory node are kept in a
+ * singly-linked list of PKRAM link structures (see above), the node has a
+ * pointer to the head of.
+ *
* The structure occupies a memory page.
*/
struct pkram_node {
@@ -68,6 +104,41 @@ static struct pkram_node *pkram_find_node(const char *name)
return NULL;
}
+static void pkram_truncate_link(struct pkram_link *link)
+{
+ struct page *page;
+ pkram_entry_t
[RFC v2 08/43] mm: PKRAM: introduce super block
The PKRAM super block is the starting point for restoring preserved
memory. By providing the super block to the new kernel at boot time,
preserved memory can be reserved and made available to be restored.
To point the kernel to the location of the super block, one passes
its pfn via the 'pkram' boot param. For that purpose, the pkram super
block pfn is exported via /sys/kernel/pkram. If none is passed, any
preserved memory will not be kept, and a new super block will be
allocated.
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 102 +++--
1 file changed, 100 insertions(+), 2 deletions(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index 975f200aef38..2809371a9aec 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -5,15 +5,18 @@
#include
#include
#include
+#include
#include
#include
#include
#include
#include
+#include
#include
#include
#include
#include
+#include
#include
#include "internal.h"
@@ -84,12 +87,38 @@ struct pkram_node {
#define PKRAM_ACCMODE_MASK 3
/*
+ * The PKRAM super block contains data needed to restore the preserved memory
+ * structure on boot. The pointer to it (pfn) should be passed via the 'pkram'
+ * boot param if one wants to restore preserved data saved by the previously
+ * executing kernel. For that purpose the kernel exports the pfn via
+ * /sys/kernel/pkram. If none is passed, preserved memory if any will not be
+ * preserved and a new clean page will be allocated for the super block.
+ *
+ * The structure occupies a memory page.
+ */
+struct pkram_super_block {
+ __u64 node_pfn; /* first element of the node list */
+};
+
+static unsigned long pkram_sb_pfn __initdata;
+static struct pkram_super_block *pkram_sb;
+
+/*
* For convenience sake PKRAM nodes are kept in an auxiliary doubly-linked list
* connected through the lru field of the page struct.
*/
static LIST_HEAD(pkram_nodes); /* linked through page::lru */
static DEFINE_MUTEX(pkram_mutex); /* serializes open/close */
+/*
+ * The PKRAM super block pfn, see above.
+ */
+static int __init parse_pkram_sb_pfn(char *arg)
+{
+ return kstrtoul(arg, 16, _sb_pfn);
+}
+early_param("pkram", parse_pkram_sb_pfn);
+
static inline struct page *pkram_alloc_page(gfp_t gfp_mask)
{
return alloc_page(gfp_mask);
@@ -275,6 +304,7 @@ static void pkram_stream_init(struct pkram_stream *ps,
* @gfp_mask specifies the memory allocation mask to be used when saving data.
*
* Error values:
+ * %ENODEV: PKRAM not available
* %ENAMETOOLONG: name len >= PKRAM_NAME_MAX
* %ENOMEM: insufficient memory available
* %EEXIST: node with specified name already exists
@@ -290,6 +320,9 @@ int pkram_prepare_save(struct pkram_stream *ps, const char
*name, gfp_t gfp_mask
struct pkram_node *node;
int err = 0;
+ if (!pkram_sb)
+ return -ENODEV;
+
if (strlen(name) >= PKRAM_NAME_MAX)
return -ENAMETOOLONG;
@@ -410,6 +443,7 @@ void pkram_discard_save(struct pkram_stream *ps)
* Returns 0 on success, -errno on failure.
*
* Error values:
+ * %ENODEV: PKRAM not available
* %ENOENT: node with specified name does not exist
* %EBUSY: save to required node has not finished yet
*
@@ -420,6 +454,9 @@ int pkram_prepare_load(struct pkram_stream *ps, const char
*name)
struct pkram_node *node;
int err = 0;
+ if (!pkram_sb)
+ return -ENODEV;
+
mutex_lock(_mutex);
node = pkram_find_node(name);
if (!node) {
@@ -809,6 +846,13 @@ static void __pkram_reboot(void)
node->node_pfn = node_pfn;
node_pfn = page_to_pfn(page);
}
+
+ /*
+* Zero out pkram_sb completely since it may have been passed from
+* the previous boot.
+*/
+ memset(pkram_sb, 0, PAGE_SIZE);
+ pkram_sb->node_pfn = node_pfn;
}
static int pkram_reboot(struct notifier_block *notifier,
@@ -816,7 +860,8 @@ static int pkram_reboot(struct notifier_block *notifier,
{
if (val != SYS_RESTART)
return NOTIFY_DONE;
- __pkram_reboot();
+ if (pkram_sb)
+ __pkram_reboot();
return NOTIFY_OK;
}
@@ -824,9 +869,62 @@ static int pkram_reboot(struct notifier_block *notifier,
.notifier_call = pkram_reboot,
};
+static ssize_t show_pkram_sb_pfn(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ unsigned long pfn = pkram_sb ? PFN_DOWN(__pa(pkram_sb)) : 0;
+
+ return sprintf(buf, "%lx\n", pfn);
+}
+
+static struct kobj_attribute pkram_sb_pfn_attr =
+ __ATTR(pkram, 0444, show_pkram_sb_pfn, NULL);
+
+static struct attribute *pkram_attrs[] = {
+ _sb_pfn_attr.attr,
+ NULL,
+};
+
+static struct attribute_group pkram_attr_group = {
+
[RFC v2 05/43] mm: PKRAM: support preserving transparent hugepages
Support preserving a transparent hugepage by recording the page order and
a flag indicating it is a THP. Use these values when the page is
restored to reconstruct the THP.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 20
1 file changed, 16 insertions(+), 4 deletions(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index 9c42db66d022..da44a6060c5f 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -21,6 +21,9 @@
#define PKRAM_ENTRY_FLAGS_SHIFT0x5
#define PKRAM_ENTRY_FLAGS_MASK 0x7f
+#define PKRAM_ENTRY_ORDER_MASK 0x1f
+
+#define PKRAM_PAGE_TRANS_HUGE 0x1 /* page is a transparent hugepage */
/*
* Keeps references to data pages saved to PKRAM.
@@ -211,7 +214,11 @@ static void pkram_add_link_entry(struct pkram_data_stream
*pds, struct page *pag
pkram_entry_t p;
short flags = 0;
+ if (PageTransHuge(page))
+ flags |= PKRAM_PAGE_TRANS_HUGE;
+
p = page_to_phys(page);
+ p |= compound_order(page);
p |= ((flags & PKRAM_ENTRY_FLAGS_MASK) << PKRAM_ENTRY_FLAGS_SHIFT);
link->entry[pds->entry_idx] = p;
pds->entry_idx++;
@@ -516,7 +523,7 @@ static int __pkram_save_page(struct pkram_access *pa,
struct page *page,
pkram_add_link_entry(pds, page);
- pa->pages.next_index++;
+ pa->pages.next_index += compound_nr(page);
return 0;
}
@@ -542,19 +549,24 @@ int pkram_save_file_page(struct pkram_access *pa, struct
page *page)
BUG_ON((node->flags & PKRAM_ACCMODE_MASK) != PKRAM_SAVE);
- BUG_ON(PageCompound(page));
-
return __pkram_save_page(pa, page, page->index);
}
static struct page *__pkram_prep_load_page(pkram_entry_t p)
{
struct page *page;
+ int order;
short flags;
flags = (p >> PKRAM_ENTRY_FLAGS_SHIFT) & PKRAM_ENTRY_FLAGS_MASK;
page = pfn_to_page(PHYS_PFN(p));
+ if (flags & PKRAM_PAGE_TRANS_HUGE) {
+ order = p & PKRAM_ENTRY_ORDER_MASK;
+ prep_compound_page(page, order);
+ prep_transhuge_page(page);
+ }
+
return page;
}
@@ -588,7 +600,7 @@ static struct page *__pkram_load_page(struct pkram_access
*pa, unsigned long *in
if (index) {
*index = pa->pages.next_index;
- pa->pages.next_index++;
+ pa->pages.next_index += compound_nr(page);
}
/* clear to avoid double free (see pkram_truncate_link()) */
--
1.8.3.1
[RFC v2 12/43] mm: PKRAM: reserve preserved memory at boot
Keep preserved pages from being recycled during boot by adding them
to the memblock reserved list during early boot. If memory reservation
fails (e.g. a region has already been reserved), all preserved pages
are dropped.
Signed-off-by: Anthony Yznaga
---
arch/x86/kernel/setup.c | 3 ++
arch/x86/mm/init_64.c | 2 ++
include/linux/pkram.h | 8 ++
mm/pkram.c | 76 +++--
4 files changed, 87 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index d883176ef2ce..fbd85964719d 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -15,6 +15,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -1146,6 +1147,8 @@ void __init setup_arch(char **cmdline_p)
initmem_init();
dma_contiguous_reserve(max_pfn_mapped << PAGE_SHIFT);
+ pkram_reserve();
+
if (boot_cpu_has(X86_FEATURE_GBPAGES))
hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT);
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index b5a3fa4033d3..8efb2fb2a88b 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -33,6 +33,7 @@
#include
#include
#include
+#include
#include
#include
@@ -1293,6 +1294,7 @@ void __init mem_init(void)
after_bootmem = 1;
x86_init.hyper.init_after_bootmem();
+ totalram_pages_add(pkram_reserved_pages);
/*
* Must be done after boot memory is put on freelist, because here we
* might set fields in deferred struct pages that have not yet been
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index 4f95d4fb5339..8d3d780d9fe1 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -99,4 +99,12 @@ int pkram_prepare_save(struct pkram_stream *ps, const char
*name,
ssize_t pkram_write(struct pkram_access *pa, const void *buf, size_t count);
size_t pkram_read(struct pkram_access *pa, void *buf, size_t count);
+#ifdef CONFIG_PKRAM
+extern unsigned long pkram_reserved_pages;
+void pkram_reserve(void);
+#else
+#define pkram_reserved_pages 0UL
+static inline void pkram_reserve(void) { }
+#endif
+
#endif /* _LINUX_PKRAM_H */
diff --git a/mm/pkram.c b/mm/pkram.c
index b4a14837946a..03731bb6af26 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -135,6 +135,8 @@ struct pkram_super_block {
static LIST_HEAD(pkram_nodes); /* linked through page::lru */
static DEFINE_MUTEX(pkram_mutex); /* serializes open/close */
+unsigned long __initdata pkram_reserved_pages;
+
/*
* The PKRAM super block pfn, see above.
*/
@@ -144,6 +146,59 @@ static int __init parse_pkram_sb_pfn(char *arg)
}
early_param("pkram", parse_pkram_sb_pfn);
+static void * __init pkram_map_meta(unsigned long pfn)
+{
+ if (pfn >= max_low_pfn)
+ return ERR_PTR(-EINVAL);
+ return pfn_to_kaddr(pfn);
+}
+
+int pkram_merge_with_reserved(void);
+/*
+ * Reserve pages that belong to preserved memory.
+ *
+ * This function should be called at boot time as early as possible to prevent
+ * preserved memory from being recycled.
+ */
+void __init pkram_reserve(void)
+{
+ int err = 0;
+
+ if (!pkram_sb_pfn)
+ return;
+
+ pr_info("PKRAM: Examining preserved memory...\n");
+
+ /* Verify that nothing else has reserved the pkram_sb page */
+ if (memblock_is_region_reserved(PFN_PHYS(pkram_sb_pfn), PAGE_SIZE)) {
+ err = -EBUSY;
+ goto out;
+ }
+
+ pkram_sb = pkram_map_meta(pkram_sb_pfn);
+ if (IS_ERR(pkram_sb)) {
+ err = PTR_ERR(pkram_sb);
+ goto out;
+ }
+ /* An empty pkram_sb is not an error */
+ if (!pkram_sb->node_pfn) {
+ pkram_sb = NULL;
+ goto done;
+ }
+
+ err = pkram_merge_with_reserved();
+out:
+ if (err) {
+ pr_err("PKRAM: Reservation failed: %d\n", err);
+ WARN_ON(pkram_reserved_pages > 0);
+ pkram_sb = NULL;
+ return;
+ }
+
+done:
+ pr_info("PKRAM: %lu pages reserved\n", pkram_reserved_pages);
+}
+
static inline struct page *pkram_alloc_page(gfp_t gfp_mask)
{
struct page *page;
@@ -163,6 +218,11 @@ static inline struct page *pkram_alloc_page(gfp_t gfp_mask)
static inline void pkram_free_page(void *addr)
{
+ /*
+* The page may have the reserved bit set since preserved pages
+* are reserved early in boot.
+*/
+ ClearPageReserved(virt_to_page(addr));
pkram_remove_identity_map(virt_to_page(addr));
free_page((unsigned long)addr);
}
@@ -201,6 +261,11 @@ static void pkram_truncate_link(struct pkram_link *link)
if (!p)
continue;
page = pfn_to_page(PHYS_PFN(p));
+ /*
+* The p
[RFC v2 15/43] PKRAM: provide a way to ban pages from use by PKRAM
Not all memory ranges can be used for saving preserved over-kexec data.
For example, a kexec kernel may be loaded before pages are preserved.
The memory regions where the kexec segments will be copied to on kexec
must not contain preserved pages or else they will be clobbered.
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 2 +
mm/pkram.c| 205 ++
2 files changed, 207 insertions(+)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index c2099a4f2004..97a7c2ac44a9 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -103,10 +103,12 @@ int pkram_prepare_save(struct pkram_stream *ps, const
char *name,
extern unsigned long pkram_reserved_pages;
void pkram_reserve(void);
void pkram_cleanup(void);
+void pkram_ban_region(unsigned long start, unsigned long end);
#else
#define pkram_reserved_pages 0UL
static inline void pkram_reserve(void) { }
static inline void pkram_cleanup(void) { }
+static inline void pkram_ban_region(unsigned long start, unsigned long end) { }
#endif
#endif /* _LINUX_PKRAM_H */
diff --git a/mm/pkram.c b/mm/pkram.c
index 8670d1633a9d..d15be75c1032 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -141,6 +141,28 @@ struct pkram_super_block {
unsigned long __initdata pkram_reserved_pages;
/*
+ * For tracking a region of memory that PKRAM is not allowed to use.
+ */
+struct banned_region {
+ unsigned long start, end; /* pfn, inclusive */
+};
+
+#define MAX_NR_BANNED (32 + MAX_NUMNODES * 2)
+
+static unsigned int nr_banned; /* number of banned regions */
+
+/* banned regions; arranged in ascending order, do not overlap */
+static struct banned_region banned[MAX_NR_BANNED];
+/*
+ * If a page allocated for PKRAM turns out to belong to a banned region,
+ * it is placed on the banned_pages list so subsequent allocation attempts
+ * do not encounter it again. The list is shrunk when system memory is low.
+ */
+static LIST_HEAD(banned_pages);/* linked through
page::lru */
+static DEFINE_SPINLOCK(banned_pages_lock);
+static unsigned long nr_banned_pages;
+
+/*
* The PKRAM super block pfn, see above.
*/
static int __init parse_pkram_sb_pfn(char *arg)
@@ -207,12 +229,116 @@ void __init pkram_reserve(void)
pr_info("PKRAM: %lu pages reserved\n", pkram_reserved_pages);
}
+/*
+ * Ban pfn range [start..end] (inclusive) from use in PKRAM.
+ */
+void pkram_ban_region(unsigned long start, unsigned long end)
+{
+ int i, merged = -1;
+
+ /* first try to merge the region with an existing one */
+ for (i = nr_banned - 1; i >= 0 && start <= banned[i].end + 1; i--) {
+ if (end + 1 >= banned[i].start) {
+ start = min(banned[i].start, start);
+ end = max(banned[i].end, end);
+ if (merged < 0)
+ merged = i;
+ } else
+ /*
+* Regions are arranged in ascending order and do not
+* intersect so the merged region cannot jump over its
+* predecessors.
+*/
+ BUG_ON(merged >= 0);
+ }
+
+ i++;
+
+ if (merged >= 0) {
+ banned[i].start = start;
+ banned[i].end = end;
+ /* shift if merged with more than one region */
+ memmove(banned + i + 1, banned + merged + 1,
+ sizeof(*banned) * (nr_banned - merged - 1));
+ nr_banned -= merged - i;
+ return;
+ }
+
+ /*
+* The region does not intersect with an existing one;
+* try to create a new one.
+*/
+ if (nr_banned == MAX_NR_BANNED) {
+ pr_err("PKRAM: Failed to ban %lu-%lu: "
+ "Too many banned regions\n", start, end);
+ return;
+ }
+
+ memmove(banned + i + 1, banned + i,
+ sizeof(*banned) * (nr_banned - i));
+ banned[i].start = start;
+ banned[i].end = end;
+ nr_banned++;
+}
+
+static void pkram_show_banned(void)
+{
+ int i;
+ unsigned long n, total = 0;
+
+ pr_info("PKRAM: banned regions:\n");
+ for (i = 0; i < nr_banned; i++) {
+ n = banned[i].end - banned[i].start + 1;
+ pr_info("%4d: [%08lx - %08lx] %ld pages\n",
+ i, banned[i].start, banned[i].end, n);
+ total += n;
+ }
+ pr_info("Total banned: %ld pages in %d regions\n",
+ total, nr_banned);
+}
+
+/*
+ * Returns true if the page may not be used for storing preserved data.
+ */
+static bool pkram_page_banned(struct page *page)
+{
+ unsigned long epfn, pfn = page_to_pfn(pag
Re: [RFC PATCH 3/5] mm: introduce VM_EXEC_KEEP
On 7/29/20 6:52 AM, Kirill A. Shutemov wrote:
> On Mon, Jul 27, 2020 at 10:11:25AM -0700, Anthony Yznaga wrote:
>> A vma with the VM_EXEC_KEEP flag is preserved across exec. For anonymous
>> vmas only. For safety, overlap with fixed address VMAs created in the new
>> mm during exec (e.g. the stack and elf load segments) is not permitted and
>> will cause the exec to fail.
>> (We are studying how to guarantee there are no conflicts. Comments welcome.)
>>
>> Signed-off-by: Steve Sistare
>> Signed-off-by: Anthony Yznaga
>> ---
>> arch/x86/Kconfig | 1 +
>> fs/exec.c | 20
>> include/linux/mm.h | 5 +
>> kernel/fork.c | 2 +-
>> mm/mmap.c | 47 +++
>> 5 files changed, 74 insertions(+), 1 deletion(-)
>>
>> diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
>> index 883da0abf779..fc36eb2f45c0 100644
>> --- a/arch/x86/Kconfig
>> +++ b/arch/x86/Kconfig
>> @@ -30,6 +30,7 @@ config X86_64
>> select MODULES_USE_ELF_RELA
>> select NEED_DMA_MAP_STATE
>> select SWIOTLB
>> +select ARCH_USES_HIGH_VMA_FLAGS
>>
>> config FORCE_DYNAMIC_FTRACE
>> def_bool y
>> diff --git a/fs/exec.c b/fs/exec.c
>> index 262112e5f9f8..1de09c4eef00 100644
>> --- a/fs/exec.c
>> +++ b/fs/exec.c
>> @@ -1069,6 +1069,20 @@ ssize_t read_code(struct file *file, unsigned long
>> addr, loff_t pos, size_t len)
>> EXPORT_SYMBOL(read_code);
>> #endif
>>
>> +static int vma_dup_some(struct mm_struct *old_mm, struct mm_struct *new_mm)
>> +{
>> +struct vm_area_struct *vma;
>> +int ret;
>> +
>> +for (vma = old_mm->mmap; vma; vma = vma->vm_next)
>> +if (vma->vm_flags & VM_EXEC_KEEP) {
>> +ret = vma_dup(vma, new_mm);
>> +if (ret)
>> +return ret;
>> +}
>> +return 0;
>> +}
>> +
>> /*
>> * Maps the mm_struct mm into the current task struct.
>> * On success, this function returns with the mutex
>> @@ -1104,6 +1118,12 @@ static int exec_mmap(struct mm_struct *mm)
>> mutex_unlock(>signal->exec_update_mutex);
>> return -EINTR;
>> }
>> +ret = vma_dup_some(old_mm, mm);
>> +if (ret) {
>> +mmap_read_unlock(old_mm);
>> +mutex_unlock(>signal->exec_update_mutex);
>> +return ret;
>> +}
>> }
>>
>> task_lock(tsk);
>> diff --git a/include/linux/mm.h b/include/linux/mm.h
>> index dc7b87310c10..1c538ba77f33 100644
>> --- a/include/linux/mm.h
>> +++ b/include/linux/mm.h
>> @@ -295,11 +295,15 @@ int overcommit_kbytes_handler(struct ctl_table *, int,
>> void *, size_t *,
>> #define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit
>> architectures */
>> #define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit
>> architectures */
>> #define VM_HIGH_ARCH_BIT_4 36 /* bit only usable on 64-bit
>> architectures */
>> +#define VM_HIGH_ARCH_BIT_5 37 /* bit only usable on 64-bit
>> architectures */
>> #define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0)
>> #define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1)
>> #define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2)
>> #define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3)
>> #define VM_HIGH_ARCH_4 BIT(VM_HIGH_ARCH_BIT_4)
>> +#define VM_EXEC_KEEPBIT(VM_HIGH_ARCH_BIT_5) /* preserve VMA across
>> exec */
>> +#else
>> +#define VM_EXEC_KEEPVM_NONE
>> #endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */
>>
>> #ifdef CONFIG_ARCH_HAS_PKEYS
>> @@ -2534,6 +2538,7 @@ extern struct vm_area_struct *copy_vma(struct
>> vm_area_struct **,
>> unsigned long addr, unsigned long len, pgoff_t pgoff,
>> bool *need_rmap_locks);
>> extern void exit_mmap(struct mm_struct *);
>> +extern int vma_dup(struct vm_area_struct *vma, struct mm_struct *mm);
>>
>> static inline int check_data_rlimit(unsigned long rlim,
>> unsigned long new,
>> diff --git a/kernel/fork.c b/kernel/fork.c
>> index efc5493203ae..15ead613714f 100644
>> --- a/kernel/fork.c
>> +++ b/kernel/fork.c
>> @@ -564,7 +564,7 @@ static __latent_entropy int dup_mmap(struct mm_struct
>> *mm,
>> tmp->anon_vma = NULL;
&g
Re: [RFC PATCH 3/5] mm: introduce VM_EXEC_KEEP
On 7/28/20 6:38 AM, ebied...@xmission.com wrote:
> Anthony Yznaga writes:
>
>> A vma with the VM_EXEC_KEEP flag is preserved across exec. For anonymous
>> vmas only. For safety, overlap with fixed address VMAs created in the new
>> mm during exec (e.g. the stack and elf load segments) is not permitted and
>> will cause the exec to fail.
>> (We are studying how to guarantee there are no conflicts. Comments welcome.)
>>
>> diff --git a/fs/exec.c b/fs/exec.c
>> index 262112e5f9f8..1de09c4eef00 100644
>> --- a/fs/exec.c
>> +++ b/fs/exec.c
>> @@ -1069,6 +1069,20 @@ ssize_t read_code(struct file *file, unsigned long
>> addr, loff_t pos, size_t len)
>> EXPORT_SYMBOL(read_code);
>> #endif
>>
>> +static int vma_dup_some(struct mm_struct *old_mm, struct mm_struct *new_mm)
>> +{
>> +struct vm_area_struct *vma;
>> +int ret;
>> +
>> +for (vma = old_mm->mmap; vma; vma = vma->vm_next)
>> +if (vma->vm_flags & VM_EXEC_KEEP) {
>> +ret = vma_dup(vma, new_mm);
>> +if (ret)
>> +return ret;
>> +}
>> +return 0;
>> +}
>> +
>> /*
>> * Maps the mm_struct mm into the current task struct.
>> * On success, this function returns with the mutex
>> @@ -1104,6 +1118,12 @@ static int exec_mmap(struct mm_struct *mm)
>> mutex_unlock(>signal->exec_update_mutex);
>> return -EINTR;
>> }
>> +ret = vma_dup_some(old_mm, mm);
> ^^
>
> Ouch! An unconditional loop through all of the vmas of the execing
> process, just in case there is a VM_EXEC_KEEP vma.
>
> I know we already walk the list in exit_mmap, but I get the feeling this
> will slow exec down when this feature is not enabled, especially when
> a process with a lot of vmas is calling exec.
Patch 4 changes this to only call vma_dup_some() if the new
binary has opted in to accepting preserved memory.
Anthony
>
>
>> +if (ret) {
>> +mmap_read_unlock(old_mm);
>> +mutex_unlock(>signal->exec_update_mutex);
>> +return ret;
>> +}
>> }
>>
>> task_lock(tsk);
Re: [RFC PATCH 0/5] madvise MADV_DOEXEC
On 7/28/20 4:34 AM, Kirill Tkhai wrote: > On 27.07.2020 20:11, Anthony Yznaga wrote: >> This patchset adds support for preserving an anonymous memory range across >> exec(3) using a new madvise MADV_DOEXEC argument. The primary benefit for >> sharing memory in this manner, as opposed to re-attaching to a named shared >> memory segment, is to ensure it is mapped at the same virtual address in >> the new process as it was in the old one. An intended use for this is to >> preserve guest memory for guests using vfio while qemu exec's an updated >> version of itself. By ensuring the memory is preserved at a fixed address, > So, the goal is an update of QEMU binary without a stopping of virtual > machine? Essentially, yes. The VM is paused very briefly. Anthony > >> vfio mappings and their associated kernel data structures can remain valid. >> In addition, for the qemu use case, qemu instances that back guest RAM with >> anonymous memory can be updated. >> >> Patches 1 and 2 ensure that loading of ELF load segments does not silently >> clobber existing VMAS, and remove assumptions that the stack is the only >> VMA in the mm when the stack is set up. Patch 1 re-introduces the use of >> MAP_FIXED_NOREPLACE to load ELF binaries that addresses the previous issues >> and could be considered on its own. >> >> Patches 3, 4, and 5 introduce the feature and an opt-in method for its use >> using an ELF note. >> >> Anthony Yznaga (5): >> elf: reintroduce using MAP_FIXED_NOREPLACE for elf executable mappings >> mm: do not assume only the stack vma exists in setup_arg_pages() >> mm: introduce VM_EXEC_KEEP >> exec, elf: require opt-in for accepting preserved mem >> mm: introduce MADV_DOEXEC >> >> arch/x86/Kconfig | 1 + >> fs/binfmt_elf.c| 196 >> + >> fs/exec.c | 33 +- >> include/linux/binfmts.h| 7 +- >> include/linux/mm.h | 5 + >> include/uapi/asm-generic/mman-common.h | 3 + >> kernel/fork.c | 2 +- >> mm/madvise.c | 25 + >> mm/mmap.c | 47 >> 9 files changed, 266 insertions(+), 53 deletions(-) >>
[RFC PATCH 5/5] mm: introduce MADV_DOEXEC
madvise MADV_DOEXEC preserves a memory range across exec. Initially
only supported for non-executable, non-stack, anonymous memory.
MADV_DONTEXEC reverts the effect of a previous MADV_DOXEXEC call and
undoes the preservation of the range. After a successful exec call,
the behavior of all ranges reverts to MADV_DONTEXEC.
Signed-off-by: Steve Sistare
Signed-off-by: Anthony Yznaga
---
include/uapi/asm-generic/mman-common.h | 3 +++
mm/madvise.c | 25 +
2 files changed, 28 insertions(+)
diff --git a/include/uapi/asm-generic/mman-common.h
b/include/uapi/asm-generic/mman-common.h
index f94f65d429be..7c5f616b28f7 100644
--- a/include/uapi/asm-generic/mman-common.h
+++ b/include/uapi/asm-generic/mman-common.h
@@ -72,6 +72,9 @@
#define MADV_COLD 20 /* deactivate these pages */
#define MADV_PAGEOUT 21 /* reclaim these pages */
+#define MADV_DOEXEC22 /* do inherit across exec */
+#define MADV_DONTEXEC 23 /* don't inherit across exec */
+
/* compatibility flags */
#define MAP_FILE 0
diff --git a/mm/madvise.c b/mm/madvise.c
index dd1d43cf026d..b447fa748649 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -103,6 +103,26 @@ static long madvise_behavior(struct vm_area_struct *vma,
case MADV_KEEPONFORK:
new_flags &= ~VM_WIPEONFORK;
break;
+ case MADV_DOEXEC:
+ /*
+* MADV_DOEXEC is only supported on private, non-executable,
+* non-stack anonymous memory and if the VM_EXEC_KEEP flag
+* is available.
+*/
+ if (!VM_EXEC_KEEP || vma->vm_file || vma->vm_flags &
(VM_EXEC|VM_SHARED|VM_STACK)) {
+ error = -EINVAL;
+ goto out;
+ }
+ new_flags |= (new_flags & ~VM_MAYEXEC) | VM_EXEC_KEEP;
+ break;
+ case MADV_DONTEXEC:
+ if (!VM_EXEC_KEEP) {
+ error = -EINVAL;
+ goto out;
+ }
+ if (new_flags & VM_EXEC_KEEP)
+ new_flags |= (new_flags & ~VM_EXEC_KEEP) | VM_MAYEXEC;
+ break;
case MADV_DONTDUMP:
new_flags |= VM_DONTDUMP;
break;
@@ -983,6 +1003,8 @@ static int madvise_inject_error(int behavior,
case MADV_SOFT_OFFLINE:
case MADV_HWPOISON:
#endif
+ case MADV_DOEXEC:
+ case MADV_DONTEXEC:
return true;
default:
@@ -1037,6 +1059,9 @@ static int madvise_inject_error(int behavior,
* MADV_DONTDUMP - the application wants to prevent pages in the given range
* from being included in its core dump.
* MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump.
+ * MADV_DOEXEC - On exec, preserve and duplicate this area in the new process
+ * if the new process allows it.
+ * MADV_DONTEXEC - Undo the effect of MADV_DOEXEC.
*
* return values:
* zero- success
--
1.8.3.1
[RFC PATCH 3/5] mm: introduce VM_EXEC_KEEP
A vma with the VM_EXEC_KEEP flag is preserved across exec. For anonymous
vmas only. For safety, overlap with fixed address VMAs created in the new
mm during exec (e.g. the stack and elf load segments) is not permitted and
will cause the exec to fail.
(We are studying how to guarantee there are no conflicts. Comments welcome.)
Signed-off-by: Steve Sistare
Signed-off-by: Anthony Yznaga
---
arch/x86/Kconfig | 1 +
fs/exec.c | 20
include/linux/mm.h | 5 +
kernel/fork.c | 2 +-
mm/mmap.c | 47 +++
5 files changed, 74 insertions(+), 1 deletion(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 883da0abf779..fc36eb2f45c0 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -30,6 +30,7 @@ config X86_64
select MODULES_USE_ELF_RELA
select NEED_DMA_MAP_STATE
select SWIOTLB
+ select ARCH_USES_HIGH_VMA_FLAGS
config FORCE_DYNAMIC_FTRACE
def_bool y
diff --git a/fs/exec.c b/fs/exec.c
index 262112e5f9f8..1de09c4eef00 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -1069,6 +1069,20 @@ ssize_t read_code(struct file *file, unsigned long addr,
loff_t pos, size_t len)
EXPORT_SYMBOL(read_code);
#endif
+static int vma_dup_some(struct mm_struct *old_mm, struct mm_struct *new_mm)
+{
+ struct vm_area_struct *vma;
+ int ret;
+
+ for (vma = old_mm->mmap; vma; vma = vma->vm_next)
+ if (vma->vm_flags & VM_EXEC_KEEP) {
+ ret = vma_dup(vma, new_mm);
+ if (ret)
+ return ret;
+ }
+ return 0;
+}
+
/*
* Maps the mm_struct mm into the current task struct.
* On success, this function returns with the mutex
@@ -1104,6 +1118,12 @@ static int exec_mmap(struct mm_struct *mm)
mutex_unlock(>signal->exec_update_mutex);
return -EINTR;
}
+ ret = vma_dup_some(old_mm, mm);
+ if (ret) {
+ mmap_read_unlock(old_mm);
+ mutex_unlock(>signal->exec_update_mutex);
+ return ret;
+ }
}
task_lock(tsk);
diff --git a/include/linux/mm.h b/include/linux/mm.h
index dc7b87310c10..1c538ba77f33 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -295,11 +295,15 @@ int overcommit_kbytes_handler(struct ctl_table *, int,
void *, size_t *,
#define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit
architectures */
#define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit
architectures */
#define VM_HIGH_ARCH_BIT_4 36 /* bit only usable on 64-bit
architectures */
+#define VM_HIGH_ARCH_BIT_5 37 /* bit only usable on 64-bit
architectures */
#define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0)
#define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1)
#define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2)
#define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3)
#define VM_HIGH_ARCH_4 BIT(VM_HIGH_ARCH_BIT_4)
+#define VM_EXEC_KEEP BIT(VM_HIGH_ARCH_BIT_5) /* preserve VMA across exec */
+#else
+#define VM_EXEC_KEEP VM_NONE
#endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */
#ifdef CONFIG_ARCH_HAS_PKEYS
@@ -2534,6 +2538,7 @@ extern struct vm_area_struct *copy_vma(struct
vm_area_struct **,
unsigned long addr, unsigned long len, pgoff_t pgoff,
bool *need_rmap_locks);
extern void exit_mmap(struct mm_struct *);
+extern int vma_dup(struct vm_area_struct *vma, struct mm_struct *mm);
static inline int check_data_rlimit(unsigned long rlim,
unsigned long new,
diff --git a/kernel/fork.c b/kernel/fork.c
index efc5493203ae..15ead613714f 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -564,7 +564,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
tmp->anon_vma = NULL;
} else if (anon_vma_fork(tmp, mpnt))
goto fail_nomem_anon_vma_fork;
- tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT);
+ tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT | VM_EXEC_KEEP);
file = tmp->vm_file;
if (file) {
struct inode *inode = file_inode(file);
diff --git a/mm/mmap.c b/mm/mmap.c
index 59a4682ebf3f..be2ff53743c3 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -3279,6 +3279,53 @@ struct vm_area_struct *copy_vma(struct vm_area_struct
**vmap,
return NULL;
}
+int vma_dup(struct vm_area_struct *old_vma, struct mm_struct *mm)
+{
+ unsigned long npages;
+ struct mm_struct *old_mm = old_vma->vm_mm;
+ struct vm_area_struct *vma;
+ int ret = -ENOMEM;
+
+ if (WARN_ON(old_vma->vm_file || old_vma->vm_ops))
+ return -EINVAL;
+
+ vma = find_vma(mm, old_vma->vm_start);
+ if (vma && vma->vm_start < old_vm
[RFC PATCH 0/5] madvise MADV_DOEXEC
This patchset adds support for preserving an anonymous memory range across exec(3) using a new madvise MADV_DOEXEC argument. The primary benefit for sharing memory in this manner, as opposed to re-attaching to a named shared memory segment, is to ensure it is mapped at the same virtual address in the new process as it was in the old one. An intended use for this is to preserve guest memory for guests using vfio while qemu exec's an updated version of itself. By ensuring the memory is preserved at a fixed address, vfio mappings and their associated kernel data structures can remain valid. In addition, for the qemu use case, qemu instances that back guest RAM with anonymous memory can be updated. Patches 1 and 2 ensure that loading of ELF load segments does not silently clobber existing VMAS, and remove assumptions that the stack is the only VMA in the mm when the stack is set up. Patch 1 re-introduces the use of MAP_FIXED_NOREPLACE to load ELF binaries that addresses the previous issues and could be considered on its own. Patches 3, 4, and 5 introduce the feature and an opt-in method for its use using an ELF note. Anthony Yznaga (5): elf: reintroduce using MAP_FIXED_NOREPLACE for elf executable mappings mm: do not assume only the stack vma exists in setup_arg_pages() mm: introduce VM_EXEC_KEEP exec, elf: require opt-in for accepting preserved mem mm: introduce MADV_DOEXEC arch/x86/Kconfig | 1 + fs/binfmt_elf.c| 196 + fs/exec.c | 33 +- include/linux/binfmts.h| 7 +- include/linux/mm.h | 5 + include/uapi/asm-generic/mman-common.h | 3 + kernel/fork.c | 2 +- mm/madvise.c | 25 + mm/mmap.c | 47 9 files changed, 266 insertions(+), 53 deletions(-) -- 1.8.3.1
[RFC PATCH 1/5] elf: reintroduce using MAP_FIXED_NOREPLACE for elf executable mappings
Commit b212921b13bd ("elf: don't use MAP_FIXED_NOREPLACE for elf
executable mappings") reverted back to using MAP_FIXED to map elf load
segments because it was found that the load segments in some binaries
overlap and can cause MAP_FIXED_NOREPLACE to fail. The original intent
of MAP_FIXED_NOREPLACE was to prevent the silent clobbering of an
existing mapping (e.g. the stack) by the elf image. To achieve this,
expand on the logic used when loading ET_DYN binaries which calculates a
total size for the image when the first segment is mapped, maps the
entire image, and then unmaps the remainder before remaining segments
are mapped. Apply this to ET_EXEC binaries as well as ET_DYN binaries
as is done now, and for both ET_EXEC and ET_DYN+INTERP use
MAP_FIXED_NOREPLACE for the initial total size mapping and MAP_FIXED for
remaining mappings. For ET_DYN w/out INTERP, continue to map at a
system-selected address in the mmap region.
Signed-off-by: Anthony Yznaga
---
fs/binfmt_elf.c | 112
1 file changed, 64 insertions(+), 48 deletions(-)
diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c
index 9fe3b51c116a..6445a6dbdb1d 100644
--- a/fs/binfmt_elf.c
+++ b/fs/binfmt_elf.c
@@ -1046,58 +1046,25 @@ static int load_elf_binary(struct linux_binprm *bprm)
vaddr = elf_ppnt->p_vaddr;
/*
-* If we are loading ET_EXEC or we have already performed
-* the ET_DYN load_addr calculations, proceed normally.
+* Map remaining segments with MAP_FIXED once the first
+* total size mapping has been done.
*/
- if (elf_ex->e_type == ET_EXEC || load_addr_set) {
+ if (load_addr_set) {
elf_flags |= MAP_FIXED;
- } else if (elf_ex->e_type == ET_DYN) {
- /*
-* This logic is run once for the first LOAD Program
-* Header for ET_DYN binaries to calculate the
-* randomization (load_bias) for all the LOAD
-* Program Headers, and to calculate the entire
-* size of the ELF mapping (total_size). (Note that
-* load_addr_set is set to true later once the
-* initial mapping is performed.)
-*
-* There are effectively two types of ET_DYN
-* binaries: programs (i.e. PIE: ET_DYN with INTERP)
-* and loaders (ET_DYN without INTERP, since they
-* _are_ the ELF interpreter). The loaders must
-* be loaded away from programs since the program
-* may otherwise collide with the loader (especially
-* for ET_EXEC which does not have a randomized
-* position). For example to handle invocations of
-* "./ld.so someprog" to test out a new version of
-* the loader, the subsequent program that the
-* loader loads must avoid the loader itself, so
-* they cannot share the same load range. Sufficient
-* room for the brk must be allocated with the
-* loader as well, since brk must be available with
-* the loader.
-*
-* Therefore, programs are loaded offset from
-* ELF_ET_DYN_BASE and loaders are loaded into the
-* independently randomized mmap region (0 load_bias
-* without MAP_FIXED).
-*/
- if (interpreter) {
- load_bias = ELF_ET_DYN_BASE;
- if (current->flags & PF_RANDOMIZE)
- load_bias += arch_mmap_rnd();
- elf_flags |= MAP_FIXED;
- } else
- load_bias = 0;
-
+ } else {
/*
-* Since load_bias is used for all subsequent loading
-* calculations, we must lower it by the first vaddr
-* so that the remaining calculations based on the
-* ELF vaddrs will be correctly offset. The result
-* is then page aligned.
+* To ensure loading does not continue if an ELF
+* LOAD segment overlaps an existing mapping (e.g.
+* the stack), for the first LOAD Program Header
+* calculate the the entire size of the ELF mapping
+* and map it
[RFC PATCH 4/5] exec, elf: require opt-in for accepting preserved mem
Don't copy preserved VMAs to the binary being exec'd unless the binary has
a "preserved-mem-ok" ELF note.
Signed-off-by: Anthony Yznaga
---
fs/binfmt_elf.c | 84 +
fs/exec.c | 17 +-
include/linux/binfmts.h | 7 -
3 files changed, 100 insertions(+), 8 deletions(-)
diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c
index 6445a6dbdb1d..46248b7b0a75 100644
--- a/fs/binfmt_elf.c
+++ b/fs/binfmt_elf.c
@@ -683,6 +683,81 @@ static unsigned long load_elf_interp(struct elfhdr
*interp_elf_ex,
return error;
}
+#define NOTES_SZ SZ_1K
+#define PRESERVED_MEM_OK_STRING"preserved-mem-ok"
+#define SZ_PRESERVED_MEM_OK_STRING sizeof(PRESERVED_MEM_OK_STRING)
+
+static int parse_elf_note(struct linux_binprm *bprm, const char *data, size_t
*off, size_t datasz)
+{
+ const struct elf_note *nhdr;
+ const char *name;
+ size_t o;
+
+ o = *off;
+ datasz -= o;
+
+ if (datasz < sizeof(*nhdr))
+ return -ENOEXEC;
+
+ nhdr = (const struct elf_note *)(data + o);
+ o += sizeof(*nhdr);
+ datasz -= sizeof(*nhdr);
+
+ /*
+* Currently only the preserved-mem-ok elf note is of interest.
+*/
+ if (nhdr->n_type != 0x07c1feed)
+ goto next;
+
+ if (nhdr->n_namesz > SZ_PRESERVED_MEM_OK_STRING)
+ return -ENOEXEC;
+
+ name = data + o;
+ if (datasz < SZ_PRESERVED_MEM_OK_STRING ||
+ strncmp(name, PRESERVED_MEM_OK_STRING, SZ_PRESERVED_MEM_OK_STRING))
+ return -ENOEXEC;
+
+ bprm->accepts_preserved_mem = 1;
+
+next:
+ o += roundup(nhdr->n_namesz, 4) + roundup(nhdr->n_descsz, 4);
+ *off = o;
+
+ return 0;
+}
+
+static int parse_elf_notes(struct linux_binprm *bprm, struct elf_phdr *phdr)
+{
+ char *notes;
+ size_t notes_sz;
+ size_t off = 0;
+ int ret;
+
+ if (!phdr)
+ return 0;
+
+ notes_sz = phdr->p_filesz;
+ if ((notes_sz > NOTES_SZ) || (notes_sz < sizeof(struct elf_note)))
+ return -ENOEXEC;
+
+ notes = kvmalloc(notes_sz, GFP_KERNEL);
+ if (!notes)
+ return -ENOMEM;
+
+ ret = elf_read(bprm->file, notes, notes_sz, phdr->p_offset);
+ if (ret < 0)
+ goto out;
+
+ while (off < notes_sz) {
+ ret = parse_elf_note(bprm, notes, , notes_sz);
+ if (ret)
+ break;
+ }
+out:
+ kvfree(notes);
+ return ret;
+}
+
/*
* These are the functions used to load ELF style executables and shared
* libraries. There is no binary dependent code anywhere else.
@@ -801,6 +876,7 @@ static int load_elf_binary(struct linux_binprm *bprm)
unsigned long error;
struct elf_phdr *elf_ppnt, *elf_phdata, *interp_elf_phdata = NULL;
struct elf_phdr *elf_property_phdata = NULL;
+ struct elf_phdr *elf_notes_phdata = NULL;
unsigned long elf_bss, elf_brk;
int bss_prot = 0;
int retval, i;
@@ -909,6 +985,10 @@ static int load_elf_binary(struct linux_binprm *bprm)
executable_stack = EXSTACK_DISABLE_X;
break;
+ case PT_NOTE:
+ elf_notes_phdata = elf_ppnt;
+ break;
+
case PT_LOPROC ... PT_HIPROC:
retval = arch_elf_pt_proc(elf_ex, elf_ppnt,
bprm->file, false,
@@ -970,6 +1050,10 @@ static int load_elf_binary(struct linux_binprm *bprm)
if (retval)
goto out_free_dentry;
+ retval = parse_elf_notes(bprm, elf_notes_phdata);
+ if (retval)
+ goto out_free_dentry;
+
/* Flush all traces of the currently running executable */
retval = begin_new_exec(bprm);
if (retval)
diff --git a/fs/exec.c b/fs/exec.c
index 1de09c4eef00..b2b046fec1f8 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -1088,10 +1088,11 @@ static int vma_dup_some(struct mm_struct *old_mm,
struct mm_struct *new_mm)
* On success, this function returns with the mutex
* exec_update_mutex locked.
*/
-static int exec_mmap(struct mm_struct *mm)
+static int exec_mmap(struct linux_binprm *bprm)
{
struct task_struct *tsk;
struct mm_struct *old_mm, *active_mm;
+ struct mm_struct *mm = bprm->mm;
int ret;
/* Notify parent that we're no longer interested in the old VM */
@@ -1118,11 +1119,13 @@ static int exec_mmap(struct mm_struct *mm)
mutex_unlock(>signal->exec_update_mutex);
return -EINTR;
}
- ret = vma_dup_some(old_mm, mm);
- if (ret) {
- mmap_read_unlock(old_mm);
-
[RFC PATCH 2/5] mm: do not assume only the stack vma exists in setup_arg_pages()
In preparation for allowing vmas to be preserved across exec do not
assume that there is no prev vma to pass to mprotect_fixup() in
setup_arg_pages().
Also, setup_arg_pages() expands the initial stack of a process by
128k or to the stack size limit, whichever is smaller. expand_stack()
assumes there is no vma between the vma passed to it and the
address to expand to, so check before calling it.
Signed-off-by: Anthony Yznaga
---
fs/exec.c | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/fs/exec.c b/fs/exec.c
index e6e8a9a70327..262112e5f9f8 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -720,7 +720,7 @@ int setup_arg_pages(struct linux_binprm *bprm,
unsigned long stack_shift;
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma = bprm->vma;
- struct vm_area_struct *prev = NULL;
+ struct vm_area_struct *prev = vma->vm_prev;
unsigned long vm_flags;
unsigned long stack_base;
unsigned long stack_size;
@@ -819,6 +819,10 @@ int setup_arg_pages(struct linux_binprm *bprm,
else
stack_base = vma->vm_start - stack_expand;
#endif
+ if (vma != find_vma(mm, stack_base)) {
+ ret = -EFAULT;
+ goto out_unlock;
+ }
current->mm->start_stack = bprm->p;
ret = expand_stack(vma, stack_base);
if (ret)
--
1.8.3.1
[PATCH 0/3] avoid unnecessary memslot rmap walks
While investigating optimizing qemu start time for large memory guests I found that kvm_mmu_slot_apply_flags() is walking rmaps to update existing sptes when creating or moving a slot but that there won't be any existing sptes to update and any sptes inserted once the new memslot is visible won't need updating. I can't find any reason for this not to be the case, but I've taken a more cautious approach to fixing this by dividing things into three patches. Anthony Yznaga (3): KVM: x86: remove unnecessary rmap walk of read-only memslots KVM: x86: avoid unnecessary rmap walks when creating/moving slots KVM: x86: minor code refactor and comments fixup around dirty logging arch/x86/kvm/x86.c | 106 + 1 file changed, 49 insertions(+), 57 deletions(-) -- 2.13.3
[PATCH 1/3] KVM: x86: remove unnecessary rmap walk of read-only memslots
There's no write access to remove. An existing memslot cannot be updated
to set or clear KVM_MEM_READONLY, and any mappings established in a newly
created or moved read-only memslot will already be read-only.
Signed-off-by: Anthony Yznaga
---
arch/x86/kvm/x86.c | 6 ++
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index c17e6eb9ad43..23fd888e52ee 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -10038,11 +10038,9 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
struct kvm_memory_slot *new)
{
- /* Still write protect RO slot */
- if (new->flags & KVM_MEM_READONLY) {
- kvm_mmu_slot_remove_write_access(kvm, new, PT_PAGE_TABLE_LEVEL);
+ /* Nothing to do for RO slots */
+ if (new->flags & KVM_MEM_READONLY)
return;
- }
/*
* Call kvm_x86_ops dirty logging hooks when they are valid.
--
2.13.3
[PATCH 2/3] KVM: x86: avoid unnecessary rmap walks when creating/moving slots
On large memory guests it has been observed that creating a memslot for a very large range can take noticeable amount of time. Investigation showed that the time is spent walking the rmaps to update existing sptes to remove write access or set/clear dirty bits to support dirty logging. These rmap walks are unnecessary when creating or moving a memslot. A newly created memslot will not have any existing mappings, and the existing mappings of a moved memslot will have been invalidated and flushed. Any mappings established once the new/moved memslot becomes visible will be set using the properties of the new slot. Signed-off-by: Anthony Yznaga --- arch/x86/kvm/x86.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 23fd888e52ee..d211c8ced6bb 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10138,7 +10138,7 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, * * FIXME: const-ify all uses of struct kvm_memory_slot. */ - if (change != KVM_MR_DELETE) + if (change == KVM_MR_FLAGS_ONLY) kvm_mmu_slot_apply_flags(kvm, (struct kvm_memory_slot *) new); /* Free the arrays associated with the old memslot. */ -- 2.13.3
[PATCH 3/3] KVM: x86: minor code refactor and comments fixup around dirty logging
Consolidate the code and correct the comments to show that the actions
taken to update existing mappings to disable or enable dirty logging
are not necessary when creating, moving, or deleting a memslot.
Signed-off-by: Anthony Yznaga
---
arch/x86/kvm/x86.c | 104 +
1 file changed, 49 insertions(+), 55 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index d211c8ced6bb..1e70d188d83a 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -10036,41 +10036,65 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
}
static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
-struct kvm_memory_slot *new)
+struct kvm_memory_slot *old,
+struct kvm_memory_slot *new,
+enum kvm_mr_change change)
{
- /* Nothing to do for RO slots */
- if (new->flags & KVM_MEM_READONLY)
+ /*
+* Nothing to do for RO slots or CREATE/MOVE/DELETE of a slot.
+* See comments below.
+*/
+ if ((change != KVM_MR_FLAGS_ONLY) || (new->flags & KVM_MEM_READONLY))
return;
/*
-* Call kvm_x86_ops dirty logging hooks when they are valid.
-*
-* kvm_x86_ops.slot_disable_log_dirty is called when:
-*
-* - KVM_MR_CREATE with dirty logging is disabled
-* - KVM_MR_FLAGS_ONLY with dirty logging is disabled in new flag
-*
-* The reason is, in case of PML, we need to set D-bit for any slots
-* with dirty logging disabled in order to eliminate unnecessary GPA
-* logging in PML buffer (and potential PML buffer full VMEXIT). This
-* guarantees leaving PML enabled during guest's lifetime won't have
-* any additional overhead from PML when guest is running with dirty
-* logging disabled for memory slots.
+* Dirty logging tracks sptes in 4k granularity, meaning that large
+* sptes have to be split. If live migration is successful, the guest
+* in the source machine will be destroyed and large sptes will be
+* created in the destination. However, if the guest continues to run
+* in the source machine (for example if live migration fails), small
+* sptes will remain around and cause bad performance.
*
-* kvm_x86_ops.slot_enable_log_dirty is called when switching new slot
-* to dirty logging mode.
+* Scan sptes if dirty logging has been stopped, dropping those
+* which can be collapsed into a single large-page spte. Later
+* page faults will create the large-page sptes.
*
-* If kvm_x86_ops dirty logging hooks are invalid, use write protect.
+* There is no need to do this in any of the following cases:
+* CREATE: No dirty mappings will already exist.
+* MOVE/DELETE: The old mappings will already have been cleaned up by
+* kvm_arch_flush_shadow_memslot()
+*/
+ if ((old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
+ !(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
+ kvm_mmu_zap_collapsible_sptes(kvm, new);
+
+ /*
+* Enable or disable dirty logging for the slot.
*
-* In case of write protect:
+* For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of the old
+* slot have been zapped so no dirty logging updates are needed for
+* the old slot.
+* For KVM_MR_CREATE and KVM_MR_MOVE, once the new slot is visible
+* any mappings that might be created in it will consume the
+* properties of the new slot and do not need to be updated here.
*
-* Write protect all pages for dirty logging.
+* When PML is enabled, the kvm_x86_ops dirty logging hooks are
+* called to enable/disable dirty logging.
*
-* All the sptes including the large sptes which point to this
-* slot are set to readonly. We can not create any new large
-* spte on this slot until the end of the logging.
+* When disabling dirty logging with PML enabled, the D-bit is set
+* for sptes in the slot in order to prevent unnecessary GPA
+* logging in the PML buffer (and potential PML buffer full VMEXIT).
+* This guarantees leaving PML enabled for the guest's lifetime
+* won't have any additional overhead from PML when the guest is
+* running with dirty logging disabled.
*
+* When enabling dirty logging, large sptes are write-protected
+* so they can be split on first write. New large sptes cannot
+* be created for this slot until the end of the logging.
* See the comments in fast_page_fault().
+* For small sptes, nothing is done if the dirty log is in the
+* initial-
Re: [RFC 14/43] mm: memblock: PKRAM: prevent memblock resize from clobbering preserved pages
On 5/11/20 6:57 AM, Mike Rapoport wrote:
> On Wed, May 06, 2020 at 05:41:40PM -0700, Anthony Yznaga wrote:
>> The size of the memblock reserved array may be increased while preserved
>> pages are being reserved. When this happens, preserved pages that have
>> not yet been reserved are at risk for being clobbered when space for a
>> larger array is allocated.
>> When called from memblock_double_array(), a wrapper around
>> memblock_find_in_range() walks the preserved pages pagetable to find
>> sufficiently sized ranges without preserved pages and passes them to
>> memblock_find_in_range().
> I'd suggest to create an array of memblock_region's that will contain
> the PKRAM ranges before kexec and pass this array to the new kernel.
> Then, somewhere in start_kerenel() replace replace
> memblock.reserved->regions with that array.
I'll look into doing this. Thanks!
Anthony
>
>> Signed-off-by: Anthony Yznaga
>> ---
>> include/linux/pkram.h | 3 +++
>> mm/memblock.c | 15 +--
>> mm/pkram.c| 51
>> +++
>> 3 files changed, 67 insertions(+), 2 deletions(-)
>>
>> diff --git a/include/linux/pkram.h b/include/linux/pkram.h
>> index edc5d8bef9d3..409022e1472f 100644
>> --- a/include/linux/pkram.h
>> +++ b/include/linux/pkram.h
>> @@ -62,6 +62,9 @@ struct page *pkram_load_page(struct pkram_stream *ps,
>> unsigned long *index,
>> ssize_t pkram_write(struct pkram_stream *ps, const void *buf, size_t count);
>> size_t pkram_read(struct pkram_stream *ps, void *buf, size_t count);
>>
>> +phys_addr_t pkram_memblock_find_in_range(phys_addr_t start, phys_addr_t end,
>> + phys_addr_t size, phys_addr_t align);
>> +
>> #ifdef CONFIG_PKRAM
>> extern unsigned long pkram_reserved_pages;
>> void pkram_reserve(void);
>> diff --git a/mm/memblock.c b/mm/memblock.c
>> index c79ba6f9920c..69ae883b8d21 100644
>> --- a/mm/memblock.c
>> +++ b/mm/memblock.c
>> @@ -16,6 +16,7 @@
>> #include
>> #include
>> #include
>> +#include
>>
>> #include
>> #include
>> @@ -349,6 +350,16 @@ phys_addr_t __init_memblock
>> memblock_find_in_range(phys_addr_t start,
>> return ret;
>> }
>>
>> +phys_addr_t __init_memblock __memblock_find_in_range(phys_addr_t start,
>> +phys_addr_t end, phys_addr_t size,
>> +phys_addr_t align)
>> +{
>> +if (IS_ENABLED(CONFIG_PKRAM))
>> +return pkram_memblock_find_in_range(start, end, size, align);
>> +else
>> +return memblock_find_in_range(start, end, size, align);
>> +}
>> +
>> static void __init_memblock memblock_remove_region(struct memblock_type
>> *type, unsigned long r)
>> {
>> type->total_size -= type->regions[r].size;
>> @@ -447,11 +458,11 @@ static int __init_memblock
>> memblock_double_array(struct memblock_type *type,
>> if (type != )
>> new_area_start = new_area_size = 0;
>>
>> -addr = memblock_find_in_range(new_area_start + new_area_size,
>> +addr = __memblock_find_in_range(new_area_start + new_area_size,
>> memblock.current_limit,
>> new_alloc_size, PAGE_SIZE);
>> if (!addr && new_area_size)
>> -addr = memblock_find_in_range(0,
>> +addr = __memblock_find_in_range(0,
>> min(new_area_start, memblock.current_limit),
>> new_alloc_size, PAGE_SIZE);
>>
>> diff --git a/mm/pkram.c b/mm/pkram.c
>> index dd3c89614010..e49c9bcd3854 100644
>> --- a/mm/pkram.c
>> +++ b/mm/pkram.c
>> @@ -1238,3 +1238,54 @@ void pkram_free_pgt(void)
>> __free_pages_core(virt_to_page(pkram_pgd), 0);
>> pkram_pgd = NULL;
>> }
>> +
>> +static int __init_memblock pkram_memblock_find_cb(struct pkram_pg_state
>> *st, unsigned long base, unsigned long size)
>> +{
>> +unsigned long end = base + size;
>> +unsigned long addr;
>> +
>> +if (size < st->min_size)
>> +return 0;
>> +
>> +addr = memblock_find_in_range(base, end, st->min_size, PAGE_SIZE);
>> +if (!addr)
>> +return 0;
>> +
>> +st->retval = addr;
>> +return 1;
&g
Re: [RFC 21/43] x86/KASLR: PKRAM: support physical kaslr
On 5/7/20 10:51 AM, Kees Cook wrote: > On Wed, May 06, 2020 at 05:41:47PM -0700, Anthony Yznaga wrote: >> Avoid regions of memory that contain preserved pages when computing >> slots used to select where to put the decompressed kernel. > This is changing the slot-walking code instead of updating > mem_avoid_overlap() -- that's where the check for a "reserved" memory > area should live. > > For example, this is how both mem_avoid_memmap() and the setup_data > memory areas are handled. > > Is there a reason mem_avoid_overlap() can't be used here? > I was thinking it would be more efficient to process just the regions that did not contain preserved pages rather than checking for preserved pages in mem_avoid_overlap(), but I see that may just be adding unnecessary complexity. I'll investigate modifying mem_avoid_overlap(). Thank you for the comments! Anthony
Re: [RFC 34/43] shmem: PKRAM: multithread preserving and restoring shmem pages
On 5/7/20 9:30 AM, Randy Dunlap wrote: > On 5/6/20 5:42 PM, Anthony Yznaga wrote: >> Improve performance by multithreading the work to preserve and restore >> shmem pages. >> >> Add 'pkram_max_threads=' kernel option to specify the maximum number >> of threads to use to preserve or restore the pages of a shmem file. >> The default is 16. > Hi, > Please document kernel boot options in > Documentation/admin-guide/kernel-parameters.txt. I'll do that. Thanks! Anthony > >> When preserving pages each thread saves chunks of a file to a pkram_obj >> until no more no more chunks are available. >> >> When restoring pages each thread loads pages using a copy of a >> pkram_stream initialized by pkram_prepare_load_obj(). Under the hood >> each thread ends up fetching and operating on pkram_link pages. >> >> Signed-off-by: Anthony Yznaga >> --- >> include/linux/pkram.h | 2 + >> mm/shmem_pkram.c | 101 >> +- >> 2 files changed, 101 insertions(+), 2 deletions(-) > thanks.
[RFC 34/43] shmem: PKRAM: multithread preserving and restoring shmem pages
Improve performance by multithreading the work to preserve and restore
shmem pages.
Add 'pkram_max_threads=' kernel option to specify the maximum number
of threads to use to preserve or restore the pages of a shmem file.
The default is 16.
When preserving pages each thread saves chunks of a file to a pkram_obj
until no more no more chunks are available.
When restoring pages each thread loads pages using a copy of a
pkram_stream initialized by pkram_prepare_load_obj(). Under the hood
each thread ends up fetching and operating on pkram_link pages.
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 2 +
mm/shmem_pkram.c | 101 +-
2 files changed, 101 insertions(+), 2 deletions(-)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index e71ccb91d6a6..bf2e138b044e 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -13,6 +13,8 @@ struct pkram_stream {
struct pkram_node *node;
struct pkram_obj *obj;
+ int error;
+
struct pkram_link *link;/* current link */
unsigned int entry_idx; /* next entry in link */
diff --git a/mm/shmem_pkram.c b/mm/shmem_pkram.c
index 2f4d0bdf3e05..4992b6c3e54e 100644
--- a/mm/shmem_pkram.c
+++ b/mm/shmem_pkram.c
@@ -126,6 +126,16 @@ static int save_file_content_range(struct address_space
*mapping,
return err;
}
+/* Completion tracking for do_save_file_content_thr() threads */
+static atomic_t pkram_save_n_undone;
+static DECLARE_COMPLETION(pkram_save_all_done_comp);
+
+static inline void pkram_save_report_one_done(void)
+{
+ if (atomic_dec_and_test(_save_n_undone))
+ complete(_save_all_done_comp);
+}
+
static int do_save_file_content(struct pkram_stream *ps)
{
int ret;
@@ -142,11 +152,55 @@ static int do_save_file_content(struct pkram_stream *ps)
return ret;
}
+static int do_save_file_content_thr(void *data)
+{
+ struct pkram_stream *ps = data;
+ struct pkram_stream pslocal = *ps;
+ int ret;
+
+ ret = do_save_file_content();
+ if (ret && !ps->error)
+ ps->error = ret;
+
+ pkram_save_report_one_done();
+ return 0;
+}
+#define PKRAM_DEFAULT_MAX_THREADS 16
+
+static int pkram_max_threads = PKRAM_DEFAULT_MAX_THREADS;
+
+static int __init set_pkram_max_threads(char *str)
+{
+ unsigned int val;
+
+ if (kstrtouint(str, 0, ))
+ return 1;
+
+ pkram_max_threads = val;
+
+ return 1;
+}
+__setup("pkram_max_threads=", set_pkram_max_threads);
+
static int save_file_content(struct pkram_stream *ps)
{
+ unsigned int thr, nr_threads;
+
+ nr_threads = num_online_cpus() - 1;
+ nr_threads = clamp_val(pkram_max_threads, 1, nr_threads);
+
ps->max_idx = DIV_ROUND_UP(i_size_read(ps->mapping->host), PAGE_SIZE);
- return do_save_file_content(ps);
+ if (nr_threads == 1)
+ return do_save_file_content(ps);
+
+ atomic_set(_save_n_undone, nr_threads);
+ for (thr = 0; thr < nr_threads; thr++)
+ kthread_run(do_save_file_content_thr, ps, "pkram_save%d", thr);
+
+ wait_for_completion(_save_all_done_comp);
+
+ return ps->error;
}
static int save_file(struct dentry *dentry, struct pkram_stream *ps)
@@ -248,7 +302,17 @@ int shmem_save_pkram(struct super_block *sb)
return err;
}
-static int load_file_content(struct pkram_stream *ps)
+/* Completion tracking for do_load_file_content_thr() threads */
+static atomic_t pkram_load_n_undone;
+static DECLARE_COMPLETION(pkram_load_all_done_comp);
+
+static inline void pkram_load_report_one_done(void)
+{
+ if (atomic_dec_and_test(_load_n_undone))
+ complete(_load_all_done_comp);
+}
+
+static int do_load_file_content(struct pkram_stream *ps)
{
unsigned long index;
struct page *page;
@@ -266,6 +330,39 @@ static int load_file_content(struct pkram_stream *ps)
return err;
}
+static int do_load_file_content_thr(void *data)
+{
+ struct pkram_stream *ps = data;
+ struct pkram_stream pslocal = *ps;
+ int ret;
+
+ ret = do_load_file_content();
+ if (ret && !ps->error)
+ ps->error = ret;
+
+ pkram_load_report_one_done();
+ return 0;
+}
+
+static int load_file_content(struct pkram_stream *ps)
+{
+ unsigned int thr, nr_threads;
+
+ nr_threads = num_online_cpus() - 1;
+ nr_threads = clamp_val(pkram_max_threads, 1, nr_threads);
+
+ if (nr_threads == 1)
+ return do_load_file_content(ps);
+
+ atomic_set(_load_n_undone, nr_threads);
+ for (thr = 0; thr < nr_threads; thr++)
+ kthread_run(do_load_file_content_thr, ps, "pkram_load%d", thr);
+
+ wait_for_completion(_load_all_done_comp);
+
+ return ps->error;
+}
+
static int load_file(struct
[RFC 27/43] x86/mm/numa: add numa_isolate_memblocks()
Provide a way for a caller external to numa to ensure memblocks in the
memblock reserved list do not cross node boundaries and have a node id
assigned to them. This will be used by PKRAM to ensure initialization of
page structs for preserved pages can be deferred and multithreaded
efficiently.
Signed-off-by: Anthony Yznaga
---
arch/x86/include/asm/numa.h | 4
arch/x86/mm/numa.c | 32
2 files changed, 24 insertions(+), 12 deletions(-)
diff --git a/arch/x86/include/asm/numa.h b/arch/x86/include/asm/numa.h
index bbfde3d2662f..f9e05f4eb1c6 100644
--- a/arch/x86/include/asm/numa.h
+++ b/arch/x86/include/asm/numa.h
@@ -40,6 +40,7 @@ static inline void set_apicid_to_node(int apicid, s16 node)
}
extern int numa_cpu_node(int cpu);
+extern void __init numa_isolate_memblocks(void);
#else /* CONFIG_NUMA */
static inline void set_apicid_to_node(int apicid, s16 node)
@@ -50,6 +51,9 @@ static inline int numa_cpu_node(int cpu)
{
return NUMA_NO_NODE;
}
+static inline void numa_isolate_memblocks(void)
+{
+}
#endif /* CONFIG_NUMA */
#ifdef CONFIG_X86_32
diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c
index 59ba008504dc..df0065e24ea5 100644
--- a/arch/x86/mm/numa.c
+++ b/arch/x86/mm/numa.c
@@ -475,6 +475,25 @@ static bool __init numa_meminfo_cover_memory(const struct
numa_meminfo *mi)
return true;
}
+void __init numa_isolate_memblocks(void)
+{
+ int i;
+
+ /*
+* Iterate over all memory known to the x86 architecture,
+* and use those ranges to set the nid in memblock.reserved.
+* This will split up the memblock regions along node
+* boundaries and will set the node IDs as well.
+*/
+ for (i = 0; i < numa_meminfo.nr_blks; i++) {
+ struct numa_memblk *mb = numa_meminfo.blk + i;
+ int ret;
+
+ ret = memblock_set_node(mb->start, mb->end - mb->start,
, mb->nid);
+ WARN_ON_ONCE(ret);
+ }
+}
+
/*
* Mark all currently memblock-reserved physical memory (which covers the
* kernel's own memory ranges) as hot-unswappable.
@@ -493,19 +512,8 @@ static void __init numa_clear_kernel_node_hotplug(void)
* used by the kernel, but those regions are not split up
* along node boundaries yet, and don't necessarily have their
* node ID set yet either.
-*
-* So iterate over all memory known to the x86 architecture,
-* and use those ranges to set the nid in memblock.reserved.
-* This will split up the memblock regions along node
-* boundaries and will set the node IDs as well.
*/
- for (i = 0; i < numa_meminfo.nr_blks; i++) {
- struct numa_memblk *mb = numa_meminfo.blk + i;
- int ret;
-
- ret = memblock_set_node(mb->start, mb->end - mb->start,
, mb->nid);
- WARN_ON_ONCE(ret);
- }
+ numa_isolate_memblocks();
/*
* Now go over all reserved memblock regions, to construct a
--
2.13.3
[RFC 29/43] memblock: PKRAM: mark memblocks that contain preserved pages
To support deferred initialization of page structs for preserved pages,
separate memblocks containing preserved pages by setting a new flag
when adding them to the memblock reserved list.
Signed-off-by: Anthony Yznaga
---
include/linux/memblock.h | 7 +++
mm/memblock.c| 8 +++-
mm/pkram.c | 4 ++--
3 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/include/linux/memblock.h b/include/linux/memblock.h
index 6bc37a731d27..27ab2b30ae1d 100644
--- a/include/linux/memblock.h
+++ b/include/linux/memblock.h
@@ -37,6 +37,7 @@ enum memblock_flags {
MEMBLOCK_HOTPLUG= 0x1, /* hotpluggable region */
MEMBLOCK_MIRROR = 0x2, /* mirrored region */
MEMBLOCK_NOMAP = 0x4, /* don't add to kernel direct mapping */
+ MEMBLOCK_PRESERVED = 0x8, /* preserved pages region */
};
/**
@@ -111,6 +112,7 @@ void memblock_allow_resize(void);
int memblock_add_node(phys_addr_t base, phys_addr_t size, int nid);
int memblock_add(phys_addr_t base, phys_addr_t size);
int memblock_remove(phys_addr_t base, phys_addr_t size);
+int __memblock_reserve(phys_addr_t base, phys_addr_t size, enum memblock_flags
flags);
int memblock_free(phys_addr_t base, phys_addr_t size);
int memblock_reserve(phys_addr_t base, phys_addr_t size);
#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
@@ -215,6 +217,11 @@ static inline bool memblock_is_nomap(struct
memblock_region *m)
return m->flags & MEMBLOCK_NOMAP;
}
+static inline bool memblock_is_preserved(struct memblock_region *m)
+{
+ return m->flags & MEMBLOCK_PRESERVED;
+}
+
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
int memblock_search_pfn_nid(unsigned long pfn, unsigned long *start_pfn,
unsigned long *end_pfn);
diff --git a/mm/memblock.c b/mm/memblock.c
index 69ae883b8d21..1a9a2055ed11 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -831,6 +831,12 @@ int __init_memblock memblock_free(phys_addr_t base,
phys_addr_t size)
return memblock_remove_range(, base, size);
}
+int __init_memblock __memblock_reserve(phys_addr_t base, phys_addr_t size,
+ enum memblock_flags flags)
+{
+ return memblock_add_range(, base, size, MAX_NUMNODES,
flags);
+}
+
int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
{
phys_addr_t end = base + size - 1;
@@ -838,7 +844,7 @@ int __init_memblock memblock_reserve(phys_addr_t base,
phys_addr_t size)
memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
, , (void *)_RET_IP_);
- return memblock_add_range(, base, size, MAX_NUMNODES,
0);
+ return __memblock_reserve(base, size, 0);
}
#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
diff --git a/mm/pkram.c b/mm/pkram.c
index 97a7dd0a5b7d..b83d31740619 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -170,7 +170,7 @@ static int __init pkram_reserve_page(unsigned long pfn)
size = PAGE_SIZE;
if (memblock_is_region_reserved(base, size) ||
- memblock_reserve(base, size) < 0)
+ __memblock_reserve(base, size, MEMBLOCK_PRESERVED) < 0)
err = -EBUSY;
if (!err)
@@ -1446,7 +1446,7 @@ static void pkram_remove_identity_map(struct page *page)
static int __init pkram_reserve_range_cb(struct pkram_pg_state *st, unsigned
long base, unsigned long size)
{
if (memblock_is_region_reserved(base, size) ||
- memblock_reserve(base, size) < 0) {
+ __memblock_reserve(base, size, MEMBLOCK_PRESERVED) < 0) {
pr_warn("PKRAM: reservations exist in [0x%lx,0x%lx]\n", base,
base + size - 1);
/*
* Set a lower bound so another walk can undo the earlier,
--
2.13.3
[RFC 23/43] mm: shmem: enable saving to PKRAM
This patch illustrates how the PKRAM API can be used for preserving tmpfs.
Two options are added to tmpfs:
The 'pkram=' option specifies the PKRAM node to load/save the
filesystem tree from/to.
The 'preserve' option initiates preservation of a read-only
filesystem tree.
If the 'pkram=' options is passed on mount, shmem will look for the
corresponding PKRAM node and load the FS tree from it.
If the 'pkram=' options was passed on mount and the 'preserve' option is
passed on remount and the filesystem is read-only, shmem will save the
FS tree to the PKRAM node.
A typical usage scenario looks like:
# mount -t tmpfs -o pkram=mytmpfs none /mnt
# echo something > /mnt/smth
# mount -o remount ro,preserve /mnt
# mount -t tmpfs -o pkram=mytmpfs none /mnt
# cat /mnt/smth
Each FS tree is saved into a PKRAM node, and each file is saved into a
PKRAM object. A byte stream written to the object is used for saving file
metadata (name, permissions, etc) while the page stream written to
the object accommodates file content pages and their offsets.
This implementation serves as a demonstration and therefore is
simplified: it supports only regular files in the root directory without
multiple hard links, and it does not save swapped out files and aborts if
any are found. However, it can be elaborated to fully support tmpfs.
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h| 1 +
include/linux/shmem_fs.h | 24 +++
mm/Makefile | 2 +-
mm/shmem.c | 66
mm/shmem_pkram.c | 381 +++
5 files changed, 473 insertions(+), 1 deletion(-)
create mode 100644 mm/shmem_pkram.c
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index 1cd518843d7a..b47b3aef16e3 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -17,6 +17,7 @@ struct pkram_stream {
unsigned int entry_idx; /* next entry in link */
unsigned long next_index;
+ struct address_space *mapping;
/* byte data */
struct page *data_page;
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index 688b92cd4ec7..f2ce9937a8f2 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -26,6 +26,11 @@ struct shmem_inode_info {
struct inodevfs_inode;
};
+#define SHMEM_PKRAM_NAME_MAX 128
+struct shmem_pkram_info {
+ char name[SHMEM_PKRAM_NAME_MAX];
+};
+
struct shmem_sb_info {
unsigned long max_blocks; /* How many blocks are allowed */
struct percpu_counter used_blocks; /* How many are allocated */
@@ -40,6 +45,8 @@ struct shmem_sb_info {
spinlock_t shrinklist_lock; /* Protects shrinklist */
struct list_head shrinklist; /* List of shinkable inodes */
unsigned long shrinklist_len; /* Length of shrinklist */
+ struct shmem_pkram_info *pkram;
+ bool preserve; /* PKRAM-enabled data is preserved */
};
static inline struct shmem_inode_info *SHMEM_I(struct inode *inode)
@@ -99,6 +106,23 @@ extern int shmem_getpage(struct inode *inode, pgoff_t index,
extern int shmem_insert_page(struct mm_struct *mm, struct inode *inode,
pgoff_t index, struct page *page);
+#ifdef CONFIG_PKRAM
+extern int shmem_parse_pkram(const char *str, struct shmem_pkram_info **pkram);
+extern void shmem_show_pkram(struct seq_file *seq, struct shmem_pkram_info
*pkram,
+ bool preserve);
+extern int shmem_save_pkram(struct super_block *sb);
+extern void shmem_load_pkram(struct super_block *sb);
+extern int shmem_release_pkram(struct super_block *sb);
+#else
+static inline int shmem_parse_pkram(const char *str,
+ struct shmem_pkram_info **pkram) { return 1; }
+static inline void shmem_show_pkram(struct seq_file *seq,
+ struct shmem_pkram_info *pkram, bool preserve) { }
+static inline int shmem_save_pkram(struct super_block *sb) { return 0; }
+static inline void shmem_load_pkram(struct super_block *sb) { }
+static inline int shmem_release_pkram(struct super_block *sb) { return 0; }
+#endif
+
static inline struct page *shmem_read_mapping_page(
struct address_space *mapping, pgoff_t index)
{
diff --git a/mm/Makefile b/mm/Makefile
index c4ad1c56e237..5c07ecaa5a38 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -112,4 +112,4 @@ obj-$(CONFIG_MEMFD_CREATE) += memfd.o
obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o
obj-$(CONFIG_PTDUMP_CORE) += ptdump.o
obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o
-obj-$(CONFIG_PKRAM) += pkram.o pkram_pagetable.o
+obj-$(CONFIG_PKRAM) += pkram.o pkram_pagetable.o shmem_pkram.o
diff --git a/mm/shmem.c b/mm/shmem.c
index 0a9a2166e51f..9c28ef657cd1 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -112,14 +112,18 @@ struct shmem_options {
unsigned long long blocks;
unsigned long long ino
[RFC 33/43] PKRAM: atomically add and remove link pages
Add and remove pkram_link pages from a pkram_obj atomically to prepare
for multithreading.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 27 ++-
1 file changed, 18 insertions(+), 9 deletions(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index 5f4e4d12865f..042c14dedc25 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -551,22 +551,31 @@ static void pkram_truncate(void)
static void pkram_add_link(struct pkram_link *link, struct pkram_obj *obj)
{
- link->link_pfn = obj->link_pfn;
- obj->link_pfn = page_to_pfn(virt_to_page(link));
+ __u64 link_pfn = page_to_pfn(virt_to_page(link));
+ __u64 *head = >link_pfn;
+
+ do {
+ link->link_pfn = *head;
+ } while (cmpxchg64(head, link->link_pfn, link_pfn) != link->link_pfn);
}
static struct pkram_link *pkram_remove_link(struct pkram_obj *obj)
{
struct pkram_link *current_link;
+ __u64 *head = >link_pfn;
+ __u64 head_pfn = *head;
+
+ while (head_pfn) {
+ current_link = pfn_to_kaddr(head_pfn);
+ if (cmpxchg64(head, head_pfn, current_link->link_pfn) ==
head_pfn) {
+ current_link->link_pfn = 0;
+ return current_link;
+ }
- if (!obj->link_pfn)
- return NULL;
-
- current_link = pfn_to_kaddr(obj->link_pfn);
- obj->link_pfn = current_link->link_pfn;
- current_link->link_pfn = 0;
+ head_pfn = *head;
+ }
- return current_link;
+ return NULL;
}
static void pkram_stream_init(struct pkram_stream *ps,
--
2.13.3
[RFC 17/43] PKRAM: provide a way to check if a memory range has preserved pages
When a kernel is loaded for kexec the address ranges where the kexec
segments will be copied to may conflict with pages already set to be
preserved. Provide a way to determine if preserved pages exist in a
specified range.
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 2 ++
mm/pkram.c| 25 +
2 files changed, 27 insertions(+)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index 1ba48442ef8e..1cd518843d7a 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -70,11 +70,13 @@ extern unsigned long pkram_reserved_pages;
void pkram_reserve(void);
void pkram_free_pgt(void);
void pkram_ban_region(unsigned long start, unsigned long end);
+int pkram_has_preserved_pages(unsigned long start, unsigned long end);
#else
#define pkram_reserved_pages 0UL
static inline void pkram_reserve(void) { }
static inline void pkram_free_pgt(void) { }
static inline void pkram_ban_region(unsigned long start, unsigned long end) { }
+static inline int pkram_has_preserved_pages(unsigned long start, unsigned long
end) { return 0; }
#endif
#endif /* _LINUX_PKRAM_H */
diff --git a/mm/pkram.c b/mm/pkram.c
index 60863c8ecbab..0aaaf9b79682 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -1499,3 +1499,28 @@ phys_addr_t __init_memblock
pkram_memblock_find_in_range(phys_addr_t start,
return st.retval;
}
+
+static int pkram_has_preserved_pages_cb(struct pkram_pg_state *st, unsigned
long base, unsigned long size)
+{
+ st->retval = 1;
+ return 1;
+}
+
+/*
+ * Check whether the memory range [start, end) contains preserved pages.
+ */
+int pkram_has_preserved_pages(unsigned long start, unsigned long end)
+{
+ struct pkram_pg_state st = {
+ .range_cb = pkram_has_preserved_pages_cb,
+ .min_addr = start,
+ .max_addr = end,
+ };
+
+ if (!pkram_pgd)
+ return 0;
+
+ pkram_walk_pgt_rev(, pkram_pgd);
+
+ return st.retval;
+}
--
2.13.3
[RFC 41/43] XArray: add xas_export_node() and xas_import_node()
Contention on the xarray lock when multiple threads are adding to the
same xarray can be mitigated by providing a way to add entries in
bulk.
Allow a caller to allocate and populate an xarray node outside of
the target xarray and then only take the xarray lock long enough to
import the node into it.
Signed-off-by: Anthony Yznaga
---
Documentation/core-api/xarray.rst | 8 +++
include/linux/xarray.h| 2 +
lib/test_xarray.c | 45 +
lib/xarray.c | 100 ++
4 files changed, 155 insertions(+)
diff --git a/Documentation/core-api/xarray.rst
b/Documentation/core-api/xarray.rst
index 640934b6f7b4..659e10df8901 100644
--- a/Documentation/core-api/xarray.rst
+++ b/Documentation/core-api/xarray.rst
@@ -444,6 +444,14 @@ called each time the XArray updates a node. This is used
by the page
cache workingset code to maintain its list of nodes which contain only
shadow entries.
+xas_export_node() is used to remove and return a node from an XArray
+while xas_import_node() is used to add a node to an XArray. Together
+these can be used, for example, to reduce lock contention when multiple
+threads are updating an XArray by allowing a caller to allocate and
+populate a node outside of the target XArray in a local XArray, export
+the node, and then take the target XArray lock just long enough to import
+the node.
+
Multi-Index Entries
---
diff --git a/include/linux/xarray.h b/include/linux/xarray.h
index 14c893433139..73bd8ccc4424 100644
--- a/include/linux/xarray.h
+++ b/include/linux/xarray.h
@@ -1504,6 +1504,8 @@ bool xas_nomem(struct xa_state *, gfp_t);
void xas_pause(struct xa_state *);
void xas_create_range(struct xa_state *);
+struct xa_node *xas_export_node(struct xa_state *xas);
+void xas_import_node(struct xa_state *xas, struct xa_node *node);
/**
* xas_reload() - Refetch an entry from the xarray.
diff --git a/lib/test_xarray.c b/lib/test_xarray.c
index d4f97925dbd8..5cfaa1720cc1 100644
--- a/lib/test_xarray.c
+++ b/lib/test_xarray.c
@@ -1682,6 +1682,50 @@ static noinline void check_destroy(struct xarray *xa)
#endif
}
+static noinline void check_export_import_1(struct xarray *xa,
+ unsigned long index, unsigned int order)
+{
+ int xa_shift = order + XA_CHUNK_SHIFT - (order % XA_CHUNK_SHIFT);
+ XA_STATE(xas, xa, index);
+ struct xa_node *node;
+ unsigned long i;
+
+ xa_store_many_order(xa, index, xa_shift);
+
+ xas_lock();
+ xas_set_order(, index, xa_shift);
+ node = xas_export_node();
+ xas_unlock();
+
+ XA_BUG_ON(xa, !xa_empty(xa));
+
+ do {
+ xas_lock();
+ xas_set_order(, index, xa_shift);
+ xas_import_node(, node);
+ xas_unlock();
+ } while (xas_nomem(, GFP_KERNEL));
+
+ for (i = index; i < index + (1UL << xa_shift); i++)
+ xa_erase_index(xa, i);
+
+ XA_BUG_ON(xa, !xa_empty(xa));
+}
+
+static noinline void check_export_import(struct xarray *xa)
+{
+ unsigned int order;
+ unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 12 : 1;
+
+ for (order = 0; order < max_order; order += XA_CHUNK_SHIFT) {
+ int xa_shift = order + XA_CHUNK_SHIFT;
+ unsigned long j;
+
+ for (j = 0; j < XA_CHUNK_SIZE; j++)
+ check_export_import_1(xa, j << xa_shift, order);
+ }
+}
+
static DEFINE_XARRAY(array);
static int xarray_checks(void)
@@ -1712,6 +1756,7 @@ static int xarray_checks(void)
check_workingset(, 0);
check_workingset(, 64);
check_workingset(, 4096);
+ check_export_import();
printk("XArray: %u of %u tests passed\n", tests_passed, tests_run);
return (tests_run == tests_passed) ? 0 : -EINVAL;
diff --git a/lib/xarray.c b/lib/xarray.c
index e9e641d3c0c3..478925780e87 100644
--- a/lib/xarray.c
+++ b/lib/xarray.c
@@ -507,6 +507,30 @@ static void xas_delete_node(struct xa_state *xas)
xas_shrink(xas);
}
+static void xas_unlink_node(struct xa_state *xas)
+{
+ struct xa_node *node = xas->xa_node;
+ struct xa_node *parent;
+
+ parent = xa_parent_locked(xas->xa, node);
+ xas->xa_node = parent;
+ xas->xa_offset = node->offset;
+
+ if (!parent) {
+ xas->xa->xa_head = NULL;
+ xas->xa_node = XAS_BOUNDS;
+ return;
+ }
+
+ parent->slots[xas->xa_offset] = NULL;
+ parent->count--;
+ XA_NODE_BUG_ON(parent, parent->count > XA_CHUNK_SIZE);
+
+ xas_update(xas, parent);
+
+ xas_delete_node(xas);
+}
+
/**
* xas_free_nodes() - Free this node and all nodes that it references
* @xas: Array operation state.
@@ -1540,6 +1564,82 @@ static void xas_set_range(struct xa_state *xas, unsigned
long
[RFC 38/43] mm: implement splicing a list of pages to the LRU
Considerable contention on the LRU lock happens when multiple threads
are used to insert pages into a shmem file in parallel. To alleviate this
provide a way for pages to be added to the same LRU to be staged so that
they can be added by splicing lists and updating stats once with the lock
held. For now only unevictable pages are supported.
Signed-off-by: Anthony Yznaga
---
include/linux/swap.h | 11 ++
mm/swap.c| 101 +++
2 files changed, 112 insertions(+)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index e1bbf7a16b27..462045f536a8 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -346,6 +346,17 @@ extern void swap_setup(void);
extern void lru_cache_add_active_or_unevictable(struct page *page,
struct vm_area_struct *vma);
+struct lru_splice {
+ struct list_headsplice;
+ struct list_head*lru_head;
+ struct pglist_data *pgdat;
+ struct lruvec *lruvec;
+ enum lru_list lru;
+ unsigned long nr_pages[MAX_NR_ZONES];
+ unsigned long pgculled;
+};
+extern void lru_splice_add_anon(struct page *page, struct lru_splice *splice);
+extern void add_splice_to_lru_list(struct lru_splice *splice);
/* linux/mm/vmscan.c */
extern unsigned long zone_reclaimable_pages(struct zone *zone);
diff --git a/mm/swap.c b/mm/swap.c
index bf9a79fed62d..848f8b516471 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -187,6 +187,107 @@ int get_kernel_page(unsigned long start, int write,
struct page **pages)
}
EXPORT_SYMBOL_GPL(get_kernel_page);
+/*
+ * Update stats and move accumulated pages from an lru_splice to the lru.
+ */
+void add_splice_to_lru_list(struct lru_splice *splice)
+{
+ struct pglist_data *pgdat = splice->pgdat;
+ struct lruvec *lruvec = splice->lruvec;
+ enum lru_list lru = splice->lru;
+ unsigned long flags = 0;
+ int zid;
+
+ spin_lock_irqsave(>lru_lock, flags);
+ for (zid = 0; zid < MAX_NR_ZONES; zid++) {
+ if (splice->nr_pages[zid])
+ update_lru_size(lruvec, lru, zid,
splice->nr_pages[zid]);
+ }
+ count_vm_events(UNEVICTABLE_PGCULLED, splice->pgculled);
+ list_splice(>splice, splice->lru_head);
+ spin_unlock_irqrestore(>lru_lock, flags);
+
+ splice->lru_head = NULL;
+ splice->pgculled = 0;
+}
+
+static void add_page_to_lru_splice(struct page *page,
+ struct lru_splice *splice,
+ struct lruvec *lruvec, enum lru_list lru)
+{
+ int zid;
+
+ if (splice->lru_head == >lists[lru]) {
+ list_add(>lru, >splice);
+ splice->nr_pages[page_zonenum(page)] += hpage_nr_pages(page);
+ return;
+ }
+
+ INIT_LIST_HEAD(>splice);
+ splice->lruvec = lruvec;
+ splice->lru_head = >lists[lru];
+ splice->lru = lru;
+ list_add(>lru, >splice);
+ for (zid = 0; zid < MAX_NR_ZONES; zid++)
+ splice->nr_pages[zid] = 0;
+ splice->nr_pages[page_zonenum(page)] = hpage_nr_pages(page);
+
+}
+
+/*
+ * Similar in functionality to __pagevec_lru_add_fn() but here the page is
+ * being added to an lru_splice and the LRU lock is not held.
+ */
+static void page_lru_splice_add(struct page *page, struct lruvec *lruvec,
struct lru_splice *splice)
+{
+ enum lru_list lru;
+ int was_unevictable = TestClearPageUnevictable(page);
+
+ VM_BUG_ON_PAGE(PageLRU(page), page);
+ /* XXX only supports unevictable pages at the moment */
+ VM_BUG_ON_PAGE(was_unevictable, page);
+
+ SetPageLRU(page);
+ smp_mb();
+
+ lru = LRU_UNEVICTABLE;
+ ClearPageActive(page);
+ SetPageUnevictable(page);
+ if (!was_unevictable)
+ splice->pgculled++;
+
+ add_page_to_lru_splice(page, splice, lruvec, lru);
+ trace_mm_lru_insertion(page, lru);
+}
+
+static void lru_splice_add(struct page *page, struct lru_splice *splice)
+{
+ struct pglist_data *pagepgdat, *pgdat = splice->pgdat;
+ struct lruvec *lruvec;
+
+ pagepgdat = page_pgdat(page);
+
+ if (pagepgdat != pgdat) {
+ if (pgdat)
+ add_splice_to_lru_list(splice);
+ splice->pgdat = pagepgdat;
+ }
+
+ lruvec = mem_cgroup_page_lruvec(page, pagepgdat);
+ page_lru_splice_add(page, lruvec, splice);
+}
+
+void lru_splice_add_anon(struct page *page, struct lru_splice *splice)
+{
+ if (PageActive(page))
+ ClearPageActive(page);
+ get_page(page);
+
+ lru_splice_add(page, splice);
+
+ put_page(page);
+}
+
static void pagevec_lru_move_fn(struct pagevec *pvec,
void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg),
void *arg)
--
2.13.3
[RFC 26/43] mm: shmem: when inserting, handle pages already charged to a memcg
If shmem_insert_page() is called to insert a page that was preserved
using PKRAM on the current boot (i.e. preserved page is restored without
an intervening kexec boot), the page will still be charged to a memory
cgroup because it is never freed. Don't try to charge it again.
Signed-off-by: Anthony Yznaga
---
mm/shmem.c | 21 +
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/mm/shmem.c b/mm/shmem.c
index 13475073fb52..1f3b43b8fa34 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -693,6 +693,7 @@ int shmem_insert_page(struct mm_struct *mm, struct inode
*inode, pgoff_t index,
struct mem_cgroup *memcg;
pgoff_t hindex = index;
bool on_lru = PageLRU(page);
+ bool has_memcg = page->mem_cgroup ? true : false;
if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
return -EFBIG;
@@ -738,20 +739,24 @@ int shmem_insert_page(struct mm_struct *mm, struct inode
*inode, pgoff_t index,
__SetPageReferenced(page);
- err = mem_cgroup_try_charge_delay(page, mm, gfp, ,
- PageTransHuge(page));
- if (err)
- goto out_unlock;
+ if (!has_memcg) {
+ err = mem_cgroup_try_charge_delay(page, mm, gfp, ,
+ PageTransHuge(page));
+ if (err)
+ goto out_unlock;
+ }
err = shmem_add_to_page_cache(page, mapping, hindex,
NULL, gfp & GFP_RECLAIM_MASK);
if (err) {
- mem_cgroup_cancel_charge(page, memcg,
- PageTransHuge(page));
+ if (!has_memcg)
+ mem_cgroup_cancel_charge(page, memcg,
+ PageTransHuge(page));
goto out_unlock;
}
- mem_cgroup_commit_charge(page, memcg, on_lru,
- PageTransHuge(page));
+ if (!has_memcg)
+ mem_cgroup_commit_charge(page, memcg, on_lru,
+ PageTransHuge(page));
if (!on_lru)
lru_cache_add_anon(page);
--
2.13.3
[RFC 43/43] PKRAM: improve index alignment of pkram_link entries
To take advantage of optimizations when adding pages to the page cache
via shmem_insert_pages(), improve the likelihood that the pages array
passed to shmem_insert_pages() starts on an aligned index. Do this
when preserving pages by starting a new pkram_link page when the current
page is aligned and the next aligned page will not fit on the pkram_link
page.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 14 --
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index ef092aa5ce7a..416c3ca4411b 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -913,11 +913,21 @@ static int __pkram_save_page(struct pkram_stream *ps,
{
struct pkram_link *link = ps->link;
struct pkram_obj *obj = ps->obj;
+ int order, align, align_cnt;
pkram_entry_t p;
- int order;
+
+ if (PageTransHuge(page)) {
+ align = 1 << (HPAGE_PMD_ORDER + XA_CHUNK_SHIFT -
(HPAGE_PMD_ORDER % XA_CHUNK_SHIFT));
+ align_cnt = align >> HPAGE_PMD_ORDER;
+ } else {
+ align = XA_CHUNK_SIZE;
+ align_cnt = XA_CHUNK_SIZE;
+ }
if (!link || ps->entry_idx >= PKRAM_LINK_ENTRIES_MAX ||
- index != ps->next_index) {
+ index != ps->next_index ||
+ (IS_ALIGNED(index, align) &&
+ (ps->entry_idx + align_cnt > PKRAM_LINK_ENTRIES_MAX))) {
struct page *link_page;
link_page = pkram_alloc_page((ps->gfp_mask & GFP_RECLAIM_MASK) |
--
2.13.3
[RFC 36/43] PKRAM: add support for loading pages in bulk
This patch adds three functions:
pkram_prepare_load_pages()
Called after calling pkram_prepare_load_obj()
pkram_load_pages()
Loads some number of pages that are contiguous by their original
file index values. The index of the first page, an array of the
page pointers, and the number of pages in the array are provided
to the caller.
pkram_finish_load_pages()
Called when no more pages will be loaded from the pkram_obj.
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 6 +++
mm/pkram.c| 106 ++
2 files changed, 112 insertions(+)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index bf2e138b044e..3f059791f88c 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -18,6 +18,9 @@ struct pkram_stream {
struct pkram_link *link;/* current link */
unsigned int entry_idx; /* next entry in link */
+ struct page **pages;
+ unsigned int nr_pages;
+
unsigned long next_index;
struct address_space *mapping;
struct mm_struct *mm;
@@ -60,14 +63,17 @@ void pkram_discard_save(struct pkram_stream *ps);
int pkram_prepare_load(struct pkram_stream *ps, const char *name);
int pkram_prepare_load_obj(struct pkram_stream *ps);
+int pkram_prepare_load_pages(struct pkram_stream *ps);
void pkram_finish_load(struct pkram_stream *ps);
void pkram_finish_load_obj(struct pkram_stream *ps);
+void pkram_finish_load_pages(struct pkram_stream *ps);
#define PKRAM_PAGE_TRANS_HUGE 0x1 /* page is a transparent hugepage */
int pkram_save_page(struct pkram_stream *ps, struct page *page, short flags);
struct page *pkram_load_page(struct pkram_stream *ps, unsigned long *index,
short *flags);
+int pkram_load_pages(struct pkram_stream *ps, unsigned long *index);
ssize_t pkram_write(struct pkram_stream *ps, const void *buf, size_t count);
size_t pkram_read(struct pkram_stream *ps, void *buf, size_t count);
diff --git a/mm/pkram.c b/mm/pkram.c
index 042c14dedc25..ef092aa5ce7a 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -820,6 +820,37 @@ int pkram_prepare_load_obj(struct pkram_stream *ps)
}
/**
+ * Initialize stream @ps for loading preserved pages from it.
+ *
+ * Returns 0 on success, -errno on failure.
+ *
+ * Error values:
+ * %ENOMEM: insufficient memory available
+ *
+ * After the load has finished, pkram_finish_load_pages() is to be called.
+ */
+int pkram_prepare_load_pages(struct pkram_stream *ps)
+{
+ BUG_ON((ps->node->flags & PKRAM_ACCMODE_MASK) != PKRAM_LOAD);
+
+ ps->pages = kzalloc(PAGE_SIZE, GFP_KERNEL);
+ if (!ps->pages)
+ return -ENOMEM;
+
+ return 0;
+}
+
+/**
+ * Finish the load of preserved pages started with pkram_prepare_load_pages()
+ */
+void pkram_finish_load_pages(struct pkram_stream *ps)
+{
+ BUG_ON((ps->node->flags & PKRAM_ACCMODE_MASK) != PKRAM_LOAD);
+
+ kfree(ps->pages);
+}
+
+/**
* Finish the load of a preserved memory object started with
* pkram_prepare_load_obj() freeing the object and any data that has not
* been loaded from it.
@@ -1066,6 +1097,81 @@ struct page *pkram_load_page(struct pkram_stream *ps,
unsigned long *index, shor
}
/**
+ * Load pages from the preserved memory object associated with stream
+ * @ps. The stream must have been initialized with pkram_prepare_load(),
+ * pkram_prepare_load_obj(), and pkram_prepare_load_pages().
+ * The page entries of a single pkram_link are processed, and the stream
+ * 'pages' buffer is populated with the page pointers.
+ *
+ * Returns 0 if one or more pages are loaded or -ENODATA if there are no
+ * pages to load.
+ *
+ * The pages loaded have an incremented refcount either because the page
+ * was initialized with a refcount of 1 at boot or because the page was
+ * subsequently preserved which increased the refcount.
+ */
+int pkram_load_pages(struct pkram_stream *ps, unsigned long *index)
+{
+ struct pkram_link *link = ps->link;
+ int nr_entries = 0;
+ int i;
+
+ if (!link) {
+ link = pkram_remove_link(ps->obj);
+ if (!link)
+ return -ENODATA;
+ }
+
+ *index = link->index;
+
+ for (i = 0; i < PKRAM_LINK_ENTRIES_MAX; i++) {
+ unsigned long p = link->entry[i];
+ struct page *page;
+ short flags;
+
+ if (!p)
+ break;
+
+ flags = (p >> PKRAM_ENTRY_FLAGS_SHIFT) & PKRAM_ENTRY_FLAGS_MASK;
+ nr_entries++;
+
+ page = pfn_to_page(PHYS_PFN(p));
+ ps->pages[i] = page;
+
+ if (flags & PKRAM_PAGE_TRANS_HUGE) {
+ int order = p & PKRAM_ENTRY_ORDER_MASK;
+ int nr_pages = 1 << order;
+
[RFC 04/43] mm: PKRAM: implement page stream operations
Using the pkram_save_page() function, one can populate PKRAM objects with
memory pages which can later be loaded using the pkram_load_page()
function. Saving a memory page to PKRAM is accomplished by recording
its pfn and incrementing its refcount so that it will not be freed after
the last user puts it.
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 5 ++
mm/pkram.c| 219 +-
2 files changed, 221 insertions(+), 3 deletions(-)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index fabde2cd8203..f338d1c2aeb6 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -12,6 +12,11 @@ struct pkram_stream {
gfp_t gfp_mask;
struct pkram_node *node;
struct pkram_obj *obj;
+
+ struct pkram_link *link;/* current link */
+ unsigned int entry_idx; /* next entry in link */
+
+ unsigned long next_index;
};
#define PKRAM_NAME_MAX 256 /* including nul */
diff --git a/mm/pkram.c b/mm/pkram.c
index 4934ffd8b019..ab3053ca3539 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
#include
#include
+#include
#include
#include
#include
@@ -10,8 +11,38 @@
#include
#include
+#include "internal.h"
+
+
+/*
+ * Represents a reference to a data page saved to PKRAM.
+ */
+typedef __u64 pkram_entry_t;
+
+#define PKRAM_ENTRY_FLAGS_SHIFT0x5
+#define PKRAM_ENTRY_FLAGS_MASK 0x7f
+
+/*
+ * Keeps references to data pages saved to PKRAM.
+ * The structure occupies a memory page.
+ */
+struct pkram_link {
+ __u64 link_pfn; /* points to the next link of the object */
+ __u64 index; /* mapping index of first pkram_entry_t */
+
+ /*
+* the array occupies the rest of the link page; if the link is not
+* full, the rest of the array must be filled with zeros
+*/
+ pkram_entry_t entry[0];
+};
+
+#define PKRAM_LINK_ENTRIES_MAX \
+ ((PAGE_SIZE-sizeof(struct pkram_link))/sizeof(pkram_entry_t))
+
struct pkram_obj {
- __u64 obj_pfn;/* points to the next object in the list */
+ __u64 link_pfn; /* points to the first link of the object */
+ __u64 obj_pfn;/* points to the next object in the list */
};
/*
@@ -19,6 +50,10 @@ struct pkram_obj {
* independently of each other. The nodes are identified by unique name
* strings.
*
+ * References to data pages saved to a preserved memory node are kept in a
+ * singly-linked list of PKRAM link structures (see above), the node has a
+ * pointer to the head of.
+ *
* The structure occupies a memory page.
*/
struct pkram_node {
@@ -68,6 +103,37 @@ static struct pkram_node *pkram_find_node(const char *name)
return NULL;
}
+static void pkram_truncate_link(struct pkram_link *link)
+{
+ struct page *page;
+ pkram_entry_t p;
+ int i;
+
+ for (i = 0; i < PKRAM_LINK_ENTRIES_MAX; i++) {
+ p = link->entry[i];
+ if (!p)
+ continue;
+ page = pfn_to_page(PHYS_PFN(p));
+ put_page(page);
+ }
+}
+
+static void pkram_truncate_obj(struct pkram_obj *obj)
+{
+ unsigned long link_pfn;
+ struct pkram_link *link;
+
+ link_pfn = obj->link_pfn;
+ while (link_pfn) {
+ link = pfn_to_kaddr(link_pfn);
+ pkram_truncate_link(link);
+ link_pfn = link->link_pfn;
+ pkram_free_page(link);
+ cond_resched();
+ }
+ obj->link_pfn = 0;
+}
+
static void pkram_truncate_node(struct pkram_node *node)
{
unsigned long obj_pfn;
@@ -76,6 +142,7 @@ static void pkram_truncate_node(struct pkram_node *node)
obj_pfn = node->obj_pfn;
while (obj_pfn) {
obj = pfn_to_kaddr(obj_pfn);
+ pkram_truncate_obj(obj);
obj_pfn = obj->obj_pfn;
pkram_free_page(obj);
cond_resched();
@@ -83,6 +150,26 @@ static void pkram_truncate_node(struct pkram_node *node)
node->obj_pfn = 0;
}
+static void pkram_add_link(struct pkram_link *link, struct pkram_obj *obj)
+{
+ link->link_pfn = obj->link_pfn;
+ obj->link_pfn = page_to_pfn(virt_to_page(link));
+}
+
+static struct pkram_link *pkram_remove_link(struct pkram_obj *obj)
+{
+ struct pkram_link *current_link;
+
+ if (!obj->link_pfn)
+ return NULL;
+
+ current_link = pfn_to_kaddr(obj->link_pfn);
+ obj->link_pfn = current_link->link_pfn;
+ current_link->link_pfn = 0;
+
+ return current_link;
+}
+
static void pkram_stream_init(struct pkram_stream *ps,
struct pkram_node *node, gfp_t gfp_mask)
{
@@ -94,6 +181,9 @@ static void pkram_stream_init(struct pkram_stream *ps,
[RFC 05/43] mm: PKRAM: support preserving transparent hugepages
Support preserving a transparent hugepage by recording the page order and
a flag indicating it is a THP. Use these values when the page is
restored to reconstruct the THP.
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 2 ++
mm/pkram.c| 20
2 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index f338d1c2aeb6..584cadb662b4 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -33,6 +33,8 @@ int pkram_prepare_load_obj(struct pkram_stream *ps);
void pkram_finish_load(struct pkram_stream *ps);
void pkram_finish_load_obj(struct pkram_stream *ps);
+#define PKRAM_PAGE_TRANS_HUGE 0x1 /* page is a transparent hugepage */
+
int pkram_save_page(struct pkram_stream *ps, struct page *page, short flags);
struct page *pkram_load_page(struct pkram_stream *ps, unsigned long *index,
short *flags);
diff --git a/mm/pkram.c b/mm/pkram.c
index ab3053ca3539..9164060e36f5 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -21,6 +21,7 @@ typedef __u64 pkram_entry_t;
#define PKRAM_ENTRY_FLAGS_SHIFT0x5
#define PKRAM_ENTRY_FLAGS_MASK 0x7f
+#define PKRAM_ENTRY_ORDER_MASK 0x1f
/*
* Keeps references to data pages saved to PKRAM.
@@ -434,6 +435,7 @@ static int __pkram_save_page(struct pkram_stream *ps,
struct pkram_link *link = ps->link;
struct pkram_obj *obj = ps->obj;
pkram_entry_t p;
+ int order;
if (!link || ps->entry_idx >= PKRAM_LINK_ENTRIES_MAX ||
index != ps->next_index) {
@@ -452,10 +454,15 @@ static int __pkram_save_page(struct pkram_stream *ps,
ps->next_index = link->index = index;
}
- ps->next_index++;
+ if (PageTransHuge(page))
+ flags |= PKRAM_PAGE_TRANS_HUGE;
+
+ order = compound_order(page);
+ ps->next_index += (1 << order);
get_page(page);
p = page_to_phys(page);
+ p |= order;
p |= ((flags & PKRAM_ENTRY_FLAGS_MASK) << PKRAM_ENTRY_FLAGS_SHIFT);
link->entry[ps->entry_idx] = p;
ps->entry_idx++;
@@ -485,8 +492,6 @@ int pkram_save_page(struct pkram_stream *ps, struct page
*page, short flags)
BUG_ON((node->flags & PKRAM_ACCMODE_MASK) != PKRAM_SAVE);
- BUG_ON(PageCompound(page));
-
return __pkram_save_page(ps, page, flags, page->index);
}
@@ -499,6 +504,7 @@ static struct page *__pkram_load_page(struct pkram_stream
*ps, unsigned long *in
struct pkram_link *link = ps->link;
struct page *page;
pkram_entry_t p;
+ int order;
short flgs;
if (!link) {
@@ -517,14 +523,20 @@ static struct page *__pkram_load_page(struct pkram_stream
*ps, unsigned long *in
BUG_ON(!p);
flgs = (p >> PKRAM_ENTRY_FLAGS_SHIFT) & PKRAM_ENTRY_FLAGS_MASK;
+ order = p & PKRAM_ENTRY_ORDER_MASK;
page = pfn_to_page(PHYS_PFN(p));
+ if (flgs & PKRAM_PAGE_TRANS_HUGE) {
+ prep_compound_page(page, order);
+ prep_transhuge_page(page);
+ }
+
if (flags)
*flags = flgs;
if (index)
*index = ps->next_index;
- ps->next_index++;
+ ps->next_index += (1 << order);
/* clear to avoid double free (see pkram_truncate_link()) */
link->entry[ps->entry_idx] = 0;
--
2.13.3
[RFC 42/43] shmem: reduce time holding xa_lock when inserting pages
Rather than adding one page at a time to the page cache and taking the
page cache xarray lock each time, where possible add pages in bulk by
first populating an xarray node outside of the page cache before taking
the lock to insert it.
When a group of pages to be inserted will fill an xarray node, add them
to a local xarray, export the xarray node, and then take the lock on the
page cache xarray and insert the node.
Signed-off-by: Anthony Yznaga
---
mm/shmem.c | 145 ++---
1 file changed, 138 insertions(+), 7 deletions(-)
diff --git a/mm/shmem.c b/mm/shmem.c
index f621d863e362..9d3c4e1f2dc1 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -732,17 +732,130 @@ static void shmem_delete_from_page_cache(struct page
*page, void *radswap)
BUG_ON(error);
}
+static int shmem_add_aligned_to_page_cache(struct page *pages[], int npages,
+ struct address_space *mapping,
+ pgoff_t index, gfp_t gfp, int order)
+{
+ int xa_shift = order + XA_CHUNK_SHIFT - (order % XA_CHUNK_SHIFT);
+ XA_STATE_ORDER(xas, >i_pages, index, xa_shift);
+ struct xa_state *xas_ptr =
+ struct xarray xa_tmp;
+ /*
+* Specify order so xas_create_range() only needs to be called once
+* to allocate the entire range. This guarantees that xas_store()
+* will not fail due to lack of memory.
+* Specify index == 0 so the minimum necessary nodes are allocated.
+*/
+ XA_STATE_ORDER(xas_tmp, _tmp, 0, xa_shift);
+ unsigned long nr = 1UL << order;
+ struct xa_node *node;
+ int i;
+
+ if (npages * nr != 1 << xa_shift) {
+ WARN_ONCE(1, "npages (%d) not aligned to xa_shift\n", npages);
+ return -EINVAL;
+ }
+ if (!IS_ALIGNED(index, 1 << xa_shift)) {
+ WARN_ONCE(1, "index (%lu) not aligned to xa_shift\n", index);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < npages; i++) {
+ VM_BUG_ON_PAGE(PageTail(pages[i]), pages[i]);
+ VM_BUG_ON_PAGE(!PageLocked(pages[i]), pages[i]);
+ VM_BUG_ON_PAGE(!PageSwapBacked(pages[i]), pages[i]);
+
+ page_ref_add(pages[i], nr);
+ pages[i]->mapping = mapping;
+ pages[i]->index = index + (i * nr);
+ }
+
+ xa_init(_tmp);
+ do {
+ xas_lock(_tmp);
+ xas_create_range(_tmp);
+ if (xas_error(_tmp))
+ goto unlock;
+ for (i = 0; i < npages; i++) {
+ int j = 0;
+next:
+ xas_store(_tmp, pages[i]);
+ if (++j < nr) {
+ xas_next(_tmp);
+ goto next;
+ }
+ if (i < npages - 1)
+ xas_next(_tmp);
+ }
+ xas_set_order(_tmp, 0, xa_shift);
+ node = xas_export_node(_tmp);
+unlock:
+ xas_unlock(_tmp);
+ } while (xas_nomem(_tmp, gfp));
+
+ if (xas_error(_tmp)) {
+ xas_ptr = _tmp;
+ goto error;
+ }
+
+ do {
+ xas_lock_irq();
+ xas_import_node(, node);
+ if (xas_error())
+ goto unlock1;
+ mapping->nrpages += nr * npages;
+ xas_unlock();
+ for (i = 0; i < npages; i++) {
+ __mod_node_page_state(page_pgdat(pages[i]),
NR_FILE_PAGES, nr);
+ __mod_node_page_state(page_pgdat(pages[i]), NR_SHMEM,
nr);
+ if (PageTransHuge(pages[i])) {
+ count_vm_event(THP_FILE_ALLOC);
+ __inc_node_page_state(pages[i], NR_SHMEM_THPS);
+ }
+ }
+ local_irq_enable();
+ break;
+unlock1:
+ xas_unlock_irq();
+ } while (xas_nomem(, gfp));
+
+ if (!xas_error())
+ return 0;
+
+error:
+ for (i = 0; i < npages; i++) {
+ pages[i]->mapping = NULL;
+ page_ref_sub(pages[i], nr);
+ }
+ return xas_error(xas_ptr);
+}
+
static int shmem_add_pages_to_cache(struct page *pages[], int npages,
struct address_space *mapping,
pgoff_t start, gfp_t gfp)
{
pgoff_t index = start;
int err = 0;
- int i;
+ int i, j;
i = 0;
while (i < npages) {
if (PageTransHuge(pages[i])) {
+ if (IS_ALIGNED(index, 4096) && i+8 <= npages) {
+ for (j =
[RFC 39/43] shmem: optimize adding pages to the LRU in shmem_insert_pages()
Reduce LRU lock contention when inserting shmem pages by staging pages to be added to the same LRU and adding them en masse. Signed-off-by: Anthony Yznaga --- mm/shmem.c | 8 +++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mm/shmem.c b/mm/shmem.c index ca5edf580f24..678a396ba8d3 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -789,9 +789,12 @@ int shmem_insert_pages(struct mm_struct *mm, struct inode *inode, pgoff_t index, struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); gfp_t gfp = mapping_gfp_mask(mapping); struct mem_cgroup *memcg; + struct lru_splice splice; int i, err; int nr = 0; + memset(, 0, sizeof(splice)); + for (i = 0; i < npages; i++) nr += compound_nr(pages[i]); @@ -866,7 +869,7 @@ int shmem_insert_pages(struct mm_struct *mm, struct inode *inode, pgoff_t index, } if (!PageLRU(pages[i])) - lru_cache_add_anon(pages[i]); + lru_splice_add_anon(pages[i], ); flush_dcache_page(pages[i]); SetPageUptodate(pages[i]); @@ -875,6 +878,9 @@ int shmem_insert_pages(struct mm_struct *mm, struct inode *inode, pgoff_t index, unlock_page(pages[i]); } + if (splice.pgdat) + add_splice_to_lru_list(); + return 0; out_truncate: -- 2.13.3
[RFC 10/43] PKRAM: add code for walking the preserved pages pagetable
Add the ability to walk the pkram pagetable from high to low addresses
and execute a callback for each contiguous range of preserved or not
preserved memory found. The reason for walking high to low is to align
with high to low memblock allocation when finding holes that memblocks
can safely be allocated from as will be seen in a later patch.
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 15 +
mm/Makefile | 2 +-
mm/pkram_pagetable.c | 169 ++
3 files changed, 185 insertions(+), 1 deletion(-)
create mode 100644 mm/pkram_pagetable.c
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index a58dd2ea835a..b6fa973d37cc 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -25,6 +25,21 @@ struct pkram_stream {
#define PKRAM_NAME_MAX 256 /* including nul */
+struct pkram_pg_state {
+ int (*range_cb)(struct pkram_pg_state *state, unsigned long base,
+ unsigned long size);
+ unsigned long curr_addr;
+ unsigned long end_addr;
+ unsigned long min_addr;
+ unsigned long max_addr;
+ unsigned long min_size;
+ bool tracking;
+ bool find_holes;
+ unsigned long retval;
+};
+
+void pkram_walk_pgt_rev(struct pkram_pg_state *st, pgd_t *pgd);
+
int pkram_prepare_save(struct pkram_stream *ps, const char *name,
gfp_t gfp_mask);
int pkram_prepare_save_obj(struct pkram_stream *ps);
diff --git a/mm/Makefile b/mm/Makefile
index 59cd381194af..c4ad1c56e237 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -112,4 +112,4 @@ obj-$(CONFIG_MEMFD_CREATE) += memfd.o
obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o
obj-$(CONFIG_PTDUMP_CORE) += ptdump.o
obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o
-obj-$(CONFIG_PKRAM) += pkram.o
+obj-$(CONFIG_PKRAM) += pkram.o pkram_pagetable.o
diff --git a/mm/pkram_pagetable.c b/mm/pkram_pagetable.c
new file mode 100644
index ..d31aa36207ba
--- /dev/null
+++ b/mm/pkram_pagetable.c
@@ -0,0 +1,169 @@
+// SPDX-License-Identifier: GPL-2.0
+#include
+#include
+#include
+
+#define pgd_none(a) (pgtable_l5_enabled() ? pgd_none(a) :
p4d_none(__p4d(pgd_val(a
+
+static int note_page_rev(struct pkram_pg_state *st, unsigned long curr_size,
bool present)
+{
+ unsigned long curr_addr = st->curr_addr;
+ bool track_page = present ^ st->find_holes;
+
+ if (!st->tracking && track_page) {
+ unsigned long end_addr = curr_addr + curr_size;
+
+ if (end_addr <= st->min_addr)
+ return 1;
+
+ st->end_addr = min(end_addr, st->max_addr);
+ st->tracking = true;
+ } else if (st->tracking) {
+ unsigned long base, size;
+
+ /* Continue tracking if lower bound has not been reached */
+ if (track_page && curr_addr && curr_addr >= st->min_addr)
+ return 0;
+
+ if (!track_page)
+ base = max(curr_addr + curr_size, st->min_addr);
+ else
+ base = st->min_addr;
+
+ size = st->end_addr - base;
+ st->tracking = false;
+
+ return st->range_cb(st, base, size);
+ }
+
+ return 0;
+}
+
+static int walk_pte_level_rev(struct pkram_pg_state *st, pmd_t addr, unsigned
long P)
+{
+ unsigned long *bitmap;
+ int present;
+ int i, ret;
+
+ bitmap = __va(pmd_val(addr));
+ for (i = PTRS_PER_PTE - 1; i >= 0; i--) {
+ unsigned long curr_addr = P + i * PAGE_SIZE;
+
+ if (curr_addr >= st->max_addr)
+ continue;
+ st->curr_addr = curr_addr;
+
+ present = test_bit(i, bitmap);
+ ret = note_page_rev(st, PAGE_SIZE, present);
+ if (ret)
+ break;
+ }
+
+ return ret;
+}
+
+static int walk_pmd_level_rev(struct pkram_pg_state *st, pud_t addr, unsigned
long P)
+{
+ pmd_t *start;
+ int i, ret;
+
+ start = (pmd_t *)pud_page_vaddr(addr) + PTRS_PER_PMD - 1;
+ for (i = PTRS_PER_PMD - 1; i >= 0; i--, start--) {
+ unsigned long curr_addr = P + i * PMD_SIZE;
+
+ if (curr_addr >= st->max_addr)
+ continue;
+ st->curr_addr = curr_addr;
+
+ if (!pmd_none(*start)) {
+ if (pmd_large(*start))
+ ret = note_page_rev(st, PMD_SIZE, true);
+ else
+ ret = walk_pte_level_rev(st, *start, curr_addr);
+ } else
+ ret = note_page_rev(st, PMD_SIZE, false);
+ if (ret)
+ break;
+ }
+
+ return ret;
+}
+
+static int wal
[RFC 15/43] PKRAM: provide a way to ban pages from use by PKRAM
Not all memory ranges can be used for saving preserved over-kexec data.
For example, a kexec kernel may be loaded before pages are preserved.
The memory regions where the kexec segments will be copied to on kexec
must not contain preserved pages or else they will be clobbered.
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 2 +
mm/pkram.c| 210 ++
2 files changed, 212 insertions(+)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index 409022e1472f..1ba48442ef8e 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -69,10 +69,12 @@ phys_addr_t pkram_memblock_find_in_range(phys_addr_t start,
phys_addr_t end,
extern unsigned long pkram_reserved_pages;
void pkram_reserve(void);
void pkram_free_pgt(void);
+void pkram_ban_region(unsigned long start, unsigned long end);
#else
#define pkram_reserved_pages 0UL
static inline void pkram_reserve(void) { }
static inline void pkram_free_pgt(void) { }
+static inline void pkram_ban_region(unsigned long start, unsigned long end) { }
#endif
#endif /* _LINUX_PKRAM_H */
diff --git a/mm/pkram.c b/mm/pkram.c
index e49c9bcd3854..60863c8ecbab 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -119,6 +119,28 @@ unsigned long __initdata pkram_reserved_pages;
static bool pkram_reservation_in_progress;
/*
+ * For tracking a region of memory that PKRAM is not allowed to use.
+ */
+struct banned_region {
+ unsigned long start, end; /* pfn, inclusive */
+};
+
+#define MAX_NR_BANNED (32 + MAX_NUMNODES * 2)
+
+static unsigned int nr_banned; /* number of banned regions */
+
+/* banned regions; arranged in ascending order, do not overlap */
+static struct banned_region banned[MAX_NR_BANNED];
+/*
+ * If a page allocated for PKRAM turns out to belong to a banned region,
+ * it is placed on the banned_pages list so subsequent allocation attempts
+ * do not encounter it again. The list is shrunk when system memory is low.
+ */
+static LIST_HEAD(banned_pages);/* linked through
page::lru */
+static DEFINE_SPINLOCK(banned_pages_lock);
+static unsigned long nr_banned_pages;
+
+/*
* The PKRAM super block pfn, see above.
*/
static int __init parse_pkram_sb_pfn(char *arg)
@@ -223,12 +245,120 @@ void __init pkram_reserve(void)
pr_info("PKRAM: %lu pages reserved\n", pkram_reserved_pages);
}
+/*
+ * Ban pfn range [start..end] (inclusive) from use in PKRAM.
+ */
+void pkram_ban_region(unsigned long start, unsigned long end)
+{
+ int i, merged = -1;
+
+ if (pkram_reservation_in_progress)
+ return;
+
+ /* first try to merge the region with an existing one */
+ for (i = nr_banned - 1; i >= 0 && start <= banned[i].end + 1; i--) {
+ if (end + 1 >= banned[i].start) {
+ start = min(banned[i].start, start);
+ end = max(banned[i].end, end);
+ if (merged < 0)
+ merged = i;
+ } else
+ /*
+* Regions are arranged in ascending order and do not
+* intersect so the merged region cannot jump over its
+* predecessors.
+*/
+ BUG_ON(merged >= 0);
+ }
+
+ i++;
+
+ if (merged >= 0) {
+ banned[i].start = start;
+ banned[i].end = end;
+ /* shift if merged with more than one region */
+ memmove(banned + i + 1, banned + merged + 1,
+ sizeof(*banned) * (nr_banned - merged - 1));
+ nr_banned -= merged - i;
+ return;
+ }
+
+ /*
+* The region does not intersect with an existing one;
+* try to create a new one.
+*/
+ if (nr_banned == MAX_NR_BANNED) {
+ pr_err("PKRAM: Failed to ban %lu-%lu: "
+ "Too many banned regions\n", start, end);
+ return;
+ }
+
+ memmove(banned + i + 1, banned + i,
+ sizeof(*banned) * (nr_banned - i));
+ banned[i].start = start;
+ banned[i].end = end;
+ nr_banned++;
+}
+
+static void pkram_show_banned(void)
+{
+ int i;
+ unsigned long n, total = 0;
+
+ pr_info("PKRAM: banned regions:\n");
+ for (i = 0; i < nr_banned; i++) {
+ n = banned[i].end - banned[i].start + 1;
+ pr_info("%4d: [%08lx - %08lx] %ld pages\n",
+ i, banned[i].start, banned[i].end, n);
+ total += n;
+ }
+ pr_info("Total banned: %ld pages in %d regions\n",
+ total, nr_banned);
+}
+
+/*
+ * Returns true if the page may not be used for storing preserved data.
+ */
+static b
[RFC 40/43] shmem: initial support for adding multiple pages to pagecache
shmem_insert_pages() currently loops over the array of pages passed
to it and calls shmem_add_to_page_cache() for each one. Prepare
for adding pages to the pagecache in bulk by adding and using a
shmem_add_pages_to_cache() call. For now it just iterates over
an array and adds pages individually, but improvements in performance
when multiple threads are adding to the same pagecache are achieved
by calling a new shmem_add_to_page_cache_fast() function that does
not check for conflicts and drops the xarray lock before updating stats.
Signed-off-by: Anthony Yznaga
---
mm/shmem.c | 95 ++
1 file changed, 84 insertions(+), 11 deletions(-)
diff --git a/mm/shmem.c b/mm/shmem.c
index 678a396ba8d3..f621d863e362 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -660,6 +660,57 @@ static int shmem_add_to_page_cache(struct page *page,
return 0;
}
+static int shmem_add_to_page_cache_fast(struct page *page,
+ struct address_space *mapping,
+ pgoff_t index, gfp_t gfp)
+{
+ XA_STATE_ORDER(xas, >i_pages, index, compound_order(page));
+ unsigned long nr = compound_nr(page);
+ unsigned long i = 0;
+
+ VM_BUG_ON_PAGE(PageTail(page), page);
+ VM_BUG_ON_PAGE(index != round_down(index, nr), page);
+ VM_BUG_ON_PAGE(!PageLocked(page), page);
+ VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
+
+ page_ref_add(page, nr);
+ page->mapping = mapping;
+ page->index = index;
+
+ do {
+ xas_lock_irq();
+ xas_create_range();
+ if (xas_error())
+ goto unlock;
+next:
+ xas_store(, page);
+ if (++i < nr) {
+ xas_next();
+ goto next;
+ }
+ mapping->nrpages += nr;
+ xas_unlock();
+ if (PageTransHuge(page)) {
+ count_vm_event(THP_FILE_ALLOC);
+ __inc_node_page_state(page, NR_SHMEM_THPS);
+ }
+ __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
+ __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr);
+ local_irq_enable();
+ break;
+unlock:
+ xas_unlock_irq();
+ } while (xas_nomem(, gfp));
+
+ if (xas_error()) {
+ page->mapping = NULL;
+ page_ref_sub(page, nr);
+ return xas_error();
+ }
+
+ return 0;
+}
+
/*
* Like delete_from_page_cache, but substitutes swap for page.
*/
@@ -681,6 +732,35 @@ static void shmem_delete_from_page_cache(struct page
*page, void *radswap)
BUG_ON(error);
}
+static int shmem_add_pages_to_cache(struct page *pages[], int npages,
+ struct address_space *mapping,
+ pgoff_t start, gfp_t gfp)
+{
+ pgoff_t index = start;
+ int err = 0;
+ int i;
+
+ i = 0;
+ while (i < npages) {
+ if (PageTransHuge(pages[i])) {
+ err = shmem_add_to_page_cache_fast(pages[i], mapping,
index, gfp);
+ if (err)
+ break;
+ index += HPAGE_PMD_NR;
+ i++;
+ continue;
+ }
+
+ err = shmem_add_to_page_cache_fast(pages[i], mapping, index,
gfp);
+ if (err)
+ break;
+ index++;
+ i++;
+ }
+
+ return err;
+}
+
int shmem_insert_page(struct mm_struct *mm, struct inode *inode, pgoff_t index,
struct page *page)
{
@@ -844,17 +924,10 @@ int shmem_insert_pages(struct mm_struct *mm, struct inode
*inode, pgoff_t index,
}
- for (i = 0; i < npages; i++) {
- err = shmem_add_to_page_cache(pages[i], mapping, index,
- NULL, gfp & GFP_RECLAIM_MASK);
- if (err)
- goto out_truncate;
-
- if (PageTransHuge(pages[i]))
- index += HPAGE_PMD_NR;
- else
- index++;
- }
+ err = shmem_add_pages_to_cache(pages, npages, mapping, index,
+ gfp & GFP_RECLAIM_MASK);
+ if (err)
+ goto out_truncate;
spin_lock(>lock);
info->alloced += nr;
--
2.13.3
[RFC 12/43] mm: PKRAM: reserve preserved memory at boot
Keep preserved pages from being recycled during boot by adding them
to the memblock reserved list during early boot. If memory reservation
fails (e.g. a region has already been reserved), all preserved pages
are dropped.
For efficiency the preserved pages pagetable is used to identify and
reserve by the contiguous ranges present rather than a page at a time.
Signed-off-by: Anthony Yznaga
---
arch/x86/kernel/setup.c | 3 +
arch/x86/mm/init_64.c | 2 +
include/linux/pkram.h | 8 +++
mm/pkram.c | 179 +++-
4 files changed, 189 insertions(+), 3 deletions(-)
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 4b3fa6cd3106..851515753ad9 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -14,6 +14,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -1158,6 +1159,8 @@ void __init setup_arch(char **cmdline_p)
initmem_init();
dma_contiguous_reserve(max_pfn_mapped << PAGE_SHIFT);
+ pkram_reserve();
+
if (boot_cpu_has(X86_FEATURE_GBPAGES))
hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT);
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 3b289c2f75cd..ae569ef6bd7d 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -33,6 +33,7 @@
#include
#include
#include
+#include
#include
#include
@@ -1244,6 +1245,7 @@ void __init mem_init(void)
after_bootmem = 1;
x86_init.hyper.init_after_bootmem();
+ totalram_pages_add(pkram_reserved_pages);
/*
* Must be done after boot memory is put on freelist, because here we
* might set fields in deferred struct pages that have not yet been
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index b6fa973d37cc..1b475f6e1598 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -61,4 +61,12 @@ struct page *pkram_load_page(struct pkram_stream *ps,
unsigned long *index,
ssize_t pkram_write(struct pkram_stream *ps, const void *buf, size_t count);
size_t pkram_read(struct pkram_stream *ps, void *buf, size_t count);
+#ifdef CONFIG_PKRAM
+extern unsigned long pkram_reserved_pages;
+void pkram_reserve(void);
+#else
+#define pkram_reserved_pages 0UL
+static inline void pkram_reserve(void) { }
+#endif
+
#endif /* _LINUX_PKRAM_H */
diff --git a/mm/pkram.c b/mm/pkram.c
index 54b2779d0813..2c323154df76 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -7,6 +7,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -105,6 +106,7 @@ static DEFINE_SPINLOCK(pkram_pgd_lock);
static int pkram_add_identity_map(struct page *page);
static void pkram_remove_identity_map(struct page *page);
+static int pkram_reserve_page_ranges(pgd_t *pgd);
/*
* For convenience sake PKRAM nodes are kept in an auxiliary doubly-linked list
@@ -113,6 +115,9 @@ static void pkram_remove_identity_map(struct page *page);
static LIST_HEAD(pkram_nodes); /* linked through page::lru */
static DEFINE_MUTEX(pkram_mutex); /* serializes open/close */
+unsigned long __initdata pkram_reserved_pages;
+static bool pkram_reservation_in_progress;
+
/*
* The PKRAM super block pfn, see above.
*/
@@ -122,6 +127,102 @@ static int __init parse_pkram_sb_pfn(char *arg)
}
early_param("pkram", parse_pkram_sb_pfn);
+static void * __init pkram_map_meta(unsigned long pfn)
+{
+ if (pfn >= max_low_pfn)
+ return ERR_PTR(-EINVAL);
+ return pfn_to_kaddr(pfn);
+}
+
+static int __init pkram_reserve_page(unsigned long pfn)
+{
+ phys_addr_t base, size;
+ int err = 0;
+
+ if (pfn >= max_pfn)
+ return -EINVAL;
+
+ base = PFN_PHYS(pfn);
+ size = PAGE_SIZE;
+
+ if (memblock_is_region_reserved(base, size) ||
+ memblock_reserve(base, size) < 0)
+ err = -EBUSY;
+
+ if (!err)
+ pkram_reserved_pages++;
+
+ return err;
+}
+
+static void __init pkram_unreserve_page(unsigned long pfn)
+{
+ memblock_free(PFN_PHYS(pfn), PAGE_SIZE);
+ pkram_reserved_pages--;
+}
+
+/*
+ * Reserved pages that belong to preserved memory.
+ *
+ * This function should be called at boot time as early as possible to prevent
+ * preserved memory from being recycled.
+ */
+void __init pkram_reserve(void)
+{
+ int err = 0;
+
+ if (!pkram_sb_pfn)
+ return;
+
+ pr_info("PKRAM: Examining preserved memory...\n");
+ pkram_reservation_in_progress = true;
+
+ err = pkram_reserve_page(pkram_sb_pfn);
+ if (err)
+ goto out;
+ pkram_sb = pkram_map_meta(pkram_sb_pfn);
+ if (IS_ERR(pkram_sb)) {
+ pkram_unreserve_page(pkram_sb_pfn);
+ err = PTR_ERR(pkram_sb);
+ goto out;
+ }
+
+ /* An empty pkram_sb is not an error */
+ if (!pkram_sb->node_pfn)
[RFC 35/43] shmem: introduce shmem_insert_pages()
Calling shmem_insert_page() to insert one page at a time does
not scale well when multiple threads are inserting pages into
the same shmem segment. This is primarily due to the locking needed
when adding to the pagecache and LRU but also due to contention
on the shmem_inode_info lock. To address the shmem_inode_info lock
and prepare for future optimizations, introduce shmem_insert_pages()
which allows a caller to pass an array of pages to be inserted into a
shmem segment.
Signed-off-by: Anthony Yznaga
---
include/linux/shmem_fs.h | 3 +-
mm/shmem.c | 114 +++
2 files changed, 116 insertions(+), 1 deletion(-)
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index f2ce9937a8f2..d308a6a154b6 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -105,7 +105,8 @@ extern int shmem_getpage(struct inode *inode, pgoff_t index,
extern int shmem_insert_page(struct mm_struct *mm, struct inode *inode,
pgoff_t index, struct page *page);
-
+extern int shmem_insert_pages(struct mm_struct *mm, struct inode *inode,
+ pgoff_t index, struct page *pages[], int npages);
#ifdef CONFIG_PKRAM
extern int shmem_parse_pkram(const char *str, struct shmem_pkram_info **pkram);
extern void shmem_show_pkram(struct seq_file *seq, struct shmem_pkram_info
*pkram,
diff --git a/mm/shmem.c b/mm/shmem.c
index 1f3b43b8fa34..ca5edf580f24 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -781,6 +781,120 @@ int shmem_insert_page(struct mm_struct *mm, struct inode
*inode, pgoff_t index,
return err;
}
+int shmem_insert_pages(struct mm_struct *mm, struct inode *inode, pgoff_t
index,
+ struct page *pages[], int npages)
+{
+ struct address_space *mapping = inode->i_mapping;
+ struct shmem_inode_info *info = SHMEM_I(inode);
+ struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
+ gfp_t gfp = mapping_gfp_mask(mapping);
+ struct mem_cgroup *memcg;
+ int i, err;
+ int nr = 0;
+
+ for (i = 0; i < npages; i++)
+ nr += compound_nr(pages[i]);
+
+ if (index + nr - 1 > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
+ return -EFBIG;
+
+retry:
+ err = 0;
+ if (!shmem_inode_acct_block(inode, nr))
+ err = -ENOSPC;
+ if (err) {
+ int retry = 5;
+
+ /*
+* Try to reclaim some space by splitting a huge page
+* beyond i_size on the filesystem.
+*/
+ while (retry--) {
+ int ret;
+
+ ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
+ if (ret == SHRINK_STOP)
+ break;
+ if (ret)
+ goto retry;
+ }
+ goto failed;
+ }
+
+ for (i = 0; i < npages; i++) {
+ if (!PageLRU(pages[i])) {
+ __SetPageLocked(pages[i]);
+ __SetPageSwapBacked(pages[i]);
+ } else {
+ lock_page(pages[i]);
+ }
+
+ __SetPageReferenced(pages[i]);
+
+ if (pages[i]->mem_cgroup)
+ continue;
+
+ err = mem_cgroup_try_charge_delay(pages[i], mm, gfp,
+ , PageTransHuge(pages[i]));
+ if (err)
+ goto out_unlock;
+
+ }
+
+ for (i = 0; i < npages; i++) {
+ err = shmem_add_to_page_cache(pages[i], mapping, index,
+ NULL, gfp & GFP_RECLAIM_MASK);
+ if (err)
+ goto out_truncate;
+
+ if (PageTransHuge(pages[i]))
+ index += HPAGE_PMD_NR;
+ else
+ index++;
+ }
+
+ spin_lock(>lock);
+ info->alloced += nr;
+ inode->i_blocks += BLOCKS_PER_PAGE * nr;
+ shmem_recalc_inode(inode);
+ spin_unlock(>lock);
+
+ for (i = 0; i < npages; i++) {
+ if (!pages[i]->mem_cgroup) {
+ mem_cgroup_commit_charge(pages[i], memcg,
+ PageLRU(pages[i]), PageTransHuge(pages[i]));
+ }
+
+ if (!PageLRU(pages[i]))
+ lru_cache_add_anon(pages[i]);
+
+ flush_dcache_page(pages[i]);
+ SetPageUptodate(pages[i]);
+ set_page_dirty(pages[i]);
+
+ unlock_page(pages[i]);
+ }
+
+ return 0;
+
+out_truncate:
+ while (--i >= 0)
+ truncate_inode_page(mapping, pages[i]);
+ i = npages;
+out_unlock:
+ while (--i >= 0) {
+ if (!pages[i]->mem_cgroup) {
+
[RFC 37/43] shmem: PKRAM: enable bulk loading of preserved pages into shmem
Make use of new interfaces for loading and inserting preserved pages
into a shmem file in bulk.
Signed-off-by: Anthony Yznaga
---
mm/shmem_pkram.c | 23 +--
1 file changed, 17 insertions(+), 6 deletions(-)
diff --git a/mm/shmem_pkram.c b/mm/shmem_pkram.c
index 4992b6c3e54e..435488368104 100644
--- a/mm/shmem_pkram.c
+++ b/mm/shmem_pkram.c
@@ -315,18 +315,29 @@ static inline void pkram_load_report_one_done(void)
static int do_load_file_content(struct pkram_stream *ps)
{
unsigned long index;
- struct page *page;
- int err = 0;
+ int i, err;
+
+ err = pkram_prepare_load_pages(ps);
+ if (err)
+ return err;
do {
- page = pkram_load_page(ps, , NULL);
- if (!page)
+ err = pkram_load_pages(ps, );
+ if (err) {
+ if (err == -ENODATA)
+ err = 0;
break;
+ }
- err = shmem_insert_page(ps->mm, ps->mapping->host, index, page);
- put_page(page);
+ err = shmem_insert_pages(ps->mm, ps->mapping->host, index,
+ps->pages, ps->nr_pages);
+
+ for (i = 0; i < ps->nr_pages; i++)
+ put_page(ps->pages[i]);
} while (!err);
+ pkram_finish_load_pages(ps);
+
return err;
}
--
2.13.3
[RFC 03/43] mm: PKRAM: implement object load and save functions
PKRAM nodes are further divided into a list of objects. After a save
operation has been initiated for a node, a save operation for an object
associated with the node is initiated by calling pkram_prepare_save_obj().
A new object is created and linked to the node. The save operation for
the object is committed by calling pkram_finish_save_obj(). After a load
operation has been initiated, pkram_prepare_load_obj() is called to
delete the next object from the node and prepare the corresponding
stream for loading data from it. After the load of object has been
finished, pkram_finish_load_obj() is called to free the object. Objects
are also deleted when a save operation is discarded.
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 1 +
mm/pkram.c| 77 ---
2 files changed, 74 insertions(+), 4 deletions(-)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index 83a0579e4c1c..fabde2cd8203 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -11,6 +11,7 @@ struct pkram_node;
struct pkram_stream {
gfp_t gfp_mask;
struct pkram_node *node;
+ struct pkram_obj *obj;
};
#define PKRAM_NAME_MAX 256 /* including nul */
diff --git a/mm/pkram.c b/mm/pkram.c
index 5c57126353ff..4934ffd8b019 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -6,9 +6,14 @@
#include
#include
#include
+#include
#include
#include
+struct pkram_obj {
+ __u64 obj_pfn;/* points to the next object in the list */
+};
+
/*
* Preserved memory is divided into nodes that can be saved or loaded
* independently of each other. The nodes are identified by unique name
@@ -18,6 +23,7 @@
*/
struct pkram_node {
__u32 flags;
+ __u64 obj_pfn;/* points to the first obj of the node */
__u8name[PKRAM_NAME_MAX];
};
@@ -62,6 +68,21 @@ static struct pkram_node *pkram_find_node(const char *name)
return NULL;
}
+static void pkram_truncate_node(struct pkram_node *node)
+{
+ unsigned long obj_pfn;
+ struct pkram_obj *obj;
+
+ obj_pfn = node->obj_pfn;
+ while (obj_pfn) {
+ obj = pfn_to_kaddr(obj_pfn);
+ obj_pfn = obj->obj_pfn;
+ pkram_free_page(obj);
+ cond_resched();
+ }
+ node->obj_pfn = 0;
+}
+
static void pkram_stream_init(struct pkram_stream *ps,
struct pkram_node *node, gfp_t gfp_mask)
{
@@ -70,6 +91,11 @@ static void pkram_stream_init(struct pkram_stream *ps,
ps->node = node;
}
+static void pkram_stream_init_obj(struct pkram_stream *ps, struct pkram_obj
*obj)
+{
+ ps->obj = obj;
+}
+
/**
* Create a preserved memory node with name @name and initialize stream @ps
* for saving data to it.
@@ -124,12 +150,31 @@ int pkram_prepare_save(struct pkram_stream *ps, const
char *name, gfp_t gfp_mask
*
* Returns 0 on success, -errno on failure.
*
+ * Error values:
+ * %ENOMEM: insufficient memory available
+ *
* After the save has finished, pkram_finish_save_obj() (or
pkram_discard_save()
* in case of failure) is to be called.
*/
int pkram_prepare_save_obj(struct pkram_stream *ps)
{
- return -ENOSYS;
+ struct pkram_node *node = ps->node;
+ struct pkram_obj *obj;
+ struct page *page;
+
+ BUG_ON((node->flags & PKRAM_ACCMODE_MASK) != PKRAM_SAVE);
+
+ page = pkram_alloc_page(GFP_KERNEL | __GFP_ZERO);
+ if (!page)
+ return -ENOMEM;
+ obj = page_address(page);
+
+ if (node->obj_pfn)
+ obj->obj_pfn = node->obj_pfn;
+ node->obj_pfn = page_to_pfn(page);
+
+ pkram_stream_init_obj(ps, obj);
+ return 0;
}
/**
@@ -137,7 +182,9 @@ int pkram_prepare_save_obj(struct pkram_stream *ps)
*/
void pkram_finish_save_obj(struct pkram_stream *ps)
{
- BUG();
+ struct pkram_node *node = ps->node;
+
+ BUG_ON((node->flags & PKRAM_ACCMODE_MASK) != PKRAM_SAVE);
}
/**
@@ -169,6 +216,7 @@ void pkram_discard_save(struct pkram_stream *ps)
pkram_delete_node(node);
mutex_unlock(_mutex);
+ pkram_truncate_node(node);
pkram_free_page(node);
}
@@ -216,11 +264,26 @@ int pkram_prepare_load(struct pkram_stream *ps, const
char *name)
*
* Returns 0 on success, -errno on failure.
*
+ * Error values:
+ * %ENODATA: Stream @ps has no preserved memory objects
+ *
* After the load has finished, pkram_finish_load_obj() is to be called.
*/
int pkram_prepare_load_obj(struct pkram_stream *ps)
{
- return -ENOSYS;
+ struct pkram_node *node = ps->node;
+ struct pkram_obj *obj;
+
+ BUG_ON((node->flags & PKRAM_ACCMODE_MASK) != PKRAM_LOAD);
+
+ if (!node->obj_pfn)
+ return -ENODATA;
+
+ obj = pfn_to_kaddr(node->obj_pfn);
+ node->obj_pfn = obj->obj_pfn;
+
+
[RFC 31/43] memblock, mm: defer initialization of preserved pages
Preserved pages are represented in the memblock reserved list, but page
structs for pages in the reserved list are initialized early while boot
is single threaded which means that a large number of preserved pages
can impact boot time. To mitigate, defer initialization of preserved
pages by skipping them when other reserved pages are initialized and
initializing them later with a separate kernel thread.
Signed-off-by: Anthony Yznaga
---
arch/x86/mm/init_64.c | 1 -
include/linux/mm.h| 2 +-
mm/memblock.c | 10 --
mm/page_alloc.c | 52 +++
4 files changed, 53 insertions(+), 12 deletions(-)
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 72662615977b..ae569ef6bd7d 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1245,7 +1245,6 @@ void __init mem_init(void)
after_bootmem = 1;
x86_init.hyper.init_after_bootmem();
- pkram_free_pgt();
totalram_pages_add(pkram_reserved_pages);
/*
* Must be done after boot memory is put on freelist, because here we
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 5a323422d783..69b9cd08c721 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2297,7 +2297,7 @@ extern void free_highmem_page(struct page *page);
extern void adjust_managed_page_count(struct page *page, long count);
extern void mem_init_print_info(const char *str);
-extern void reserve_bootmem_region(phys_addr_t start, phys_addr_t end);
+extern void reserve_bootmem_region(phys_addr_t start, phys_addr_t end, int
nid);
/* Free the reserved page into the buddy system, so it gets managed. */
static inline void __free_reserved_page(struct page *page)
diff --git a/mm/memblock.c b/mm/memblock.c
index 33597f352dc0..5524edbaf691 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -2042,11 +2042,17 @@ static unsigned long __init
free_low_memory_core_early(void)
unsigned long count = 0;
phys_addr_t start, end;
u64 i;
+ enum memblock_flags exclude;
memblock_clear_hotplug(0, -1);
- for_each_reserved_mem_region(i, , )
- reserve_bootmem_region(start, end);
+ if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT))
+ exclude = MEMBLOCK_PRESERVED;
+ else
+ exclude = MEMBLOCK_NONE;
+
+ for_each_reserved_mem_range(i, 0, exclude, , , NULL)
+ reserve_bootmem_region(start, end, -1);
/*
* We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 69827d4fa052..afd97b31725e 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -68,6 +68,7 @@
#include
#include
#include
+#include
#include
#include
@@ -1408,15 +1409,18 @@ static void __meminit __init_single_page(struct page
*page, unsigned long pfn,
}
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
-static void __meminit init_reserved_page(unsigned long pfn)
+static void __meminit init_reserved_page(unsigned long pfn, int nid)
{
pg_data_t *pgdat;
- int nid, zid;
+ int zid;
- if (!early_page_uninitialised(pfn))
- return;
+ if (nid == -1) {
+ if (!early_page_uninitialised(pfn))
+ return;
+
+ nid = early_pfn_to_nid(pfn);
+ }
- nid = early_pfn_to_nid(pfn);
pgdat = NODE_DATA(nid);
for (zid = 0; zid < MAX_NR_ZONES; zid++) {
@@ -1428,7 +1432,7 @@ static void __meminit init_reserved_page(unsigned long
pfn)
__init_single_page(pfn_to_page(pfn), pfn, zid, nid);
}
#else
-static inline void init_reserved_page(unsigned long pfn)
+static inline void init_reserved_page(unsigned long pfn, int nid)
{
}
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
@@ -1439,7 +1443,7 @@ static inline void init_reserved_page(unsigned long pfn)
* marks the pages PageReserved. The remaining valid pages are later
* sent to the buddy page allocator.
*/
-void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
+void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end, int
nid)
{
unsigned long start_pfn = PFN_DOWN(start);
unsigned long end_pfn = PFN_UP(end);
@@ -1448,7 +1452,7 @@ void __meminit reserve_bootmem_region(phys_addr_t start,
phys_addr_t end)
if (pfn_valid(start_pfn)) {
struct page *page = pfn_to_page(start_pfn);
- init_reserved_page(start_pfn);
+ init_reserved_page(start_pfn, nid);
/* Avoid false-positive PageTail() */
INIT_LIST_HEAD(>lru);
@@ -1876,6 +1880,34 @@ static int __init deferred_init_memmap(void *data)
return 0;
}
+#ifdef CONFIG_PKRAM
+static int __init deferred_init_preserved(void *dummy)
+{
+ unsigned long start = jiffies;
+ unsigned long nr_pages = 0;
+ phys_a
[RFC 32/43] shmem: PKRAM: preserve shmem files a chunk at a time
To prepare for multithreading the work done to a preserve a file,
divide the work into subranges of the total index range of the file.
The chunk size is a rather arbitrary 256k indices.
A new API call, pkram_prepare_save_chunk(), is added. It is called
after calling pkram_prepare_save_obj(), and it initializes pkram_stream
with the index range of the next available range of pages to save.
find_get_pages_range() can then be used to get the pages in the range.
When no more index ranges are available, pkram_prepare_save_chunk()
returns -ENODATA.
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 6 +
mm/pkram.c| 26 +
mm/shmem_pkram.c | 63 +++
3 files changed, 75 insertions(+), 20 deletions(-)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index cbb79d2803c0..e71ccb91d6a6 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -20,6 +20,11 @@ struct pkram_stream {
struct address_space *mapping;
struct mm_struct *mm;
+ unsigned long start_idx;/* first index in range to save */
+ unsigned long end_idx; /* last index in range to save */
+ unsigned long max_idx; /* maximum index to save */
+ atomic64_t *next_idx; /* first index of next range to save */
+
/* byte data */
struct page *data_page;
unsigned int data_offset;
@@ -46,6 +51,7 @@ void pkram_free_pgt_walk_pgd(pgd_t *pgd);
int pkram_prepare_save(struct pkram_stream *ps, const char *name,
gfp_t gfp_mask);
int pkram_prepare_save_obj(struct pkram_stream *ps);
+int pkram_prepare_save_chunk(struct pkram_stream *ps);
void pkram_finish_save(struct pkram_stream *ps);
void pkram_finish_save_obj(struct pkram_stream *ps);
void pkram_discard_save(struct pkram_stream *ps);
diff --git a/mm/pkram.c b/mm/pkram.c
index b83d31740619..5f4e4d12865f 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -638,6 +638,25 @@ int pkram_prepare_save(struct pkram_stream *ps, const char
*name, gfp_t gfp_mask
return 0;
}
+unsigned long max_pages_per_chunk = 512 * 512;
+
+/*
+ * Initialize the stream @ps for the next index range to save.
+ *
+ * Returns 0 on success, -ENODATA if no index range is available
+ *
+ */
+int pkram_prepare_save_chunk(struct pkram_stream *ps)
+{
+ ps->start_idx = atomic64_fetch_add(max_pages_per_chunk, ps->next_idx);
+ if (ps->start_idx >= ps->max_idx)
+ return -ENODATA;
+
+ ps->end_idx = ps->start_idx + max_pages_per_chunk - 1;
+
+ return 0;
+}
+
/**
* Create a preserved memory object and initialize stream @ps for saving data
* to it.
@@ -667,6 +686,11 @@ int pkram_prepare_save_obj(struct pkram_stream *ps)
obj->obj_pfn = node->obj_pfn;
node->obj_pfn = page_to_pfn(page);
+ ps->next_idx = kmalloc(sizeof(atomic64_t), GFP_KERNEL);
+ if (!ps->next_idx)
+ return -ENOMEM;
+ atomic64_set(ps->next_idx, 0);
+
pkram_stream_init_obj(ps, obj);
return 0;
}
@@ -679,6 +703,8 @@ void pkram_finish_save_obj(struct pkram_stream *ps)
struct pkram_node *node = ps->node;
BUG_ON((node->flags & PKRAM_ACCMODE_MASK) != PKRAM_SAVE);
+
+ kfree(ps->next_idx);
}
/**
diff --git a/mm/shmem_pkram.c b/mm/shmem_pkram.c
index c97d64393822..2f4d0bdf3e05 100644
--- a/mm/shmem_pkram.c
+++ b/mm/shmem_pkram.c
@@ -74,58 +74,81 @@ static int save_page(struct page *page, struct pkram_stream
*ps)
return err;
}
-static int save_file_content(struct pkram_stream *ps)
+static int save_file_content_range(struct address_space *mapping,
+ struct pkram_stream *ps)
{
+ unsigned long index, end;
struct pagevec pvec;
- pgoff_t indices[PAGEVEC_SIZE];
- pgoff_t index = 0;
struct page *page;
- int i, err = 0;
+ int err = 0;
+ int i;
+
+ index = ps->start_idx;
+ end = ps->end_idx;
pagevec_init();
for ( ; ; ) {
- pvec.nr = find_get_entries(ps->mapping, index, PAGEVEC_SIZE,
- pvec.pages, indices);
+ pvec.nr = find_get_pages_range(mapping, , end,
+ PAGEVEC_SIZE, pvec.pages);
if (!pvec.nr)
break;
- for (i = 0; i < pagevec_count(); i++) {
+ for (i = 0; i < pagevec_count(); ) {
page = pvec.pages[i];
- index = indices[i];
-
- if (WARN_ON_ONCE(xa_is_value(page))) {
- err = -EINVAL;
- break;
- }
-
lock_page(page);
if (PageTransTail(page)) {
[RFC 28/43] PKRAM: ensure memblocks with preserved pages init'd for numa
In order to facilitate fast initialization of page structs for
preserved pages, memblocks with preserved pages must not cross
numa node boundaries and must have a node id assigned to them.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 10 ++
1 file changed, 10 insertions(+)
diff --git a/mm/pkram.c b/mm/pkram.c
index a5e539052af6..97a7dd0a5b7d 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -21,6 +21,7 @@
#include
#include
+#include
#include "internal.h"
@@ -242,6 +243,15 @@ void __init pkram_reserve(void)
return;
}
+ /*
+* Fix up the reserved memblock list to ensure the
+* memblock regions are split along node boundaries
+* and have a node ID set. This will allow the page
+* structs for the preserved pages to be initialized
+* more efficiently.
+*/
+ numa_isolate_memblocks();
+
done:
pr_info("PKRAM: %lu pages reserved\n", pkram_reserved_pages);
}
--
2.13.3
[RFC 30/43] memblock: add for_each_reserved_mem_range()
To support deferred initialization of page structs for preserved
pages, add an iterator of the memblock reserved list that can select or
exclude ranges based on memblock flags.
Signed-off-by: Anthony Yznaga
---
include/linux/memblock.h | 10 ++
mm/memblock.c| 51 +++-
2 files changed, 60 insertions(+), 1 deletion(-)
diff --git a/include/linux/memblock.h b/include/linux/memblock.h
index 27ab2b30ae1d..f348ebb750c9 100644
--- a/include/linux/memblock.h
+++ b/include/linux/memblock.h
@@ -145,6 +145,11 @@ void __next_mem_range_rev(u64 *idx, int nid, enum
memblock_flags flags,
void __next_reserved_mem_region(u64 *idx, phys_addr_t *out_start,
phys_addr_t *out_end);
+void __next_reserved_mem_range(u64 *idx, enum memblock_flags flags,
+ enum memblock_flags exclflags,
+ phys_addr_t *out_start, phys_addr_t *out_end,
+ int *out_nid);
+
void __memblock_free_late(phys_addr_t base, phys_addr_t size);
/**
@@ -202,6 +207,11 @@ void __memblock_free_late(phys_addr_t base, phys_addr_t
size);
i != (u64)ULLONG_MAX; \
__next_reserved_mem_region(, p_start, p_end))
+#define for_each_reserved_mem_range(i, flags, exclflags, p_start, p_end,
p_nid)\
+ for (i = 0UL, __next_reserved_mem_range(, flags, exclflags, p_start,
p_end, p_nid); \
+i != (u64)ULLONG_MAX; \
+__next_reserved_mem_range(, flags, exclflags, p_start, p_end,
p_nid))
+
static inline bool memblock_is_hotpluggable(struct memblock_region *m)
{
return m->flags & MEMBLOCK_HOTPLUG;
diff --git a/mm/memblock.c b/mm/memblock.c
index 1a9a2055ed11..33597f352dc0 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -987,6 +987,55 @@ void __init_memblock __next_reserved_mem_region(u64 *idx,
*idx = ULLONG_MAX;
}
+/**
+ * __next_reserved_mem_range - next function for for_each_reserved_range()
+ * @idx: pointer to u64 loop variable
+ * @flags: pick blocks based on memory attributes
+ * @exclflags: exclude blocks based on memory attributes
+ * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
+ * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
+ * @out_nid: ptr to int for nid of the range, can be %NULL
+ *
+ * Iterate over all reserved memory ranges.
+ */
+void __init_memblock __next_reserved_mem_range(u64 *idx,
+ enum memblock_flags flags,
+ enum memblock_flags exclflags,
+ phys_addr_t *out_start,
+ phys_addr_t *out_end, int *out_nid)
+{
+ struct memblock_type *type =
+ int _idx = *idx;
+
+ for (; _idx < type->cnt; _idx++) {
+ struct memblock_region *r = >regions[_idx];
+ phys_addr_t base = r->base;
+ phys_addr_t size = r->size;
+
+ /* skip preserved pages */
+ if ((exclflags & MEMBLOCK_PRESERVED) &&
memblock_is_preserved(r))
+ continue;
+
+ /* skip non-preserved pages */
+ if ((flags & MEMBLOCK_PRESERVED) && !memblock_is_preserved(r))
+ continue;
+
+ if (out_start)
+ *out_start = base;
+ if (out_end)
+ *out_end = base + size - 1;
+ if (out_nid)
+ *out_nid = r->nid;
+
+ _idx++;
+ *idx = (u64)_idx;
+ return;
+ }
+
+ /* signal end of iteration */
+ *idx = ULLONG_MAX;
+}
+
static bool should_skip_region(struct memblock_region *m, int nid, int flags)
{
int m_nid = memblock_get_region_node(m);
@@ -1011,7 +1060,7 @@ static bool should_skip_region(struct memblock_region *m,
int nid, int flags)
}
/**
- * __next_mem_range - next function for for_each_free_mem_range() etc.
+ * __next__mem_range - next function for for_each_free_mem_range() etc.
* @idx: pointer to u64 loop variable
* @nid: node selector, %NUMA_NO_NODE for all nodes
* @flags: pick from blocks based on memory attributes
--
2.13.3
[RFC 11/43] PKRAM: pass the preserved pages pagetable to the next kernel
Add a pointer to the pagetable to the pkram_super_block page.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 20 +---
1 file changed, 13 insertions(+), 7 deletions(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index 5a7b8f61a55d..54b2779d0813 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -94,6 +94,7 @@ struct pkram_node {
*/
struct pkram_super_block {
__u64 node_pfn; /* first element of the node list */
+ __u64 pgd_pfn;
};
static unsigned long pkram_sb_pfn __initdata;
@@ -769,15 +770,20 @@ static void __pkram_reboot(void)
struct page *page;
struct pkram_node *node;
unsigned long node_pfn = 0;
-
- list_for_each_entry_reverse(page, _nodes, lru) {
- node = page_address(page);
- if (WARN_ON(node->flags & PKRAM_ACCMODE_MASK))
- continue;
- node->node_pfn = node_pfn;
- node_pfn = page_to_pfn(page);
+ unsigned long pgd_pfn = 0;
+
+ if (pkram_pgd) {
+ list_for_each_entry_reverse(page, _nodes, lru) {
+ node = page_address(page);
+ if (WARN_ON(node->flags & PKRAM_ACCMODE_MASK))
+ continue;
+ node->node_pfn = node_pfn;
+ node_pfn = page_to_pfn(page);
+ }
+ pgd_pfn = page_to_pfn(virt_to_page(pkram_pgd));
}
pkram_sb->node_pfn = node_pfn;
+ pkram_sb->pgd_pfn = pgd_pfn;
}
static int pkram_reboot(struct notifier_block *notifier,
--
2.13.3
[RFC 09/43] PKRAM: build a physical mapping pagetable of pages to be preserved
Future patches will need a way to efficiently identify physically
contiguous ranges of preserved pages regardless of their virtual
addresses as well as a way to identify ranges that do not contain
preserved pages. To facilitate this all pages to be preserved across
kexec are added to an identity mapping-style pagetable that is passed
to the next kernel.
The pagetable makes use of the existing architecture definitions for
building a memory mapping pagetable with the primary difference being
that a bitmap is used to represent the presence or absence of preserved
pages at the PTE level.
In general both metadata pages and data pages must be added to the
pagetable. A mapping for a metadata page can be added when the page is
allocated, but there is an exception: for the pagetable pages themselves
mappings are added after they are allocated to avoid recursion.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 233 -
1 file changed, 230 insertions(+), 3 deletions(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index 70f2219e6218..5a7b8f61a55d 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -99,6 +99,12 @@ struct pkram_super_block {
static unsigned long pkram_sb_pfn __initdata;
static struct pkram_super_block *pkram_sb;
+static pgd_t *pkram_pgd;
+static DEFINE_SPINLOCK(pkram_pgd_lock);
+
+static int pkram_add_identity_map(struct page *page);
+static void pkram_remove_identity_map(struct page *page);
+
/*
* For convenience sake PKRAM nodes are kept in an auxiliary doubly-linked list
* connected through the lru field of the page struct.
@@ -115,13 +121,31 @@ static int __init parse_pkram_sb_pfn(char *arg)
}
early_param("pkram", parse_pkram_sb_pfn);
+static inline struct page *__pkram_alloc_page(gfp_t gfp_mask, bool add_to_map)
+{
+ struct page *page;
+ int err;
+
+ page = alloc_page(gfp_mask);
+ if (page && add_to_map) {
+ err = pkram_add_identity_map(page);
+ if (err) {
+ __free_page(page);
+ page = NULL;
+ }
+ }
+
+ return page;
+}
+
static inline struct page *pkram_alloc_page(gfp_t gfp_mask)
{
- return alloc_page(gfp_mask);
+ return __pkram_alloc_page(gfp_mask, true);
}
static inline void pkram_free_page(void *addr)
{
+ pkram_remove_identity_map(virt_to_page(addr));
free_page((unsigned long)addr);
}
@@ -159,6 +183,7 @@ static void pkram_truncate_link(struct pkram_link *link)
if (!p)
continue;
page = pfn_to_page(PHYS_PFN(p));
+ pkram_remove_identity_map(page);
put_page(page);
}
}
@@ -547,10 +572,15 @@ static int __pkram_save_page(struct pkram_stream *ps,
int pkram_save_page(struct pkram_stream *ps, struct page *page, short flags)
{
struct pkram_node *node = ps->node;
+ int err;
BUG_ON((node->flags & PKRAM_ACCMODE_MASK) != PKRAM_SAVE);
- return __pkram_save_page(ps, page, flags, page->index);
+ err = __pkram_save_page(ps, page, flags, page->index);
+ if (!err)
+ err = pkram_add_identity_map(page);
+
+ return err;
}
/*
@@ -599,6 +629,8 @@ static struct page *__pkram_load_page(struct pkram_stream
*ps, unsigned long *in
/* clear to avoid double free (see pkram_truncate_link()) */
link->entry[ps->entry_idx] = 0;
+ pkram_remove_identity_map(page);
+
ps->entry_idx++;
if (ps->entry_idx >= PKRAM_LINK_ENTRIES_MAX ||
!link->entry[ps->entry_idx]) {
@@ -791,7 +823,7 @@ static int __init pkram_init_sb(void)
if (!pkram_sb) {
struct page *page;
- page = pkram_alloc_page(GFP_KERNEL | __GFP_ZERO);
+ page = __pkram_alloc_page(GFP_KERNEL | __GFP_ZERO, false);
if (!page) {
pr_err("PKRAM: Failed to allocate super block\n");
return 0;
@@ -821,3 +853,198 @@ static int __init pkram_init(void)
return 0;
}
module_init(pkram_init);
+
+static unsigned long *pkram_alloc_pte_bitmap(void)
+{
+ return page_address(__pkram_alloc_page(GFP_KERNEL | __GFP_ZERO, false));
+}
+
+static void pkram_free_pte_bitmap(void *bitmap)
+{
+ pkram_remove_identity_map(virt_to_page(bitmap));
+ free_page((unsigned long)bitmap);
+}
+
+#define set_p4d(p4dp, p4d) WRITE_ONCE(*(p4dp), (p4d))
+
+static int pkram_add_identity_map(struct page *page)
+{
+ unsigned long orig_paddr, paddr;
+ unsigned long *bitmap;
+ int result = -ENOMEM;
+ unsigned int index;
+ struct page *pg;
+ LIST_HEAD(list);
+ pgd_t *pgd;
+ p4d_t *p4d;
+ pud_t *pud;
+ pmd_t *pmd;
+
+ if (!pkram_pgd) {
+ spin_lock(_pgd_lock);
+ if (!pkram_pgd) {
+
[RFC 21/43] x86/KASLR: PKRAM: support physical kaslr
Avoid regions of memory that contain preserved pages when computing
slots used to select where to put the decompressed kernel.
Signed-off-by: Anthony Yznaga
---
arch/x86/boot/compressed/Makefile | 3 +
arch/x86/boot/compressed/kaslr.c | 67 ++
arch/x86/boot/compressed/misc.h | 19 +++
arch/x86/boot/compressed/pkram.c | 252 ++
4 files changed, 320 insertions(+), 21 deletions(-)
create mode 100644 arch/x86/boot/compressed/pkram.c
diff --git a/arch/x86/boot/compressed/Makefile
b/arch/x86/boot/compressed/Makefile
index 5f7c262bcc99..ba0d76c53574 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -84,6 +84,9 @@ ifdef CONFIG_X86_64
vmlinux-objs-$(CONFIG_RANDOMIZE_BASE) += $(obj)/kaslr_64.o
vmlinux-objs-y += $(obj)/mem_encrypt.o
vmlinux-objs-y += $(obj)/pgtable_64.o
+ifdef CONFIG_RANDOMIZE_BASE
+ vmlinux-objs-$(CONFIG_PKRAM) += $(obj)/pkram.o
+endif
endif
vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o
diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c
index d7408af55738..3f0a6fb15ac2 100644
--- a/arch/x86/boot/compressed/kaslr.c
+++ b/arch/x86/boot/compressed/kaslr.c
@@ -613,31 +613,16 @@ static unsigned long slots_fetch_random(void)
return 0;
}
-static void __process_mem_region(struct mem_vector *entry,
-unsigned long minimum,
-unsigned long image_size)
+void ___process_mem_region(struct mem_vector *entry,
+ unsigned long minimum,
+ unsigned long image_size)
{
struct mem_vector region, overlap;
- unsigned long start_orig, end;
+ unsigned long start_orig;
struct mem_vector cur_entry;
- /* On 32-bit, ignore entries entirely above our maximum. */
- if (IS_ENABLED(CONFIG_X86_32) && entry->start >= KERNEL_IMAGE_SIZE)
- return;
-
- /* Ignore entries entirely below our minimum. */
- if (entry->start + entry->size < minimum)
- return;
-
- /* Ignore entries above memory limit */
- end = min(entry->size + entry->start, mem_limit);
- if (entry->start >= end)
- return;
- cur_entry.start = entry->start;
- cur_entry.size = end - entry->start;
-
- region.start = cur_entry.start;
- region.size = cur_entry.size;
+ region.start = cur_entry.start = entry->start;
+ region.size = cur_entry.size = entry->size;
/* Give up if slot area array is full. */
while (slot_area_index < MAX_SLOT_AREA) {
@@ -691,6 +676,39 @@ static void __process_mem_region(struct mem_vector *entry,
}
}
+static void __process_mem_region(struct mem_vector *entry,
+unsigned long minimum,
+unsigned long image_size)
+{
+ struct mem_vector region, overlap;
+ unsigned long start_orig, end;
+ struct mem_vector cur_entry;
+
+ /* On 32-bit, ignore entries entirely above our maximum. */
+ if (IS_ENABLED(CONFIG_X86_32) && entry->start >= KERNEL_IMAGE_SIZE)
+ return;
+
+ /* Ignore entries entirely below our minimum. */
+ if (entry->start + entry->size < minimum)
+ return;
+
+ /* Ignore entries above memory limit */
+ end = min(entry->size + entry->start, mem_limit);
+ if (entry->start >= end)
+ return;
+ cur_entry.start = entry->start;
+ cur_entry.size = end - entry->start;
+
+ /* Return if region can't contain decompressed kernel */
+ if (cur_entry.size < image_size)
+ return;
+
+ if (pkram_enabled())
+ return pkram_process_mem_region(_entry, minimum,
image_size);
+ else
+ return ___process_mem_region(_entry, minimum, image_size);
+}
+
static bool process_mem_region(struct mem_vector *region,
unsigned long long minimum,
unsigned long long image_size)
@@ -902,6 +920,8 @@ void choose_random_location(unsigned long input,
return;
}
+ pkram_init();
+
#ifdef CONFIG_X86_5LEVEL
if (__read_cr4() & X86_CR4_LA57) {
__pgtable_l5_enabled = 1;
@@ -952,3 +972,8 @@ void choose_random_location(unsigned long input,
random_addr = find_random_virt_addr(LOAD_PHYSICAL_ADDR,
output_size);
*virt_addr = random_addr;
}
+
+int slot_areas_full(void)
+{
+ return slot_area_index == MAX_SLOT_AREA;
+}
diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h
index 726e264410ff..ca1a8ae5ebe9 100644
--- a/arch/x86/boot/compressed/misc.h
+++ b/arch/x86/boot/compressed/misc.h
@@ -117,6 +117,25 @@ static inline void console_init(void)
{ }
#endi
[RFC 18/43] kexec: PKRAM: avoid clobbering already preserved pages
Ensure destination ranges of the kexec segments do not overlap
with any kernel pages marked to be preserved across kexec.
For kexec_load, return EADDRNOTAVAIL if overlap is detected.
For kexec_file_load, skip ranges containing preserved pages when
seaching for available ranges to use.
Signed-off-by: Anthony Yznaga
---
kernel/kexec_core.c | 3 +++
kernel/kexec_file.c | 5 +
2 files changed, 8 insertions(+)
diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c
index c19c0dad1ebe..8c24b546352e 100644
--- a/kernel/kexec_core.c
+++ b/kernel/kexec_core.c
@@ -37,6 +37,7 @@
#include
#include
#include
+#include
#include
#include
@@ -176,6 +177,8 @@ int sanity_check_segment_list(struct kimage *image)
return -EADDRNOTAVAIL;
if (mend >= KEXEC_DESTINATION_MEMORY_LIMIT)
return -EADDRNOTAVAIL;
+ if (pkram_has_preserved_pages(mstart, mend))
+ return -EADDRNOTAVAIL;
}
/* Verify our destination addresses do not overlap.
diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c
index f57f72237859..7b14e1b1a178 100644
--- a/kernel/kexec_file.c
+++ b/kernel/kexec_file.c
@@ -498,6 +498,11 @@ static int locate_mem_hole_top_down(unsigned long start,
unsigned long end,
continue;
}
+ if (pkram_has_preserved_pages(temp_start, temp_end + 1)) {
+ temp_start = temp_start - PAGE_SIZE;
+ continue;
+ }
+
/* We found a suitable memory range */
break;
} while (1);
--
2.13.3
[RFC 01/43] mm: add PKRAM API stubs and Kconfig
Preserved-across-kexec memory or PKRAM is a method for saving memory pages of the currently executing kernel and restoring them after kexec boot into a new one. This can be utilized for preserving guest VM state, large in-memory databases, process memory, etc. across reboot. While DRAM-as-PMEM or actual persistent memory could be used to accomplish these things, PKRAM provides the latency of DRAM with the flexibility of dynamically determining the amount of memory to preserve. The proposed API: * Preserved memory is divided into nodes which can be saved or loaded independently of each other. The nodes are identified by unique name strings. A PKRAM node is created when save is initiated by calling pkram_prepare_save(). A PKRAM node is removed when load is initiated by calling pkram_prepare_load(). See below * A node is further divided into objects. An object represents a grouping of associated pages and any relevant metadata preserved with them. For example, the pages and attributes of a file. * For saving/loading data from a PKRAM node/object an instance of the pkram_stream struct is used. The struct is initialized by calling pkram_prepare_save() for saving data or pkram_prepare_load() for loading data. After save (load) is complete, pkram_finish_save() (pkram_finish_load()) must be called. If an error occurred during save, the saved data and the PKRAM node may be freed by calling pkram_discard_save() instead of pkram_finish_save(). * Both page data and byte data can separately be streamed to a PKRAM object. pkram_save_page() and pkram_load_page() are used to stream page data while pkram_write() and pkram_read() are used to stream byte data. A sequence of operations for saving/loading data from PKRAM would look like: * For saving data to PKRAM: /* create a PKRAM node and do initial stream setup */ pkram_prepare_save() /* create a PKRAM object associated with the PKRAM node and complete stream initialization */ pkram_prepare_save_obj() /* save data to the node/object */ pkram_save_page()[,...] /* for page stream, or pkram_write()[,...] * ... for byte stream */ pkram_finish_save_obj() /* commit the save or discard and delete the node */ pkram_finish_save() /* on success, or pkram_discard_save() * ... in case of error */ * For loading data from PKRAM: /* remove a PKRAM node from the list and do initial stream setup */ pkram_prepare_load() /* Remove a PKRAM object from the node and complete stream initializtion for loading data from it. */ pkram_prepare_load_obj() /* load data from the node/object */ pkram_load_page()[,...] /* for page stream, or pkram_read()[,...]* ... for byte stream */ /* free the object */ pkram_finish_load_obj() /* free the node */ pkram_finish_load() Originally-by: Vladimir Davydov Signed-off-by: Anthony Yznaga --- include/linux/pkram.h | 32 ++ mm/Kconfig| 9 +++ mm/Makefile | 1 + mm/pkram.c| 169 ++ 4 files changed, 211 insertions(+) create mode 100644 include/linux/pkram.h create mode 100644 mm/pkram.c diff --git a/include/linux/pkram.h b/include/linux/pkram.h new file mode 100644 index ..4c4e13311ec8 --- /dev/null +++ b/include/linux/pkram.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_PKRAM_H +#define _LINUX_PKRAM_H + +#include +#include +#include + +struct pkram_stream; + +#define PKRAM_NAME_MAX 256 /* including nul */ + +int pkram_prepare_save(struct pkram_stream *ps, const char *name, + gfp_t gfp_mask); +int pkram_prepare_save_obj(struct pkram_stream *ps); +void pkram_finish_save(struct pkram_stream *ps); +void pkram_finish_save_obj(struct pkram_stream *ps); +void pkram_discard_save(struct pkram_stream *ps); + +int pkram_prepare_load(struct pkram_stream *ps, const char *name); +int pkram_prepare_load_obj(struct pkram_stream *ps); +void pkram_finish_load(struct pkram_stream *ps); +void pkram_finish_load_obj(struct pkram_stream *ps); + +int pkram_save_page(struct pkram_stream *ps, struct page *page, short flags); +struct page *pkram_load_page(struct pkram_stream *ps, unsigned long *index, +short *flags); + +ssize_t pkram_write(struct pkram_stream *ps, const void *buf, size_t count); +size_t pkram_read(struct pkram_stream *ps, void *buf, size_t count); + +#endif /* _LINUX_PKRAM_H */ diff --git a/mm/Kconfig b/mm/Kconfig index c1acc34c1c35..bddf20ecf6e1 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -867,4 +867,13 @@ config ARCH_HAS_HUGEPD config MAPPING_DIRTY_HELPERS bool +config PKRAM + bool "Preserved-over-kexec memory storage" + default n + help + This option adds the kernel API that enables saving me
[RFC 22/43] mm: shmem: introduce shmem_insert_page
The function inserts a page into a shmem file at a specified offset.
The page can be a regular PAGE_SIZE page or a transparent huge page.
If there is something at the offset (page or swap), the function fails.
The function will be used by the next patch.
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
include/linux/shmem_fs.h | 3 ++
mm/shmem.c | 95
2 files changed, 98 insertions(+)
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index 7a35a6901221..688b92cd4ec7 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -96,6 +96,9 @@ enum sgp_type {
extern int shmem_getpage(struct inode *inode, pgoff_t index,
struct page **pagep, enum sgp_type sgp);
+extern int shmem_insert_page(struct mm_struct *mm, struct inode *inode,
+ pgoff_t index, struct page *page);
+
static inline struct page *shmem_read_mapping_page(
struct address_space *mapping, pgoff_t index)
{
diff --git a/mm/shmem.c b/mm/shmem.c
index bd8840082c94..0a9a2166e51f 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -677,6 +677,101 @@ static void shmem_delete_from_page_cache(struct page
*page, void *radswap)
BUG_ON(error);
}
+int shmem_insert_page(struct mm_struct *mm, struct inode *inode, pgoff_t index,
+ struct page *page)
+{
+ struct address_space *mapping = inode->i_mapping;
+ struct shmem_inode_info *info = SHMEM_I(inode);
+ struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
+ gfp_t gfp = mapping_gfp_mask(mapping);
+ int err;
+ int nr = 1;
+ struct mem_cgroup *memcg;
+ pgoff_t hindex = index;
+ bool on_lru = PageLRU(page);
+
+ if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
+ return -EFBIG;
+
+ if (PageTransHuge(page))
+ nr = HPAGE_PMD_NR;
+ else
+ nr = 1;
+retry:
+ err = 0;
+ if (!shmem_inode_acct_block(inode, nr))
+ err = -ENOSPC;
+ if (err) {
+ int retry = 5;
+
+ /*
+* Try to reclaim some space by splitting a huge page
+* beyond i_size on the filesystem.
+*/
+ while (retry--) {
+ int ret;
+
+ ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
+ if (ret == SHRINK_STOP)
+ break;
+ if (ret)
+ goto retry;
+ }
+ goto failed;
+ }
+
+ if (!on_lru) {
+ __SetPageLocked(page);
+ __SetPageSwapBacked(page);
+ } else {
+ lock_page(page);
+ }
+
+ if (PageTransHuge(page))
+ hindex = round_down(index, HPAGE_PMD_NR);
+ else
+ hindex = index;
+
+ __SetPageReferenced(page);
+
+ err = mem_cgroup_try_charge_delay(page, mm, gfp, ,
+ PageTransHuge(page));
+ if (err)
+ goto out_unlock;
+
+ err = shmem_add_to_page_cache(page, mapping, hindex,
+ NULL, gfp & GFP_RECLAIM_MASK);
+ if (err) {
+ mem_cgroup_cancel_charge(page, memcg,
+ PageTransHuge(page));
+ goto out_unlock;
+ }
+ mem_cgroup_commit_charge(page, memcg, on_lru,
+ PageTransHuge(page));
+
+ if (!on_lru)
+ lru_cache_add_anon(page);
+
+ spin_lock(>lock);
+ info->alloced += compound_nr(page);
+ inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page);
+ shmem_recalc_inode(inode);
+ spin_unlock(>lock);
+
+ flush_dcache_page(page);
+ SetPageUptodate(page);
+ set_page_dirty(page);
+
+ unlock_page(page);
+ return 0;
+
+out_unlock:
+ unlock_page(page);
+ shmem_inode_unacct_blocks(inode, nr);
+failed:
+ return err;
+}
+
/*
* Remove swap entry from page cache, free the swap and its page cache.
*/
--
2.13.3
[RFC 24/43] mm: shmem: prevent swapping of PKRAM-enabled tmpfs pages
Workaround the limitation that shmem pages must be in memory in order
to be preserved by preventing them from being swapped out in the first
place. Do this by marking shmem pages associated with a PKRAM node
as unevictable.
Signed-off-by: Anthony Yznaga
---
mm/shmem.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/mm/shmem.c b/mm/shmem.c
index 9c28ef657cd1..13475073fb52 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -2360,6 +2360,8 @@ static struct inode *shmem_get_inode(struct super_block
*sb, const struct inode
INIT_LIST_HEAD(>swaplist);
simple_xattrs_init(>xattrs);
cache_no_acl(inode);
+ if (sbinfo->pkram)
+ mapping_set_unevictable(inode->i_mapping);
switch (mode & S_IFMT) {
default:
--
2.13.3
[RFC 08/43] mm: PKRAM: introduce super block
The PKRAM super block is the starting point for restoring preserved
memory. By providing the super block to the new kernel at boot time,
preserved memory can be reserved and made available to be restored.
To point the kernel to the location of the super block, one passes
its pfn via the 'pkram' boot param. For that purpose, the pkram super
block pfn is exported via /sys/kernel/pkram. If none is passed, any
preserved memory will not be kept, and a new super block will be
allocated.
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 96 --
1 file changed, 94 insertions(+), 2 deletions(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index 44fadb70acf6..70f2219e6218 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -5,15 +5,18 @@
#include
#include
#include
+#include
#include
#include
#include
#include
#include
+#include
#include
#include
#include
#include
+#include
#include
#include "internal.h"
@@ -80,12 +83,38 @@ struct pkram_node {
#define PKRAM_ACCMODE_MASK 3
/*
+ * The PKRAM super block contains data needed to restore the preserved memory
+ * structure on boot. The pointer to it (pfn) should be passed via the 'pkram'
+ * boot param if one wants to restore preserved data saved by the previously
+ * executing kernel. For that purpose the kernel exports the pfn via
+ * /sys/kernel/pkram. If none is passed, preserved memory if any will not be
+ * preserved and a new clean page will be allocated for the super block.
+ *
+ * The structure occupies a memory page.
+ */
+struct pkram_super_block {
+ __u64 node_pfn; /* first element of the node list */
+};
+
+static unsigned long pkram_sb_pfn __initdata;
+static struct pkram_super_block *pkram_sb;
+
+/*
* For convenience sake PKRAM nodes are kept in an auxiliary doubly-linked list
* connected through the lru field of the page struct.
*/
static LIST_HEAD(pkram_nodes); /* linked through page::lru */
static DEFINE_MUTEX(pkram_mutex); /* serializes open/close */
+/*
+ * The PKRAM super block pfn, see above.
+ */
+static int __init parse_pkram_sb_pfn(char *arg)
+{
+ return kstrtoul(arg, 16, _sb_pfn);
+}
+early_param("pkram", parse_pkram_sb_pfn);
+
static inline struct page *pkram_alloc_page(gfp_t gfp_mask)
{
return alloc_page(gfp_mask);
@@ -209,6 +238,7 @@ static void pkram_stream_init_obj(struct pkram_stream *ps,
struct pkram_obj *obj
* @gfp_mask specifies the memory allocation mask to be used when saving data.
*
* Error values:
+ * %ENODEV: PKRAM not available
* %ENAMETOOLONG: name len >= PKRAM_NAME_MAX
* %ENOMEM: insufficient memory available
* %EEXIST: node with specified name already exists
@@ -224,6 +254,9 @@ int pkram_prepare_save(struct pkram_stream *ps, const char
*name, gfp_t gfp_mask
struct pkram_node *node;
int err = 0;
+ if (!pkram_sb)
+ return -ENODEV;
+
if (strlen(name) >= PKRAM_NAME_MAX)
return -ENAMETOOLONG;
@@ -333,6 +366,7 @@ void pkram_discard_save(struct pkram_stream *ps)
* Returns 0 on success, -errno on failure.
*
* Error values:
+ * %ENODEV: PKRAM not available
* %ENOENT: node with specified name does not exist
* %EBUSY: save to required node has not finished yet
*
@@ -343,6 +377,9 @@ int pkram_prepare_load(struct pkram_stream *ps, const char
*name)
struct pkram_node *node;
int err = 0;
+ if (!pkram_sb)
+ return -ENODEV;
+
mutex_lock(_mutex);
node = pkram_find_node(name);
if (!node) {
@@ -708,6 +745,7 @@ static void __pkram_reboot(void)
node->node_pfn = node_pfn;
node_pfn = page_to_pfn(page);
}
+ pkram_sb->node_pfn = node_pfn;
}
static int pkram_reboot(struct notifier_block *notifier,
@@ -715,7 +753,8 @@ static int pkram_reboot(struct notifier_block *notifier,
{
if (val != SYS_RESTART)
return NOTIFY_DONE;
- __pkram_reboot();
+ if (pkram_sb)
+ __pkram_reboot();
return NOTIFY_OK;
}
@@ -723,9 +762,62 @@ static struct notifier_block pkram_reboot_notifier = {
.notifier_call = pkram_reboot,
};
+static ssize_t show_pkram_sb_pfn(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ unsigned long pfn = pkram_sb ? PFN_DOWN(__pa(pkram_sb)) : 0;
+
+ return sprintf(buf, "%lx\n", pfn);
+}
+
+static struct kobj_attribute pkram_sb_pfn_attr =
+ __ATTR(pkram, 0444, show_pkram_sb_pfn, NULL);
+
+static struct attribute *pkram_attrs[] = {
+ _sb_pfn_attr.attr,
+ NULL,
+};
+
+static struct attribute_group pkram_attr_group = {
+ .attrs = pkram_attrs,
+};
+
+/* returns non-zero on success */
+static int __init pkram_init_sb(void)
+{
+ unsigned long pfn;
+ struct
[RFC 25/43] mm: shmem: specify the mm to use when inserting pages
Explicitly specify the mm to pass to shmem_insert_page() when
the pkram_stream is initialized rather than use the mm of the
current thread. This will allow for multiple kernel threads to
target the same mm when inserting pages in parallel.
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 1 +
mm/pkram.c| 1 +
mm/shmem_pkram.c | 2 +-
3 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index b47b3aef16e3..cbb79d2803c0 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -18,6 +18,7 @@ struct pkram_stream {
unsigned long next_index;
struct address_space *mapping;
+ struct mm_struct *mm;
/* byte data */
struct page *data_page;
diff --git a/mm/pkram.c b/mm/pkram.c
index 4d4d836fea53..a5e539052af6 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -565,6 +565,7 @@ static void pkram_stream_init(struct pkram_stream *ps,
memset(ps, 0, sizeof(*ps));
ps->gfp_mask = gfp_mask;
ps->node = node;
+ ps->mm = current->mm;
}
static void pkram_stream_init_obj(struct pkram_stream *ps, struct pkram_obj
*obj)
diff --git a/mm/shmem_pkram.c b/mm/shmem_pkram.c
index 3fa9cfbe0003..c97d64393822 100644
--- a/mm/shmem_pkram.c
+++ b/mm/shmem_pkram.c
@@ -236,7 +236,7 @@ static int load_file_content(struct pkram_stream *ps)
if (!page)
break;
- err = shmem_insert_page(current->mm, ps->mapping->host, index,
page);
+ err = shmem_insert_page(ps->mm, ps->mapping->host, index, page);
put_page(page);
} while (!err);
--
2.13.3
[RFC 20/43] PKRAM: disable feature when running the kdump kernel
The kdump kernel should not preserve or restore pages.
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index 95e691382721..4d4d836fea53 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -1,4 +1,5 @@
// SPDX-License-Identifier: GPL-2.0
+#include
#include
#include
#include
@@ -193,7 +194,7 @@ void __init pkram_reserve(void)
{
int err = 0;
- if (!pkram_sb_pfn)
+ if (!pkram_sb_pfn || is_kdump_kernel())
return;
pr_info("PKRAM: Examining preserved memory...\n");
@@ -305,6 +306,9 @@ static void pkram_show_banned(void)
int i;
unsigned long n, total = 0;
+ if (is_kdump_kernel())
+ return;
+
pr_info("PKRAM: banned regions:\n");
for (i = 0; i < nr_banned; i++) {
n = banned[i].end - banned[i].start + 1;
@@ -1223,7 +1227,7 @@ static int __init pkram_init_sb(void)
static int __init pkram_init(void)
{
- if (pkram_init_sb()) {
+ if (!is_kdump_kernel() && pkram_init_sb()) {
register_reboot_notifier(_reboot_notifier);
register_shrinker(_pages_shrinker);
sysfs_update_group(kernel_kobj, _attr_group);
--
2.13.3
[RFC 19/43] mm: PKRAM: allow preserved memory to be freed from userspace
To free all space utilized for preserved memory, one can write 0 to
/sys/kernel/pkram. This will destroy all PKRAM nodes that are not
currently being read or written.
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
mm/pkram.c | 39 ++-
1 file changed, 38 insertions(+), 1 deletion(-)
diff --git a/mm/pkram.c b/mm/pkram.c
index 0aaaf9b79682..95e691382721 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -509,6 +509,32 @@ static void pkram_truncate_node(struct pkram_node *node)
node->obj_pfn = 0;
}
+/*
+ * Free all nodes that are not under operation.
+ */
+static void pkram_truncate(void)
+{
+ struct page *page, *tmp;
+ struct pkram_node *node;
+ LIST_HEAD(dispose);
+
+ mutex_lock(_mutex);
+ list_for_each_entry_safe(page, tmp, _nodes, lru) {
+ node = page_address(page);
+ if (!(node->flags & PKRAM_ACCMODE_MASK))
+ list_move(>lru, );
+ }
+ mutex_unlock(_mutex);
+
+ while (!list_empty()) {
+ page = list_first_entry(, struct page, lru);
+ list_del(>lru);
+ node = page_address(page);
+ pkram_truncate_node(node);
+ pkram_free_page(node);
+ }
+}
+
static void pkram_add_link(struct pkram_link *link, struct pkram_obj *obj)
{
link->link_pfn = obj->link_pfn;
@@ -1141,8 +1167,19 @@ static ssize_t show_pkram_sb_pfn(struct kobject *kobj,
return sprintf(buf, "%lx\n", pfn);
}
+static ssize_t store_pkram_sb_pfn(struct kobject *kobj,
+ struct kobj_attribute *attr, const char *buf, size_t count)
+{
+ int val;
+
+ if (kstrtoint(buf, 0, ) || val)
+ return -EINVAL;
+ pkram_truncate();
+ return count;
+}
+
static struct kobj_attribute pkram_sb_pfn_attr =
- __ATTR(pkram, 0444, show_pkram_sb_pfn, NULL);
+ __ATTR(pkram, 0644, show_pkram_sb_pfn, store_pkram_sb_pfn);
static struct attribute *pkram_attrs[] = {
_sb_pfn_attr.attr,
--
2.13.3
[RFC 02/43] mm: PKRAM: implement node load and save functions
Preserved memory is divided into nodes which can be saved and loaded
independently of each other. PKRAM nodes are kept on a list and
identified by unique names. Whenever a save operation is initiated by
calling pkram_prepare_save(), a new node is created and linked to the
list. When the save operation has been committed by calling
pkram_finish_save(), the node becomes loadable. A load operation can be
then initiated by calling pkram_prepare_load() which deletes the node
from the list and prepares the corresponding stream for loading data
from it. After the load has been finished, the pkram_finish_load()
function must be called to free the node. Nodes are also deleted when a
save operation is discarded, i.e. pkram_discard_save() is called instead
of pkram_finish_save().
Originally-by: Vladimir Davydov
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 7 ++-
mm/pkram.c| 148 --
2 files changed, 149 insertions(+), 6 deletions(-)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index 4c4e13311ec8..83a0579e4c1c 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -6,7 +6,12 @@
#include
#include
-struct pkram_stream;
+struct pkram_node;
+
+struct pkram_stream {
+ gfp_t gfp_mask;
+ struct pkram_node *node;
+};
#define PKRAM_NAME_MAX 256 /* including nul */
diff --git a/mm/pkram.c b/mm/pkram.c
index d6f2f79d4852..5c57126353ff 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -2,16 +2,85 @@
#include
#include
#include
+#include
#include
+#include
#include
+#include
#include
+/*
+ * Preserved memory is divided into nodes that can be saved or loaded
+ * independently of each other. The nodes are identified by unique name
+ * strings.
+ *
+ * The structure occupies a memory page.
+ */
+struct pkram_node {
+ __u32 flags;
+
+ __u8name[PKRAM_NAME_MAX];
+};
+
+#define PKRAM_SAVE 1
+#define PKRAM_LOAD 2
+#define PKRAM_ACCMODE_MASK 3
+
+static LIST_HEAD(pkram_nodes); /* linked through page::lru */
+static DEFINE_MUTEX(pkram_mutex); /* serializes open/close */
+
+static inline struct page *pkram_alloc_page(gfp_t gfp_mask)
+{
+ return alloc_page(gfp_mask);
+}
+
+static inline void pkram_free_page(void *addr)
+{
+ free_page((unsigned long)addr);
+}
+
+static inline void pkram_insert_node(struct pkram_node *node)
+{
+ list_add(_to_page(node)->lru, _nodes);
+}
+
+static inline void pkram_delete_node(struct pkram_node *node)
+{
+ list_del(_to_page(node)->lru);
+}
+
+static struct pkram_node *pkram_find_node(const char *name)
+{
+ struct page *page;
+ struct pkram_node *node;
+
+ list_for_each_entry(page, _nodes, lru) {
+ node = page_address(page);
+ if (strcmp(node->name, name) == 0)
+ return node;
+ }
+ return NULL;
+}
+
+static void pkram_stream_init(struct pkram_stream *ps,
+struct pkram_node *node, gfp_t gfp_mask)
+{
+ memset(ps, 0, sizeof(*ps));
+ ps->gfp_mask = gfp_mask;
+ ps->node = node;
+}
+
/**
* Create a preserved memory node with name @name and initialize stream @ps
* for saving data to it.
*
* @gfp_mask specifies the memory allocation mask to be used when saving data.
*
+ * Error values:
+ * %ENAMETOOLONG: name len >= PKRAM_NAME_MAX
+ * %ENOMEM: insufficient memory available
+ * %EEXIST: node with specified name already exists
+ *
* Returns 0 on success, -errno on failure.
*
* After the save has finished, pkram_finish_save() (or pkram_discard_save() in
@@ -19,7 +88,34 @@
*/
int pkram_prepare_save(struct pkram_stream *ps, const char *name, gfp_t
gfp_mask)
{
- return -ENOSYS;
+ struct page *page;
+ struct pkram_node *node;
+ int err = 0;
+
+ if (strlen(name) >= PKRAM_NAME_MAX)
+ return -ENAMETOOLONG;
+
+ page = pkram_alloc_page(GFP_KERNEL | __GFP_ZERO);
+ if (!page)
+ return -ENOMEM;
+ node = page_address(page);
+
+ node->flags = PKRAM_SAVE;
+ strcpy(node->name, name);
+
+ mutex_lock(_mutex);
+ if (!pkram_find_node(name))
+ pkram_insert_node(node);
+ else
+ err = -EEXIST;
+ mutex_unlock(_mutex);
+ if (err) {
+ __free_page(page);
+ return err;
+ }
+
+ pkram_stream_init(ps, node, gfp_mask);
+ return 0;
}
/**
@@ -50,7 +146,12 @@ void pkram_finish_save_obj(struct pkram_stream *ps)
*/
void pkram_finish_save(struct pkram_stream *ps)
{
- BUG();
+ struct pkram_node *node = ps->node;
+
+ BUG_ON((node->flags & PKRAM_ACCMODE_MASK) != PKRAM_SAVE);
+
+ smp_wmb();
+ node->flags &= ~PKRAM_ACCMODE_MASK;
}
/**
@@ -60,7 +161,15 @@ void pkram_finish_save(struct pkr
[RFC 14/43] mm: memblock: PKRAM: prevent memblock resize from clobbering preserved pages
The size of the memblock reserved array may be increased while preserved
pages are being reserved. When this happens, preserved pages that have
not yet been reserved are at risk for being clobbered when space for a
larger array is allocated.
When called from memblock_double_array(), a wrapper around
memblock_find_in_range() walks the preserved pages pagetable to find
sufficiently sized ranges without preserved pages and passes them to
memblock_find_in_range().
Signed-off-by: Anthony Yznaga
---
include/linux/pkram.h | 3 +++
mm/memblock.c | 15 +--
mm/pkram.c| 51 +++
3 files changed, 67 insertions(+), 2 deletions(-)
diff --git a/include/linux/pkram.h b/include/linux/pkram.h
index edc5d8bef9d3..409022e1472f 100644
--- a/include/linux/pkram.h
+++ b/include/linux/pkram.h
@@ -62,6 +62,9 @@ struct page *pkram_load_page(struct pkram_stream *ps,
unsigned long *index,
ssize_t pkram_write(struct pkram_stream *ps, const void *buf, size_t count);
size_t pkram_read(struct pkram_stream *ps, void *buf, size_t count);
+phys_addr_t pkram_memblock_find_in_range(phys_addr_t start, phys_addr_t end,
+phys_addr_t size, phys_addr_t align);
+
#ifdef CONFIG_PKRAM
extern unsigned long pkram_reserved_pages;
void pkram_reserve(void);
diff --git a/mm/memblock.c b/mm/memblock.c
index c79ba6f9920c..69ae883b8d21 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -16,6 +16,7 @@
#include
#include
#include
+#include
#include
#include
@@ -349,6 +350,16 @@ phys_addr_t __init_memblock
memblock_find_in_range(phys_addr_t start,
return ret;
}
+phys_addr_t __init_memblock __memblock_find_in_range(phys_addr_t start,
+ phys_addr_t end, phys_addr_t size,
+ phys_addr_t align)
+{
+ if (IS_ENABLED(CONFIG_PKRAM))
+ return pkram_memblock_find_in_range(start, end, size, align);
+ else
+ return memblock_find_in_range(start, end, size, align);
+}
+
static void __init_memblock memblock_remove_region(struct memblock_type *type,
unsigned long r)
{
type->total_size -= type->regions[r].size;
@@ -447,11 +458,11 @@ static int __init_memblock memblock_double_array(struct
memblock_type *type,
if (type != )
new_area_start = new_area_size = 0;
- addr = memblock_find_in_range(new_area_start + new_area_size,
+ addr = __memblock_find_in_range(new_area_start + new_area_size,
memblock.current_limit,
new_alloc_size, PAGE_SIZE);
if (!addr && new_area_size)
- addr = memblock_find_in_range(0,
+ addr = __memblock_find_in_range(0,
min(new_area_start, memblock.current_limit),
new_alloc_size, PAGE_SIZE);
diff --git a/mm/pkram.c b/mm/pkram.c
index dd3c89614010..e49c9bcd3854 100644
--- a/mm/pkram.c
+++ b/mm/pkram.c
@@ -1238,3 +1238,54 @@ void pkram_free_pgt(void)
__free_pages_core(virt_to_page(pkram_pgd), 0);
pkram_pgd = NULL;
}
+
+static int __init_memblock pkram_memblock_find_cb(struct pkram_pg_state *st,
unsigned long base, unsigned long size)
+{
+ unsigned long end = base + size;
+ unsigned long addr;
+
+ if (size < st->min_size)
+ return 0;
+
+ addr = memblock_find_in_range(base, end, st->min_size, PAGE_SIZE);
+ if (!addr)
+ return 0;
+
+ st->retval = addr;
+ return 1;
+}
+
+/*
+ * It may be necessary to allocate a larger reserved memblock array
+ * while populating it with ranges of preserved pages. To avoid
+ * trampling preserved pages that have not yet been added to the
+ * memblock reserved list this function implements a wrapper around
+ * memblock_find_in_range() that restricts searches to subranges
+ * that do not contain preserved pages.
+ */
+phys_addr_t __init_memblock pkram_memblock_find_in_range(phys_addr_t start,
+ phys_addr_t end, phys_addr_t size,
+ phys_addr_t align)
+{
+ struct pkram_pg_state st = {
+ .range_cb = pkram_memblock_find_cb,
+ .min_addr = start,
+ .max_addr = end,
+ .min_size = PAGE_ALIGN(size),
+ .find_holes = true,
+ };
+
+ if (!pkram_reservation_in_progress)
+ return memblock_find_in_range(start, end, size, align);
+
+ if (!pkram_pgd) {
+ WARN_ONCE(1, "No preserved pages pagetable\n");
+ return memblock_find_in_range(start, end, size, align);
+ }
+
+ WARN_ONCE(memblock_bottom_up(), "PKRAM: bottom up memblock allocation
not yet su
