From: Zi Yan <z...@nvidia.com> No functionality is changed. Prepare for the following patches, which add parallel, concurrent page migration modes in conjunction to the existing modes.
Signed-off-by: Zi Yan <z...@nvidia.com> --- fs/aio.c | 10 +++++----- fs/f2fs/data.c | 4 ++-- fs/hugetlbfs/inode.c | 2 +- fs/iomap.c | 2 +- fs/ubifs/file.c | 2 +- include/linux/migrate_mode.h | 2 ++ mm/balloon_compaction.c | 2 +- mm/compaction.c | 22 +++++++++++----------- mm/migrate.c | 18 +++++++++--------- mm/zsmalloc.c | 2 +- 10 files changed, 34 insertions(+), 32 deletions(-) diff --git a/fs/aio.c b/fs/aio.c index 38b741a..0a88dfd 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -389,7 +389,7 @@ static int aio_migratepage(struct address_space *mapping, struct page *new, * happen under the ctx->completion_lock. That does not work with the * migration workflow of MIGRATE_SYNC_NO_COPY. */ - if (mode == MIGRATE_SYNC_NO_COPY) + if ((mode & MIGRATE_MODE_MASK) == MIGRATE_SYNC_NO_COPY) return -EINVAL; rc = 0; @@ -1300,10 +1300,10 @@ static long read_events(struct kioctx *ctx, long min_nr, long nr, * Create an aio_context capable of receiving at least nr_events. * ctxp must not point to an aio_context that already exists, and * must be initialized to 0 prior to the call. On successful - * creation of the aio_context, *ctxp is filled in with the resulting + * creation of the aio_context, *ctxp is filled in with the resulting * handle. May fail with -EINVAL if *ctxp is not initialized, - * if the specified nr_events exceeds internal limits. May fail - * with -EAGAIN if the specified nr_events exceeds the user's limit + * if the specified nr_events exceeds internal limits. May fail + * with -EAGAIN if the specified nr_events exceeds the user's limit * of available events. May fail with -ENOMEM if insufficient kernel * resources are available. May fail with -EFAULT if an invalid * pointer is passed for ctxp. Will fail with -ENOSYS if not @@ -1373,7 +1373,7 @@ COMPAT_SYSCALL_DEFINE2(io_setup, unsigned, nr_events, u32 __user *, ctx32p) #endif /* sys_io_destroy: - * Destroy the aio_context specified. May cancel any outstanding + * Destroy the aio_context specified. May cancel any outstanding * AIOs and block on completion. Will fail with -ENOSYS if not * implemented. May fail with -EINVAL if the context pointed to * is invalid. diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 97279441..e7f0e3a 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -2792,7 +2792,7 @@ int f2fs_migrate_page(struct address_space *mapping, /* migrating an atomic written page is safe with the inmem_lock hold */ if (atomic_written) { - if (mode != MIGRATE_SYNC) + if ((mode & MIGRATE_MODE_MASK) != MIGRATE_SYNC) return -EBUSY; if (!mutex_trylock(&fi->inmem_lock)) return -EAGAIN; @@ -2825,7 +2825,7 @@ int f2fs_migrate_page(struct address_space *mapping, f2fs_clear_page_private(page); } - if (mode != MIGRATE_SYNC_NO_COPY) + if ((mode & MIGRATE_MODE_MASK) != MIGRATE_SYNC_NO_COPY) migrate_page_copy(newpage, page); else migrate_page_states(newpage, page); diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index ec32fec..04ba8bb 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -885,7 +885,7 @@ static int hugetlbfs_migrate_page(struct address_space *mapping, set_page_private(page, 0); } - if (mode != MIGRATE_SYNC_NO_COPY) + if ((mode & MIGRATE_MODE_MASK) != MIGRATE_SYNC_NO_COPY) migrate_page_copy(newpage, page); else migrate_page_states(newpage, page); diff --git a/fs/iomap.c b/fs/iomap.c index abdd18e..8ee3f9f 100644 --- a/fs/iomap.c +++ b/fs/iomap.c @@ -584,7 +584,7 @@ iomap_migrate_page(struct address_space *mapping, struct page *newpage, SetPagePrivate(newpage); } - if (mode != MIGRATE_SYNC_NO_COPY) + if ((mode & MIGRATE_MODE_MASK) != MIGRATE_SYNC_NO_COPY) migrate_page_copy(newpage, page); else migrate_page_states(newpage, page); diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c index 5d2ffb1..2bb8788 100644 --- a/fs/ubifs/file.c +++ b/fs/ubifs/file.c @@ -1490,7 +1490,7 @@ static int ubifs_migrate_page(struct address_space *mapping, SetPagePrivate(newpage); } - if (mode != MIGRATE_SYNC_NO_COPY) + if ((mode & MIGRATE_MODE_MASK) != MIGRATE_SYNC_NO_COPY) migrate_page_copy(newpage, page); else migrate_page_states(newpage, page); diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h index 883c992..59d75fc 100644 --- a/include/linux/migrate_mode.h +++ b/include/linux/migrate_mode.h @@ -17,6 +17,8 @@ enum migrate_mode { MIGRATE_SYNC_LIGHT, MIGRATE_SYNC, MIGRATE_SYNC_NO_COPY, + + MIGRATE_MODE_MASK = 3, }; #endif /* MIGRATE_MODE_H_INCLUDED */ diff --git a/mm/balloon_compaction.c b/mm/balloon_compaction.c index ef858d5..5acb55f 100644 --- a/mm/balloon_compaction.c +++ b/mm/balloon_compaction.c @@ -158,7 +158,7 @@ int balloon_page_migrate(struct address_space *mapping, * is unlikely to be use with ballon pages. See include/linux/hmm.h for * user of the MIGRATE_SYNC_NO_COPY mode. */ - if (mode == MIGRATE_SYNC_NO_COPY) + if ((mode & MIGRATE_MODE_MASK) == MIGRATE_SYNC_NO_COPY) return -EINVAL; VM_BUG_ON_PAGE(!PageLocked(page), page); diff --git a/mm/compaction.c b/mm/compaction.c index f171a83..bfcbe08 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -408,7 +408,7 @@ static void update_cached_migrate(struct compact_control *cc, unsigned long pfn) if (pfn > zone->compact_cached_migrate_pfn[0]) zone->compact_cached_migrate_pfn[0] = pfn; - if (cc->mode != MIGRATE_ASYNC && + if ((cc->mode & MIGRATE_MODE_MASK) != MIGRATE_ASYNC && pfn > zone->compact_cached_migrate_pfn[1]) zone->compact_cached_migrate_pfn[1] = pfn; } @@ -475,7 +475,7 @@ static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags, struct compact_control *cc) { /* Track if the lock is contended in async mode */ - if (cc->mode == MIGRATE_ASYNC && !cc->contended) { + if (((cc->mode & MIGRATE_MODE_MASK) == MIGRATE_ASYNC) && !cc->contended) { if (spin_trylock_irqsave(lock, *flags)) return true; @@ -792,7 +792,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, */ while (unlikely(too_many_isolated(pgdat))) { /* async migration should just abort */ - if (cc->mode == MIGRATE_ASYNC) + if ((cc->mode & MIGRATE_MODE_MASK) == MIGRATE_ASYNC) return 0; congestion_wait(BLK_RW_ASYNC, HZ/10); @@ -803,7 +803,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, cond_resched(); - if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) { + if (cc->direct_compaction && ((cc->mode & MIGRATE_MODE_MASK) == MIGRATE_ASYNC)) { skip_on_failure = true; next_skip_pfn = block_end_pfn(low_pfn, cc->order); } @@ -1117,7 +1117,7 @@ static bool suitable_migration_source(struct compact_control *cc, if (pageblock_skip_persistent(page)) return false; - if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction) + if (((cc->mode & MIGRATE_MODE_MASK) != MIGRATE_ASYNC) || !cc->direct_compaction) return true; block_mt = get_pageblock_migratetype(page); @@ -1216,7 +1216,7 @@ fast_isolate_around(struct compact_control *cc, unsigned long pfn, unsigned long return; /* Minimise scanning during async compaction */ - if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC) + if (cc->direct_compaction && (cc->mode & MIGRATE_MODE_MASK) == MIGRATE_ASYNC) return; /* Pageblock boundaries */ @@ -1448,7 +1448,7 @@ static void isolate_freepages(struct compact_control *cc) block_end_pfn = min(block_start_pfn + pageblock_nr_pages, zone_end_pfn(zone)); low_pfn = pageblock_end_pfn(cc->migrate_pfn); - stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1; + stride = (cc->mode & MIGRATE_MODE_MASK) == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1; /* * Isolate free pages until enough are available to migrate the @@ -1734,7 +1734,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, struct page *page; const isolate_mode_t isolate_mode = (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) | - (cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0); + (((cc->mode & MIGRATE_MODE_MASK) != MIGRATE_SYNC) ? ISOLATE_ASYNC_MIGRATE : 0); bool fast_find_block; /* @@ -1907,7 +1907,7 @@ static enum compact_result __compact_finished(struct compact_control *cc) * to sync compaction, as async compaction operates * on pageblocks of the same migratetype. */ - if (cc->mode == MIGRATE_ASYNC || + if ((cc->mode & MIGRATE_MODE_MASK) == MIGRATE_ASYNC || IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages)) { return COMPACT_SUCCESS; @@ -2063,7 +2063,7 @@ compact_zone(struct compact_control *cc, struct capture_control *capc) unsigned long start_pfn = cc->zone->zone_start_pfn; unsigned long end_pfn = zone_end_pfn(cc->zone); unsigned long last_migrated_pfn; - const bool sync = cc->mode != MIGRATE_ASYNC; + const bool sync = (cc->mode & MIGRATE_MODE_MASK) != MIGRATE_ASYNC; bool update_cached; cc->migratetype = gfpflags_to_migratetype(cc->gfp_mask); @@ -2195,7 +2195,7 @@ compact_zone(struct compact_control *cc, struct capture_control *capc) * order-aligned block, so skip the rest of it. */ if (cc->direct_compaction && - (cc->mode == MIGRATE_ASYNC)) { + ((cc->mode & MIGRATE_MODE_MASK) == MIGRATE_ASYNC)) { cc->migrate_pfn = block_end_pfn( cc->migrate_pfn - 1, cc->order); /* Draining pcplists is useless in this case */ diff --git a/mm/migrate.c b/mm/migrate.c index ac6f493..c161c03 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -691,7 +691,7 @@ int migrate_page(struct address_space *mapping, if (rc != MIGRATEPAGE_SUCCESS) return rc; - if (mode != MIGRATE_SYNC_NO_COPY) + if ((mode & MIGRATE_MODE_MASK) != MIGRATE_SYNC_NO_COPY) migrate_page_copy(newpage, page); else migrate_page_states(newpage, page); @@ -707,7 +707,7 @@ static bool buffer_migrate_lock_buffers(struct buffer_head *head, struct buffer_head *bh = head; /* Simple case, sync compaction */ - if (mode != MIGRATE_ASYNC) { + if ((mode & MIGRATE_MODE_MASK) != MIGRATE_ASYNC) { do { lock_buffer(bh); bh = bh->b_this_page; @@ -804,7 +804,7 @@ static int __buffer_migrate_page(struct address_space *mapping, SetPagePrivate(newpage); - if (mode != MIGRATE_SYNC_NO_COPY) + if ((mode & MIGRATE_MODE_MASK) != MIGRATE_SYNC_NO_COPY) migrate_page_copy(newpage, page); else migrate_page_states(newpage, page); @@ -895,7 +895,7 @@ static int fallback_migrate_page(struct address_space *mapping, { if (PageDirty(page)) { /* Only writeback pages in full synchronous migration */ - switch (mode) { + switch (mode & MIGRATE_MODE_MASK) { case MIGRATE_SYNC: case MIGRATE_SYNC_NO_COPY: break; @@ -911,7 +911,7 @@ static int fallback_migrate_page(struct address_space *mapping, */ if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL)) - return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY; + return (mode & MIGRATE_MODE_MASK) == MIGRATE_SYNC ? -EAGAIN : -EBUSY; return migrate_page(mapping, newpage, page, mode); } @@ -1009,7 +1009,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage, bool is_lru = !__PageMovable(page); if (!trylock_page(page)) { - if (!force || mode == MIGRATE_ASYNC) + if (!force || ((mode & MIGRATE_MODE_MASK) == MIGRATE_ASYNC)) goto out; /* @@ -1038,7 +1038,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage, * the retry loop is too short and in the sync-light case, * the overhead of stalling is too much */ - switch (mode) { + switch (mode & MIGRATE_MODE_MASK) { case MIGRATE_SYNC: case MIGRATE_SYNC_NO_COPY: break; @@ -1303,9 +1303,9 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, return -ENOMEM; if (!trylock_page(hpage)) { - if (!force) + if (!force || ((mode & MIGRATE_MODE_MASK) != MIGRATE_SYNC)) goto out; - switch (mode) { + switch (mode & MIGRATE_MODE_MASK) { case MIGRATE_SYNC: case MIGRATE_SYNC_NO_COPY: break; diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 0787d33..018bb51 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -1981,7 +1981,7 @@ static int zs_page_migrate(struct address_space *mapping, struct page *newpage, * happen under the zs lock, which does not work with * MIGRATE_SYNC_NO_COPY workflow. */ - if (mode == MIGRATE_SYNC_NO_COPY) + if ((mode & MIGRATE_MODE_MASK) == MIGRATE_SYNC_NO_COPY) return -EINVAL; VM_BUG_ON_PAGE(!PageMovable(page), page); -- 2.7.4