Current pageblock_flags is only 4 bits, so it has to share a char size
in cmpxchg when get set, the false sharing cause perf drop.
If we incrase the bits up to 8, false sharing would gone in cmpxchg. and
the only cost is half char per pageblock, which is half char per 128MB
on x86, 4 chars in 1 GB.
Signed-off-by: Alex Shi <alex....@linux.alibaba.com>
Cc: Andrew Morton <a...@linux-foundation.org>
Cc: Mel Gorman <mgor...@techsingularity.net>
Cc: linux-kernel@vger.kernel.org
Cc: linux...@kvack.org
---
include/linux/page-isolation.h | 2 +-
include/linux/pageblock-flags.h | 2 +-
mm/page_alloc.c | 10 +++-------
3 files changed, 5 insertions(+), 9 deletions(-)
diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h
index 572458016331..baa2e1ecc134 100644
--- a/include/linux/page-isolation.h
+++ b/include/linux/page-isolation.h
@@ -35,7 +35,7 @@ static inline bool is_migrate_isolate(int migratetype)
struct page *has_unmovable_pages(struct zone *zone, struct page *page,
int migratetype, int flags);
-void set_pageblock_migratetype(struct page *page, int migratetype);
+void set_pageblock_migratetype(struct page *page, unsigned char migratetype);
int move_freepages_block(struct zone *zone, struct page *page,
int migratetype, int *num_movable);
diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h
index d189441568eb..f785c9d6d68c 100644
--- a/include/linux/pageblock-flags.h
+++ b/include/linux/pageblock-flags.h
@@ -25,7 +25,7 @@ enum pageblock_bits {
* Assume the bits will always align on a word. If this assumption
* changes then get/set pageblock needs updating.
*/
- NR_PAGEBLOCK_BITS
+ NR_PAGEBLOCK_BITS = BITS_PER_BYTE
};
#ifdef CONFIG_HUGETLB_PAGE
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 81e96d4d9c42..3688e6b83318 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -517,19 +517,15 @@ void set_pfnblock_flags_mask(struct page *page, unsigned
long flags,
unsigned long bitidx, byte_bitidx;
unsigned char old_byte, byte;
- BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
+ BUILD_BUG_ON(NR_PAGEBLOCK_BITS != BITS_PER_BYTE);
BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
bitmap = get_pageblock_bitmap(page, pfn);
bitidx = pfn_to_bitidx(page, pfn);
byte_bitidx = bitidx / BITS_PER_BYTE;
- bitidx &= (BITS_PER_BYTE-1);
VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
- mask <<= bitidx;
- flags <<= bitidx;
-
byte = READ_ONCE(bitmap[byte_bitidx]);
for (;;) {
old_byte = cmpxchg(&bitmap[byte_bitidx], byte, (byte & ~mask) |
flags);
@@ -539,13 +535,13 @@ void set_pfnblock_flags_mask(struct page *page, unsigned
long flags,
}
}
-void set_pageblock_migratetype(struct page *page, int migratetype)
+void set_pageblock_migratetype(struct page *page, unsigned char migratetype)
{
if (unlikely(page_group_by_mobility_disabled &&
migratetype < MIGRATE_PCPTYPES))
migratetype = MIGRATE_UNMOVABLE;
- set_pfnblock_flags_mask(page, (unsigned long)migratetype,
+ set_pfnblock_flags_mask(page, migratetype,
page_to_pfn(page), MIGRATETYPE_MASK);
}
--
1.8.3.1
