[PATCH 49/49] mm: consolidate struct page power-of-2 size checks for HVO
Muchun Song
songmuchun at bytedance.com
Sun Apr 5 22:52:40 AEST 2026
The Hugepage Vmemmap Optimization (HVO) requires that struct page
size is a power of two. This size is evaluated by the C compiler
and currently cannot be natively evaluated by Kconfig. Therefore,
the condition is_power_of_2(sizeof(struct page)) was scattered
across several macros and static inline functions.
Extract the check into a preprocessor macro
STRUCT_PAGE_SIZE_IS_POWER_OF_2 evaluated during the Kbuild process.
Define SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED as a master toggle
that is 1 only if both Kconfig CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION
and the power of 2 size check are true.
This allows us to completely remove all scattered sizeof(struct page)
checks, making the code much cleaner and eliminating redundant logic.
Additionally, mm/hugetlb_vmemmap.c and its corresponding header are now
guarded by SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED. This brings an added
benefit: when struct page size is not a power of 2, the compiler can
entirely optimize away the unused functions in mm/hugetlb_vmemmap.c,
reducing kernel image size.
Signed-off-by: Muchun Song <songmuchun at bytedance.com>
---
include/linux/mm_types.h | 2 ++
include/linux/mm_types_task.h | 4 ++++
include/linux/mmzone.h | 32 +++++++++++++++-----------------
include/linux/page-flags.h | 28 ++++------------------------
kernel/bounds.c | 2 ++
mm/hugetlb_vmemmap.c | 2 ++
mm/hugetlb_vmemmap.h | 4 +---
mm/internal.h | 3 ---
mm/sparse.c | 6 ++----
mm/util.c | 2 +-
10 files changed, 33 insertions(+), 52 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index a308e2c23b82..6de6c0c20f8b 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -15,7 +15,9 @@
#include <linux/cpumask.h>
#include <linux/uprobes.h>
#include <linux/rcupdate.h>
+#ifndef __GENERATING_BOUNDS_H
#include <linux/page-flags-layout.h>
+#endif
#include <linux/workqueue.h>
#include <linux/seqlock.h>
#include <linux/percpu_counter.h>
diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h
index 11bf319d78ec..09e5039fff97 100644
--- a/include/linux/mm_types_task.h
+++ b/include/linux/mm_types_task.h
@@ -17,7 +17,11 @@
#include <asm/tlbbatch.h>
#endif
+#ifndef __GENERATING_BOUNDS_H
#define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8)
+#else
+#define ALLOC_SPLIT_PTLOCKS 0
+#endif
/*
* When updating this, please also update struct resident_page_types[] in
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index a6900f585f9b..3a46cb0bfaaa 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -96,27 +96,26 @@
#define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER)
-/*
- * Hugepage Vmemmap Optimization (HVO) requires struct pages of the head page to
- * be naturally aligned with regard to the folio size.
- *
- * HVO which is only active if the size of struct page is a power of 2.
- */
-#define MAX_FOLIO_VMEMMAP_ALIGN \
- (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION) && \
- is_power_of_2(sizeof(struct page)) ? \
- MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0)
-
/* The number of vmemmap pages required by a vmemmap-optimized folio. */
#define OPTIMIZED_FOLIO_VMEMMAP_PAGES 1
#define OPTIMIZED_FOLIO_VMEMMAP_SIZE (OPTIMIZED_FOLIO_VMEMMAP_PAGES * PAGE_SIZE)
#define OPTIMIZED_FOLIO_VMEMMAP_PAGE_STRUCTS (OPTIMIZED_FOLIO_VMEMMAP_SIZE / sizeof(struct page))
#define OPTIMIZABLE_FOLIO_MIN_ORDER (ilog2(OPTIMIZED_FOLIO_VMEMMAP_PAGE_STRUCTS) + 1)
+#if defined(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION) && STRUCT_PAGE_SIZE_IS_POWER_OF_2
+#define SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED 1
+/*
+ * Hugepage Vmemmap Optimization (HVO) requires struct pages of the head page to
+ * be naturally aligned with regard to the folio size.
+ */
+#define MAX_FOLIO_VMEMMAP_ALIGN (MAX_FOLIO_NR_PAGES * sizeof(struct page))
#define __NR_OPTIMIZABLE_FOLIO_SIZES (MAX_FOLIO_ORDER - OPTIMIZABLE_FOLIO_MIN_ORDER + 1)
#define NR_OPTIMIZABLE_FOLIO_SIZES \
- ((__NR_OPTIMIZABLE_FOLIO_SIZES > 0 && \
- IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION)) ? __NR_OPTIMIZABLE_FOLIO_SIZES : 0)
+ (__NR_OPTIMIZABLE_FOLIO_SIZES > 0 ? __NR_OPTIMIZABLE_FOLIO_SIZES : 0)
+#else
+#define MAX_FOLIO_VMEMMAP_ALIGN 0
+#define NR_OPTIMIZABLE_FOLIO_SIZES 0
+#endif
enum migratetype {
MIGRATE_UNMOVABLE,
@@ -2015,7 +2014,7 @@ struct mem_section {
*/
struct page_ext *page_ext;
#endif
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION
+#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED
/*
* The order of compound pages in this section. Typically, the section
* holds compound pages of this order; a larger compound page will span
@@ -2208,7 +2207,7 @@ static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long
}
#endif
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION
+#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED
static inline void section_set_order(struct mem_section *section, unsigned int order)
{
VM_BUG_ON(section->order && order && section->order != order);
@@ -2267,8 +2266,7 @@ static inline void section_set_compound_range(unsigned long pfn,
static inline bool section_vmemmap_optimizable(const struct mem_section *section)
{
- return is_power_of_2(sizeof(struct page)) &&
- section_order(section) >= OPTIMIZABLE_FOLIO_MIN_ORDER;
+ return section_order(section) >= OPTIMIZABLE_FOLIO_MIN_ORDER;
}
void sparse_init_early_section(int nid, struct page *map, unsigned long pnum,
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 12665b34586c..bea934d49750 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -198,32 +198,12 @@ enum pageflags {
#ifndef __GENERATING_BOUNDS_H
-/*
- * For tail pages, if the size of struct page is power-of-2 ->compound_info
- * encodes the mask that converts the address of the tail page address to
- * the head page address.
- *
- * Otherwise, ->compound_info has direct pointer to head pages.
- */
-static __always_inline bool compound_info_has_mask(void)
-{
- /*
- * The approach with mask would work in the wider set of conditions,
- * but it requires validating that struct pages are naturally aligned
- * for all orders up to the MAX_FOLIO_ORDER, which can be tricky.
- */
- if (!IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION))
- return false;
-
- return is_power_of_2(sizeof(struct page));
-}
-
static __always_inline unsigned long _compound_head(const struct page *page)
{
unsigned long info = READ_ONCE(page->compound_info);
unsigned long mask;
- if (!compound_info_has_mask()) {
+ if (!IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED)) {
/* Bit 0 encodes PageTail() */
if (info & 1)
return info - 1;
@@ -232,8 +212,8 @@ static __always_inline unsigned long _compound_head(const struct page *page)
}
/*
- * If compound_info_has_mask() is true the rest of the info encodes
- * the mask that converts the address of the tail page to the head page.
+ * If HVO is enabled the rest of the info encodes the mask that converts
+ * the address of the tail page to the head page.
*
* No need to clear bit 0 in the mask as 'page' always has it clear.
*
@@ -257,7 +237,7 @@ static __always_inline void set_compound_head(struct page *tail,
unsigned int shift;
unsigned long mask;
- if (!compound_info_has_mask()) {
+ if (!IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED)) {
WRITE_ONCE(tail->compound_info, (unsigned long)head | 1);
return;
}
diff --git a/kernel/bounds.c b/kernel/bounds.c
index 02b619eb6106..ff2ec3834d32 100644
--- a/kernel/bounds.c
+++ b/kernel/bounds.c
@@ -8,6 +8,7 @@
#define __GENERATING_BOUNDS_H
#define COMPILE_OFFSETS
/* Include headers that define the enum constants of interest */
+#include <linux/mm_types.h>
#include <linux/page-flags.h>
#include <linux/mmzone.h>
#include <linux/kbuild.h>
@@ -30,6 +31,7 @@ int main(void)
DEFINE(LRU_GEN_WIDTH, 0);
DEFINE(__LRU_REFS_WIDTH, 0);
#endif
+ DEFINE(STRUCT_PAGE_SIZE_IS_POWER_OF_2, is_power_of_2(sizeof(struct page)));
/* End of constants */
return 0;
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index d595ef759bc2..0347341be156 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -21,6 +21,7 @@
#include "hugetlb_vmemmap.h"
#include "internal.h"
+#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED
/**
* struct vmemmap_remap_walk - walk vmemmap page table
*
@@ -693,3 +694,4 @@ static int __init hugetlb_vmemmap_init(void)
return 0;
}
late_initcall(hugetlb_vmemmap_init);
+#endif
diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h
index 0022f9c5a101..bd576ef41ee7 100644
--- a/mm/hugetlb_vmemmap.h
+++ b/mm/hugetlb_vmemmap.h
@@ -12,7 +12,7 @@
#include <linux/io.h>
#include <linux/memblock.h>
-#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
+#if defined(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) && defined(SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED)
int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio);
long hugetlb_vmemmap_restore_folios(const struct hstate *h,
struct list_head *folio_list,
@@ -34,8 +34,6 @@ static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate
{
int size = hugetlb_vmemmap_size(h) - OPTIMIZED_FOLIO_VMEMMAP_SIZE;
- if (!is_power_of_2(sizeof(struct page)))
- return 0;
return size > 0 ? size : 0;
}
#else
diff --git a/mm/internal.h b/mm/internal.h
index 02064f21bfe1..121c9076f09a 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1026,9 +1026,6 @@ static inline bool vmemmap_page_optimizable(const struct page *page)
unsigned long pfn = page_to_pfn(page);
unsigned int order = section_order(__pfn_to_section(pfn));
- if (!is_power_of_2(sizeof(struct page)))
- return false;
-
return (pfn & ((1L << order) - 1)) >= OPTIMIZED_FOLIO_VMEMMAP_PAGE_STRUCTS;
}
diff --git a/mm/sparse.c b/mm/sparse.c
index 77bb0113bac5..7375f66a58d5 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -404,10 +404,8 @@ void __init sparse_init(void)
unsigned long pnum_end, pnum_begin, map_count = 1;
int nid_begin;
- if (compound_info_has_mask()) {
- VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0),
- MAX_FOLIO_VMEMMAP_ALIGN));
- }
+ VM_WARN_ON_ONCE(IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED) &&
+ !IS_ALIGNED((unsigned long)pfn_to_page(0), MAX_FOLIO_VMEMMAP_ALIGN));
pnum_begin = first_present_section_nr();
nid_begin = sparse_early_nid(__nr_to_section(pnum_begin));
diff --git a/mm/util.c b/mm/util.c
index f063fd4de1e8..783b2081ea74 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -1348,7 +1348,7 @@ void snapshot_page(struct page_snapshot *ps, const struct page *page)
foliop = (struct folio *)page;
} else {
/* See compound_head() */
- if (compound_info_has_mask()) {
+ if (IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED)) {
unsigned long p = (unsigned long)page;
foliop = (struct folio *)(p & info);
--
2.20.1
More information about the Linuxppc-dev
mailing list