[PATCH] erofs: Consider NUMA affinity when allocating memory for per-CPU pcpubuf

[Li RongQing]

per-CPU pcpubufs are dominantly accessed from their own local CPUs,
so allocate them node-local to improve performance.

Signed-off-by: Li RongQing <lirongqing at baidu.com>
---
 fs/erofs/internal.h |  1 +
 fs/erofs/pcpubuf.c  |  5 +++--
 fs/erofs/utils.c    | 14 ++++++++++++++
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
index 39c6711..94c8b62 100644
--- a/fs/erofs/internal.h
+++ b/fs/erofs/internal.h
@@ -446,6 +446,7 @@ int __init erofs_init_sysfs(void);
 void erofs_exit_sysfs(void);
 
 struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp);
+struct page *erofs_allocpage_node(struct page **pagepool, gfp_t gfp, int nid);
 static inline void erofs_pagepool_add(struct page **pagepool, struct page *page)
 {
 	set_page_private(page, (unsigned long)*pagepool);
diff --git a/fs/erofs/pcpubuf.c b/fs/erofs/pcpubuf.c
index c7a4b1d..b0b05a3 100644
--- a/fs/erofs/pcpubuf.c
+++ b/fs/erofs/pcpubuf.c
@@ -62,16 +62,17 @@ int erofs_pcpubuf_growsize(unsigned int nrpages)
 	for_each_possible_cpu(cpu) {
 		struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
 		struct page **pages, **oldpages;
+		int nid = cpu_to_node(cpu);
 		void *ptr, *old_ptr;
 
-		pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL);
+		pages = kmalloc_array_node(nrpages, sizeof(*pages), GFP_KERNEL, nid);
 		if (!pages) {
 			ret = -ENOMEM;
 			break;
 		}
 
 		for (i = 0; i < nrpages; ++i) {
-			pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL);
+			pages[i] = erofs_allocpage_node(&pagepool, GFP_KERNEL, nid);
 			if (!pages[i]) {
 				ret = -ENOMEM;
 				oldpages = pages;
diff --git a/fs/erofs/utils.c b/fs/erofs/utils.c
index 518bdd6..ba5ba68 100644
--- a/fs/erofs/utils.c
+++ b/fs/erofs/utils.c
@@ -18,6 +18,20 @@ struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp)
 	return page;
 }
 
+
+struct page *erofs_allocpage_node(struct page **pagepool, gfp_t gfp, int nid)
+{
+	struct page *page = *pagepool;
+
+	if (page) {
+		DBG_BUGON(page_ref_count(page) != 1);
+		*pagepool = (struct page *)page_private(page);
+	} else {
+		page = alloc_pages_node(nid, gfp, 0);
+	}
+	return page;
+}
+
 void erofs_release_pages(struct page **pagepool)
 {
 	while (*pagepool) {
-- 
2.9.4