[PATCH 6/6] hugetlb: support free hugepage pre zero out

[Liang Li]

This patch add support of pre zero out free hugepage, we can use
this feature to speed up page population and page fault handing.

Cc: Alexander Duyck <alexander.h.duyck@xxxxxxxxxxxxxxx>
Cc: Mel Gorman <mgorman@xxxxxxxxxxxxxxxxxxx>
Cc: Andrea Arcangeli <aarcange@xxxxxxxxxx>
Cc: Dan Williams <dan.j.williams@xxxxxxxxx>
Cc: Dave Hansen <dave.hansen@xxxxxxxxx>
Cc: David Hildenbrand <david@xxxxxxxxxx>
Cc: Michal Hocko <mhocko@xxxxxxxx>
Cc: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
Cc: Alex Williamson <alex.williamson@xxxxxxxxxx>
Cc: Michael S. Tsirkin <mst@xxxxxxxxxx>
Cc: Jason Wang <jasowang@xxxxxxxxxx>
Cc: Liang Li <liliang324@xxxxxxxxx>
Signed-off-by: Liang Li <liliangleo@xxxxxxxxxxxxxx>
---
 include/linux/page-flags.h |  12 ++
 mm/Kconfig                 |  10 ++
 mm/huge_memory.c           |   3 +-
 mm/hugetlb.c               | 243 +++++++++++++++++++++++++++++++++++++
 mm/memory.c                |   4 +
 5 files changed, 271 insertions(+), 1 deletion(-)

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index ec5d0290e0ee..f177c5e85632 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -173,6 +173,9 @@ enum pageflags {
 
 	/* Only valid for buddy pages. Used to track pages that are reported */
 	PG_reported = PG_uptodate,
+
+	/* Only valid for hugetlb pages. Used to mark zero pages */
+	PG_zero = PG_slab,
 };
 
 #ifndef __GENERATING_BOUNDS_H
@@ -451,6 +454,15 @@ PAGEFLAG(Idle, idle, PF_ANY)
  */
 __PAGEFLAG(Reported, reported, PF_NO_COMPOUND)
 
+/*
+ * PageZero() is used to track hugetlb free pages within the free list
+ * of hugetlbfs. We can use the non-atomic version of the test and set
+ * operations as both should be shielded with the hugetlb lock to prevent
+ * any possible races on the setting or clearing of the bit.
+ */
+__PAGEFLAG(Zero, zero, PF_ONLY_HEAD)
+
+
 /*
  * On an anonymous page mapped into a user virtual memory area,
  * page->mapping points to its anon_vma, not to a struct address_space;
diff --git a/mm/Kconfig b/mm/Kconfig
index 630cde982186..1d91e182825d 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -254,6 +254,16 @@ config PAGE_REPORTING
 	  those pages to another entity, such as a hypervisor, so that the
 	  memory can be freed within the host for other uses.
 
+#
+# support for pre zero out hugetlbfs free page
+config PREZERO_HPAGE
+	bool "Pre zero out hugetlbfs free page"
+	def_bool n
+	depends on PAGE_REPORTING
+	help
+	  Allows pre zero out hugetlbfs free pages in freelist based on free
+	  page reporting
+
 #
 # support for page migration
 #
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 9237976abe72..4ff99724d669 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2407,7 +2407,8 @@ static void __split_huge_page_tail(struct page *head, int tail,
 #ifdef CONFIG_64BIT
 			 (1L << PG_arch_2) |
 #endif
-			 (1L << PG_dirty)));
+			 (1L << PG_dirty) |
+			 (1L << PG_zero)));
 
 	/* ->mapping in first tail page is compound_mapcount */
 	VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 0fccd5f96954..2029668a0864 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1029,6 +1029,7 @@ static void enqueue_huge_page(struct hstate *h, struct page *page)
 	list_move(&page->lru, &h->hugepage_freelists[nid]);
 	h->free_huge_pages++;
 	h->free_huge_pages_node[nid]++;
+	__ClearPageZero(page);
 	if (hugepage_reported(page))
 		__ClearPageReported(page);
 	hugepage_reporting_notify_free(h->order);
@@ -1315,6 +1316,7 @@ static void update_and_free_page(struct hstate *h, struct page *page)
 	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page_rsvd(page), page);
 	set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
 	set_page_refcounted(page);
+	__ClearPageZero(page);
 	if (hstate_is_gigantic(h)) {
 		/*
 		 * Temporarily drop the hugetlb_lock, because
@@ -2963,6 +2965,237 @@ static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
 	return retval;
 }
 
+#ifdef CONFIG_PREZERO_HPAGE
+
+#define PRE_ZERO_STOP	0
+#define PRE_ZERO_RUN	1
+
+static int mini_page_order;
+static unsigned long batch_size = 16 * 1024 * 1024;
+static unsigned long delay_millisecs = 1000;
+static unsigned long prezero_enable __read_mostly;
+static DEFINE_MUTEX(kprezerod_mutex);
+static struct page_reporting_dev_info pre_zero_hpage_dev_info;
+
+static int zero_out_pages(struct page_reporting_dev_info *pr_dev_info,
+			   struct scatterlist *sgl, unsigned int nents)
+{
+	struct scatterlist *sg = sgl;
+
+	might_sleep();
+	do {
+		struct page *page = sg_page(sg);
+		unsigned int i, order = get_order(sg->length);
+
+		VM_BUG_ON(PageBuddy(page) || buddy_order(page));
+
+		if (PageZero(page))
+			continue;
+		for (i = 0; i < (1 << order); i++) {
+			cond_resched();
+			clear_highpage(page + i);
+		}
+		__SetPageZero(page);
+	} while ((sg = sg_next(sg)));
+
+	return 0;
+}
+
+static int start_kprezerod(void)
+{
+	int err = 0;
+
+	if (prezero_enable == PRE_ZERO_RUN) {
+		pre_zero_hpage_dev_info.report = zero_out_pages;
+		pre_zero_hpage_dev_info.mini_order = mini_page_order;
+		pre_zero_hpage_dev_info.batch_size = batch_size;
+		pre_zero_hpage_dev_info.delay_jiffies = msecs_to_jiffies(delay_millisecs);
+
+		err = hugepage_reporting_register(&pre_zero_hpage_dev_info);
+		pr_info("Pre zero hugepage enabled\n");
+	} else {
+		hugepage_reporting_unregister(&pre_zero_hpage_dev_info);
+		pr_info("Pre zero hugepage disabled\n");
+	}
+
+	return err;
+}
+
+static int restart_kprezerod(void)
+{
+	int err = 0;
+
+	mutex_lock(&kprezerod_mutex);
+	if (prezero_enable == PRE_ZERO_RUN) {
+		hugepage_reporting_unregister(&pre_zero_hpage_dev_info);
+
+		pre_zero_hpage_dev_info.report = zero_out_pages;
+		pre_zero_hpage_dev_info.mini_order = mini_page_order;
+		pre_zero_hpage_dev_info.batch_size = batch_size;
+		pre_zero_hpage_dev_info.delay_jiffies = msecs_to_jiffies(delay_millisecs);
+
+		err = hugepage_reporting_register(&pre_zero_hpage_dev_info);
+	}
+	mutex_unlock(&kprezerod_mutex);
+
+	return err;
+}
+
+static ssize_t enabled_show(struct kobject *kobj,
+			    struct kobj_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%lu\n", prezero_enable);
+}
+
+static ssize_t enabled_store(struct kobject *kobj,
+			     struct kobj_attribute *attr,
+			     const char *buf, size_t count)
+{
+	ssize_t ret = 0;
+	unsigned long flags;
+	int err;
+
+	err = kstrtoul(buf, 10, &flags);
+	if (err || flags > UINT_MAX)
+		return -EINVAL;
+	if (flags > PRE_ZERO_RUN)
+		return -EINVAL;
+
+	mutex_lock(&kprezerod_mutex);
+	if (prezero_enable != flags) {
+		prezero_enable = flags;
+		ret = start_kprezerod();
+	}
+	mutex_unlock(&kprezerod_mutex);
+
+	return count;
+}
+
+static struct kobj_attribute enabled_attr =
+	__ATTR(enabled, 0644, enabled_show, enabled_store);
+
+
+static ssize_t batch_size_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%lu\n", batch_size);
+}
+
+static ssize_t batch_size_store(struct kobject *kobj,
+				struct kobj_attribute *attr,
+				const char *buf, size_t count)
+{
+	unsigned long size;
+	int err;
+
+	err = kstrtoul(buf, 10, &size);
+	if (err || size >= UINT_MAX)
+		return -EINVAL;
+
+	batch_size = size;
+
+	restart_kprezerod();
+	return count;
+}
+
+static struct kobj_attribute batch_size_attr =
+	__ATTR(batch_size, 0644, batch_size_show, batch_size_store);
+
+static ssize_t delay_millisecs_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%lu\n", delay_millisecs);
+}
+
+static ssize_t delay_millisecs_store(struct kobject *kobj,
+				struct kobj_attribute *attr,
+				const char *buf, size_t count)
+{
+	unsigned long msecs;
+	int err;
+
+	err = kstrtoul(buf, 10, &msecs);
+	if (err || msecs >= UINT_MAX)
+		return -EINVAL;
+
+	delay_millisecs = msecs;
+
+	restart_kprezerod();
+
+	return count;
+}
+
+static struct kobj_attribute wake_delay_millisecs_attr =
+	__ATTR(delay_millisecs, 0644, delay_millisecs_show,
+		delay_millisecs_store);
+
+static ssize_t mini_order_show(struct kobject *kobj,
+				struct kobj_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%u\n", mini_page_order);
+}
+
+static ssize_t mini_order_store(struct kobject *kobj,
+				struct kobj_attribute *attr,
+				const char *buf, size_t count)
+{
+	unsigned int order;
+	int err;
+
+	err = kstrtouint(buf, 10, &order);
+	if (err || order >= MAX_ORDER)
+		return -EINVAL;
+
+	if (mini_page_order != order) {
+		mutex_lock(&kprezerod_mutex);
+		mini_page_order = order;
+		mutex_unlock(&kprezerod_mutex);
+	}
+
+	restart_kprezerod();
+	return count;
+}
+
+static struct kobj_attribute mini_order_attr =
+	__ATTR(mini_order, 0644, mini_order_show, mini_order_store);
+
+static struct attribute *pre_zero_attr[] = {
+	&enabled_attr.attr,
+	&mini_order_attr.attr,
+	&wake_delay_millisecs_attr.attr,
+	&batch_size_attr.attr,
+	NULL,
+};
+
+static struct attribute_group pre_zero_attr_group = {
+	.attrs = pre_zero_attr,
+};
+
+static int __init zeropage_init_sysfs(struct kobject *parent_kobj)
+{
+	int err;
+	struct kobject *pre_zero_kobj;
+
+	pre_zero_kobj = kobject_create_and_add("pre_zero", parent_kobj);
+	if (unlikely(!pre_zero_kobj)) {
+		pr_err("pre_zero: failed to create pre_zero kobject\n");
+		return -ENOMEM;
+	}
+
+	err = sysfs_create_group(pre_zero_kobj, &pre_zero_attr_group);
+	if (err) {
+		pr_err("pre_zero: failed to register pre_zero group\n");
+		goto delete_obj;
+	}
+
+	return 0;
+
+delete_obj:
+	kobject_put(pre_zero_kobj);
+	return err;
+}
+#endif
+
 static void __init hugetlb_sysfs_init(void)
 {
 	struct hstate *h;
@@ -2978,6 +3211,16 @@ static void __init hugetlb_sysfs_init(void)
 		if (err)
 			pr_err("HugeTLB: Unable to add hstate %s", h->name);
 	}
+
+	if (err)
+		return;
+#ifdef CONFIG_PREZERO_HPAGE
+	err = zeropage_init_sysfs(hugepages_kobj);
+	if (err)
+		return;
+
+	start_kprezerod();
+#endif
 }
 
 #ifdef CONFIG_NUMA
diff --git a/mm/memory.c b/mm/memory.c
index 7d608765932b..e98eed1a59a5 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -5100,6 +5100,10 @@ void clear_huge_page(struct page *page,
 	unsigned long addr = addr_hint &
 		~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
 
+	if (PageZero(page)) {
+		__ClearPageZero(page);
+		return;
+	}
 	if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
 		clear_gigantic_page(page, addr, pages_per_huge_page);
 		return;
-- 
2.18.2