[PATCH] mm: fix sleeping copy_huge_page called from atomic context

[Andrea Righi]

copy_huge_page() can be called with mapping->private_lock held from
__buffer_migrate_page() -> migrate_page_copy(), so it is not safe to
do a cond_resched() in this context.

Introduce migrate_page_copy_nowait() and copy_huge_page_nowait()
variants that can be used from an atomic context.

The downside of this change is that we may experience temporary soft
lockups when copying large huge pages in very slow systems, but this
allows to prevent potential deadlocks.

Link: https://syzkaller.appspot.com/bug?id=683b472eb7539d56da69de85f4bfb4b9af67f7ec
Fixes: 79789db03fdd ("mm: Make copy_huge_page() always available")
Signed-off-by: Andrea Righi <andrea.righi@xxxxxxxxxxxxx>
---
 include/linux/migrate.h | 10 +++++++++-
 include/linux/mm.h      | 10 +++++++++-
 mm/migrate.c            |  8 ++++----
 mm/util.c               |  5 +++--
 4 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/include/linux/migrate.h b/include/linux/migrate.h
index c8077e936691..3dc6dab9a3f7 100644
--- a/include/linux/migrate.h
+++ b/include/linux/migrate.h
@@ -52,7 +52,15 @@ extern struct page *alloc_migration_target(struct page *page, unsigned long priv
 extern int isolate_movable_page(struct page *page, isolate_mode_t mode);
 
 extern void migrate_page_states(struct page *newpage, struct page *page);
-extern void migrate_page_copy(struct page *newpage, struct page *page);
+extern void __migrate_page_copy(struct page *newpage, struct page *page, bool atomic);
+static inline void migrate_page_copy(struct page *newpage, struct page *page)
+{
+	return __migrate_page_copy(newpage, page, false);
+}
+static inline void migrate_page_copy_nowait(struct page *newpage, struct page *page)
+{
+	return __migrate_page_copy(newpage, page, true);
+}
 extern int migrate_huge_page_move_mapping(struct address_space *mapping,
 				  struct page *newpage, struct page *page);
 extern int migrate_page_move_mapping(struct address_space *mapping,
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 73a52aba448f..1c96bb084366 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -907,7 +907,15 @@ void __put_page(struct page *page);
 void put_pages_list(struct list_head *pages);
 
 void split_page(struct page *page, unsigned int order);
-void copy_huge_page(struct page *dst, struct page *src);
+void __copy_huge_page(struct page *dst, struct page *src, bool atomic);
+static inline void copy_huge_page(struct page *dst, struct page *src)
+{
+	__copy_huge_page(dst, src, false);
+}
+static inline void copy_huge_page_nowait(struct page *dst, struct page *src)
+{
+	__copy_huge_page(dst, src, true);
+}
 
 /*
  * Compound pages have a destructor function.  Provide a
diff --git a/mm/migrate.c b/mm/migrate.c
index 1852d787e6ab..d8bc0586d157 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -613,16 +613,16 @@ void migrate_page_states(struct page *newpage, struct page *page)
 }
 EXPORT_SYMBOL(migrate_page_states);
 
-void migrate_page_copy(struct page *newpage, struct page *page)
+void __migrate_page_copy(struct page *newpage, struct page *page, bool atomic)
 {
 	if (PageHuge(page) || PageTransHuge(page))
-		copy_huge_page(newpage, page);
+		__copy_huge_page(newpage, page, atomic);
 	else
 		copy_highpage(newpage, page);
 
 	migrate_page_states(newpage, page);
 }
-EXPORT_SYMBOL(migrate_page_copy);
+EXPORT_SYMBOL(__migrate_page_copy);
 
 /************************************************************
  *                    Migration functions
@@ -755,7 +755,7 @@ static int __buffer_migrate_page(struct address_space *mapping,
 	} while (bh != head);
 
 	if (mode != MIGRATE_SYNC_NO_COPY)
-		migrate_page_copy(newpage, page);
+		migrate_page_copy_nowait(newpage, page);
 	else
 		migrate_page_states(newpage, page);
 
diff --git a/mm/util.c b/mm/util.c
index bacabe446906..f84e65643d1d 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -750,12 +750,13 @@ int __page_mapcount(struct page *page)
 }
 EXPORT_SYMBOL_GPL(__page_mapcount);
 
-void copy_huge_page(struct page *dst, struct page *src)
+void __copy_huge_page(struct page *dst, struct page *src, bool atomic)
 {
 	unsigned i, nr = compound_nr(src);
 
 	for (i = 0; i < nr; i++) {
-		cond_resched();
+		if (!atomic)
+			cond_resched();
 		copy_highpage(nth_page(dst, i), nth_page(src, i));
 	}
 }
-- 
2.32.0