anolis: mm: Readd DMA page copy support for page migration

ANBZ: #5398

Previous DMA page copy for page migration only supports synchronous
migration. However, synchronous migration can only migrate pages one by one
after commit 2ef7dbb269 ("migrate_pages: try migrate in batch
asynchronously firstly"), which leads to the efficiency of DMA page copy
not being utilized. To continue using DMA page copy, this patch readd DMA
page copy for page migration by extending it to asynchronous migration.

To ensure the asynchronous migration will not be blocked when using DMA
page copy, force it to use DMA migrate polling instead of interrupt in
asynchronous migration mode by DMA_MIGRATE_POLLING introduced in the
previous patch.

When DMA page copy succeeds, the migrate_page_move should be informed not
to copy these pages. For synchronous migration, there is already a
MIGRATE_SYNC_NO_COPY mode available. For asynchronous migration, a new
MIGRATE_ASYNC_NO_COPY mode is introduced to deal with it.

Another problem is that DMA page copy is not suitable for all scenarios.
Now limit DMA page copy to LRU pages with four migratepage functions:
migrate_page(), buffer_migrate_page(), buffer_migrate_page_norefs() and
fallback_migrate_page(). DMA page copy will only be used if all pages in
the batch can support DMA copy.

Finally, to avoid the performance overhead caused by DMA page copy when
there are fewer pages to migrate, introduce a new dma_migrate_min_pages
interface in /sys/kernel/mm/migrate/. Only when the number of pages to be
moved exceeds dma_migration_min_pages, will DMA page copy be applied. Now
this value defaults to 32.

After adding this patch, asynchronous migration will use the DMA page copy
with the dma_migration_min_pages limitation. Synchronous migration can also
use the DMA page copy in the first asynchronous trying.

This patch series is tested on a 2-socket Intel server like the author
Huang Ying did.

- Run pmbench memory accessing benchmark with default parameters for around
  100 seconds.
- Run migratepages to migrate pages of pmbench between node 0 and node 1
  back and forth around 60 seconds.

Test results show that the batch page migration performance increases by
67.55%. With DMA page copy, the batch page migration performance increases
by 166.98%.

Signed-off-by: Shawn Wang <shawnwang@linux.alibaba.com>
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Reviewed-by: Xu Yu <xuyu@linux.alibaba.com>
Link: https://gitee.com/anolis/cloud-kernel/pulls/1720

include/linux/migrate.h

@@ -40,7 +40,7 @@ enum dma_migrate_mode {
 	DMA_MIGRATE_INTERRUPT
 };
 
-extern bool migrate_use_dma(void);
+extern bool migrate_use_dma(int nr_move_pages);
 extern int dma_migrate_pages_copy(const struct list_head *pages,
 				  const struct list_head *new_pages,
 				  enum dma_migrate_mode mode);

include/linux/migrate_mode.h

@@ -11,12 +11,17 @@
  *	with the CPU. Instead, page copy happens outside the migratepage()
  *	callback and is likely using a DMA engine. See migrate_vma() and HMM
  *	(mm/hmm.c) for users of this mode.
+ * MIGRATE_ASYNC_NO_COPY will not block when migrating pages but will not copy
+ *	pages with the CPU. Instead, page copy happens outside the migratepages()
+ *	callback and is likely using a DMA engine. See migrate_pages_batch() for
+ *	users of this mode.
  */
 enum migrate_mode {
 	MIGRATE_ASYNC,
 	MIGRATE_SYNC_LIGHT,
 	MIGRATE_SYNC,
 	MIGRATE_SYNC_NO_COPY,
+	MIGRATE_ASYNC_NO_COPY,
 };
 
 #endif		/* MIGRATE_MODE_H_INCLUDED */

mm/dma_page_copy.c

@@ -10,6 +10,7 @@
 static bool dma_migrate_enabled __read_mostly;
 static unsigned int dma_migrate_segment = 32;
 static bool dma_migrate_polling __read_mostly = true;
+static int dma_migrate_min_pages __read_mostly = 32;
 
 static void dma_async_callback(void *arg)
 {
@@ -116,9 +117,10 @@ unmap_sg:
 	return err;
 }
 
-bool migrate_use_dma(void)
+bool migrate_use_dma(int nr_move_pages)
 {
-	return READ_ONCE(dma_migrate_enabled);
+	return READ_ONCE(dma_migrate_enabled) &&
+	       nr_move_pages >= READ_ONCE(dma_migrate_min_pages);
 }
 
 int dma_migrate_pages_copy(const struct list_head *pages,
@@ -276,10 +278,36 @@ static struct kobj_attribute dma_migrate_polling_attr =
 	__ATTR(dma_migrate_polling, 0644, migrate_dma_polling_show,
 	       migrate_dma_polling_store);
 
+static ssize_t dma_migrate_min_pages_show(struct kobject *kobj,
+					  struct kobj_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "%d\n", dma_migrate_min_pages);
+}
+
+static ssize_t dma_migrate_min_pages_store(struct kobject *kobj,
+					   struct kobj_attribute *attr,
+					   const char *buf, size_t count)
+{
+	unsigned int nr_min_pages;
+	int err;
+
+	err = kstrtouint(buf, 10, &nr_min_pages);
+	if (err)
+		return -EINVAL;
+
+	dma_migrate_min_pages = nr_min_pages;
+
+	return count;
+}
+static struct kobj_attribute dma_migrate_min_pages_attr =
+	__ATTR(dma_migrate_min_pages, 0644, dma_migrate_min_pages_show,
+	       dma_migrate_min_pages_store);
+
 static struct attribute *migrate_attrs[] = {
 	&dma_migrate_enabled_attr.attr,
 	&dma_migrate_segment_attr.attr,
 	&dma_migrate_polling_attr.attr,
+	&dma_migrate_min_pages_attr.attr,
 	NULL,
 };
 

mm/migrate.c

@@ -733,7 +733,7 @@ int migrate_page(struct address_space *mapping,
 	if (rc != MIGRATEPAGE_SUCCESS)
 		return rc;
 
-	if (mode != MIGRATE_SYNC_NO_COPY)
+	if (mode != MIGRATE_SYNC_NO_COPY && mode != MIGRATE_ASYNC_NO_COPY)
 		migrate_page_copy(newpage, page);
 	else
 		migrate_page_states(newpage, page);
@@ -749,7 +749,7 @@ static bool buffer_migrate_lock_buffers(struct buffer_head *head,
 	struct buffer_head *bh = head;
 
 	/* Simple case, sync compaction */
-	if (mode != MIGRATE_ASYNC) {
+	if (mode != MIGRATE_ASYNC && mode != MIGRATE_ASYNC_NO_COPY) {
 		do {
 			lock_buffer(bh);
 			bh = bh->b_this_page;
@@ -840,7 +840,7 @@ recheck_buffers:
 
 	} while (bh != head);
 
-	if (mode != MIGRATE_SYNC_NO_COPY)
+	if (mode != MIGRATE_SYNC_NO_COPY && mode != MIGRATE_ASYNC_NO_COPY)
 		migrate_page_copy(newpage, page);
 	else
 		migrate_page_states(newpage, page);
@@ -1616,6 +1616,52 @@ static int migrate_hugetlbs(struct list_head *from, new_page_t get_new_page,
 	return nr_failed;
 }
 
+static int migrate_pages_copy(struct list_head *pages,
+			      struct list_head *new_pages,
+			      int nr_move_pages,
+			      enum migrate_mode mode)
+{
+	int (*migratepage) (struct address_space *, struct page *,
+			    struct page *, enum migrate_mode);
+	struct address_space *mapping;
+	enum dma_migrate_mode dma_mode;
+	struct page *page;
+	bool is_lru;
+
+	if (!migrate_use_dma(nr_move_pages))
+		return -EINVAL;
+
+	/*
+	 * Check if all pages support being copied via dma. At this point
+	 * the validation is safe, cause all pages are under the page lock.
+	 */
+	list_for_each_entry(page, pages, lru) {
+		mapping = page_mapping(page);
+		is_lru = !__PageMovable(page);
+		if (likely(is_lru)) {
+			/* migrate_page is supported when mapping is NULL */
+			if (mapping) {
+				migratepage = mapping->a_ops->migratepage;
+				/*
+				 * Currently only three migratepage callbacks
+				 * and fallback_migrate_page are supported.
+				 */
+				if (migratepage &&
+				    migratepage != migrate_page &&
+				    migratepage != buffer_migrate_page &&
+				    migratepage != buffer_migrate_page_norefs)
+					return -EINVAL;
+			}
+		} else {
+			return -EINVAL;
+		}
+	}
+
+	dma_mode = (mode == MIGRATE_ASYNC) ? DMA_MIGRATE_POLLING :
+					     DMA_MIGRATE_DEFAULT;
+	return dma_migrate_pages_copy(pages, new_pages, dma_mode);
+}
+
 /*
  * migrate_pages_batch() first unmaps pages in the from list as many as
  * possible, then move the unmapped pages.
@@ -1634,6 +1680,7 @@ int migrate_pages_batch(struct list_head *from, new_page_t get_new_page,
 	int retry = 1;
 	int thp_retry = 1;
 	int nr_failed = 0;
+	int nr_move_pages = 0;
 	int nr_retry_pages = 0;
 	int pass = 0;
 	bool is_thp = false;
@@ -1733,6 +1780,7 @@ int migrate_pages_batch(struct list_head *from, new_page_t get_new_page,
 			case MIGRATEPAGE_UNMAP:
 				list_move_tail(&page->lru, &unmap_pages);
 				list_add_tail(&dst->lru, &new_pages);
+				nr_move_pages += nr_pages;
 				break;
 			default:
 				/*
@@ -1758,6 +1806,13 @@ move:
 	/* Flush TLBs for all unmapped pages */
 	try_to_unmap_flush();
 
+	if (!migrate_pages_copy(&unmap_pages, &new_pages, nr_move_pages, mode)) {
+		if (mode == MIGRATE_ASYNC)
+			mode = MIGRATE_ASYNC_NO_COPY;
+		else
+			mode = MIGRATE_SYNC_NO_COPY;
+	}
+
 	retry = 1;
 	for (pass = 0; pass < nr_pass && (retry || thp_retry); pass++) {
 		retry = 0;