anolis: mm: Add ioremap_nohuge() func support

ANBZ: #2077

In some VGPU scenario, if the GPU card does not support SRIOV feature,
the BAR1 of the GPU needs to map kernel physical address in the way of
4K pages, so it can keep the same as the VGPU memory management, But
when CONFIG_HAVE_ARCH_HUGE_VMAP is enabled, this feature may cause
related GPU card functions to be abnormal, On the other hand,
forcibly disable CONFIG_HAVE_ARCH_HUGE_VMAP will increases the memory
overhead of page tables (up to 80G MMIO area on a single GPU).

Therefore, the kernel needs to support a new ioremap_nohuge func, which
uses 4K pages to map I/O areas. In the VGPU VFIO driver, ioremap and
ioremap_nohuge can be selected according to the GPU card type.

Link: https://gitee.com/anolis/cloud-kernel/pulls/697
Signed-off-by: Xin Hao <xhao@linux.alibaba.com>
Reviewed-by: Xu Yu <xuyu@linux.alibaba.com>

arch/arm64/include/asm/io.h

@@ -166,9 +166,12 @@ extern void __memset_io(volatile void __iomem *, int, size_t);
 extern void __iomem *__ioremap(phys_addr_t phys_addr, size_t size, pgprot_t prot);
 extern void iounmap(volatile void __iomem *addr);
 extern void __iomem *ioremap_cache(phys_addr_t phys_addr, size_t size);
+extern void __iomem *__ioremap_nohuge(phys_addr_t phys_addr, size_t size, pgprot_t prot);
 
 #define ioremap(addr, size)		__ioremap((addr), (size), __pgprot(PROT_DEVICE_nGnRE))
 #define ioremap_wc(addr, size)		__ioremap((addr), (size), __pgprot(PROT_NORMAL_NC))
+#define ioremap_nohuge(addr, size)		\
+	__ioremap_nohuge((addr), (size), __pgprot(PROT_DEVICE_nGnRE))
 
 /*
  * PCI configuration space mapping function.

arch/arm64/mm/ioremap.c

@@ -18,8 +18,9 @@
 #include <asm/fixmap.h>
 #include <asm/tlbflush.h>
 
-static void __iomem *__ioremap_caller(phys_addr_t phys_addr, size_t size,
-				      pgprot_t prot, void *caller)
+static void __iomem *__do_ioremap_caller(phys_addr_t phys_addr, size_t size,
+					 pgprot_t prot, void *caller,
+					 bool hugevmap_enabled)
 {
 	unsigned long last_addr;
 	unsigned long offset = phys_addr & ~PAGE_MASK;
@@ -53,7 +54,11 @@ static void __iomem *__ioremap_caller(phys_addr_t phys_addr, size_t size,
 	addr = (unsigned long)area->addr;
 	area->phys_addr = phys_addr;
 
-	err = ioremap_page_range(addr, addr + size, phys_addr, prot);
+	if (hugevmap_enabled)
+		err = ioremap_page_range(addr, addr + size, phys_addr, prot);
+	else
+		err = ioremap_nohuge_page_range(addr, addr + size, phys_addr, prot);
+
 	if (err) {
 		vunmap((void *)addr);
 		return NULL;
@@ -62,6 +67,18 @@ static void __iomem *__ioremap_caller(phys_addr_t phys_addr, size_t size,
 	return (void __iomem *)(offset + addr);
 }
 
+static void __iomem *__ioremap_caller(phys_addr_t phys_addr, size_t size,
+				      pgprot_t prot, void *caller)
+{
+	return __do_ioremap_caller(phys_addr, size, prot, caller, true);
+}
+
+static void __iomem *__ioremap_nohuge_caller(phys_addr_t phys_addr, size_t size,
+					     pgprot_t prot, void *caller)
+{
+	return __do_ioremap_caller(phys_addr, size, prot, caller, false);
+}
+
 void __iomem *__ioremap(phys_addr_t phys_addr, size_t size, pgprot_t prot)
 {
 	return __ioremap_caller(phys_addr, size, prot,
@@ -69,6 +86,14 @@ void __iomem *__ioremap(phys_addr_t phys_addr, size_t size, pgprot_t prot)
 }
 EXPORT_SYMBOL(__ioremap);
 
+void __iomem *__ioremap_nohuge(phys_addr_t phys_addr, size_t size,
+			       pgprot_t prot)
+{
+	return __ioremap_nohuge_caller(phys_addr, size, prot,
+				       __builtin_return_address(0));
+}
+EXPORT_SYMBOL(__ioremap_nohuge);
+
 void iounmap(volatile void __iomem *io_addr)
 {
 	unsigned long addr = (unsigned long)io_addr & PAGE_MASK;

arch/x86/include/asm/io.h

@@ -206,6 +206,9 @@ extern void __iomem *ioremap_encrypted(resource_size_t phys_addr, unsigned long
 void __iomem *ioremap(resource_size_t offset, unsigned long size);
 #define ioremap ioremap
 
+void __iomem *ioremap_nohuge(resource_size_t offset, unsigned long size);
+#define ioremap_nohuge ioremap_nohuge
+
 extern void iounmap(volatile void __iomem *addr);
 #define iounmap iounmap
 

arch/x86/mm/ioremap.c

@@ -175,8 +175,9 @@ static void __ioremap_check_mem(resource_size_t addr, unsigned long size,
  * caller shouldn't need to know that small detail.
  */
 static void __iomem *
-__ioremap_caller(resource_size_t phys_addr, unsigned long size,
-		 enum page_cache_mode pcm, void *caller, bool encrypted)
+__do_ioremap_caller(resource_size_t phys_addr, unsigned long size,
+		    enum page_cache_mode pcm, void *caller,
+		    bool encrypted, bool hugevmap_enabled)
 {
 	unsigned long offset, vaddr;
 	resource_size_t last_addr;
@@ -281,8 +282,13 @@ __ioremap_caller(resource_size_t phys_addr, unsigned long size,
 	if (memtype_kernel_map_sync(phys_addr, size, pcm))
 		goto err_free_area;
 
-	if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot))
-		goto err_free_area;
+	if (hugevmap_enabled) {
+		if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot))
+			goto err_free_area;
+	} else {
+		if (ioremap_nohuge_page_range(vaddr, vaddr + size, phys_addr, prot))
+			goto err_free_area;
+	}
 
 	ret_addr = (void __iomem *) (vaddr + offset);
 	mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
@@ -302,6 +308,22 @@ err_free_memtype:
 	return NULL;
 }
 
+static void __iomem *
+__ioremap_caller(resource_size_t phys_addr, unsigned long size,
+		 enum page_cache_mode pcm, void *caller, bool encrypted)
+{
+	return __do_ioremap_caller(phys_addr, size, pcm, caller,
+				   encrypted, true);
+}
+
+static void __iomem *
+__ioremap_nohuge_caller(resource_size_t phys_addr, unsigned long size,
+			enum page_cache_mode pcm, void *caller, bool encrypted)
+{
+	return __do_ioremap_caller(phys_addr, size, pcm, caller,
+				   encrypted, false);
+}
+
 /**
  * ioremap     -   map bus memory into CPU space
  * @phys_addr:    bus address of the memory
@@ -340,6 +362,23 @@ void __iomem *ioremap(resource_size_t phys_addr, unsigned long size)
 }
 EXPORT_SYMBOL(ioremap);
 
+void __iomem *ioremap_nohuge(resource_size_t phys_addr, unsigned long size)
+{
+	/*
+	 * Ideally, this should be:
+	 *	pat_enabled() ? _PAGE_CACHE_MODE_UC : _PAGE_CACHE_MODE_UC_MINUS;
+	 *
+	 * Till we fix all X drivers to use ioremap_wc(), we will use
+	 * UC MINUS. Drivers that are certain they need or can already
+	 * be converted over to strong UC can use ioremap_uc().
+	 */
+	enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC_MINUS;
+
+	return __ioremap_nohuge_caller(phys_addr, size, pcm,
+				       __builtin_return_address(0), false);
+}
+EXPORT_SYMBOL(ioremap_nohuge);
+
 /**
  * ioremap_uc     -   map bus memory into CPU space as strongly uncachable
  * @phys_addr:    bus address of the memory

include/linux/io.h

@@ -23,12 +23,20 @@ void __iowrite64_copy(void __iomem *to, const void *from, size_t count);
 #ifdef CONFIG_MMU
 int ioremap_page_range(unsigned long addr, unsigned long end,
 		       phys_addr_t phys_addr, pgprot_t prot);
+int ioremap_nohuge_page_range(unsigned long addr, unsigned long end,
+			      phys_addr_t phys_addr, pgprot_t prot);
 #else
 static inline int ioremap_page_range(unsigned long addr, unsigned long end,
 				     phys_addr_t phys_addr, pgprot_t prot)
 {
 	return 0;
 }
+
+static inline int ioremap_nohuge_page_range(unsigned long addr, unsigned long end,
+					    phys_addr_t phys_addr, pgprot_t prot)
+{
+	return 0;
+}
 #endif
 
 /*

mm/ioremap.c

@@ -34,6 +34,12 @@ int ioremap_page_range(unsigned long addr,
 	return vmap_range(addr, end, phys_addr, prot, iomap_max_page_shift);
 }
 
+int ioremap_nohuge_page_range(unsigned long addr, unsigned long end,
+			      phys_addr_t phys_addr, pgprot_t prot)
+{
+	return vmap_range(addr, end, phys_addr, prot, PAGE_SHIFT);
+}
+
 #ifdef CONFIG_GENERIC_IOREMAP
 void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot)
 {