From a82dbc46e2343c394d3edcabcbc73ea6e9f403da Mon Sep 17 00:00:00 2001 From: Jichao Zou Date: Thu, 12 Aug 2021 18:20:21 +0800 Subject: [PATCH] cma:optimize cma allocation. Pre-allocate CMA memory that configured in device tree, this greatly improves the CMA memory allocation efficiency, cma_[alloc|free] is less than 1ms, old way is took a few ms to tens or hundreds ms. Signed-off-by: Jichao Zou --- include/linux/cma.h | 3 ++- kernel/dma/contiguous.c | 2 +- mm/cma.c | 35 ++++++++++++++++++++++++++++++++--- mm/cma.h | 1 + 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/include/linux/cma.h b/include/linux/cma.h index 53fd8c3cdbd0..68bc147a82a7 100644 --- a/include/linux/cma.h +++ b/include/linux/cma.h @@ -43,7 +43,8 @@ static inline int __init cma_declare_contiguous(phys_addr_t base, extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, unsigned int order_per_bit, const char *name, - struct cma **res_cma); + struct cma **res_cma, + unsigned long node); extern struct page *cma_alloc(struct cma *cma, unsigned long count, unsigned int align, bool no_warn); extern bool cma_release(struct cma *cma, const struct page *pages, unsigned long count); diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index 3d63d91cba5c..d77c2745244c 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -421,7 +421,7 @@ static int __init rmem_cma_setup(struct reserved_mem *rmem) return -EINVAL; } - err = cma_init_reserved_mem(rmem->base, rmem->size, 0, rmem->name, &cma); + err = cma_init_reserved_mem(rmem->base, rmem->size, 0, rmem->name, &cma, node); if (err) { pr_err("Reserved memory: unable to setup CMA region\n"); return err; diff --git a/mm/cma.c b/mm/cma.c index 995e15480937..c5682d03c5e9 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -32,6 +32,7 @@ #include #include #include +#include #include "cma.h" @@ -124,6 +125,17 @@ static void __init cma_activate_area(struct cma *cma) INIT_HLIST_HEAD(&cma->mem_head); spin_lock_init(&cma->mem_head_lock); #endif + if (cma->preallocated_cma) { + struct acr_info info = {0}; + + pfn = base_pfn; + if (!alloc_contig_range(pfn, pfn + cma->count, MIGRATE_CMA, GFP_KERNEL, &info)) { + pr_info("CMA area %s be pre-allocated successfully\n", cma->name); + } else { + cma->preallocated_cma = false; + pr_err("CMA area %s be pre-allocated failure\n", cma->name); + } + } return; @@ -159,13 +171,15 @@ core_initcall(cma_init_reserved_areas); * the area will be set to "cmaN", where N is a running counter of * used areas. * @res_cma: Pointer to store the created cma region. + * @node: CMA memory dtsi node. * * This function creates custom contiguous area from already reserved memory. */ int __init cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, unsigned int order_per_bit, const char *name, - struct cma **res_cma) + struct cma **res_cma, + unsigned long node) { struct cma *cma; phys_addr_t alignment; @@ -204,6 +218,9 @@ int __init cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, cma->base_pfn = PFN_DOWN(base); cma->count = size >> PAGE_SHIFT; cma->order_per_bit = order_per_bit; + if (node) + cma->preallocated_cma = of_get_flat_dt_prop(node, "linux,preallocated-cma", NULL); + *res_cma = cma; cma_area_count++; totalcma_pages += (size / PAGE_SIZE); @@ -369,7 +386,7 @@ int __init cma_declare_contiguous_nid(phys_addr_t base, base = addr; } - ret = cma_init_reserved_mem(base, size, order_per_bit, name, res_cma); + ret = cma_init_reserved_mem(base, size, order_per_bit, name, res_cma, 0); if (ret) goto free_mem; @@ -471,6 +488,16 @@ struct page *cma_alloc(struct cma *cma, unsigned long count, spin_unlock_irq(&cma->lock); pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit); + + /* + * cma bitmap should ensure that pfn is in the cma. + */ + if (cma->preallocated_cma) { + BUG_ON(pfn + count > cma->base_pfn + cma->count); + page = pfn_to_page(pfn); + ret = 0; + break; + } ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA, GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0)); @@ -551,7 +578,9 @@ bool cma_release(struct cma *cma, const struct page *pages, VM_BUG_ON(pfn + count > cma->base_pfn + cma->count); - free_contig_range(pfn, count); + if (!cma->preallocated_cma) + free_contig_range(pfn, count); + cma_clear_bitmap(cma, pfn, count); trace_cma_release(cma->name, pfn, pages, count); diff --git a/mm/cma.h b/mm/cma.h index 2c775877eae2..1778cb0e68c4 100644 --- a/mm/cma.h +++ b/mm/cma.h @@ -30,6 +30,7 @@ struct cma { /* kobject requires dynamic object */ struct cma_kobject *cma_kobj; #endif + bool preallocated_cma; }; extern struct cma cma_areas[MAX_CMA_AREAS]; -- 2.25.1