diff --git a/Documentation/dev-tools/kfence.rst b/Documentation/dev-tools/kfence.rst index 63c3467e4358..59795ae048d8 100644 --- a/Documentation/dev-tools/kfence.rst +++ b/Documentation/dev-tools/kfence.rst @@ -112,6 +112,31 @@ be recovered to the origin PUD. An exception is the user input less than 131071 in boot cmdline. See mode 1 of the following examples. +Set a pool limit on various memory +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Like crashkernel, the user can limit the size of kfence pool by setting +``kfence.booting_max`` in boot command line. A reasonable config can be:: + + kfence.booting_max=0-128M:0,128M-256M:1M,256M-:2M + +So that: + On machines with memory of [0, 128M), kfence will not be enabled. + + On machines with memory of [128M, 256M), kfence will allocate at most 1MB + for kfence pool. (which means num_objects = 127 on page_size = 4KB) + + On machines with memory larger than 256M, kfence will allocate at most 2MB + for kfence pool. (which means num_objects = 255 on page_size = 4KB) + +Notes: + This config only sets the upper limit, so if the user sets num_objects = 127 + and ``kfence.booting_max=0-:2M``, kfence will still allocate 1MB for pool. + + This config only works for upstream mode. (pool_size < 1GB and + sample_interval > 0) Because if the user want to use debug mode, he must + focus on the specific machine and not need this general setting. + Examples ~~~~~~~~ diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index c7e0fecd6f8b..47ecd409236b 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -624,17 +624,27 @@ static phys_addr_t __init arm64_kfence_alloc_pool(void) if (!kfence_early_init) return 0; + if (update_kfence_booting_max()) { + if (!kfence_num_objects) + goto nokfence; + + kfence_pool_size = (kfence_num_objects + 1) * 2 * PAGE_SIZE; + } + kfence_pool = memblock_phys_alloc(kfence_pool_size, PAGE_SIZE); if (!kfence_pool) { pr_err("failed to allocate kfence pool\n"); - kfence_early_init = false; - return 0; + goto nokfence; } /* Temporarily mark as NOMAP. */ memblock_mark_nomap(kfence_pool, kfence_pool_size); return kfence_pool; + +nokfence: + kfence_early_init = false; + return 0; } static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) diff --git a/include/linux/kfence.h b/include/linux/kfence.h index 95be008745e3..b5b7a804ee62 100644 --- a/include/linux/kfence.h +++ b/include/linux/kfence.h @@ -106,6 +106,17 @@ void __init kfence_alloc_pool(void); */ void __init kfence_init(void); +/** + * update_kfence_booting_max() - analyse the max num_objects from cmdline + * + * Read the config from boot cmdline and limit kfence pool size. + * This function is called by kfence itself (e.g., kfence_alloc_pool()), or, + * by specific arch alloc (e.g., arm64_kfence_alloc_pool()). + * + * Return: 1 if kfence_num_objects is changed, otherwise 0. + */ +int __init update_kfence_booting_max(void); + /** * kfence_shutdown_cache() - handle shutdown_cache() for KFENCE objects * @s: cache being shut down diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 4673b871fa86..b4a7cdd47c6d 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -1484,6 +1484,103 @@ static bool update_kfence_node_map(void) return true; } +/* + * Get the last kfence.booting_max= from boot cmdline. + * Mainly copied from get_last_crashkernel(). + */ +static __init char *get_last_kfence_booting_max(char *name) +{ + char *p = boot_command_line, *ck_cmdline = NULL; + + /* find kfence.booting_max and use the last one if there are more */ + p = strstr(p, name); + while (p) { + char *end_p = strchr(p, ' '); + + if (!end_p) + end_p = p + strlen(p); + ck_cmdline = p; + p = strstr(p+1, name); + } + + if (!ck_cmdline) + return NULL; + + ck_cmdline += strlen(name); + return ck_cmdline; +} + +/* + * This function parses command lines in the format + * + * kfence.booting_max=ramsize-range:size[,...] + * + * The function returns 0 on success and -EINVAL on failure. + * Mainly copied from parse_crashkernel_mem(). + */ +static int __init parse_kfence_booting_max(char *cmdline, + unsigned long long system_ram, + unsigned long long *reserve_max) +{ + char *cur = cmdline, *tmp; + + /* for each entry of the comma-separated list */ + do { + unsigned long long start, end = ULLONG_MAX, size; + + /* get the start of the range */ + start = memparse(cur, &tmp); + if (cur == tmp) { + pr_warn("kfence.booting_max: Memory value expected\n"); + return -EINVAL; + } + cur = tmp; + if (*cur != '-') { + pr_warn("kfence.booting_max: '-' expected\n"); + return -EINVAL; + } + cur++; + + /* if no ':' is here, than we read the end */ + if (*cur != ':') { + end = memparse(cur, &tmp); + if (cur == tmp) { + pr_warn("kfence.booting_max: Memory value expected\n"); + return -EINVAL; + } + cur = tmp; + if (end <= start) { + pr_warn("kfence.booting_max: end <= start\n"); + return -EINVAL; + } + } + + if (*cur != ':') { + pr_warn("kfence.booting_max: ':' expected\n"); + return -EINVAL; + } + cur++; + + size = memparse(cur, &tmp); + if (cur == tmp) { + pr_warn("kfence.booting_max: Memory value expected\n"); + return -EINVAL; + } + cur = tmp; + + /* match ? */ + if (system_ram >= start && system_ram < end) { + *reserve_max = size; + break; + } + } while (*cur++ == ','); + + if (!*reserve_max) + pr_info("kfence.booting_max size resulted in zero bytes, disabled\n"); + + return 0; +} + /* === DebugFS Interface ==================================================== */ static inline void print_pool_size(struct seq_file *seq, unsigned long byte) @@ -1699,6 +1796,51 @@ static DECLARE_DELAYED_WORK(kfence_timer, toggle_allocation_gate); /* === Public interface ===================================================== */ +int __init update_kfence_booting_max(void) +{ + static bool done __initdata; + + unsigned long long parse_mem = PUD_SIZE; + unsigned long nr_pages, nr_obj_max; + char *cmdline; + int ret; + + /* + * We may reach here twice because some arch like aarch64 + * will call this function first. + */ + if (done) + return 0; + done = true; + + /* Boot cmdline is not set. Just leave. */ + cmdline = get_last_kfence_booting_max("kfence.booting_max="); + if (!cmdline) + return 0; + + ret = parse_kfence_booting_max(cmdline, memblock_phys_mem_size(), &parse_mem); + /* disable booting kfence on parsing fail. */ + if (ret) + goto nokfence; + + nr_pages = min_t(unsigned long, parse_mem, PUD_SIZE) / PAGE_SIZE; + /* We need at least 4 pages to enable KFENCE. */ + if (nr_pages < 4) + goto nokfence; + + nr_obj_max = nr_pages / 2 - 1; + if (kfence_num_objects > nr_obj_max) { + kfence_num_objects = nr_obj_max; + return 1; + } + + return 0; + +nokfence: + kfence_num_objects = 0; + return 1; +} + void __init kfence_alloc_pool(void) { int node; @@ -1707,13 +1849,22 @@ void __init kfence_alloc_pool(void) if (!READ_ONCE(kfence_sample_interval)) return; - /* - * Not allow both pool size < 1GiB and enabling node mode. - * Not allow both pool size < 1GiB and non-interval alloc. - */ - if (kfence_num_objects < KFENCE_MAX_OBJECTS_PER_AREA && - (kfence_pool_node_mode || kfence_sample_interval < 0)) - goto fail; + if (kfence_num_objects < KFENCE_MAX_OBJECTS_PER_AREA) { + /* + * Not allow both pool size < 1GiB and enabling node mode. + * Not allow both pool size < 1GiB and non-interval alloc. + */ + if (kfence_pool_node_mode || kfence_sample_interval < 0) + goto fail; + + /* + * Only limit upstream mode for online environment, + * as it makes no sense for limiting debug setup. + */ + update_kfence_booting_max(); + if (!kfence_num_objects) + goto fail; + } kfence_num_objects_stat = memblock_alloc(sizeof(struct kfence_alloc_node_cond) * nr_node_ids, PAGE_SIZE);