136 lines
3.6 KiB
C
136 lines
3.6 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
|
|
*
|
|
* scx_arena_pool: kernel-side sub-allocator over BPF-arena pages.
|
|
*
|
|
* Each chunk added to @sch->arena_pool comes from one
|
|
* bpf_arena_alloc_pages_sleepable() call and is registered at the
|
|
* kernel-side mapping address. Callers translate to the BPF-arena form
|
|
* themselves if needed.
|
|
*
|
|
* Allocations grow the pool on demand. Underlying arena pages are released
|
|
* when the arena map itself is torn down.
|
|
*
|
|
* Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
|
|
* Copyright (c) 2026 Tejun Heo <tj@kernel.org>
|
|
*/
|
|
#include <linux/genalloc.h>
|
|
|
|
#include "internal.h"
|
|
#include "arena.h"
|
|
|
|
enum scx_arena_consts {
|
|
SCX_ARENA_MIN_ORDER = 3, /* 8-byte minimum sub-allocation */
|
|
SCX_ARENA_GROW_PAGES = 4, /* per growth */
|
|
};
|
|
|
|
s32 scx_arena_pool_init(struct scx_sched *sch)
|
|
{
|
|
if (!sch->arena_map)
|
|
return 0;
|
|
|
|
sch->arena_pool = gen_pool_create(SCX_ARENA_MIN_ORDER, NUMA_NO_NODE);
|
|
if (!sch->arena_pool)
|
|
return -ENOMEM;
|
|
return 0;
|
|
}
|
|
|
|
static void scx_arena_clear_chunk(struct gen_pool *pool, struct gen_pool_chunk *chunk,
|
|
void *data)
|
|
{
|
|
int order = pool->min_alloc_order;
|
|
size_t chunk_sz = chunk->end_addr - chunk->start_addr + 1;
|
|
unsigned long end_bit = chunk_sz >> order;
|
|
unsigned long b, e;
|
|
|
|
for_each_set_bitrange(b, e, chunk->bits, end_bit)
|
|
gen_pool_free(pool, chunk->start_addr + (b << order),
|
|
(e - b) << order);
|
|
}
|
|
|
|
/*
|
|
* Tear down the pool. Outstanding gen_pool allocations are freed via
|
|
* scx_arena_clear_chunk() so gen_pool_destroy() doesn't BUG. The underlying
|
|
* arena pages are released when the arena map itself is torn down.
|
|
*/
|
|
void scx_arena_pool_destroy(struct scx_sched *sch)
|
|
{
|
|
if (!sch->arena_pool)
|
|
return;
|
|
gen_pool_for_each_chunk(sch->arena_pool, scx_arena_clear_chunk, NULL);
|
|
gen_pool_destroy(sch->arena_pool);
|
|
sch->arena_pool = NULL;
|
|
}
|
|
|
|
/*
|
|
* Grow the pool by @page_cnt pages. bpf_arena_alloc_pages_sleepable() and
|
|
* gen_pool_add() (which calls vzalloc(GFP_KERNEL)) require a sleepable
|
|
* context.
|
|
*/
|
|
static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt)
|
|
{
|
|
u64 kern_vm_start;
|
|
u32 uaddr32;
|
|
void *p;
|
|
int ret;
|
|
|
|
if (!sch->arena_map || !sch->arena_pool)
|
|
return -EINVAL;
|
|
|
|
p = bpf_arena_alloc_pages_sleepable(sch->arena_map, NULL,
|
|
page_cnt, NUMA_NO_NODE, 0);
|
|
if (!p)
|
|
return -ENOMEM;
|
|
|
|
uaddr32 = (u32)(unsigned long)p;
|
|
/* arena.o, which defines these, is built only on MMU && 64BIT */
|
|
#if defined(CONFIG_MMU) && defined(CONFIG_64BIT)
|
|
kern_vm_start = bpf_arena_map_kern_vm_start(sch->arena_map);
|
|
#else
|
|
kern_vm_start = 0;
|
|
#endif
|
|
|
|
ret = gen_pool_add(sch->arena_pool, kern_vm_start + uaddr32,
|
|
page_cnt * PAGE_SIZE, NUMA_NO_NODE);
|
|
if (ret) {
|
|
bpf_arena_free_pages_non_sleepable(sch->arena_map, p, page_cnt);
|
|
return ret;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Allocate @size bytes from the arena pool. Returns kernel VA on success, NULL
|
|
* on failure. May grow the pool via scx_arena_grow() which sleeps. Caller must
|
|
* be in a GFP_KERNEL context.
|
|
*/
|
|
void *scx_arena_alloc(struct scx_sched *sch, size_t size)
|
|
{
|
|
unsigned long kern_va;
|
|
u32 page_cnt;
|
|
|
|
might_sleep();
|
|
|
|
if (!sch->arena_pool)
|
|
return NULL;
|
|
|
|
while (true) {
|
|
kern_va = gen_pool_alloc(sch->arena_pool, size);
|
|
if (kern_va)
|
|
break;
|
|
page_cnt = max_t(u32, SCX_ARENA_GROW_PAGES,
|
|
(size + PAGE_SIZE - 1) >> PAGE_SHIFT);
|
|
if (scx_arena_grow(sch, page_cnt))
|
|
return NULL;
|
|
}
|
|
|
|
return (void *)kern_va;
|
|
}
|
|
|
|
void scx_arena_free(struct scx_sched *sch, void *kern_va, size_t size)
|
|
{
|
|
if (sch->arena_pool && kern_va)
|
|
gen_pool_free(sch->arena_pool, (unsigned long)kern_va, size);
|
|
}
|