lastdragon/linux-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resguardo inicial

Browse Source
This commit is contained in:
lastdragon
2026-09-06 12:53:21 -06:00
parent 662efe3c2a
commit ad2088d178
94853 changed files with 43200404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linux-7.2.3/tools/testing/selftests/bpf/.gitignore
+50
View File
@@ -0,0 +1,50 @@
# SPDX-License-Identifier: GPL-2.0-only
bpftool
bpf-helpers*
bpf-syscall*
test_verifier
test_maps
test_lru_map
test_tag
FEATURE-DUMP.libbpf
FEATURE-DUMP.selftests
fixdep
/test_progs
/test_progs-no_alu32
/test_progs-bpf_gcc
/test_progs-cpuv4
test_verifier_log
feature
urandom_read
test_sockmap
test_lirc_mode2_user
flow_dissector_load
test_tcpnotify_user
test_libbpf
test_cpp
*.d
*.subskel.h
*.skel.h
*.lskel.h
/no_alu32
/bpf_gcc
/cpuv4
/host-tools
/tools
/bench
/veristat
/sign-file
/uprobe_multi
*.ko
*.tmp
xskxceiver
xdp_redirect_multi
xdp_synproxy
xdp_hw_metadata
xdp_features
verification_cert.h
*.BTF
*.BTF_ids
*.BTF.base
usdt_1
usdt_2
linux-7.2.3/tools/testing/selftests/bpf/DENYLIST
+7
View File
@@ -0,0 +1,7 @@
# TEMPORARY
# Alphabetical order
get_stack_raw_tp # spams with kernel warnings until next bpf -> bpf-next merge
stacktrace_build_id
stacktrace_build_id_nmi
task_fd_query_rawtp
varlen
linux-7.2.3/tools/testing/selftests/bpf/DENYLIST.asan
+3
View File
@@ -0,0 +1,3 @@
*arena*
task_local_data
uprobe_multi_test
linux-7.2.3/tools/testing/selftests/bpf/DENYLIST.riscv64
+3
View File
@@ -0,0 +1,3 @@
# riscv64 deny list for BPF CI and local vmtest
exceptions # JIT does not support exceptions
tailcalls/tailcall_bpf2bpf* # JIT does not support mixing bpf2bpf and tailcalls
linux-7.2.3/tools/testing/selftests/bpf/DENYLIST.s390x
+5
View File
@@ -0,0 +1,5 @@
# TEMPORARY
# Alphabetical order
exe_ctx # execution context check (e.g., hardirq, softirq, etc)
get_stack_raw_tp # user_stack corrupted user stack (no backchain userspace)
stacktrace_build_id # compare_map_keys stackid_hmap vs. stackmap err -2 errno 2 (?)
linux-7.2.3/tools/testing/selftests/bpf/Makefile
+1070
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/tools/testing/selftests/bpf/Makefile.docs
+77
View File
@@ -0,0 +1,77 @@
# SPDX-License-Identifier: GPL-2.0-only
include ../../../scripts/Makefile.include
include ../../../scripts/utilities.mak
INSTALL ?= install
RM ?= rm -f
RMDIR ?= rmdir --ignore-fail-on-non-empty
prefix ?= /usr/local
mandir ?= $(prefix)/man
man2dir = $(mandir)/man2
man7dir = $(mandir)/man7
SYSCALL_RST = bpf-syscall.rst
MAN2_RST = $(SYSCALL_RST)
HELPERS_RST = bpf-helpers.rst
MAN7_RST = $(HELPERS_RST)
_DOC_MAN2 = $(patsubst %.rst,%.2,$(MAN2_RST))
DOC_MAN2 = $(addprefix $(OUTPUT),$(_DOC_MAN2))
_DOC_MAN7 = $(patsubst %.rst,%.7,$(MAN7_RST))
DOC_MAN7 = $(addprefix $(OUTPUT),$(_DOC_MAN7))
DOCTARGETS := helpers syscall
docs: $(DOCTARGETS)
syscall: man2
helpers: man7
man2: $(DOC_MAN2)
man7: $(DOC_MAN7)
RST2MAN_DEP := $(shell command -v rst2man 2>/dev/null)
# Configure make rules for the man page bpf-$1.$2.
# $1 - target for scripts/bpf_doc.py
# $2 - man page section to generate the troff file
define DOCS_RULES =
$(OUTPUT)bpf-$1.rst: ../../../../include/uapi/linux/bpf.h
$$(QUIET_GEN)../../../../scripts/bpf_doc.py $1 \
--filename $$< > $$@
$(OUTPUT)%.$2: $(OUTPUT)%.rst
ifndef RST2MAN_DEP
$$(error "rst2man not found, but required to generate man pages")
endif
$$(QUIET_GEN)rst2man --exit-status=1 $$< > $$@.tmp
$$(QUIET_GEN)mv $$@.tmp $$@
docs-clean-$1:
$$(call QUIET_CLEAN, eBPF_$1-manpage)
$(Q)$(RM) $$(DOC_MAN$2) $(OUTPUT)bpf-$1.rst
docs-install-$1: docs
$$(call QUIET_INSTALL, eBPF_$1-manpage)
$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$$(man$2dir)
$(Q)$(INSTALL) -m 644 $$(DOC_MAN$2) $(DESTDIR)$$(man$2dir)
docs-uninstall-$1:
$$(call QUIET_UNINST, eBPF_$1-manpage)
$(Q)$(RM) $$(addprefix $(DESTDIR)$$(man$2dir)/,$$(_DOC_MAN$2))
$(Q)$(RMDIR) $(DESTDIR)$$(man$2dir)
.PHONY: $1 docs-clean-$1 docs-install-$1 docs-uninstall-$1
endef
# Create the make targets to generate manual pages by name and section
$(eval $(call DOCS_RULES,helpers,7))
$(eval $(call DOCS_RULES,syscall,2))
docs-clean: $(foreach doctarget,$(DOCTARGETS), docs-clean-$(doctarget))
docs-install: $(foreach doctarget,$(DOCTARGETS), docs-install-$(doctarget))
docs-uninstall: $(foreach doctarget,$(DOCTARGETS), docs-uninstall-$(doctarget))
.PHONY: docs docs-clean docs-install docs-uninstall man2 man7
linux-7.2.3/tools/testing/selftests/bpf/README.rst
+339
View File
@@ -0,0 +1,339 @@
==================
BPF Selftest Notes
==================
General instructions on running selftests can be found in
`Documentation/bpf/bpf_devel_QA.rst`__.
__ /Documentation/bpf/bpf_devel_QA.rst#q-how-to-run-bpf-selftests
=============
BPF CI System
=============
BPF employs a continuous integration (CI) system to check patch submission in an
automated fashion. The system runs selftests for each patch in a series. Results
are propagated to patchwork, where failures are highlighted similar to
violations of other checks (such as additional warnings being emitted or a
``scripts/checkpatch.pl`` reported deficiency):
https://patchwork.kernel.org/project/netdevbpf/list/?delegate=121173
The CI system executes tests on multiple architectures. It uses a kernel
configuration derived from both the generic and architecture specific config
file fragments below ``tools/testing/selftests/bpf/`` (e.g., ``config`` and
``config.x86_64``).
Denylisting Tests
=================
It is possible for some architectures to not have support for all BPF features.
In such a case tests in CI may fail. An example of such a shortcoming is BPF
trampoline support on IBM's s390x architecture. For cases like this, an in-tree
deny list file, located at ``tools/testing/selftests/bpf/DENYLIST.<arch>``, can
be used to prevent the test from running on such an architecture.
In addition to that, the generic ``tools/testing/selftests/bpf/DENYLIST`` is
honored on every architecture running tests.
These files are organized in three columns. The first column lists the test in
question. This can be the name of a test suite or of an individual test. The
remaining two columns provide additional meta data that helps identify and
classify the entry: column two is a copy and paste of the error being reported
when running the test in the setting in question. The third column, if
available, summarizes the underlying problem. A value of ``trampoline``, for
example, indicates that lack of trampoline support is causing the test to fail.
This last entry helps identify tests that can be re-enabled once such support is
added.
=========================
Running Selftests in a VM
=========================
It's now possible to run the selftests using ``tools/testing/selftests/bpf/vmtest.sh``.
The script tries to ensure that the tests are run with the same environment as they
would be run post-submit in the CI used by the Maintainers, with the exception
that deny lists are not automatically honored.
This script uses the in-tree kernel configuration and downloads a VM userspace
image from the system used by the CI. It builds the kernel (without overwriting
your existing Kconfig), recompiles the bpf selftests, runs them (by default
``tools/testing/selftests/bpf/test_progs``) and saves the resulting output (by
default in ``~/.bpf_selftests``).
Script dependencies:
- clang (preferably built from sources, https://github.com/llvm/llvm-project);
- pahole (preferably built from sources, https://git.kernel.org/pub/scm/devel/pahole/pahole.git/);
- qemu;
- docutils (for ``rst2man``);
- libcap-devel.
For more information about using the script, run:
.. code-block:: console
$ tools/testing/selftests/bpf/vmtest.sh -h
In case of linker errors when running selftests, try using static linking:
.. code-block:: console
$ LDLIBS=-static EXTRA_LDFLAGS=-static PKG_CONFIG='pkg-config --static' vmtest.sh
.. note:: Some distros may not support static linking.
.. note:: The script uses pahole and clang based on host environment setting.
If you want to change pahole and llvm, you can change `PATH` environment
variable in the beginning of script.
Running vmtest on RV64
======================
To speed up testing and avoid various dependency issues, it is recommended to
run vmtest in a Docker container. Before running vmtest, we need to prepare
Docker container and local rootfs image. The overall steps are as follows:
1. Create Docker container as shown in link [0].
2. Use mkrootfs_debian.sh script [1] to build local rootfs image:
.. code-block:: console
$ sudo ./mkrootfs_debian.sh --arch riscv64 --distro noble
3. Start Docker container [0] and run vmtest in the container:
.. code-block:: console
$ PLATFORM=riscv64 CROSS_COMPILE=riscv64-linux-gnu- \
tools/testing/selftests/bpf/vmtest.sh \
-l <path of local rootfs image> -- \
./test_progs -d \
\"$(cat tools/testing/selftests/bpf/DENYLIST.riscv64 \
| cut -d'#' -f1 \
| sed -e 's/^[[:space:]]*//' \
-e 's/[[:space:]]*$//' \
| tr -s '\n' ',' \
)\"
Link: https://github.com/pulehui/riscv-bpf-vmtest.git [0]
Link: https://github.com/libbpf/ci/blob/main/rootfs/mkrootfs_debian.sh [1]
Additional information about selftest failures are
documented here.
profiler[23] test failures with clang/llvm <12.0.0
==================================================
With clang/llvm <12.0.0, the profiler[23] test may fail.
The symptom looks like
.. code-block:: c
// r9 is a pointer to map_value
// r7 is a scalar
17: bf 96 00 00 00 00 00 00 r6 = r9
18: 0f 76 00 00 00 00 00 00 r6 += r7
math between map_value pointer and register with unbounded min value is not allowed
// the instructions below will not be seen in the verifier log
19: a5 07 01 00 01 01 00 00 if r7 < 257 goto +1
20: bf 96 00 00 00 00 00 00 r6 = r9
// r6 is used here
The verifier will reject such code with above error.
At insn 18 the r7 is indeed unbounded. The later insn 19 checks the bounds and
the insn 20 undoes map_value addition. It is currently impossible for the
verifier to understand such speculative pointer arithmetic.
Hence `this patch`__ addresses it on the compiler side. It was committed on llvm 12.
__ https://github.com/llvm/llvm-project/commit/ddf1864ace484035e3cde5e83b3a31ac81e059c6
The corresponding C code
.. code-block:: c
for (int i = 0; i < MAX_CGROUPS_PATH_DEPTH; i++) {
filepart_length = bpf_probe_read_str(payload, ...);
if (filepart_length <= MAX_PATH) {
barrier_var(filepart_length); // workaround
payload += filepart_length;
}
}
bpf_iter test failures with clang/llvm 10.0.0
=============================================
With clang/llvm 10.0.0, the following two bpf_iter tests failed:
* ``bpf_iter/ipv6_route``
* ``bpf_iter/netlink``
The symptom for ``bpf_iter/ipv6_route`` looks like
.. code-block:: c
2: (79) r8 = *(u64 *)(r1 +8)
...
14: (bf) r2 = r8
15: (0f) r2 += r1
; BPF_SEQ_PRINTF(seq, "%pi6 %02x ", &rt->fib6_dst.addr, rt->fib6_dst.plen);
16: (7b) *(u64 *)(r8 +64) = r2
only read is supported
The symptom for ``bpf_iter/netlink`` looks like
.. code-block:: c
; struct netlink_sock *nlk = ctx->sk;
2: (79) r7 = *(u64 *)(r1 +8)
...
15: (bf) r2 = r7
16: (0f) r2 += r1
; BPF_SEQ_PRINTF(seq, "%pK %-3d ", s, s->sk_protocol);
17: (7b) *(u64 *)(r7 +0) = r2
only read is supported
This is due to a llvm BPF backend bug. `The fix`__
has been pushed to llvm 10.x release branch and will be
available in 10.0.1. The patch is available in llvm 11.0.0 trunk.
__ https://github.com/llvm/llvm-project/commit/3cb7e7bf959dcd3b8080986c62e10a75c7af43f0
bpf_verif_scale/loop6.bpf.o test failure with Clang 12
======================================================
With Clang 12, the following bpf_verif_scale test failed:
* ``bpf_verif_scale/loop6.bpf.o``
The verifier output looks like
.. code-block:: c
R1 type=ctx expected=fp
The sequence of 8193 jumps is too complex.
The reason is compiler generating the following code
.. code-block:: c
; for (i = 0; (i < VIRTIO_MAX_SGS) && (i < num); i++) {
14: 16 05 40 00 00 00 00 00 if w5 == 0 goto +64 <LBB0_6>
15: bc 51 00 00 00 00 00 00 w1 = w5
16: 04 01 00 00 ff ff ff ff w1 += -1
17: 67 05 00 00 20 00 00 00 r5 <<= 32
18: 77 05 00 00 20 00 00 00 r5 >>= 32
19: a6 01 01 00 05 00 00 00 if w1 < 5 goto +1 <LBB0_4>
20: b7 05 00 00 06 00 00 00 r5 = 6
00000000000000a8 <LBB0_4>:
21: b7 02 00 00 00 00 00 00 r2 = 0
22: b7 01 00 00 00 00 00 00 r1 = 0
; for (i = 0; (i < VIRTIO_MAX_SGS) && (i < num); i++) {
23: 7b 1a e0 ff 00 00 00 00 *(u64 *)(r10 - 32) = r1
24: 7b 5a c0 ff 00 00 00 00 *(u64 *)(r10 - 64) = r5
Note that insn #15 has w1 = w5 and w1 is refined later but
r5(w5) is eventually saved on stack at insn #24 for later use.
This cause later verifier failure. The bug has been `fixed`__ in
Clang 13.
__ https://github.com/llvm/llvm-project/commit/1959ead525b8830cc8a345f45e1c3ef9902d3229
BPF CO-RE-based tests and Clang version
=======================================
A set of selftests use BPF target-specific built-ins, which might require
bleeding-edge Clang versions (Clang 12 nightly at this time).
Few sub-tests of core_reloc test suit (part of test_progs test runner) require
the following built-ins, listed with corresponding Clang diffs introducing
them to Clang/LLVM. These sub-tests are going to be skipped if Clang is too
old to support them, they shouldn't cause build failures or runtime test
failures:
- __builtin_btf_type_id() [0_, 1_, 2_];
- __builtin_preserve_type_info(), __builtin_preserve_enum_value() [3_, 4_].
.. _0: https://github.com/llvm/llvm-project/commit/6b01b465388b204d543da3cf49efd6080db094a9
.. _1: https://github.com/llvm/llvm-project/commit/072cde03aaa13a2c57acf62d79876bf79aa1919f
.. _2: https://github.com/llvm/llvm-project/commit/00602ee7ef0bf6c68d690a2bd729c12b95c95c99
.. _3: https://github.com/llvm/llvm-project/commit/6d218b4adb093ff2e9764febbbc89f429412006c
.. _4: https://github.com/llvm/llvm-project/commit/6d6750696400e7ce988d66a1a00e1d0cb32815f8
Floating-point tests and Clang version
======================================
Certain selftests, e.g. core_reloc, require support for the floating-point
types, which was introduced in `Clang 13`__. The older Clang versions will
either crash when compiling these tests, or generate an incorrect BTF.
__ https://github.com/llvm/llvm-project/commit/a7137b238a07d9399d3ae96c0b461571bd5aa8b2
Kernel function call test and Clang version
===========================================
Some selftests (e.g. kfunc_call and bpf_tcp_ca) require a LLVM support
to generate extern function in BTF. It was introduced in `Clang 13`__.
Without it, the error from compiling bpf selftests looks like:
.. code-block:: console
libbpf: failed to find BTF for extern 'tcp_slow_start' [25] section: -2
__ https://github.com/llvm/llvm-project/commit/886f9ff53155075bd5f1e994f17b85d1e1b7470c
btf_tag test and Clang version
==============================
The btf_tag selftest requires LLVM support to recognize the btf_decl_tag and
btf_type_tag attributes. They are introduced in `Clang 14` [0_, 1_].
The subtests ``btf_type_tag_user_{mod1, mod2, vmlinux}`` also requires
pahole version ``1.23``.
Without them, the btf_tag selftest will be skipped and you will observe:
.. code-block:: console
#<test_num> btf_tag:SKIP
.. _0: https://github.com/llvm/llvm-project/commit/a162b67c98066218d0d00aa13b99afb95d9bb5e6
.. _1: https://github.com/llvm/llvm-project/commit/3466e00716e12e32fdb100e3fcfca5c2b3e8d784
Clang dependencies for static linking tests
===========================================
linked_vars, linked_maps, and linked_funcs tests depend on `Clang fix`__ to
generate valid BTF information for weak variables. Please make sure you use
Clang that contains the fix.
__ https://github.com/llvm/llvm-project/commit/968292cb93198442138128d850fd54dc7edc0035
Clang relocation changes
========================
Clang 13 patch `clang reloc patch`_ made some changes on relocations such
that existing relocation types are broken into more types and
each new type corresponds to only one way to resolve relocation.
See `kernel llvm reloc`_ for more explanation and some examples.
Using clang 13 to compile old libbpf which has static linker support,
there will be a compilation failure::
libbpf: ELF relo #0 in section #6 has unexpected type 2 in .../bpf_tcp_nogpl.bpf.o
Here, ``type 2`` refers to new relocation type ``R_BPF_64_ABS64``.
To fix this issue, user newer libbpf.
.. Links
.. _clang reloc patch: https://github.com/llvm/llvm-project/commit/6a2ea84600ba4bd3b2733bd8f08f5115eb32164b
.. _kernel llvm reloc: /Documentation/bpf/llvm_reloc.rst
Clang dependencies for the u32 spill test (xdpwall)
===================================================
The xdpwall selftest requires a change in `Clang 14`__.
Without it, the xdpwall selftest will fail and the error message
from running test_progs will look like:
.. code-block:: console
test_xdpwall:FAIL:Does LLVM have https://github.com/llvm/llvm-project/commit/ea72b0319d7b0f0c2fcf41d121afa5d031b319d5? unexpected error: -4007
__ https://github.com/llvm/llvm-project/commit/ea72b0319d7b0f0c2fcf41d121afa5d031b319d5
linux-7.2.3/tools/testing/selftests/bpf/autoconf_helper.h
+9
View File
@@ -0,0 +1,9 @@
// SPDX-License-Identifier: GPL-2.0-only
#ifdef HAVE_GENHDR
# include "autoconf.h"
#else
# if defined(__i386) || defined(__x86_64) || defined(__s390x__) || defined(__aarch64__)
# define CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 1
# endif
#endif
linux-7.2.3/tools/testing/selftests/bpf/bench.c
+808
View File
@@ -0,0 +1,808 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#define _GNU_SOURCE
#include <argp.h>
#include <linux/compiler.h>
#include <sys/time.h>
#include <sched.h>
#include <fcntl.h>
#include <pthread.h>
#include <sys/sysinfo.h>
#include <signal.h>
#include "bench.h"
#include "bpf_util.h"
#include "testing_helpers.h"
struct env env = {
.warmup_sec = 1,
.duration_sec = 5,
.affinity = false,
.quiet = false,
.consumer_cnt = 0,
.producer_cnt = 1,
};
static int libbpf_print_fn(enum libbpf_print_level level,
const char *format, va_list args)
{
if (level == LIBBPF_DEBUG && !env.verbose)
return 0;
return vfprintf(stderr, format, args);
}
void setup_libbpf(void)
{
libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
libbpf_set_print(libbpf_print_fn);
}
void false_hits_report_progress(int iter, struct bench_res *res, long delta_ns)
{
long total = res->false_hits + res->hits + res->drops;
printf("Iter %3d (%7.3lfus): ",
iter, (delta_ns - 1000000000) / 1000.0);
printf("%ld false hits of %ld total operations. Percentage = %2.2f %%\n",
res->false_hits, total, ((float)res->false_hits / total) * 100);
}
void false_hits_report_final(struct bench_res res[], int res_cnt)
{
long total_hits = 0, total_drops = 0, total_false_hits = 0, total_ops = 0;
int i;
for (i = 0; i < res_cnt; i++) {
total_hits += res[i].hits;
total_false_hits += res[i].false_hits;
total_drops += res[i].drops;
}
total_ops = total_hits + total_false_hits + total_drops;
printf("Summary: %ld false hits of %ld total operations. ",
total_false_hits, total_ops);
printf("Percentage = %2.2f %%\n",
((float)total_false_hits / total_ops) * 100);
}
void hits_drops_report_progress(int iter, struct bench_res *res, long delta_ns)
{
double hits_per_sec, drops_per_sec;
double hits_per_prod;
hits_per_sec = res->hits / 1000000.0 / (delta_ns / 1000000000.0);
hits_per_prod = hits_per_sec / env.producer_cnt;
drops_per_sec = res->drops / 1000000.0 / (delta_ns / 1000000000.0);
printf("Iter %3d (%7.3lfus): ",
iter, (delta_ns - 1000000000) / 1000.0);
printf("hits %8.3lfM/s (%7.3lfM/prod), drops %8.3lfM/s, total operations %8.3lfM/s\n",
hits_per_sec, hits_per_prod, drops_per_sec, hits_per_sec + drops_per_sec);
}
void
grace_period_latency_basic_stats(struct bench_res res[], int res_cnt, struct basic_stats *gp_stat)
{
int i;
memset(gp_stat, 0, sizeof(struct basic_stats));
for (i = 0; i < res_cnt; i++)
gp_stat->mean += res[i].gp_ns / 1000.0 / (double)res[i].gp_ct / (0.0 + res_cnt);
#define IT_MEAN_DIFF (res[i].gp_ns / 1000.0 / (double)res[i].gp_ct - gp_stat->mean)
if (res_cnt > 1) {
for (i = 0; i < res_cnt; i++)
gp_stat->stddev += (IT_MEAN_DIFF * IT_MEAN_DIFF) / (res_cnt - 1.0);
}
gp_stat->stddev = sqrt(gp_stat->stddev);
#undef IT_MEAN_DIFF
}
void
grace_period_ticks_basic_stats(struct bench_res res[], int res_cnt, struct basic_stats *gp_stat)
{
int i;
memset(gp_stat, 0, sizeof(struct basic_stats));
for (i = 0; i < res_cnt; i++)
gp_stat->mean += res[i].stime / (double)res[i].gp_ct / (0.0 + res_cnt);
#define IT_MEAN_DIFF (res[i].stime / (double)res[i].gp_ct - gp_stat->mean)
if (res_cnt > 1) {
for (i = 0; i < res_cnt; i++)
gp_stat->stddev += (IT_MEAN_DIFF * IT_MEAN_DIFF) / (res_cnt - 1.0);
}
gp_stat->stddev = sqrt(gp_stat->stddev);
#undef IT_MEAN_DIFF
}
void hits_drops_report_final(struct bench_res res[], int res_cnt)
{
int i;
double hits_mean = 0.0, drops_mean = 0.0, total_ops_mean = 0.0;
double hits_stddev = 0.0, drops_stddev = 0.0, total_ops_stddev = 0.0;
double total_ops;
for (i = 0; i < res_cnt; i++) {
hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
drops_mean += res[i].drops / 1000000.0 / (0.0 + res_cnt);
}
total_ops_mean = hits_mean + drops_mean;
if (res_cnt > 1) {
for (i = 0; i < res_cnt; i++) {
hits_stddev += (hits_mean - res[i].hits / 1000000.0) *
(hits_mean - res[i].hits / 1000000.0) /
(res_cnt - 1.0);
drops_stddev += (drops_mean - res[i].drops / 1000000.0) *
(drops_mean - res[i].drops / 1000000.0) /
(res_cnt - 1.0);
total_ops = res[i].hits + res[i].drops;
total_ops_stddev += (total_ops_mean - total_ops / 1000000.0) *
(total_ops_mean - total_ops / 1000000.0) /
(res_cnt - 1.0);
}
hits_stddev = sqrt(hits_stddev);
drops_stddev = sqrt(drops_stddev);
total_ops_stddev = sqrt(total_ops_stddev);
}
printf("Summary: hits %8.3lf \u00B1 %5.3lfM/s (%7.3lfM/prod), ",
hits_mean, hits_stddev, hits_mean / env.producer_cnt);
printf("drops %8.3lf \u00B1 %5.3lfM/s, ",
drops_mean, drops_stddev);
printf("total operations %8.3lf \u00B1 %5.3lfM/s\n",
total_ops_mean, total_ops_stddev);
}
void ops_report_progress(int iter, struct bench_res *res, long delta_ns)
{
double hits_per_sec, hits_per_prod;
hits_per_sec = res->hits / 1000000.0 / (delta_ns / 1000000000.0);
hits_per_prod = hits_per_sec / env.producer_cnt;
printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0);
printf("hits %8.3lfM/s (%7.3lfM/prod)\n", hits_per_sec, hits_per_prod);
}
void ops_report_final(struct bench_res res[], int res_cnt)
{
double hits_mean = 0.0, hits_stddev = 0.0;
int i;
for (i = 0; i < res_cnt; i++)
hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
if (res_cnt > 1) {
for (i = 0; i < res_cnt; i++)
hits_stddev += (hits_mean - res[i].hits / 1000000.0) *
(hits_mean - res[i].hits / 1000000.0) /
(res_cnt - 1.0);
hits_stddev = sqrt(hits_stddev);
}
printf("Summary: throughput %8.3lf \u00B1 %5.3lf M ops/s (%7.3lfM ops/prod), ",
hits_mean, hits_stddev, hits_mean / env.producer_cnt);
printf("latency %8.3lf ns/op\n", 1000.0 / hits_mean * env.producer_cnt);
}
void local_storage_report_progress(int iter, struct bench_res *res,
long delta_ns)
{
double important_hits_per_sec, hits_per_sec;
double delta_sec = delta_ns / 1000000000.0;
hits_per_sec = res->hits / 1000000.0 / delta_sec;
important_hits_per_sec = res->important_hits / 1000000.0 / delta_sec;
printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0);
printf("hits %8.3lfM/s ", hits_per_sec);
printf("important_hits %8.3lfM/s\n", important_hits_per_sec);
}
void local_storage_report_final(struct bench_res res[], int res_cnt)
{
double important_hits_mean = 0.0, important_hits_stddev = 0.0;
double hits_mean = 0.0, hits_stddev = 0.0;
int i;
for (i = 0; i < res_cnt; i++) {
hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
important_hits_mean += res[i].important_hits / 1000000.0 / (0.0 + res_cnt);
}
if (res_cnt > 1) {
for (i = 0; i < res_cnt; i++) {
hits_stddev += (hits_mean - res[i].hits / 1000000.0) *
(hits_mean - res[i].hits / 1000000.0) /
(res_cnt - 1.0);
important_hits_stddev +=
(important_hits_mean - res[i].important_hits / 1000000.0) *
(important_hits_mean - res[i].important_hits / 1000000.0) /
(res_cnt - 1.0);
}
hits_stddev = sqrt(hits_stddev);
important_hits_stddev = sqrt(important_hits_stddev);
}
printf("Summary: hits throughput %8.3lf \u00B1 %5.3lf M ops/s, ",
hits_mean, hits_stddev);
printf("hits latency %8.3lf ns/op, ", 1000.0 / hits_mean);
printf("important_hits throughput %8.3lf \u00B1 %5.3lf M ops/s\n",
important_hits_mean, important_hits_stddev);
}
const char *argp_program_version = "benchmark";
const char *argp_program_bug_address = "<bpf@vger.kernel.org>";
const char argp_program_doc[] =
"benchmark Generic benchmarking framework.\n"
"\n"
"This tool runs benchmarks.\n"
"\n"
"USAGE: benchmark <bench-name>\n"
"\n"
"EXAMPLES:\n"
" # run 'count-local' benchmark with 1 producer and 1 consumer\n"
" benchmark count-local\n"
" # run 'count-local' with 16 producer and 8 consumer thread, pinned to CPUs\n"
" benchmark -p16 -c8 -a count-local\n";
enum {
ARG_PROD_AFFINITY_SET = 1000,
ARG_CONS_AFFINITY_SET = 1001,
};
static const struct argp_option opts[] = {
{ "list", 'l', NULL, 0, "List available benchmarks"},
{ "duration", 'd', "SEC", 0, "Duration of benchmark, seconds"},
{ "warmup", 'w', "SEC", 0, "Warm-up period, seconds"},
{ "producers", 'p', "NUM", 0, "Number of producer threads"},
{ "consumers", 'c', "NUM", 0, "Number of consumer threads"},
{ "verbose", 'v', NULL, 0, "Verbose debug output"},
{ "affinity", 'a', NULL, 0, "Set consumer/producer thread affinity"},
{ "quiet", 'q', NULL, 0, "Be more quiet"},
{ "stacktrace", 's', NULL, 0, "Get stack trace"},
{ "prod-affinity", ARG_PROD_AFFINITY_SET, "CPUSET", 0,
"Set of CPUs for producer threads; implies --affinity"},
{ "cons-affinity", ARG_CONS_AFFINITY_SET, "CPUSET", 0,
"Set of CPUs for consumer threads; implies --affinity"},
{},
};
extern struct argp bench_ringbufs_argp;
extern struct argp bench_bloom_map_argp;
extern struct argp bench_bpf_loop_argp;
extern struct argp bench_local_storage_argp;
extern struct argp bench_local_storage_rcu_tasks_trace_argp;
extern struct argp bench_strncmp_argp;
extern struct argp bench_hashmap_lookup_argp;
extern struct argp bench_local_storage_create_argp;
extern struct argp bench_htab_mem_argp;
extern struct argp bench_trigger_batch_argp;
extern struct argp bench_crypto_argp;
extern struct argp bench_sockmap_argp;
extern struct argp bench_lpm_trie_map_argp;
extern struct argp bench_xdp_lb_argp;
static const struct argp_child bench_parsers[] = {
{ &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 },
{ &bench_bloom_map_argp, 0, "Bloom filter map benchmark", 0 },
{ &bench_bpf_loop_argp, 0, "bpf_loop helper benchmark", 0 },
{ &bench_local_storage_argp, 0, "local_storage benchmark", 0 },
{ &bench_strncmp_argp, 0, "bpf_strncmp helper benchmark", 0 },
{ &bench_local_storage_rcu_tasks_trace_argp, 0,
"local_storage RCU Tasks Trace slowdown benchmark", 0 },
{ &bench_hashmap_lookup_argp, 0, "Hashmap lookup benchmark", 0 },
{ &bench_local_storage_create_argp, 0, "local-storage-create benchmark", 0 },
{ &bench_htab_mem_argp, 0, "hash map memory benchmark", 0 },
{ &bench_trigger_batch_argp, 0, "BPF triggering benchmark", 0 },
{ &bench_crypto_argp, 0, "bpf crypto benchmark", 0 },
{ &bench_sockmap_argp, 0, "bpf sockmap benchmark", 0 },
{ &bench_lpm_trie_map_argp, 0, "LPM trie map benchmark", 0 },
{ &bench_xdp_lb_argp, 0, "XDP load-balancer benchmark", 0 },
{},
};
/* Make pos_args global, so that we can run argp_parse twice, if necessary */
static int pos_args;
static error_t parse_arg(int key, char *arg, struct argp_state *state)
{
switch (key) {
case 'v':
env.verbose = true;
break;
case 'l':
env.list = true;
break;
case 'd':
env.duration_sec = strtol(arg, NULL, 10);
if (env.duration_sec <= 0) {
fprintf(stderr, "Invalid duration: %s\n", arg);
argp_usage(state);
}
break;
case 'w':
env.warmup_sec = strtol(arg, NULL, 10);
if (env.warmup_sec <= 0) {
fprintf(stderr, "Invalid warm-up duration: %s\n", arg);
argp_usage(state);
}
break;
case 'p':
env.producer_cnt = strtol(arg, NULL, 10);
if (env.producer_cnt < 0) {
fprintf(stderr, "Invalid producer count: %s\n", arg);
argp_usage(state);
}
break;
case 'c':
env.consumer_cnt = strtol(arg, NULL, 10);
if (env.consumer_cnt < 0) {
fprintf(stderr, "Invalid consumer count: %s\n", arg);
argp_usage(state);
}
break;
case 'a':
env.affinity = true;
break;
case 'q':
env.quiet = true;
break;
case 's':
env.stacktrace = true;
break;
case ARG_PROD_AFFINITY_SET:
env.affinity = true;
if (parse_num_list(arg, &env.prod_cpus.cpus,
&env.prod_cpus.cpus_len)) {
fprintf(stderr, "Invalid format of CPU set for producers.");
argp_usage(state);
}
break;
case ARG_CONS_AFFINITY_SET:
env.affinity = true;
if (parse_num_list(arg, &env.cons_cpus.cpus,
&env.cons_cpus.cpus_len)) {
fprintf(stderr, "Invalid format of CPU set for consumers.");
argp_usage(state);
}
break;
case ARGP_KEY_ARG:
if (pos_args++) {
fprintf(stderr,
"Unrecognized positional argument: %s\n", arg);
argp_usage(state);
}
env.bench_name = strdup(arg);
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
static void parse_cmdline_args_init(int argc, char **argv)
{
static const struct argp argp = {
.options = opts,
.parser = parse_arg,
.doc = argp_program_doc,
.children = bench_parsers,
};
if (argp_parse(&argp, argc, argv, 0, NULL, NULL))
exit(1);
}
static void parse_cmdline_args_final(int argc, char **argv)
{
struct argp_child bench_parsers[2] = {};
const struct argp argp = {
.options = opts,
.parser = parse_arg,
.doc = argp_program_doc,
.children = bench_parsers,
};
/* Parse arguments the second time with the correct set of parsers */
if (bench->argp) {
bench_parsers[0].argp = bench->argp;
bench_parsers[0].header = bench->name;
pos_args = 0;
if (argp_parse(&argp, argc, argv, 0, NULL, NULL))
exit(1);
}
}
static void collect_measurements(long delta_ns);
static __u64 last_time_ns;
static void sigalarm_handler(int signo)
{
long new_time_ns = get_time_ns();
long delta_ns = new_time_ns - last_time_ns;
collect_measurements(delta_ns);
last_time_ns = new_time_ns;
}
/* set up periodic 1-second timer */
static void setup_timer()
{
static struct sigaction sigalarm_action = {
.sa_handler = sigalarm_handler,
};
struct itimerval timer_settings = {};
int err;
last_time_ns = get_time_ns();
err = sigaction(SIGALRM, &sigalarm_action, NULL);
if (err < 0) {
fprintf(stderr, "failed to install SIGALRM handler: %d\n", -errno);
exit(1);
}
timer_settings.it_interval.tv_sec = 1;
timer_settings.it_value.tv_sec = 1;
err = setitimer(ITIMER_REAL, &timer_settings, NULL);
if (err < 0) {
fprintf(stderr, "failed to arm interval timer: %d\n", -errno);
exit(1);
}
}
static void set_thread_affinity(pthread_t thread, int cpu)
{
cpu_set_t cpuset;
int err;
CPU_ZERO(&cpuset);
CPU_SET(cpu, &cpuset);
err = pthread_setaffinity_np(thread, sizeof(cpuset), &cpuset);
if (err) {
fprintf(stderr, "setting affinity to CPU #%d failed: %d\n",
cpu, -err);
exit(1);
}
}
static int next_cpu(struct cpu_set *cpu_set)
{
if (cpu_set->cpus) {
int i;
/* find next available CPU */
for (i = cpu_set->next_cpu; i < cpu_set->cpus_len; i++) {
if (cpu_set->cpus[i]) {
cpu_set->next_cpu = i + 1;
return i;
}
}
fprintf(stderr, "Not enough CPUs specified, need CPU #%d or higher.\n", i);
exit(1);
}
return cpu_set->next_cpu++ % env.nr_cpus;
}
static struct bench_state {
int res_cnt;
struct bench_res *results;
pthread_t *consumers;
pthread_t *producers;
} state;
const struct bench *bench = NULL;
extern const struct bench bench_count_global;
extern const struct bench bench_count_local;
extern const struct bench bench_rename_base;
extern const struct bench bench_rename_kprobe;
extern const struct bench bench_rename_kretprobe;
extern const struct bench bench_rename_rawtp;
extern const struct bench bench_rename_fentry;
extern const struct bench bench_rename_fexit;
/* pure counting benchmarks to establish theoretical limits */
extern const struct bench bench_trig_usermode_count;
extern const struct bench bench_trig_syscall_count;
extern const struct bench bench_trig_kernel_count;
/* batched, staying mostly in-kernel benchmarks */
extern const struct bench bench_trig_kprobe;
extern const struct bench bench_trig_kretprobe;
extern const struct bench bench_trig_kprobe_multi;
extern const struct bench bench_trig_kretprobe_multi;
extern const struct bench bench_trig_fentry;
extern const struct bench bench_trig_kprobe_multi_all;
extern const struct bench bench_trig_kretprobe_multi_all;
extern const struct bench bench_trig_fexit;
extern const struct bench bench_trig_fmodret;
extern const struct bench bench_trig_tp;
extern const struct bench bench_trig_rawtp;
/* uprobe/uretprobe benchmarks */
extern const struct bench bench_trig_uprobe_nop;
extern const struct bench bench_trig_uretprobe_nop;
extern const struct bench bench_trig_uprobe_push;
extern const struct bench bench_trig_uretprobe_push;
extern const struct bench bench_trig_uprobe_ret;
extern const struct bench bench_trig_uretprobe_ret;
extern const struct bench bench_trig_uprobe_multi_nop;
extern const struct bench bench_trig_uretprobe_multi_nop;
extern const struct bench bench_trig_uprobe_multi_push;
extern const struct bench bench_trig_uretprobe_multi_push;
extern const struct bench bench_trig_uprobe_multi_ret;
extern const struct bench bench_trig_uretprobe_multi_ret;
#ifdef __x86_64__
extern const struct bench bench_trig_uprobe_nop5;
extern const struct bench bench_trig_uretprobe_nop5;
extern const struct bench bench_trig_uprobe_multi_nop5;
extern const struct bench bench_trig_uretprobe_multi_nop5;
extern const struct bench bench_trig_usdt_nop;
extern const struct bench bench_trig_usdt_nop5;
#endif
extern const struct bench bench_rb_libbpf;
extern const struct bench bench_rb_custom;
extern const struct bench bench_pb_libbpf;
extern const struct bench bench_pb_custom;
extern const struct bench bench_bloom_lookup;
extern const struct bench bench_bloom_update;
extern const struct bench bench_bloom_false_positive;
extern const struct bench bench_hashmap_without_bloom;
extern const struct bench bench_hashmap_with_bloom;
extern const struct bench bench_bpf_loop;
extern const struct bench bench_strncmp_no_helper;
extern const struct bench bench_strncmp_helper;
extern const struct bench bench_bpf_hashmap_full_update;
extern const struct bench bench_bpf_rhashmap_full_update;
extern const struct bench bench_local_storage_cache_seq_get;
extern const struct bench bench_local_storage_cache_interleaved_get;
extern const struct bench bench_local_storage_cache_hashmap_control;
extern const struct bench bench_local_storage_tasks_trace;
extern const struct bench bench_bpf_hashmap_lookup;
extern const struct bench bench_bpf_rhashmap_lookup;
extern const struct bench bench_local_storage_create;
extern const struct bench bench_htab_mem;
extern const struct bench bench_rhtab_mem;
extern const struct bench bench_crypto_encrypt;
extern const struct bench bench_crypto_decrypt;
extern const struct bench bench_sockmap;
extern const struct bench bench_lpm_trie_noop;
extern const struct bench bench_lpm_trie_baseline;
extern const struct bench bench_lpm_trie_lookup;
extern const struct bench bench_lpm_trie_insert;
extern const struct bench bench_lpm_trie_update;
extern const struct bench bench_lpm_trie_delete;
extern const struct bench bench_lpm_trie_free;
extern const struct bench bench_bpf_nop;
extern const struct bench bench_xdp_lb;
static const struct bench *benchs[] = {
&bench_count_global,
&bench_count_local,
&bench_rename_base,
&bench_rename_kprobe,
&bench_rename_kretprobe,
&bench_rename_rawtp,
&bench_rename_fentry,
&bench_rename_fexit,
/* pure counting benchmarks for establishing theoretical limits */
&bench_trig_usermode_count,
&bench_trig_kernel_count,
&bench_trig_syscall_count,
/* batched, staying mostly in-kernel triggers */
&bench_trig_kprobe,
&bench_trig_kretprobe,
&bench_trig_kprobe_multi,
&bench_trig_kretprobe_multi,
&bench_trig_fentry,
&bench_trig_kprobe_multi_all,
&bench_trig_kretprobe_multi_all,
&bench_trig_fexit,
&bench_trig_fmodret,
&bench_trig_tp,
&bench_trig_rawtp,
/* uprobes */
&bench_trig_uprobe_nop,
&bench_trig_uretprobe_nop,
&bench_trig_uprobe_push,
&bench_trig_uretprobe_push,
&bench_trig_uprobe_ret,
&bench_trig_uretprobe_ret,
&bench_trig_uprobe_multi_nop,
&bench_trig_uretprobe_multi_nop,
&bench_trig_uprobe_multi_push,
&bench_trig_uretprobe_multi_push,
&bench_trig_uprobe_multi_ret,
&bench_trig_uretprobe_multi_ret,
#ifdef __x86_64__
&bench_trig_uprobe_nop5,
&bench_trig_uretprobe_nop5,
&bench_trig_uprobe_multi_nop5,
&bench_trig_uretprobe_multi_nop5,
&bench_trig_usdt_nop,
&bench_trig_usdt_nop5,
#endif
/* ringbuf/perfbuf benchmarks */
&bench_rb_libbpf,
&bench_rb_custom,
&bench_pb_libbpf,
&bench_pb_custom,
&bench_bloom_lookup,
&bench_bloom_update,
&bench_bloom_false_positive,
&bench_hashmap_without_bloom,
&bench_hashmap_with_bloom,
&bench_bpf_loop,
&bench_strncmp_no_helper,
&bench_strncmp_helper,
&bench_bpf_hashmap_full_update,
&bench_bpf_rhashmap_full_update,
&bench_local_storage_cache_seq_get,
&bench_local_storage_cache_interleaved_get,
&bench_local_storage_cache_hashmap_control,
&bench_local_storage_tasks_trace,
&bench_bpf_hashmap_lookup,
&bench_bpf_rhashmap_lookup,
&bench_local_storage_create,
&bench_htab_mem,
&bench_rhtab_mem,
&bench_crypto_encrypt,
&bench_crypto_decrypt,
&bench_sockmap,
&bench_lpm_trie_noop,
&bench_lpm_trie_baseline,
&bench_lpm_trie_lookup,
&bench_lpm_trie_insert,
&bench_lpm_trie_update,
&bench_lpm_trie_delete,
&bench_lpm_trie_free,
&bench_bpf_nop,
&bench_xdp_lb,
};
static void find_benchmark(void)
{
int i;
if (!env.bench_name) {
fprintf(stderr, "benchmark name is not specified\n");
exit(1);
}
for (i = 0; i < ARRAY_SIZE(benchs); i++) {
if (strcmp(benchs[i]->name, env.bench_name) == 0) {
bench = benchs[i];
break;
}
}
if (!bench) {
fprintf(stderr, "benchmark '%s' not found\n", env.bench_name);
exit(1);
}
}
static void setup_benchmark(void)
{
int i, err;
if (!env.quiet)
printf("Setting up benchmark '%s'...\n", bench->name);
state.producers = calloc(env.producer_cnt, sizeof(*state.producers));
state.consumers = calloc(env.consumer_cnt, sizeof(*state.consumers));
state.results = calloc(env.duration_sec + env.warmup_sec + 2,
sizeof(*state.results));
if (!state.producers || !state.consumers || !state.results)
exit(1);
if (bench->validate)
bench->validate();
if (bench->setup)
bench->setup();
for (i = 0; i < env.consumer_cnt; i++) {
if (!bench->consumer_thread) {
fprintf(stderr, "benchmark doesn't support consumers!\n");
exit(1);
}
err = pthread_create(&state.consumers[i], NULL,
bench->consumer_thread, (void *)(long)i);
if (err) {
fprintf(stderr, "failed to create consumer thread #%d: %d\n",
i, -err);
exit(1);
}
if (env.affinity)
set_thread_affinity(state.consumers[i],
next_cpu(&env.cons_cpus));
}
/* unless explicit producer CPU list is specified, continue after
* last consumer CPU
*/
if (!env.prod_cpus.cpus)
env.prod_cpus.next_cpu = env.cons_cpus.next_cpu;
for (i = 0; i < env.producer_cnt; i++) {
if (!bench->producer_thread) {
fprintf(stderr, "benchmark doesn't support producers!\n");
exit(1);
}
err = pthread_create(&state.producers[i], NULL,
bench->producer_thread, (void *)(long)i);
if (err) {
fprintf(stderr, "failed to create producer thread #%d: %d\n",
i, -err);
exit(1);
}
if (env.affinity)
set_thread_affinity(state.producers[i],
next_cpu(&env.prod_cpus));
}
if (!env.quiet)
printf("Benchmark '%s' started.\n", bench->name);
}
static pthread_mutex_t bench_done_mtx = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t bench_done = PTHREAD_COND_INITIALIZER;
void bench_force_done(void)
{
pthread_mutex_lock(&bench_done_mtx);
pthread_cond_signal(&bench_done);
pthread_mutex_unlock(&bench_done_mtx);
}
static void collect_measurements(long delta_ns) {
int iter = state.res_cnt++;
struct bench_res *res = &state.results[iter];
bench->measure(res);
if (bench->report_progress)
bench->report_progress(iter, res, delta_ns);
if (iter == env.duration_sec + env.warmup_sec)
bench_force_done();
}
int main(int argc, char **argv)
{
env.nr_cpus = get_nprocs();
parse_cmdline_args_init(argc, argv);
if (env.list) {
int i;
printf("Available benchmarks:\n");
for (i = 0; i < ARRAY_SIZE(benchs); i++) {
printf("- %s\n", benchs[i]->name);
}
return 0;
}
find_benchmark();
parse_cmdline_args_final(argc, argv);
setup_benchmark();
setup_timer();
pthread_mutex_lock(&bench_done_mtx);
pthread_cond_wait(&bench_done, &bench_done_mtx);
pthread_mutex_unlock(&bench_done_mtx);
if (bench->report_final)
/* skip first sample */
bench->report_final(state.results + env.warmup_sec,
state.res_cnt - env.warmup_sec);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/bench.h
+101
View File
@@ -0,0 +1,101 @@
/* SPDX-License-Identifier: GPL-2.0 */
#pragma once
#include <stdlib.h>
#include <stdbool.h>
#include <linux/err.h>
#include <errno.h>
#include <unistd.h>
#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include <math.h>
#include <time.h>
#include <sys/syscall.h>
#include <limits.h>
struct cpu_set {
bool *cpus;
int cpus_len;
int next_cpu;
};
struct env {
char *bench_name;
int duration_sec;
int warmup_sec;
bool verbose;
bool list;
bool affinity;
bool quiet;
bool stacktrace;
int consumer_cnt;
int producer_cnt;
int nr_cpus;
struct cpu_set prod_cpus;
struct cpu_set cons_cpus;
};
struct basic_stats {
double mean;
double stddev;
};
struct bench_res {
long hits;
long drops;
long false_hits;
long important_hits;
unsigned long gp_ns;
unsigned long gp_ct;
unsigned int stime;
unsigned long duration_ns;
};
struct bench {
const char *name;
const struct argp *argp;
void (*validate)(void);
void (*setup)(void);
void *(*producer_thread)(void *ctx);
void *(*consumer_thread)(void *ctx);
void (*measure)(struct bench_res* res);
void (*report_progress)(int iter, struct bench_res* res, long delta_ns);
void (*report_final)(struct bench_res res[], int res_cnt);
};
struct counter {
long value;
} __attribute__((aligned(128)));
extern struct env env;
extern const struct bench *bench;
void setup_libbpf(void);
void bench_force_done(void);
void hits_drops_report_progress(int iter, struct bench_res *res, long delta_ns);
void hits_drops_report_final(struct bench_res res[], int res_cnt);
void false_hits_report_progress(int iter, struct bench_res *res, long delta_ns);
void false_hits_report_final(struct bench_res res[], int res_cnt);
void ops_report_progress(int iter, struct bench_res *res, long delta_ns);
void ops_report_final(struct bench_res res[], int res_cnt);
void local_storage_report_progress(int iter, struct bench_res *res,
long delta_ns);
void local_storage_report_final(struct bench_res res[], int res_cnt);
void grace_period_latency_basic_stats(struct bench_res res[], int res_cnt,
struct basic_stats *gp_stat);
void grace_period_ticks_basic_stats(struct bench_res res[], int res_cnt,
struct basic_stats *gp_stat);
static inline void atomic_inc(long *value)
{
(void)__atomic_add_fetch(value, 1, __ATOMIC_RELAXED);
}
static inline void atomic_add(long *value, long n)
{
(void)__atomic_add_fetch(value, n, __ATOMIC_RELAXED);
}
static inline long atomic_swap(long *value, long n)
{
return __atomic_exchange_n(value, n, __ATOMIC_RELAXED);
}
linux-7.2.3/tools/testing/selftests/bpf/bench_bpf_timing.h
+50
View File
@@ -0,0 +1,50 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#ifndef __BENCH_BPF_TIMING_H__
#define __BENCH_BPF_TIMING_H__
#include <stdbool.h>
#include <linux/types.h>
#include "bench.h"
#ifndef BENCH_NR_SAMPLES
#define BENCH_NR_SAMPLES 4096
#endif
#ifndef BENCH_NR_CPUS
#define BENCH_NR_CPUS 256
#endif
typedef void (*bpf_bench_run_fn)(void *ctx);
struct bpf_bench_timing {
__u64 (*samples)[BENCH_NR_SAMPLES]; /* skel->bss->timing_samples */
__u32 *idx; /* skel->bss->timing_idx */
volatile __u32 *timing_enabled; /* &skel->bss->timing_enabled */
volatile __u32 *batch_iters_bss; /* &skel->bss->batch_iters */
__u32 batch_iters;
__u32 target_samples;
__u32 nr_cpus;
int warmup_ticks;
bool done;
bool machine_readable;
};
#define BENCH_TIMING_INIT(t, skel, iters) do { \
(t)->samples = (skel)->bss->timing_samples; \
(t)->idx = (skel)->bss->timing_idx; \
(t)->timing_enabled = &(skel)->bss->timing_enabled; \
(t)->batch_iters_bss = &(skel)->bss->batch_iters; \
(t)->batch_iters = (iters); \
(t)->target_samples = 200; \
(t)->nr_cpus = env.nr_cpus; \
(t)->warmup_ticks = 0; \
(t)->done = false; \
(t)->machine_readable = false; \
} while (0)
void bpf_bench_timing_measure(struct bpf_bench_timing *t, struct bench_res *res);
void bpf_bench_timing_report(struct bpf_bench_timing *t, const char *name, const char *desc);
void bpf_bench_calibrate(struct bpf_bench_timing *t, bpf_bench_run_fn run_fn, void *ctx);
#endif /* __BENCH_BPF_TIMING_H__ */
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_bloom_filter_map.c
+477
View File
@@ -0,0 +1,477 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2021 Facebook */
#include <argp.h>
#include <linux/log2.h>
#include <pthread.h>
#include "bench.h"
#include "bloom_filter_bench.skel.h"
#include "bpf_util.h"
static struct ctx {
bool use_array_map;
bool use_hashmap;
bool hashmap_use_bloom;
bool count_false_hits;
struct bloom_filter_bench *skel;
int bloom_fd;
int hashmap_fd;
int array_map_fd;
pthread_mutex_t map_done_mtx;
pthread_cond_t map_done_cv;
bool map_done;
bool map_prepare_err;
__u32 next_map_idx;
} ctx = {
.map_done_mtx = PTHREAD_MUTEX_INITIALIZER,
.map_done_cv = PTHREAD_COND_INITIALIZER,
};
struct stat {
__u32 stats[3];
};
static struct {
__u32 nr_entries;
__u8 nr_hash_funcs;
__u8 value_size;
} args = {
.nr_entries = 1000,
.nr_hash_funcs = 3,
.value_size = 8,
};
enum {
ARG_NR_ENTRIES = 3000,
ARG_NR_HASH_FUNCS = 3001,
ARG_VALUE_SIZE = 3002,
};
static const struct argp_option opts[] = {
{ "nr_entries", ARG_NR_ENTRIES, "NR_ENTRIES", 0,
"Set number of expected unique entries in the bloom filter"},
{ "nr_hash_funcs", ARG_NR_HASH_FUNCS, "NR_HASH_FUNCS", 0,
"Set number of hash functions in the bloom filter"},
{ "value_size", ARG_VALUE_SIZE, "VALUE_SIZE", 0,
"Set value size (in bytes) of bloom filter entries"},
{},
};
static error_t parse_arg(int key, char *arg, struct argp_state *state)
{
long ret;
switch (key) {
case ARG_NR_ENTRIES:
ret = strtol(arg, NULL, 10);
if (ret < 1 || ret > UINT_MAX) {
fprintf(stderr, "Invalid nr_entries count.");
argp_usage(state);
}
args.nr_entries = ret;
break;
case ARG_NR_HASH_FUNCS:
ret = strtol(arg, NULL, 10);
if (ret < 1 || ret > 15) {
fprintf(stderr,
"The bloom filter must use 1 to 15 hash functions.");
argp_usage(state);
}
args.nr_hash_funcs = ret;
break;
case ARG_VALUE_SIZE:
ret = strtol(arg, NULL, 10);
if (ret < 2 || ret > 256) {
fprintf(stderr,
"Invalid value size. Must be between 2 and 256 bytes");
argp_usage(state);
}
args.value_size = ret;
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
/* exported into benchmark runner */
const struct argp bench_bloom_map_argp = {
.options = opts,
.parser = parse_arg,
};
static void validate(void)
{
if (env.consumer_cnt != 0) {
fprintf(stderr,
"The bloom filter benchmarks do not support consumer\n");
exit(1);
}
}
static inline void trigger_bpf_program(void)
{
syscall(__NR_getpgid);
}
static void *producer(void *input)
{
while (true)
trigger_bpf_program();
return NULL;
}
static void *map_prepare_thread(void *arg)
{
__u32 val_size, i;
void *val = NULL;
int err;
val_size = args.value_size;
val = malloc(val_size);
if (!val) {
ctx.map_prepare_err = true;
goto done;
}
while (true) {
i = __atomic_add_fetch(&ctx.next_map_idx, 1, __ATOMIC_RELAXED);
if (i > args.nr_entries)
break;
again:
/* Populate hashmap, bloom filter map, and array map with the same
* random values
*/
err = syscall(__NR_getrandom, val, val_size, 0);
if (err != val_size) {
ctx.map_prepare_err = true;
fprintf(stderr, "failed to get random value: %d\n", -errno);
break;
}
if (ctx.use_hashmap) {
err = bpf_map_update_elem(ctx.hashmap_fd, val, val, BPF_NOEXIST);
if (err) {
if (err != -EEXIST) {
ctx.map_prepare_err = true;
fprintf(stderr, "failed to add elem to hashmap: %d\n",
-errno);
break;
}
goto again;
}
}
i--;
if (ctx.use_array_map) {
err = bpf_map_update_elem(ctx.array_map_fd, &i, val, 0);
if (err) {
ctx.map_prepare_err = true;
fprintf(stderr, "failed to add elem to array map: %d\n", -errno);
break;
}
}
if (ctx.use_hashmap && !ctx.hashmap_use_bloom)
continue;
err = bpf_map_update_elem(ctx.bloom_fd, NULL, val, 0);
if (err) {
ctx.map_prepare_err = true;
fprintf(stderr,
"failed to add elem to bloom filter map: %d\n", -errno);
break;
}
}
done:
pthread_mutex_lock(&ctx.map_done_mtx);
ctx.map_done = true;
pthread_cond_signal(&ctx.map_done_cv);
pthread_mutex_unlock(&ctx.map_done_mtx);
if (val)
free(val);
return NULL;
}
static void populate_maps(void)
{
unsigned int nr_cpus = bpf_num_possible_cpus();
pthread_t map_thread;
int i, err, nr_rand_bytes;
ctx.bloom_fd = bpf_map__fd(ctx.skel->maps.bloom_map);
ctx.hashmap_fd = bpf_map__fd(ctx.skel->maps.hashmap);
ctx.array_map_fd = bpf_map__fd(ctx.skel->maps.array_map);
for (i = 0; i < nr_cpus; i++) {
err = pthread_create(&map_thread, NULL, map_prepare_thread,
NULL);
if (err) {
fprintf(stderr, "failed to create pthread: %d\n", -errno);
exit(1);
}
}
pthread_mutex_lock(&ctx.map_done_mtx);
while (!ctx.map_done)
pthread_cond_wait(&ctx.map_done_cv, &ctx.map_done_mtx);
pthread_mutex_unlock(&ctx.map_done_mtx);
if (ctx.map_prepare_err)
exit(1);
nr_rand_bytes = syscall(__NR_getrandom, ctx.skel->bss->rand_vals,
ctx.skel->rodata->nr_rand_bytes, 0);
if (nr_rand_bytes != ctx.skel->rodata->nr_rand_bytes) {
fprintf(stderr, "failed to get random bytes\n");
exit(1);
}
}
static void check_args(void)
{
if (args.value_size < 8) {
__u64 nr_unique_entries = 1ULL << (args.value_size * 8);
if (args.nr_entries > nr_unique_entries) {
fprintf(stderr,
"Not enough unique values for the nr_entries requested\n");
exit(1);
}
}
}
static struct bloom_filter_bench *setup_skeleton(void)
{
struct bloom_filter_bench *skel;
check_args();
setup_libbpf();
skel = bloom_filter_bench__open();
if (!skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
skel->rodata->hashmap_use_bloom = ctx.hashmap_use_bloom;
skel->rodata->count_false_hits = ctx.count_false_hits;
/* Resize number of entries */
bpf_map__set_max_entries(skel->maps.hashmap, args.nr_entries);
bpf_map__set_max_entries(skel->maps.array_map, args.nr_entries);
bpf_map__set_max_entries(skel->maps.bloom_map, args.nr_entries);
/* Set value size */
bpf_map__set_value_size(skel->maps.array_map, args.value_size);
bpf_map__set_value_size(skel->maps.bloom_map, args.value_size);
bpf_map__set_value_size(skel->maps.hashmap, args.value_size);
/* For the hashmap, we use the value as the key as well */
bpf_map__set_key_size(skel->maps.hashmap, args.value_size);
skel->bss->value_size = args.value_size;
/* Set number of hash functions */
bpf_map__set_map_extra(skel->maps.bloom_map, args.nr_hash_funcs);
if (bloom_filter_bench__load(skel)) {
fprintf(stderr, "failed to load skeleton\n");
exit(1);
}
return skel;
}
static void bloom_lookup_setup(void)
{
struct bpf_link *link;
ctx.use_array_map = true;
ctx.skel = setup_skeleton();
populate_maps();
link = bpf_program__attach(ctx.skel->progs.bloom_lookup);
if (!link) {
fprintf(stderr, "failed to attach program!\n");
exit(1);
}
}
static void bloom_update_setup(void)
{
struct bpf_link *link;
ctx.use_array_map = true;
ctx.skel = setup_skeleton();
populate_maps();
link = bpf_program__attach(ctx.skel->progs.bloom_update);
if (!link) {
fprintf(stderr, "failed to attach program!\n");
exit(1);
}
}
static void false_positive_setup(void)
{
struct bpf_link *link;
ctx.use_hashmap = true;
ctx.hashmap_use_bloom = true;
ctx.count_false_hits = true;
ctx.skel = setup_skeleton();
populate_maps();
link = bpf_program__attach(ctx.skel->progs.bloom_hashmap_lookup);
if (!link) {
fprintf(stderr, "failed to attach program!\n");
exit(1);
}
}
static void hashmap_with_bloom_setup(void)
{
struct bpf_link *link;
ctx.use_hashmap = true;
ctx.hashmap_use_bloom = true;
ctx.skel = setup_skeleton();
populate_maps();
link = bpf_program__attach(ctx.skel->progs.bloom_hashmap_lookup);
if (!link) {
fprintf(stderr, "failed to attach program!\n");
exit(1);
}
}
static void hashmap_no_bloom_setup(void)
{
struct bpf_link *link;
ctx.use_hashmap = true;
ctx.skel = setup_skeleton();
populate_maps();
link = bpf_program__attach(ctx.skel->progs.bloom_hashmap_lookup);
if (!link) {
fprintf(stderr, "failed to attach program!\n");
exit(1);
}
}
static void measure(struct bench_res *res)
{
unsigned long total_hits = 0, total_drops = 0, total_false_hits = 0;
static unsigned long last_hits, last_drops, last_false_hits;
unsigned int nr_cpus = bpf_num_possible_cpus();
int hit_key, drop_key, false_hit_key;
int i;
hit_key = ctx.skel->rodata->hit_key;
drop_key = ctx.skel->rodata->drop_key;
false_hit_key = ctx.skel->rodata->false_hit_key;
if (ctx.skel->bss->error != 0) {
fprintf(stderr, "error (%d) when searching the bloom filter\n",
ctx.skel->bss->error);
exit(1);
}
for (i = 0; i < nr_cpus; i++) {
struct stat *s = (void *)&ctx.skel->bss->percpu_stats[i];
total_hits += s->stats[hit_key];
total_drops += s->stats[drop_key];
total_false_hits += s->stats[false_hit_key];
}
res->hits = total_hits - last_hits;
res->drops = total_drops - last_drops;
res->false_hits = total_false_hits - last_false_hits;
last_hits = total_hits;
last_drops = total_drops;
last_false_hits = total_false_hits;
}
const struct bench bench_bloom_lookup = {
.name = "bloom-lookup",
.argp = &bench_bloom_map_argp,
.validate = validate,
.setup = bloom_lookup_setup,
.producer_thread = producer,
.measure = measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_bloom_update = {
.name = "bloom-update",
.argp = &bench_bloom_map_argp,
.validate = validate,
.setup = bloom_update_setup,
.producer_thread = producer,
.measure = measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_bloom_false_positive = {
.name = "bloom-false-positive",
.argp = &bench_bloom_map_argp,
.validate = validate,
.setup = false_positive_setup,
.producer_thread = producer,
.measure = measure,
.report_progress = false_hits_report_progress,
.report_final = false_hits_report_final,
};
const struct bench bench_hashmap_without_bloom = {
.name = "hashmap-without-bloom",
.argp = &bench_bloom_map_argp,
.validate = validate,
.setup = hashmap_no_bloom_setup,
.producer_thread = producer,
.measure = measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_hashmap_with_bloom = {
.name = "hashmap-with-bloom",
.argp = &bench_bloom_map_argp,
.validate = validate,
.setup = hashmap_with_bloom_setup,
.producer_thread = producer,
.measure = measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_bpf_crypto.c
+185
View File
@@ -0,0 +1,185 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#include <argp.h>
#include "bench.h"
#include "crypto_bench.skel.h"
#define MAX_CIPHER_LEN 32
static char *input;
static struct crypto_ctx {
struct crypto_bench *skel;
int pfd;
} ctx;
static struct crypto_args {
u32 crypto_len;
char *crypto_cipher;
} args = {
.crypto_len = 16,
.crypto_cipher = "ecb(aes)",
};
enum {
ARG_CRYPTO_LEN = 5000,
ARG_CRYPTO_CIPHER = 5001,
};
static const struct argp_option opts[] = {
{ "crypto-len", ARG_CRYPTO_LEN, "CRYPTO_LEN", 0,
"Set the length of crypto buffer" },
{ "crypto-cipher", ARG_CRYPTO_CIPHER, "CRYPTO_CIPHER", 0,
"Set the cipher to use (default:ecb(aes))" },
{},
};
static error_t crypto_parse_arg(int key, char *arg, struct argp_state *state)
{
switch (key) {
case ARG_CRYPTO_LEN:
args.crypto_len = strtoul(arg, NULL, 10);
if (!args.crypto_len ||
args.crypto_len > sizeof(ctx.skel->bss->dst)) {
fprintf(stderr, "Invalid crypto buffer len (limit %zu)\n",
sizeof(ctx.skel->bss->dst));
argp_usage(state);
}
break;
case ARG_CRYPTO_CIPHER:
args.crypto_cipher = strdup(arg);
if (!strlen(args.crypto_cipher) ||
strlen(args.crypto_cipher) > MAX_CIPHER_LEN) {
fprintf(stderr, "Invalid crypto cipher len (limit %d)\n",
MAX_CIPHER_LEN);
argp_usage(state);
}
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
const struct argp bench_crypto_argp = {
.options = opts,
.parser = crypto_parse_arg,
};
static void crypto_validate(void)
{
if (env.consumer_cnt != 0) {
fprintf(stderr, "bpf crypto benchmark doesn't support consumer!\n");
exit(1);
}
}
static void crypto_setup(void)
{
LIBBPF_OPTS(bpf_test_run_opts, opts);
int err, pfd;
size_t i, sz;
sz = args.crypto_len;
if (!sz || sz > sizeof(ctx.skel->bss->dst)) {
fprintf(stderr, "invalid encrypt buffer size (source %zu, target %zu)\n",
sz, sizeof(ctx.skel->bss->dst));
exit(1);
}
setup_libbpf();
ctx.skel = crypto_bench__open();
if (!ctx.skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
snprintf(ctx.skel->bss->cipher, 128, "%s", args.crypto_cipher);
memcpy(ctx.skel->bss->key, "12345678testtest", 16);
ctx.skel->bss->key_len = 16;
ctx.skel->bss->authsize = 0;
srandom(time(NULL));
input = malloc(sz);
for (i = 0; i < sz - 1; i++)
input[i] = '1' + random() % 9;
input[sz - 1] = '\0';
ctx.skel->rodata->len = args.crypto_len;
err = crypto_bench__load(ctx.skel);
if (err) {
fprintf(stderr, "failed to load skeleton\n");
crypto_bench__destroy(ctx.skel);
exit(1);
}
pfd = bpf_program__fd(ctx.skel->progs.crypto_setup);
if (pfd < 0) {
fprintf(stderr, "failed to get fd for setup prog\n");
crypto_bench__destroy(ctx.skel);
exit(1);
}
err = bpf_prog_test_run_opts(pfd, &opts);
if (err || ctx.skel->bss->status) {
fprintf(stderr, "failed to run setup prog: err %d, status %d\n",
err, ctx.skel->bss->status);
crypto_bench__destroy(ctx.skel);
exit(1);
}
}
static void crypto_encrypt_setup(void)
{
crypto_setup();
ctx.pfd = bpf_program__fd(ctx.skel->progs.crypto_encrypt);
}
static void crypto_decrypt_setup(void)
{
crypto_setup();
ctx.pfd = bpf_program__fd(ctx.skel->progs.crypto_decrypt);
}
static void crypto_measure(struct bench_res *res)
{
res->hits = atomic_swap(&ctx.skel->bss->hits, 0);
}
static void *crypto_producer(void *unused)
{
LIBBPF_OPTS(bpf_test_run_opts, opts,
.repeat = 64,
.data_in = input,
.data_size_in = args.crypto_len,
);
while (true)
(void)bpf_prog_test_run_opts(ctx.pfd, &opts);
return NULL;
}
const struct bench bench_crypto_encrypt = {
.name = "crypto-encrypt",
.argp = &bench_crypto_argp,
.validate = crypto_validate,
.setup = crypto_encrypt_setup,
.producer_thread = crypto_producer,
.measure = crypto_measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_crypto_decrypt = {
.name = "crypto-decrypt",
.argp = &bench_crypto_argp,
.validate = crypto_validate,
.setup = crypto_decrypt_setup,
.producer_thread = crypto_producer,
.measure = crypto_measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_full_update.c
+119
View File
@@ -0,0 +1,119 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2022 Bytedance */
#include "bench.h"
#include "bpf_hashmap_full_update_bench.skel.h"
#include "bpf_util.h"
/* BPF triggering benchmarks */
static struct ctx {
struct bpf_hashmap_full_update_bench *skel;
} ctx;
#define MAX_LOOP_NUM 10000
static void validate(void)
{
if (env.consumer_cnt != 0) {
fprintf(stderr, "benchmark doesn't support consumer!\n");
exit(1);
}
}
static void *producer(void *input)
{
while (true) {
/* trigger the bpf program */
syscall(__NR_getpgid);
}
return NULL;
}
static void measure(struct bench_res *res)
{
}
static void hashmap_full_update_setup(enum bpf_map_type map_type)
{
struct bpf_link *link;
int map_fd, i, max_entries;
setup_libbpf();
ctx.skel = bpf_hashmap_full_update_bench__open();
if (!ctx.skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
bpf_map__set_type(ctx.skel->maps.hash_map_bench, map_type);
if (map_type == BPF_MAP_TYPE_RHASH)
bpf_map__set_map_flags(ctx.skel->maps.hash_map_bench,
BPF_F_NO_PREALLOC);
if (bpf_hashmap_full_update_bench__load(ctx.skel)) {
fprintf(stderr, "failed to load skeleton\n");
exit(1);
}
ctx.skel->bss->nr_loops = MAX_LOOP_NUM;
link = bpf_program__attach(ctx.skel->progs.benchmark);
if (!link) {
fprintf(stderr, "failed to attach program!\n");
exit(1);
}
/* fill hash_map */
map_fd = bpf_map__fd(ctx.skel->maps.hash_map_bench);
max_entries = bpf_map__max_entries(ctx.skel->maps.hash_map_bench);
for (i = 0; i < max_entries; i++)
bpf_map_update_elem(map_fd, &i, &i, BPF_ANY);
}
static void setup(void)
{
hashmap_full_update_setup(BPF_MAP_TYPE_HASH);
}
static void rhash_setup(void)
{
hashmap_full_update_setup(BPF_MAP_TYPE_RHASH);
}
static void hashmap_report_final(struct bench_res res[], int res_cnt)
{
unsigned int nr_cpus = bpf_num_possible_cpus();
int i;
for (i = 0; i < nr_cpus; i++) {
u64 time = ctx.skel->bss->percpu_time[i];
if (!time)
continue;
printf("%d:hash_map_full_perf %lld events per sec\n",
i, ctx.skel->bss->nr_loops * 1000000000ll / time);
}
}
const struct bench bench_bpf_hashmap_full_update = {
.name = "bpf-hashmap-full-update",
.validate = validate,
.setup = setup,
.producer_thread = producer,
.measure = measure,
.report_progress = NULL,
.report_final = hashmap_report_final,
};
const struct bench bench_bpf_rhashmap_full_update = {
.name = "bpf-rhashmap-full-update",
.validate = validate,
.setup = rhash_setup,
.producer_thread = producer,
.measure = measure,
.report_progress = NULL,
.report_final = hashmap_report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_lookup.c
+304
View File
@@ -0,0 +1,304 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2023 Isovalent */
#include <sys/random.h>
#include <argp.h>
#include "bench.h"
#include "bpf_hashmap_lookup.skel.h"
#include "bpf_util.h"
/* BPF triggering benchmarks */
static struct ctx {
struct bpf_hashmap_lookup *skel;
} ctx;
/* only available to kernel, so define it here */
#define BPF_MAX_LOOPS (1<<23)
#define MAX_KEY_SIZE 1024 /* the size of the key map */
static struct {
__u32 key_size;
__u32 map_flags;
__u32 max_entries;
__u32 nr_entries;
__u32 nr_loops;
} args = {
.key_size = 4,
.map_flags = 0,
.max_entries = 1000,
.nr_entries = 500,
.nr_loops = 1000000,
};
enum {
ARG_KEY_SIZE = 8001,
ARG_MAP_FLAGS,
ARG_MAX_ENTRIES,
ARG_NR_ENTRIES,
ARG_NR_LOOPS,
};
static const struct argp_option opts[] = {
{ "key_size", ARG_KEY_SIZE, "KEY_SIZE", 0,
"The hashmap key size (max 1024)"},
{ "map_flags", ARG_MAP_FLAGS, "MAP_FLAGS", 0,
"The hashmap flags passed to BPF_MAP_CREATE"},
{ "max_entries", ARG_MAX_ENTRIES, "MAX_ENTRIES", 0,
"The hashmap max entries"},
{ "nr_entries", ARG_NR_ENTRIES, "NR_ENTRIES", 0,
"The number of entries to insert/lookup"},
{ "nr_loops", ARG_NR_LOOPS, "NR_LOOPS", 0,
"The number of loops for the benchmark"},
{},
};
static error_t parse_arg(int key, char *arg, struct argp_state *state)
{
long ret;
switch (key) {
case ARG_KEY_SIZE:
ret = strtol(arg, NULL, 10);
if (ret < 1 || ret > MAX_KEY_SIZE) {
fprintf(stderr, "invalid key_size");
argp_usage(state);
}
args.key_size = ret;
break;
case ARG_MAP_FLAGS:
ret = strtol(arg, NULL, 0);
if (ret < 0 || ret > UINT_MAX) {
fprintf(stderr, "invalid map_flags");
argp_usage(state);
}
args.map_flags = ret;
break;
case ARG_MAX_ENTRIES:
ret = strtol(arg, NULL, 10);
if (ret < 1 || ret > UINT_MAX) {
fprintf(stderr, "invalid max_entries");
argp_usage(state);
}
args.max_entries = ret;
break;
case ARG_NR_ENTRIES:
ret = strtol(arg, NULL, 10);
if (ret < 1 || ret > UINT_MAX) {
fprintf(stderr, "invalid nr_entries");
argp_usage(state);
}
args.nr_entries = ret;
break;
case ARG_NR_LOOPS:
ret = strtol(arg, NULL, 10);
if (ret < 1 || ret > BPF_MAX_LOOPS) {
fprintf(stderr, "invalid nr_loops: %ld (min=1 max=%u)\n",
ret, BPF_MAX_LOOPS);
argp_usage(state);
}
args.nr_loops = ret;
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
const struct argp bench_hashmap_lookup_argp = {
.options = opts,
.parser = parse_arg,
};
static void validate(void)
{
if (env.consumer_cnt != 0) {
fprintf(stderr, "benchmark doesn't support consumer!\n");
exit(1);
}
if (args.nr_entries > args.max_entries) {
fprintf(stderr, "args.nr_entries is too big! (max %u, got %u)\n",
args.max_entries, args.nr_entries);
exit(1);
}
}
static void *producer(void *input)
{
while (true) {
/* trigger the bpf program */
syscall(__NR_getpgid);
}
return NULL;
}
static void measure(struct bench_res *res)
{
}
static inline void patch_key(u32 i, u32 *key)
{
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
*key = i + 1;
#else
*key = __builtin_bswap32(i + 1);
#endif
/* the rest of key is random */
}
static void hashmap_lookup_setup(enum bpf_map_type map_type)
{
struct bpf_link *link;
__u32 map_flags;
int map_fd;
int ret;
int i;
setup_libbpf();
ctx.skel = bpf_hashmap_lookup__open();
if (!ctx.skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
map_flags = args.map_flags;
if (map_type == BPF_MAP_TYPE_RHASH)
map_flags |= BPF_F_NO_PREALLOC;
bpf_map__set_type(ctx.skel->maps.hash_map_bench, map_type);
bpf_map__set_max_entries(ctx.skel->maps.hash_map_bench, args.max_entries);
bpf_map__set_key_size(ctx.skel->maps.hash_map_bench, args.key_size);
bpf_map__set_value_size(ctx.skel->maps.hash_map_bench, 8);
bpf_map__set_map_flags(ctx.skel->maps.hash_map_bench, map_flags);
ctx.skel->bss->nr_entries = args.nr_entries;
ctx.skel->bss->nr_loops = args.nr_loops / args.nr_entries;
if (args.key_size > 4) {
for (i = 1; i < args.key_size/4; i++)
ctx.skel->bss->key[i] = 2654435761 * i;
}
ret = bpf_hashmap_lookup__load(ctx.skel);
if (ret) {
bpf_hashmap_lookup__destroy(ctx.skel);
fprintf(stderr, "failed to load map: %s", strerror(-ret));
exit(1);
}
/* fill in the hash_map */
map_fd = bpf_map__fd(ctx.skel->maps.hash_map_bench);
for (u64 i = 0; i < args.nr_entries; i++) {
patch_key(i, ctx.skel->bss->key);
bpf_map_update_elem(map_fd, ctx.skel->bss->key, &i, BPF_ANY);
}
link = bpf_program__attach(ctx.skel->progs.benchmark);
if (!link) {
fprintf(stderr, "failed to attach program!\n");
exit(1);
}
}
static void setup(void)
{
hashmap_lookup_setup(BPF_MAP_TYPE_HASH);
}
static void rhash_setup(void)
{
hashmap_lookup_setup(BPF_MAP_TYPE_RHASH);
}
static inline double events_from_time(u64 time)
{
if (time)
return args.nr_loops * 1000000000llu / time / 1000000.0L;
return 0;
}
static int compute_events(u64 *times, double *events_mean, double *events_stddev, u64 *mean_time)
{
int i, n = 0;
*events_mean = 0;
*events_stddev = 0;
*mean_time = 0;
for (i = 0; i < 32; i++) {
if (!times[i])
break;
*mean_time += times[i];
*events_mean += events_from_time(times[i]);
n += 1;
}
if (!n)
return 0;
*mean_time /= n;
*events_mean /= n;
if (n > 1) {
for (i = 0; i < n; i++) {
double events_i = *events_mean - events_from_time(times[i]);
*events_stddev += events_i * events_i / (n - 1);
}
*events_stddev = sqrt(*events_stddev);
}
return n;
}
static void hashmap_report_final(struct bench_res res[], int res_cnt)
{
unsigned int nr_cpus = bpf_num_possible_cpus();
double events_mean, events_stddev;
u64 mean_time;
int i, n;
for (i = 0; i < nr_cpus; i++) {
n = compute_events(ctx.skel->bss->percpu_times[i], &events_mean,
&events_stddev, &mean_time);
if (n == 0)
continue;
if (env.quiet) {
/* we expect only one cpu to be present */
if (env.affinity)
printf("%.3lf\n", events_mean);
else
printf("cpu%02d %.3lf\n", i, events_mean);
} else {
printf("cpu%02d: lookup %.3lfM ± %.3lfM events/sec"
" (approximated from %d samples of ~%lums)\n",
i, events_mean, 2*events_stddev,
n, mean_time / 1000000);
}
}
}
const struct bench bench_bpf_hashmap_lookup = {
.name = "bpf-hashmap-lookup",
.argp = &bench_hashmap_lookup_argp,
.validate = validate,
.setup = setup,
.producer_thread = producer,
.measure = measure,
.report_progress = NULL,
.report_final = hashmap_report_final,
};
const struct bench bench_bpf_rhashmap_lookup = {
.name = "bpf-rhashmap-lookup",
.argp = &bench_hashmap_lookup_argp,
.validate = validate,
.setup = rhash_setup,
.producer_thread = producer,
.measure = measure,
.report_progress = NULL,
.report_final = hashmap_report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_bpf_loop.c
+100
View File
@@ -0,0 +1,100 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2021 Facebook */
#include <argp.h>
#include "bench.h"
#include "bpf_loop_bench.skel.h"
/* BPF triggering benchmarks */
static struct ctx {
struct bpf_loop_bench *skel;
} ctx;
static struct {
__u32 nr_loops;
} args = {
.nr_loops = 10,
};
enum {
ARG_NR_LOOPS = 4000,
};
static const struct argp_option opts[] = {
{ "nr_loops", ARG_NR_LOOPS, "nr_loops", 0,
"Set number of loops for the bpf_loop helper"},
{},
};
static error_t parse_arg(int key, char *arg, struct argp_state *state)
{
switch (key) {
case ARG_NR_LOOPS:
args.nr_loops = strtol(arg, NULL, 10);
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
/* exported into benchmark runner */
const struct argp bench_bpf_loop_argp = {
.options = opts,
.parser = parse_arg,
};
static void validate(void)
{
if (env.consumer_cnt != 0) {
fprintf(stderr, "benchmark doesn't support consumer!\n");
exit(1);
}
}
static void *producer(void *input)
{
while (true)
/* trigger the bpf program */
syscall(__NR_getpgid);
return NULL;
}
static void measure(struct bench_res *res)
{
res->hits = atomic_swap(&ctx.skel->bss->hits, 0);
}
static void setup(void)
{
struct bpf_link *link;
setup_libbpf();
ctx.skel = bpf_loop_bench__open_and_load();
if (!ctx.skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
link = bpf_program__attach(ctx.skel->progs.benchmark);
if (!link) {
fprintf(stderr, "failed to attach program!\n");
exit(1);
}
ctx.skel->bss->nr_loops = args.nr_loops;
}
const struct bench bench_bpf_loop = {
.name = "bpf-loop",
.argp = &bench_bpf_loop_argp,
.validate = validate,
.setup = setup,
.producer_thread = producer,
.measure = measure,
.report_progress = ops_report_progress,
.report_final = ops_report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_bpf_nop.c
+84
View File
@@ -0,0 +1,84 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#include "bench.h"
#include "bench_bpf_timing.h"
#include "bpf_nop_bench.skel.h"
#include "bpf_util.h"
static struct ctx {
struct bpf_nop_bench *skel;
struct bpf_bench_timing timing;
int prog_fd;
} ctx;
static void nop_validate(void)
{
if (env.consumer_cnt != 0) {
fprintf(stderr, "benchmark doesn't support consumers\n");
exit(1);
}
}
static void nop_run_once(void *unused __always_unused)
{
LIBBPF_OPTS(bpf_test_run_opts, topts);
bpf_prog_test_run_opts(ctx.prog_fd, &topts);
}
static void nop_setup(void)
{
struct bpf_nop_bench *skel;
int err;
setup_libbpf();
skel = bpf_nop_bench__open();
if (!skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
err = bpf_nop_bench__load(skel);
if (err) {
fprintf(stderr, "failed to load skeleton: %s\n", strerror(-err));
bpf_nop_bench__destroy(skel);
exit(1);
}
ctx.skel = skel;
ctx.prog_fd = bpf_program__fd(skel->progs.bench_nop);
BENCH_TIMING_INIT(&ctx.timing, skel, 0);
bpf_bench_calibrate(&ctx.timing, nop_run_once, NULL);
env.duration_sec = 600;
}
static void *nop_producer(void *input)
{
while (true)
nop_run_once(NULL);
return NULL;
}
static void nop_measure(struct bench_res *res)
{
bpf_bench_timing_measure(&ctx.timing, res);
}
static void nop_report_final(struct bench_res res[], int res_cnt)
{
bpf_bench_timing_report(&ctx.timing, "bpf-nop", NULL);
}
const struct bench bench_bpf_nop = {
.name = "bpf-nop",
.validate = nop_validate,
.setup = nop_setup,
.producer_thread = nop_producer,
.measure = nop_measure,
.report_final = nop_report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_bpf_timing.c
+298
View File
@@ -0,0 +1,298 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include "bench_bpf_timing.h"
#include "bpf_util.h"
struct timing_stats {
double min, max;
double median, p99;
double mean, stddev;
int count;
};
static int cmp_double(const void *a, const void *b)
{
double da = *(const double *)a;
double db = *(const double *)b;
if (da < db)
return -1;
if (da > db)
return 1;
return 0;
}
static double percentile(const double *sorted, int n, double pct)
{
int idx = (int)(n * pct / 100.0);
if (idx >= n)
idx = n - 1;
return sorted[idx];
}
static int collect_samples(struct bpf_bench_timing *t,
double *out, int max_out)
{
unsigned int nr_cpus = bpf_num_possible_cpus();
__u32 timed_iters = t->batch_iters;
int total = 0;
if (nr_cpus > BENCH_NR_CPUS)
nr_cpus = BENCH_NR_CPUS;
for (unsigned int cpu = 0; cpu < nr_cpus; cpu++) {
__u32 count = t->idx[cpu];
if (count > BENCH_NR_SAMPLES)
count = BENCH_NR_SAMPLES;
for (__u32 i = 0; i < count && total < max_out; i++) {
__u64 sample = t->samples[cpu][i];
if (sample == 0)
continue;
out[total++] = (double)sample / timed_iters;
}
}
qsort(out, total, sizeof(double), cmp_double);
return total;
}
static int filter_outliers_iqr(double *sorted, int n)
{
double q1, q3, iqr, lo, hi;
int start = 0, end = n;
if (n < 8)
return n;
q1 = sorted[n / 4];
q3 = sorted[3 * n / 4];
iqr = q3 - q1;
lo = q1 - 1.5 * iqr;
hi = q3 + 1.5 * iqr;
while (start < end && sorted[start] < lo)
start++;
while (end > start && sorted[end - 1] > hi)
end--;
if (start > 0)
memmove(sorted, sorted + start, (end - start) * sizeof(double));
return end - start;
}
static void compute_stats(const double *sorted, int n,
struct timing_stats *s)
{
double sum = 0, var_sum = 0;
memset(s, 0, sizeof(*s));
s->count = n;
if (n == 0)
return;
s->min = sorted[0];
s->max = sorted[n - 1];
s->median = sorted[n / 2];
s->p99 = percentile(sorted, n, 99);
for (int i = 0; i < n; i++)
sum += sorted[i];
s->mean = sum / n;
for (int i = 0; i < n; i++) {
double d = sorted[i] - s->mean;
var_sum += d * d;
}
s->stddev = n > 1 ? sqrt(var_sum / (n - 1)) : 0;
}
void bpf_bench_timing_measure(struct bpf_bench_timing *t, struct bench_res *res)
{
unsigned int nr_cpus;
__u32 total_samples;
int i;
t->warmup_ticks++;
if (t->warmup_ticks < env.warmup_sec)
return;
if (t->warmup_ticks == env.warmup_sec) {
*t->timing_enabled = 1;
return;
}
nr_cpus = bpf_num_possible_cpus();
if (nr_cpus > BENCH_NR_CPUS)
nr_cpus = BENCH_NR_CPUS;
total_samples = 0;
for (i = 0; i < (int)nr_cpus; i++) {
__u32 cnt = t->idx[i];
if (cnt > BENCH_NR_SAMPLES)
cnt = BENCH_NR_SAMPLES;
total_samples += cnt;
}
if (total_samples >= (__u32)env.producer_cnt * t->target_samples && !t->done) {
t->done = true;
*t->timing_enabled = 0;
bench_force_done();
}
}
void bpf_bench_timing_report(struct bpf_bench_timing *t, const char *name, const char *description)
{
int max_out = BENCH_NR_CPUS * BENCH_NR_SAMPLES;
struct timing_stats s;
double *all;
int total;
all = calloc(max_out, sizeof(*all));
if (!all) {
fprintf(stderr, "failed to allocate timing buffer\n");
return;
}
total = collect_samples(t, all, max_out);
if (total == 0) {
printf("No timing samples collected.\n");
free(all);
return;
}
total = filter_outliers_iqr(all, total);
compute_stats(all, total, &s);
if (t->machine_readable) {
printf("RESULT scenario=%s samples=%d median=%.2f stddev=%.2f cv=%.2f min=%.2f "
"p99=%.2f max=%.2f\n", name, total, s.median, s.stddev,
s.mean > 0 ? s.stddev / s.mean * 100.0 : 0.0, s.min, s.p99, s.max);
} else {
printf("%s: median %.2f ns/op, stddev %.2f, p99 %.2f (%d samples)\n", name,
s.median, s.stddev, s.p99, total);
}
free(all);
}
#define CALIBRATE_SEED_BATCH 100
#define CALIBRATE_MIN_BATCH 100
#define CALIBRATE_MAX_BATCH 10000000
#define CALIBRATE_TARGET_MS 10
#define CALIBRATE_RUNS 5
#define PROPORTIONALITY_TOL 0.05 /* 5% */
static void reset_timing(struct bpf_bench_timing *t)
{
*t->timing_enabled = 0;
memset(t->samples, 0, sizeof(__u64) * BENCH_NR_CPUS * BENCH_NR_SAMPLES);
memset(t->idx, 0, sizeof(__u32) * BENCH_NR_CPUS);
}
static __u64 measure_elapsed(struct bpf_bench_timing *t, bpf_bench_run_fn run_fn, void *run_ctx,
__u32 iters, int runs)
{
__u64 buf[CALIBRATE_RUNS];
int n = 0, i, j;
reset_timing(t);
*t->batch_iters_bss = iters;
*t->timing_enabled = 1;
for (i = 0; i < runs; i++)
run_fn(run_ctx);
*t->timing_enabled = 0;
for (i = 0; i < BENCH_NR_CPUS && n < runs; i++) {
__u32 cnt = t->idx[i];
for (j = 0; j < (int)cnt && n < runs; j++)
buf[n++] = t->samples[i][j];
}
if (n == 0)
return 0;
for (i = 1; i < n; i++) {
__u64 key = buf[i];
j = i - 1;
while (j >= 0 && buf[j] > key) {
buf[j + 1] = buf[j];
j--;
}
buf[j + 1] = key;
}
return buf[n / 2];
}
static __u32 compute_batch_iters(__u64 per_op_ns)
{
__u64 target_ns = (__u64)CALIBRATE_TARGET_MS * 1000000ULL;
__u32 iters;
if (per_op_ns == 0)
return CALIBRATE_MIN_BATCH;
iters = target_ns / per_op_ns;
if (iters < CALIBRATE_MIN_BATCH)
iters = CALIBRATE_MIN_BATCH;
if (iters > CALIBRATE_MAX_BATCH)
iters = CALIBRATE_MAX_BATCH;
return iters;
}
void bpf_bench_calibrate(struct bpf_bench_timing *t, bpf_bench_run_fn run_fn, void *run_ctx)
{
__u64 elapsed, per_op_ns;
__u64 time_n, time_2n;
double ratio;
elapsed = measure_elapsed(t, run_fn, run_ctx, CALIBRATE_SEED_BATCH, CALIBRATE_RUNS);
if (elapsed == 0) {
fprintf(stderr, "calibration: no timing samples, using default\n");
t->batch_iters = 10000;
*t->batch_iters_bss = t->batch_iters;
reset_timing(t);
return;
}
per_op_ns = elapsed / CALIBRATE_SEED_BATCH;
t->batch_iters = compute_batch_iters(per_op_ns);
time_n = measure_elapsed(t, run_fn, run_ctx, t->batch_iters, CALIBRATE_RUNS);
time_2n = measure_elapsed(t, run_fn, run_ctx, t->batch_iters * 2, CALIBRATE_RUNS);
if (time_n > 0 && time_2n > 0) {
ratio = (double)time_2n / (double)time_n;
if (fabs(ratio - 2.0) / 2.0 > PROPORTIONALITY_TOL)
fprintf(stderr,
"WARNING: proportionality check failed (2N/N ratio=%.3f, "
"expected=2.000, error=%.1f%%)\n System noise may be affecting "
"results.\n",
ratio, fabs(ratio - 2.0) / 2.0 * 100.0);
}
*t->batch_iters_bss = t->batch_iters;
reset_timing(t);
}
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_count.c
+79
View File
@@ -0,0 +1,79 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include "bench.h"
/* COUNT-GLOBAL benchmark */
static struct count_global_ctx {
struct counter hits;
} count_global_ctx;
static void *count_global_producer(void *input)
{
struct count_global_ctx *ctx = &count_global_ctx;
while (true) {
atomic_inc(&ctx->hits.value);
}
return NULL;
}
static void count_global_measure(struct bench_res *res)
{
struct count_global_ctx *ctx = &count_global_ctx;
res->hits = atomic_swap(&ctx->hits.value, 0);
}
/* COUNT-local benchmark */
static struct count_local_ctx {
struct counter *hits;
} count_local_ctx;
static void count_local_setup(void)
{
struct count_local_ctx *ctx = &count_local_ctx;
ctx->hits = calloc(env.producer_cnt, sizeof(*ctx->hits));
if (!ctx->hits)
exit(1);
}
static void *count_local_producer(void *input)
{
struct count_local_ctx *ctx = &count_local_ctx;
int idx = (long)input;
while (true) {
atomic_inc(&ctx->hits[idx].value);
}
return NULL;
}
static void count_local_measure(struct bench_res *res)
{
struct count_local_ctx *ctx = &count_local_ctx;
int i;
for (i = 0; i < env.producer_cnt; i++) {
res->hits += atomic_swap(&ctx->hits[i].value, 0);
}
}
const struct bench bench_count_global = {
.name = "count-global",
.producer_thread = count_global_producer,
.measure = count_global_measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_count_local = {
.name = "count-local",
.setup = count_local_setup,
.producer_thread = count_local_producer,
.measure = count_local_measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_htab_mem.c
+381
View File
@@ -0,0 +1,381 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) 2023. Huawei Technologies Co., Ltd */
#include <argp.h>
#include <stdbool.h>
#include <pthread.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/param.h>
#include <fcntl.h>
#include "bench.h"
#include "bpf_util.h"
#include "cgroup_helpers.h"
#include "htab_mem_bench.skel.h"
struct htab_mem_use_case {
const char *name;
const char **progs;
/* Do synchronization between addition thread and deletion thread */
bool need_sync;
};
static struct htab_mem_ctx {
const struct htab_mem_use_case *uc;
struct htab_mem_bench *skel;
pthread_barrier_t *notify;
int fd;
} ctx;
const char *ow_progs[] = {"overwrite", NULL};
const char *batch_progs[] = {"batch_add_batch_del", NULL};
const char *add_del_progs[] = {"add_only", "del_only", NULL};
const static struct htab_mem_use_case use_cases[] = {
{ .name = "overwrite", .progs = ow_progs },
{ .name = "batch_add_batch_del", .progs = batch_progs },
{ .name = "add_del_on_diff_cpu", .progs = add_del_progs, .need_sync = true },
};
static struct htab_mem_args {
u32 value_size;
const char *use_case;
bool preallocated;
} args = {
.value_size = 8,
.use_case = "overwrite",
.preallocated = false,
};
enum {
ARG_VALUE_SIZE = 10000,
ARG_USE_CASE = 10001,
ARG_PREALLOCATED = 10002,
};
static const struct argp_option opts[] = {
{ "value-size", ARG_VALUE_SIZE, "VALUE_SIZE", 0,
"Set the value size of hash map (default 8)" },
{ "use-case", ARG_USE_CASE, "USE_CASE", 0,
"Set the use case of hash map: overwrite|batch_add_batch_del|add_del_on_diff_cpu" },
{ "preallocated", ARG_PREALLOCATED, NULL, 0, "use preallocated hash map" },
{},
};
static error_t htab_mem_parse_arg(int key, char *arg, struct argp_state *state)
{
switch (key) {
case ARG_VALUE_SIZE:
args.value_size = strtoul(arg, NULL, 10);
if (args.value_size > 4096) {
fprintf(stderr, "too big value size %u\n", args.value_size);
argp_usage(state);
}
break;
case ARG_USE_CASE:
args.use_case = strdup(arg);
if (!args.use_case) {
fprintf(stderr, "no mem for use-case\n");
argp_usage(state);
}
break;
case ARG_PREALLOCATED:
args.preallocated = true;
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
const struct argp bench_htab_mem_argp = {
.options = opts,
.parser = htab_mem_parse_arg,
};
static void htab_mem_validate(void)
{
if (!strcmp(use_cases[2].name, args.use_case) && env.producer_cnt % 2) {
fprintf(stderr, "%s needs an even number of producers\n", args.use_case);
exit(1);
}
}
static int htab_mem_bench_init_barriers(void)
{
pthread_barrier_t *barriers;
unsigned int i, nr;
if (!ctx.uc->need_sync)
return 0;
nr = (env.producer_cnt + 1) / 2;
barriers = calloc(nr, sizeof(*barriers));
if (!barriers)
return -1;
/* Used for synchronization between two threads */
for (i = 0; i < nr; i++)
pthread_barrier_init(&barriers[i], NULL, 2);
ctx.notify = barriers;
return 0;
}
static void htab_mem_bench_exit_barriers(void)
{
unsigned int i, nr;
if (!ctx.notify)
return;
nr = (env.producer_cnt + 1) / 2;
for (i = 0; i < nr; i++)
pthread_barrier_destroy(&ctx.notify[i]);
free(ctx.notify);
}
static const struct htab_mem_use_case *htab_mem_find_use_case_or_exit(const char *name)
{
unsigned int i;
for (i = 0; i < ARRAY_SIZE(use_cases); i++) {
if (!strcmp(name, use_cases[i].name))
return &use_cases[i];
}
fprintf(stderr, "no such use-case: %s\n", name);
fprintf(stderr, "available use case:");
for (i = 0; i < ARRAY_SIZE(use_cases); i++)
fprintf(stderr, " %s", use_cases[i].name);
fprintf(stderr, "\n");
exit(1);
}
static void htab_mem_setup_impl(enum bpf_map_type map_type)
{
struct bpf_map *map;
const char **names;
int err;
setup_libbpf();
ctx.uc = htab_mem_find_use_case_or_exit(args.use_case);
err = htab_mem_bench_init_barriers();
if (err) {
fprintf(stderr, "failed to init barrier\n");
exit(1);
}
ctx.fd = cgroup_setup_and_join("/htab_mem");
if (ctx.fd < 0)
goto cleanup;
ctx.skel = htab_mem_bench__open();
if (!ctx.skel) {
fprintf(stderr, "failed to open skeleton\n");
goto cleanup;
}
map = ctx.skel->maps.htab;
bpf_map__set_type(map, map_type);
bpf_map__set_value_size(map, args.value_size);
/* Ensure that different CPUs can operate on different subset */
bpf_map__set_max_entries(map, MAX(8192, 64 * env.nr_cpus));
if (map_type != BPF_MAP_TYPE_RHASH && args.preallocated)
bpf_map__set_map_flags(map, bpf_map__map_flags(map) & ~BPF_F_NO_PREALLOC);
names = ctx.uc->progs;
while (*names) {
struct bpf_program *prog;
prog = bpf_object__find_program_by_name(ctx.skel->obj, *names);
if (!prog) {
fprintf(stderr, "no such program %s\n", *names);
goto cleanup;
}
bpf_program__set_autoload(prog, true);
names++;
}
ctx.skel->bss->nr_thread = env.producer_cnt;
err = htab_mem_bench__load(ctx.skel);
if (err) {
fprintf(stderr, "failed to load skeleton\n");
goto cleanup;
}
err = htab_mem_bench__attach(ctx.skel);
if (err) {
fprintf(stderr, "failed to attach skeleton\n");
goto cleanup;
}
return;
cleanup:
htab_mem_bench__destroy(ctx.skel);
htab_mem_bench_exit_barriers();
if (ctx.fd >= 0) {
close(ctx.fd);
cleanup_cgroup_environment();
}
exit(1);
}
static void htab_mem_setup(void)
{
htab_mem_setup_impl(BPF_MAP_TYPE_HASH);
}
static void rhtab_mem_setup(void)
{
htab_mem_setup_impl(BPF_MAP_TYPE_RHASH);
}
static void htab_mem_add_fn(pthread_barrier_t *notify)
{
while (true) {
/* Do addition */
(void)syscall(__NR_getpgid, 0);
/* Notify deletion thread to do deletion */
pthread_barrier_wait(notify);
/* Wait for deletion to complete */
pthread_barrier_wait(notify);
}
}
static void htab_mem_delete_fn(pthread_barrier_t *notify)
{
while (true) {
/* Wait for addition to complete */
pthread_barrier_wait(notify);
/* Do deletion */
(void)syscall(__NR_getppid);
/* Notify addition thread to do addition */
pthread_barrier_wait(notify);
}
}
static void *htab_mem_producer(void *arg)
{
pthread_barrier_t *notify;
int seq;
if (!ctx.uc->need_sync) {
while (true)
(void)syscall(__NR_getpgid, 0);
return NULL;
}
seq = (long)arg;
notify = &ctx.notify[seq / 2];
if (seq & 1)
htab_mem_delete_fn(notify);
else
htab_mem_add_fn(notify);
return NULL;
}
static void htab_mem_read_mem_cgrp_file(const char *name, unsigned long *value)
{
char buf[32];
ssize_t got;
int fd;
fd = openat(ctx.fd, name, O_RDONLY);
if (fd < 0) {
/* cgroup v1 ? */
fprintf(stderr, "no %s\n", name);
*value = 0;
return;
}
got = read(fd, buf, sizeof(buf) - 1);
close(fd);
if (got <= 0) {
*value = 0;
return;
}
buf[got] = 0;
*value = strtoull(buf, NULL, 0);
}
static void htab_mem_measure(struct bench_res *res)
{
res->hits = atomic_swap(&ctx.skel->bss->op_cnt, 0) / env.producer_cnt;
htab_mem_read_mem_cgrp_file("memory.current", &res->gp_ct);
}
static void htab_mem_report_progress(int iter, struct bench_res *res, long delta_ns)
{
double loop, mem;
loop = res->hits / 1000.0 / (delta_ns / 1000000000.0);
mem = res->gp_ct / 1048576.0;
printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0);
printf("per-prod-op %7.2lfk/s, memory usage %7.2lfMiB\n", loop, mem);
}
static void htab_mem_report_final(struct bench_res res[], int res_cnt)
{
double mem_mean = 0.0, mem_stddev = 0.0;
double loop_mean = 0.0, loop_stddev = 0.0;
unsigned long peak_mem;
int i;
for (i = 0; i < res_cnt; i++) {
loop_mean += res[i].hits / 1000.0 / (0.0 + res_cnt);
mem_mean += res[i].gp_ct / 1048576.0 / (0.0 + res_cnt);
}
if (res_cnt > 1) {
for (i = 0; i < res_cnt; i++) {
loop_stddev += (loop_mean - res[i].hits / 1000.0) *
(loop_mean - res[i].hits / 1000.0) /
(res_cnt - 1.0);
mem_stddev += (mem_mean - res[i].gp_ct / 1048576.0) *
(mem_mean - res[i].gp_ct / 1048576.0) /
(res_cnt - 1.0);
}
loop_stddev = sqrt(loop_stddev);
mem_stddev = sqrt(mem_stddev);
}
htab_mem_read_mem_cgrp_file("memory.peak", &peak_mem);
printf("Summary: per-prod-op %7.2lf \u00B1 %7.2lfk/s, memory usage %7.2lf \u00B1 %7.2lfMiB,"
" peak memory usage %7.2lfMiB\n",
loop_mean, loop_stddev, mem_mean, mem_stddev, peak_mem / 1048576.0);
close(ctx.fd);
cleanup_cgroup_environment();
}
static void rhtab_mem_validate(void)
{
if (args.preallocated) {
fprintf(stderr, "rhash map does not support preallocation\n");
exit(1);
}
htab_mem_validate();
}
const struct bench bench_htab_mem = {
.name = "htab-mem",
.argp = &bench_htab_mem_argp,
.validate = htab_mem_validate,
.setup = htab_mem_setup,
.producer_thread = htab_mem_producer,
.measure = htab_mem_measure,
.report_progress = htab_mem_report_progress,
.report_final = htab_mem_report_final,
};
const struct bench bench_rhtab_mem = {
.name = "rhtab-mem",
.argp = &bench_htab_mem_argp,
.validate = rhtab_mem_validate,
.setup = rhtab_mem_setup,
.producer_thread = htab_mem_producer,
.measure = htab_mem_measure,
.report_progress = htab_mem_report_progress,
.report_final = htab_mem_report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_local_storage.c
+282
View File
@@ -0,0 +1,282 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
#include <argp.h>
#include <linux/btf.h>
#include "local_storage_bench.skel.h"
#include "bench.h"
#include <test_btf.h>
static struct {
__u32 nr_maps;
__u32 hashmap_nr_keys_used;
} args = {
.nr_maps = 1000,
.hashmap_nr_keys_used = 1000,
};
enum {
ARG_NR_MAPS = 6000,
ARG_HASHMAP_NR_KEYS_USED = 6001,
};
static const struct argp_option opts[] = {
{ "nr_maps", ARG_NR_MAPS, "NR_MAPS", 0,
"Set number of local_storage maps"},
{ "hashmap_nr_keys_used", ARG_HASHMAP_NR_KEYS_USED, "NR_KEYS",
0, "When doing hashmap test, set number of hashmap keys test uses"},
{},
};
static error_t parse_arg(int key, char *arg, struct argp_state *state)
{
long ret;
switch (key) {
case ARG_NR_MAPS:
ret = strtol(arg, NULL, 10);
if (ret < 1 || ret > UINT_MAX) {
fprintf(stderr, "invalid nr_maps");
argp_usage(state);
}
args.nr_maps = ret;
break;
case ARG_HASHMAP_NR_KEYS_USED:
ret = strtol(arg, NULL, 10);
if (ret < 1 || ret > UINT_MAX) {
fprintf(stderr, "invalid hashmap_nr_keys_used");
argp_usage(state);
}
args.hashmap_nr_keys_used = ret;
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
const struct argp bench_local_storage_argp = {
.options = opts,
.parser = parse_arg,
};
/* Keep in sync w/ array of maps in bpf */
#define MAX_NR_MAPS 1000
/* keep in sync w/ same define in bpf */
#define HASHMAP_SZ 4194304
static void validate(void)
{
if (env.producer_cnt != 1) {
fprintf(stderr, "benchmark doesn't support multi-producer!\n");
exit(1);
}
if (env.consumer_cnt != 0) {
fprintf(stderr, "benchmark doesn't support consumer!\n");
exit(1);
}
if (args.nr_maps > MAX_NR_MAPS) {
fprintf(stderr, "nr_maps must be <= 1000\n");
exit(1);
}
if (args.hashmap_nr_keys_used > HASHMAP_SZ) {
fprintf(stderr, "hashmap_nr_keys_used must be <= %u\n", HASHMAP_SZ);
exit(1);
}
}
static struct {
struct local_storage_bench *skel;
void *bpf_obj;
struct bpf_map *array_of_maps;
} ctx;
static void prepopulate_hashmap(int fd)
{
int i, key, val;
/* local_storage gets will have BPF_LOCAL_STORAGE_GET_F_CREATE flag set, so
* populate the hashmap for a similar comparison
*/
for (i = 0; i < HASHMAP_SZ; i++) {
key = val = i;
if (bpf_map_update_elem(fd, &key, &val, 0)) {
fprintf(stderr, "Error prepopulating hashmap (key %d)\n", key);
exit(1);
}
}
}
static void __setup(struct bpf_program *prog, bool hashmap)
{
struct bpf_map *inner_map;
int i, fd, mim_fd, err;
LIBBPF_OPTS(bpf_map_create_opts, create_opts);
if (!hashmap)
create_opts.map_flags = BPF_F_NO_PREALLOC;
ctx.skel->rodata->num_maps = args.nr_maps;
ctx.skel->rodata->hashmap_num_keys = args.hashmap_nr_keys_used;
inner_map = bpf_map__inner_map(ctx.array_of_maps);
create_opts.btf_key_type_id = bpf_map__btf_key_type_id(inner_map);
create_opts.btf_value_type_id = bpf_map__btf_value_type_id(inner_map);
err = local_storage_bench__load(ctx.skel);
if (err) {
fprintf(stderr, "Error loading skeleton\n");
goto err_out;
}
create_opts.btf_fd = bpf_object__btf_fd(ctx.skel->obj);
mim_fd = bpf_map__fd(ctx.array_of_maps);
if (mim_fd < 0) {
fprintf(stderr, "Error getting map_in_map fd\n");
goto err_out;
}
for (i = 0; i < args.nr_maps; i++) {
if (hashmap)
fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(int),
sizeof(int), HASHMAP_SZ, &create_opts);
else
fd = bpf_map_create(BPF_MAP_TYPE_TASK_STORAGE, NULL, sizeof(int),
sizeof(int), 0, &create_opts);
if (fd < 0) {
fprintf(stderr, "Error creating map %d: %d\n", i, fd);
goto err_out;
}
if (hashmap)
prepopulate_hashmap(fd);
err = bpf_map_update_elem(mim_fd, &i, &fd, 0);
if (err) {
fprintf(stderr, "Error updating array-of-maps w/ map %d\n", i);
goto err_out;
}
}
if (!bpf_program__attach(prog)) {
fprintf(stderr, "Error attaching bpf program\n");
goto err_out;
}
return;
err_out:
exit(1);
}
static void hashmap_setup(void)
{
struct local_storage_bench *skel;
setup_libbpf();
skel = local_storage_bench__open();
ctx.skel = skel;
ctx.array_of_maps = skel->maps.array_of_hash_maps;
skel->rodata->use_hashmap = 1;
skel->rodata->interleave = 0;
__setup(skel->progs.get_local, true);
}
static void local_storage_cache_get_setup(void)
{
struct local_storage_bench *skel;
setup_libbpf();
skel = local_storage_bench__open();
ctx.skel = skel;
ctx.array_of_maps = skel->maps.array_of_local_storage_maps;
skel->rodata->use_hashmap = 0;
skel->rodata->interleave = 0;
__setup(skel->progs.get_local, false);
}
static void local_storage_cache_get_interleaved_setup(void)
{
struct local_storage_bench *skel;
setup_libbpf();
skel = local_storage_bench__open();
ctx.skel = skel;
ctx.array_of_maps = skel->maps.array_of_local_storage_maps;
skel->rodata->use_hashmap = 0;
skel->rodata->interleave = 1;
__setup(skel->progs.get_local, false);
}
static void measure(struct bench_res *res)
{
res->hits = atomic_swap(&ctx.skel->bss->hits, 0);
res->important_hits = atomic_swap(&ctx.skel->bss->important_hits, 0);
}
static inline void trigger_bpf_program(void)
{
syscall(__NR_getpgid);
}
static void *producer(void *input)
{
while (true)
trigger_bpf_program();
return NULL;
}
/* cache sequential and interleaved get benchs test local_storage get
* performance, specifically they demonstrate performance cliff of
* current list-plus-cache local_storage model.
*
* cache sequential get: call bpf_task_storage_get on n maps in order
* cache interleaved get: like "sequential get", but interleave 4 calls to the
* 'important' map (idx 0 in array_of_maps) for every 10 calls. Goal
* is to mimic environment where many progs are accessing their local_storage
* maps, with 'our' prog needing to access its map more often than others
*/
const struct bench bench_local_storage_cache_seq_get = {
.name = "local-storage-cache-seq-get",
.argp = &bench_local_storage_argp,
.validate = validate,
.setup = local_storage_cache_get_setup,
.producer_thread = producer,
.measure = measure,
.report_progress = local_storage_report_progress,
.report_final = local_storage_report_final,
};
const struct bench bench_local_storage_cache_interleaved_get = {
.name = "local-storage-cache-int-get",
.argp = &bench_local_storage_argp,
.validate = validate,
.setup = local_storage_cache_get_interleaved_setup,
.producer_thread = producer,
.measure = measure,
.report_progress = local_storage_report_progress,
.report_final = local_storage_report_final,
};
const struct bench bench_local_storage_cache_hashmap_control = {
.name = "local-storage-cache-hashmap-control",
.argp = &bench_local_storage_argp,
.validate = validate,
.setup = hashmap_setup,
.producer_thread = producer,
.measure = measure,
.report_progress = local_storage_report_progress,
.report_final = local_storage_report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_local_storage_create.c
+249
View File
@@ -0,0 +1,249 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */
#include <sys/types.h>
#include <sys/socket.h>
#include <pthread.h>
#include <argp.h>
#include "bench.h"
#include "bench_local_storage_create.skel.h"
struct thread {
int *fds;
pthread_t *pthds;
int *pthd_results;
};
static struct bench_local_storage_create *skel;
static struct thread *threads;
static long create_owner_errs;
static int storage_type = BPF_MAP_TYPE_SK_STORAGE;
static int batch_sz = 32;
enum {
ARG_BATCH_SZ = 9000,
ARG_STORAGE_TYPE = 9001,
};
static const struct argp_option opts[] = {
{ "batch-size", ARG_BATCH_SZ, "BATCH_SIZE", 0,
"The number of storage creations in each batch" },
{ "storage-type", ARG_STORAGE_TYPE, "STORAGE_TYPE", 0,
"The type of local storage to test (socket or task)" },
{},
};
static error_t parse_arg(int key, char *arg, struct argp_state *state)
{
int ret;
switch (key) {
case ARG_BATCH_SZ:
ret = atoi(arg);
if (ret < 1) {
fprintf(stderr, "invalid batch-size\n");
argp_usage(state);
}
batch_sz = ret;
break;
case ARG_STORAGE_TYPE:
if (!strcmp(arg, "task")) {
storage_type = BPF_MAP_TYPE_TASK_STORAGE;
} else if (!strcmp(arg, "socket")) {
storage_type = BPF_MAP_TYPE_SK_STORAGE;
} else {
fprintf(stderr, "invalid storage-type (socket or task)\n");
argp_usage(state);
}
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
const struct argp bench_local_storage_create_argp = {
.options = opts,
.parser = parse_arg,
};
static void validate(void)
{
if (env.consumer_cnt != 0) {
fprintf(stderr,
"local-storage-create benchmark does not need consumer\n");
exit(1);
}
}
static void setup(void)
{
int i;
skel = bench_local_storage_create__open_and_load();
if (!skel) {
fprintf(stderr, "error loading skel\n");
exit(1);
}
skel->bss->bench_pid = getpid();
if (storage_type == BPF_MAP_TYPE_SK_STORAGE) {
if (!bpf_program__attach(skel->progs.socket_post_create)) {
fprintf(stderr, "Error attaching bpf program\n");
exit(1);
}
} else {
if (!bpf_program__attach(skel->progs.sched_process_fork)) {
fprintf(stderr, "Error attaching bpf program\n");
exit(1);
}
}
threads = calloc(env.producer_cnt, sizeof(*threads));
if (!threads) {
fprintf(stderr, "cannot alloc thread_res\n");
exit(1);
}
for (i = 0; i < env.producer_cnt; i++) {
struct thread *t = &threads[i];
if (storage_type == BPF_MAP_TYPE_SK_STORAGE) {
t->fds = malloc(batch_sz * sizeof(*t->fds));
if (!t->fds) {
fprintf(stderr, "cannot alloc t->fds\n");
exit(1);
}
} else {
t->pthds = malloc(batch_sz * sizeof(*t->pthds));
if (!t->pthds) {
fprintf(stderr, "cannot alloc t->pthds\n");
exit(1);
}
t->pthd_results = malloc(batch_sz * sizeof(*t->pthd_results));
if (!t->pthd_results) {
fprintf(stderr, "cannot alloc t->pthd_results\n");
exit(1);
}
}
}
}
static void measure(struct bench_res *res)
{
res->hits = atomic_swap(&skel->bss->create_cnts, 0);
}
static void *sk_producer(void *input)
{
struct thread *t = &threads[(long)(input)];
int *fds = t->fds;
int i;
while (true) {
for (i = 0; i < batch_sz; i++) {
fds[i] = socket(AF_INET6, SOCK_DGRAM, 0);
if (fds[i] == -1)
atomic_inc(&create_owner_errs);
}
for (i = 0; i < batch_sz; i++) {
if (fds[i] != -1)
close(fds[i]);
}
}
return NULL;
}
static void *thread_func(void *arg)
{
return NULL;
}
static void *task_producer(void *input)
{
struct thread *t = &threads[(long)(input)];
pthread_t *pthds = t->pthds;
int *pthd_results = t->pthd_results;
int i;
while (true) {
for (i = 0; i < batch_sz; i++) {
pthd_results[i] = pthread_create(&pthds[i], NULL, thread_func, NULL);
if (pthd_results[i])
atomic_inc(&create_owner_errs);
}
for (i = 0; i < batch_sz; i++) {
if (!pthd_results[i])
pthread_join(pthds[i], NULL);
}
}
return NULL;
}
static void *producer(void *input)
{
if (storage_type == BPF_MAP_TYPE_SK_STORAGE)
return sk_producer(input);
else
return task_producer(input);
}
static void report_progress(int iter, struct bench_res *res, long delta_ns)
{
double creates_per_sec;
creates_per_sec = res->hits / 1000.0 / (delta_ns / 1000000000.0);
printf("Iter %3d (%7.3lfus): ",
iter, (delta_ns - 1000000000) / 1000.0);
printf("creates %8.3lfk/s (%7.3lfk/prod)\n",
creates_per_sec, creates_per_sec / env.producer_cnt);
}
static void report_final(struct bench_res res[], int res_cnt)
{
double creates_mean = 0.0, creates_stddev = 0.0;
long total_creates = 0;
int i;
for (i = 0; i < res_cnt; i++) {
creates_mean += res[i].hits / 1000.0 / (0.0 + res_cnt);
total_creates += res[i].hits;
}
if (res_cnt > 1) {
for (i = 0; i < res_cnt; i++)
creates_stddev += (creates_mean - res[i].hits / 1000.0) *
(creates_mean - res[i].hits / 1000.0) /
(res_cnt - 1.0);
creates_stddev = sqrt(creates_stddev);
}
printf("Summary: creates %8.3lf \u00B1 %5.3lfk/s (%7.3lfk/prod), %ld total\n",
creates_mean, creates_stddev, creates_mean / env.producer_cnt,
total_creates);
if (create_owner_errs || skel->bss->create_errs)
printf("%s() errors %ld create_errs %ld\n",
storage_type == BPF_MAP_TYPE_SK_STORAGE ?
"socket" : "pthread_create",
create_owner_errs,
skel->bss->create_errs);
}
/* Benchmark performance of creating bpf local storage */
const struct bench bench_local_storage_create = {
.name = "local-storage-create",
.argp = &bench_local_storage_create_argp,
.validate = validate,
.setup = setup,
.producer_thread = producer,
.measure = measure,
.report_progress = report_progress,
.report_final = report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_local_storage_rcu_tasks_trace.c
+263
View File
@@ -0,0 +1,263 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
#include <argp.h>
#include <sys/prctl.h>
#include "local_storage_rcu_tasks_trace_bench.skel.h"
#include "bench.h"
#include <signal.h>
static struct {
__u32 nr_procs;
__u32 kthread_pid;
} args = {
.nr_procs = 1000,
.kthread_pid = 0,
};
enum {
ARG_NR_PROCS = 7000,
ARG_KTHREAD_PID = 7001,
};
static const struct argp_option opts[] = {
{ "nr_procs", ARG_NR_PROCS, "NR_PROCS", 0,
"Set number of user processes to spin up"},
{ "kthread_pid", ARG_KTHREAD_PID, "PID", 0,
"Pid of rcu_tasks_trace kthread for ticks tracking"},
{},
};
static error_t parse_arg(int key, char *arg, struct argp_state *state)
{
long ret;
switch (key) {
case ARG_NR_PROCS:
ret = strtol(arg, NULL, 10);
if (ret < 1 || ret > UINT_MAX) {
fprintf(stderr, "invalid nr_procs\n");
argp_usage(state);
}
args.nr_procs = ret;
break;
case ARG_KTHREAD_PID:
ret = strtol(arg, NULL, 10);
if (ret < 1) {
fprintf(stderr, "invalid kthread_pid\n");
argp_usage(state);
}
args.kthread_pid = ret;
break;
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
const struct argp bench_local_storage_rcu_tasks_trace_argp = {
.options = opts,
.parser = parse_arg,
};
#define MAX_SLEEP_PROCS 150000
static void validate(void)
{
if (env.producer_cnt != 1) {
fprintf(stderr, "benchmark doesn't support multi-producer!\n");
exit(1);
}
if (env.consumer_cnt != 0) {
fprintf(stderr, "benchmark doesn't support consumer!\n");
exit(1);
}
if (args.nr_procs > MAX_SLEEP_PROCS) {
fprintf(stderr, "benchmark supports up to %u sleeper procs!\n",
MAX_SLEEP_PROCS);
exit(1);
}
}
static long kthread_pid_ticks(void)
{
char procfs_path[100];
long stime;
FILE *f;
if (!args.kthread_pid)
return -1;
sprintf(procfs_path, "/proc/%u/stat", args.kthread_pid);
f = fopen(procfs_path, "r");
if (!f) {
fprintf(stderr, "couldn't open %s, exiting\n", procfs_path);
goto err_out;
}
if (fscanf(f, "%*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %ld", &stime) != 1) {
fprintf(stderr, "fscanf of %s failed, exiting\n", procfs_path);
goto err_out;
}
fclose(f);
return stime;
err_out:
if (f)
fclose(f);
exit(1);
return 0;
}
static struct {
struct local_storage_rcu_tasks_trace_bench *skel;
long prev_kthread_stime;
} ctx;
static void sleep_and_loop(void)
{
while (true) {
sleep(rand() % 4);
syscall(__NR_getpgid);
}
}
static void local_storage_tasks_trace_setup(void)
{
int i, err, forkret, runner_pid;
runner_pid = getpid();
for (i = 0; i < args.nr_procs; i++) {
forkret = fork();
if (forkret < 0) {
fprintf(stderr, "Error forking sleeper proc %u of %u, exiting\n", i,
args.nr_procs);
goto err_out;
}
if (!forkret) {
err = prctl(PR_SET_PDEATHSIG, SIGKILL);
if (err < 0) {
fprintf(stderr, "prctl failed with err %d, exiting\n", errno);
goto err_out;
}
if (getppid() != runner_pid) {
fprintf(stderr, "Runner died while spinning up procs, exiting\n");
goto err_out;
}
sleep_and_loop();
}
}
printf("Spun up %u procs (our pid %d)\n", args.nr_procs, runner_pid);
setup_libbpf();
ctx.skel = local_storage_rcu_tasks_trace_bench__open_and_load();
if (!ctx.skel) {
fprintf(stderr, "Error doing open_and_load, exiting\n");
goto err_out;
}
ctx.prev_kthread_stime = kthread_pid_ticks();
if (!bpf_program__attach(ctx.skel->progs.get_local)) {
fprintf(stderr, "Error attaching bpf program\n");
goto err_out;
}
if (!bpf_program__attach(ctx.skel->progs.pregp_step)) {
fprintf(stderr, "Error attaching bpf program\n");
goto err_out;
}
if (!bpf_program__attach(ctx.skel->progs.postgp)) {
fprintf(stderr, "Error attaching bpf program\n");
goto err_out;
}
return;
err_out:
exit(1);
}
static void measure(struct bench_res *res)
{
long ticks;
res->gp_ct = atomic_swap(&ctx.skel->bss->gp_hits, 0);
res->gp_ns = atomic_swap(&ctx.skel->bss->gp_times, 0);
ticks = kthread_pid_ticks();
res->stime = ticks - ctx.prev_kthread_stime;
ctx.prev_kthread_stime = ticks;
}
static void *producer(void *input)
{
while (true)
syscall(__NR_getpgid);
return NULL;
}
static void report_progress(int iter, struct bench_res *res, long delta_ns)
{
if (ctx.skel->bss->unexpected) {
fprintf(stderr, "Error: Unexpected order of bpf prog calls (postgp after pregp).");
fprintf(stderr, "Data can't be trusted, exiting\n");
exit(1);
}
if (env.quiet)
return;
printf("Iter %d\t avg tasks_trace grace period latency\t%lf ns\n",
iter, res->gp_ns / (double)res->gp_ct);
printf("Iter %d\t avg ticks per tasks_trace grace period\t%lf\n",
iter, res->stime / (double)res->gp_ct);
}
static void report_final(struct bench_res res[], int res_cnt)
{
struct basic_stats gp_stat;
grace_period_latency_basic_stats(res, res_cnt, &gp_stat);
printf("SUMMARY tasks_trace grace period latency");
printf("\tavg %.3lf us\tstddev %.3lf us\n", gp_stat.mean, gp_stat.stddev);
grace_period_ticks_basic_stats(res, res_cnt, &gp_stat);
printf("SUMMARY ticks per tasks_trace grace period");
printf("\tavg %.3lf\tstddev %.3lf\n", gp_stat.mean, gp_stat.stddev);
}
/* local-storage-tasks-trace: Benchmark performance of BPF local_storage's use
* of RCU Tasks-Trace.
*
* Stress RCU Tasks Trace by forking many tasks, all of which do no work aside
* from sleep() loop, and creating/destroying BPF task-local storage on wakeup.
* The number of forked tasks is configurable.
*
* exercising code paths which call call_rcu_tasks_trace while there are many
* thousands of tasks on the system should result in RCU Tasks-Trace having to
* do a noticeable amount of work.
*
* This should be observable by measuring rcu_tasks_trace_kthread CPU usage
* after the grace period has ended, or by measuring grace period latency.
*
* This benchmark uses both approaches, attaching to rcu_tasks_trace_pregp_step
* and rcu_tasks_trace_postgp functions to measure grace period latency and
* using /proc/PID/stat to measure rcu_tasks_trace_kthread kernel ticks
*/
const struct bench bench_local_storage_tasks_trace = {
.name = "local-storage-tasks-trace",
.argp = &bench_local_storage_rcu_tasks_trace_argp,
.validate = validate,
.setup = local_storage_tasks_trace_setup,
.producer_thread = producer,
.measure = measure,
.report_progress = report_progress,
.report_final = report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c
+555
View File
@@ -0,0 +1,555 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Cloudflare */
/*
* All of these benchmarks operate on tries with keys in the range
* [0, args.nr_entries), i.e. there are no gaps or partially filled
* branches of the trie for any key < args.nr_entries.
*
* This gives an idea of worst-case behaviour.
*/
#include <argp.h>
#include <linux/time64.h>
#include <linux/if_ether.h>
#include "lpm_trie_bench.skel.h"
#include "lpm_trie_map.skel.h"
#include "bench.h"
#include "testing_helpers.h"
#include "progs/lpm_trie.h"
static struct ctx {
struct lpm_trie_bench *bench;
} ctx;
static struct {
__u32 nr_entries;
__u32 prefixlen;
bool random;
} args = {
.nr_entries = 0,
.prefixlen = 32,
.random = false,
};
enum {
ARG_NR_ENTRIES = 9000,
ARG_PREFIX_LEN,
ARG_RANDOM,
};
static const struct argp_option opts[] = {
{ "nr_entries", ARG_NR_ENTRIES, "NR_ENTRIES", 0,
"Number of unique entries in the LPM trie" },
{ "prefix_len", ARG_PREFIX_LEN, "PREFIX_LEN", 0,
"Number of prefix bits to use in the LPM trie" },
{ "random", ARG_RANDOM, NULL, 0, "Access random keys during op" },
{},
};
static error_t lpm_parse_arg(int key, char *arg, struct argp_state *state)
{
long ret;
switch (key) {
case ARG_NR_ENTRIES:
ret = strtol(arg, NULL, 10);
if (ret < 1 || ret > UINT_MAX) {
fprintf(stderr, "Invalid nr_entries count.");
argp_usage(state);
}
args.nr_entries = ret;
break;
case ARG_PREFIX_LEN:
ret = strtol(arg, NULL, 10);
if (ret < 1 || ret > UINT_MAX) {
fprintf(stderr, "Invalid prefix_len value.");
argp_usage(state);
}
args.prefixlen = ret;
break;
case ARG_RANDOM:
args.random = true;
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
const struct argp bench_lpm_trie_map_argp = {
.options = opts,
.parser = lpm_parse_arg,
};
static void validate_common(void)
{
if (env.consumer_cnt != 0) {
fprintf(stderr, "benchmark doesn't support consumer\n");
exit(1);
}
if (args.nr_entries == 0) {
fprintf(stderr, "Missing --nr_entries parameter\n");
exit(1);
}
if ((1UL << args.prefixlen) < args.nr_entries) {
fprintf(stderr, "prefix_len value too small for nr_entries\n");
exit(1);
}
}
static void lpm_insert_validate(void)
{
validate_common();
if (env.producer_cnt != 1) {
fprintf(stderr, "lpm-trie-insert requires a single producer\n");
exit(1);
}
if (args.random) {
fprintf(stderr, "lpm-trie-insert does not support --random\n");
exit(1);
}
}
static void lpm_delete_validate(void)
{
validate_common();
if (env.producer_cnt != 1) {
fprintf(stderr, "lpm-trie-delete requires a single producer\n");
exit(1);
}
if (args.random) {
fprintf(stderr, "lpm-trie-delete does not support --random\n");
exit(1);
}
}
static void lpm_free_validate(void)
{
validate_common();
if (env.producer_cnt != 1) {
fprintf(stderr, "lpm-trie-free requires a single producer\n");
exit(1);
}
if (args.random) {
fprintf(stderr, "lpm-trie-free does not support --random\n");
exit(1);
}
}
static struct trie_key *keys;
static __u32 *vals;
static void fill_map(int map_fd)
{
int err;
DECLARE_LIBBPF_OPTS(bpf_map_batch_opts, opts,
.elem_flags = 0,
.flags = 0,
);
err = bpf_map_update_batch(map_fd, keys, vals, &args.nr_entries, &opts);
if (err) {
fprintf(stderr, "failed to batch update keys to map: %d\n",
-err);
exit(1);
}
}
static void empty_map(int map_fd)
{
int err;
DECLARE_LIBBPF_OPTS(bpf_map_batch_opts, opts,
.elem_flags = 0,
.flags = 0,
);
err = bpf_map_delete_batch(map_fd, keys, &args.nr_entries, &opts);
if (err) {
fprintf(stderr, "failed to batch delete keys for map: %d\n",
-err);
exit(1);
}
}
static void attach_prog(void)
{
int i;
ctx.bench = lpm_trie_bench__open_and_load();
if (!ctx.bench) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
ctx.bench->bss->nr_entries = args.nr_entries;
ctx.bench->bss->prefixlen = args.prefixlen;
ctx.bench->bss->random = args.random;
if (lpm_trie_bench__attach(ctx.bench)) {
fprintf(stderr, "failed to attach skeleton\n");
exit(1);
}
keys = calloc(args.nr_entries, sizeof(*keys));
vals = calloc(args.nr_entries, sizeof(*vals));
for (i = 0; i < args.nr_entries; i++) {
struct trie_key *k = &keys[i];
__u32 *v = &vals[i];
k->prefixlen = args.prefixlen;
k->data = i;
*v = 1;
}
}
static void attach_prog_and_fill_map(void)
{
int fd;
attach_prog();
fd = bpf_map__fd(ctx.bench->maps.trie_map);
fill_map(fd);
}
static void lpm_noop_setup(void)
{
attach_prog();
ctx.bench->bss->op = LPM_OP_NOOP;
}
static void lpm_baseline_setup(void)
{
attach_prog();
ctx.bench->bss->op = LPM_OP_BASELINE;
}
static void lpm_lookup_setup(void)
{
attach_prog_and_fill_map();
ctx.bench->bss->op = LPM_OP_LOOKUP;
}
static void lpm_insert_setup(void)
{
attach_prog();
ctx.bench->bss->op = LPM_OP_INSERT;
}
static void lpm_update_setup(void)
{
attach_prog_and_fill_map();
ctx.bench->bss->op = LPM_OP_UPDATE;
}
static void lpm_delete_setup(void)
{
attach_prog_and_fill_map();
ctx.bench->bss->op = LPM_OP_DELETE;
}
static void lpm_free_setup(void)
{
attach_prog();
ctx.bench->bss->op = LPM_OP_FREE;
}
static void lpm_measure(struct bench_res *res)
{
res->hits = atomic_swap(&ctx.bench->bss->hits, 0);
res->duration_ns = atomic_swap(&ctx.bench->bss->duration_ns, 0);
}
static void bench_reinit_map(void)
{
int fd = bpf_map__fd(ctx.bench->maps.trie_map);
switch (ctx.bench->bss->op) {
case LPM_OP_INSERT:
/* trie_map needs to be emptied */
empty_map(fd);
break;
case LPM_OP_DELETE:
/* trie_map needs to be refilled */
fill_map(fd);
break;
default:
fprintf(stderr, "Unexpected REINIT return code for op %d\n",
ctx.bench->bss->op);
exit(1);
}
}
/* For NOOP, BASELINE, LOOKUP, INSERT, UPDATE, and DELETE */
static void *lpm_producer(void *unused __always_unused)
{
int err;
char in[ETH_HLEN]; /* unused */
LIBBPF_OPTS(bpf_test_run_opts, opts, .data_in = in,
.data_size_in = sizeof(in), .repeat = 1, );
while (true) {
int fd = bpf_program__fd(ctx.bench->progs.run_bench);
err = bpf_prog_test_run_opts(fd, &opts);
if (err) {
fprintf(stderr, "failed to run BPF prog: %d\n", err);
exit(1);
}
/* Check for kernel error code */
if ((int)opts.retval < 0) {
fprintf(stderr, "BPF prog returned error: %d\n",
opts.retval);
exit(1);
}
switch (opts.retval) {
case LPM_BENCH_SUCCESS:
break;
case LPM_BENCH_REINIT_MAP:
bench_reinit_map();
break;
default:
fprintf(stderr, "Unexpected BPF prog return code %d for op %d\n",
opts.retval, ctx.bench->bss->op);
exit(1);
}
}
return NULL;
}
static void *lpm_free_producer(void *unused __always_unused)
{
while (true) {
struct lpm_trie_map *skel;
skel = lpm_trie_map__open_and_load();
if (!skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
fill_map(bpf_map__fd(skel->maps.trie_free_map));
lpm_trie_map__destroy(skel);
}
return NULL;
}
/*
* The standard bench op_report_*() functions assume measurements are
* taken over a 1-second interval but operations that modify the map
* (INSERT, DELETE, and FREE) cannot run indefinitely without
* "resetting" the map to the initial state. Depending on the size of
* the map, this likely needs to happen before the 1-second timer fires.
*
* Calculate the fraction of a second over which the op measurement was
* taken (to ignore any time spent doing the reset) and report the
* throughput results per second.
*/
static void frac_second_report_progress(int iter, struct bench_res *res,
long delta_ns, double rate_divisor,
char rate)
{
double hits_per_sec, hits_per_prod;
hits_per_sec = res->hits / rate_divisor /
(res->duration_ns / (double)NSEC_PER_SEC);
hits_per_prod = hits_per_sec / env.producer_cnt;
printf("Iter %3d (%7.3lfus): ", iter,
(delta_ns - NSEC_PER_SEC) / 1000.0);
printf("hits %8.3lf%c/s (%7.3lf%c/prod)\n", hits_per_sec, rate,
hits_per_prod, rate);
}
static void frac_second_report_final(struct bench_res res[], int res_cnt,
double lat_divisor, double rate_divisor,
char rate, const char *unit)
{
double hits_mean = 0.0, hits_stddev = 0.0;
double latency = 0.0;
int i;
for (i = 0; i < res_cnt; i++) {
double val = res[i].hits / rate_divisor /
(res[i].duration_ns / (double)NSEC_PER_SEC);
hits_mean += val / (0.0 + res_cnt);
latency += res[i].duration_ns / res[i].hits / (0.0 + res_cnt);
}
if (res_cnt > 1) {
for (i = 0; i < res_cnt; i++) {
double val =
res[i].hits / rate_divisor /
(res[i].duration_ns / (double)NSEC_PER_SEC);
hits_stddev += (hits_mean - val) * (hits_mean - val) /
(res_cnt - 1.0);
}
hits_stddev = sqrt(hits_stddev);
}
printf("Summary: throughput %8.3lf \u00B1 %5.3lf %c ops/s (%7.3lf%c ops/prod), ",
hits_mean, hits_stddev, rate, hits_mean / env.producer_cnt,
rate);
printf("latency %8.3lf %s/op\n",
latency / lat_divisor / env.producer_cnt, unit);
}
static void insert_ops_report_progress(int iter, struct bench_res *res,
long delta_ns)
{
double rate_divisor = 1000000.0;
char rate = 'M';
frac_second_report_progress(iter, res, delta_ns, rate_divisor, rate);
}
static void delete_ops_report_progress(int iter, struct bench_res *res,
long delta_ns)
{
double rate_divisor = 1000000.0;
char rate = 'M';
frac_second_report_progress(iter, res, delta_ns, rate_divisor, rate);
}
static void free_ops_report_progress(int iter, struct bench_res *res,
long delta_ns)
{
double rate_divisor = 1000.0;
char rate = 'K';
frac_second_report_progress(iter, res, delta_ns, rate_divisor, rate);
}
static void insert_ops_report_final(struct bench_res res[], int res_cnt)
{
double lat_divisor = 1.0;
double rate_divisor = 1000000.0;
const char *unit = "ns";
char rate = 'M';
frac_second_report_final(res, res_cnt, lat_divisor, rate_divisor, rate,
unit);
}
static void delete_ops_report_final(struct bench_res res[], int res_cnt)
{
double lat_divisor = 1.0;
double rate_divisor = 1000000.0;
const char *unit = "ns";
char rate = 'M';
frac_second_report_final(res, res_cnt, lat_divisor, rate_divisor, rate,
unit);
}
static void free_ops_report_final(struct bench_res res[], int res_cnt)
{
double lat_divisor = 1000000.0;
double rate_divisor = 1000.0;
const char *unit = "ms";
char rate = 'K';
frac_second_report_final(res, res_cnt, lat_divisor, rate_divisor, rate,
unit);
}
/* noop bench measures harness-overhead */
const struct bench bench_lpm_trie_noop = {
.name = "lpm-trie-noop",
.argp = &bench_lpm_trie_map_argp,
.validate = validate_common,
.setup = lpm_noop_setup,
.producer_thread = lpm_producer,
.measure = lpm_measure,
.report_progress = ops_report_progress,
.report_final = ops_report_final,
};
/* baseline overhead for lookup and update */
const struct bench bench_lpm_trie_baseline = {
.name = "lpm-trie-baseline",
.argp = &bench_lpm_trie_map_argp,
.validate = validate_common,
.setup = lpm_baseline_setup,
.producer_thread = lpm_producer,
.measure = lpm_measure,
.report_progress = ops_report_progress,
.report_final = ops_report_final,
};
/* measure cost of doing a lookup on existing entries in a full trie */
const struct bench bench_lpm_trie_lookup = {
.name = "lpm-trie-lookup",
.argp = &bench_lpm_trie_map_argp,
.validate = validate_common,
.setup = lpm_lookup_setup,
.producer_thread = lpm_producer,
.measure = lpm_measure,
.report_progress = ops_report_progress,
.report_final = ops_report_final,
};
/* measure cost of inserting new entries into an empty trie */
const struct bench bench_lpm_trie_insert = {
.name = "lpm-trie-insert",
.argp = &bench_lpm_trie_map_argp,
.validate = lpm_insert_validate,
.setup = lpm_insert_setup,
.producer_thread = lpm_producer,
.measure = lpm_measure,
.report_progress = insert_ops_report_progress,
.report_final = insert_ops_report_final,
};
/* measure cost of updating existing entries in a full trie */
const struct bench bench_lpm_trie_update = {
.name = "lpm-trie-update",
.argp = &bench_lpm_trie_map_argp,
.validate = validate_common,
.setup = lpm_update_setup,
.producer_thread = lpm_producer,
.measure = lpm_measure,
.report_progress = ops_report_progress,
.report_final = ops_report_final,
};
/* measure cost of deleting existing entries from a full trie */
const struct bench bench_lpm_trie_delete = {
.name = "lpm-trie-delete",
.argp = &bench_lpm_trie_map_argp,
.validate = lpm_delete_validate,
.setup = lpm_delete_setup,
.producer_thread = lpm_producer,
.measure = lpm_measure,
.report_progress = delete_ops_report_progress,
.report_final = delete_ops_report_final,
};
/* measure cost of freeing a full trie */
const struct bench bench_lpm_trie_free = {
.name = "lpm-trie-free",
.argp = &bench_lpm_trie_map_argp,
.validate = lpm_free_validate,
.setup = lpm_free_setup,
.producer_thread = lpm_free_producer,
.measure = lpm_measure,
.report_progress = free_ops_report_progress,
.report_final = free_ops_report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_rename.c
+167
View File
@@ -0,0 +1,167 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <fcntl.h>
#include "bench.h"
#include "test_overhead.skel.h"
/* BPF triggering benchmarks */
static struct ctx {
struct test_overhead *skel;
struct counter hits;
int fd;
} ctx;
static void validate(void)
{
if (env.producer_cnt != 1) {
fprintf(stderr, "benchmark doesn't support multi-producer!\n");
exit(1);
}
if (env.consumer_cnt != 0) {
fprintf(stderr, "benchmark doesn't support consumer!\n");
exit(1);
}
}
static void *producer(void *input)
{
char buf[] = "test_overhead";
int err;
while (true) {
err = write(ctx.fd, buf, sizeof(buf));
if (err < 0) {
fprintf(stderr, "write failed\n");
exit(1);
}
atomic_inc(&ctx.hits.value);
}
}
static void measure(struct bench_res *res)
{
res->hits = atomic_swap(&ctx.hits.value, 0);
}
static void setup_ctx(void)
{
setup_libbpf();
ctx.skel = test_overhead__open_and_load();
if (!ctx.skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
ctx.fd = open("/proc/self/comm", O_WRONLY|O_TRUNC);
if (ctx.fd < 0) {
fprintf(stderr, "failed to open /proc/self/comm: %d\n", -errno);
exit(1);
}
}
static void attach_bpf(struct bpf_program *prog)
{
struct bpf_link *link;
link = bpf_program__attach(prog);
if (!link) {
fprintf(stderr, "failed to attach program!\n");
exit(1);
}
}
static void setup_base(void)
{
setup_ctx();
}
static void setup_kprobe(void)
{
setup_ctx();
attach_bpf(ctx.skel->progs.prog1);
}
static void setup_kretprobe(void)
{
setup_ctx();
attach_bpf(ctx.skel->progs.prog2);
}
static void setup_rawtp(void)
{
setup_ctx();
attach_bpf(ctx.skel->progs.prog3);
}
static void setup_fentry(void)
{
setup_ctx();
attach_bpf(ctx.skel->progs.prog4);
}
static void setup_fexit(void)
{
setup_ctx();
attach_bpf(ctx.skel->progs.prog5);
}
const struct bench bench_rename_base = {
.name = "rename-base",
.validate = validate,
.setup = setup_base,
.producer_thread = producer,
.measure = measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_rename_kprobe = {
.name = "rename-kprobe",
.validate = validate,
.setup = setup_kprobe,
.producer_thread = producer,
.measure = measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_rename_kretprobe = {
.name = "rename-kretprobe",
.validate = validate,
.setup = setup_kretprobe,
.producer_thread = producer,
.measure = measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_rename_rawtp = {
.name = "rename-rawtp",
.validate = validate,
.setup = setup_rawtp,
.producer_thread = producer,
.measure = measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_rename_fentry = {
.name = "rename-fentry",
.validate = validate,
.setup = setup_fentry,
.producer_thread = producer,
.measure = measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_rename_fexit = {
.name = "rename-fexit",
.validate = validate,
.setup = setup_fexit,
.producer_thread = producer,
.measure = measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_ringbufs.c
+619
View File
@@ -0,0 +1,619 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <asm/barrier.h>
#include <linux/perf_event.h>
#include <linux/ring_buffer.h>
#include <sys/epoll.h>
#include <sys/mman.h>
#include <argp.h>
#include <stdlib.h>
#include "bench.h"
#include "ringbuf_bench.skel.h"
#include "perfbuf_bench.skel.h"
static struct {
bool back2back;
int batch_cnt;
bool sampled;
int sample_rate;
int ringbuf_sz; /* per-ringbuf, in bytes */
bool ringbuf_use_output; /* use slower output API */
int perfbuf_sz; /* per-CPU size, in pages */
bool overwrite;
bool bench_producer;
} args = {
.back2back = false,
.batch_cnt = 500,
.sampled = false,
.sample_rate = 500,
.ringbuf_sz = 512 * 1024,
.ringbuf_use_output = false,
.perfbuf_sz = 128,
.overwrite = false,
.bench_producer = false,
};
enum {
ARG_RB_BACK2BACK = 2000,
ARG_RB_USE_OUTPUT = 2001,
ARG_RB_BATCH_CNT = 2002,
ARG_RB_SAMPLED = 2003,
ARG_RB_SAMPLE_RATE = 2004,
ARG_RB_OVERWRITE = 2005,
ARG_RB_BENCH_PRODUCER = 2006,
};
static const struct argp_option opts[] = {
{ "rb-b2b", ARG_RB_BACK2BACK, NULL, 0, "Back-to-back mode"},
{ "rb-use-output", ARG_RB_USE_OUTPUT, NULL, 0, "Use bpf_ringbuf_output() instead of bpf_ringbuf_reserve()"},
{ "rb-batch-cnt", ARG_RB_BATCH_CNT, "CNT", 0, "Set BPF-side record batch count"},
{ "rb-sampled", ARG_RB_SAMPLED, NULL, 0, "Notification sampling"},
{ "rb-sample-rate", ARG_RB_SAMPLE_RATE, "RATE", 0, "Notification sample rate"},
{ "rb-overwrite", ARG_RB_OVERWRITE, NULL, 0, "Overwrite mode"},
{ "rb-bench-producer", ARG_RB_BENCH_PRODUCER, NULL, 0, "Benchmark producer"},
{},
};
static error_t parse_arg(int key, char *arg, struct argp_state *state)
{
switch (key) {
case ARG_RB_BACK2BACK:
args.back2back = true;
break;
case ARG_RB_USE_OUTPUT:
args.ringbuf_use_output = true;
break;
case ARG_RB_BATCH_CNT:
args.batch_cnt = strtol(arg, NULL, 10);
if (args.batch_cnt < 0) {
fprintf(stderr, "Invalid batch count.");
argp_usage(state);
}
break;
case ARG_RB_SAMPLED:
args.sampled = true;
break;
case ARG_RB_SAMPLE_RATE:
args.sample_rate = strtol(arg, NULL, 10);
if (args.sample_rate < 0) {
fprintf(stderr, "Invalid perfbuf sample rate.");
argp_usage(state);
}
break;
case ARG_RB_OVERWRITE:
args.overwrite = true;
break;
case ARG_RB_BENCH_PRODUCER:
args.bench_producer = true;
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
/* exported into benchmark runner */
const struct argp bench_ringbufs_argp = {
.options = opts,
.parser = parse_arg,
};
/* RINGBUF-LIBBPF benchmark */
static struct counter buf_hits;
static inline void bufs_trigger_batch(void)
{
(void)syscall(__NR_getpgid);
}
static void bufs_validate(void)
{
if (args.bench_producer && strcmp(env.bench_name, "rb-libbpf")) {
fprintf(stderr, "--rb-bench-producer only works with rb-libbpf!\n");
exit(1);
}
if (args.overwrite && !args.bench_producer) {
fprintf(stderr, "overwrite mode only works with --rb-bench-producer for now!\n");
exit(1);
}
if (args.bench_producer && env.consumer_cnt != 0) {
fprintf(stderr, "no consumer is needed for --rb-bench-producer!\n");
exit(1);
}
if (args.bench_producer && args.back2back) {
fprintf(stderr, "back-to-back mode makes no sense for --rb-bench-producer!\n");
exit(1);
}
if (args.bench_producer && args.sampled) {
fprintf(stderr, "sampling mode makes no sense for --rb-bench-producer!\n");
exit(1);
}
if (!args.bench_producer && env.consumer_cnt != 1) {
fprintf(stderr, "benchmarks without --rb-bench-producer require exactly one consumer!\n");
exit(1);
}
if (args.back2back && env.producer_cnt > 1) {
fprintf(stderr, "back-to-back mode makes sense only for single-producer case!\n");
exit(1);
}
}
static void *bufs_sample_producer(void *input)
{
if (args.back2back) {
/* initial batch to get everything started */
bufs_trigger_batch();
return NULL;
}
while (true)
bufs_trigger_batch();
return NULL;
}
static struct ringbuf_libbpf_ctx {
struct ringbuf_bench *skel;
struct ring_buffer *ringbuf;
} ringbuf_libbpf_ctx;
static void ringbuf_libbpf_measure(struct bench_res *res)
{
struct ringbuf_libbpf_ctx *ctx = &ringbuf_libbpf_ctx;
if (args.bench_producer)
res->hits = atomic_swap(&ctx->skel->bss->hits, 0);
else
res->hits = atomic_swap(&buf_hits.value, 0);
res->drops = atomic_swap(&ctx->skel->bss->dropped, 0);
}
static struct ringbuf_bench *ringbuf_setup_skeleton(void)
{
__u32 flags;
struct bpf_map *ringbuf;
struct ringbuf_bench *skel;
setup_libbpf();
skel = ringbuf_bench__open();
if (!skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
skel->rodata->batch_cnt = args.batch_cnt;
skel->rodata->use_output = args.ringbuf_use_output ? 1 : 0;
skel->rodata->bench_producer = args.bench_producer;
if (args.sampled)
/* record data + header take 16 bytes */
skel->rodata->wakeup_data_size = args.sample_rate * 16;
ringbuf = skel->maps.ringbuf;
if (args.overwrite) {
flags = bpf_map__map_flags(ringbuf) | BPF_F_RB_OVERWRITE;
bpf_map__set_map_flags(ringbuf, flags);
}
bpf_map__set_max_entries(ringbuf, args.ringbuf_sz);
if (ringbuf_bench__load(skel)) {
fprintf(stderr, "failed to load skeleton\n");
exit(1);
}
return skel;
}
static int buf_process_sample(void *ctx, void *data, size_t len)
{
atomic_inc(&buf_hits.value);
return 0;
}
static void ringbuf_libbpf_setup(void)
{
struct ringbuf_libbpf_ctx *ctx = &ringbuf_libbpf_ctx;
struct bpf_link *link;
int map_fd;
ctx->skel = ringbuf_setup_skeleton();
map_fd = bpf_map__fd(ctx->skel->maps.ringbuf);
ctx->ringbuf = ring_buffer__new(map_fd, buf_process_sample, NULL, NULL);
if (!ctx->ringbuf) {
fprintf(stderr, "failed to create ringbuf\n");
exit(1);
}
link = bpf_program__attach(ctx->skel->progs.bench_ringbuf);
if (!link) {
fprintf(stderr, "failed to attach program!\n");
exit(1);
}
}
static void *ringbuf_libbpf_consumer(void *input)
{
struct ringbuf_libbpf_ctx *ctx = &ringbuf_libbpf_ctx;
while (ring_buffer__poll(ctx->ringbuf, -1) >= 0) {
if (args.back2back)
bufs_trigger_batch();
}
fprintf(stderr, "ringbuf polling failed!\n");
return NULL;
}
/* RINGBUF-CUSTOM benchmark */
struct ringbuf_custom {
__u64 *consumer_pos;
__u64 *producer_pos;
__u64 mask;
void *data;
int map_fd;
};
static struct ringbuf_custom_ctx {
struct ringbuf_bench *skel;
struct ringbuf_custom ringbuf;
int epoll_fd;
struct epoll_event event;
} ringbuf_custom_ctx;
static void ringbuf_custom_measure(struct bench_res *res)
{
struct ringbuf_custom_ctx *ctx = &ringbuf_custom_ctx;
res->hits = atomic_swap(&buf_hits.value, 0);
res->drops = atomic_swap(&ctx->skel->bss->dropped, 0);
}
static void ringbuf_custom_setup(void)
{
struct ringbuf_custom_ctx *ctx = &ringbuf_custom_ctx;
const size_t page_size = getpagesize();
struct bpf_link *link;
struct ringbuf_custom *r;
void *tmp;
int err;
ctx->skel = ringbuf_setup_skeleton();
ctx->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
if (ctx->epoll_fd < 0) {
fprintf(stderr, "failed to create epoll fd: %d\n", -errno);
exit(1);
}
r = &ctx->ringbuf;
r->map_fd = bpf_map__fd(ctx->skel->maps.ringbuf);
r->mask = args.ringbuf_sz - 1;
/* Map writable consumer page */
tmp = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
r->map_fd, 0);
if (tmp == MAP_FAILED) {
fprintf(stderr, "failed to mmap consumer page: %d\n", -errno);
exit(1);
}
r->consumer_pos = tmp;
/* Map read-only producer page and data pages. */
tmp = mmap(NULL, page_size + 2 * args.ringbuf_sz, PROT_READ, MAP_SHARED,
r->map_fd, page_size);
if (tmp == MAP_FAILED) {
fprintf(stderr, "failed to mmap data pages: %d\n", -errno);
exit(1);
}
r->producer_pos = tmp;
r->data = tmp + page_size;
ctx->event.events = EPOLLIN;
err = epoll_ctl(ctx->epoll_fd, EPOLL_CTL_ADD, r->map_fd, &ctx->event);
if (err < 0) {
fprintf(stderr, "failed to epoll add ringbuf: %d\n", -errno);
exit(1);
}
link = bpf_program__attach(ctx->skel->progs.bench_ringbuf);
if (!link) {
fprintf(stderr, "failed to attach program\n");
exit(1);
}
}
#define RINGBUF_BUSY_BIT (1 << 31)
#define RINGBUF_DISCARD_BIT (1 << 30)
#define RINGBUF_META_LEN 8
static inline int roundup_len(__u32 len)
{
/* clear out top 2 bits */
len <<= 2;
len >>= 2;
/* add length prefix */
len += RINGBUF_META_LEN;
/* round up to 8 byte alignment */
return (len + 7) / 8 * 8;
}
static void ringbuf_custom_process_ring(struct ringbuf_custom *r)
{
unsigned long cons_pos, prod_pos;
int *len_ptr, len;
bool got_new_data;
cons_pos = smp_load_acquire(r->consumer_pos);
while (true) {
got_new_data = false;
prod_pos = smp_load_acquire(r->producer_pos);
while (cons_pos < prod_pos) {
len_ptr = r->data + (cons_pos & r->mask);
len = smp_load_acquire(len_ptr);
/* sample not committed yet, bail out for now */
if (len & RINGBUF_BUSY_BIT)
return;
got_new_data = true;
cons_pos += roundup_len(len);
atomic_inc(&buf_hits.value);
}
if (got_new_data)
smp_store_release(r->consumer_pos, cons_pos);
else
break;
}
}
static void *ringbuf_custom_consumer(void *input)
{
struct ringbuf_custom_ctx *ctx = &ringbuf_custom_ctx;
int cnt;
do {
if (args.back2back)
bufs_trigger_batch();
cnt = epoll_wait(ctx->epoll_fd, &ctx->event, 1, -1);
if (cnt > 0)
ringbuf_custom_process_ring(&ctx->ringbuf);
} while (cnt >= 0);
fprintf(stderr, "ringbuf polling failed!\n");
return 0;
}
/* PERFBUF-LIBBPF benchmark */
static struct perfbuf_libbpf_ctx {
struct perfbuf_bench *skel;
struct perf_buffer *perfbuf;
} perfbuf_libbpf_ctx;
static void perfbuf_measure(struct bench_res *res)
{
struct perfbuf_libbpf_ctx *ctx = &perfbuf_libbpf_ctx;
res->hits = atomic_swap(&buf_hits.value, 0);
res->drops = atomic_swap(&ctx->skel->bss->dropped, 0);
}
static struct perfbuf_bench *perfbuf_setup_skeleton(void)
{
struct perfbuf_bench *skel;
setup_libbpf();
skel = perfbuf_bench__open();
if (!skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
skel->rodata->batch_cnt = args.batch_cnt;
if (perfbuf_bench__load(skel)) {
fprintf(stderr, "failed to load skeleton\n");
exit(1);
}
return skel;
}
static enum bpf_perf_event_ret
perfbuf_process_sample_raw(void *input_ctx, int cpu,
struct perf_event_header *e)
{
switch (e->type) {
case PERF_RECORD_SAMPLE:
atomic_inc(&buf_hits.value);
break;
case PERF_RECORD_LOST:
break;
default:
return LIBBPF_PERF_EVENT_ERROR;
}
return LIBBPF_PERF_EVENT_CONT;
}
static void perfbuf_libbpf_setup(void)
{
struct perfbuf_libbpf_ctx *ctx = &perfbuf_libbpf_ctx;
struct perf_event_attr attr;
struct bpf_link *link;
ctx->skel = perfbuf_setup_skeleton();
memset(&attr, 0, sizeof(attr));
attr.config = PERF_COUNT_SW_BPF_OUTPUT;
attr.type = PERF_TYPE_SOFTWARE;
attr.sample_type = PERF_SAMPLE_RAW;
/* notify only every Nth sample */
if (args.sampled) {
attr.sample_period = args.sample_rate;
attr.wakeup_events = args.sample_rate;
} else {
attr.sample_period = 1;
attr.wakeup_events = 1;
}
if (args.sample_rate > args.batch_cnt) {
fprintf(stderr, "sample rate %d is too high for given batch count %d\n",
args.sample_rate, args.batch_cnt);
exit(1);
}
ctx->perfbuf = perf_buffer__new_raw(bpf_map__fd(ctx->skel->maps.perfbuf),
args.perfbuf_sz, &attr,
perfbuf_process_sample_raw, NULL, NULL);
if (!ctx->perfbuf) {
fprintf(stderr, "failed to create perfbuf\n");
exit(1);
}
link = bpf_program__attach(ctx->skel->progs.bench_perfbuf);
if (!link) {
fprintf(stderr, "failed to attach program\n");
exit(1);
}
}
static void *perfbuf_libbpf_consumer(void *input)
{
struct perfbuf_libbpf_ctx *ctx = &perfbuf_libbpf_ctx;
while (perf_buffer__poll(ctx->perfbuf, -1) >= 0) {
if (args.back2back)
bufs_trigger_batch();
}
fprintf(stderr, "perfbuf polling failed!\n");
return NULL;
}
/* PERFBUF-CUSTOM benchmark */
/* copies of internal libbpf definitions */
struct perf_cpu_buf {
struct perf_buffer *pb;
void *base; /* mmap()'ed memory */
void *buf; /* for reconstructing segmented data */
size_t buf_size;
int fd;
int cpu;
int map_key;
};
struct perf_buffer {
perf_buffer_event_fn event_cb;
perf_buffer_sample_fn sample_cb;
perf_buffer_lost_fn lost_cb;
void *ctx; /* passed into callbacks */
size_t page_size;
size_t mmap_size;
struct perf_cpu_buf **cpu_bufs;
struct epoll_event *events;
int cpu_cnt; /* number of allocated CPU buffers */
int epoll_fd; /* perf event FD */
int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
};
static void *perfbuf_custom_consumer(void *input)
{
struct perfbuf_libbpf_ctx *ctx = &perfbuf_libbpf_ctx;
struct perf_buffer *pb = ctx->perfbuf;
struct perf_cpu_buf *cpu_buf;
struct perf_event_mmap_page *header;
size_t mmap_mask = pb->mmap_size - 1;
struct perf_event_header *ehdr;
__u64 data_head, data_tail;
size_t ehdr_size;
void *base;
int i, cnt;
while (true) {
if (args.back2back)
bufs_trigger_batch();
cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, -1);
if (cnt <= 0) {
fprintf(stderr, "perf epoll failed: %d\n", -errno);
exit(1);
}
for (i = 0; i < cnt; ++i) {
cpu_buf = pb->events[i].data.ptr;
header = cpu_buf->base;
base = ((void *)header) + pb->page_size;
data_head = ring_buffer_read_head(header);
data_tail = header->data_tail;
while (data_head != data_tail) {
ehdr = base + (data_tail & mmap_mask);
ehdr_size = ehdr->size;
if (ehdr->type == PERF_RECORD_SAMPLE)
atomic_inc(&buf_hits.value);
data_tail += ehdr_size;
}
ring_buffer_write_tail(header, data_tail);
}
}
return NULL;
}
const struct bench bench_rb_libbpf = {
.name = "rb-libbpf",
.argp = &bench_ringbufs_argp,
.validate = bufs_validate,
.setup = ringbuf_libbpf_setup,
.producer_thread = bufs_sample_producer,
.consumer_thread = ringbuf_libbpf_consumer,
.measure = ringbuf_libbpf_measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_rb_custom = {
.name = "rb-custom",
.argp = &bench_ringbufs_argp,
.validate = bufs_validate,
.setup = ringbuf_custom_setup,
.producer_thread = bufs_sample_producer,
.consumer_thread = ringbuf_custom_consumer,
.measure = ringbuf_custom_measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_pb_libbpf = {
.name = "pb-libbpf",
.argp = &bench_ringbufs_argp,
.validate = bufs_validate,
.setup = perfbuf_libbpf_setup,
.producer_thread = bufs_sample_producer,
.consumer_thread = perfbuf_libbpf_consumer,
.measure = perfbuf_measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_pb_custom = {
.name = "pb-custom",
.argp = &bench_ringbufs_argp,
.validate = bufs_validate,
.setup = perfbuf_libbpf_setup,
.producer_thread = bufs_sample_producer,
.consumer_thread = perfbuf_custom_consumer,
.measure = perfbuf_measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_sockmap.c
+599
View File
@@ -0,0 +1,599 @@
// SPDX-License-Identifier: GPL-2.0
#include <error.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/sendfile.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <argp.h>
#include "bench.h"
#include "bench_sockmap_prog.skel.h"
#include "bpf_util.h"
#define FILE_SIZE (128 * 1024)
#define DATA_REPEAT_SIZE 10
static const char snd_data[DATA_REPEAT_SIZE] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
/* c1 <-> [p1, p2] <-> c2
* RX bench(BPF_SK_SKB_STREAM_VERDICT):
* ARG_FW_RX_PASS:
* send(p2) -> recv(c2) -> bpf skb passthrough -> recv(c2)
* ARG_FW_RX_VERDICT_EGRESS:
* send(c1) -> verdict skb to tx queuec of p2 -> recv(c2)
* ARG_FW_RX_VERDICT_INGRESS:
* send(c1) -> verdict skb to rx queuec of c2 -> recv(c2)
*
* TX bench(BPF_SK_MSG_VERDIC):
* ARG_FW_TX_PASS:
* send(p2) -> bpf msg passthrough -> send(p2) -> recv(c2)
* ARG_FW_TX_VERDICT_INGRESS:
* send(p2) -> verdict msg to rx queue of c2 -> recv(c2)
* ARG_FW_TX_VERDICT_EGRESS:
* send(p1) -> verdict msg to tx queue of p2 -> recv(c2)
*/
enum SOCKMAP_ARG_FLAG {
ARG_FW_RX_NORMAL = 11000,
ARG_FW_RX_PASS,
ARG_FW_RX_VERDICT_EGRESS,
ARG_FW_RX_VERDICT_INGRESS,
ARG_FW_TX_NORMAL,
ARG_FW_TX_PASS,
ARG_FW_TX_VERDICT_INGRESS,
ARG_FW_TX_VERDICT_EGRESS,
ARG_CTL_RX_STRP,
ARG_CONSUMER_DELAY_TIME,
ARG_PRODUCER_DURATION,
};
#define TXMODE_NORMAL() \
((ctx.mode) == ARG_FW_TX_NORMAL)
#define TXMODE_BPF_INGRESS() \
((ctx.mode) == ARG_FW_TX_VERDICT_INGRESS)
#define TXMODE_BPF_EGRESS() \
((ctx.mode) == ARG_FW_TX_VERDICT_EGRESS)
#define TXMODE_BPF_PASS() \
((ctx.mode) == ARG_FW_TX_PASS)
#define TXMODE_BPF() ( \
TXMODE_BPF_PASS() || \
TXMODE_BPF_INGRESS() || \
TXMODE_BPF_EGRESS())
#define TXMODE() ( \
TXMODE_NORMAL() || \
TXMODE_BPF())
#define RXMODE_NORMAL() \
((ctx.mode) == ARG_FW_RX_NORMAL)
#define RXMODE_BPF_PASS() \
((ctx.mode) == ARG_FW_RX_PASS)
#define RXMODE_BPF_VERDICT_EGRESS() \
((ctx.mode) == ARG_FW_RX_VERDICT_EGRESS)
#define RXMODE_BPF_VERDICT_INGRESS() \
((ctx.mode) == ARG_FW_RX_VERDICT_INGRESS)
#define RXMODE_BPF_VERDICT() ( \
RXMODE_BPF_VERDICT_INGRESS() || \
RXMODE_BPF_VERDICT_EGRESS())
#define RXMODE_BPF() ( \
RXMODE_BPF_PASS() || \
RXMODE_BPF_VERDICT())
#define RXMODE() ( \
RXMODE_NORMAL() || \
RXMODE_BPF())
static struct socmap_ctx {
struct bench_sockmap_prog *skel;
enum SOCKMAP_ARG_FLAG mode;
#define c1 fds[0]
#define p1 fds[1]
#define c2 fds[2]
#define p2 fds[3]
#define sfd fds[4]
int fds[5];
long send_calls;
long read_calls;
long prod_send;
long user_read;
int file_size;
int delay_consumer;
int prod_run_time;
int strp_size;
} ctx = {
.prod_send = 0,
.user_read = 0,
.file_size = FILE_SIZE,
.mode = ARG_FW_RX_VERDICT_EGRESS,
.fds = {0},
.delay_consumer = 0,
.prod_run_time = 0,
.strp_size = 0,
};
static void bench_sockmap_prog_destroy(void)
{
int i;
for (i = 0; i < ARRAY_SIZE(ctx.fds); i++) {
if (ctx.fds[i] > 0)
close(ctx.fds[i]);
}
bench_sockmap_prog__destroy(ctx.skel);
}
static void init_addr(struct sockaddr_storage *ss,
socklen_t *len)
{
struct sockaddr_in *addr4 = memset(ss, 0, sizeof(*ss));
addr4->sin_family = AF_INET;
addr4->sin_port = 0;
addr4->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
*len = sizeof(*addr4);
}
static bool set_non_block(int fd, bool blocking)
{
int flags = fcntl(fd, F_GETFL, 0);
if (flags == -1)
return false;
flags = blocking ? (flags | O_NONBLOCK) : (flags & ~O_NONBLOCK);
return (fcntl(fd, F_SETFL, flags) == 0);
}
static int create_pair(int *c, int *p, int type)
{
struct sockaddr_storage addr;
int err, cfd, pfd;
socklen_t addr_len = sizeof(struct sockaddr_storage);
err = getsockname(ctx.sfd, (struct sockaddr *)&addr, &addr_len);
if (err) {
fprintf(stderr, "getsockname error %d\n", errno);
return err;
}
cfd = socket(AF_INET, type, 0);
if (cfd < 0) {
fprintf(stderr, "socket error %d\n", errno);
return err;
}
err = connect(cfd, (struct sockaddr *)&addr, addr_len);
if (err && errno != EINPROGRESS) {
fprintf(stderr, "connect error %d\n", errno);
return err;
}
pfd = accept(ctx.sfd, NULL, NULL);
if (pfd < 0) {
fprintf(stderr, "accept error %d\n", errno);
return err;
}
*c = cfd;
*p = pfd;
return 0;
}
static int create_sockets(void)
{
struct sockaddr_storage addr;
int err, one = 1;
socklen_t addr_len;
init_addr(&addr, &addr_len);
ctx.sfd = socket(AF_INET, SOCK_STREAM, 0);
if (ctx.sfd < 0) {
fprintf(stderr, "socket error:%d\n", errno);
return ctx.sfd;
}
err = setsockopt(ctx.sfd, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one));
if (err) {
fprintf(stderr, "setsockopt error:%d\n", errno);
return err;
}
err = bind(ctx.sfd, (struct sockaddr *)&addr, addr_len);
if (err) {
fprintf(stderr, "bind error:%d\n", errno);
return err;
}
err = listen(ctx.sfd, SOMAXCONN);
if (err) {
fprintf(stderr, "listen error:%d\n", errno);
return err;
}
err = create_pair(&ctx.c1, &ctx.p1, SOCK_STREAM);
if (err) {
fprintf(stderr, "create_pair 1 error\n");
return err;
}
err = create_pair(&ctx.c2, &ctx.p2, SOCK_STREAM);
if (err) {
fprintf(stderr, "create_pair 2 error\n");
return err;
}
printf("create socket fd c1:%d p1:%d c2:%d p2:%d\n",
ctx.c1, ctx.p1, ctx.c2, ctx.p2);
return 0;
}
static void validate(void)
{
if (env.consumer_cnt != 2 || env.producer_cnt != 1 ||
!env.affinity)
goto err;
return;
err:
fprintf(stderr, "argument '-c 2 -p 1 -a' is necessary");
exit(1);
}
static int setup_rx_sockmap(void)
{
int verdict, pass, parser, map;
int zero = 0, one = 1;
int err;
parser = bpf_program__fd(ctx.skel->progs.prog_skb_parser);
verdict = bpf_program__fd(ctx.skel->progs.prog_skb_verdict);
pass = bpf_program__fd(ctx.skel->progs.prog_skb_pass);
map = bpf_map__fd(ctx.skel->maps.sock_map_rx);
if (ctx.strp_size != 0) {
ctx.skel->bss->pkt_size = ctx.strp_size;
err = bpf_prog_attach(parser, map, BPF_SK_SKB_STREAM_PARSER, 0);
if (err)
return err;
}
if (RXMODE_BPF_VERDICT())
err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0);
else if (RXMODE_BPF_PASS())
err = bpf_prog_attach(pass, map, BPF_SK_SKB_STREAM_VERDICT, 0);
if (err)
return err;
if (RXMODE_BPF_PASS())
return bpf_map_update_elem(map, &zero, &ctx.c2, BPF_NOEXIST);
err = bpf_map_update_elem(map, &zero, &ctx.p1, BPF_NOEXIST);
if (err < 0)
return err;
if (RXMODE_BPF_VERDICT_INGRESS()) {
ctx.skel->bss->verdict_dir = BPF_F_INGRESS;
err = bpf_map_update_elem(map, &one, &ctx.c2, BPF_NOEXIST);
} else {
err = bpf_map_update_elem(map, &one, &ctx.p2, BPF_NOEXIST);
}
if (err < 0)
return err;
return 0;
}
static int setup_tx_sockmap(void)
{
int zero = 0, one = 1;
int prog, map;
int err;
map = bpf_map__fd(ctx.skel->maps.sock_map_tx);
prog = TXMODE_BPF_PASS() ?
bpf_program__fd(ctx.skel->progs.prog_skmsg_pass) :
bpf_program__fd(ctx.skel->progs.prog_skmsg_verdict);
err = bpf_prog_attach(prog, map, BPF_SK_MSG_VERDICT, 0);
if (err)
return err;
if (TXMODE_BPF_EGRESS()) {
err = bpf_map_update_elem(map, &zero, &ctx.p1, BPF_NOEXIST);
err |= bpf_map_update_elem(map, &one, &ctx.p2, BPF_NOEXIST);
} else {
ctx.skel->bss->verdict_dir = BPF_F_INGRESS;
err = bpf_map_update_elem(map, &zero, &ctx.p2, BPF_NOEXIST);
err |= bpf_map_update_elem(map, &one, &ctx.c2, BPF_NOEXIST);
}
if (err < 0)
return err;
return 0;
}
static void setup(void)
{
int err;
ctx.skel = bench_sockmap_prog__open_and_load();
if (!ctx.skel) {
fprintf(stderr, "error loading skel\n");
exit(1);
}
if (create_sockets()) {
fprintf(stderr, "create_net_mode error\n");
goto err;
}
if (RXMODE_BPF()) {
err = setup_rx_sockmap();
if (err) {
fprintf(stderr, "setup_rx_sockmap error:%d\n", err);
goto err;
}
} else if (TXMODE_BPF()) {
err = setup_tx_sockmap();
if (err) {
fprintf(stderr, "setup_tx_sockmap error:%d\n", err);
goto err;
}
} else {
fprintf(stderr, "unknown sockmap bench mode: %d\n", ctx.mode);
goto err;
}
return;
err:
bench_sockmap_prog_destroy();
exit(1);
}
static void measure(struct bench_res *res)
{
res->drops = atomic_swap(&ctx.prod_send, 0);
res->hits = atomic_swap(&ctx.skel->bss->process_byte, 0);
res->false_hits = atomic_swap(&ctx.user_read, 0);
res->important_hits = atomic_swap(&ctx.send_calls, 0);
res->important_hits |= atomic_swap(&ctx.read_calls, 0) << 32;
}
static void verify_data(int *check_pos, char *buf, int rcv)
{
for (int i = 0 ; i < rcv; i++) {
if (buf[i] != snd_data[(*check_pos) % DATA_REPEAT_SIZE]) {
fprintf(stderr, "verify data fail");
exit(1);
}
(*check_pos)++;
if (*check_pos >= FILE_SIZE)
*check_pos = 0;
}
}
static void *consumer(void *input)
{
int rcv, sent;
int check_pos = 0;
int tid = (long)input;
int recv_buf_size = FILE_SIZE;
char *buf = malloc(recv_buf_size);
int delay_read = ctx.delay_consumer;
if (!buf) {
fprintf(stderr, "fail to init read buffer");
return NULL;
}
while (true) {
if (tid == 1) {
/* consumer 1 is unused for tx test and stream verdict test */
if (RXMODE_BPF() || TXMODE())
return NULL;
/* it's only for RX_NORMAL which service as reserve-proxy mode */
rcv = read(ctx.p1, buf, recv_buf_size);
if (rcv < 0) {
fprintf(stderr, "fail to read p1");
return NULL;
}
sent = send(ctx.p2, buf, recv_buf_size, 0);
if (sent < 0) {
fprintf(stderr, "fail to send p2");
return NULL;
}
} else {
if (delay_read != 0) {
if (delay_read < 0)
return NULL;
sleep(delay_read);
delay_read = 0;
}
/* read real endpoint by consumer 0 */
atomic_inc(&ctx.read_calls);
rcv = read(ctx.c2, buf, recv_buf_size);
if (rcv < 0 && errno != EAGAIN) {
fprintf(stderr, "%s fail to read c2 %d\n", __func__, errno);
return NULL;
}
verify_data(&check_pos, buf, rcv);
atomic_add(&ctx.user_read, rcv);
}
}
return NULL;
}
static void *producer(void *input)
{
int off = 0, fp, need_sent, sent;
int file_size = ctx.file_size;
struct timespec ts1, ts2;
int target;
FILE *file;
file = tmpfile();
if (!file) {
fprintf(stderr, "create file for sendfile");
return NULL;
}
/* we need simple verify */
for (int i = 0; i < file_size; i++) {
if (fwrite(&snd_data[off], sizeof(char), 1, file) != 1) {
fprintf(stderr, "init tmpfile error");
return NULL;
}
if (++off >= sizeof(snd_data))
off = 0;
}
fflush(file);
fseek(file, 0, SEEK_SET);
fp = fileno(file);
need_sent = file_size;
clock_gettime(CLOCK_MONOTONIC, &ts1);
if (RXMODE_BPF_VERDICT())
target = ctx.c1;
else if (TXMODE_BPF_EGRESS())
target = ctx.p1;
else
target = ctx.p2;
set_non_block(target, true);
while (true) {
if (ctx.prod_run_time) {
clock_gettime(CLOCK_MONOTONIC, &ts2);
if (ts2.tv_sec - ts1.tv_sec > ctx.prod_run_time)
return NULL;
}
errno = 0;
atomic_inc(&ctx.send_calls);
sent = sendfile(target, fp, NULL, need_sent);
if (sent < 0) {
if (errno != EAGAIN && errno != ENOMEM && errno != ENOBUFS) {
fprintf(stderr, "sendfile return %d, errorno %d:%s\n",
sent, errno, strerror(errno));
return NULL;
}
continue;
} else if (sent < need_sent) {
need_sent -= sent;
atomic_add(&ctx.prod_send, sent);
continue;
}
atomic_add(&ctx.prod_send, need_sent);
need_sent = file_size;
lseek(fp, 0, SEEK_SET);
}
return NULL;
}
static void report_progress(int iter, struct bench_res *res, long delta_ns)
{
double speed_mbs, prod_mbs, bpf_mbs, send_hz, read_hz;
prod_mbs = res->drops / 1000000.0 / (delta_ns / 1000000000.0);
speed_mbs = res->false_hits / 1000000.0 / (delta_ns / 1000000000.0);
bpf_mbs = res->hits / 1000000.0 / (delta_ns / 1000000000.0);
send_hz = (res->important_hits & 0xFFFFFFFF) / (delta_ns / 1000000000.0);
read_hz = (res->important_hits >> 32) / (delta_ns / 1000000000.0);
printf("Iter %3d (%7.3lfus): ",
iter, (delta_ns - 1000000000) / 1000.0);
printf("Send Speed %8.3lf MB/s (%8.3lf calls/s), BPF Speed %8.3lf MB/s, "
"Rcv Speed %8.3lf MB/s (%8.3lf calls/s)\n",
prod_mbs, send_hz, bpf_mbs, speed_mbs, read_hz);
}
static void report_final(struct bench_res res[], int res_cnt)
{
double verdict_mbs_mean = 0.0;
long verdict_total = 0;
int i;
for (i = 0; i < res_cnt; i++) {
verdict_mbs_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
verdict_total += res[i].hits / 1000000.0;
}
printf("Summary: total trans %8.3lu MB \u00B1 %5.3lf MB/s\n",
verdict_total, verdict_mbs_mean);
}
static const struct argp_option opts[] = {
{ "rx-normal", ARG_FW_RX_NORMAL, NULL, 0,
"simple reserve-proxy mode, no bfp enabled"},
{ "rx-pass", ARG_FW_RX_PASS, NULL, 0,
"run bpf prog but no redir applied"},
{ "rx-strp", ARG_CTL_RX_STRP, "Byte", 0,
"enable strparser and set the encapsulation size"},
{ "rx-verdict-egress", ARG_FW_RX_VERDICT_EGRESS, NULL, 0,
"forward data with bpf(stream verdict)"},
{ "rx-verdict-ingress", ARG_FW_RX_VERDICT_INGRESS, NULL, 0,
"forward data with bpf(stream verdict)"},
{ "tx-normal", ARG_FW_TX_NORMAL, NULL, 0,
"simple c-s mode, no bfp enabled"},
{ "tx-pass", ARG_FW_TX_PASS, NULL, 0,
"run bpf prog but no redir applied"},
{ "tx-verdict-ingress", ARG_FW_TX_VERDICT_INGRESS, NULL, 0,
"forward msg to ingress queue of another socket"},
{ "tx-verdict-egress", ARG_FW_TX_VERDICT_EGRESS, NULL, 0,
"forward msg to egress queue of another socket"},
{ "delay-consumer", ARG_CONSUMER_DELAY_TIME, "SEC", 0,
"delay consumer start"},
{ "producer-duration", ARG_PRODUCER_DURATION, "SEC", 0,
"producer duration"},
{},
};
static error_t parse_arg(int key, char *arg, struct argp_state *state)
{
switch (key) {
case ARG_FW_RX_NORMAL...ARG_FW_TX_VERDICT_EGRESS:
ctx.mode = key;
break;
case ARG_CONSUMER_DELAY_TIME:
ctx.delay_consumer = strtol(arg, NULL, 10);
break;
case ARG_PRODUCER_DURATION:
ctx.prod_run_time = strtol(arg, NULL, 10);
break;
case ARG_CTL_RX_STRP:
ctx.strp_size = strtol(arg, NULL, 10);
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
/* exported into benchmark runner */
const struct argp bench_sockmap_argp = {
.options = opts,
.parser = parse_arg,
};
/* Benchmark performance of creating bpf local storage */
const struct bench bench_sockmap = {
.name = "sockmap",
.argp = &bench_sockmap_argp,
.validate = validate,
.setup = setup,
.producer_thread = producer,
.consumer_thread = consumer,
.measure = measure,
.report_progress = report_progress,
.report_final = report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_strncmp.c
+156
View File
@@ -0,0 +1,156 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) 2021. Huawei Technologies Co., Ltd */
#include <argp.h>
#include "bench.h"
#include "strncmp_bench.skel.h"
static struct strncmp_ctx {
struct strncmp_bench *skel;
} ctx;
static struct strncmp_args {
u32 cmp_str_len;
} args = {
.cmp_str_len = 32,
};
enum {
ARG_CMP_STR_LEN = 5000,
};
static const struct argp_option opts[] = {
{ "cmp-str-len", ARG_CMP_STR_LEN, "CMP_STR_LEN", 0,
"Set the length of compared string" },
{},
};
static error_t strncmp_parse_arg(int key, char *arg, struct argp_state *state)
{
switch (key) {
case ARG_CMP_STR_LEN:
args.cmp_str_len = strtoul(arg, NULL, 10);
if (!args.cmp_str_len ||
args.cmp_str_len >= sizeof(ctx.skel->bss->str)) {
fprintf(stderr, "Invalid cmp str len (limit %zu)\n",
sizeof(ctx.skel->bss->str));
argp_usage(state);
}
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
const struct argp bench_strncmp_argp = {
.options = opts,
.parser = strncmp_parse_arg,
};
static void strncmp_validate(void)
{
if (env.consumer_cnt != 0) {
fprintf(stderr, "strncmp benchmark doesn't support consumer!\n");
exit(1);
}
}
static void strncmp_setup(void)
{
int err;
char *target;
size_t i, sz;
sz = sizeof(ctx.skel->rodata->target);
if (!sz || sz < sizeof(ctx.skel->bss->str)) {
fprintf(stderr, "invalid string size (target %zu, src %zu)\n",
sz, sizeof(ctx.skel->bss->str));
exit(1);
}
setup_libbpf();
ctx.skel = strncmp_bench__open();
if (!ctx.skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
srandom(time(NULL));
target = ctx.skel->rodata->target;
for (i = 0; i < sz - 1; i++)
target[i] = '1' + random() % 9;
target[sz - 1] = '\0';
ctx.skel->rodata->cmp_str_len = args.cmp_str_len;
memcpy(ctx.skel->bss->str, target, args.cmp_str_len);
ctx.skel->bss->str[args.cmp_str_len] = '\0';
/* Make bss->str < rodata->target */
ctx.skel->bss->str[args.cmp_str_len - 1] -= 1;
err = strncmp_bench__load(ctx.skel);
if (err) {
fprintf(stderr, "failed to load skeleton\n");
strncmp_bench__destroy(ctx.skel);
exit(1);
}
}
static void strncmp_attach_prog(struct bpf_program *prog)
{
struct bpf_link *link;
link = bpf_program__attach(prog);
if (!link) {
fprintf(stderr, "failed to attach program!\n");
exit(1);
}
}
static void strncmp_no_helper_setup(void)
{
strncmp_setup();
strncmp_attach_prog(ctx.skel->progs.strncmp_no_helper);
}
static void strncmp_helper_setup(void)
{
strncmp_setup();
strncmp_attach_prog(ctx.skel->progs.strncmp_helper);
}
static void *strncmp_producer(void *ctx)
{
while (true)
(void)syscall(__NR_getpgid);
return NULL;
}
static void strncmp_measure(struct bench_res *res)
{
res->hits = atomic_swap(&ctx.skel->bss->hits, 0);
}
const struct bench bench_strncmp_no_helper = {
.name = "strncmp-no-helper",
.argp = &bench_strncmp_argp,
.validate = strncmp_validate,
.setup = strncmp_no_helper_setup,
.producer_thread = strncmp_producer,
.measure = strncmp_measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
const struct bench bench_strncmp_helper = {
.name = "strncmp-helper",
.argp = &bench_strncmp_argp,
.validate = strncmp_validate,
.setup = strncmp_helper_setup,
.producer_thread = strncmp_producer,
.measure = strncmp_measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_trigger.c
+674
View File
@@ -0,0 +1,674 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#define _GNU_SOURCE
#include <argp.h>
#include <unistd.h>
#include <stdint.h>
#include "bpf_util.h"
#include "bench.h"
#include "trigger_bench.skel.h"
#include "trace_helpers.h"
#define MAX_TRIG_BATCH_ITERS 1000
static struct {
__u32 batch_iters;
} args = {
.batch_iters = 100,
};
enum {
ARG_TRIG_BATCH_ITERS = 7000,
};
static const struct argp_option opts[] = {
{ "trig-batch-iters", ARG_TRIG_BATCH_ITERS, "BATCH_ITER_CNT", 0,
"Number of in-kernel iterations per one driver test run"},
{},
};
static error_t parse_arg(int key, char *arg, struct argp_state *state)
{
long ret;
switch (key) {
case ARG_TRIG_BATCH_ITERS:
ret = strtol(arg, NULL, 10);
if (ret < 1 || ret > MAX_TRIG_BATCH_ITERS) {
fprintf(stderr, "invalid --trig-batch-iters value (should be between %d and %d)\n",
1, MAX_TRIG_BATCH_ITERS);
argp_usage(state);
}
args.batch_iters = ret;
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
const struct argp bench_trigger_batch_argp = {
.options = opts,
.parser = parse_arg,
};
/* adjust slot shift in inc_hits() if changing */
#define MAX_BUCKETS 256
#pragma GCC diagnostic ignored "-Wattributes"
/* BPF triggering benchmarks */
static struct trigger_ctx {
struct trigger_bench *skel;
bool usermode_counters;
int driver_prog_fd;
} ctx;
static struct counter base_hits[MAX_BUCKETS];
static __always_inline void inc_counter(struct counter *counters)
{
static __thread int tid = 0;
unsigned slot;
if (unlikely(tid == 0))
tid = sys_gettid();
/* multiplicative hashing, it's fast */
slot = 2654435769U * tid;
slot >>= 24;
atomic_inc(&base_hits[slot].value); /* use highest byte as an index */
}
static long sum_and_reset_counters(struct counter *counters)
{
int i;
long sum = 0;
for (i = 0; i < MAX_BUCKETS; i++)
sum += atomic_swap(&counters[i].value, 0);
return sum;
}
static void trigger_validate(void)
{
if (env.consumer_cnt != 0) {
fprintf(stderr, "benchmark doesn't support consumer!\n");
exit(1);
}
}
static void *trigger_producer(void *input)
{
if (ctx.usermode_counters) {
while (true) {
(void)syscall(__NR_getpgid);
inc_counter(base_hits);
}
} else {
while (true)
(void)syscall(__NR_getpgid);
}
return NULL;
}
static void *trigger_producer_batch(void *input)
{
int fd = ctx.driver_prog_fd ?: bpf_program__fd(ctx.skel->progs.trigger_driver);
while (true)
bpf_prog_test_run_opts(fd, NULL);
return NULL;
}
static void trigger_measure(struct bench_res *res)
{
if (ctx.usermode_counters)
res->hits = sum_and_reset_counters(base_hits);
else
res->hits = sum_and_reset_counters(ctx.skel->bss->hits);
}
static void setup_ctx(void)
{
setup_libbpf();
ctx.skel = trigger_bench__open();
if (!ctx.skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
/* default "driver" BPF program */
bpf_program__set_autoload(ctx.skel->progs.trigger_driver, true);
ctx.skel->rodata->batch_iters = args.batch_iters;
ctx.skel->rodata->stacktrace = env.stacktrace;
}
static void load_ctx(void)
{
int err;
err = trigger_bench__load(ctx.skel);
if (err) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
}
static void attach_bpf(struct bpf_program *prog)
{
struct bpf_link *link;
link = bpf_program__attach(prog);
if (!link) {
fprintf(stderr, "failed to attach program!\n");
exit(1);
}
}
static void trigger_syscall_count_setup(void)
{
ctx.usermode_counters = true;
}
/* Batched, staying mostly in-kernel triggering setups */
static void trigger_kernel_count_setup(void)
{
setup_ctx();
bpf_program__set_autoload(ctx.skel->progs.trigger_driver, false);
bpf_program__set_autoload(ctx.skel->progs.trigger_kernel_count, true);
load_ctx();
/* override driver program */
ctx.driver_prog_fd = bpf_program__fd(ctx.skel->progs.trigger_kernel_count);
}
static void trigger_kprobe_setup(void)
{
setup_ctx();
bpf_program__set_autoload(ctx.skel->progs.bench_trigger_kprobe, true);
load_ctx();
attach_bpf(ctx.skel->progs.bench_trigger_kprobe);
}
static void trigger_kretprobe_setup(void)
{
setup_ctx();
bpf_program__set_autoload(ctx.skel->progs.bench_trigger_kretprobe, true);
load_ctx();
attach_bpf(ctx.skel->progs.bench_trigger_kretprobe);
}
static void trigger_kprobe_multi_setup(void)
{
setup_ctx();
bpf_program__set_autoload(ctx.skel->progs.bench_trigger_kprobe_multi, true);
load_ctx();
attach_bpf(ctx.skel->progs.bench_trigger_kprobe_multi);
}
static void trigger_kretprobe_multi_setup(void)
{
setup_ctx();
bpf_program__set_autoload(ctx.skel->progs.bench_trigger_kretprobe_multi, true);
load_ctx();
attach_bpf(ctx.skel->progs.bench_trigger_kretprobe_multi);
}
static void trigger_fentry_setup(void)
{
setup_ctx();
bpf_program__set_autoload(ctx.skel->progs.bench_trigger_fentry, true);
load_ctx();
attach_bpf(ctx.skel->progs.bench_trigger_fentry);
}
static void attach_ksyms_all(struct bpf_program *empty, bool kretprobe)
{
LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
struct bpf_link *link = NULL;
struct ksyms *ksyms = NULL;
/* Some recursive functions will be skipped in
* bpf_get_ksyms -> skip_entry, as they can introduce sufficient
* overhead. However, it's difficut to skip all the recursive
* functions for a debug kernel.
*
* So, don't run the kprobe-multi-all and kretprobe-multi-all on
* a debug kernel.
*/
if (bpf_get_ksyms(&ksyms, true)) {
fprintf(stderr, "failed to get ksyms\n");
exit(1);
}
opts.syms = (const char **)ksyms->filtered_syms;
opts.cnt = ksyms->filtered_cnt;
opts.retprobe = kretprobe;
/* attach empty to all the kernel functions except bpf_get_numa_node_id. */
link = bpf_program__attach_kprobe_multi_opts(empty, NULL, &opts);
free_kallsyms_local(ksyms);
if (!link) {
fprintf(stderr, "failed to attach bpf_program__attach_kprobe_multi_opts to all\n");
exit(1);
}
}
static void trigger_kprobe_multi_all_setup(void)
{
struct bpf_program *prog, *empty;
setup_ctx();
empty = ctx.skel->progs.bench_kprobe_multi_empty;
prog = ctx.skel->progs.bench_trigger_kprobe_multi;
bpf_program__set_autoload(empty, true);
bpf_program__set_autoload(prog, true);
load_ctx();
attach_ksyms_all(empty, false);
attach_bpf(prog);
}
static void trigger_kretprobe_multi_all_setup(void)
{
struct bpf_program *prog, *empty;
setup_ctx();
empty = ctx.skel->progs.bench_kretprobe_multi_empty;
prog = ctx.skel->progs.bench_trigger_kretprobe_multi;
bpf_program__set_autoload(empty, true);
bpf_program__set_autoload(prog, true);
load_ctx();
attach_ksyms_all(empty, true);
attach_bpf(prog);
}
static void trigger_fexit_setup(void)
{
setup_ctx();
bpf_program__set_autoload(ctx.skel->progs.bench_trigger_fexit, true);
load_ctx();
attach_bpf(ctx.skel->progs.bench_trigger_fexit);
}
static void trigger_fmodret_setup(void)
{
setup_ctx();
bpf_program__set_autoload(ctx.skel->progs.trigger_driver, false);
bpf_program__set_autoload(ctx.skel->progs.trigger_driver_kfunc, true);
bpf_program__set_autoload(ctx.skel->progs.bench_trigger_fmodret, true);
load_ctx();
/* override driver program */
ctx.driver_prog_fd = bpf_program__fd(ctx.skel->progs.trigger_driver_kfunc);
attach_bpf(ctx.skel->progs.bench_trigger_fmodret);
}
static void trigger_tp_setup(void)
{
setup_ctx();
bpf_program__set_autoload(ctx.skel->progs.trigger_driver, false);
bpf_program__set_autoload(ctx.skel->progs.trigger_driver_kfunc, true);
bpf_program__set_autoload(ctx.skel->progs.bench_trigger_tp, true);
load_ctx();
/* override driver program */
ctx.driver_prog_fd = bpf_program__fd(ctx.skel->progs.trigger_driver_kfunc);
attach_bpf(ctx.skel->progs.bench_trigger_tp);
}
static void trigger_rawtp_setup(void)
{
setup_ctx();
bpf_program__set_autoload(ctx.skel->progs.trigger_driver, false);
bpf_program__set_autoload(ctx.skel->progs.trigger_driver_kfunc, true);
bpf_program__set_autoload(ctx.skel->progs.bench_trigger_rawtp, true);
load_ctx();
/* override driver program */
ctx.driver_prog_fd = bpf_program__fd(ctx.skel->progs.trigger_driver_kfunc);
attach_bpf(ctx.skel->progs.bench_trigger_rawtp);
}
/* make sure call is not inlined and not avoided by compiler, so __weak and
* inline asm volatile in the body of the function
*
* There is a performance difference between uprobing at nop location vs other
* instructions. So use two different targets, one of which starts with nop
* and another doesn't.
*
* GCC doesn't generate stack setup preamble for these functions due to them
* having no input arguments and doing nothing in the body.
*/
__nocf_check __weak void uprobe_target_nop(void)
{
asm volatile ("nop");
}
__weak void opaque_noop_func(void)
{
}
__nocf_check __weak int uprobe_target_push(void)
{
/* overhead of function call is negligible compared to uprobe
* triggering, so this shouldn't affect benchmark results much
*/
opaque_noop_func();
return 1;
}
__nocf_check __weak void uprobe_target_ret(void)
{
asm volatile ("");
}
static void *uprobe_producer_count(void *input)
{
while (true) {
uprobe_target_nop();
inc_counter(base_hits);
}
return NULL;
}
static void *uprobe_producer_nop(void *input)
{
while (true)
uprobe_target_nop();
return NULL;
}
static void *uprobe_producer_push(void *input)
{
while (true)
uprobe_target_push();
return NULL;
}
static void *uprobe_producer_ret(void *input)
{
while (true)
uprobe_target_ret();
return NULL;
}
#ifdef __x86_64__
__nocf_check __weak void uprobe_target_nop5(void)
{
asm volatile (".byte 0x0f, 0x1f, 0x44, 0x00, 0x00");
}
static void *uprobe_producer_nop5(void *input)
{
while (true)
uprobe_target_nop5();
return NULL;
}
void usdt_1(void);
void usdt_2(void);
static void *uprobe_producer_usdt_nop(void *input)
{
while (true)
usdt_1();
return NULL;
}
static void *uprobe_producer_usdt_nop5(void *input)
{
while (true)
usdt_2();
return NULL;
}
#endif
static void usetup(bool use_retprobe, bool use_multi, void *target_addr)
{
size_t uprobe_offset;
struct bpf_link *link;
int err;
setup_libbpf();
ctx.skel = trigger_bench__open();
if (!ctx.skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
if (use_multi)
bpf_program__set_autoload(ctx.skel->progs.bench_trigger_uprobe_multi, true);
else
bpf_program__set_autoload(ctx.skel->progs.bench_trigger_uprobe, true);
err = trigger_bench__load(ctx.skel);
if (err) {
fprintf(stderr, "failed to load skeleton\n");
exit(1);
}
uprobe_offset = get_uprobe_offset(target_addr);
if (use_multi) {
LIBBPF_OPTS(bpf_uprobe_multi_opts, opts,
.retprobe = use_retprobe,
.cnt = 1,
.offsets = &uprobe_offset,
);
link = bpf_program__attach_uprobe_multi(
ctx.skel->progs.bench_trigger_uprobe_multi,
-1 /* all PIDs */, "/proc/self/exe", NULL, &opts);
ctx.skel->links.bench_trigger_uprobe_multi = link;
} else {
link = bpf_program__attach_uprobe(ctx.skel->progs.bench_trigger_uprobe,
use_retprobe,
-1 /* all PIDs */,
"/proc/self/exe",
uprobe_offset);
ctx.skel->links.bench_trigger_uprobe = link;
}
if (!link) {
fprintf(stderr, "failed to attach %s!\n", use_multi ? "multi-uprobe" : "uprobe");
exit(1);
}
}
static void usermode_count_setup(void)
{
ctx.usermode_counters = true;
}
static void uprobe_nop_setup(void)
{
usetup(false, false /* !use_multi */, &uprobe_target_nop);
}
static void uretprobe_nop_setup(void)
{
usetup(true, false /* !use_multi */, &uprobe_target_nop);
}
static void uprobe_push_setup(void)
{
usetup(false, false /* !use_multi */, &uprobe_target_push);
}
static void uretprobe_push_setup(void)
{
usetup(true, false /* !use_multi */, &uprobe_target_push);
}
static void uprobe_ret_setup(void)
{
usetup(false, false /* !use_multi */, &uprobe_target_ret);
}
static void uretprobe_ret_setup(void)
{
usetup(true, false /* !use_multi */, &uprobe_target_ret);
}
static void uprobe_multi_nop_setup(void)
{
usetup(false, true /* use_multi */, &uprobe_target_nop);
}
static void uretprobe_multi_nop_setup(void)
{
usetup(true, true /* use_multi */, &uprobe_target_nop);
}
static void uprobe_multi_push_setup(void)
{
usetup(false, true /* use_multi */, &uprobe_target_push);
}
static void uretprobe_multi_push_setup(void)
{
usetup(true, true /* use_multi */, &uprobe_target_push);
}
static void uprobe_multi_ret_setup(void)
{
usetup(false, true /* use_multi */, &uprobe_target_ret);
}
static void uretprobe_multi_ret_setup(void)
{
usetup(true, true /* use_multi */, &uprobe_target_ret);
}
#ifdef __x86_64__
static void uprobe_nop5_setup(void)
{
usetup(false, false /* !use_multi */, &uprobe_target_nop5);
}
static void uretprobe_nop5_setup(void)
{
usetup(true, false /* !use_multi */, &uprobe_target_nop5);
}
static void uprobe_multi_nop5_setup(void)
{
usetup(false, true /* use_multi */, &uprobe_target_nop5);
}
static void uretprobe_multi_nop5_setup(void)
{
usetup(true, true /* use_multi */, &uprobe_target_nop5);
}
static void usdt_setup(const char *name)
{
struct bpf_link *link;
int err;
setup_libbpf();
ctx.skel = trigger_bench__open();
if (!ctx.skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
bpf_program__set_autoload(ctx.skel->progs.bench_trigger_usdt, true);
err = trigger_bench__load(ctx.skel);
if (err) {
fprintf(stderr, "failed to load skeleton\n");
exit(1);
}
link = bpf_program__attach_usdt(ctx.skel->progs.bench_trigger_usdt,
0 /*self*/, "/proc/self/exe",
"optimized_attach", name, NULL);
if (libbpf_get_error(link)) {
fprintf(stderr, "failed to attach optimized_attach:%s usdt probe\n", name);
exit(1);
}
ctx.skel->links.bench_trigger_usdt = link;
}
static void usdt_nop_setup(void)
{
usdt_setup("usdt_1");
}
static void usdt_nop5_setup(void)
{
usdt_setup("usdt_2");
}
#endif
const struct bench bench_trig_syscall_count = {
.name = "trig-syscall-count",
.validate = trigger_validate,
.setup = trigger_syscall_count_setup,
.producer_thread = trigger_producer,
.measure = trigger_measure,
.report_progress = hits_drops_report_progress,
.report_final = hits_drops_report_final,
};
/* batched (staying mostly in kernel) kprobe/fentry benchmarks */
#define BENCH_TRIG_KERNEL(KIND, NAME) \
const struct bench bench_trig_##KIND = { \
.name = "trig-" NAME, \
.setup = trigger_##KIND##_setup, \
.producer_thread = trigger_producer_batch, \
.measure = trigger_measure, \
.report_progress = hits_drops_report_progress, \
.report_final = hits_drops_report_final, \
.argp = &bench_trigger_batch_argp, \
}
BENCH_TRIG_KERNEL(kernel_count, "kernel-count");
BENCH_TRIG_KERNEL(kprobe, "kprobe");
BENCH_TRIG_KERNEL(kretprobe, "kretprobe");
BENCH_TRIG_KERNEL(kprobe_multi, "kprobe-multi");
BENCH_TRIG_KERNEL(kretprobe_multi, "kretprobe-multi");
BENCH_TRIG_KERNEL(fentry, "fentry");
BENCH_TRIG_KERNEL(kprobe_multi_all, "kprobe-multi-all");
BENCH_TRIG_KERNEL(kretprobe_multi_all, "kretprobe-multi-all");
BENCH_TRIG_KERNEL(fexit, "fexit");
BENCH_TRIG_KERNEL(fmodret, "fmodret");
BENCH_TRIG_KERNEL(tp, "tp");
BENCH_TRIG_KERNEL(rawtp, "rawtp");
/* uprobe benchmarks */
#define BENCH_TRIG_USERMODE(KIND, PRODUCER, NAME) \
const struct bench bench_trig_##KIND = { \
.name = "trig-" NAME, \
.validate = trigger_validate, \
.setup = KIND##_setup, \
.producer_thread = uprobe_producer_##PRODUCER, \
.measure = trigger_measure, \
.report_progress = hits_drops_report_progress, \
.report_final = hits_drops_report_final, \
}
BENCH_TRIG_USERMODE(usermode_count, count, "usermode-count");
BENCH_TRIG_USERMODE(uprobe_nop, nop, "uprobe-nop");
BENCH_TRIG_USERMODE(uprobe_push, push, "uprobe-push");
BENCH_TRIG_USERMODE(uprobe_ret, ret, "uprobe-ret");
BENCH_TRIG_USERMODE(uretprobe_nop, nop, "uretprobe-nop");
BENCH_TRIG_USERMODE(uretprobe_push, push, "uretprobe-push");
BENCH_TRIG_USERMODE(uretprobe_ret, ret, "uretprobe-ret");
BENCH_TRIG_USERMODE(uprobe_multi_nop, nop, "uprobe-multi-nop");
BENCH_TRIG_USERMODE(uprobe_multi_push, push, "uprobe-multi-push");
BENCH_TRIG_USERMODE(uprobe_multi_ret, ret, "uprobe-multi-ret");
BENCH_TRIG_USERMODE(uretprobe_multi_nop, nop, "uretprobe-multi-nop");
BENCH_TRIG_USERMODE(uretprobe_multi_push, push, "uretprobe-multi-push");
BENCH_TRIG_USERMODE(uretprobe_multi_ret, ret, "uretprobe-multi-ret");
#ifdef __x86_64__
BENCH_TRIG_USERMODE(uprobe_nop5, nop5, "uprobe-nop5");
BENCH_TRIG_USERMODE(uretprobe_nop5, nop5, "uretprobe-nop5");
BENCH_TRIG_USERMODE(uprobe_multi_nop5, nop5, "uprobe-multi-nop5");
BENCH_TRIG_USERMODE(uretprobe_multi_nop5, nop5, "uretprobe-multi-nop5");
BENCH_TRIG_USERMODE(usdt_nop, usdt_nop, "usdt-nop");
BENCH_TRIG_USERMODE(usdt_nop5, usdt_nop5, "usdt-nop5");
#endif
linux-7.2.3/tools/testing/selftests/bpf/benchs/bench_xdp_lb.c
+1124
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/tools/testing/selftests/bpf/benchs/run_bench_bloom_filter_map.sh
Executable
+45
View File
@@ -0,0 +1,45 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
source ./benchs/run_common.sh
set -eufo pipefail
header "Bloom filter map"
for v in 2 4 8 16 40; do
for t in 1 4 8 12 16; do
for h in {1..10}; do
subtitle "value_size: $v bytes, # threads: $t, # hashes: $h"
for e in 10000 50000 75000 100000 250000 500000 750000 1000000 2500000 5000000; do
printf "%'d entries -\n" $e
printf "\t"
summarize "Lookups, total operations: " \
"$($RUN_BENCH -p $t --nr_hash_funcs $h --nr_entries $e --value_size $v bloom-lookup)"
printf "\t"
summarize "Updates, total operations: " \
"$($RUN_BENCH -p $t --nr_hash_funcs $h --nr_entries $e --value_size $v bloom-update)"
printf "\t"
summarize_percentage "False positive rate: " \
"$($RUN_BENCH -p $t --nr_hash_funcs $h --nr_entries $e --value_size $v bloom-false-positive)"
done
printf "\n"
done
done
done
header "Hashmap without bloom filter vs. hashmap with bloom filter (throughput, 8 threads)"
for v in 2 4 8 16 40; do
for h in {1..10}; do
subtitle "value_size: $v, # hashes: $h"
for e in 10000 50000 75000 100000 250000 500000 750000 1000000 2500000 5000000; do
printf "%'d entries -\n" $e
printf "\t"
summarize_total "Hashmap without bloom filter: " \
"$($RUN_BENCH --nr_hash_funcs $h --nr_entries $e --value_size $v -p 8 hashmap-without-bloom)"
printf "\t"
summarize_total "Hashmap with bloom filter: " \
"$($RUN_BENCH --nr_hash_funcs $h --nr_entries $e --value_size $v -p 8 hashmap-with-bloom)"
done
printf "\n"
done
done
linux-7.2.3/tools/testing/selftests/bpf/benchs/run_bench_bpf_hashmap_full_update.sh
Executable
+11
View File
@@ -0,0 +1,11 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
source ./benchs/run_common.sh
set -eufo pipefail
nr_threads=`expr $(cat /proc/cpuinfo | grep "processor"| wc -l) - 1`
summary=$($RUN_BENCH -p $nr_threads bpf-hashmap-full-update)
printf "$summary"
printf "\n"
linux-7.2.3/tools/testing/selftests/bpf/benchs/run_bench_bpf_loop.sh
Executable
+15
View File
@@ -0,0 +1,15 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
source ./benchs/run_common.sh
set -eufo pipefail
for t in 1 4 8 12 16; do
for i in 10 100 500 1000 5000 10000 50000 100000 500000 1000000; do
subtitle "nr_loops: $i, nr_threads: $t"
summarize_ops "bpf_loop: " \
"$($RUN_BENCH -p $t --nr_loops $i bpf-loop)"
printf "\n"
done
done
linux-7.2.3/tools/testing/selftests/bpf/benchs/run_bench_htab_mem.sh
Executable
+40
View File
@@ -0,0 +1,40 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
source ./benchs/run_common.sh
set -eufo pipefail
htab_mem()
{
echo -n "per-prod-op: "
echo -n "$*" | sed -E "s/.* per-prod-op\s+([0-9]+\.[0-9]+ ± [0-9]+\.[0-9]+k\/s).*/\1/"
echo -n -e ", avg mem: "
echo -n "$*" | sed -E "s/.* memory usage\s+([0-9]+\.[0-9]+ ± [0-9]+\.[0-9]+MiB).*/\1/"
echo -n ", peak mem: "
echo "$*" | sed -E "s/.* peak memory usage\s+([0-9]+\.[0-9]+MiB).*/\1/"
}
summarize_htab_mem()
{
local bench="$1"
local summary=$(echo $2 | tail -n1)
printf "%-20s %s\n" "$bench" "$(htab_mem $summary)"
}
htab_mem_bench()
{
local name
for name in overwrite batch_add_batch_del add_del_on_diff_cpu
do
summarize_htab_mem "$name" "$($RUN_BENCH htab-mem --use-case $name -p8 "$@")"
done
}
header "preallocated"
htab_mem_bench "--preallocated"
header "normal bpf ma"
htab_mem_bench
linux-7.2.3/tools/testing/selftests/bpf/benchs/run_bench_local_storage.sh
Executable
+24
View File
@@ -0,0 +1,24 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
source ./benchs/run_common.sh
set -eufo pipefail
header "Hashmap Control"
for i in 10 1000 10000 100000 4194304; do
subtitle "num keys: $i"
summarize_local_storage "hashmap (control) sequential get: "\
"$(./bench --nr_maps 1 --hashmap_nr_keys_used=$i local-storage-cache-hashmap-control)"
printf "\n"
done
header "Local Storage"
for i in 1 10 16 17 24 32 100 1000; do
subtitle "num_maps: $i"
summarize_local_storage "local_storage cache sequential get: "\
"$(./bench --nr_maps $i local-storage-cache-seq-get)"
summarize_local_storage "local_storage cache interleaved get: "\
"$(./bench --nr_maps $i local-storage-cache-int-get)"
printf "\n"
done
linux-7.2.3/tools/testing/selftests/bpf/benchs/run_bench_local_storage_rcu_tasks_trace.sh
Executable
+11
View File
@@ -0,0 +1,11 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
kthread_pid=`pgrep rcu_tasks_trace_kthread`
if [ -z $kthread_pid ]; then
echo "error: Couldn't find rcu_tasks_trace_kthread"
exit 1
fi
./bench --nr_procs 15000 --kthread_pid $kthread_pid -d 600 --quiet local-storage-tasks-trace
linux-7.2.3/tools/testing/selftests/bpf/benchs/run_bench_rename.sh
Executable
+9
View File
@@ -0,0 +1,9 @@
#!/bin/bash
set -eufo pipefail
for i in base kprobe kretprobe rawtp fentry fexit
do
summary=$(sudo ./bench -w2 -d5 -a rename-$i | tail -n1 | cut -d'(' -f1 | cut -d' ' -f3-)
printf "%-10s: %s\n" $i "$summary"
done
linux-7.2.3/tools/testing/selftests/bpf/benchs/run_bench_ringbufs.sh
Executable
+60
View File
@@ -0,0 +1,60 @@
#!/bin/bash
source ./benchs/run_common.sh
set -eufo pipefail
RUN_RB_BENCH="$RUN_BENCH -c1"
header "Single-producer, parallel producer"
for b in rb-libbpf rb-custom pb-libbpf pb-custom; do
summarize $b "$($RUN_RB_BENCH $b)"
done
header "Single-producer, parallel producer, sampled notification"
for b in rb-libbpf rb-custom pb-libbpf pb-custom; do
summarize $b "$($RUN_RB_BENCH --rb-sampled $b)"
done
header "Single-producer, back-to-back mode"
for b in rb-libbpf rb-custom pb-libbpf pb-custom; do
summarize $b "$($RUN_RB_BENCH --rb-b2b $b)"
summarize $b-sampled "$($RUN_RB_BENCH --rb-sampled --rb-b2b $b)"
done
header "Ringbuf back-to-back, effect of sample rate"
for b in 1 5 10 25 50 100 250 500 1000 2000 3000; do
summarize "rb-sampled-$b" "$($RUN_RB_BENCH --rb-b2b --rb-batch-cnt $b --rb-sampled --rb-sample-rate $b rb-custom)"
done
header "Perfbuf back-to-back, effect of sample rate"
for b in 1 5 10 25 50 100 250 500 1000 2000 3000; do
summarize "pb-sampled-$b" "$($RUN_RB_BENCH --rb-b2b --rb-batch-cnt $b --rb-sampled --rb-sample-rate $b pb-custom)"
done
header "Ringbuf back-to-back, reserve+commit vs output"
summarize "reserve" "$($RUN_RB_BENCH --rb-b2b rb-custom)"
summarize "output" "$($RUN_RB_BENCH --rb-b2b --rb-use-output rb-custom)"
header "Ringbuf sampled, reserve+commit vs output"
summarize "reserve-sampled" "$($RUN_RB_BENCH --rb-sampled rb-custom)"
summarize "output-sampled" "$($RUN_RB_BENCH --rb-sampled --rb-use-output rb-custom)"
header "Single-producer, consumer/producer competing on the same CPU, low batch count"
for b in rb-libbpf rb-custom pb-libbpf pb-custom; do
summarize $b "$($RUN_RB_BENCH --rb-batch-cnt 1 --rb-sample-rate 1 --prod-affinity 0 --cons-affinity 0 $b)"
done
header "Ringbuf, multi-producer contention"
for b in 1 2 3 4 8 12 16 20 24 28 32 36 40 44 48 52; do
summarize "rb-libbpf nr_prod $b" "$($RUN_RB_BENCH -p$b --rb-batch-cnt 50 rb-libbpf)"
done
header "Perfbuf, multi-producer"
for b in 1 2 3 4 8 12 16 20 24 28 32 36 40 44 48 52; do
summarize "pb-libbpf nr_prod $b" "$($RUN_RB_BENCH -p$b --rb-batch-cnt 50 --rb-sample-rate 50 pb-libbpf)"
done
header "Ringbuf, multi-producer contention in overwrite mode, no consumer"
for b in 1 2 3 4 8 12 16 20 24 28 32 36 40 44 48 52; do
summarize "rb-prod nr_prod $b" "$($RUN_BENCH -p$b --rb-batch-cnt 50 --rb-overwrite --rb-bench-producer rb-libbpf)"
done
linux-7.2.3/tools/testing/selftests/bpf/benchs/run_bench_strncmp.sh
Executable
+12
View File
@@ -0,0 +1,12 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
source ./benchs/run_common.sh
set -eufo pipefail
for s in 1 8 64 512 2048 4095; do
for b in no-helper helper; do
summarize ${b}-${s} "$($RUN_BENCH --cmp-str-len=$s strncmp-${b})"
done
done
linux-7.2.3/tools/testing/selftests/bpf/benchs/run_bench_trigger.sh
Executable
+23
View File
@@ -0,0 +1,23 @@
#!/bin/bash
set -eufo pipefail
def_tests=( \
usermode-count kernel-count syscall-count \
fentry fexit fmodret \
rawtp tp \
kprobe kprobe-multi kprobe-multi-all \
kretprobe kretprobe-multi kretprobe-multi-all \
)
tests=("$@")
if [ ${#tests[@]} -eq 0 ]; then
tests=("${def_tests[@]}")
fi
p=${PROD_CNT:-1}
for t in "${tests[@]}"; do
summary=$(sudo ./bench -w2 -d5 -a -p$p trig-$t | tail -n1 | cut -d'(' -f1 | cut -d' ' -f3-)
printf "%-15s: %s\n" $t "$summary"
done
linux-7.2.3/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh
Executable
+9
View File
@@ -0,0 +1,9 @@
#!/bin/bash
set -eufo pipefail
for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop5} usdt-nop usdt-nop5
do
summary=$(sudo ./bench -w2 -d5 -a trig-$i | tail -n1 | cut -d'(' -f1 | cut -d' ' -f3-)
printf "%-15s: %s\n" $i "$summary"
done
linux-7.2.3/tools/testing/selftests/bpf/benchs/run_bench_xdp_lb.sh
Executable
+79
View File
@@ -0,0 +1,79 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
source ./benchs/run_common.sh
set -eufo pipefail
WARMUP=${WARMUP:-3}
RUN="sudo ./bench -q -w${WARMUP} -a xdp-lb --machine-readable"
SEP=" +----------------------------------+----------+---------+----------+"
HDR=" | %-32s | %8s | %7s | %8s |\n"
ROW=" | %-32s | %8s | %7s | %8s |\n"
function group_header()
{
printf "%s\n" "$SEP"
printf "$HDR" "$1" "p50" "stddev" "p99"
printf "%s\n" "$SEP"
}
function rval()
{
echo "$1" | sed -nE "s/.*$2=([^ ]+).*/\1/p"
}
function run_scenario()
{
local sc="$1"
shift
local output rline
output=$($RUN --scenario "$sc" "$@" 2>&1) || true
rline=$(echo "$output" | grep '^RESULT ' || true)
if [ -z "$rline" ]; then
printf "$ROW" "$sc" "ERR" "-" "-"
return
fi
printf "$ROW" "$sc" \
"$(rval "$rline" median)" \
"$(rval "$rline" stddev)" \
"$(rval "$rline" p99)"
}
header "XDP load-balancer benchmark"
group_header "Single-flow baseline"
for sc in tcp-v4-lru-hit tcp-v4-ch \
tcp-v6-lru-hit tcp-v6-ch \
udp-v4-lru-hit udp-v6-lru-hit \
tcp-v4v6-lru-hit; do
run_scenario "$sc"
done
group_header "Diverse flows (4K src addrs)"
for sc in tcp-v4-lru-diverse tcp-v4-ch-diverse \
tcp-v6-lru-diverse tcp-v6-ch-diverse \
udp-v4-lru-diverse; do
run_scenario "$sc"
done
group_header "TCP flags"
run_scenario tcp-v4-syn
run_scenario tcp-v4-rst-miss
group_header "LRU stress"
run_scenario tcp-v4-lru-miss
run_scenario udp-v4-lru-miss
run_scenario tcp-v4-lru-warmup
group_header "Early exits"
for sc in pass-v4-no-vip pass-v6-no-vip pass-v4-icmp pass-non-ip drop-v4-frag drop-v4-options \
drop-v6-frag; do
run_scenario "$sc"
done
printf "%s\n" "$SEP"
linux-7.2.3/tools/testing/selftests/bpf/benchs/run_common.sh
+92
View File
@@ -0,0 +1,92 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
RUN_BENCH="sudo ./bench -w3 -d10 -a"
function header()
{
local len=${#1}
printf "\n%s\n" "$1"
for i in $(seq 1 $len); do printf '='; done
printf '\n'
}
function subtitle()
{
local len=${#1}
printf "\t%s\n" "$1"
}
function hits()
{
echo "$*" | sed -E "s/.*hits\s+([0-9]+\.[0-9]+ ± [0-9]+\.[0-9]+M\/s).*/\1/"
}
function drops()
{
echo "$*" | sed -E "s/.*drops\s+([0-9]+\.[0-9]+ ± [0-9]+\.[0-9]+M\/s).*/\1/"
}
function percentage()
{
echo "$*" | sed -E "s/.*Percentage\s=\s+([0-9]+\.[0-9]+).*/\1/"
}
function ops()
{
echo -n "throughput: "
echo -n "$*" | sed -E "s/.*throughput\s+([0-9]+\.[0-9]+ ± [0-9]+\.[0-9]+\sM\sops\/s).*/\1/"
echo -n -e ", latency: "
echo "$*" | sed -E "s/.*latency\s+([0-9]+\.[0-9]+\sns\/op).*/\1/"
}
function local_storage()
{
echo -n "hits throughput: "
echo -n "$*" | sed -E "s/.* hits throughput\s+([0-9]+\.[0-9]+ ± [0-9]+\.[0-9]+\sM\sops\/s).*/\1/"
echo -n -e ", hits latency: "
echo -n "$*" | sed -E "s/.* hits latency\s+([0-9]+\.[0-9]+\sns\/op).*/\1/"
echo -n ", important_hits throughput: "
echo "$*" | sed -E "s/.*important_hits throughput\s+([0-9]+\.[0-9]+ ± [0-9]+\.[0-9]+\sM\sops\/s).*/\1/"
}
function total()
{
echo "$*" | sed -E "s/.*total operations\s+([0-9]+\.[0-9]+ ± [0-9]+\.[0-9]+M\/s).*/\1/"
}
function summarize()
{
bench="$1"
summary=$(echo $2 | tail -n1)
printf "%-20s %s (drops %s)\n" "$bench" "$(hits $summary)" "$(drops $summary)"
}
function summarize_percentage()
{
bench="$1"
summary=$(echo $2 | tail -n1)
printf "%-20s %s%%\n" "$bench" "$(percentage $summary)"
}
function summarize_ops()
{
bench="$1"
summary=$(echo $2 | tail -n1)
printf "%-20s %s\n" "$bench" "$(ops $summary)"
}
function summarize_local_storage()
{
bench="$1"
summary=$(echo $2 | tail -n1)
printf "%-20s %s\n" "$bench" "$(local_storage $summary)"
}
function summarize_total()
{
bench="$1"
summary=$(echo $2 | tail -n1)
printf "%-20s %s\n" "$bench" "$(total $summary)"
}
linux-7.2.3/tools/testing/selftests/bpf/bpf_arena_alloc.h
+67
View File
@@ -0,0 +1,67 @@
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#pragma once
#include <bpf_arena_common.h>
#ifndef __round_mask
#define __round_mask(x, y) ((__typeof__(x))((y)-1))
#endif
#ifndef round_up
#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
#endif
#ifdef __BPF__
#define NR_CPUS (sizeof(struct cpumask) * 8)
static void __arena * __arena page_frag_cur_page[NR_CPUS];
static int __arena page_frag_cur_offset[NR_CPUS];
/* Simple page_frag allocator */
static inline void __arena* bpf_alloc(unsigned int size)
{
__u64 __arena *obj_cnt;
__u32 cpu = bpf_get_smp_processor_id();
void __arena *page = page_frag_cur_page[cpu];
int __arena *cur_offset = &page_frag_cur_offset[cpu];
int offset;
size = round_up(size, 8);
if (size >= PAGE_SIZE - 8)
return NULL;
if (!page) {
refill:
page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
if (!page)
return NULL;
cast_kern(page);
page_frag_cur_page[cpu] = page;
*cur_offset = PAGE_SIZE - 8;
obj_cnt = page + PAGE_SIZE - 8;
*obj_cnt = 0;
} else {
cast_kern(page);
obj_cnt = page + PAGE_SIZE - 8;
}
offset = *cur_offset - size;
if (offset < 0)
goto refill;
(*obj_cnt)++;
*cur_offset = offset;
return page + offset;
}
static inline void bpf_free(void __arena *addr)
{
__u64 __arena *obj_cnt;
addr = (void __arena *)(((long)addr) & ~(PAGE_SIZE - 1));
obj_cnt = addr + PAGE_SIZE - 8;
if (--(*obj_cnt) == 0)
bpf_arena_free_pages(&arena, addr, 1);
}
#else
static inline void __arena* bpf_alloc(unsigned int size) { return NULL; }
static inline void bpf_free(void __arena *addr) {}
#endif
linux-7.2.3/tools/testing/selftests/bpf/bpf_arena_htab.h
+99
View File
@@ -0,0 +1,99 @@
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#pragma once
#include <errno.h>
#include "bpf_arena_alloc.h"
#include "bpf_arena_list.h"
struct htab_bucket {
struct arena_list_head head;
};
typedef struct htab_bucket __arena htab_bucket_t;
struct htab {
htab_bucket_t *buckets;
int n_buckets;
};
static inline htab_bucket_t *__select_bucket(struct htab __arena *htab, __u32 hash)
{
htab_bucket_t *b = htab->buckets;
cast_kern(b);
return &b[hash & (htab->n_buckets - 1)];
}
static inline arena_list_head_t *select_bucket(struct htab __arena *htab, __u32 hash)
{
return &__select_bucket(htab, hash)->head;
}
struct hashtab_elem {
int hash;
int key;
int value;
struct arena_list_node hash_node;
};
typedef struct hashtab_elem __arena hashtab_elem_t;
static hashtab_elem_t *lookup_elem_raw(arena_list_head_t *head, __u32 hash, int key)
{
hashtab_elem_t *l;
list_for_each_entry(l, head, hash_node)
if (l->hash == hash && l->key == key)
return l;
return NULL;
}
static int htab_hash(int key)
{
return key;
}
__weak int htab_lookup_elem(struct htab __arena *htab, int key)
{
hashtab_elem_t *l_old;
arena_list_head_t *head;
cast_kern(htab);
head = select_bucket(htab, key);
l_old = lookup_elem_raw(head, htab_hash(key), key);
if (l_old)
return l_old->value;
return 0;
}
__weak int htab_update_elem(struct htab __arena *htab, int key, int value)
{
hashtab_elem_t *l_new = NULL, *l_old;
arena_list_head_t *head;
cast_kern(htab);
head = select_bucket(htab, key);
l_old = lookup_elem_raw(head, htab_hash(key), key);
l_new = bpf_alloc(sizeof(*l_new));
if (!l_new)
return -ENOMEM;
l_new->key = key;
l_new->hash = htab_hash(key);
l_new->value = value;
list_add_head(&l_new->hash_node, head);
if (l_old) {
list_del(&l_old->hash_node);
bpf_free(l_old);
}
return 0;
}
void htab_init(struct htab __arena *htab)
{
void __arena *buckets = bpf_arena_alloc_pages(&arena, NULL, 2, NUMA_NO_NODE, 0);
cast_user(buckets);
htab->buckets = buckets;
htab->n_buckets = 2 * PAGE_SIZE / sizeof(struct htab_bucket);
}
linux-7.2.3/tools/testing/selftests/bpf/bpf_arena_list.h
+90
View File
@@ -0,0 +1,90 @@
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#pragma once
#include <bpf_arena_common.h>
struct arena_list_node;
typedef struct arena_list_node __arena arena_list_node_t;
struct arena_list_node {
arena_list_node_t *next;
arena_list_node_t * __arena *pprev;
};
struct arena_list_head {
struct arena_list_node __arena *first;
};
typedef struct arena_list_head __arena arena_list_head_t;
#define list_entry(ptr, type, member) arena_container_of(ptr, type, member)
#define list_entry_safe(ptr, type, member) \
({ typeof(*ptr) * ___ptr = (ptr); \
___ptr ? ({ cast_kern(___ptr); list_entry(___ptr, type, member); }) : NULL; \
})
#ifndef __BPF__
static inline void *bpf_iter_num_new(struct bpf_iter_num *it, int i, int j) { return NULL; }
static inline void bpf_iter_num_destroy(struct bpf_iter_num *it) {}
static inline bool bpf_iter_num_next(struct bpf_iter_num *it) { return true; }
#define cond_break ({})
#define can_loop true
#endif
/* Safely walk link list elements. Deletion of elements is allowed. */
#define list_for_each_entry(pos, head, member) \
for (void * ___tmp = (pos = list_entry_safe((head)->first, \
typeof(*(pos)), member), \
(void *)0); \
pos && ({ ___tmp = (void *)pos->member.next; 1; }) && can_loop; \
pos = list_entry_safe((void __arena *)___tmp, typeof(*(pos)), member))
static inline void list_add_head(arena_list_node_t *n, arena_list_head_t *h)
{
arena_list_node_t *first = h->first, * __arena *tmp;
cast_user(first);
cast_kern(n);
WRITE_ONCE(n->next, first);
cast_kern(first);
if (first) {
tmp = &n->next;
cast_user(tmp);
WRITE_ONCE(first->pprev, tmp);
}
cast_user(n);
WRITE_ONCE(h->first, n);
tmp = &h->first;
cast_user(tmp);
cast_kern(n);
WRITE_ONCE(n->pprev, tmp);
}
static inline void __list_del(arena_list_node_t *n)
{
arena_list_node_t *next = n->next;
arena_list_node_t * __arena *pprev = n->pprev;
cast_user(next);
cast_kern(pprev);
WRITE_ONCE(*pprev, next);
if (next) {
cast_user(pprev);
cast_kern(next);
WRITE_ONCE(next->pprev, pprev);
}
}
#define POISON_POINTER_DELTA 0
#define LIST_POISON1 ((void __arena *) 0x100 + POISON_POINTER_DELTA)
#define LIST_POISON2 ((void __arena *) 0x122 + POISON_POINTER_DELTA)
static inline void list_del(arena_list_node_t *n)
{
__list_del(n);
n->next = LIST_POISON1;
n->pprev = LIST_POISON2;
}
linux-7.2.3/tools/testing/selftests/bpf/bpf_arena_strsearch.h
+128
View File
@@ -0,0 +1,128 @@
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#pragma once
#include <bpf_arena_common.h>
__noinline int bpf_arena_strlen(const char __arena *s)
{
const char __arena *sc;
for (sc = s; *sc != '\0'; ++sc)
cond_break;
return sc - s;
}
/**
* glob_match - Shell-style pattern matching, like !fnmatch(pat, str, 0)
* @pat: Shell-style pattern to match, e.g. "*.[ch]".
* @str: String to match. The pattern must match the entire string.
*
* Perform shell-style glob matching, returning true (1) if the match
* succeeds, or false (0) if it fails. Equivalent to !fnmatch(@pat, @str, 0).
*
* Pattern metacharacters are ?, *, [ and \.
* (And, inside character classes, !, - and ].)
*
* This is small and simple implementation intended for device blacklists
* where a string is matched against a number of patterns. Thus, it
* does not preprocess the patterns. It is non-recursive, and run-time
* is at most quadratic: strlen(@str)*strlen(@pat).
*
* An example of the worst case is glob_match("*aaaaa", "aaaaaaaaaa");
* it takes 6 passes over the pattern before matching the string.
*
* Like !fnmatch(@pat, @str, 0) and unlike the shell, this does NOT
* treat / or leading . specially; it isn't actually used for pathnames.
*
* Note that according to glob(7) (and unlike bash), character classes
* are complemented by a leading !; this does not support the regex-style
* [^a-z] syntax.
*
* An opening bracket without a matching close is matched literally.
*/
__noinline bool glob_match(char const __arena *pat, char const __arena *str)
{
/*
* Backtrack to previous * on mismatch and retry starting one
* character later in the string. Because * matches all characters
* (no exception for /), it can be easily proved that there's
* never a need to backtrack multiple levels.
*/
char const __arena *back_pat = NULL, *back_str;
/*
* Loop over each token (character or class) in pat, matching
* it against the remaining unmatched tail of str. Return false
* on mismatch, or true after matching the trailing nul bytes.
*/
for (;;) {
unsigned char c = *str++;
unsigned char d = *pat++;
switch (d) {
case '?': /* Wildcard: anything but nul */
if (c == '\0')
return false;
break;
case '*': /* Any-length wildcard */
if (*pat == '\0') /* Optimize trailing * case */
return true;
back_pat = pat;
back_str = --str; /* Allow zero-length match */
break;
case '[': { /* Character class */
bool match = false, inverted = (*pat == '!');
char const __arena *class = pat + inverted;
unsigned char a = *class++;
/*
* Iterate over each span in the character class.
* A span is either a single character a, or a
* range a-b. The first span may begin with ']'.
*/
do {
unsigned char b = a;
if (a == '\0') /* Malformed */
goto literal;
if (class[0] == '-' && class[1] != ']') {
b = class[1];
if (b == '\0')
goto literal;
class += 2;
/* Any special action if a > b? */
}
match |= (a <= c && c <= b);
cond_break;
} while ((a = *class++) != ']');
if (match == inverted)
goto backtrack;
pat = class;
}
break;
case '\\':
d = *pat++;
__attribute__((__fallthrough__));
default: /* Literal character */
literal:
if (c == d) {
if (d == '\0')
return true;
break;
}
backtrack:
if (c == '\0' || !back_pat)
return false; /* No point continuing */
/* Try again from last *, one character later in str. */
pat = back_pat;
str = ++back_str;
break;
}
cond_break;
}
return false;
}
linux-7.2.3/tools/testing/selftests/bpf/bpf_experimental.h
+522
View File
@@ -0,0 +1,522 @@
#ifndef __BPF_EXPERIMENTAL__
#define __BPF_EXPERIMENTAL__
#include <vmlinux.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
#include <bpf_may_goto.h>
#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node)))
/* Convenience macro to wrap over bpf_obj_new */
#define bpf_obj_new(type) ((type *)bpf_obj_new(bpf_core_type_id_local(type)))
/* Convenience macro to wrap over bpf_percpu_obj_new */
#define bpf_percpu_obj_new(type) ((type __percpu_kptr *)bpf_percpu_obj_new(bpf_core_type_id_local(type)))
struct bpf_iter_task_vma;
extern int bpf_iter_task_vma_new(struct bpf_iter_task_vma *it,
struct task_struct *task,
__u64 addr) __ksym;
extern struct vm_area_struct *bpf_iter_task_vma_next(struct bpf_iter_task_vma *it) __ksym;
extern void bpf_iter_task_vma_destroy(struct bpf_iter_task_vma *it) __ksym;
/* Description
* Throw a BPF exception from the program, immediately terminating its
* execution and unwinding the stack. The supplied 'cookie' parameter
* will be the return value of the program when an exception is thrown,
* and the default exception callback is used. Otherwise, if an exception
* callback is set using the '__exception_cb(callback)' declaration tag
* on the main program, the 'cookie' parameter will be the callback's only
* input argument.
*
* Thus, in case of default exception callback, 'cookie' is subjected to
* constraints on the program's return value (as with R0 on exit).
* Otherwise, the return value of the marked exception callback will be
* subjected to the same checks.
*
* Note that throwing an exception with lingering resources (locks,
* references, etc.) will lead to a verification error.
*
* Note that callbacks *cannot* call this helper.
* Returns
* Never.
* Throws
* An exception with the specified 'cookie' value.
*/
extern void bpf_throw(u64 cookie) __ksym;
/* Description
* Acquire a reference on the exe_file member field belonging to the
* mm_struct that is nested within the supplied task_struct. The supplied
* task_struct must be trusted/referenced.
* Returns
* A referenced file pointer pointing to the exe_file member field of the
* mm_struct nested in the supplied task_struct, or NULL.
*/
extern struct file *bpf_get_task_exe_file(struct task_struct *task) __ksym;
/* Description
* Release a reference on the supplied file. The supplied file must be
* acquired.
*/
extern void bpf_put_file(struct file *file) __ksym;
/* Description
* Resolve a pathname for the supplied path and store it in the supplied
* buffer. The supplied path must be trusted/referenced.
* Returns
* A positive integer corresponding to the length of the resolved pathname,
* including the NULL termination character, stored in the supplied
* buffer. On error, a negative integer is returned.
*/
extern int bpf_path_d_path(const struct path *path, char *buf, size_t buf__sz) __ksym;
/* This macro must be used to mark the exception callback corresponding to the
* main program. For example:
*
* int exception_cb(u64 cookie) {
* return cookie;
* }
*
* SEC("tc")
* __exception_cb(exception_cb)
* int main_prog(struct __sk_buff *ctx) {
* ...
* return TC_ACT_OK;
* }
*
* Here, exception callback for the main program will be 'exception_cb'. Note
* that this attribute can only be used once, and multiple exception callbacks
* specified for the main program will lead to verification error.
*/
#define __exception_cb(name) __attribute__((btf_decl_tag("exception_callback:" #name)))
#define __bpf_assert_signed(x) _Generic((x), \
unsigned long: 0, \
unsigned long long: 0, \
signed long: 1, \
signed long long: 1 \
)
#define __bpf_assert_check(LHS, op, RHS) \
_Static_assert(sizeof(&(LHS)), "1st argument must be an lvalue expression"); \
_Static_assert(sizeof(LHS) == 8, "Only 8-byte integers are supported\n"); \
_Static_assert(__builtin_constant_p(__bpf_assert_signed(LHS)), "internal static assert"); \
_Static_assert(__builtin_constant_p((RHS)), "2nd argument must be a constant expression")
#define __bpf_assert(LHS, op, cons, RHS, VAL) \
({ \
(void)bpf_throw; \
asm volatile ("if %[lhs] " op " %[rhs] goto +2; r1 = %[value]; call bpf_throw" \
: : [lhs] "r"(LHS), [rhs] cons(RHS), [value] "ri"(VAL) : ); \
})
#define __bpf_assert_op_sign(LHS, op, cons, RHS, VAL, supp_sign) \
({ \
__bpf_assert_check(LHS, op, RHS); \
if (__bpf_assert_signed(LHS) && !(supp_sign)) \
__bpf_assert(LHS, "s" #op, cons, RHS, VAL); \
else \
__bpf_assert(LHS, #op, cons, RHS, VAL); \
})
#define __bpf_assert_op(LHS, op, RHS, VAL, supp_sign) \
({ \
if (sizeof(typeof(RHS)) == 8) { \
const typeof(RHS) rhs_var = (RHS); \
__bpf_assert_op_sign(LHS, op, "r", rhs_var, VAL, supp_sign); \
} else { \
__bpf_assert_op_sign(LHS, op, "i", RHS, VAL, supp_sign); \
} \
})
#define __cmp_cannot_be_signed(x) \
__builtin_strcmp(#x, "==") == 0 || __builtin_strcmp(#x, "!=") == 0 || \
__builtin_strcmp(#x, "&") == 0
#define __is_signed_type(type) (((type)(-1)) < (type)1)
#define __bpf_cmp(LHS, OP, PRED, RHS, DEFAULT) \
({ \
__label__ l_true; \
bool ret = DEFAULT; \
asm volatile goto("if %[lhs] " OP " %[rhs] goto %l[l_true]" \
:: [lhs] "r"((short)LHS), [rhs] PRED (RHS) :: l_true); \
ret = !DEFAULT; \
l_true: \
ret; \
})
/* C type conversions coupled with comparison operator are tricky.
* Make sure BPF program is compiled with -Wsign-compare then
* __lhs OP __rhs below will catch the mistake.
* Be aware that we check only __lhs to figure out the sign of compare.
*/
#define _bpf_cmp(LHS, OP, RHS, UNLIKELY) \
({ \
typeof(LHS) __lhs = (LHS); \
typeof(RHS) __rhs = (RHS); \
bool ret; \
_Static_assert(sizeof(&(LHS)), "1st argument must be an lvalue expression"); \
(void)(__lhs OP __rhs); \
if (__cmp_cannot_be_signed(OP) || !__is_signed_type(typeof(__lhs))) { \
if (sizeof(__rhs) == 8) \
/* "i" will truncate 64-bit constant into s32, \
* so we have to use extra register via "r". \
*/ \
ret = __bpf_cmp(__lhs, #OP, "r", __rhs, UNLIKELY); \
else \
ret = __bpf_cmp(__lhs, #OP, "ri", __rhs, UNLIKELY); \
} else { \
if (sizeof(__rhs) == 8) \
ret = __bpf_cmp(__lhs, "s"#OP, "r", __rhs, UNLIKELY); \
else \
ret = __bpf_cmp(__lhs, "s"#OP, "ri", __rhs, UNLIKELY); \
} \
ret; \
})
#ifndef bpf_cmp_unlikely
#define bpf_cmp_unlikely(LHS, OP, RHS) _bpf_cmp(LHS, OP, RHS, true)
#endif
#ifndef bpf_cmp_likely
#define bpf_cmp_likely(LHS, OP, RHS) \
({ \
bool ret = 0; \
if (__builtin_strcmp(#OP, "==") == 0) \
ret = _bpf_cmp(LHS, !=, RHS, false); \
else if (__builtin_strcmp(#OP, "!=") == 0) \
ret = _bpf_cmp(LHS, ==, RHS, false); \
else if (__builtin_strcmp(#OP, "<=") == 0) \
ret = _bpf_cmp(LHS, >, RHS, false); \
else if (__builtin_strcmp(#OP, "<") == 0) \
ret = _bpf_cmp(LHS, >=, RHS, false); \
else if (__builtin_strcmp(#OP, ">") == 0) \
ret = _bpf_cmp(LHS, <=, RHS, false); \
else if (__builtin_strcmp(#OP, ">=") == 0) \
ret = _bpf_cmp(LHS, <, RHS, false); \
else \
asm volatile("r0 " #OP " invalid compare"); \
ret; \
})
#endif
#ifndef bpf_nop_mov
#define bpf_nop_mov(var) \
asm volatile("%[reg]=%[reg]"::[reg]"r"((short)var))
#endif
/* emit instruction:
* rX = rX .off = BPF_ADDR_SPACE_CAST .imm32 = (dst_as << 16) | src_as
*/
#ifndef bpf_addr_space_cast
#define bpf_addr_space_cast(var, dst_as, src_as)\
asm volatile(".byte 0xBF; \
.ifc %[reg], r0; \
.byte 0x00; \
.endif; \
.ifc %[reg], r1; \
.byte 0x11; \
.endif; \
.ifc %[reg], r2; \
.byte 0x22; \
.endif; \
.ifc %[reg], r3; \
.byte 0x33; \
.endif; \
.ifc %[reg], r4; \
.byte 0x44; \
.endif; \
.ifc %[reg], r5; \
.byte 0x55; \
.endif; \
.ifc %[reg], r6; \
.byte 0x66; \
.endif; \
.ifc %[reg], r7; \
.byte 0x77; \
.endif; \
.ifc %[reg], r8; \
.byte 0x88; \
.endif; \
.ifc %[reg], r9; \
.byte 0x99; \
.endif; \
.short %[off]; \
.long %[as]" \
: [reg]"+r"(var) \
: [off]"i"(BPF_ADDR_SPACE_CAST) \
, [as]"i"((dst_as << 16) | src_as));
#endif
void bpf_preempt_disable(void) __weak __ksym;
void bpf_preempt_enable(void) __weak __ksym;
typedef struct {
} __bpf_preempt_t;
static inline __bpf_preempt_t __bpf_preempt_constructor(void)
{
__bpf_preempt_t ret = {};
bpf_preempt_disable();
return ret;
}
static inline void __bpf_preempt_destructor(__bpf_preempt_t *t)
{
bpf_preempt_enable();
}
#define bpf_guard_preempt() \
__bpf_preempt_t ___bpf_apply(preempt, __COUNTER__) \
__attribute__((__unused__, __cleanup__(__bpf_preempt_destructor))) = \
__bpf_preempt_constructor()
/* Description
* Assert that a conditional expression is true.
* Returns
* Void.
* Throws
* An exception with the value zero when the assertion fails.
*/
#define bpf_assert(cond) if (!(cond)) bpf_throw(0);
/* Description
* Assert that a conditional expression is true.
* Returns
* Void.
* Throws
* An exception with the specified value when the assertion fails.
*/
#define bpf_assert_with(cond, value) if (!(cond)) bpf_throw(value);
/* Description
* Assert that LHS is in the range [BEG, END] (inclusive of both). This
* statement updates the known bounds of LHS during verification. Note
* that both BEG and END must be constant values, and must fit within the
* data type of LHS.
* Returns
* Void.
* Throws
* An exception with the value zero when the assertion fails.
*/
#define bpf_assert_range(LHS, BEG, END) \
({ \
_Static_assert(BEG <= END, "BEG must be <= END"); \
barrier_var(LHS); \
__bpf_assert_op(LHS, >=, BEG, 0, false); \
__bpf_assert_op(LHS, <=, END, 0, false); \
})
/* Description
* Assert that LHS is in the range [BEG, END] (inclusive of both). This
* statement updates the known bounds of LHS during verification. Note
* that both BEG and END must be constant values, and must fit within the
* data type of LHS.
* Returns
* Void.
* Throws
* An exception with the specified value when the assertion fails.
*/
#define bpf_assert_range_with(LHS, BEG, END, value) \
({ \
_Static_assert(BEG <= END, "BEG must be <= END"); \
barrier_var(LHS); \
__bpf_assert_op(LHS, >=, BEG, value, false); \
__bpf_assert_op(LHS, <=, END, value, false); \
})
struct bpf_iter_css_task;
struct cgroup_subsys_state;
extern int bpf_iter_css_task_new(struct bpf_iter_css_task *it,
struct cgroup_subsys_state *css, unsigned int flags) __weak __ksym;
extern struct task_struct *bpf_iter_css_task_next(struct bpf_iter_css_task *it) __weak __ksym;
extern void bpf_iter_css_task_destroy(struct bpf_iter_css_task *it) __weak __ksym;
struct bpf_iter_task;
extern int bpf_iter_task_new(struct bpf_iter_task *it,
struct task_struct *task, unsigned int flags) __weak __ksym;
extern struct task_struct *bpf_iter_task_next(struct bpf_iter_task *it) __weak __ksym;
extern void bpf_iter_task_destroy(struct bpf_iter_task *it) __weak __ksym;
struct bpf_iter_css;
extern int bpf_iter_css_new(struct bpf_iter_css *it,
struct cgroup_subsys_state *start, unsigned int flags) __weak __ksym;
extern struct cgroup_subsys_state *bpf_iter_css_next(struct bpf_iter_css *it) __weak __ksym;
extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym;
extern int bpf_wq_init(struct bpf_wq *wq, void *p__map, unsigned int flags) __weak __ksym;
extern int bpf_wq_start(struct bpf_wq *wq, unsigned int flags) __weak __ksym;
struct bpf_iter_kmem_cache;
extern int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache *it) __weak __ksym;
extern struct kmem_cache *bpf_iter_kmem_cache_next(struct bpf_iter_kmem_cache *it) __weak __ksym;
extern void bpf_iter_kmem_cache_destroy(struct bpf_iter_kmem_cache *it) __weak __ksym;
struct bpf_iter_dmabuf;
extern int bpf_iter_dmabuf_new(struct bpf_iter_dmabuf *it) __weak __ksym;
extern struct dma_buf *bpf_iter_dmabuf_next(struct bpf_iter_dmabuf *it) __weak __ksym;
extern void bpf_iter_dmabuf_destroy(struct bpf_iter_dmabuf *it) __weak __ksym;
extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str,
struct bpf_dynptr *value_p) __weak __ksym;
#define PREEMPT_BITS 8
#define SOFTIRQ_BITS 8
#define HARDIRQ_BITS 4
#define NMI_BITS 4
#define PREEMPT_SHIFT 0
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
#define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS)
#define __IRQ_MASK(x) ((1UL << (x))-1)
#define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
#define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
#define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT)
#define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT)
extern bool CONFIG_PREEMPT_RT __kconfig __weak;
#ifdef bpf_target_x86
extern const int __preempt_count __ksym __weak;
struct pcpu_hot___local {
int preempt_count;
} __attribute__((preserve_access_index));
extern struct pcpu_hot___local pcpu_hot __ksym __weak;
#endif
struct task_struct___preempt_rt {
int softirq_disable_cnt;
} __attribute__((preserve_access_index));
#ifdef bpf_target_s390
extern struct lowcore *bpf_get_lowcore(void) __weak __ksym;
#endif
static inline int get_preempt_count(void)
{
#if defined(bpf_target_x86)
/* By default, read the per-CPU __preempt_count. */
if (bpf_ksym_exists(&__preempt_count))
return *(int *) bpf_this_cpu_ptr(&__preempt_count);
/*
* If __preempt_count does not exist, try to read preempt_count under
* struct pcpu_hot. Between v6.1 and v6.14 -- more specifically,
* [64701838bf057, 46e8fff6d45fe), preempt_count had been managed
* under struct pcpu_hot.
*/
if (bpf_core_field_exists(pcpu_hot.preempt_count))
return ((struct pcpu_hot___local *)
bpf_this_cpu_ptr(&pcpu_hot))->preempt_count;
#elif defined(bpf_target_arm64)
return bpf_get_current_task_btf()->thread_info.preempt.count;
#elif defined(bpf_target_powerpc)
return bpf_get_current_task_btf()->thread_info.preempt_count;
#elif defined(bpf_target_s390)
return bpf_get_lowcore()->preempt_count;
#elif defined(bpf_target_loongarch)
return bpf_get_current_task_btf()->thread_info.preempt_count;
#endif
return 0;
}
/* Description
* Report whether it is in interrupt context. Only works on the following archs:
* * x86
* * arm64
* * powerpc64
* * s390x
* * loongarch
*/
static inline int bpf_in_interrupt(void)
{
struct task_struct___preempt_rt *tsk;
int pcnt;
pcnt = get_preempt_count();
if (!CONFIG_PREEMPT_RT)
return pcnt & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_MASK);
tsk = (void *) bpf_get_current_task_btf();
return (pcnt & (NMI_MASK | HARDIRQ_MASK)) |
(tsk->softirq_disable_cnt & SOFTIRQ_MASK);
}
/* Description
* Report whether it is in NMI context. Only works on the following archs:
* * x86
* * arm64
* * powerpc64
* * s390x
* * loongarch
*/
static inline int bpf_in_nmi(void)
{
return get_preempt_count() & NMI_MASK;
}
/* Description
* Report whether it is in hard IRQ context. Only works on the following archs:
* * x86
* * arm64
* * powerpc64
* * s390x
* * loongarch
*/
static inline int bpf_in_hardirq(void)
{
return get_preempt_count() & HARDIRQ_MASK;
}
/* Description
* Report whether it is in softirq context. Only works on the following archs:
* * x86
* * arm64
* * powerpc64
* * s390x
* * loongarch
*/
static inline int bpf_in_serving_softirq(void)
{
struct task_struct___preempt_rt *tsk;
int pcnt;
pcnt = get_preempt_count();
if (!CONFIG_PREEMPT_RT)
return (pcnt & SOFTIRQ_MASK) & SOFTIRQ_OFFSET;
tsk = (void *) bpf_get_current_task_btf();
return (tsk->softirq_disable_cnt & SOFTIRQ_MASK) & SOFTIRQ_OFFSET;
}
/* Description
* Report whether it is in task context. Only works on the following archs:
* * x86
* * arm64
* * powerpc64
* * s390x
* * loongarch
*/
static inline int bpf_in_task(void)
{
struct task_struct___preempt_rt *tsk;
int pcnt;
pcnt = get_preempt_count();
if (!CONFIG_PREEMPT_RT)
return !(pcnt & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));
tsk = (void *) bpf_get_current_task_btf();
return !((pcnt & (NMI_MASK | HARDIRQ_MASK)) |
((tsk->softirq_disable_cnt & SOFTIRQ_MASK) & SOFTIRQ_OFFSET));
}
#endif
linux-7.2.3/tools/testing/selftests/bpf/bpf_kfuncs.h
+95
View File
@@ -0,0 +1,95 @@
#ifndef __BPF_KFUNCS__
#define __BPF_KFUNCS__
struct bpf_sock_addr_kern;
/* Description
* Initializes an skb-type dynptr
* Returns
* Error code
*/
extern int bpf_dynptr_from_skb(struct __sk_buff *skb, __u64 flags,
struct bpf_dynptr *ptr__uninit) __ksym __weak;
/* Description
* Initializes an xdp-type dynptr
* Returns
* Error code
*/
extern int bpf_dynptr_from_xdp(struct xdp_md *xdp, __u64 flags,
struct bpf_dynptr *ptr__uninit) __ksym __weak;
extern int bpf_dynptr_from_skb_meta(struct __sk_buff *skb, __u64 flags,
struct bpf_dynptr *ptr__uninit) __ksym __weak;
/* Description
* Obtain a read-only pointer to the dynptr's data
* Returns
* Either a direct pointer to the dynptr data or a pointer to the user-provided
* buffer if unable to obtain a direct pointer
*/
extern void *bpf_dynptr_slice(const struct bpf_dynptr *ptr, __u64 offset,
void *buffer, __u64 buffer__szk) __ksym __weak;
/* Description
* Obtain a read-write pointer to the dynptr's data
* Returns
* Either a direct pointer to the dynptr data or a pointer to the user-provided
* buffer if unable to obtain a direct pointer
*/
extern void *bpf_dynptr_slice_rdwr(const struct bpf_dynptr *ptr, __u64 offset, void *buffer,
__u64 buffer__szk) __ksym __weak;
extern int bpf_dynptr_adjust(struct bpf_dynptr *ptr, __u64 start, __u64 end) __ksym __weak;
extern bool bpf_dynptr_is_null(const struct bpf_dynptr *ptr) __ksym __weak;
extern bool bpf_dynptr_is_rdonly(const struct bpf_dynptr *ptr) __ksym __weak;
extern __u64 bpf_dynptr_size(const struct bpf_dynptr *ptr) __ksym __weak;
extern int bpf_dynptr_clone(const struct bpf_dynptr *ptr, struct bpf_dynptr *clone__init) __ksym __weak;
/* Description
* Modify the address of a AF_UNIX sockaddr.
* Returns
* -EINVAL if the address size is too big or, 0 if the sockaddr was successfully modified.
*/
extern int bpf_sock_addr_set_sun_path(struct bpf_sock_addr_kern *sa_kern,
const __u8 *sun_path, __u32 sun_path__sz) __ksym;
/* Description
* Allocate and configure a reqsk and link it with a listener and skb.
* Returns
* Error code
*/
struct sock;
struct bpf_tcp_req_attrs;
extern int bpf_sk_assign_tcp_reqsk(struct __sk_buff *skb, struct sock *sk,
struct bpf_tcp_req_attrs *attrs, int attrs__sz) __ksym;
void *bpf_cast_to_kern_ctx(void *) __ksym;
extern void *bpf_rdonly_cast(const void *obj, __u32 btf_id) __ksym __weak;
extern int bpf_get_file_xattr(struct file *file, const char *name,
struct bpf_dynptr *value_ptr) __ksym;
extern int bpf_get_fsverity_digest(struct file *file, const struct bpf_dynptr *digest_ptr) __ksym;
extern struct bpf_key *bpf_lookup_user_key(__s32 serial, __u64 flags) __ksym;
extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym;
extern void bpf_key_put(struct bpf_key *key) __ksym;
extern int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_ptr,
const struct bpf_dynptr *sig_ptr,
struct bpf_key *trusted_keyring) __ksym;
struct dentry;
/* Description
* Returns xattr of a dentry
* Returns
* Error code
*/
extern int bpf_get_dentry_xattr(struct dentry *dentry, const char *name,
struct bpf_dynptr *value_ptr) __ksym __weak;
extern int bpf_set_dentry_xattr(struct dentry *dentry, const char *name__str,
const struct bpf_dynptr *value_p, int flags) __ksym __weak;
extern int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name__str) __ksym __weak;
#endif
linux-7.2.3/tools/testing/selftests/bpf/bpf_legacy.h
+23
View File
@@ -0,0 +1,23 @@
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
#ifndef __BPF_LEGACY__
#define __BPF_LEGACY__
#if __GNUC__ && !__clang__
/* Functions to emit BPF_LD_ABS and BPF_LD_IND instructions. We
* provide the "standard" names as synonyms of the corresponding GCC
* builtins. Note how the SKB argument is ignored.
*/
#define load_byte(skb, off) __builtin_bpf_load_byte(off)
#define load_half(skb, off) __builtin_bpf_load_half(off)
#define load_word(skb, off) __builtin_bpf_load_word(off)
#else
/* llvm builtin functions that eBPF C program may use to
* emit BPF_LD_ABS and BPF_LD_IND instructions
*/
unsigned long long load_byte(void *skb, unsigned long long off) asm("llvm.bpf.load.byte");
unsigned long long load_half(void *skb, unsigned long long off) asm("llvm.bpf.load.half");
unsigned long long load_word(void *skb, unsigned long long off) asm("llvm.bpf.load.word");
#endif
#endif
linux-7.2.3/tools/testing/selftests/bpf/bpf_rand.h
+80
View File
@@ -0,0 +1,80 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __BPF_RAND__
#define __BPF_RAND__
#include <stdint.h>
#include <stdlib.h>
#include <time.h>
static inline uint64_t bpf_rand_mask(uint64_t mask)
{
return (((uint64_t)(uint32_t)rand()) |
((uint64_t)(uint32_t)rand() << 32)) & mask;
}
#define bpf_rand_ux(x, m) \
static inline uint64_t bpf_rand_u##x(int shift) \
{ \
return bpf_rand_mask((m)) << shift; \
}
bpf_rand_ux( 8, 0xffULL)
bpf_rand_ux(16, 0xffffULL)
bpf_rand_ux(24, 0xffffffULL)
bpf_rand_ux(32, 0xffffffffULL)
bpf_rand_ux(40, 0xffffffffffULL)
bpf_rand_ux(48, 0xffffffffffffULL)
bpf_rand_ux(56, 0xffffffffffffffULL)
bpf_rand_ux(64, 0xffffffffffffffffULL)
static inline void bpf_semi_rand_init(void)
{
srand(time(NULL));
}
static inline uint64_t bpf_semi_rand_get(void)
{
switch (rand() % 39) {
case 0: return 0x000000ff00000000ULL | bpf_rand_u8(0);
case 1: return 0xffffffff00000000ULL | bpf_rand_u16(0);
case 2: return 0x00000000ffff0000ULL | bpf_rand_u16(0);
case 3: return 0x8000000000000000ULL | bpf_rand_u32(0);
case 4: return 0x00000000f0000000ULL | bpf_rand_u32(0);
case 5: return 0x0000000100000000ULL | bpf_rand_u24(0);
case 6: return 0x800ff00000000000ULL | bpf_rand_u32(0);
case 7: return 0x7fffffff00000000ULL | bpf_rand_u32(0);
case 8: return 0xffffffffffffff00ULL ^ bpf_rand_u32(24);
case 9: return 0xffffffffffffff00ULL | bpf_rand_u8(0);
case 10: return 0x0000000010000000ULL | bpf_rand_u32(0);
case 11: return 0xf000000000000000ULL | bpf_rand_u8(0);
case 12: return 0x0000f00000000000ULL | bpf_rand_u8(8);
case 13: return 0x000000000f000000ULL | bpf_rand_u8(16);
case 14: return 0x0000000000000f00ULL | bpf_rand_u8(32);
case 15: return 0x00fff00000000f00ULL | bpf_rand_u8(48);
case 16: return 0x00007fffffffffffULL ^ bpf_rand_u32(1);
case 17: return 0xffff800000000000ULL | bpf_rand_u8(4);
case 18: return 0xffff800000000000ULL | bpf_rand_u8(20);
case 19: return (0xffffffc000000000ULL + 0x80000ULL) | bpf_rand_u32(0);
case 20: return (0xffffffc000000000ULL - 0x04000000ULL) | bpf_rand_u32(0);
case 21: return 0x0000000000000000ULL | bpf_rand_u8(55) | bpf_rand_u32(20);
case 22: return 0xffffffffffffffffULL ^ bpf_rand_u8(3) ^ bpf_rand_u32(40);
case 23: return 0x0000000000000000ULL | bpf_rand_u8(bpf_rand_u8(0) % 64);
case 24: return 0x0000000000000000ULL | bpf_rand_u16(bpf_rand_u8(0) % 64);
case 25: return 0xffffffffffffffffULL ^ bpf_rand_u8(bpf_rand_u8(0) % 64);
case 26: return 0xffffffffffffffffULL ^ bpf_rand_u40(bpf_rand_u8(0) % 64);
case 27: return 0x0000800000000000ULL;
case 28: return 0x8000000000000000ULL;
case 29: return 0x0000000000000000ULL;
case 30: return 0xffffffffffffffffULL;
case 31: return bpf_rand_u16(bpf_rand_u8(0) % 64);
case 32: return bpf_rand_u24(bpf_rand_u8(0) % 64);
case 33: return bpf_rand_u32(bpf_rand_u8(0) % 64);
case 34: return bpf_rand_u40(bpf_rand_u8(0) % 64);
case 35: return bpf_rand_u48(bpf_rand_u8(0) % 64);
case 36: return bpf_rand_u56(bpf_rand_u8(0) % 64);
case 37: return bpf_rand_u64(bpf_rand_u8(0) % 64);
default: return bpf_rand_u64(0);
}
}
#endif /* __BPF_RAND__ */
linux-7.2.3/tools/testing/selftests/bpf/bpf_sockopt_helpers.h
+21
View File
@@ -0,0 +1,21 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include <sys/socket.h>
#include <bpf/bpf_helpers.h>
int get_set_sk_priority(void *ctx)
{
int prio;
/* Verify that context allows calling bpf_getsockopt and
* bpf_setsockopt by reading and writing back socket
* priority.
*/
if (bpf_getsockopt(ctx, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(prio)))
return 0;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(prio)))
return 0;
return 1;
}
linux-7.2.3/tools/testing/selftests/bpf/bpf_util.h
+96
View File
@@ -0,0 +1,96 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __BPF_UTIL__
#define __BPF_UTIL__
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <syscall.h>
#include <bpf/libbpf.h> /* libbpf_num_possible_cpus */
#include <linux/args.h>
static inline unsigned int bpf_num_possible_cpus(void)
{
int possible_cpus = libbpf_num_possible_cpus();
if (possible_cpus < 0) {
printf("Failed to get # of possible cpus: '%s'!\n",
strerror(-possible_cpus));
exit(1);
}
return possible_cpus;
}
/*
* Simplified strscpy() implementation. The kernel one is in lib/string.c
*/
static inline ssize_t sized_strscpy(char *dest, const char *src, size_t count)
{
long res = 0;
if (count == 0)
return -E2BIG;
while (count > 1) {
char c;
c = src[res];
dest[res] = c;
if (!c)
return res;
res++;
count--;
}
/* Force NUL-termination. */
dest[res] = '\0';
/* Return E2BIG if the source didn't stop */
return src[res] ? -E2BIG : res;
}
#define __strscpy0(dst, src, ...) \
sized_strscpy(dst, src, sizeof(dst))
#define __strscpy1(dst, src, size) \
sized_strscpy(dst, src, size)
#undef strscpy /* Redefine the placeholder from tools/include/linux/string.h */
#define strscpy(dst, src, ...) \
CONCATENATE(__strscpy, COUNT_ARGS(__VA_ARGS__))(dst, src, __VA_ARGS__)
#define __bpf_percpu_val_align __attribute__((__aligned__(8)))
#define BPF_DECLARE_PERCPU(type, name) \
struct { type v; /* padding */ } __bpf_percpu_val_align \
name[bpf_num_possible_cpus()]
#define bpf_percpu(name, cpu) name[(cpu)].v
#ifndef ARRAY_SIZE
# define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
#endif
#ifndef sizeof_field
#define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
#endif
#ifndef offsetofend
#define offsetofend(TYPE, MEMBER) \
(offsetof(TYPE, MEMBER) + sizeof_field(TYPE, MEMBER))
#endif
/* Availability of gettid across glibc versions is hit-and-miss, therefore
* fallback to syscall in this macro and use it everywhere.
*/
#ifndef sys_gettid
#define sys_gettid() syscall(SYS_gettid)
#endif
/* and poison usage to ensure it does not creep back in. */
#pragma GCC poison gettid
#ifndef ENOTSUPP
#define ENOTSUPP 524
#endif
#endif /* __BPF_UTIL__ */
linux-7.2.3/tools/testing/selftests/bpf/bpftool_helpers.c
+86
View File
@@ -0,0 +1,86 @@
// SPDX-License-Identifier: GPL-2.0-only
#include <unistd.h>
#include <string.h>
#include <stdbool.h>
#include <limits.h>
#include "bpf_util.h"
#include "bpftool_helpers.h"
#define BPFTOOL_FULL_CMD_MAX_LEN (PATH_MAX * 2)
#define BPFTOOL_DEFAULT_PATH "tools/sbin/bpftool"
static int detect_bpftool_path(char *buffer, size_t size)
{
char tmp[PATH_MAX];
const char *env_path;
/* First, check if BPFTOOL environment variable is set */
env_path = getenv("BPFTOOL");
if (env_path && access(env_path, X_OK) == 0) {
strscpy(buffer, env_path, size);
return 0;
} else if (env_path) {
fprintf(stderr, "bpftool '%s' doesn't exist or is not executable\n", env_path);
return 1;
}
/* Check default bpftool location (will work if we are running the
* default flavor of test_progs)
*/
snprintf(tmp, sizeof(tmp), "./%s", BPFTOOL_DEFAULT_PATH);
if (access(tmp, X_OK) == 0) {
strscpy(buffer, tmp, size);
return 0;
}
/* Check alternate bpftool location (will work if we are running a
* specific flavor of test_progs, e.g. cpuv4 or no_alu32)
*/
snprintf(tmp, sizeof(tmp), "../%s", BPFTOOL_DEFAULT_PATH);
if (access(tmp, X_OK) == 0) {
strscpy(buffer, tmp, size);
return 0;
}
fprintf(stderr, "Failed to detect bpftool path, use BPFTOOL env var to override\n");
return 1;
}
static int run_command(char *args, char *output_buf, size_t output_max_len)
{
static char bpftool_path[PATH_MAX] = {};
bool suppress_output = !(output_buf && output_max_len);
char command[BPFTOOL_FULL_CMD_MAX_LEN];
FILE *f;
int ret;
/* Detect and cache bpftool binary location */
if (bpftool_path[0] == 0 && detect_bpftool_path(bpftool_path, sizeof(bpftool_path)))
return 1;
ret = snprintf(command, sizeof(command), "%s %s%s",
bpftool_path, args,
suppress_output ? " > /dev/null 2>&1" : "");
f = popen(command, "r");
if (!f)
return 1;
if (!suppress_output)
fread(output_buf, 1, output_max_len, f);
ret = pclose(f);
return ret;
}
int run_bpftool_command(char *args)
{
return run_command(args, NULL, 0);
}
int get_bpftool_command_output(char *args, char *output_buf, size_t output_max_len)
{
return run_command(args, output_buf, output_max_len);
}
linux-7.2.3/tools/testing/selftests/bpf/bpftool_helpers.h
+11
View File
@@ -0,0 +1,11 @@
/* SPDX-License-Identifier: GPL-2.0-only */
#pragma once
#include <stdlib.h>
#include <stdio.h>
#include <stdbool.h>
#define MAX_BPFTOOL_CMD_LEN (256)
int run_bpftool_command(char *args);
int get_bpftool_command_output(char *args, char *output_buf, size_t output_max_len);
linux-7.2.3/tools/testing/selftests/bpf/btf_helpers.c
+292
View File
@@ -0,0 +1,292 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <stdio.h>
#include <errno.h>
#include <bpf/btf.h>
#include <bpf/libbpf.h>
#include "test_progs.h"
static const char * const btf_kind_str_mapping[] = {
[BTF_KIND_UNKN] = "UNKNOWN",
[BTF_KIND_INT] = "INT",
[BTF_KIND_PTR] = "PTR",
[BTF_KIND_ARRAY] = "ARRAY",
[BTF_KIND_STRUCT] = "STRUCT",
[BTF_KIND_UNION] = "UNION",
[BTF_KIND_ENUM] = "ENUM",
[BTF_KIND_FWD] = "FWD",
[BTF_KIND_TYPEDEF] = "TYPEDEF",
[BTF_KIND_VOLATILE] = "VOLATILE",
[BTF_KIND_CONST] = "CONST",
[BTF_KIND_RESTRICT] = "RESTRICT",
[BTF_KIND_FUNC] = "FUNC",
[BTF_KIND_FUNC_PROTO] = "FUNC_PROTO",
[BTF_KIND_VAR] = "VAR",
[BTF_KIND_DATASEC] = "DATASEC",
[BTF_KIND_FLOAT] = "FLOAT",
[BTF_KIND_DECL_TAG] = "DECL_TAG",
[BTF_KIND_TYPE_TAG] = "TYPE_TAG",
[BTF_KIND_ENUM64] = "ENUM64",
};
static const char *btf_kind_str(__u16 kind)
{
if (kind > BTF_KIND_ENUM64)
return "UNKNOWN";
return btf_kind_str_mapping[kind];
}
static const char *btf_int_enc_str(__u8 encoding)
{
switch (encoding) {
case 0:
return "(none)";
case BTF_INT_SIGNED:
return "SIGNED";
case BTF_INT_CHAR:
return "CHAR";
case BTF_INT_BOOL:
return "BOOL";
default:
return "UNKN";
}
}
static const char *btf_var_linkage_str(__u32 linkage)
{
switch (linkage) {
case BTF_VAR_STATIC:
return "static";
case BTF_VAR_GLOBAL_ALLOCATED:
return "global-alloc";
default:
return "(unknown)";
}
}
static const char *btf_func_linkage_str(const struct btf_type *t)
{
switch (btf_vlen(t)) {
case BTF_FUNC_STATIC:
return "static";
case BTF_FUNC_GLOBAL:
return "global";
case BTF_FUNC_EXTERN:
return "extern";
default:
return "(unknown)";
}
}
static const char *btf_str(const struct btf *btf, __u32 off)
{
if (!off)
return "(anon)";
return btf__str_by_offset(btf, off) ?: "(invalid)";
}
int fprintf_btf_type_raw(FILE *out, const struct btf *btf, __u32 id)
{
const struct btf_type *t;
int kind, i;
__u32 vlen;
t = btf__type_by_id(btf, id);
if (!t)
return -EINVAL;
vlen = btf_vlen(t);
kind = btf_kind(t);
fprintf(out, "[%u] %s '%s'", id, btf_kind_str(kind), btf_str(btf, t->name_off));
switch (kind) {
case BTF_KIND_INT:
fprintf(out, " size=%u bits_offset=%u nr_bits=%u encoding=%s",
t->size, btf_int_offset(t), btf_int_bits(t),
btf_int_enc_str(btf_int_encoding(t)));
break;
case BTF_KIND_PTR:
case BTF_KIND_CONST:
case BTF_KIND_VOLATILE:
case BTF_KIND_RESTRICT:
case BTF_KIND_TYPEDEF:
case BTF_KIND_TYPE_TAG:
fprintf(out, " type_id=%u", t->type);
break;
case BTF_KIND_ARRAY: {
const struct btf_array *arr = btf_array(t);
fprintf(out, " type_id=%u index_type_id=%u nr_elems=%u",
arr->type, arr->index_type, arr->nelems);
break;
}
case BTF_KIND_STRUCT:
case BTF_KIND_UNION: {
const struct btf_member *m = btf_members(t);
fprintf(out, " size=%u vlen=%u", t->size, vlen);
for (i = 0; i < vlen; i++, m++) {
__u32 bit_off, bit_sz;
bit_off = btf_member_bit_offset(t, i);
bit_sz = btf_member_bitfield_size(t, i);
fprintf(out, "\n\t'%s' type_id=%u bits_offset=%u",
btf_str(btf, m->name_off), m->type, bit_off);
if (bit_sz)
fprintf(out, " bitfield_size=%u", bit_sz);
}
break;
}
case BTF_KIND_ENUM: {
const struct btf_enum *v = btf_enum(t);
const char *fmt_str;
fmt_str = btf_kflag(t) ? "\n\t'%s' val=%d" : "\n\t'%s' val=%u";
fprintf(out, " encoding=%s size=%u vlen=%u",
btf_kflag(t) ? "SIGNED" : "UNSIGNED", t->size, vlen);
for (i = 0; i < vlen; i++, v++) {
fprintf(out, fmt_str,
btf_str(btf, v->name_off), v->val);
}
break;
}
case BTF_KIND_ENUM64: {
const struct btf_enum64 *v = btf_enum64(t);
const char *fmt_str;
fmt_str = btf_kflag(t) ? "\n\t'%s' val=%lld" : "\n\t'%s' val=%llu";
fprintf(out, " encoding=%s size=%u vlen=%u",
btf_kflag(t) ? "SIGNED" : "UNSIGNED", t->size, vlen);
for (i = 0; i < vlen; i++, v++) {
fprintf(out, fmt_str,
btf_str(btf, v->name_off),
((__u64)v->val_hi32 << 32) | v->val_lo32);
}
break;
}
case BTF_KIND_FWD:
fprintf(out, " fwd_kind=%s", btf_kflag(t) ? "union" : "struct");
break;
case BTF_KIND_FUNC:
fprintf(out, " type_id=%u linkage=%s", t->type, btf_func_linkage_str(t));
break;
case BTF_KIND_FUNC_PROTO: {
const struct btf_param *p = btf_params(t);
fprintf(out, " ret_type_id=%u vlen=%u", t->type, vlen);
for (i = 0; i < vlen; i++, p++) {
fprintf(out, "\n\t'%s' type_id=%u",
btf_str(btf, p->name_off), p->type);
}
break;
}
case BTF_KIND_VAR:
fprintf(out, " type_id=%u, linkage=%s",
t->type, btf_var_linkage_str(btf_var(t)->linkage));
break;
case BTF_KIND_DATASEC: {
const struct btf_var_secinfo *v = btf_var_secinfos(t);
fprintf(out, " size=%u vlen=%u", t->size, vlen);
for (i = 0; i < vlen; i++, v++) {
fprintf(out, "\n\ttype_id=%u offset=%u size=%u",
v->type, v->offset, v->size);
}
break;
}
case BTF_KIND_FLOAT:
fprintf(out, " size=%u", t->size);
break;
case BTF_KIND_DECL_TAG:
fprintf(out, " type_id=%u component_idx=%d",
t->type, btf_decl_tag(t)->component_idx);
break;
default:
break;
}
return 0;
}
/* Print raw BTF type dump into a local buffer and return string pointer back.
* Buffer *will* be overwritten by subsequent btf_type_raw_dump() calls
*/
const char *btf_type_raw_dump(const struct btf *btf, int type_id)
{
static char buf[16 * 1024];
FILE *buf_file;
buf_file = fmemopen(buf, sizeof(buf) - 1, "w");
if (!buf_file) {
fprintf(stderr, "Failed to open memstream: %d\n", errno);
return NULL;
}
fprintf_btf_type_raw(buf_file, btf, type_id);
fflush(buf_file);
fclose(buf_file);
return buf;
}
int btf_validate_raw(struct btf *btf, int nr_types, const char *exp_types[])
{
int i;
bool ok = true;
ASSERT_EQ(btf__type_cnt(btf) - 1, nr_types, "btf_nr_types");
for (i = 1; i <= nr_types; i++) {
if (!ASSERT_STREQ(btf_type_raw_dump(btf, i), exp_types[i - 1], "raw_dump"))
ok = false;
}
return ok;
}
static void btf_dump_printf(void *ctx, const char *fmt, va_list args)
{
vfprintf(ctx, fmt, args);
}
/* Print BTF-to-C dump into a local buffer and return string pointer back.
* Buffer *will* be overwritten by subsequent btf_type_raw_dump() calls
*/
const char *btf_type_c_dump(const struct btf *btf)
{
static char buf[16 * 1024];
FILE *buf_file;
struct btf_dump *d = NULL;
int err, i;
buf_file = fmemopen(buf, sizeof(buf) - 1, "w");
if (!buf_file) {
fprintf(stderr, "Failed to open memstream: %d\n", errno);
return NULL;
}
d = btf_dump__new(btf, btf_dump_printf, buf_file, NULL);
if (libbpf_get_error(d)) {
fprintf(stderr, "Failed to create btf_dump instance: %ld\n", libbpf_get_error(d));
goto err_out;
}
for (i = 1; i < btf__type_cnt(btf); i++) {
err = btf_dump__dump_type(d, i);
if (err) {
fprintf(stderr, "Failed to dump type [%d]: %d\n", i, err);
goto err_out;
}
}
btf_dump__free(d);
fflush(buf_file);
fclose(buf_file);
return buf;
err_out:
btf_dump__free(d);
fclose(buf_file);
return NULL;
}
linux-7.2.3/tools/testing/selftests/bpf/btf_helpers.h
+19
View File
@@ -0,0 +1,19 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2020 Facebook */
#ifndef __BTF_HELPERS_H
#define __BTF_HELPERS_H
#include <stdio.h>
#include <bpf/btf.h>
int fprintf_btf_type_raw(FILE *out, const struct btf *btf, __u32 id);
const char *btf_type_raw_dump(const struct btf *btf, int type_id);
int btf_validate_raw(struct btf *btf, int nr_types, const char *exp_types[]);
#define VALIDATE_RAW_BTF(btf, raw_types...) \
btf_validate_raw(btf, \
sizeof((const char *[]){raw_types})/sizeof(void *),\
(const char *[]){raw_types})
const char *btf_type_c_dump(const struct btf *btf);
#endif
linux-7.2.3/tools/testing/selftests/bpf/cap_helpers.c
+67
View File
@@ -0,0 +1,67 @@
// SPDX-License-Identifier: GPL-2.0
#include "cap_helpers.h"
/* Avoid including <sys/capability.h> from the libcap-devel package,
* so directly declare them here and use them from glibc.
*/
int capget(cap_user_header_t header, cap_user_data_t data);
int capset(cap_user_header_t header, const cap_user_data_t data);
int cap_enable_effective(__u64 caps, __u64 *old_caps)
{
struct __user_cap_data_struct data[_LINUX_CAPABILITY_U32S_3];
struct __user_cap_header_struct hdr = {
.version = _LINUX_CAPABILITY_VERSION_3,
};
__u32 cap0 = caps;
__u32 cap1 = caps >> 32;
int err;
err = capget(&hdr, data);
if (err)
return -errno;
if (old_caps)
*old_caps = (__u64)(data[1].effective) << 32 | data[0].effective;
if ((data[0].effective & cap0) == cap0 &&
(data[1].effective & cap1) == cap1)
return 0;
data[0].effective |= cap0;
data[1].effective |= cap1;
err = capset(&hdr, data);
if (err)
return -errno;
return 0;
}
int cap_disable_effective(__u64 caps, __u64 *old_caps)
{
struct __user_cap_data_struct data[_LINUX_CAPABILITY_U32S_3];
struct __user_cap_header_struct hdr = {
.version = _LINUX_CAPABILITY_VERSION_3,
};
__u32 cap0 = caps;
__u32 cap1 = caps >> 32;
int err;
err = capget(&hdr, data);
if (err)
return -errno;
if (old_caps)
*old_caps = (__u64)(data[1].effective) << 32 | data[0].effective;
if (!(data[0].effective & cap0) && !(data[1].effective & cap1))
return 0;
data[0].effective &= ~cap0;
data[1].effective &= ~cap1;
err = capset(&hdr, data);
if (err)
return -errno;
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/cap_helpers.h
+20
View File
@@ -0,0 +1,20 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __CAP_HELPERS_H
#define __CAP_HELPERS_H
#include <linux/types.h>
#include <linux/capability.h>
#include <errno.h>
#ifndef CAP_PERFMON
#define CAP_PERFMON 38
#endif
#ifndef CAP_BPF
#define CAP_BPF 39
#endif
int cap_enable_effective(__u64 caps, __u64 *old_caps);
int cap_disable_effective(__u64 caps, __u64 *old_caps);
#endif
linux-7.2.3/tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h
+25
View File
@@ -0,0 +1,25 @@
/* SPDX-License-Identifier: GPL-2.0 */
BPF_RETVAL_HOOK(ingress, "cgroup_skb/ingress", __sk_buff, -EINVAL)
BPF_RETVAL_HOOK(egress, "cgroup_skb/egress", __sk_buff, -EINVAL)
BPF_RETVAL_HOOK(sock_create, "cgroup/sock_create", bpf_sock, 0)
BPF_RETVAL_HOOK(sock_ops, "sockops", bpf_sock_ops, -EINVAL)
BPF_RETVAL_HOOK(dev, "cgroup/dev", bpf_cgroup_dev_ctx, 0)
BPF_RETVAL_HOOK(bind4, "cgroup/bind4", bpf_sock_addr, 0)
BPF_RETVAL_HOOK(bind6, "cgroup/bind6", bpf_sock_addr, 0)
BPF_RETVAL_HOOK(connect4, "cgroup/connect4", bpf_sock_addr, 0)
BPF_RETVAL_HOOK(connect6, "cgroup/connect6", bpf_sock_addr, 0)
BPF_RETVAL_HOOK(post_bind4, "cgroup/post_bind4", bpf_sock_addr, 0)
BPF_RETVAL_HOOK(post_bind6, "cgroup/post_bind6", bpf_sock_addr, 0)
BPF_RETVAL_HOOK(sendmsg4, "cgroup/sendmsg4", bpf_sock_addr, 0)
BPF_RETVAL_HOOK(sendmsg6, "cgroup/sendmsg6", bpf_sock_addr, 0)
BPF_RETVAL_HOOK(sysctl, "cgroup/sysctl", bpf_sysctl, 0)
BPF_RETVAL_HOOK(recvmsg4, "cgroup/recvmsg4", bpf_sock_addr, -EINVAL)
BPF_RETVAL_HOOK(recvmsg6, "cgroup/recvmsg6", bpf_sock_addr, -EINVAL)
BPF_RETVAL_HOOK(getsockopt, "cgroup/getsockopt", bpf_sockopt, 0)
BPF_RETVAL_HOOK(setsockopt, "cgroup/setsockopt", bpf_sockopt, 0)
BPF_RETVAL_HOOK(getpeername4, "cgroup/getpeername4", bpf_sock_addr, -EINVAL)
BPF_RETVAL_HOOK(getpeername6, "cgroup/getpeername6", bpf_sock_addr, -EINVAL)
BPF_RETVAL_HOOK(getsockname4, "cgroup/getsockname4", bpf_sock_addr, -EINVAL)
BPF_RETVAL_HOOK(getsockname6, "cgroup/getsockname6", bpf_sock_addr, -EINVAL)
BPF_RETVAL_HOOK(sock_release, "cgroup/sock_release", bpf_sock, 0)
linux-7.2.3/tools/testing/selftests/bpf/cgroup_helpers.c
+751
View File
@@ -0,0 +1,751 @@
// SPDX-License-Identifier: GPL-2.0
#define _GNU_SOURCE
#include <sched.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/xattr.h>
#include <linux/limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <linux/sched.h>
#include <fcntl.h>
#include <unistd.h>
#include <ftw.h>
#include "cgroup_helpers.h"
#include "bpf_util.h"
/*
* To avoid relying on the system setup, when setup_cgroup_env is called
* we create a new mount namespace, and cgroup namespace. The cgroupv2
* root is mounted at CGROUP_MOUNT_PATH. Unfortunately, most people don't
* have cgroupv2 enabled at this point in time. It's easier to create our
* own mount namespace and manage it ourselves. We assume /mnt exists.
*
* Related cgroupv1 helpers are named *classid*(), since we only use the
* net_cls controller for tagging net_cls.classid. We assume the default
* mount under /sys/fs/cgroup/net_cls, which should be the case for the
* vast majority of users.
*/
#define WALK_FD_LIMIT 16
#define CGROUP_MOUNT_PATH "/mnt"
#define CGROUP_MOUNT_DFLT "/sys/fs/cgroup"
#define NETCLS_MOUNT_PATH CGROUP_MOUNT_DFLT "/net_cls"
#define CGROUP_WORK_DIR "/cgroup-test-work-dir"
#define format_cgroup_path_pid(buf, path, pid) \
snprintf(buf, sizeof(buf), "%s%s%d%s", CGROUP_MOUNT_PATH, \
CGROUP_WORK_DIR, pid, path)
#define format_cgroup_path(buf, path) \
format_cgroup_path_pid(buf, path, getpid())
#define format_parent_cgroup_path(buf, path) \
format_cgroup_path_pid(buf, path, getppid())
#define format_classid_path_pid(buf, pid) \
snprintf(buf, sizeof(buf), "%s%s%d", NETCLS_MOUNT_PATH, \
CGROUP_WORK_DIR, pid)
#define format_classid_path(buf) \
format_classid_path_pid(buf, getpid())
static __thread bool cgroup_workdir_mounted;
static void __cleanup_cgroup_environment(void);
static int __enable_controllers(const char *cgroup_path, const char *controllers)
{
char path[PATH_MAX + 1];
char enable[PATH_MAX + 1];
char *c, *c2;
int fd, cfd;
ssize_t len;
/* If not controllers are passed, enable all available controllers */
if (!controllers) {
snprintf(path, sizeof(path), "%s/cgroup.controllers",
cgroup_path);
fd = open(path, O_RDONLY);
if (fd < 0) {
log_err("Opening cgroup.controllers: %s", path);
return 1;
}
len = read(fd, enable, sizeof(enable) - 1);
if (len < 0) {
close(fd);
log_err("Reading cgroup.controllers: %s", path);
return 1;
} else if (len == 0) { /* No controllers to enable */
close(fd);
return 0;
}
enable[len] = 0;
close(fd);
} else {
strscpy(enable, controllers);
}
snprintf(path, sizeof(path), "%s/cgroup.subtree_control", cgroup_path);
cfd = open(path, O_RDWR);
if (cfd < 0) {
log_err("Opening cgroup.subtree_control: %s", path);
return 1;
}
for (c = strtok_r(enable, " ", &c2); c; c = strtok_r(NULL, " ", &c2)) {
if (dprintf(cfd, "+%s\n", c) <= 0) {
log_err("Enabling controller %s: %s", c, path);
close(cfd);
return 1;
}
}
close(cfd);
return 0;
}
/**
* enable_controllers() - Enable cgroup v2 controllers
* @relative_path: The cgroup path, relative to the workdir
* @controllers: List of controllers to enable in cgroup.controllers format
*
*
* Enable given cgroup v2 controllers, if @controllers is NULL, enable all
* available controllers.
*
* If successful, 0 is returned.
*/
int enable_controllers(const char *relative_path, const char *controllers)
{
char cgroup_path[PATH_MAX + 1];
format_cgroup_path(cgroup_path, relative_path);
return __enable_controllers(cgroup_path, controllers);
}
static int __write_cgroup_file(const char *cgroup_path, const char *file,
const char *buf)
{
char file_path[PATH_MAX + 1];
int fd;
snprintf(file_path, sizeof(file_path), "%s/%s", cgroup_path, file);
fd = open(file_path, O_RDWR);
if (fd < 0) {
log_err("Opening %s", file_path);
return 1;
}
if (dprintf(fd, "%s", buf) <= 0) {
log_err("Writing to %s", file_path);
close(fd);
return 1;
}
close(fd);
return 0;
}
/**
* write_cgroup_file() - Write to a cgroup file
* @relative_path: The cgroup path, relative to the workdir
* @file: The name of the file in cgroupfs to write to
* @buf: Buffer to write to the file
*
* Write to a file in the given cgroup's directory.
*
* If successful, 0 is returned.
*/
int write_cgroup_file(const char *relative_path, const char *file,
const char *buf)
{
char cgroup_path[PATH_MAX - 24];
format_cgroup_path(cgroup_path, relative_path);
return __write_cgroup_file(cgroup_path, file, buf);
}
/**
* write_cgroup_file_parent() - Write to a cgroup file in the parent process
* workdir
* @relative_path: The cgroup path, relative to the parent process workdir
* @file: The name of the file in cgroupfs to write to
* @buf: Buffer to write to the file
*
* Write to a file in the given cgroup's directory under the parent process
* workdir.
*
* If successful, 0 is returned.
*/
int write_cgroup_file_parent(const char *relative_path, const char *file,
const char *buf)
{
char cgroup_path[PATH_MAX - 24];
format_parent_cgroup_path(cgroup_path, relative_path);
return __write_cgroup_file(cgroup_path, file, buf);
}
/**
* setup_cgroup_environment() - Setup the cgroup environment
*
* After calling this function, cleanup_cgroup_environment should be called
* once testing is complete.
*
* This function will print an error to stderr and return 1 if it is unable
* to setup the cgroup environment. If setup is successful, 0 is returned.
*/
int setup_cgroup_environment(void)
{
char cgroup_workdir[PATH_MAX - 24];
format_cgroup_path(cgroup_workdir, "");
if (mkdir(CGROUP_MOUNT_PATH, 0777) && errno != EEXIST) {
log_err("mkdir mount");
return 1;
}
if (unshare(CLONE_NEWNS)) {
log_err("unshare");
return 1;
}
if (mount("none", "/", NULL, MS_REC | MS_PRIVATE, NULL)) {
log_err("mount fakeroot");
return 1;
}
if (mount("none", CGROUP_MOUNT_PATH, "cgroup2", 0, NULL) && errno != EBUSY) {
log_err("mount cgroup2");
return 1;
}
cgroup_workdir_mounted = true;
/* Cleanup existing failed runs, now that the environment is setup */
__cleanup_cgroup_environment();
if (mkdir(cgroup_workdir, 0777) && errno != EEXIST) {
log_err("mkdir cgroup work dir");
return 1;
}
/* Enable all available controllers to increase test coverage */
if (__enable_controllers(CGROUP_MOUNT_PATH, NULL) ||
__enable_controllers(cgroup_workdir, NULL))
return 1;
return 0;
}
static int nftwfunc(const char *filename, const struct stat *statptr,
int fileflags, struct FTW *pfwt)
{
if ((fileflags & FTW_D) && rmdir(filename))
log_err("Removing cgroup: %s", filename);
return 0;
}
static int join_cgroup_from_top(const char *cgroup_path)
{
char cgroup_procs_path[PATH_MAX + 1];
pid_t pid = getpid();
int fd, rc = 0;
snprintf(cgroup_procs_path, sizeof(cgroup_procs_path),
"%s/cgroup.procs", cgroup_path);
fd = open(cgroup_procs_path, O_WRONLY);
if (fd < 0) {
log_err("Opening Cgroup Procs: %s", cgroup_procs_path);
return 1;
}
if (dprintf(fd, "%d\n", pid) < 0) {
log_err("Joining Cgroup");
rc = 1;
}
close(fd);
return rc;
}
/**
* join_cgroup() - Join a cgroup
* @relative_path: The cgroup path, relative to the workdir, to join
*
* This function expects a cgroup to already be created, relative to the cgroup
* work dir, and it joins it. For example, passing "/my-cgroup" as the path
* would actually put the calling process into the cgroup
* "/cgroup-test-work-dir/my-cgroup"
*
* On success, it returns 0, otherwise on failure it returns 1.
*/
int join_cgroup(const char *relative_path)
{
char cgroup_path[PATH_MAX + 1];
format_cgroup_path(cgroup_path, relative_path);
return join_cgroup_from_top(cgroup_path);
}
/**
* join_root_cgroup() - Join the root cgroup
*
* This function joins the root cgroup.
*
* On success, it returns 0, otherwise on failure it returns 1.
*/
int join_root_cgroup(void)
{
return join_cgroup_from_top(CGROUP_MOUNT_PATH);
}
/**
* join_parent_cgroup() - Join a cgroup in the parent process workdir
* @relative_path: The cgroup path, relative to parent process workdir, to join
*
* See join_cgroup().
*
* On success, it returns 0, otherwise on failure it returns 1.
*/
int join_parent_cgroup(const char *relative_path)
{
char cgroup_path[PATH_MAX + 1];
format_parent_cgroup_path(cgroup_path, relative_path);
return join_cgroup_from_top(cgroup_path);
}
/**
* set_cgroup_xattr() - Set xattr on a cgroup dir
* @relative_path: The cgroup path, relative to the workdir, to set xattr
* @name: xattr name
* @value: xattr value
*
* This function set xattr on cgroup dir.
*
* On success, it returns 0, otherwise on failure it returns -1.
*/
int set_cgroup_xattr(const char *relative_path,
const char *name,
const char *value)
{
char cgroup_path[PATH_MAX + 1];
format_cgroup_path(cgroup_path, relative_path);
return setxattr(cgroup_path, name, value, strlen(value) + 1, 0);
}
/**
* __cleanup_cgroup_environment() - Delete temporary cgroups
*
* This is a helper for cleanup_cgroup_environment() that is responsible for
* deletion of all temporary cgroups that have been created during the test.
*/
static void __cleanup_cgroup_environment(void)
{
char cgroup_workdir[PATH_MAX + 1];
format_cgroup_path(cgroup_workdir, "");
join_cgroup_from_top(CGROUP_MOUNT_PATH);
nftw(cgroup_workdir, nftwfunc, WALK_FD_LIMIT, FTW_DEPTH | FTW_MOUNT);
}
/**
* cleanup_cgroup_environment() - Cleanup Cgroup Testing Environment
*
* This is an idempotent function to delete all temporary cgroups that
* have been created during the test and unmount the cgroup testing work
* directory.
*
* At call time, it moves the calling process to the root cgroup, and then
* runs the deletion process. It is idempotent, and should not fail, unless
* a process is lingering.
*
* On failure, it will print an error to stderr, and try to continue.
*/
void cleanup_cgroup_environment(void)
{
__cleanup_cgroup_environment();
if (cgroup_workdir_mounted && umount(CGROUP_MOUNT_PATH))
log_err("umount cgroup2");
cgroup_workdir_mounted = false;
}
/**
* get_root_cgroup() - Get the FD of the root cgroup
*
* On success, it returns the file descriptor. On failure, it returns -1.
* If there is a failure, it prints the error to stderr.
*/
int get_root_cgroup(void)
{
int fd;
fd = open(CGROUP_MOUNT_PATH, O_RDONLY);
if (fd < 0) {
log_err("Opening root cgroup");
return -1;
}
return fd;
}
/*
* remove_cgroup() - Remove a cgroup
* @relative_path: The cgroup path, relative to the workdir, to remove
*
* This function expects a cgroup to already be created, relative to the cgroup
* work dir. It also expects the cgroup doesn't have any children or live
* processes and it removes the cgroup.
*
* On failure, it will print an error to stderr.
*/
void remove_cgroup(const char *relative_path)
{
char cgroup_path[PATH_MAX + 1];
format_cgroup_path(cgroup_path, relative_path);
if (rmdir(cgroup_path))
log_err("rmdiring cgroup %s .. %s", relative_path, cgroup_path);
}
/*
* remove_cgroup_pid() - Remove a cgroup setup by process identified by PID
* @relative_path: The cgroup path, relative to the workdir, to remove
* @pid: PID to be used to find cgroup_path
*
* This function expects a cgroup to already be created, relative to the cgroup
* work dir. It also expects the cgroup doesn't have any children or live
* processes and it removes the cgroup.
*
* On failure, it will print an error to stderr.
*/
void remove_cgroup_pid(const char *relative_path, int pid)
{
char cgroup_path[PATH_MAX + 1];
format_cgroup_path_pid(cgroup_path, relative_path, pid);
if (rmdir(cgroup_path))
log_err("rmdiring cgroup %s .. %s", relative_path, cgroup_path);
}
/**
* create_and_get_cgroup() - Create a cgroup, relative to workdir, and get the FD
* @relative_path: The cgroup path, relative to the workdir, to join
*
* This function creates a cgroup under the top level workdir and returns the
* file descriptor. It is idempotent.
*
* On success, it returns the file descriptor. On failure it returns -1.
* If there is a failure, it prints the error to stderr.
*/
int create_and_get_cgroup(const char *relative_path)
{
char cgroup_path[PATH_MAX + 1];
int fd;
format_cgroup_path(cgroup_path, relative_path);
if (mkdir(cgroup_path, 0777) && errno != EEXIST) {
log_err("mkdiring cgroup %s .. %s", relative_path, cgroup_path);
return -1;
}
fd = open(cgroup_path, O_RDONLY);
if (fd < 0) {
log_err("Opening Cgroup");
return -1;
}
return fd;
}
/**
* get_cgroup_id_from_path - Get cgroup id for a particular cgroup path
* @cgroup_workdir: The absolute cgroup path
*
* On success, it returns the cgroup id. On failure it returns 0,
* which is an invalid cgroup id.
* If there is a failure, it prints the error to stderr.
*/
static unsigned long long get_cgroup_id_from_path(const char *cgroup_workdir)
{
int dirfd, err, flags, mount_id, fhsize;
union {
unsigned long long cgid;
unsigned char raw_bytes[8];
} id;
struct file_handle *fhp, *fhp2;
unsigned long long ret = 0;
dirfd = AT_FDCWD;
flags = 0;
fhsize = sizeof(*fhp);
fhp = calloc(1, fhsize);
if (!fhp) {
log_err("calloc");
return 0;
}
err = name_to_handle_at(dirfd, cgroup_workdir, fhp, &mount_id, flags);
if (err >= 0 || fhp->handle_bytes != 8) {
log_err("name_to_handle_at");
goto free_mem;
}
fhsize = sizeof(struct file_handle) + fhp->handle_bytes;
fhp2 = realloc(fhp, fhsize);
if (!fhp2) {
log_err("realloc");
goto free_mem;
}
err = name_to_handle_at(dirfd, cgroup_workdir, fhp2, &mount_id, flags);
fhp = fhp2;
if (err < 0) {
log_err("name_to_handle_at");
goto free_mem;
}
memcpy(id.raw_bytes, fhp->f_handle, 8);
ret = id.cgid;
free_mem:
free(fhp);
return ret;
}
unsigned long long get_cgroup_id(const char *relative_path)
{
char cgroup_workdir[PATH_MAX + 1];
format_cgroup_path(cgroup_workdir, relative_path);
return get_cgroup_id_from_path(cgroup_workdir);
}
int cgroup_setup_and_join(const char *path) {
int cg_fd;
if (setup_cgroup_environment()) {
fprintf(stderr, "Failed to setup cgroup environment\n");
return -EINVAL;
}
cg_fd = create_and_get_cgroup(path);
if (cg_fd < 0) {
fprintf(stderr, "Failed to create test cgroup\n");
cleanup_cgroup_environment();
return cg_fd;
}
if (join_cgroup(path)) {
fprintf(stderr, "Failed to join cgroup\n");
cleanup_cgroup_environment();
return -EINVAL;
}
return cg_fd;
}
/**
* setup_classid_environment() - Setup the cgroupv1 net_cls environment
*
* This function should only be called in a custom mount namespace, e.g.
* created by running setup_cgroup_environment.
*
* After calling this function, cleanup_classid_environment should be called
* once testing is complete.
*
* This function will print an error to stderr and return 1 if it is unable
* to setup the cgroup environment. If setup is successful, 0 is returned.
*/
int setup_classid_environment(void)
{
char cgroup_workdir[PATH_MAX + 1];
format_classid_path(cgroup_workdir);
if (mount("tmpfs", CGROUP_MOUNT_DFLT, "tmpfs", 0, NULL) &&
errno != EBUSY) {
log_err("mount cgroup base");
return 1;
}
if (mkdir(NETCLS_MOUNT_PATH, 0777) && errno != EEXIST) {
log_err("mkdir cgroup net_cls");
return 1;
}
if (mount("net_cls", NETCLS_MOUNT_PATH, "cgroup", 0, "net_cls")) {
if (errno != EBUSY) {
log_err("mount cgroup net_cls");
return 1;
}
if (rmdir(NETCLS_MOUNT_PATH)) {
log_err("rmdir cgroup net_cls");
return 1;
}
if (umount(CGROUP_MOUNT_DFLT)) {
log_err("umount cgroup base");
return 1;
}
}
cleanup_classid_environment();
if (mkdir(cgroup_workdir, 0777) && errno != EEXIST) {
log_err("mkdir cgroup work dir");
return 1;
}
return 0;
}
/**
* set_classid() - Set a cgroupv1 net_cls classid
*
* Writes the classid into the cgroup work dir's net_cls.classid
* file in order to later on trigger socket tagging.
*
* We leverage the current pid as the classid, ensuring unique identification.
*
* On success, it returns 0, otherwise on failure it returns 1. If there
* is a failure, it prints the error to stderr.
*/
int set_classid(void)
{
char cgroup_workdir[PATH_MAX - 42];
char cgroup_classid_path[PATH_MAX + 1];
int fd, rc = 0;
format_classid_path(cgroup_workdir);
snprintf(cgroup_classid_path, sizeof(cgroup_classid_path),
"%s/net_cls.classid", cgroup_workdir);
fd = open(cgroup_classid_path, O_WRONLY);
if (fd < 0) {
log_err("Opening cgroup classid: %s", cgroup_classid_path);
return 1;
}
if (dprintf(fd, "%u\n", getpid()) < 0) {
log_err("Setting cgroup classid");
rc = 1;
}
close(fd);
return rc;
}
/**
* join_classid() - Join a cgroupv1 net_cls classid
*
* This function expects the cgroup work dir to be already created, as we
* join it here. This causes the process sockets to be tagged with the given
* net_cls classid.
*
* On success, it returns 0, otherwise on failure it returns 1.
*/
int join_classid(void)
{
char cgroup_workdir[PATH_MAX + 1];
format_classid_path(cgroup_workdir);
return join_cgroup_from_top(cgroup_workdir);
}
/**
* cleanup_classid_environment() - Cleanup the cgroupv1 net_cls environment
*
* At call time, it moves the calling process to the root cgroup, and then
* runs the deletion process.
*
* On failure, it will print an error to stderr, and try to continue.
*/
void cleanup_classid_environment(void)
{
char cgroup_workdir[PATH_MAX + 1];
format_classid_path(cgroup_workdir);
join_cgroup_from_top(NETCLS_MOUNT_PATH);
nftw(cgroup_workdir, nftwfunc, WALK_FD_LIMIT, FTW_DEPTH | FTW_MOUNT);
}
/**
* get_classid_cgroup_id - Get the cgroup id of a net_cls cgroup
*/
unsigned long long get_classid_cgroup_id(void)
{
char cgroup_workdir[PATH_MAX + 1];
format_classid_path(cgroup_workdir);
return get_cgroup_id_from_path(cgroup_workdir);
}
/**
* get_cgroup1_hierarchy_id - Retrieves the ID of a cgroup1 hierarchy from the cgroup1 subsys name.
* @subsys_name: The cgroup1 subsys name, which can be retrieved from /proc/self/cgroup. It can be
* a named cgroup like "name=systemd", a controller name like "net_cls", or multi-controllers like
* "net_cls,net_prio".
*/
int get_cgroup1_hierarchy_id(const char *subsys_name)
{
char *c, *c2, *c3, *c4;
bool found = false;
char line[1024];
FILE *file;
int i, id;
if (!subsys_name)
return -1;
file = fopen("/proc/self/cgroup", "r");
if (!file) {
log_err("fopen /proc/self/cgroup");
return -1;
}
while (fgets(line, 1024, file)) {
i = 0;
for (c = strtok_r(line, ":", &c2); c && i < 2; c = strtok_r(NULL, ":", &c2)) {
if (i == 0) {
id = strtol(c, NULL, 10);
} else if (i == 1) {
if (!strcmp(c, subsys_name)) {
found = true;
break;
}
/* Multiple subsystems may share one single mount point */
for (c3 = strtok_r(c, ",", &c4); c3;
c3 = strtok_r(NULL, ",", &c4)) {
if (!strcmp(c, subsys_name)) {
found = true;
break;
}
}
}
i++;
}
if (found)
break;
}
fclose(file);
return found ? id : -1;
}
/**
* open_classid() - Open a cgroupv1 net_cls classid
*
* This function expects the cgroup work dir to be already created, as we
* open it here.
*
* On success, it returns the file descriptor. On failure it returns -1.
*/
int open_classid(void)
{
char cgroup_workdir[PATH_MAX + 1];
format_classid_path(cgroup_workdir);
return open(cgroup_workdir, O_RDONLY);
}
linux-7.2.3/tools/testing/selftests/bpf/cgroup_helpers.h
+46
View File
@@ -0,0 +1,46 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __CGROUP_HELPERS_H
#define __CGROUP_HELPERS_H
#include <errno.h>
#include <string.h>
#define clean_errno() (errno == 0 ? "None" : strerror(errno))
#define log_err(MSG, ...) fprintf(stderr, "(%s:%d: errno: %s) " MSG "\n", \
__FILE__, __LINE__, clean_errno(), ##__VA_ARGS__)
/* cgroupv2 related */
int enable_controllers(const char *relative_path, const char *controllers);
int write_cgroup_file(const char *relative_path, const char *file,
const char *buf);
int write_cgroup_file_parent(const char *relative_path, const char *file,
const char *buf);
int cgroup_setup_and_join(const char *relative_path);
int get_root_cgroup(void);
int create_and_get_cgroup(const char *relative_path);
void remove_cgroup(const char *relative_path);
void remove_cgroup_pid(const char *relative_path, int pid);
unsigned long long get_cgroup_id(const char *relative_path);
int get_cgroup1_hierarchy_id(const char *subsys_name);
int join_cgroup(const char *relative_path);
int join_root_cgroup(void);
int join_parent_cgroup(const char *relative_path);
int set_cgroup_xattr(const char *relative_path,
const char *name,
const char *value);
int setup_cgroup_environment(void);
void cleanup_cgroup_environment(void);
/* cgroupv1 related */
int set_classid(void);
int join_classid(void);
unsigned long long get_classid_cgroup_id(void);
int open_classid(void);
int setup_classid_environment(void);
void cleanup_classid_environment(void);
#endif /* __CGROUP_HELPERS_H */
linux-7.2.3/tools/testing/selftests/bpf/cgroup_iter_memcg.h
+16
View File
@@ -0,0 +1,16 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#ifndef __CGROUP_ITER_MEMCG_H
#define __CGROUP_ITER_MEMCG_H
struct memcg_query {
/* some node_stat_item's */
unsigned long nr_anon_mapped;
unsigned long nr_shmem;
unsigned long nr_file_pages;
unsigned long nr_file_mapped;
/* some vm_event_item */
unsigned long pgfault;
};
#endif /* __CGROUP_ITER_MEMCG_H */
linux-7.2.3/tools/testing/selftests/bpf/cgroup_tcp_skb.h
+35
View File
@@ -0,0 +1,35 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */
/* Define states of a socket to tracking messages sending to and from the
* socket.
*
* These states are based on rfc9293 with some modifications to support
* tracking of messages sent out from a socket. For example, when a SYN is
* received, a new socket is transiting to the SYN_RECV state defined in
* rfc9293. But, we put it in SYN_RECV_SENDING_SYN_ACK state and when
* SYN-ACK is sent out, it moves to SYN_RECV state. With this modification,
* we can track the message sent out from a socket.
*/
#ifndef __CGROUP_TCP_SKB_H__
#define __CGROUP_TCP_SKB_H__
enum {
INIT,
CLOSED,
SYN_SENT,
SYN_RECV_SENDING_SYN_ACK,
SYN_RECV,
ESTABLISHED,
FIN_WAIT1,
FIN_WAIT2,
CLOSE_WAIT_SENDING_ACK,
CLOSE_WAIT,
CLOSING,
LAST_ACK,
TIME_WAIT_SENDING_ACK,
TIME_WAIT,
};
#endif /* __CGROUP_TCP_SKB_H__ */
linux-7.2.3/tools/testing/selftests/bpf/config
+138
View File
@@ -0,0 +1,138 @@
CONFIG_BLK_DEV_LOOP=y
CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y
CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=1
CONFIG_BPF=y
CONFIG_BPF_EVENTS=y
CONFIG_BPF_JIT=y
CONFIG_BPF_KPROBE_OVERRIDE=y
CONFIG_BPF_LIRC_MODE2=y
CONFIG_BPF_LSM=y
CONFIG_BPF_STREAM_PARSER=y
CONFIG_BPF_SYSCALL=y
# CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set
CONFIG_CGROUP_BPF=y
CONFIG_CRYPTO_HMAC=y
CONFIG_CRYPTO_SHA256=y
CONFIG_CRYPTO_USER_API=y
CONFIG_CRYPTO_USER_API_HASH=y
CONFIG_CRYPTO_USER_API_SKCIPHER=y
CONFIG_CRYPTO_SKCIPHER=y
CONFIG_CRYPTO_ECB=y
CONFIG_CRYPTO_AES=y
CONFIG_DEBUG_INFO=y
CONFIG_DEBUG_INFO_BTF=y
CONFIG_DEBUG_INFO_DWARF4=y
CONFIG_DMABUF_HEAPS=y
CONFIG_DMABUF_HEAPS_SYSTEM=y
CONFIG_DUMMY=y
CONFIG_DYNAMIC_FTRACE=y
CONFIG_FPROBE=y
CONFIG_FTRACE_SYSCALLS=y
CONFIG_FUNCTION_ERROR_INJECTION=y
CONFIG_FUNCTION_TRACER=y
CONFIG_FS_VERITY=y
CONFIG_GENEVE=y
CONFIG_IKCONFIG=y
CONFIG_IKCONFIG_PROC=y
CONFIG_IMA=y
CONFIG_IMA_READ_POLICY=y
CONFIG_IMA_WRITE_POLICY=y
CONFIG_INET_ESP=y
CONFIG_IP_NF_FILTER=y
CONFIG_IP_NF_RAW=y
CONFIG_IP_NF_TARGET_SYNPROXY=y
CONFIG_IPV6=y
CONFIG_IPV6_FOU=y
CONFIG_IPV6_FOU_TUNNEL=y
CONFIG_IPV6_GRE=y
CONFIG_IPV6_IOAM6_LWTUNNEL=y
CONFIG_IPV6_SEG6_BPF=y
CONFIG_IPV6_SEG6_LWTUNNEL=y
CONFIG_IPV6_SIT=y
CONFIG_IPV6_TUNNEL=y
CONFIG_KEYS=y
CONFIG_LIRC=y
CONFIG_LIVEPATCH=y
CONFIG_LWTUNNEL=y
CONFIG_LWTUNNEL_BPF=y
CONFIG_MODULE_SIG=y
CONFIG_MODULE_SRCVERSION_ALL=y
CONFIG_MODULE_UNLOAD=y
CONFIG_MODULES=y
CONFIG_MODVERSIONS=y
CONFIG_MPLS=y
CONFIG_MPLS_IPTUNNEL=y
CONFIG_MPLS_ROUTING=y
CONFIG_MPTCP=y
CONFIG_NET_ACT_GACT=y
CONFIG_NET_ACT_MIRRED=y
CONFIG_NET_ACT_SKBMOD=y
CONFIG_NET_CLS=y
CONFIG_NET_CLS_ACT=y
CONFIG_NET_CLS_BPF=y
CONFIG_NET_CLS_FLOWER=y
CONFIG_NET_CLS_MATCHALL=y
CONFIG_NET_FOU=y
CONFIG_NET_FOU_IP_TUNNELS=y
CONFIG_NET_IPGRE=y
CONFIG_NET_IPGRE_DEMUX=y
CONFIG_NET_IPIP=y
CONFIG_NET_MPLS_GSO=y
CONFIG_NET_SCH_BPF=y
CONFIG_NET_SCH_FQ=y
CONFIG_NET_SCH_INGRESS=y
CONFIG_NET_SCH_HTB=y
CONFIG_NET_SCH_RED=y
CONFIG_NET_SCHED=y
CONFIG_NETDEVSIM=y
CONFIG_NETFILTER=y
CONFIG_NETFILTER_ADVANCED=y
CONFIG_NETFILTER_SYNPROXY=y
CONFIG_NETFILTER_XT_CONNMARK=y
CONFIG_NETFILTER_XT_MATCH_STATE=y
CONFIG_NETFILTER_XT_TARGET_CT=y
CONFIG_NETKIT=y
CONFIG_NF_CONNTRACK=y
CONFIG_NF_CONNTRACK_MARK=y
CONFIG_NF_CONNTRACK_ZONES=y
CONFIG_NF_DEFRAG_IPV4=y
CONFIG_NF_DEFRAG_IPV6=y
CONFIG_NF_TABLES=y
CONFIG_NF_TABLES_INET=y
CONFIG_NF_TABLES_NETDEV=y
CONFIG_NF_TABLES_IPV4=y
CONFIG_NF_TABLES_IPV6=y
CONFIG_NETFILTER_INGRESS=y
CONFIG_IP_NF_IPTABLES_LEGACY=y
CONFIG_IP6_NF_IPTABLES_LEGACY=y
CONFIG_NETFILTER_XTABLES_LEGACY=y
CONFIG_NF_FLOW_TABLE=y
CONFIG_NF_FLOW_TABLE_INET=y
CONFIG_NETFILTER_NETLINK=y
CONFIG_NFT_FLOW_OFFLOAD=y
CONFIG_IP_NF_IPTABLES=y
CONFIG_IP6_NF_IPTABLES=y
CONFIG_IP6_NF_FILTER=y
CONFIG_NF_NAT=y
CONFIG_PACKET=y
CONFIG_RC_CORE=y
CONFIG_SAMPLES=y
CONFIG_SAMPLE_LIVEPATCH=m
CONFIG_SECURITY=y
CONFIG_SECURITYFS=y
CONFIG_SYN_COOKIES=y
CONFIG_TEST_BPF=m
CONFIG_UDMABUF=y
CONFIG_USERFAULTFD=y
CONFIG_VSOCKETS=y
CONFIG_VXLAN=y
CONFIG_XDP_SOCKETS=y
CONFIG_XFRM_INTERFACE=y
CONFIG_TCP_CONG_DCTCP=y
CONFIG_TCP_CONG_BBR=y
CONFIG_INFINIBAND=y
CONFIG_SMC=y
CONFIG_SMC_HS_CTRL_BPF=y
CONFIG_DIBS=y
CONFIG_DIBS_LO=y
CONFIG_PM_WAKELOCKS=y
linux-7.2.3/tools/testing/selftests/bpf/config.aarch64
+154
View File
@@ -0,0 +1,154 @@
CONFIG_ARCH_VEXPRESS=y
CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y
CONFIG_ARM_SMMU_V3=y
CONFIG_ATA=y
CONFIG_AUDIT=y
CONFIG_BINFMT_MISC=y
CONFIG_BLK_CGROUP=y
CONFIG_BLK_DEV_BSGLIB=y
CONFIG_BLK_DEV_INITRD=y
CONFIG_BLK_DEV_IO_TRACE=y
CONFIG_BLK_DEV_RAM=y
CONFIG_BLK_DEV_SD=y
CONFIG_BONDING=y
CONFIG_BPF_JIT_ALWAYS_ON=y
CONFIG_BPF_JIT_DEFAULT_ON=y
CONFIG_BPF_PRELOAD_UMD=y
CONFIG_BPF_PRELOAD=y
CONFIG_BRIDGE=m
CONFIG_CGROUP_CPUACCT=y
CONFIG_CGROUP_DEVICE=y
CONFIG_CGROUP_FREEZER=y
CONFIG_CGROUP_HUGETLB=y
CONFIG_CGROUP_NET_CLASSID=y
CONFIG_CGROUP_PERF=y
CONFIG_CGROUP_PIDS=y
CONFIG_CGROUP_SCHED=y
CONFIG_CGROUPS=y
CONFIG_CHECKPOINT_RESTORE=y
CONFIG_CHR_DEV_SG=y
CONFIG_COMPAT=y
CONFIG_CPUSETS=y
CONFIG_CRASH_DUMP=y
CONFIG_CRYPTO_USER_API_RNG=y
CONFIG_DEBUG_ATOMIC_SLEEP=y
CONFIG_DEBUG_INFO_REDUCED=n
CONFIG_DEBUG_LIST=y
CONFIG_DEBUG_LOCKDEP=y
CONFIG_DEBUG_NOTIFIERS=y
CONFIG_DEBUG_PAGEALLOC=y
CONFIG_DEBUG_SECTION_MISMATCH=y
CONFIG_DEBUG_SG=y
CONFIG_DETECT_HUNG_TASK=y
CONFIG_DEVTMPFS_MOUNT=y
CONFIG_DEVTMPFS=y
CONFIG_DRM=y
CONFIG_EXPERT=y
CONFIG_EXT4_FS_POSIX_ACL=y
CONFIG_EXT4_FS_SECURITY=y
CONFIG_EXT4_FS=y
CONFIG_FANOTIFY=y
CONFIG_FB=y
CONFIG_FUNCTION_PROFILER=y
CONFIG_FUSE_FS=y
CONFIG_FW_CFG_SYSFS_CMDLINE=y
CONFIG_FW_CFG_SYSFS=y
CONFIG_GDB_SCRIPTS=y
CONFIG_HAVE_EBPF_JIT=y
CONFIG_HAVE_KPROBES_ON_FTRACE=y
CONFIG_HAVE_KPROBES=y
CONFIG_HAVE_KRETPROBES=y
CONFIG_HEADERS_INSTALL=y
CONFIG_HIGH_RES_TIMERS=y
CONFIG_HUGETLBFS=y
CONFIG_HW_RANDOM=y
CONFIG_HZ_100=y
CONFIG_IDLE_PAGE_TRACKING=y
CONFIG_IKHEADERS=y
CONFIG_INET6_ESP=y
CONFIG_INET=y
CONFIG_INPUT_EVDEV=y
CONFIG_IP_ADVANCED_ROUTER=y
CONFIG_IP_MULTICAST=y
CONFIG_IP_MULTIPLE_TABLES=y
CONFIG_IPV6_SEG6_LWTUNNEL=y
CONFIG_IPVLAN=y
CONFIG_JUMP_LABEL=y
CONFIG_KERNEL_UNCOMPRESSED=y
CONFIG_KPROBES_ON_FTRACE=y
CONFIG_KPROBES=y
CONFIG_KRETPROBES=y
CONFIG_KSM=y
CONFIG_LATENCYTOP=y
CONFIG_LIVEPATCH=y
CONFIG_LOCK_STAT=y
CONFIG_MACVLAN=y
CONFIG_MACVTAP=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_MAILBOX=y
CONFIG_MEMCG=y
CONFIG_MEMORY_HOTPLUG=y
CONFIG_MEMORY_HOTREMOVE=y
CONFIG_NAMESPACES=y
CONFIG_NET_ACT_BPF=y
CONFIG_NETDEVICES=y
CONFIG_NETFILTER_XT_MATCH_BPF=y
CONFIG_NETFILTER_XT_TARGET_MARK=y
CONFIG_NET_KEY=y
CONFIG_NET_VRF=y
CONFIG_NET=y
CONFIG_NLMON=y
CONFIG_NO_HZ_IDLE=y
CONFIG_NR_CPUS=256
CONFIG_NUMA=y
CONFIG_OVERLAY_FS=y
CONFIG_PACKET_DIAG=y
CONFIG_PANIC_ON_OOPS=y
CONFIG_PARTITION_ADVANCED=y
CONFIG_PCI_HOST_GENERIC=y
CONFIG_PCI=y
CONFIG_PL320_MBOX=y
CONFIG_POSIX_MQUEUE=y
CONFIG_PROC_KCORE=y
CONFIG_PROFILING=y
CONFIG_PROVE_LOCKING=y
CONFIG_PTDUMP_DEBUGFS=y
CONFIG_RC_DEVICES=y
CONFIG_RC_LOOPBACK=y
CONFIG_RTC_CLASS=y
CONFIG_RTC_DRV_PL031=y
CONFIG_RT_GROUP_SCHED=y
CONFIG_SAMPLE_SECCOMP=y
CONFIG_SAMPLES=y
CONFIG_SCHED_AUTOGROUP=y
CONFIG_SCHED_TRACER=y
CONFIG_SCSI_CONSTANTS=y
CONFIG_SCSI_LOGGING=y
CONFIG_SCSI_SCAN_ASYNC=y
CONFIG_SCSI=y
CONFIG_SECURITY_NETWORK=y
CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
CONFIG_SERIAL_AMBA_PL011=y
CONFIG_STACK_TRACER=y
CONFIG_STATIC_KEYS_SELFTEST=y
CONFIG_SYSVIPC=y
CONFIG_TASK_DELAY_ACCT=y
CONFIG_TASK_IO_ACCOUNTING=y
CONFIG_TASKSTATS=y
CONFIG_TASK_XACCT=y
CONFIG_TCG_TIS=y
CONFIG_TCG_TPM=y
CONFIG_TCP_CONG_ADVANCED=y
CONFIG_TLS=y
CONFIG_TMPFS_POSIX_ACL=y
CONFIG_TMPFS=y
CONFIG_TRACER_SNAPSHOT_PER_CPU_SWAP=y
CONFIG_TRANSPARENT_HUGEPAGE=y
CONFIG_TUN=y
CONFIG_UNIX=y
CONFIG_UPROBES=y
CONFIG_USER_NS=y
CONFIG_VETH=y
CONFIG_VLAN_8021Q=y
CONFIG_VSOCKETS_LOOPBACK=y
CONFIG_XFRM_USER=y
linux-7.2.3/tools/testing/selftests/bpf/config.ppc64el
+92
View File
@@ -0,0 +1,92 @@
CONFIG_ALTIVEC=y
CONFIG_AUDIT=y
CONFIG_BLK_CGROUP=y
CONFIG_BLK_DEV_INITRD=y
CONFIG_BLK_DEV_RAM=y
CONFIG_BONDING=y
CONFIG_BPF_JIT_ALWAYS_ON=y
CONFIG_BPF_PRELOAD_UMD=y
CONFIG_BPF_PRELOAD=y
CONFIG_CGROUP_CPUACCT=y
CONFIG_CGROUP_DEVICE=y
CONFIG_CGROUP_FREEZER=y
CONFIG_CGROUP_HUGETLB=y
CONFIG_CGROUP_NET_CLASSID=y
CONFIG_CGROUP_PERF=y
CONFIG_CGROUP_PIDS=y
CONFIG_CGROUP_SCHED=y
CONFIG_CGROUPS=y
CONFIG_CMDLINE_BOOL=y
CONFIG_CMDLINE="console=hvc0 wg.success=hvc1 panic_on_warn=1"
CONFIG_CPU_LITTLE_ENDIAN=y
CONFIG_CPUSETS=y
CONFIG_DEBUG_ATOMIC_SLEEP=y
CONFIG_DEBUG_FS=y
CONFIG_DETECT_HUNG_TASK=y
CONFIG_DEVTMPFS_MOUNT=y
CONFIG_DEVTMPFS=y
CONFIG_EXPERT=y
CONFIG_EXT4_FS_POSIX_ACL=y
CONFIG_EXT4_FS_SECURITY=y
CONFIG_EXT4_FS=y
CONFIG_FRAME_POINTER=y
CONFIG_FRAME_WARN=1280
CONFIG_HARDLOCKUP_DETECTOR=y
CONFIG_HIGH_RES_TIMERS=y
CONFIG_HUGETLBFS=y
CONFIG_HVC_CONSOLE=y
CONFIG_INET=y
CONFIG_IP_ADVANCED_ROUTER=y
CONFIG_IP_MULTICAST=y
CONFIG_IP_MULTIPLE_TABLES=y
CONFIG_IPV6_SEG6_LWTUNNEL=y
CONFIG_JUMP_LABEL=y
CONFIG_KALLSYMS_ALL=y
CONFIG_KPROBES=y
CONFIG_MEMCG=y
CONFIG_NAMESPACES=y
CONFIG_NET_ACT_BPF=y
CONFIG_NETDEVICES=y
CONFIG_NETFILTER_XT_MATCH_BPF=y
CONFIG_NET_L3_MASTER_DEV=y
CONFIG_NET_VRF=y
CONFIG_NET=y
CONFIG_NO_HZ_IDLE=y
CONFIG_NONPORTABLE=y
CONFIG_NR_CPUS=256
CONFIG_PANIC_ON_OOPS=y
CONFIG_PARTITION_ADVANCED=y
CONFIG_PCI_HOST_GENERIC=y
CONFIG_PCI=y
CONFIG_POSIX_MQUEUE=y
CONFIG_PPC64=y
CONFIG_PPC_OF_BOOT_TRAMPOLINE=y
CONFIG_PPC_PSERIES=y
CONFIG_PPC_RADIX_MMU=y
CONFIG_PRINTK_TIME=y
CONFIG_PROC_KCORE=y
CONFIG_PROFILING=y
CONFIG_RCU_CPU_STALL_TIMEOUT=60
CONFIG_RT_GROUP_SCHED=y
CONFIG_SECTION_MISMATCH_WARN_ONLY=y
CONFIG_SECURITY_NETWORK=y
CONFIG_SERIAL_8250_CONSOLE=y
CONFIG_SERIAL_8250=y
CONFIG_SERIAL_OF_PLATFORM=y
CONFIG_SMP=y
CONFIG_SOC_VIRT=y
CONFIG_SYSVIPC=y
CONFIG_TCP_CONG_ADVANCED=y
CONFIG_THREAD_SHIFT=14
CONFIG_TLS=y
CONFIG_TMPFS_POSIX_ACL=y
CONFIG_TMPFS=y
CONFIG_TUN=y
CONFIG_UNIX=y
CONFIG_UPROBES=y
CONFIG_USER_NS=y
CONFIG_VETH=y
CONFIG_VLAN_8021Q=y
CONFIG_VSOCKETS_LOOPBACK=y
CONFIG_VSX=y
CONFIG_XFRM_USER=y
linux-7.2.3/tools/testing/selftests/bpf/config.riscv64
+83
View File
@@ -0,0 +1,83 @@
CONFIG_AUDIT=y
CONFIG_BLK_CGROUP=y
CONFIG_BLK_DEV_INITRD=y
CONFIG_BLK_DEV_RAM=y
CONFIG_BONDING=y
CONFIG_BPF_JIT_ALWAYS_ON=y
CONFIG_BPF_PRELOAD=y
CONFIG_BPF_PRELOAD_UMD=y
CONFIG_CGROUPS=y
CONFIG_CGROUP_CPUACCT=y
CONFIG_CGROUP_DEVICE=y
CONFIG_CGROUP_FREEZER=y
CONFIG_CGROUP_HUGETLB=y
CONFIG_CGROUP_NET_CLASSID=y
CONFIG_CGROUP_PERF=y
CONFIG_CGROUP_PIDS=y
CONFIG_CGROUP_SCHED=y
CONFIG_CPUSETS=y
CONFIG_DEBUG_ATOMIC_SLEEP=y
CONFIG_DEBUG_FS=y
CONFIG_DETECT_HUNG_TASK=y
CONFIG_DEVTMPFS=y
CONFIG_DEVTMPFS_MOUNT=y
CONFIG_EXPERT=y
CONFIG_EXT4_FS=y
CONFIG_EXT4_FS_POSIX_ACL=y
CONFIG_EXT4_FS_SECURITY=y
CONFIG_FRAME_POINTER=y
CONFIG_HARDLOCKUP_DETECTOR=y
CONFIG_HIGH_RES_TIMERS=y
CONFIG_HUGETLBFS=y
CONFIG_INET=y
CONFIG_IPV6_SEG6_LWTUNNEL=y
CONFIG_IP_ADVANCED_ROUTER=y
CONFIG_IP_MULTICAST=y
CONFIG_IP_MULTIPLE_TABLES=y
CONFIG_JUMP_LABEL=y
CONFIG_KALLSYMS_ALL=y
CONFIG_KPROBES=y
CONFIG_MEMCG=y
CONFIG_NAMESPACES=y
CONFIG_NET=y
CONFIG_NETDEVICES=y
CONFIG_NETFILTER_XT_MATCH_BPF=y
CONFIG_NET_ACT_BPF=y
CONFIG_NET_L3_MASTER_DEV=y
CONFIG_NET_VRF=y
CONFIG_NONPORTABLE=y
CONFIG_NO_HZ_IDLE=y
CONFIG_NR_CPUS=256
CONFIG_PANIC_ON_OOPS=y
CONFIG_PARTITION_ADVANCED=y
CONFIG_PCI=y
CONFIG_PCI_HOST_GENERIC=y
CONFIG_POSIX_MQUEUE=y
CONFIG_PRINTK_TIME=y
CONFIG_PROC_KCORE=y
CONFIG_PROFILING=y
CONFIG_RCU_CPU_STALL_TIMEOUT=60
CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS=y
CONFIG_RISCV_ISA_C=y
CONFIG_RISCV_PMU=y
CONFIG_RISCV_PMU_SBI=y
CONFIG_RT_GROUP_SCHED=y
CONFIG_SECURITY_NETWORK=y
CONFIG_SERIAL_8250=y
CONFIG_SERIAL_8250_CONSOLE=y
CONFIG_SERIAL_OF_PLATFORM=y
CONFIG_SMP=y
CONFIG_SOC_VIRT=y
CONFIG_SYSVIPC=y
CONFIG_TCP_CONG_ADVANCED=y
CONFIG_TLS=y
CONFIG_TMPFS=y
CONFIG_TMPFS_POSIX_ACL=y
CONFIG_TUN=y
CONFIG_UNIX=y
CONFIG_UPROBES=y
CONFIG_USER_NS=y
CONFIG_VETH=y
CONFIG_VLAN_8021Q=y
CONFIG_VSOCKETS_LOOPBACK=y
CONFIG_XFRM_USER=y
linux-7.2.3/tools/testing/selftests/bpf/config.s390x
+125
View File
@@ -0,0 +1,125 @@
CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y
CONFIG_AUDIT=y
CONFIG_BLK_CGROUP=y
CONFIG_BLK_DEV_INITRD=y
CONFIG_BLK_DEV_IO_TRACE=y
CONFIG_BLK_DEV_RAM=y
CONFIG_BONDING=y
CONFIG_BPF_JIT_ALWAYS_ON=y
CONFIG_BPF_JIT_DEFAULT_ON=y
CONFIG_BPF_PRELOAD=y
CONFIG_BPF_PRELOAD_UMD=y
CONFIG_CGROUP_CPUACCT=y
CONFIG_CGROUP_DEVICE=y
CONFIG_CGROUP_FREEZER=y
CONFIG_CGROUP_HUGETLB=y
CONFIG_CGROUP_NET_CLASSID=y
CONFIG_CGROUP_PERF=y
CONFIG_CGROUP_PIDS=y
CONFIG_CGROUP_SCHED=y
CONFIG_CGROUPS=y
CONFIG_CHECKPOINT_RESTORE=y
CONFIG_CPUSETS=y
CONFIG_CRASH_DUMP=y
CONFIG_CRYPTO_USER_API_RNG=y
CONFIG_DEBUG_ATOMIC_SLEEP=y
CONFIG_DEBUG_LIST=y
CONFIG_DEBUG_LOCKDEP=y
CONFIG_DEBUG_NOTIFIERS=y
CONFIG_DEBUG_PAGEALLOC=y
CONFIG_DEBUG_SECTION_MISMATCH=y
CONFIG_DEBUG_SG=y
CONFIG_DETECT_HUNG_TASK=y
CONFIG_DEVTMPFS=y
CONFIG_EXPERT=y
CONFIG_EXT4_FS=y
CONFIG_EXT4_FS_POSIX_ACL=y
CONFIG_EXT4_FS_SECURITY=y
CONFIG_FANOTIFY=y
CONFIG_FUNCTION_PROFILER=y
CONFIG_GDB_SCRIPTS=y
CONFIG_HAVE_EBPF_JIT=y
CONFIG_HAVE_KPROBES=y
CONFIG_HAVE_KPROBES_ON_FTRACE=y
CONFIG_HAVE_KRETPROBES=y
CONFIG_HAVE_MARCH_Z10_FEATURES=y
CONFIG_HAVE_MARCH_Z196_FEATURES=y
CONFIG_HEADERS_INSTALL=y
CONFIG_HIGH_RES_TIMERS=y
CONFIG_HUGETLBFS=y
CONFIG_HW_RANDOM=y
CONFIG_HZ_100=y
CONFIG_IDLE_PAGE_TRACKING=y
CONFIG_IKHEADERS=y
CONFIG_INET6_ESP=y
CONFIG_INET=y
CONFIG_IP_ADVANCED_ROUTER=y
CONFIG_IP_MULTICAST=y
CONFIG_IP_MULTIPLE_TABLES=y
CONFIG_IPV6_SEG6_LWTUNNEL=y
CONFIG_IPVLAN=y
CONFIG_JUMP_LABEL=y
CONFIG_KERNEL_UNCOMPRESSED=y
CONFIG_KPROBES=y
CONFIG_KPROBES_ON_FTRACE=y
CONFIG_KRETPROBES=y
CONFIG_KSM=y
CONFIG_LATENCYTOP=y
CONFIG_LIVEPATCH=y
CONFIG_LOCK_STAT=y
CONFIG_MACVLAN=y
CONFIG_MACVTAP=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_MARCH_Z196=y
CONFIG_MARCH_Z196_TUNE=y
CONFIG_MEMCG=y
CONFIG_MEMORY_HOTPLUG=y
CONFIG_MEMORY_HOTREMOVE=y
CONFIG_NAMESPACES=y
CONFIG_NET=y
CONFIG_NET_ACT_BPF=y
CONFIG_NET_KEY=y
CONFIG_NET_VRF=y
CONFIG_NETDEVICES=y
CONFIG_NETFILTER_XT_MATCH_BPF=y
CONFIG_NETFILTER_XT_TARGET_MARK=y
CONFIG_NO_HZ_IDLE=y
CONFIG_NR_CPUS=256
CONFIG_NUMA=y
CONFIG_PANIC_ON_OOPS=y
CONFIG_PARTITION_ADVANCED=y
CONFIG_PCI=y
CONFIG_POSIX_MQUEUE=y
CONFIG_PROC_KCORE=y
CONFIG_PROFILING=y
CONFIG_PROVE_LOCKING=y
CONFIG_PTDUMP_DEBUGFS=y
CONFIG_RC_DEVICES=y
CONFIG_RC_LOOPBACK=y
CONFIG_RT_GROUP_SCHED=y
CONFIG_SAMPLE_SECCOMP=y
CONFIG_SAMPLES=y
CONFIG_SCHED_TRACER=y
CONFIG_SCSI=y
CONFIG_SECURITY_NETWORK=y
CONFIG_STACK_TRACER=y
CONFIG_STATIC_KEYS_SELFTEST=y
CONFIG_SYSVIPC=y
CONFIG_TASK_DELAY_ACCT=y
CONFIG_TASK_IO_ACCOUNTING=y
CONFIG_TASK_XACCT=y
CONFIG_TASKSTATS=y
CONFIG_TCP_CONG_ADVANCED=y
CONFIG_TLS=y
CONFIG_TMPFS=y
CONFIG_TMPFS_POSIX_ACL=y
CONFIG_TRACER_SNAPSHOT_PER_CPU_SWAP=y
CONFIG_TRANSPARENT_HUGEPAGE=y
CONFIG_TUN=y
CONFIG_UNIX=y
CONFIG_UPROBES=y
CONFIG_USER_NS=y
CONFIG_VETH=y
CONFIG_VLAN_8021Q=y
CONFIG_VSOCKETS_LOOPBACK=y
CONFIG_XFRM_USER=y
linux-7.2.3/tools/testing/selftests/bpf/config.vm
+15
View File
@@ -0,0 +1,15 @@
CONFIG_9P_FS_POSIX_ACL=y
CONFIG_9P_FS_SECURITY=y
CONFIG_9P_FS=y
CONFIG_CRYPTO_DEV_VIRTIO=y
CONFIG_FUSE_FS=y
CONFIG_FUSE_PASSTHROUGH=y
CONFIG_NET_9P_VIRTIO=y
CONFIG_NET_9P=y
CONFIG_VIRTIO_BALLOON=y
CONFIG_VIRTIO_BLK=y
CONFIG_VIRTIO_CONSOLE=y
CONFIG_VIRTIO_FS=y
CONFIG_VIRTIO_NET=y
CONFIG_VIRTIO_PCI=y
CONFIG_VIRTIO_VSOCKETS_COMMON=y
linux-7.2.3/tools/testing/selftests/bpf/config.x86_64
+227
View File
@@ -0,0 +1,227 @@
CONFIG_AGP=y
CONFIG_AGP_AMD64=y
CONFIG_AGP_INTEL=y
CONFIG_AGP_SIS=y
CONFIG_AGP_VIA=y
CONFIG_AMIGA_PARTITION=y
CONFIG_AUDIT=y
CONFIG_BACKLIGHT_CLASS_DEVICE=y
CONFIG_BINFMT_MISC=y
CONFIG_BLK_CGROUP=y
CONFIG_BLK_CGROUP_IOLATENCY=y
CONFIG_BLK_DEV_BSGLIB=y
CONFIG_BLK_DEV_IO_TRACE=y
CONFIG_BLK_DEV_RAM=y
CONFIG_BLK_DEV_RAM_SIZE=16384
CONFIG_BLK_DEV_THROTTLING=y
CONFIG_BONDING=y
CONFIG_BOOTTIME_TRACING=y
CONFIG_BPF_JIT_ALWAYS_ON=y
CONFIG_BPF_PRELOAD=y
CONFIG_BPF_PRELOAD_UMD=y
CONFIG_BSD_DISKLABEL=y
CONFIG_BSD_PROCESS_ACCT=y
CONFIG_CFS_BANDWIDTH=y
CONFIG_CGROUP_CPUACCT=y
CONFIG_CGROUP_DEVICE=y
CONFIG_CGROUP_FREEZER=y
CONFIG_CGROUP_HUGETLB=y
CONFIG_CGROUP_PERF=y
CONFIG_CGROUP_SCHED=y
CONFIG_CGROUPS=y
CONFIG_CMA=y
CONFIG_CMA_AREAS=7
CONFIG_COMPAT_32BIT_TIME=y
CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
CONFIG_CPU_FREQ_GOV_ONDEMAND=y
CONFIG_CPU_FREQ_GOV_USERSPACE=y
CONFIG_CPU_FREQ_STAT=y
CONFIG_CPU_IDLE_GOV_LADDER=y
CONFIG_CPUSETS=y
CONFIG_CRYPTO_BLAKE2B=y
CONFIG_CRYPTO_SEQIV=y
CONFIG_CRYPTO_XXHASH=y
CONFIG_DCB=y
CONFIG_DEBUG_ATOMIC_SLEEP=y
CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_DEBUG_MEMORY_INIT=y
CONFIG_DEFAULT_FQ_CODEL=y
CONFIG_DEFAULT_RENO=y
CONFIG_DEFAULT_SECURITY_DAC=y
CONFIG_DEVTMPFS=y
CONFIG_DEVTMPFS_MOUNT=y
CONFIG_DMA_CMA=y
CONFIG_DNS_RESOLVER=y
CONFIG_EFI=y
CONFIG_EFI_STUB=y
CONFIG_EXPERT=y
CONFIG_EXT4_FS=y
CONFIG_EXT4_FS_POSIX_ACL=y
CONFIG_EXT4_FS_SECURITY=y
CONFIG_FAIL_FUNCTION=y
CONFIG_FAULT_INJECTION=y
CONFIG_FAULT_INJECTION_DEBUG_FS=y
CONFIG_FB=y
CONFIG_FB_MODE_HELPERS=y
CONFIG_FB_TILEBLITTING=y
CONFIG_FB_VESA=y
CONFIG_FONT_8x16=y
CONFIG_FONT_MINI_4x6=y
CONFIG_FONTS=y
CONFIG_FRAMEBUFFER_CONSOLE=y
CONFIG_FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y
CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y
CONFIG_FW_LOADER_USER_HELPER=y
CONFIG_GART_IOMMU=y
CONFIG_GENERIC_PHY=y
CONFIG_HARDLOCKUP_DETECTOR=y
CONFIG_HID_A4TECH=y
CONFIG_HID_BELKIN=y
CONFIG_HID_CHERRY=y
CONFIG_HID_CYPRESS=y
CONFIG_HID_DRAGONRISE=y
CONFIG_HID_EZKEY=y
CONFIG_HID_GREENASIA=y
CONFIG_HID_GYRATION=y
CONFIG_HID_KENSINGTON=y
CONFIG_HID_KYE=y
CONFIG_HID_MICROSOFT=y
CONFIG_HID_MONTEREY=y
CONFIG_HID_PANTHERLORD=y
CONFIG_HID_PETALYNX=y
CONFIG_HID_SMARTJOYPLUS=y
CONFIG_HID_SUNPLUS=y
CONFIG_HID_TOPSEED=y
CONFIG_HID_TWINHAN=y
CONFIG_HID_ZEROPLUS=y
CONFIG_HIGH_RES_TIMERS=y
CONFIG_HPET=y
CONFIG_HUGETLBFS=y
CONFIG_HWPOISON_INJECT=y
CONFIG_HZ_1000=y
CONFIG_INET=y
CONFIG_INPUT_EVDEV=y
CONFIG_INTEL_POWERCLAMP=y
CONFIG_IP_ADVANCED_ROUTER=y
CONFIG_IP_MROUTE=y
CONFIG_IP_MULTICAST=y
CONFIG_IP_MULTIPLE_TABLES=y
CONFIG_IP_PIMSM_V1=y
CONFIG_IP_PIMSM_V2=y
CONFIG_IP_ROUTE_MULTIPATH=y
CONFIG_IP_ROUTE_VERBOSE=y
CONFIG_IPV6_MIP6=y
CONFIG_IPV6_ROUTE_INFO=y
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_IPV6_SEG6_LWTUNNEL=y
CONFIG_IPV6_SUBTREES=y
CONFIG_IRQ_POLL=y
CONFIG_JUMP_LABEL=y
CONFIG_KARMA_PARTITION=y
CONFIG_KEXEC=y
CONFIG_KPROBES=y
CONFIG_KSM=y
CONFIG_LEGACY_VSYSCALL_NONE=y
CONFIG_LOG_BUF_SHIFT=21
CONFIG_LOG_CPU_MAX_BUF_SHIFT=0
CONFIG_LOGO=y
CONFIG_LSM="selinux,bpf,integrity"
CONFIG_MAC_PARTITION=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_MCORE2=y
CONFIG_MEMCG=y
CONFIG_MEMORY_FAILURE=y
CONFIG_MINIX_SUBPARTITION=y
CONFIG_NAMESPACES=y
CONFIG_NET=y
CONFIG_NET_ACT_BPF=y
CONFIG_NET_CLS_CGROUP=y
CONFIG_NET_EMATCH=y
CONFIG_NET_IPGRE_BROADCAST=y
CONFIG_NET_L3_MASTER_DEV=y
CONFIG_NET_SCH_DEFAULT=y
CONFIG_NET_SCH_FQ_CODEL=y
CONFIG_NET_TC_SKB_EXT=y
CONFIG_NET_VRF=y
CONFIG_NETDEVICES=y
CONFIG_NETFILTER_NETLINK_LOG=y
CONFIG_NETFILTER_NETLINK_QUEUE=y
CONFIG_NETFILTER_XT_MATCH_BPF=y
CONFIG_NETFILTER_XT_MATCH_STATISTIC=y
CONFIG_NETLABEL=y
CONFIG_NLS_ASCII=y
CONFIG_NLS_CODEPAGE_437=y
CONFIG_NLS_DEFAULT="utf8"
CONFIG_NO_HZ=y
CONFIG_NR_CPUS=128
CONFIG_NUMA=y
CONFIG_NUMA_BALANCING=y
CONFIG_NVMEM=y
CONFIG_OSF_PARTITION=y
CONFIG_PANIC_ON_OOPS=y
CONFIG_PARTITION_ADVANCED=y
CONFIG_PCI=y
CONFIG_PCI_IOV=y
CONFIG_PCI_MSI=y
CONFIG_PCIEPORTBUS=y
CONFIG_PHYSICAL_ALIGN=0x1000000
CONFIG_POSIX_MQUEUE=y
CONFIG_POWER_SUPPLY=y
CONFIG_PREEMPT=y
CONFIG_PRINTK_TIME=y
CONFIG_PROC_KCORE=y
CONFIG_PROFILING=y
CONFIG_PROVE_LOCKING=y
CONFIG_PTP_1588_CLOCK=y
CONFIG_RC_DEVICES=y
CONFIG_RC_LOOPBACK=y
CONFIG_RCU_CPU_STALL_TIMEOUT=60
CONFIG_SCHED_STACK_END_CHECK=y
CONFIG_SCHEDSTATS=y
CONFIG_SECURITY_NETWORK=y
CONFIG_SECURITY_SELINUX=y
CONFIG_SERIAL_8250=y
CONFIG_SERIAL_8250_CONSOLE=y
CONFIG_SERIAL_8250_DETECT_IRQ=y
CONFIG_SERIAL_8250_EXTENDED=y
CONFIG_SERIAL_8250_MANY_PORTS=y
CONFIG_SERIAL_8250_NR_UARTS=32
CONFIG_SERIAL_8250_RSA=y
CONFIG_SERIAL_8250_SHARE_IRQ=y
CONFIG_SERIAL_NONSTANDARD=y
CONFIG_SERIO_LIBPS2=y
CONFIG_SGI_PARTITION=y
CONFIG_SMP=y
CONFIG_SOLARIS_X86_PARTITION=y
CONFIG_SUN_PARTITION=y
CONFIG_SYNC_FILE=y
CONFIG_SYSVIPC=y
CONFIG_TASK_DELAY_ACCT=y
CONFIG_TASK_IO_ACCOUNTING=y
CONFIG_TASK_XACCT=y
CONFIG_TASKSTATS=y
CONFIG_TCP_CONG_ADVANCED=y
CONFIG_TCP_MD5SIG=y
CONFIG_TLS=y
CONFIG_TMPFS=y
CONFIG_TMPFS_POSIX_ACL=y
CONFIG_TRANSPARENT_HUGEPAGE=y
CONFIG_TRANSPARENT_HUGEPAGE_MADVISE=y
CONFIG_TUN=y
CONFIG_UNIX=y
CONFIG_UNIXWARE_DISKLABEL=y
CONFIG_USER_NS=y
CONFIG_VALIDATE_FS_PARSER=y
CONFIG_VETH=y
CONFIG_VIRT_DRIVERS=y
CONFIG_VLAN_8021Q=y
CONFIG_VSOCKETS_LOOPBACK=y
CONFIG_X86_ACPI_CPUFREQ=y
CONFIG_X86_CPUID=y
CONFIG_X86_MSR=y
CONFIG_X86_POWERNOW_K8=y
CONFIG_XDP_SOCKETS_DIAG=y
CONFIG_XFRM_SUB_POLICY=y
CONFIG_XFRM_USER=y
CONFIG_ZEROPLUS_FF=y
linux-7.2.3/tools/testing/selftests/bpf/default.profraw
BIN
View File
Binary file not shown.
linux-7.2.3/tools/testing/selftests/bpf/disasm.c
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../kernel/bpf/disasm.c
linux-7.2.3/tools/testing/selftests/bpf/disasm.h
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../kernel/bpf/disasm.h
linux-7.2.3/tools/testing/selftests/bpf/disasm_helpers.c
+69
View File
@@ -0,0 +1,69 @@
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
#include <bpf/bpf.h>
#include "disasm.h"
struct print_insn_context {
char scratch[16];
char *buf;
size_t sz;
};
static void print_insn_cb(void *private_data, const char *fmt, ...)
{
struct print_insn_context *ctx = private_data;
va_list args;
va_start(args, fmt);
vsnprintf(ctx->buf, ctx->sz, fmt, args);
va_end(args);
}
static const char *print_call_cb(void *private_data, const struct bpf_insn *insn)
{
struct print_insn_context *ctx = private_data;
/* For pseudo calls verifier.c:jit_subprogs() hides original
* imm to insn->off and changes insn->imm to be an index of
* the subprog instead.
*/
if (insn->src_reg == BPF_PSEUDO_CALL) {
snprintf(ctx->scratch, sizeof(ctx->scratch), "%+d", insn->off);
return ctx->scratch;
}
return NULL;
}
struct bpf_insn *disasm_insn(struct bpf_insn *insn, char *buf, size_t buf_sz)
{
struct print_insn_context ctx = {
.buf = buf,
.sz = buf_sz,
};
struct bpf_insn_cbs cbs = {
.cb_print = print_insn_cb,
.cb_call = print_call_cb,
.private_data = &ctx,
};
char *tmp, *pfx_end, *sfx_start;
bool double_insn;
int len;
print_bpf_insn(&cbs, insn, true);
/* We share code with kernel BPF disassembler, it adds '(FF) ' prefix
* for each instruction (FF stands for instruction `code` byte).
* Remove the prefix inplace, and also simplify call instructions.
* E.g.: "(85) call foo#10" -> "call foo".
* Also remove newline in the end (the 'max(strlen(buf) - 1, 0)' thing).
*/
pfx_end = buf + 5;
sfx_start = buf + max((int)strlen(buf) - 1, 0);
if (strncmp(pfx_end, "call ", 5) == 0 && (tmp = strrchr(buf, '#')))
sfx_start = tmp;
len = sfx_start - pfx_end;
memmove(buf, pfx_end, len);
buf[len] = 0;
double_insn = insn->code == (BPF_LD | BPF_IMM | BPF_DW);
return insn + (double_insn ? 2 : 1);
}
linux-7.2.3/tools/testing/selftests/bpf/disasm_helpers.h
+12
View File
@@ -0,0 +1,12 @@
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
#ifndef __DISASM_HELPERS_H
#define __DISASM_HELPERS_H
#include <stdlib.h>
struct bpf_insn;
struct bpf_insn *disasm_insn(struct bpf_insn *insn, char *buf, size_t buf_sz);
#endif /* __DISASM_HELPERS_H */
linux-7.2.3/tools/testing/selftests/bpf/flow_dissector_load.c
+111
View File
@@ -0,0 +1,111 @@
// SPDX-License-Identifier: GPL-2.0
#include <error.h>
#include <errno.h>
#include <getopt.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include "flow_dissector_load.h"
const char *cfg_pin_path = "/sys/fs/bpf/flow_dissector";
const char *cfg_map_name = "jmp_table";
bool cfg_attach = true;
char *cfg_prog_name;
char *cfg_path_name;
static void load_and_attach_program(void)
{
int prog_fd, ret;
struct bpf_object *obj;
/* Use libbpf 1.0 API mode */
libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
ret = bpf_flow_load(&obj, cfg_path_name, cfg_prog_name,
cfg_map_name, NULL, &prog_fd, NULL);
if (ret)
error(1, 0, "bpf_flow_load %s", cfg_path_name);
ret = bpf_prog_attach(prog_fd, 0 /* Ignore */, BPF_FLOW_DISSECTOR, 0);
if (ret)
error(1, 0, "bpf_prog_attach %s", cfg_path_name);
ret = bpf_object__pin(obj, cfg_pin_path);
if (ret)
error(1, 0, "bpf_object__pin %s", cfg_pin_path);
}
static void detach_program(void)
{
char command[64];
int ret;
ret = bpf_prog_detach(0, BPF_FLOW_DISSECTOR);
if (ret)
error(1, 0, "bpf_prog_detach");
/* To unpin, it is necessary and sufficient to just remove this dir */
sprintf(command, "rm -r %s", cfg_pin_path);
ret = system(command);
if (ret)
error(1, errno, "%s", command);
}
static void parse_opts(int argc, char **argv)
{
bool attach = false;
bool detach = false;
int c;
while ((c = getopt(argc, argv, "adp:s:")) != -1) {
switch (c) {
case 'a':
if (detach)
error(1, 0, "attach/detach are exclusive");
attach = true;
break;
case 'd':
if (attach)
error(1, 0, "attach/detach are exclusive");
detach = true;
break;
case 'p':
if (cfg_path_name)
error(1, 0, "only one path can be given");
cfg_path_name = optarg;
break;
case 's':
if (cfg_prog_name)
error(1, 0, "only one prog can be given");
cfg_prog_name = optarg;
break;
}
}
if (detach)
cfg_attach = false;
if (cfg_attach && !cfg_path_name)
error(1, 0, "must provide a path to the BPF program");
if (cfg_attach && !cfg_prog_name)
error(1, 0, "must provide a section name");
}
int main(int argc, char **argv)
{
parse_opts(argc, argv);
if (cfg_attach)
load_and_attach_program();
else
detach_program();
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/flow_dissector_load.h
+68
View File
@@ -0,0 +1,68 @@
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
#ifndef FLOW_DISSECTOR_LOAD
#define FLOW_DISSECTOR_LOAD
#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include "testing_helpers.h"
static inline int bpf_flow_load(struct bpf_object **obj,
const char *path,
const char *prog_name,
const char *map_name,
const char *keys_map_name,
int *prog_fd,
int *keys_fd)
{
struct bpf_program *prog, *main_prog;
struct bpf_map *prog_array, *keys;
int prog_array_fd;
int ret, fd, i;
ret = bpf_prog_test_load(path, BPF_PROG_TYPE_FLOW_DISSECTOR, obj,
prog_fd);
if (ret)
return ret;
main_prog = bpf_object__find_program_by_name(*obj, prog_name);
if (!main_prog)
return -1;
*prog_fd = bpf_program__fd(main_prog);
if (*prog_fd < 0)
return -1;
prog_array = bpf_object__find_map_by_name(*obj, map_name);
if (!prog_array)
return -1;
prog_array_fd = bpf_map__fd(prog_array);
if (prog_array_fd < 0)
return -1;
if (keys_map_name && keys_fd) {
keys = bpf_object__find_map_by_name(*obj, keys_map_name);
if (!keys)
return -1;
*keys_fd = bpf_map__fd(keys);
if (*keys_fd < 0)
return -1;
}
i = 0;
bpf_object__for_each_program(prog, *obj) {
fd = bpf_program__fd(prog);
if (fd < 0)
return fd;
if (fd != *prog_fd) {
bpf_map_update_elem(prog_array_fd, &i, &fd, BPF_ANY);
++i;
}
}
return 0;
}
#endif /* FLOW_DISSECTOR_LOAD */
linux-7.2.3/tools/testing/selftests/bpf/generate_udp_fragments.py
Executable
+90
View File
@@ -0,0 +1,90 @@
#!/bin/env python3
# SPDX-License-Identifier: GPL-2.0
"""
This script helps generate fragmented UDP packets.
While it is technically possible to dynamically generate
fragmented packets in C, it is much harder to read and write
said code. `scapy` is relatively industry standard and really
easy to read / write.
So we choose to write this script that generates a valid C
header. Rerun script and commit generated file after any
modifications.
"""
import argparse
import os
from scapy.all import *
# These constants must stay in sync with `ip_check_defrag.c`
VETH1_ADDR = "172.16.1.200"
VETH0_ADDR6 = "fc00::100"
VETH1_ADDR6 = "fc00::200"
CLIENT_PORT = 48878
SERVER_PORT = 48879
MAGIC_MESSAGE = "THIS IS THE ORIGINAL MESSAGE, PLEASE REASSEMBLE ME"
def print_header(f):
f.write("// SPDX-License-Identifier: GPL-2.0\n")
f.write("/* DO NOT EDIT -- this file is generated */\n")
f.write("\n")
f.write("#ifndef _IP_CHECK_DEFRAG_FRAGS_H\n")
f.write("#define _IP_CHECK_DEFRAG_FRAGS_H\n")
f.write("\n")
f.write("#include <stdint.h>\n")
f.write("\n")
def print_frags(f, frags, v6):
for idx, frag in enumerate(frags):
# 10 bytes per line to keep width in check
chunks = [frag[i : i + 10] for i in range(0, len(frag), 10)]
chunks_fmted = [", ".join([str(hex(b)) for b in chunk]) for chunk in chunks]
suffix = "6" if v6 else ""
f.write(f"static uint8_t frag{suffix}_{idx}[] = {{\n")
for chunk in chunks_fmted:
f.write(f"\t{chunk},\n")
f.write(f"}};\n")
def print_trailer(f):
f.write("\n")
f.write("#endif /* _IP_CHECK_DEFRAG_FRAGS_H */\n")
def main(f):
# srcip of 0 is filled in by IP_HDRINCL
sip = "0.0.0.0"
sip6 = VETH0_ADDR6
dip = VETH1_ADDR
dip6 = VETH1_ADDR6
sport = CLIENT_PORT
dport = SERVER_PORT
payload = MAGIC_MESSAGE.encode()
# Disable UDPv4 checksums to keep code simpler
pkt = IP(src=sip,dst=dip) / UDP(sport=sport,dport=dport,chksum=0) / Raw(load=payload)
# UDPv6 requires a checksum
# Also pin the ipv6 fragment header ID, otherwise it's a random value
pkt6 = IPv6(src=sip6,dst=dip6) / IPv6ExtHdrFragment(id=0xBEEF) / UDP(sport=sport,dport=dport) / Raw(load=payload)
frags = [f.build() for f in pkt.fragment(24)]
frags6 = [f.build() for f in fragment6(pkt6, 72)]
print_header(f)
print_frags(f, frags, False)
print_frags(f, frags6, True)
print_trailer(f)
if __name__ == "__main__":
dir = os.path.dirname(os.path.realpath(__file__))
header = f"{dir}/ip_check_defrag_frags.h"
with open(header, "w") as f:
main(f)
linux-7.2.3/tools/testing/selftests/bpf/gnu/stubs.h
+1
View File
@@ -0,0 +1 @@
/* dummy .h to trick /usr/include/features.h to work with 'clang --target=bpf' */
linux-7.2.3/tools/testing/selftests/bpf/ima_setup.sh
Executable
+156
View File
@@ -0,0 +1,156 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
set -e
set -u
set -o pipefail
IMA_POLICY_FILE="/sys/kernel/security/ima/policy"
TEST_BINARY="/bin/true"
VERBOSE="${SELFTESTS_VERBOSE:=0}"
LOG_FILE="$(mktemp /tmp/ima_setup.XXXX.log)"
usage()
{
echo "Usage: $0 <setup|cleanup|run|modify-bin|restore-bin|load-policy> <existing_tmp_dir>"
exit 1
}
ensure_mount_securityfs()
{
local securityfs_dir=$(grep "securityfs" /proc/mounts | awk '{print $2}')
if [ -z "${securityfs_dir}" ]; then
securityfs_dir=/sys/kernel/security
mount -t securityfs security "${securityfs_dir}"
fi
if [ ! -d "${securityfs_dir}" ]; then
echo "${securityfs_dir}: securityfs is not mounted" && exit 1
fi
}
setup()
{
local tmp_dir="$1"
local mount_img="${tmp_dir}/test.img"
local mount_dir="${tmp_dir}/mnt"
local copied_bin_path="${mount_dir}/$(basename ${TEST_BINARY})"
mkdir -p ${mount_dir}
dd if=/dev/zero of="${mount_img}" bs=1M count=10
losetup -f "${mount_img}"
local loop_device=$(losetup -a | grep ${mount_img:?} | cut -d ":" -f1)
mkfs.ext2 "${loop_device:?}"
mount "${loop_device}" "${mount_dir}"
cp "${TEST_BINARY}" "${mount_dir}"
local mount_uuid="$(blkid ${loop_device} | sed 's/.*UUID="\([^"]*\)".*/\1/')"
ensure_mount_securityfs
echo "measure func=BPRM_CHECK fsuuid=${mount_uuid}" > ${IMA_POLICY_FILE}
echo "measure func=BPRM_CHECK fsuuid=${mount_uuid}" > ${mount_dir}/policy_test
}
cleanup() {
local tmp_dir="$1"
local mount_img="${tmp_dir}/test.img"
local mount_dir="${tmp_dir}/mnt"
local loop_devices=$(losetup -a | grep ${mount_img:?} | cut -d ":" -f1)
for loop_dev in "${loop_devices}"; do
losetup -d $loop_dev
done
umount ${mount_dir}
rm -rf ${tmp_dir}
}
run()
{
local tmp_dir="$1"
local mount_dir="${tmp_dir}/mnt"
local copied_bin_path="${mount_dir}/$(basename ${TEST_BINARY})"
exec "${copied_bin_path}"
}
modify_bin()
{
local tmp_dir="$1"
local mount_dir="${tmp_dir}/mnt"
local copied_bin_path="${mount_dir}/$(basename ${TEST_BINARY})"
echo "mod" >> "${copied_bin_path}"
}
restore_bin()
{
local tmp_dir="$1"
local mount_dir="${tmp_dir}/mnt"
local copied_bin_path="${mount_dir}/$(basename ${TEST_BINARY})"
truncate -s -4 "${copied_bin_path}"
}
load_policy()
{
local tmp_dir="$1"
local mount_dir="${tmp_dir}/mnt"
echo ${mount_dir}/policy_test > ${IMA_POLICY_FILE} 2> /dev/null
}
catch()
{
local exit_code="$1"
local log_file="$2"
if [[ "${exit_code}" -ne 0 ]]; then
cat "${log_file}" >&3
fi
rm -f "${log_file}"
exit ${exit_code}
}
main()
{
[[ $# -ne 2 ]] && usage
local action="$1"
local tmp_dir="$2"
[[ ! -d "${tmp_dir}" ]] && echo "Directory ${tmp_dir} doesn't exist" && exit 1
if [[ "${action}" == "setup" ]]; then
setup "${tmp_dir}"
elif [[ "${action}" == "cleanup" ]]; then
cleanup "${tmp_dir}"
elif [[ "${action}" == "run" ]]; then
run "${tmp_dir}"
elif [[ "${action}" == "modify-bin" ]]; then
modify_bin "${tmp_dir}"
elif [[ "${action}" == "restore-bin" ]]; then
restore_bin "${tmp_dir}"
elif [[ "${action}" == "load-policy" ]]; then
load_policy "${tmp_dir}"
else
echo "Unknown action: ${action}"
exit 1
fi
}
trap 'catch "$?" "${LOG_FILE}"' EXIT
if [[ "${VERBOSE}" -eq 0 ]]; then
# Save the stderr to 3 so that we can output back to
# it incase of an error.
exec 3>&2 1>"${LOG_FILE}" 2>&1
fi
main "$@"
rm -f "${LOG_FILE}"
linux-7.2.3/tools/testing/selftests/bpf/io_helpers.c
+21
View File
@@ -0,0 +1,21 @@
// SPDX-License-Identifier: GPL-2.0
#include <sys/select.h>
#include <unistd.h>
#include <errno.h>
int read_with_timeout(int fd, char *buf, size_t count, long usec)
{
const long M = 1000 * 1000;
struct timeval tv = { usec / M, usec % M };
fd_set fds;
int err;
FD_ZERO(&fds);
FD_SET(fd, &fds);
err = select(fd + 1, &fds, NULL, NULL, &tv);
if (err < 0)
return err;
if (FD_ISSET(fd, &fds))
return read(fd, buf, count);
return -EAGAIN;
}
linux-7.2.3/tools/testing/selftests/bpf/io_helpers.h
+7
View File
@@ -0,0 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
#include <unistd.h>
/* As a regular read(2), but allows to specify a timeout in micro-seconds.
* Returns -EAGAIN on timeout.
*/
int read_with_timeout(int fd, char *buf, size_t count, long usec);
linux-7.2.3/tools/testing/selftests/bpf/ip_check_defrag_frags.h
+57
View File
@@ -0,0 +1,57 @@
// SPDX-License-Identifier: GPL-2.0
/* DO NOT EDIT -- this file is generated */
#ifndef _IP_CHECK_DEFRAG_FRAGS_H
#define _IP_CHECK_DEFRAG_FRAGS_H
#include <stdint.h>
static uint8_t frag_0[] = {
0x45, 0x0, 0x0, 0x2c, 0x0, 0x1, 0x20, 0x0, 0x40, 0x11,
0xac, 0xe8, 0x0, 0x0, 0x0, 0x0, 0xac, 0x10, 0x1, 0xc8,
0xbe, 0xee, 0xbe, 0xef, 0x0, 0x3a, 0x0, 0x0, 0x54, 0x48,
0x49, 0x53, 0x20, 0x49, 0x53, 0x20, 0x54, 0x48, 0x45, 0x20,
0x4f, 0x52, 0x49, 0x47,
};
static uint8_t frag_1[] = {
0x45, 0x0, 0x0, 0x2c, 0x0, 0x1, 0x20, 0x3, 0x40, 0x11,
0xac, 0xe5, 0x0, 0x0, 0x0, 0x0, 0xac, 0x10, 0x1, 0xc8,
0x49, 0x4e, 0x41, 0x4c, 0x20, 0x4d, 0x45, 0x53, 0x53, 0x41,
0x47, 0x45, 0x2c, 0x20, 0x50, 0x4c, 0x45, 0x41, 0x53, 0x45,
0x20, 0x52, 0x45, 0x41,
};
static uint8_t frag_2[] = {
0x45, 0x0, 0x0, 0x1e, 0x0, 0x1, 0x0, 0x6, 0x40, 0x11,
0xcc, 0xf0, 0x0, 0x0, 0x0, 0x0, 0xac, 0x10, 0x1, 0xc8,
0x53, 0x53, 0x45, 0x4d, 0x42, 0x4c, 0x45, 0x20, 0x4d, 0x45,
};
static uint8_t frag6_0[] = {
0x60, 0x0, 0x0, 0x0, 0x0, 0x20, 0x2c, 0x40, 0xfc, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x1, 0x0, 0xfc, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0,
0x11, 0x0, 0x0, 0x1, 0x0, 0x0, 0xbe, 0xef, 0xbe, 0xee,
0xbe, 0xef, 0x0, 0x3a, 0xd0, 0xf8, 0x54, 0x48, 0x49, 0x53,
0x20, 0x49, 0x53, 0x20, 0x54, 0x48, 0x45, 0x20, 0x4f, 0x52,
0x49, 0x47,
};
static uint8_t frag6_1[] = {
0x60, 0x0, 0x0, 0x0, 0x0, 0x20, 0x2c, 0x40, 0xfc, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x1, 0x0, 0xfc, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0,
0x11, 0x0, 0x0, 0x19, 0x0, 0x0, 0xbe, 0xef, 0x49, 0x4e,
0x41, 0x4c, 0x20, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45,
0x2c, 0x20, 0x50, 0x4c, 0x45, 0x41, 0x53, 0x45, 0x20, 0x52,
0x45, 0x41,
};
static uint8_t frag6_2[] = {
0x60, 0x0, 0x0, 0x0, 0x0, 0x12, 0x2c, 0x40, 0xfc, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x1, 0x0, 0xfc, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0,
0x11, 0x0, 0x0, 0x30, 0x0, 0x0, 0xbe, 0xef, 0x53, 0x53,
0x45, 0x4d, 0x42, 0x4c, 0x45, 0x20, 0x4d, 0x45,
};
#endif /* _IP_CHECK_DEFRAG_FRAGS_H */
linux-7.2.3/tools/testing/selftests/bpf/jit_disasm_helpers.c
+254
View File
@@ -0,0 +1,254 @@
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include <test_progs.h>
#ifdef HAVE_LLVM_SUPPORT
#include <llvm-c/Core.h>
#include <llvm-c/Disassembler.h>
#include <llvm-c/Target.h>
#include <llvm-c/TargetMachine.h>
/* The intent is to use get_jited_program_text() for small test
* programs written in BPF assembly, thus assume that 32 local labels
* would be sufficient.
*/
#define MAX_LOCAL_LABELS 32
/* Local labels are encoded as 'L42', this requires 4 bytes of storage:
* 3 characters + zero byte
*/
#define LOCAL_LABEL_LEN 4
static bool llvm_initialized;
struct local_labels {
bool print_phase;
__u32 prog_len;
__u32 cnt;
__u32 pcs[MAX_LOCAL_LABELS];
char names[MAX_LOCAL_LABELS][LOCAL_LABEL_LEN];
};
static const char *lookup_symbol(void *data, uint64_t ref_value, uint64_t *ref_type,
uint64_t ref_pc, const char **ref_name)
{
struct local_labels *labels = data;
uint64_t type = *ref_type;
int i;
*ref_type = LLVMDisassembler_ReferenceType_InOut_None;
*ref_name = NULL;
if (type != LLVMDisassembler_ReferenceType_In_Branch)
return NULL;
/* Depending on labels->print_phase either discover local labels or
* return a name assigned with local jump target:
* - if print_phase is true and ref_value is in labels->pcs,
* return corresponding labels->name.
* - if print_phase is false, save program-local jump targets
* in labels->pcs;
*/
if (labels->print_phase) {
for (i = 0; i < labels->cnt; ++i)
if (labels->pcs[i] == ref_value)
return labels->names[i];
} else {
if (labels->cnt < MAX_LOCAL_LABELS && ref_value < labels->prog_len)
labels->pcs[labels->cnt++] = ref_value;
}
return NULL;
}
static int disasm_insn(LLVMDisasmContextRef ctx, uint8_t *image, __u32 len, __u32 pc,
char *buf, __u32 buf_sz)
{
int i, cnt;
cnt = LLVMDisasmInstruction(ctx, image + pc, len - pc, pc,
buf, buf_sz);
if (cnt > 0)
return cnt;
PRINT_FAIL("Can't disasm instruction at offset %d:", pc);
for (i = 0; i < 16 && pc + i < len; ++i)
printf(" %02x", image[pc + i]);
printf("\n");
return -EINVAL;
}
static int cmp_u32(const void *_a, const void *_b)
{
__u32 a = *(__u32 *)_a;
__u32 b = *(__u32 *)_b;
if (a < b)
return -1;
if (a > b)
return 1;
return 0;
}
static int disasm_one_func(FILE *text_out, uint8_t *image, __u32 len)
{
char *label, *colon, *triple = NULL;
LLVMDisasmContextRef ctx = NULL;
struct local_labels labels = {};
__u32 *label_pc, pc;
int i, cnt, err = 0;
char buf[64];
char *cpu, *features;
triple = LLVMGetDefaultTargetTriple();
cpu = LLVMGetHostCPUName();
features = LLVMGetHostCPUFeatures();
ctx = LLVMCreateDisasmCPUFeatures(triple, cpu, features, &labels, 0, NULL, lookup_symbol);
LLVMDisposeMessage(cpu);
LLVMDisposeMessage(features);
if (!ASSERT_OK_PTR(ctx, "LLVMCreateDisasmCPUFeatures")) {
err = -EINVAL;
goto out;
}
cnt = LLVMSetDisasmOptions(ctx, LLVMDisassembler_Option_PrintImmHex);
if (!ASSERT_EQ(cnt, 1, "LLVMSetDisasmOptions")) {
err = -EINVAL;
goto out;
}
/* discover labels */
labels.prog_len = len;
pc = 0;
while (pc < len) {
cnt = disasm_insn(ctx, image, len, pc, buf, 1);
if (cnt < 0) {
err = cnt;
goto out;
}
pc += cnt;
}
qsort(labels.pcs, labels.cnt, sizeof(*labels.pcs), cmp_u32);
/* gcc is unable to infer upper bound for labels.cnt and
* assumes it to be U32_MAX. U32_MAX takes 10 decimal digits.
* snprintf below prints into labels.names[*], which has space
* only for two digits and a letter. To avoid truncation
* warning use (i < MAX_LOCAL_LABELS), which informs gcc about
* printed value upper bound.
*/
for (i = 0; i < labels.cnt && i < MAX_LOCAL_LABELS; ++i)
snprintf(labels.names[i], sizeof(labels.names[i]), "L%d", i);
/* now print with labels */
labels.print_phase = true;
pc = 0;
while (pc < len) {
cnt = disasm_insn(ctx, image, len, pc, buf, sizeof(buf));
if (cnt < 0) {
err = cnt;
goto out;
}
label_pc = bsearch(&pc, labels.pcs, labels.cnt, sizeof(*labels.pcs), cmp_u32);
label = "";
colon = "";
if (label_pc) {
label = labels.names[label_pc - labels.pcs];
colon = ":";
}
fprintf(text_out, "%x:\t", pc);
for (i = 0; i < cnt; ++i)
fprintf(text_out, "%02x ", image[pc + i]);
for (i = cnt * 3; i < 12 * 3; ++i)
fputc(' ', text_out);
fprintf(text_out, "%s%s%s\n", label, colon, buf);
pc += cnt;
}
out:
if (triple)
LLVMDisposeMessage(triple);
if (ctx)
LLVMDisasmDispose(ctx);
return err;
}
int get_jited_program_text(int fd, char *text, size_t text_sz)
{
struct bpf_prog_info info = {};
__u32 info_len = sizeof(info);
__u32 jited_funcs, len, pc;
__u32 *func_lens = NULL;
FILE *text_out = NULL;
uint8_t *image = NULL;
int i, err = 0;
if (!llvm_initialized) {
LLVMInitializeAllTargetInfos();
LLVMInitializeAllTargetMCs();
LLVMInitializeAllDisassemblers();
llvm_initialized = 1;
}
text_out = fmemopen(text, text_sz, "w");
if (!ASSERT_OK_PTR(text_out, "open_memstream")) {
err = -errno;
goto out;
}
/* first call is to find out jited program len */
err = bpf_prog_get_info_by_fd(fd, &info, &info_len);
if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd #1"))
goto out;
len = info.jited_prog_len;
image = malloc(len);
if (!ASSERT_OK_PTR(image, "malloc(info.jited_prog_len)")) {
err = -ENOMEM;
goto out;
}
jited_funcs = info.nr_jited_func_lens;
func_lens = malloc(jited_funcs * sizeof(__u32));
if (!ASSERT_OK_PTR(func_lens, "malloc(info.nr_jited_func_lens)")) {
err = -ENOMEM;
goto out;
}
memset(&info, 0, sizeof(info));
info.jited_prog_insns = (__u64)image;
info.jited_prog_len = len;
info.jited_func_lens = (__u64)func_lens;
info.nr_jited_func_lens = jited_funcs;
err = bpf_prog_get_info_by_fd(fd, &info, &info_len);
if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd #2"))
goto out;
for (pc = 0, i = 0; i < jited_funcs; ++i) {
fprintf(text_out, "func #%d:\n", i);
disasm_one_func(text_out, image + pc, func_lens[i]);
fprintf(text_out, "\n");
pc += func_lens[i];
}
out:
if (text_out)
fclose(text_out);
if (image)
free(image);
if (func_lens)
free(func_lens);
return err;
}
#else /* HAVE_LLVM_SUPPORT */
int get_jited_program_text(int fd, char *text, size_t text_sz)
{
if (env.verbosity >= VERBOSE_VERY)
printf("compiled w/o llvm development libraries, can't dis-assembly binary code");
return -EOPNOTSUPP;
}
#endif /* HAVE_LLVM_SUPPORT */
linux-7.2.3/tools/testing/selftests/bpf/jit_disasm_helpers.h
+10
View File
@@ -0,0 +1,10 @@
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
#ifndef __JIT_DISASM_HELPERS_H
#define __JIT_DISASM_HELPERS_H
#include <stddef.h>
int get_jited_program_text(int fd, char *text, size_t text_sz);
#endif /* __JIT_DISASM_HELPERS_H */
linux-7.2.3/tools/testing/selftests/bpf/json_writer.c
Symlink
+1
View File
@@ -0,0 +1 @@
../../../bpf/bpftool/json_writer.c
linux-7.2.3/tools/testing/selftests/bpf/json_writer.h
Symlink
+1
View File
@@ -0,0 +1 @@
../../../bpf/bpftool/json_writer.h
linux-7.2.3/tools/testing/selftests/bpf/libarena/Makefile
+92
View File
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
# Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
.PHONY: clean
# Defaults for standalone builds
CLANG ?= clang
BPFTOOL ?= bpftool
LDLIBS ?= -lbpf -lelf -lz -lrt -lpthread -lzstd
ifeq ($(V),1)
Q =
msg =
else
Q ?= @
msg = @printf ' %-8s%s %s%s\n' "$(1)" "$(if $(2), [$(2)])" "$(notdir $(3))" "$(if $(4), $(4))";
endif
IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \
grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__')
BPF_TARGET_ENDIAN ?= $(if $(IS_LITTLE_ENDIAN),--target=bpfel,--target=bpfeb)
LIBARENA=$(abspath .)
BPFDIR=$(abspath $(LIBARENA)/..)
INCLUDE_DIR ?= $(BPFDIR)/tools/include
LIBBPF_INCLUDE ?= $(INCLUDE_DIR)
# Scan src/ and selftests/ to generate the final binaries
LIBARENA_SOURCES = $(wildcard $(LIBARENA)/src/*.bpf.c) $(wildcard $(LIBARENA)/selftests/*.bpf.c)
LIBARENA_OBJECTS = $(notdir $(LIBARENA_SOURCES:.bpf.c=.bpf.o))
LIBARENA_OBJECTS_ASAN = $(notdir $(LIBARENA_SOURCES:.bpf.c=_asan.bpf.o))
INCLUDES = -I$(LIBARENA)/include -I$(BPFDIR)
ifneq ($(INCLUDE_DIR),)
INCLUDES += -I$(INCLUDE_DIR)
endif
ifneq ($(LIBBPF_INCLUDE),)
INCLUDES += -I$(LIBBPF_INCLUDE)
endif
ASAN_FLAGS = -fsanitize=kernel-address -fno-stack-protector -fno-builtin
ASAN_FLAGS += -mllvm -asan-instrument-address-spaces=1 -mllvm -asan-shadow-addr-space=1
ASAN_FLAGS += -mllvm -asan-use-stack-safety=0 -mllvm -asan-stack=0
ASAN_FLAGS += -mllvm -asan-kernel=1
ASAN_FLAGS += -mllvm -asan-constructor-kind=none
ASAN_FLAGS += -mllvm -asan-destructor-kind=none
# ENABLE_ATOMICS_TESTS required because we use arena spinlocks
override BPF_CFLAGS += -DENABLE_ATOMICS_TESTS
override BPF_CFLAGS += -O2 -g
override BPF_CFLAGS += -Wno-incompatible-pointer-types-discards-qualifiers
# Required for suppressing harmless vmlinux.h-related warnings.
override BPF_CFLAGS += -Wno-missing-declarations
override BPF_CFLAGS += $(INCLUDES)
CFLAGS = -O2 -no-pie
CFLAGS += $(INCLUDES)
vpath %.bpf.c $(LIBARENA)/src $(LIBARENA)/selftests
vpath %.c $(LIBARENA)/src $(LIBARENA)/selftests
skeletons: libarena.skel.h libarena_asan.skel.h
.PHONY: skeletons
libarena_asan.skel.h: libarena_asan.bpf.o
$(call msg,GEN-SKEL,libarena,$@)
$(Q)$(BPFTOOL) gen skeleton $< name "libarena_asan" > $@
libarena.skel.h: libarena.bpf.o
$(call msg,GEN-SKEL,libarena,$@)
$(Q)$(BPFTOOL) gen skeleton $< name "libarena" > $@
libarena_asan.bpf.o: $(LIBARENA_OBJECTS_ASAN)
$(call msg,GEN-OBJ,libarena,$@)
$(Q)$(BPFTOOL) gen object $@ $^
libarena.bpf.o: $(LIBARENA_OBJECTS)
$(call msg,GEN-OBJ,libarena,$@)
$(Q)$(BPFTOOL) gen object $@ $^
%_asan.bpf.o: %.bpf.c
$(call msg,CLNG-BPF,libarena,$@)
$(Q)$(CLANG) $(BPF_CFLAGS) $(ASAN_FLAGS) -DBPF_ARENA_ASAN $(BPF_TARGET_ENDIAN) -c $< -o $@
%.bpf.o: %.bpf.c
$(call msg,CLNG-BPF,libarena,$@)
$(Q)$(CLANG) $(BPF_CFLAGS) $(BPF_TARGET_ENDIAN) -c $< -o $@
clean:
$(Q)rm -f *.skel.h *.bpf.o *.linked*.o
linux-7.2.3/tools/testing/selftests/bpf/libarena/include/bpf_arena_common.h
+74
View File
@@ -0,0 +1,74 @@
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#pragma once
#ifndef WRITE_ONCE
#define WRITE_ONCE(x, val) ((*(volatile typeof(x) *) &(x)) = (val))
#endif
#ifndef NUMA_NO_NODE
#define NUMA_NO_NODE (-1)
#endif
#ifndef arena_container_of
#define arena_container_of(ptr, type, member) \
({ \
void __arena *__mptr = (void __arena *)(ptr); \
((type *)(__mptr - offsetof(type, member))); \
})
#endif
#ifdef __BPF__ /* when compiled as bpf program */
#ifndef PAGE_SIZE
#define PAGE_SIZE __PAGE_SIZE
/*
* for older kernels try sizeof(struct genradix_node)
* or flexible:
* static inline long __bpf_page_size(void) {
* return bpf_core_enum_value(enum page_size_enum___l, __PAGE_SIZE___l) ?: sizeof(struct genradix_node);
* }
* but generated code is not great.
*/
#endif
#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) && !defined(BPF_ARENA_FORCE_ASM)
#define __arena __attribute__((address_space(1))) __attribute__((btf_type_tag("arena")))
#define __arena_global __attribute__((address_space(1)))
#define cast_kern(ptr) /* nop for bpf prog. emitted by LLVM */
#define cast_user(ptr) /* nop for bpf prog. emitted by LLVM */
#else
#define __arena __attribute__((btf_type_tag("arena")))
#define __arena_global SEC(".addr_space.1")
#define cast_kern(ptr) bpf_addr_space_cast(ptr, 0, 1)
#define cast_user(ptr) bpf_addr_space_cast(ptr, 1, 0)
#endif
void __arena* bpf_arena_alloc_pages(void *map, void __arena *addr, __u32 page_cnt,
int node_id, __u64 flags) __ksym __weak;
int bpf_arena_reserve_pages(void *map, void __arena *addr, __u32 page_cnt) __ksym __weak;
void bpf_arena_free_pages(void *map, void __arena *ptr, __u32 page_cnt) __ksym __weak;
#define arena_base(map) ((void __arena *)((struct bpf_arena *)(map))->user_vm_start)
#else /* when compiled as user space code */
#define __arena
#define cast_kern(ptr) /* nop for user space */
#define cast_user(ptr) /* nop for user space */
__weak char arena[1];
#ifndef offsetof
#define offsetof(type, member) ((unsigned long)&((type *)0)->member)
#endif
static inline void __arena* bpf_arena_alloc_pages(void *map, void *addr, __u32 page_cnt,
int node_id, __u64 flags)
{
return NULL;
}
static inline void bpf_arena_free_pages(void *map, void __arena *ptr, __u32 page_cnt)
{
}
#endif
linux-7.2.3/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h
+543
View File
@@ -0,0 +1,543 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#ifndef BPF_ARENA_SPIN_LOCK_H
#define BPF_ARENA_SPIN_LOCK_H
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf_atomic.h>
#define arch_mcs_spin_lock_contended_label(l, label) smp_cond_load_acquire_label(l, VAL, label)
#define arch_mcs_spin_unlock_contended(l) smp_store_release((l), 1)
#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
#define EBUSY 16
#define EOPNOTSUPP 95
#define ETIMEDOUT 110
extern unsigned long CONFIG_NR_CPUS __kconfig;
/*
* Typically, we'd just rely on the definition in vmlinux.h for qspinlock, but
* PowerPC overrides the definition to define lock->val as u32 instead of
* atomic_t, leading to compilation errors. Import a local definition below so
* that we don't depend on the vmlinux.h version.
*/
struct __qspinlock {
union {
atomic_t val;
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
struct {
u8 locked;
u8 pending;
};
struct {
u16 locked_pending;
u16 tail;
};
#else
struct {
u16 tail;
u16 locked_pending;
};
struct {
u8 reserved[2];
u8 pending;
u8 locked;
};
#endif
};
};
#define arena_spinlock_t struct __qspinlock
/* FIXME: Using typedef causes CO-RE relocation error */
/* typedef struct qspinlock arena_spinlock_t; */
struct arena_mcs_spinlock {
struct arena_mcs_spinlock __arena *next;
int locked;
int count;
};
struct arena_qnode {
struct arena_mcs_spinlock mcs;
};
#define _Q_MAX_NODES 4
#define _Q_PENDING_LOOPS 1
/*
* Bitfields in the atomic value:
*
* 0- 7: locked byte
* 8: pending
* 9-15: not used
* 16-17: tail index
* 18-31: tail cpu (+1)
*/
#define _Q_MAX_CPUS 1024
#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
<< _Q_ ## type ## _OFFSET)
#define _Q_LOCKED_OFFSET 0
#define _Q_LOCKED_BITS 8
#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
#define _Q_PENDING_BITS 8
#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
#define _Q_TAIL_IDX_BITS 2
#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
#define _Q_TAIL_CPU_OFFSET (_Q_TAIL_IDX_OFFSET + _Q_TAIL_IDX_BITS)
#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
#define _Q_TAIL_OFFSET _Q_TAIL_IDX_OFFSET
#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
/*
* The qnodes are marked __weak so we can define them in the header
* while still ensuring all compilation units use the same struct
* instance.
*/
struct arena_qnode __weak __arena __hidden qnodes[_Q_MAX_CPUS][_Q_MAX_NODES];
static inline u32 encode_tail(int cpu, int idx)
{
u32 tail;
tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
return tail;
}
static inline struct arena_mcs_spinlock __arena *decode_tail(u32 tail)
{
u32 cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
u32 idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
return &qnodes[cpu][idx].mcs;
}
static inline
struct arena_mcs_spinlock __arena *grab_mcs_node(struct arena_mcs_spinlock __arena *base, int idx)
{
return &((struct arena_qnode __arena *)base + idx)->mcs;
}
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
/**
* xchg_tail - Put in the new queue tail code word & retrieve previous one
* @lock : Pointer to queued spinlock structure
* @tail : The new queue tail code word
* Return: The previous queue tail code word
*
* xchg(lock, tail)
*
* p,*,* -> n,*,* ; prev = xchg(lock, node)
*/
static __always_inline u32 xchg_tail(arena_spinlock_t __arena *lock, u32 tail)
{
u32 old, new;
old = atomic_read(&lock->val);
do {
new = (old & _Q_LOCKED_PENDING_MASK) | tail;
/*
* We can use relaxed semantics since the caller ensures that
* the MCS node is properly initialized before updating the
* tail.
*/
/* These loops are not expected to stall, but we still need to
* prove to the verifier they will terminate eventually.
*/
cond_break_label(out);
} while (!atomic_try_cmpxchg_relaxed(&lock->val, &old, new));
return old;
out:
bpf_printk("RUNTIME ERROR: %s unexpected cond_break exit!!!", __func__);
return old;
}
/**
* clear_pending - clear the pending bit.
* @lock: Pointer to queued spinlock structure
*
* *,1,* -> *,0,*
*/
static __always_inline void clear_pending(arena_spinlock_t __arena *lock)
{
WRITE_ONCE(lock->pending, 0);
}
/**
* clear_pending_set_locked - take ownership and clear the pending bit.
* @lock: Pointer to queued spinlock structure
*
* *,1,0 -> *,0,1
*
* Lock stealing is not allowed if this function is used.
*/
static __always_inline void clear_pending_set_locked(arena_spinlock_t __arena *lock)
{
WRITE_ONCE(lock->locked_pending, _Q_LOCKED_VAL);
}
/**
* set_locked - Set the lock bit and own the lock
* @lock: Pointer to queued spinlock structure
*
* *,*,0 -> *,0,1
*/
static __always_inline void set_locked(arena_spinlock_t __arena *lock)
{
WRITE_ONCE(lock->locked, _Q_LOCKED_VAL);
}
static __always_inline
u32 arena_fetch_set_pending_acquire(arena_spinlock_t __arena *lock)
{
u32 old, new;
old = atomic_read(&lock->val);
do {
new = old | _Q_PENDING_VAL;
/*
* These loops are not expected to stall, but we still need to
* prove to the verifier they will terminate eventually.
*/
cond_break_label(out);
} while (!atomic_try_cmpxchg_acquire(&lock->val, &old, new));
return old;
out:
bpf_printk("RUNTIME ERROR: %s unexpected cond_break exit!!!", __func__);
return old;
}
/**
* arena_spin_trylock - try to acquire the queued spinlock
* @lock : Pointer to queued spinlock structure
* Return: 1 if lock acquired, 0 if failed
*/
static __always_inline int arena_spin_trylock(arena_spinlock_t __arena *lock)
{
int val = atomic_read(&lock->val);
if (unlikely(val))
return 0;
return likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL));
}
__noinline __weak
int arena_spin_lock_slowpath(arena_spinlock_t __arena *lock, u32 val)
{
struct arena_mcs_spinlock __arena *prev, *next, *node0, *node;
int ret = -ETIMEDOUT;
u32 old, tail;
int idx;
/*
* Wait for in-progress pending->locked hand-overs with a bounded
* number of spins so that we guarantee forward progress.
*
* 0,1,0 -> 0,0,1
*/
if (val == _Q_PENDING_VAL) {
int cnt = _Q_PENDING_LOOPS;
val = atomic_cond_read_relaxed_label(&lock->val,
(VAL != _Q_PENDING_VAL) || !cnt--,
release_err);
}
/*
* If we observe any contention; queue.
*/
if (val & ~_Q_LOCKED_MASK)
goto queue;
/*
* trylock || pending
*
* 0,0,* -> 0,1,* -> 0,0,1 pending, trylock
*/
val = arena_fetch_set_pending_acquire(lock);
/*
* If we observe contention, there is a concurrent locker.
*
* Undo and queue; our setting of PENDING might have made the
* n,0,0 -> 0,0,0 transition fail and it will now be waiting
* on @next to become !NULL.
*/
if (unlikely(val & ~_Q_LOCKED_MASK)) {
/* Undo PENDING if we set it. */
if (!(val & _Q_PENDING_MASK))
clear_pending(lock);
goto queue;
}
/*
* We're pending, wait for the owner to go away.
*
* 0,1,1 -> *,1,0
*
* this wait loop must be a load-acquire such that we match the
* store-release that clears the locked bit and create lock
* sequentiality; this is because not all
* clear_pending_set_locked() implementations imply full
* barriers.
*/
if (val & _Q_LOCKED_MASK)
(void)smp_cond_load_acquire_label(&lock->locked, !VAL, release_err);
/*
* take ownership and clear the pending bit.
*
* 0,1,0 -> 0,0,1
*/
clear_pending_set_locked(lock);
return 0;
/*
* End of pending bit optimistic spinning and beginning of MCS
* queuing.
*/
queue:
node0 = &(qnodes[bpf_get_smp_processor_id()])[0].mcs;
idx = node0->count++;
tail = encode_tail(bpf_get_smp_processor_id(), idx);
/*
* 4 nodes are allocated based on the assumption that there will not be
* nested NMIs taking spinlocks. That may not be true in some
* architectures even though the chance of needing more than 4 nodes
* will still be extremely unlikely. When that happens, we simply return
* an error. Original qspinlock has a trylock fallback in this case.
*/
if (unlikely(idx >= _Q_MAX_NODES)) {
ret = -EBUSY;
goto release_node_err;
}
node = grab_mcs_node(node0, idx);
/*
* Ensure that we increment the head node->count before initialising
* the actual node. If the compiler is kind enough to reorder these
* stores, then an IRQ could overwrite our assignments.
*/
barrier();
node->locked = 0;
node->next = NULL;
/*
* We touched a (possibly) cold cacheline in the per-cpu queue node;
* attempt the trylock once more in the hope someone let go while we
* weren't watching.
*/
if (arena_spin_trylock(lock))
goto release;
/*
* Ensure that the initialisation of @node is complete before we
* publish the updated tail via xchg_tail() and potentially link
* @node into the waitqueue via WRITE_ONCE(prev->next, node) below.
*/
smp_wmb();
/*
* Publish the updated tail.
* We have already touched the queueing cacheline; don't bother with
* pending stuff.
*
* p,*,* -> n,*,*
*/
old = xchg_tail(lock, tail);
next = NULL;
/*
* if there was a previous node; link it and wait until reaching the
* head of the waitqueue.
*/
if (old & _Q_TAIL_MASK) {
prev = decode_tail(old);
/* Link @node into the waitqueue. */
WRITE_ONCE(prev->next, node);
(void)arch_mcs_spin_lock_contended_label(&node->locked, release_node_err);
/*
* While waiting for the MCS lock, the next pointer may have
* been set by another lock waiter. We cannot prefetch here
* due to lack of equivalent instruction in BPF ISA.
*/
next = READ_ONCE(node->next);
}
/*
* we're at the head of the waitqueue, wait for the owner & pending to
* go away.
*
* *,x,y -> *,0,0
*
* this wait loop must use a load-acquire such that we match the
* store-release that clears the locked bit and create lock
* sequentiality; this is because the set_locked() function below
* does not imply a full barrier.
*/
val = atomic_cond_read_acquire_label(&lock->val, !(VAL & _Q_LOCKED_PENDING_MASK),
release_node_err);
/*
* claim the lock:
*
* n,0,0 -> 0,0,1 : lock, uncontended
* *,*,0 -> *,*,1 : lock, contended
*
* If the queue head is the only one in the queue (lock value == tail)
* and nobody is pending, clear the tail code and grab the lock.
* Otherwise, we only need to grab the lock.
*/
/*
* In the PV case we might already have _Q_LOCKED_VAL set, because
* of lock stealing; therefore we must also allow:
*
* n,0,1 -> 0,0,1
*
* Note: at this point: (val & _Q_PENDING_MASK) == 0, because of the
* above wait condition, therefore any concurrent setting of
* PENDING will make the uncontended transition fail.
*/
if ((val & _Q_TAIL_MASK) == tail) {
if (atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL))
goto release; /* No contention */
}
/*
* Either somebody is queued behind us or _Q_PENDING_VAL got set
* which will then detect the remaining tail and queue behind us
* ensuring we'll see a @next.
*/
set_locked(lock);
/*
* contended path; wait for next if not observed yet, release.
*/
if (!next)
next = smp_cond_load_relaxed_label(&node->next, (VAL), release_node_err);
arch_mcs_spin_unlock_contended(&next->locked);
release:;
/*
* release the node
*
* Doing a normal dec vs this_cpu_dec is fine. An upper context always
* decrements count it incremented before returning, thus we're fine.
* For contexts interrupting us, they either observe our dec or not.
* Just ensure the compiler doesn't reorder this statement, as a
* this_cpu_dec implicitly implied that.
*/
barrier();
node0->count--;
return 0;
release_node_err:
barrier();
node0->count--;
goto release_err;
release_err:
return ret;
}
/**
* arena_spin_lock - acquire a queued spinlock
* @lock: Pointer to queued spinlock structure
*
* On error, returned value will be negative.
* On success, zero is returned.
*
* The return value _must_ be tested against zero for success,
* instead of checking it against negative, for passing the
* BPF verifier.
*
* The user should do:
* if (arena_spin_lock(...) != 0) // failure
* or
* if (arena_spin_lock(...) == 0) // success
* or
* if (arena_spin_lock(...)) // failure
* or
* if (!arena_spin_lock(...)) // success
* instead of:
* if (arena_spin_lock(...) < 0) // failure
*
* The return value can still be inspected later.
*/
static __always_inline int arena_spin_lock(arena_spinlock_t __arena *lock)
{
int val = 0;
if (CONFIG_NR_CPUS > 1024)
return -EOPNOTSUPP;
bpf_preempt_disable();
if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL)))
return 0;
val = arena_spin_lock_slowpath(lock, val);
/* FIXME: bpf_assert_range(-MAX_ERRNO, 0) once we have it working for all cases. */
if (val)
bpf_preempt_enable();
return val;
}
/**
* arena_spin_unlock - release a queued spinlock
* @lock : Pointer to queued spinlock structure
*/
static __always_inline void arena_spin_unlock(arena_spinlock_t __arena *lock)
{
/*
* unlock() needs release semantics:
*/
smp_store_release(&lock->locked, 0);
bpf_preempt_enable();
}
#define arena_spin_lock_irqsave(lock, flags) \
({ \
int __ret; \
bpf_local_irq_save(&(flags)); \
__ret = arena_spin_lock((lock)); \
if (__ret) \
bpf_local_irq_restore(&(flags)); \
(__ret); \
})
#define arena_spin_unlock_irqrestore(lock, flags) \
({ \
arena_spin_unlock((lock)); \
bpf_local_irq_restore(&(flags)); \
})
#endif
#endif /* BPF_ARENA_SPIN_LOCK_H */
linux-7.2.3/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h
+142
View File
@@ -0,0 +1,142 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#ifndef BPF_ATOMIC_H
#define BPF_ATOMIC_H
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf_may_goto.h>
extern bool CONFIG_X86_64 __kconfig __weak;
/*
* __unqual_typeof(x) - Declare an unqualified scalar type, leaving
* non-scalar types unchanged,
*
* Prefer C11 _Generic for better compile-times and simpler code. Note: 'char'
* is not type-compatible with 'signed char', and we define a separate case.
*
* This is copied verbatim from kernel's include/linux/compiler_types.h, but
* with default expression (for pointers) changed from (x) to (typeof(x)0).
*
* This is because LLVM has a bug where for lvalue (x), it does not get rid of
* an extra address_space qualifier, but does in case of rvalue (typeof(x)0).
* Hence, for pointers, we need to create an rvalue expression to get the
* desired type. See https://github.com/llvm/llvm-project/issues/53400.
*/
#define __scalar_type_to_expr_cases(type) \
unsigned type : (unsigned type)0, signed type : (signed type)0
#define __unqual_typeof(x) \
typeof(_Generic((x), \
char: (char)0, \
__scalar_type_to_expr_cases(char), \
__scalar_type_to_expr_cases(short), \
__scalar_type_to_expr_cases(int), \
__scalar_type_to_expr_cases(long), \
__scalar_type_to_expr_cases(long long), \
default: (typeof(x))0))
/* No-op for BPF */
#define cpu_relax() ({})
#define READ_ONCE(x) (*(volatile typeof(x) *)&(x))
#ifndef WRITE_ONCE
#define WRITE_ONCE(x, val) ((*(volatile typeof(x) *)&(x)) = (val))
#endif
#define cmpxchg(p, old, new) __sync_val_compare_and_swap((p), old, new)
#define try_cmpxchg(p, pold, new) \
({ \
__unqual_typeof(*(pold)) __o = *(pold); \
__unqual_typeof(*(p)) __r = cmpxchg(p, __o, new); \
if (__r != __o) \
*(pold) = __r; \
__r == __o; \
})
#define try_cmpxchg_relaxed(p, pold, new) try_cmpxchg(p, pold, new)
#define try_cmpxchg_acquire(p, pold, new) try_cmpxchg(p, pold, new)
#define smp_mb() \
({ \
volatile unsigned long __val; \
__sync_fetch_and_add(&__val, 0); \
})
#define smp_rmb() \
({ \
if (!CONFIG_X86_64) \
smp_mb(); \
else \
barrier(); \
})
#define smp_wmb() \
({ \
if (!CONFIG_X86_64) \
smp_mb(); \
else \
barrier(); \
})
/* Control dependency provides LOAD->STORE, provide LOAD->LOAD */
#define smp_acquire__after_ctrl_dep() ({ smp_rmb(); })
#define smp_load_acquire(p) \
({ \
__unqual_typeof(*(p)) __v = READ_ONCE(*(p)); \
if (!CONFIG_X86_64) \
smp_mb(); \
barrier(); \
__v; \
})
#define smp_store_release(p, val) \
({ \
if (!CONFIG_X86_64) \
smp_mb(); \
barrier(); \
WRITE_ONCE(*(p), val); \
})
#define smp_cond_load_relaxed_label(p, cond_expr, label) \
({ \
typeof(p) __ptr = (p); \
__unqual_typeof(*(p)) VAL; \
for (;;) { \
VAL = (__unqual_typeof(*(p)))READ_ONCE(*__ptr); \
if (cond_expr) \
break; \
cond_break_label(label); \
cpu_relax(); \
} \
(typeof(*(p)))VAL; \
})
#define smp_cond_load_acquire_label(p, cond_expr, label) \
({ \
__unqual_typeof(*p) __val = \
smp_cond_load_relaxed_label(p, cond_expr, label); \
smp_acquire__after_ctrl_dep(); \
(typeof(*(p)))__val; \
})
#define atomic_read(p) READ_ONCE((p)->counter)
#define atomic_cond_read_relaxed_label(p, cond_expr, label) \
smp_cond_load_relaxed_label(&(p)->counter, cond_expr, label)
#define atomic_cond_read_acquire_label(p, cond_expr, label) \
smp_cond_load_acquire_label(&(p)->counter, cond_expr, label)
#define atomic_try_cmpxchg_relaxed(p, pold, new) \
try_cmpxchg_relaxed(&(p)->counter, pold, new)
#define atomic_try_cmpxchg_acquire(p, pold, new) \
try_cmpxchg_acquire(&(p)->counter, pold, new)
#endif /* BPF_ATOMIC_H */
linux-7.2.3/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h
+84
View File
@@ -0,0 +1,84 @@
#pragma once
/*
* Note that cond_break can only be portably used in the body of a breakable
* construct, whereas can_loop can be used anywhere.
*/
#ifdef __BPF_FEATURE_MAY_GOTO
#define can_loop \
({ __label__ l_break, l_continue; \
bool ret = true; \
asm volatile goto("may_goto %l[l_break]" \
:::: l_break); \
goto l_continue; \
l_break: ret = false; \
l_continue:; \
ret; \
})
#define __cond_break(expr) \
({ __label__ l_break, l_continue; \
asm volatile goto("may_goto %l[l_break]" \
:::: l_break); \
goto l_continue; \
l_break: expr; \
l_continue:; \
})
#else
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define can_loop \
({ __label__ l_break, l_continue; \
bool ret = true; \
asm volatile goto("1:.byte 0xe5; \
.byte 0; \
.long ((%l[l_break] - 1b - 8) / 8) & 0xffff; \
.short 0" \
:::: l_break); \
goto l_continue; \
l_break: ret = false; \
l_continue:; \
ret; \
})
#define __cond_break(expr) \
({ __label__ l_break, l_continue; \
asm volatile goto("1:.byte 0xe5; \
.byte 0; \
.long ((%l[l_break] - 1b - 8) / 8) & 0xffff; \
.short 0" \
:::: l_break); \
goto l_continue; \
l_break: expr; \
l_continue:; \
})
#else
#define can_loop \
({ __label__ l_break, l_continue; \
bool ret = true; \
asm volatile goto("1:.byte 0xe5; \
.byte 0; \
.long (((%l[l_break] - 1b - 8) / 8) & 0xffff) << 16; \
.short 0" \
:::: l_break); \
goto l_continue; \
l_break: ret = false; \
l_continue:; \
ret; \
})
#define __cond_break(expr) \
({ __label__ l_break, l_continue; \
asm volatile goto("1:.byte 0xe5; \
.byte 0; \
.long (((%l[l_break] - 1b - 8) / 8) & 0xffff) << 16; \
.short 0" \
:::: l_break); \
goto l_continue; \
l_break: expr; \
l_continue:; \
})
#endif
#endif
#define cond_break __cond_break(break)
#define cond_break_label(label) __cond_break(goto label)
linux-7.2.3/tools/testing/selftests/bpf/libarena/include/libarena/asan.h
+101
View File
@@ -0,0 +1,101 @@
// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#pragma once
struct asan_init_args {
u64 arena_all_pages;
u64 arena_globals_pages;
};
int asan_init(struct asan_init_args *args);
extern volatile u64 __asan_shadow_memory_dynamic_address;
extern volatile u32 asan_reported;
extern volatile bool asan_inited;
extern volatile bool asan_report_once;
#ifdef __BPF__
#define ASAN_SHADOW_SHIFT 3
#define ASAN_SHADOW_SCALE (1ULL << ASAN_SHADOW_SHIFT)
#define ASAN_GRANULE_MASK ((1ULL << ASAN_SHADOW_SHIFT) - 1)
#define ASAN_GRANULE(addr) ((s8)((u32)(u64)((addr)) & ASAN_GRANULE_MASK))
#define __noasan __attribute__((no_sanitize("address")))
#ifdef BPF_ARENA_ASAN
static inline
s8 __arena *mem_to_shadow(void __arena *addr)
{
return (s8 __arena *)(((u32)(u64)addr >> ASAN_SHADOW_SHIFT) +
__asan_shadow_memory_dynamic_address);
}
__weak __noasan
bool asan_ready(void)
{
return __asan_shadow_memory_dynamic_address;
}
int asan_poison(void __arena *addr, s8 val, size_t size);
int asan_unpoison(void __arena *addr, size_t size);
bool asan_shadow_set(void __arena *addr);
/*
* Dummy calls to ensure the ASAN runtime's BTF information is present
* in every object file when compiling the runtime and local BPF code
* separately. The runtime calls are injected into the LLVM IR file
*/
#define DECLARE_ASAN_LOAD_STORE_SIZE(size) \
void __asan_store##size(intptr_t addr); \
void __asan_store##size##_noabort(intptr_t addr); \
void __asan_load##size(intptr_t addr); \
void __asan_load##size##_noabort(intptr_t addr); \
void __asan_report_store##size(intptr_t addr); \
void __asan_report_store##size##_noabort(intptr_t addr); \
void __asan_report_load##size(intptr_t addr); \
void __asan_report_load##size##_noabort(intptr_t addr);
DECLARE_ASAN_LOAD_STORE_SIZE(1);
DECLARE_ASAN_LOAD_STORE_SIZE(2);
DECLARE_ASAN_LOAD_STORE_SIZE(4);
DECLARE_ASAN_LOAD_STORE_SIZE(8);
void __asan_storeN(intptr_t addr, ssize_t size);
void __asan_storeN_noabort(intptr_t addr, ssize_t size);
void __asan_loadN(intptr_t addr, ssize_t size);
void __asan_loadN_noabort(intptr_t addr, ssize_t size);
/*
* Force LLVM to emit BTF information for the stubs,
* because the ASAN pass in LLVM by itself doesn't.
*/
#define ASAN_LOAD_STORE_SIZE(size) \
__asan_store##size, \
__asan_store##size##_noabort, \
__asan_load##size, \
__asan_load##size##_noabort, \
__asan_report_store##size, \
__asan_report_store##size##_noabort, \
__asan_report_load##size, \
__asan_report_load##size##_noabort
__attribute__((used))
static void (*__asan_btf_anchors[])(intptr_t) = {
ASAN_LOAD_STORE_SIZE(1),
ASAN_LOAD_STORE_SIZE(2),
ASAN_LOAD_STORE_SIZE(4),
ASAN_LOAD_STORE_SIZE(8),
};
#else /* BPF_ARENA_ASAN */
static inline int asan_poison(void __arena *addr, s8 val, size_t size) { return 0; }
static inline int asan_unpoison(void __arena *addr, size_t size) { return 0; }
static inline bool asan_shadow_set(void __arena *addr) { return 0; }
__weak bool asan_ready(void) { return true; }
#endif /* BPF_ARENA_ASAN */
#endif /* __BPF__ */
linux-7.2.3/tools/testing/selftests/bpf/libarena/include/libarena/buddy.h
+81
View File
@@ -0,0 +1,81 @@
// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#pragma once
enum buddy_consts {
/*
* Minimum allocation is 1 << BUDDY_MIN_ALLOC_SHIFT.
* Larger sizes increase internal fragmentation, but smaller
* sizes increase the space overhead of the block metadata.
*/
BUDDY_MIN_ALLOC_SHIFT = 4,
BUDDY_MIN_ALLOC_BYTES = 1 << BUDDY_MIN_ALLOC_SHIFT,
/*
* How many orders the buddy allocator can serve. Minimum block
* size is 1 << BUDDY_MIN_ALLOC_SHIFT, maximum block size is
* 1 << (BUDDY_MIN_ALLOC_SHIFT + BUDDY_CHUNK_NUM_ORDERS - 1):
* Each block has size 1 << BUDDY_MIN_ALLOC_SHIFT, and the
* allocation orders are in [0, BUDDY_CHUNK_NUM_ORDERS).
* We keep two blocks of the maximum size to retain the
* property in the code that all blocks have a buddy.
* Higher values increase the maximum allocation size,
* but also the size of the metadata for each block.
*/
BUDDY_CHUNK_NUM_ORDERS = 1 << 4,
BUDDY_CHUNK_BYTES = BUDDY_MIN_ALLOC_BYTES << (BUDDY_CHUNK_NUM_ORDERS),
/* Offset of the buddy header within a free block, see buddy.bpf.c for details */
BUDDY_HEADER_OFF = 8,
/* The maximum number of blocks a chunk may have to track. */
BUDDY_CHUNK_ITEMS = 1 << (BUDDY_CHUNK_NUM_ORDERS),
BUDDY_CHUNK_OFFSET_MASK = BUDDY_CHUNK_BYTES - 1,
/*
* Alignment for chunk allocations based on bpf_arena_alloc_pages.
* The arena allocation kfunc does not have an alignment argument,
* but that is required for all block calculations in the chunk to
* work.
*/
BUDDY_VADDR_OFFSET = BUDDY_CHUNK_BYTES,
/* Total arena virtual address space the allocator can consume. */
BUDDY_VADDR_SIZE = BUDDY_CHUNK_BYTES << 10
};
struct buddy_header {
u32 prev_index; /* "Pointer" to the previous available allocation of the same size. */
u32 next_index; /* Same for the next allocation. */
};
/*
* We bring memory into the allocator 1 MiB at a time.
*/
struct buddy_chunk {
/* The order of the current allocation for a item. 4 bits per order. */
u8 orders[BUDDY_CHUNK_ITEMS / 2];
/*
* Bit to denote whether chunk is allocated. Size of the allocated/free
* chunk found from the orders array.
*/
u8 allocated[BUDDY_CHUNK_ITEMS / 8];
/* Freelists for O(1) allocation. */
u64 freelists[BUDDY_CHUNK_NUM_ORDERS];
struct buddy_chunk __arena *next;
};
struct buddy {
struct buddy_chunk __arena *first_chunk; /* Pointer to the chunk linked list. */
arena_spinlock_t lock; /* Allocator lock */
u64 vaddr; /* Allocation into reserved vaddr */
};
#ifdef __BPF__
int buddy_init(struct buddy __arena *buddy);
int buddy_destroy(struct buddy __arena *buddy);
int buddy_free(struct buddy __arena *buddy, void __arena *free);
void __arena *buddy_alloc(struct buddy __arena *buddy, size_t size);
#endif /* __BPF__ */
linux-7.2.3/tools/testing/selftests/bpf/libarena/include/libarena/common.h
+93
View File
@@ -0,0 +1,93 @@
// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#pragma once
#ifdef __BPF__
#include <vmlinux.h>
#include <bpf_arena_common.h>
#include <bpf_arena_spin_lock.h>
#include <asm-generic/errno.h>
#ifndef __BPF_FEATURE_ADDR_SPACE_CAST
#error "Arena allocators require bpf_addr_space_cast feature"
#endif
#define arena_stdout(fmt, ...) bpf_stream_printk(1, (fmt), ##__VA_ARGS__)
#define arena_stderr(fmt, ...) bpf_stream_printk(2, (fmt), ##__VA_ARGS__)
#ifndef __maybe_unused
#define __maybe_unused __attribute__((__unused__))
#endif
#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8)))
#define ARENA_PAGES (1UL << (32 - __builtin_ffs(__PAGE_SIZE) + 1))
struct {
__uint(type, BPF_MAP_TYPE_ARENA);
__uint(map_flags, BPF_F_MMAPABLE);
__uint(max_entries, ARENA_PAGES); /* number of pages */
#if defined(__TARGET_ARCH_arm64) || defined(__aarch64__)
__ulong(map_extra, (1ull << 32)); /* start of mmap() region */
#else
__ulong(map_extra, (1ull << 44)); /* start of mmap() region */
#endif
} arena __weak SEC(".maps");
/*
* This is a variable used to aid verification. The may_goto directive
* permits open-coded for loops, but requires that the index variable is
* imprecise. To force the variable to be imprecise, initialize it with
* the opaque volatile variable 0 instead of the constant 0.
*/
extern const volatile u32 zero;
extern volatile u64 asan_violated;
int arena_fls(__u64 word);
void __arena *arena_malloc(size_t size);
void arena_free(void __arena *ptr);
/*
* The verifier associates arenas with programs by checking LD.IMM
* instruction operands for an arena and populating the program state
* with the first instance it finds. This requires accessing our global
* arena variable, but subprogs do not necessarily do so while still
* using pointers from that arena. Insert an LD.IMM instruction to
* access the arena and help the verifier.
*/
#define arena_subprog_init() do { asm volatile ("" :: "r"(&arena)); } while (0)
#else /* ! __BPF__ */
#include <stdint.h>
#define __arena
typedef uint8_t u8;
typedef uint16_t u16;
typedef uint32_t u32;
typedef uint64_t u64;
typedef int8_t s8;
typedef int16_t s16;
typedef int32_t s32;
typedef int64_t s64;
/* Dummy "definition" for userspace. */
#define arena_spinlock_t int
#endif /* __BPF__ */
struct arena_get_info_args {
void __arena *arena_base;
};
struct arena_alloc_reserve_args {
u64 nr_pages;
};
/* Reasonable default number of pages reserved by arena_alloc_reserve. */
#define ARENA_RESERVE_PAGES_DFL (8)
linux-7.2.3/tools/testing/selftests/bpf/libarena/include/libarena/rbtree.h
+83
View File
@@ -0,0 +1,83 @@
/* SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause */
#pragma once
#define RB_MAXLVL_PRINT (16)
struct rbnode;
struct rbnode {
struct rbnode __arena *parent;
union {
struct {
struct rbnode __arena *left;
struct rbnode __arena *right;
};
struct rbnode __arena *child[2];
};
uint64_t key;
/* Used as a linked list or to store KV pairs. */
union {
struct rbnode __arena *next;
uint64_t value;
};
bool is_red;
};
/*
* Does the rbtree allocate its own nodes, or do they get
* allocated by the caller?
*/
enum rbtree_alloc {
RB_ALLOC,
RB_NOALLOC,
};
/*
* Specify the behavior of rbtree insertions when the key is
* already present in the tree.
*
* RB_DEFAULT: Default behavior, reject the new insert.
*
* RB_UPDATE: Update the existing value in the rbtree.
* This updates the node itself, not just the value in
* the existing node.
*
* RB_DUPLICATE: Allow nodes with identical keys in the rbtree.
* Finding/popping/removing a key acts on any of the nodes
* with the appropriate key - there is no ordering by time
* of insertion.
*/
enum rbtree_insert_mode {
RB_DEFAULT,
RB_UPDATE,
RB_DUPLICATE,
};
struct rbtree {
struct rbnode __arena *root;
enum rbtree_alloc alloc;
enum rbtree_insert_mode insert;
};
#ifdef __BPF__
struct rbtree __arena *rb_create(enum rbtree_alloc alloc, enum rbtree_insert_mode insert);
int rb_destroy(struct rbtree __arena *rbtree);
int rb_insert(struct rbtree __arena *rbtree, u64 key, u64 value);
int rb_remove(struct rbtree __arena *rbtree, u64 key);
int rb_find(struct rbtree __arena *rbtree, u64 key, u64 *value);
int rb_print(struct rbtree __arena *rbtree);
int rb_least(struct rbtree __arena *rbtree, u64 *key, u64 *value);
int rb_pop(struct rbtree __arena *rbtree, u64 *key, u64 *value);
int rb_insert_node(struct rbtree __arena *rbtree, struct rbnode __arena *node);
int rb_remove_node(struct rbtree __arena *rbtree, struct rbnode __arena *node);
struct rbnode __arena *rb_node_alloc(u64 key, u64 value);
void rb_node_free(struct rbnode __arena *rbnode);
int rb_integrity_check(struct rbtree __arena *rbtree);
#endif /* __BPF__ */
linux-7.2.3/tools/testing/selftests/bpf/libarena/include/libarena/spmc.h
+27
View File
@@ -0,0 +1,27 @@
/* SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause */
#pragma once
struct spmc_arr;
#define SPMC_ARR_BASESZ 128
#define SPMC_ARR_ORDERS 10
struct spmc_arr {
u64 __arena *data;
u64 order;
};
struct spmc {
volatile struct spmc_arr __arena *cur;
volatile u64 top;
volatile u64 bottom;
struct spmc_arr arr[SPMC_ARR_ORDERS];
};
int spmc_owned_add(struct spmc __arena *spmc, u64 val);
int spmc_owned_remove(struct spmc __arena *spmc, u64 *val);
int spmc_steal(struct spmc __arena *spmc, u64 *val);
struct spmc __arena *spmc_create(void);
int spmc_destroy(struct spmc __arena *spmc);
linux-7.2.3/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h
+138
View File
@@ -0,0 +1,138 @@
// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#pragma once
#include <errno.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/mman.h>
#include <bpf/libbpf.h>
#include <bpf/bpf.h>
static inline int libarena_run_prog(int prog_fd)
{
LIBBPF_OPTS(bpf_test_run_opts, opts);
int ret;
ret = bpf_prog_test_run_opts(prog_fd, &opts);
if (ret)
return ret;
return opts.retval;
}
static inline bool libarena_is_test_prog(const char *name)
{
return strstr(name, "test_") == name;
}
static inline bool libarena_is_asan_test_prog(const char *name)
{
return strstr(name, "asan_test") == name;
}
static inline bool libarena_is_parallel_test_prog(const char *name)
{
return strstr(name, "parallel_test") == name;
}
static inline int libarena_run_prog_args(int prog_fd, void *args, size_t argsize)
{
LIBBPF_OPTS(bpf_test_run_opts, opts);
int ret;
opts.ctx_in = args;
opts.ctx_size_in = argsize;
ret = bpf_prog_test_run_opts(prog_fd, &opts);
return ret ?: opts.retval;
}
static inline int libarena_get_arena_base(int arena_get_info_fd,
void **arena_base)
{
LIBBPF_OPTS(bpf_test_run_opts, opts);
struct arena_get_info_args args = { .arena_base = NULL };
int ret;
opts.ctx_in = &args;
opts.ctx_size_in = sizeof(args);
ret = bpf_prog_test_run_opts(arena_get_info_fd, &opts);
if (ret)
return ret;
if (opts.retval)
return opts.retval;
*arena_base = args.arena_base;
return 0;
}
static inline int libarena_get_globals_pages(int arena_get_globals_fd,
size_t arena_all_pages,
u64 *globals_pages)
{
size_t pgsize = sysconf(_SC_PAGESIZE);
void *arena_base;
ssize_t i;
u8 *vec;
int ret;
ret = libarena_get_arena_base(arena_get_globals_fd, &arena_base);
if (ret)
return ret;
if (!arena_base)
return -EINVAL;
vec = calloc(arena_all_pages, sizeof(*vec));
if (!vec)
return -ENOMEM;
if (mincore(arena_base, arena_all_pages * pgsize, vec) < 0) {
ret = -errno;
free(vec);
return ret;
}
*globals_pages = 0;
for (i = arena_all_pages - 1; i >= 0; i--) {
if (!(vec[i] & 0x1))
break;
*globals_pages += 1;
}
free(vec);
return 0;
}
static inline int libarena_asan_init(int arena_asan_init_fd,
int asan_init_fd,
size_t arena_all_pages)
{
LIBBPF_OPTS(bpf_test_run_opts, opts);
struct asan_init_args args;
u64 globals_pages;
int ret;
ret = libarena_get_globals_pages(arena_asan_init_fd,
arena_all_pages, &globals_pages);
if (ret)
return ret;
args = (struct asan_init_args){
.arena_all_pages = arena_all_pages,
.arena_globals_pages = globals_pages,
};
opts.ctx_in = &args;
opts.ctx_size_in = sizeof(args);
ret = bpf_prog_test_run_opts(asan_init_fd, &opts);
if (ret)
return ret;
return opts.retval;
}

Some files were not shown because too many files have changed in this diff Show More