lastdragon/linux-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resguardo inicial

Browse Source
This commit is contained in:
lastdragon
2026-09-06 12:53:21 -06:00
parent 662efe3c2a
commit ad2088d178
94853 changed files with 43200404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linux-7.2.3/tools/testing/selftests/bpf/progs/access_map_in_map.c
+93
View File
@@ -0,0 +1,93 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) 2023. Huawei Technologies Co., Ltd */
#include <linux/bpf.h>
#include <time.h>
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
struct inner_map_type {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(key_size, 4);
__uint(value_size, 4);
__uint(max_entries, 1);
} inner_map SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
__type(key, int);
__type(value, int);
__uint(max_entries, 1);
__array(values, struct inner_map_type);
} outer_array_map SEC(".maps") = {
.values = {
[0] = &inner_map,
},
};
struct {
__uint(type, BPF_MAP_TYPE_HASH_OF_MAPS);
__type(key, int);
__type(value, int);
__uint(max_entries, 1);
__array(values, struct inner_map_type);
} outer_htab_map SEC(".maps") = {
.values = {
[0] = &inner_map,
},
};
char _license[] SEC("license") = "GPL";
int tgid = 0;
static int acc_map_in_map(void *outer_map)
{
int i, key, value = 0xdeadbeef;
void *inner_map;
if ((bpf_get_current_pid_tgid() >> 32) != tgid)
return 0;
/* Find nonexistent inner map */
key = 1;
inner_map = bpf_map_lookup_elem(outer_map, &key);
if (inner_map)
return 0;
/* Find the old inner map */
key = 0;
inner_map = bpf_map_lookup_elem(outer_map, &key);
if (!inner_map)
return 0;
/* Wait for the old inner map to be replaced */
for (i = 0; i < 2048; i++)
bpf_map_update_elem(inner_map, &key, &value, 0);
return 0;
}
SEC("?kprobe/" SYS_PREFIX "sys_getpgid")
int access_map_in_array(void *ctx)
{
return acc_map_in_map(&outer_array_map);
}
SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
int sleepable_access_map_in_array(void *ctx)
{
return acc_map_in_map(&outer_array_map);
}
SEC("?kprobe/" SYS_PREFIX "sys_getpgid")
int access_map_in_htab(void *ctx)
{
return acc_map_in_map(&outer_htab_map);
}
SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
int sleepable_access_map_in_htab(void *ctx)
{
return acc_map_in_map(&outer_htab_map);
}
linux-7.2.3/tools/testing/selftests/bpf/progs/arena_atomics.c
+397
View File
@@ -0,0 +1,397 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <stdbool.h>
#include <stdatomic.h>
#include <bpf_arena_common.h>
#include "../../../include/linux/filter.h"
#include "bpf_misc.h"
struct {
__uint(type, BPF_MAP_TYPE_ARENA);
__uint(map_flags, BPF_F_MMAPABLE);
__uint(max_entries, 10); /* number of pages */
#ifdef __TARGET_ARCH_arm64
__ulong(map_extra, 0x1ull << 32); /* start of mmap() region */
#else
__ulong(map_extra, 0x1ull << 44); /* start of mmap() region */
#endif
} arena SEC(".maps");
#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
bool skip_all_tests __attribute((__section__(".data"))) = false;
#else
bool skip_all_tests = true;
#endif
#if defined(ENABLE_ATOMICS_TESTS) && \
defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64))
bool skip_lacq_srel_tests __attribute((__section__(".data"))) = false;
#else
bool skip_lacq_srel_tests = true;
#endif
__u32 pid = 0;
__u64 __arena_global add64_value = 1;
__u64 __arena_global add64_result = 0;
__u32 __arena_global add32_value = 1;
__u32 __arena_global add32_result = 0;
__u64 __arena_global add_stack_value_copy = 0;
__u64 __arena_global add_stack_result = 0;
__u64 __arena_global add_noreturn_value = 1;
SEC("raw_tp/sys_enter")
int add(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
__u64 add_stack_value = 1;
add64_result = __sync_fetch_and_add(&add64_value, 2);
add32_result = __sync_fetch_and_add(&add32_value, 2);
add_stack_result = __sync_fetch_and_add(&add_stack_value, 2);
add_stack_value_copy = add_stack_value;
__sync_fetch_and_add(&add_noreturn_value, 2);
#endif
return 0;
}
__s64 __arena_global sub64_value = 1;
__s64 __arena_global sub64_result = 0;
__s32 __arena_global sub32_value = 1;
__s32 __arena_global sub32_result = 0;
__s64 __arena_global sub_stack_value_copy = 0;
__s64 __arena_global sub_stack_result = 0;
__s64 __arena_global sub_noreturn_value = 1;
SEC("raw_tp/sys_enter")
int sub(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
__u64 sub_stack_value = 1;
sub64_result = __sync_fetch_and_sub(&sub64_value, 2);
sub32_result = __sync_fetch_and_sub(&sub32_value, 2);
sub_stack_result = __sync_fetch_and_sub(&sub_stack_value, 2);
sub_stack_value_copy = sub_stack_value;
__sync_fetch_and_sub(&sub_noreturn_value, 2);
#endif
return 0;
}
#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING
_Atomic __u64 __arena_global and64_value = (0x110ull << 32);
_Atomic __u32 __arena_global and32_value = 0x110;
#else
__u64 __arena_global and64_value = (0x110ull << 32);
__u32 __arena_global and32_value = 0x110;
#endif
SEC("raw_tp/sys_enter")
int and(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING
__c11_atomic_fetch_and(&and64_value, 0x011ull << 32, memory_order_relaxed);
__c11_atomic_fetch_and(&and32_value, 0x011, memory_order_relaxed);
#else
__sync_fetch_and_and(&and64_value, 0x011ull << 32);
__sync_fetch_and_and(&and32_value, 0x011);
#endif
#endif
return 0;
}
#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING
_Atomic __u32 __arena_global or32_value = 0x110;
_Atomic __u64 __arena_global or64_value = (0x110ull << 32);
#else
__u32 __arena_global or32_value = 0x110;
__u64 __arena_global or64_value = (0x110ull << 32);
#endif
SEC("raw_tp/sys_enter")
int or(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING
__c11_atomic_fetch_or(&or64_value, 0x011ull << 32, memory_order_relaxed);
__c11_atomic_fetch_or(&or32_value, 0x011, memory_order_relaxed);
#else
__sync_fetch_and_or(&or64_value, 0x011ull << 32);
__sync_fetch_and_or(&or32_value, 0x011);
#endif
#endif
return 0;
}
#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING
_Atomic __u64 __arena_global xor64_value = (0x110ull << 32);
_Atomic __u32 __arena_global xor32_value = 0x110;
#else
__u64 __arena_global xor64_value = (0x110ull << 32);
__u32 __arena_global xor32_value = 0x110;
#endif
SEC("raw_tp/sys_enter")
int xor(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING
__c11_atomic_fetch_xor(&xor64_value, 0x011ull << 32, memory_order_relaxed);
__c11_atomic_fetch_xor(&xor32_value, 0x011, memory_order_relaxed);
#else
__sync_fetch_and_xor(&xor64_value, 0x011ull << 32);
__sync_fetch_and_xor(&xor32_value, 0x011);
#endif
#endif
return 0;
}
__u32 __arena_global cmpxchg32_value = 1;
__u32 __arena_global cmpxchg32_result_fail = 0;
__u32 __arena_global cmpxchg32_result_succeed = 0;
__u64 __arena_global cmpxchg64_value = 1;
__u64 __arena_global cmpxchg64_result_fail = 0;
__u64 __arena_global cmpxchg64_result_succeed = 0;
SEC("raw_tp/sys_enter")
int cmpxchg(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
cmpxchg64_result_fail = __sync_val_compare_and_swap(&cmpxchg64_value, 0, 3);
cmpxchg64_result_succeed = __sync_val_compare_and_swap(&cmpxchg64_value, 1, 2);
cmpxchg32_result_fail = __sync_val_compare_and_swap(&cmpxchg32_value, 0, 3);
cmpxchg32_result_succeed = __sync_val_compare_and_swap(&cmpxchg32_value, 1, 2);
#endif
return 0;
}
__u64 __arena_global xchg64_value = 1;
__u64 __arena_global xchg64_result = 0;
__u32 __arena_global xchg32_value = 1;
__u32 __arena_global xchg32_result = 0;
SEC("raw_tp/sys_enter")
int xchg(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
__u64 val64 = 2;
__u32 val32 = 2;
xchg64_result = __sync_lock_test_and_set(&xchg64_value, val64);
xchg32_result = __sync_lock_test_and_set(&xchg32_value, val32);
#endif
return 0;
}
__u64 __arena_global uaf_sink;
volatile __u64 __arena_global uaf_recovery_fails;
SEC("syscall")
int uaf(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#if defined(ENABLE_ATOMICS_TESTS) && !defined(__TARGET_ARCH_arm64) && \
!defined(__TARGET_ARCH_x86)
__u32 __arena *page32;
__u64 __arena *page64;
void __arena *page;
page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
bpf_arena_free_pages(&arena, page, 1);
uaf_recovery_fails = 24;
page32 = (__u32 __arena *)page;
uaf_sink += __sync_fetch_and_add(page32, 1);
uaf_recovery_fails -= 1;
__sync_add_and_fetch(page32, 1);
uaf_recovery_fails -= 1;
uaf_sink += __sync_fetch_and_sub(page32, 1);
uaf_recovery_fails -= 1;
__sync_sub_and_fetch(page32, 1);
uaf_recovery_fails -= 1;
uaf_sink += __sync_fetch_and_and(page32, 1);
uaf_recovery_fails -= 1;
__sync_and_and_fetch(page32, 1);
uaf_recovery_fails -= 1;
uaf_sink += __sync_fetch_and_or(page32, 1);
uaf_recovery_fails -= 1;
__sync_or_and_fetch(page32, 1);
uaf_recovery_fails -= 1;
uaf_sink += __sync_fetch_and_xor(page32, 1);
uaf_recovery_fails -= 1;
__sync_xor_and_fetch(page32, 1);
uaf_recovery_fails -= 1;
uaf_sink += __sync_val_compare_and_swap(page32, 0, 1);
uaf_recovery_fails -= 1;
uaf_sink += __sync_lock_test_and_set(page32, 1);
uaf_recovery_fails -= 1;
page64 = (__u64 __arena *)page;
uaf_sink += __sync_fetch_and_add(page64, 1);
uaf_recovery_fails -= 1;
__sync_add_and_fetch(page64, 1);
uaf_recovery_fails -= 1;
uaf_sink += __sync_fetch_and_sub(page64, 1);
uaf_recovery_fails -= 1;
__sync_sub_and_fetch(page64, 1);
uaf_recovery_fails -= 1;
uaf_sink += __sync_fetch_and_and(page64, 1);
uaf_recovery_fails -= 1;
__sync_and_and_fetch(page64, 1);
uaf_recovery_fails -= 1;
uaf_sink += __sync_fetch_and_or(page64, 1);
uaf_recovery_fails -= 1;
__sync_or_and_fetch(page64, 1);
uaf_recovery_fails -= 1;
uaf_sink += __sync_fetch_and_xor(page64, 1);
uaf_recovery_fails -= 1;
__sync_xor_and_fetch(page64, 1);
uaf_recovery_fails -= 1;
uaf_sink += __sync_val_compare_and_swap(page64, 0, 1);
uaf_recovery_fails -= 1;
uaf_sink += __sync_lock_test_and_set(page64, 1);
uaf_recovery_fails -= 1;
#endif
return 0;
}
#if __clang_major__ >= 18
__u8 __arena_global load_acquire8_value = 0x12;
__u16 __arena_global load_acquire16_value = 0x1234;
__u32 __arena_global load_acquire32_value = 0x12345678;
__u64 __arena_global load_acquire64_value = 0x1234567890abcdef;
__u8 __arena_global load_acquire8_result = 0;
__u16 __arena_global load_acquire16_result = 0;
__u32 __arena_global load_acquire32_result = 0;
__u64 __arena_global load_acquire64_result = 0;
#else
/* clang-17 crashes if the .addr_space.1 ELF section has holes. Work around
* this issue by defining the below variables as 64-bit.
*/
__u64 __arena_global load_acquire8_value;
__u64 __arena_global load_acquire16_value;
__u64 __arena_global load_acquire32_value;
__u64 __arena_global load_acquire64_value;
__u64 __arena_global load_acquire8_result;
__u64 __arena_global load_acquire16_result;
__u64 __arena_global load_acquire32_result;
__u64 __arena_global load_acquire64_result;
#endif
SEC("raw_tp/sys_enter")
int load_acquire(const void *ctx)
{
#if defined(ENABLE_ATOMICS_TESTS) && \
defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64))
#define LOAD_ACQUIRE_ARENA(SIZEOP, SIZE, SRC, DST) \
{ asm volatile ( \
"r1 = %[" #SRC "] ll;" \
"r1 = addr_space_cast(r1, 0x0, 0x1);" \
".8byte %[load_acquire_insn];" \
"r3 = %[" #DST "] ll;" \
"r3 = addr_space_cast(r3, 0x0, 0x1);" \
"*(" #SIZE " *)(r3 + 0) = r2;" \
: \
: __imm_addr(SRC), \
__imm_insn(load_acquire_insn, \
BPF_ATOMIC_OP(BPF_##SIZEOP, BPF_LOAD_ACQ, \
BPF_REG_2, BPF_REG_1, 0)), \
__imm_addr(DST) \
: __clobber_all); } \
LOAD_ACQUIRE_ARENA(B, u8, load_acquire8_value, load_acquire8_result)
LOAD_ACQUIRE_ARENA(H, u16, load_acquire16_value,
load_acquire16_result)
LOAD_ACQUIRE_ARENA(W, u32, load_acquire32_value,
load_acquire32_result)
LOAD_ACQUIRE_ARENA(DW, u64, load_acquire64_value,
load_acquire64_result)
#undef LOAD_ACQUIRE_ARENA
#endif
return 0;
}
#if __clang_major__ >= 18
__u8 __arena_global store_release8_result = 0;
__u16 __arena_global store_release16_result = 0;
__u32 __arena_global store_release32_result = 0;
__u64 __arena_global store_release64_result = 0;
#else
/* clang-17 crashes if the .addr_space.1 ELF section has holes. Work around
* this issue by defining the below variables as 64-bit.
*/
__u64 __arena_global store_release8_result;
__u64 __arena_global store_release16_result;
__u64 __arena_global store_release32_result;
__u64 __arena_global store_release64_result;
#endif
SEC("raw_tp/sys_enter")
int store_release(const void *ctx)
{
#if defined(ENABLE_ATOMICS_TESTS) && \
defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64))
#define STORE_RELEASE_ARENA(SIZEOP, DST, VAL) \
{ asm volatile ( \
"r1 = " VAL ";" \
"r2 = %[" #DST "] ll;" \
"r2 = addr_space_cast(r2, 0x0, 0x1);" \
".8byte %[store_release_insn];" \
: \
: __imm_addr(DST), \
__imm_insn(store_release_insn, \
BPF_ATOMIC_OP(BPF_##SIZEOP, BPF_STORE_REL, \
BPF_REG_2, BPF_REG_1, 0)) \
: __clobber_all); } \
STORE_RELEASE_ARENA(B, store_release8_result, "0x12")
STORE_RELEASE_ARENA(H, store_release16_result, "0x1234")
STORE_RELEASE_ARENA(W, store_release32_result, "0x12345678")
STORE_RELEASE_ARENA(DW, store_release64_result,
"0x1234567890abcdef ll")
#undef STORE_RELEASE_ARENA
#endif
return 0;
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/arena_htab.c
+59
View File
@@ -0,0 +1,59 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#define BPF_NO_KFUNC_PROTOTYPES
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
#include "bpf_experimental.h"
struct {
__uint(type, BPF_MAP_TYPE_ARENA);
__uint(map_flags, BPF_F_MMAPABLE);
__uint(max_entries, 100); /* number of pages */
} arena SEC(".maps");
#include "bpf_arena_htab.h"
void __arena *htab_for_user;
bool skip = false;
int zero = 0;
char __arena arr1[100000];
char arr2[1000];
SEC("syscall")
int arena_htab_llvm(void *ctx)
{
#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM)
struct htab __arena *htab;
char __arena *arr = arr1;
__u64 i;
htab = bpf_alloc(sizeof(*htab));
cast_kern(htab);
htab_init(htab);
cast_kern(arr);
/* first run. No old elems in the table */
for (i = zero; i < 100000 && can_loop; i++) {
htab_update_elem(htab, i, i);
arr[i] = i;
}
/* should replace some elems with new ones */
for (i = zero; i < 1000 && can_loop; i++) {
htab_update_elem(htab, i, i);
/* Access mem to make the verifier use bounded loop logic */
arr2[i] = i;
}
cast_user(htab);
htab_for_user = htab;
#else
skip = true;
#endif
return 0;
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/arena_htab_asm.c
+5
View File
@@ -0,0 +1,5 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#define BPF_ARENA_FORCE_ASM
#define arena_htab_llvm arena_htab_asm
#include "arena_htab.c"
linux-7.2.3/tools/testing/selftests/bpf/progs/arena_list.c
+99
View File
@@ -0,0 +1,99 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#define BPF_NO_KFUNC_PROTOTYPES
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
#include "bpf_experimental.h"
struct {
__uint(type, BPF_MAP_TYPE_ARENA);
__uint(map_flags, BPF_F_MMAPABLE);
__uint(max_entries, 100); /* number of pages */
#ifdef __TARGET_ARCH_arm64
__ulong(map_extra, 0x1ull << 32); /* start of mmap() region */
#else
__ulong(map_extra, 0x1ull << 44); /* start of mmap() region */
#endif
} arena SEC(".maps");
#include "bpf_arena_alloc.h"
#include "bpf_arena_list.h"
struct elem {
struct arena_list_node node;
__u64 value;
};
struct arena_list_head __arena *list_head;
int list_sum;
int cnt;
bool skip = false;
const volatile bool nonsleepable = false;
#ifdef __BPF_FEATURE_ADDR_SPACE_CAST
long __arena arena_sum;
int __arena test_val = 1;
struct arena_list_head __arena global_head;
#else
long arena_sum SEC(".addr_space.1");
int test_val SEC(".addr_space.1");
#endif
int zero;
void bpf_rcu_read_lock(void) __ksym;
void bpf_rcu_read_unlock(void) __ksym;
SEC("syscall")
int arena_list_add(void *ctx)
{
#ifdef __BPF_FEATURE_ADDR_SPACE_CAST
__u64 i;
list_head = &global_head;
for (i = zero; i < cnt && can_loop; i++) {
struct elem __arena *n = bpf_alloc(sizeof(*n));
test_val++;
n->value = i;
arena_sum += i;
list_add_head(&n->node, list_head);
}
#else
skip = true;
#endif
return 0;
}
SEC("syscall")
int arena_list_del(void *ctx)
{
#ifdef __BPF_FEATURE_ADDR_SPACE_CAST
struct elem __arena *n;
int sum = 0;
/* Take rcu_read_lock to test non-sleepable context */
if (nonsleepable)
bpf_rcu_read_lock();
arena_sum = 0;
list_for_each_entry(n, list_head, node) {
sum += n->value;
arena_sum += n->value;
list_del(&n->node);
bpf_free(n);
}
list_sum = sum;
if (nonsleepable)
bpf_rcu_read_unlock();
#else
skip = true;
#endif
return 0;
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/arena_spin_lock.c
+55
View File
@@ -0,0 +1,55 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#include <vmlinux.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
#include <bpf_arena_common.h>
#include <bpf_arena_spin_lock.h>
struct {
__uint(type, BPF_MAP_TYPE_ARENA);
__uint(map_flags, BPF_F_MMAPABLE);
__uint(max_entries, 100); /* number of pages */
#ifdef __TARGET_ARCH_arm64
__ulong(map_extra, 0x1ull << 32); /* start of mmap() region */
#else
__ulong(map_extra, 0x1ull << 44); /* start of mmap() region */
#endif
} arena SEC(".maps");
int cs_count;
#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
arena_spinlock_t __arena lock;
int test_skip = 1;
#else
int test_skip = 2;
#endif
int counter;
int limit;
SEC("tc")
int prog(void *ctx)
{
int ret = -2;
#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
unsigned long flags;
if ((ret = arena_spin_lock_irqsave(&lock, flags))) {
if (ret == -EOPNOTSUPP)
test_skip = 3;
return ret;
}
if (counter != limit)
counter++;
bpf_repeat(cs_count);
ret = 0;
arena_spin_unlock_irqrestore(&lock, flags);
#endif
return ret;
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/arena_strsearch.c
+146
View File
@@ -0,0 +1,146 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#include <vmlinux.h>
#include "bpf_experimental.h"
struct {
__uint(type, BPF_MAP_TYPE_ARENA);
__uint(map_flags, BPF_F_MMAPABLE);
__uint(max_entries, 100); /* number of pages */
} arena SEC(".maps");
#include "bpf_arena_strsearch.h"
struct glob_test {
char const __arena *pat, *str;
bool expected;
};
static bool test(char const __arena *pat, char const __arena *str, bool expected)
{
bool match = glob_match(pat, str);
bool success = match == expected;
/* bpf_printk("glob_match %s %s res %d ok %d", pat, str, match, success); */
return success;
}
/*
* The tests are all jammed together in one array to make it simpler
* to place that array in the .init.rodata section. The obvious
* "array of structures containing char *" has no way to force the
* pointed-to strings to be in a particular section.
*
* Anyway, a test consists of:
* 1. Expected glob_match result: '1' or '0'.
* 2. Pattern to match: null-terminated string
* 3. String to match against: null-terminated string
*
* The list of tests is terminated with a final '\0' instead of
* a glob_match result character.
*/
static const char __arena glob_tests[] =
/* Some basic tests */
"1" "a\0" "a\0"
"0" "a\0" "b\0"
"0" "a\0" "aa\0"
"0" "a\0" "\0"
"1" "\0" "\0"
"0" "\0" "a\0"
/* Simple character class tests */
"1" "[a]\0" "a\0"
"0" "[a]\0" "b\0"
"0" "[!a]\0" "a\0"
"1" "[!a]\0" "b\0"
"1" "[ab]\0" "a\0"
"1" "[ab]\0" "b\0"
"0" "[ab]\0" "c\0"
"1" "[!ab]\0" "c\0"
"1" "[a-c]\0" "b\0"
"0" "[a-c]\0" "d\0"
/* Corner cases in character class parsing */
"1" "[a-c-e-g]\0" "-\0"
"0" "[a-c-e-g]\0" "d\0"
"1" "[a-c-e-g]\0" "f\0"
"1" "[]a-ceg-ik[]\0" "a\0"
"1" "[]a-ceg-ik[]\0" "]\0"
"1" "[]a-ceg-ik[]\0" "[\0"
"1" "[]a-ceg-ik[]\0" "h\0"
"0" "[]a-ceg-ik[]\0" "f\0"
"0" "[!]a-ceg-ik[]\0" "h\0"
"0" "[!]a-ceg-ik[]\0" "]\0"
"1" "[!]a-ceg-ik[]\0" "f\0"
/* Simple wild cards */
"1" "?\0" "a\0"
"0" "?\0" "aa\0"
"0" "??\0" "a\0"
"1" "?x?\0" "axb\0"
"0" "?x?\0" "abx\0"
"0" "?x?\0" "xab\0"
/* Asterisk wild cards (backtracking) */
"0" "*??\0" "a\0"
"1" "*??\0" "ab\0"
"1" "*??\0" "abc\0"
"1" "*??\0" "abcd\0"
"0" "??*\0" "a\0"
"1" "??*\0" "ab\0"
"1" "??*\0" "abc\0"
"1" "??*\0" "abcd\0"
"0" "?*?\0" "a\0"
"1" "?*?\0" "ab\0"
"1" "?*?\0" "abc\0"
"1" "?*?\0" "abcd\0"
"1" "*b\0" "b\0"
"1" "*b\0" "ab\0"
"0" "*b\0" "ba\0"
"1" "*b\0" "bb\0"
"1" "*b\0" "abb\0"
"1" "*b\0" "bab\0"
"1" "*bc\0" "abbc\0"
"1" "*bc\0" "bc\0"
"1" "*bc\0" "bbc\0"
"1" "*bc\0" "bcbc\0"
/* Multiple asterisks (complex backtracking) */
"1" "*ac*\0" "abacadaeafag\0"
"1" "*ac*ae*ag*\0" "abacadaeafag\0"
"1" "*a*b*[bc]*[ef]*g*\0" "abacadaeafag\0"
"0" "*a*b*[ef]*[cd]*g*\0" "abacadaeafag\0"
"1" "*abcd*\0" "abcabcabcabcdefg\0"
"1" "*ab*cd*\0" "abcabcabcabcdefg\0"
"1" "*abcd*abcdef*\0" "abcabcdabcdeabcdefg\0"
"0" "*abcd*\0" "abcabcabcabcefg\0"
"0" "*ab*cd*\0" "abcabcabcabcefg\0";
bool skip = false;
SEC("syscall")
int arena_strsearch(void *ctx)
{
unsigned successes = 0;
unsigned n = 0;
char const __arena *p = glob_tests;
/*
* Tests are jammed together in a string. The first byte is '1'
* or '0' to indicate the expected outcome, or '\0' to indicate the
* end of the tests. Then come two null-terminated strings: the
* pattern and the string to match it against.
*/
while (*p) {
bool expected = *p++ & 1;
char const __arena *pat = p;
cond_break;
p += bpf_arena_strlen(p) + 1;
successes += test(pat, p, expected);
p += bpf_arena_strlen(p) + 1;
n++;
}
n -= successes;
/* bpf_printk("glob: %u self-tests passed, %u failed\n", successes, n); */
return n ? -1 : 0;
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/async_stack_depth.c
+61
View File
@@ -0,0 +1,61 @@
// SPDX-License-Identifier: GPL-2.0
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
struct hmap_elem {
struct bpf_timer timer;
};
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 64);
__type(key, int);
__type(value, struct hmap_elem);
} hmap SEC(".maps");
__attribute__((noinline))
static int timer_cb(void *map, int *key, struct bpf_timer *timer)
{
volatile char buf[256] = {};
return buf[69];
}
__attribute__((noinline))
static int bad_timer_cb(void *map, int *key, struct bpf_timer *timer)
{
volatile char buf[300] = {};
return buf[255] + timer_cb(NULL, NULL, NULL);
}
SEC("tc")
__failure __msg("combined stack size of 2 calls is")
int pseudo_call_check(struct __sk_buff *ctx)
{
struct hmap_elem *elem;
volatile char buf[256] = {};
elem = bpf_map_lookup_elem(&hmap, &(int){0});
if (!elem)
return 0;
timer_cb(NULL, NULL, NULL);
return bpf_timer_set_callback(&elem->timer, timer_cb) + buf[0];
}
SEC("tc")
__failure __msg("combined stack size of 2 calls is")
int async_call_root_check(struct __sk_buff *ctx)
{
struct hmap_elem *elem;
volatile char buf[256] = {};
elem = bpf_map_lookup_elem(&hmap, &(int){0});
if (!elem)
return 0;
return bpf_timer_set_callback(&elem->timer, bad_timer_cb) + buf[0];
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/atomic_bounds.c
+24
View File
@@ -0,0 +1,24 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <stdbool.h>
#ifdef ENABLE_ATOMICS_TESTS
bool skip_tests __attribute((__section__(".data"))) = false;
#else
bool skip_tests = true;
#endif
SEC("fentry/bpf_fentry_test1")
int BPF_PROG(sub, int x)
{
#ifdef ENABLE_ATOMICS_TESTS
int a = 0;
int b = __sync_fetch_and_add(&a, 1);
/* b is certainly 0 here. Can the verifier tell? */
while (b)
continue;
#endif
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/atomics.c
+170
View File
@@ -0,0 +1,170 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <stdbool.h>
#ifdef ENABLE_ATOMICS_TESTS
bool skip_tests __attribute((__section__(".data"))) = false;
#else
bool skip_tests = true;
#endif
__u32 pid = 0;
__u64 add64_value = 1;
__u64 add64_result = 0;
__u32 add32_value = 1;
__u32 add32_result = 0;
__u64 add_stack_value_copy = 0;
__u64 add_stack_result = 0;
__u64 add_noreturn_value = 1;
SEC("raw_tp/sys_enter")
int add(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
__u64 add_stack_value = 1;
add64_result = __sync_fetch_and_add(&add64_value, 2);
add32_result = __sync_fetch_and_add(&add32_value, 2);
add_stack_result = __sync_fetch_and_add(&add_stack_value, 2);
add_stack_value_copy = add_stack_value;
__sync_fetch_and_add(&add_noreturn_value, 2);
#endif
return 0;
}
__s64 sub64_value = 1;
__s64 sub64_result = 0;
__s32 sub32_value = 1;
__s32 sub32_result = 0;
__s64 sub_stack_value_copy = 0;
__s64 sub_stack_result = 0;
__s64 sub_noreturn_value = 1;
SEC("raw_tp/sys_enter")
int sub(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
__u64 sub_stack_value = 1;
sub64_result = __sync_fetch_and_sub(&sub64_value, 2);
sub32_result = __sync_fetch_and_sub(&sub32_value, 2);
sub_stack_result = __sync_fetch_and_sub(&sub_stack_value, 2);
sub_stack_value_copy = sub_stack_value;
__sync_fetch_and_sub(&sub_noreturn_value, 2);
#endif
return 0;
}
__u64 and64_value = (0x110ull << 32);
__u64 and64_result = 0;
__u32 and32_value = 0x110;
__u32 and32_result = 0;
__u64 and_noreturn_value = (0x110ull << 32);
SEC("raw_tp/sys_enter")
int and(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
and64_result = __sync_fetch_and_and(&and64_value, 0x011ull << 32);
and32_result = __sync_fetch_and_and(&and32_value, 0x011);
__sync_fetch_and_and(&and_noreturn_value, 0x011ull << 32);
#endif
return 0;
}
__u64 or64_value = (0x110ull << 32);
__u64 or64_result = 0;
__u32 or32_value = 0x110;
__u32 or32_result = 0;
__u64 or_noreturn_value = (0x110ull << 32);
SEC("raw_tp/sys_enter")
int or(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
or64_result = __sync_fetch_and_or(&or64_value, 0x011ull << 32);
or32_result = __sync_fetch_and_or(&or32_value, 0x011);
__sync_fetch_and_or(&or_noreturn_value, 0x011ull << 32);
#endif
return 0;
}
__u64 xor64_value = (0x110ull << 32);
__u64 xor64_result = 0;
__u32 xor32_value = 0x110;
__u32 xor32_result = 0;
__u64 xor_noreturn_value = (0x110ull << 32);
SEC("raw_tp/sys_enter")
int xor(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
xor64_result = __sync_fetch_and_xor(&xor64_value, 0x011ull << 32);
xor32_result = __sync_fetch_and_xor(&xor32_value, 0x011);
__sync_fetch_and_xor(&xor_noreturn_value, 0x011ull << 32);
#endif
return 0;
}
__u64 cmpxchg64_value = 1;
__u64 cmpxchg64_result_fail = 0;
__u64 cmpxchg64_result_succeed = 0;
__u32 cmpxchg32_value = 1;
__u32 cmpxchg32_result_fail = 0;
__u32 cmpxchg32_result_succeed = 0;
SEC("raw_tp/sys_enter")
int cmpxchg(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
cmpxchg64_result_fail = __sync_val_compare_and_swap(&cmpxchg64_value, 0, 3);
cmpxchg64_result_succeed = __sync_val_compare_and_swap(&cmpxchg64_value, 1, 2);
cmpxchg32_result_fail = __sync_val_compare_and_swap(&cmpxchg32_value, 0, 3);
cmpxchg32_result_succeed = __sync_val_compare_and_swap(&cmpxchg32_value, 1, 2);
#endif
return 0;
}
__u64 xchg64_value = 1;
__u64 xchg64_result = 0;
__u32 xchg32_value = 1;
__u32 xchg32_result = 0;
SEC("raw_tp/sys_enter")
int xchg(const void *ctx)
{
if (pid != (bpf_get_current_pid_tgid() >> 32))
return 0;
#ifdef ENABLE_ATOMICS_TESTS
__u64 val64 = 2;
__u32 val32 = 2;
xchg64_result = __sync_lock_test_and_set(&xchg64_value, val64);
xchg32_result = __sync_lock_test_and_set(&xchg32_value, val32);
#endif
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bad_struct_ops.c
+25
View File
@@ -0,0 +1,25 @@
// SPDX-License-Identifier: GPL-2.0
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include "../test_kmods/bpf_testmod.h"
char _license[] SEC("license") = "GPL";
SEC("struct_ops/test_1")
int BPF_PROG(test_1) { return 0; }
SEC("struct_ops/test_2")
int BPF_PROG(test_2) { return 0; }
SEC(".struct_ops.link")
struct bpf_testmod_ops testmod_1 = {
.test_1 = (void *)test_1,
.test_2 = (void *)test_2
};
SEC(".struct_ops.link")
struct bpf_testmod_ops2 testmod_2 = {
.test_1 = (void *)test_1
};
linux-7.2.3/tools/testing/selftests/bpf/progs/bad_struct_ops2.c
+14
View File
@@ -0,0 +1,14 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
/* This is an unused struct_ops program, it lacks corresponding
* struct_ops map, which provides attachment information.
* W/o additional configuration attempt to load such
* BPF object file would fail.
*/
SEC("struct_ops/foo")
void foo(void) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/bench_bpf_timing.bpf.h
+69
View File
@@ -0,0 +1,69 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#ifndef __BENCH_BPF_TIMING_BPF_H__
#define __BENCH_BPF_TIMING_BPF_H__
#include <stdbool.h>
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include <bpf_may_goto.h>
#ifndef BENCH_NR_SAMPLES
#define BENCH_NR_SAMPLES 4096
#endif
#ifndef BENCH_NR_CPUS
#define BENCH_NR_CPUS 256
#endif
#define BENCH_CPU_MASK (BENCH_NR_CPUS - 1)
__u64 timing_samples[BENCH_NR_CPUS][BENCH_NR_SAMPLES];
__u32 timing_idx[BENCH_NR_CPUS];
volatile __u32 batch_iters;
volatile __u32 timing_enabled;
static __always_inline void bench_record_sample(__u64 elapsed_ns)
{
__u32 cpu, idx;
if (!timing_enabled)
return;
cpu = bpf_get_smp_processor_id() & BENCH_CPU_MASK;
idx = timing_idx[cpu];
if (idx >= BENCH_NR_SAMPLES)
return;
timing_samples[cpu][idx] = elapsed_ns;
timing_idx[cpu] = idx + 1;
}
/*
* @body: expression to time; return value (int) stored in __bench_result.
* @reset: undo body's side-effects so each iteration starts identically.
* May reference __bench_result. Use ({}) for empty reset.
*
* Runs batch_iters timed iterations, then one untimed iteration whose
* return value the macro evaluates to (for validation).
*/
#define BENCH_BPF_LOOP(body, reset) ({ \
__u64 __bench_start = bpf_ktime_get_ns(); \
__u32 __bench_i; \
int __bench_result; \
\
for (__bench_i = 0; \
__bench_i < batch_iters && can_loop; \
__bench_i++) { \
__bench_result = (body); \
reset; \
} \
\
bench_record_sample(bpf_ktime_get_ns() - __bench_start); \
\
__bench_result = (body); \
__bench_result; \
})
#endif /* __BENCH_BPF_TIMING_BPF_H__ */
linux-7.2.3/tools/testing/selftests/bpf/progs/bench_local_storage_create.c
+72
View File
@@ -0,0 +1,72 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */
#include "vmlinux.h"
#include "bpf_tracing_net.h"
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_helpers.h>
long create_errs = 0;
long create_cnts = 0;
__u32 bench_pid = 0;
struct storage {
__u8 data[64];
};
struct {
__uint(type, BPF_MAP_TYPE_SK_STORAGE);
__uint(map_flags, BPF_F_NO_PREALLOC);
__type(key, int);
__type(value, struct storage);
} sk_storage_map SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
__uint(map_flags, BPF_F_NO_PREALLOC);
__type(key, int);
__type(value, struct storage);
} task_storage_map SEC(".maps");
SEC("tp_btf/sched_process_fork")
int BPF_PROG(sched_process_fork, struct task_struct *parent, struct task_struct *child)
{
struct storage *stg;
if (parent->tgid != bench_pid)
return 0;
stg = bpf_task_storage_get(&task_storage_map, child, NULL,
BPF_LOCAL_STORAGE_GET_F_CREATE);
if (stg)
__sync_fetch_and_add(&create_cnts, 1);
else
__sync_fetch_and_add(&create_errs, 1);
return 0;
}
SEC("lsm.s/socket_post_create")
int BPF_PROG(socket_post_create, struct socket *sock, int family, int type,
int protocol, int kern)
{
struct sock *sk = sock->sk;
struct storage *stg;
__u32 pid;
pid = bpf_get_current_pid_tgid() >> 32;
if (pid != bench_pid || !sk)
return 0;
stg = bpf_sk_storage_get(&sk_storage_map, sk, NULL,
BPF_LOCAL_STORAGE_GET_F_CREATE);
if (stg)
__sync_fetch_and_add(&create_cnts, 1);
else
__sync_fetch_and_add(&create_errs, 1);
return 0;
}
char __license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/bench_sockmap_prog.c
+65
View File
@@ -0,0 +1,65 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>
long process_byte = 0;
int verdict_dir = 0;
int dropped = 0;
int pkt_size = 0;
struct {
__uint(type, BPF_MAP_TYPE_SOCKMAP);
__uint(max_entries, 20);
__type(key, int);
__type(value, int);
} sock_map_rx SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_SOCKMAP);
__uint(max_entries, 20);
__type(key, int);
__type(value, int);
} sock_map_tx SEC(".maps");
SEC("sk_skb/stream_parser")
int prog_skb_parser(struct __sk_buff *skb)
{
return pkt_size;
}
SEC("sk_skb/stream_verdict")
int prog_skb_verdict(struct __sk_buff *skb)
{
int one = 1;
int ret = bpf_sk_redirect_map(skb, &sock_map_rx, one, verdict_dir);
if (ret == SK_DROP)
dropped++;
__sync_fetch_and_add(&process_byte, skb->len);
return ret;
}
SEC("sk_skb/stream_verdict")
int prog_skb_pass(struct __sk_buff *skb)
{
__sync_fetch_and_add(&process_byte, skb->len);
return SK_PASS;
}
SEC("sk_msg")
int prog_skmsg_verdict(struct sk_msg_md *msg)
{
int one = 1;
__sync_fetch_and_add(&process_byte, msg->size);
return bpf_msg_redirect_map(msg, &sock_map_tx, one, verdict_dir);
}
SEC("sk_msg")
int prog_skmsg_pass(struct sk_msg_md *msg)
{
__sync_fetch_and_add(&process_byte, msg->size);
return SK_PASS;
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/bind4_prog.c
+167
View File
@@ -0,0 +1,167 @@
// SPDX-License-Identifier: GPL-2.0
#include <string.h>
#include <linux/stddef.h>
#include <linux/bpf.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <linux/if.h>
#include <errno.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>
#include "bind_prog.h"
#define SERV4_IP 0xc0a801feU /* 192.168.1.254 */
#define SERV4_PORT 4040
#define SERV4_REWRITE_IP 0x7f000001U /* 127.0.0.1 */
#define SERV4_REWRITE_PORT 4444
#ifndef IFNAMSIZ
#define IFNAMSIZ 16
#endif
static __inline int bind_to_device(struct bpf_sock_addr *ctx)
{
char veth1[IFNAMSIZ] = "test_sock_addr1";
char veth2[IFNAMSIZ] = "test_sock_addr2";
char missing[IFNAMSIZ] = "nonexistent_dev";
char del_bind[IFNAMSIZ] = "";
int veth1_idx, veth2_idx;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE,
&veth1, sizeof(veth1)))
return 1;
if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
&veth1_idx, sizeof(veth1_idx)) || !veth1_idx)
return 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE,
&veth2, sizeof(veth2)))
return 1;
if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
&veth2_idx, sizeof(veth2_idx)) || !veth2_idx ||
veth1_idx == veth2_idx)
return 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE,
&missing, sizeof(missing)) != -ENODEV)
return 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
&veth1_idx, sizeof(veth1_idx)))
return 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE,
&del_bind, sizeof(del_bind)))
return 1;
return 0;
}
static __inline int bind_reuseport(struct bpf_sock_addr *ctx)
{
int val = 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_REUSEPORT,
&val, sizeof(val)))
return 1;
if (bpf_getsockopt(ctx, SOL_SOCKET, SO_REUSEPORT,
&val, sizeof(val)) || !val)
return 1;
val = 0;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_REUSEPORT,
&val, sizeof(val)))
return 1;
if (bpf_getsockopt(ctx, SOL_SOCKET, SO_REUSEPORT,
&val, sizeof(val)) || val)
return 1;
return 0;
}
static __inline int misc_opts(struct bpf_sock_addr *ctx, int opt)
{
int old, tmp, new = 0xeb9f;
/* Socket in test case has guarantee that old never equals to new. */
if (bpf_getsockopt(ctx, SOL_SOCKET, opt, &old, sizeof(old)) ||
old == new)
return 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, opt, &new, sizeof(new)))
return 1;
if (bpf_getsockopt(ctx, SOL_SOCKET, opt, &tmp, sizeof(tmp)) ||
tmp != new)
return 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, opt, &old, sizeof(old)))
return 1;
return 0;
}
SEC("cgroup/bind4")
int bind_v4_prog(struct bpf_sock_addr *ctx)
{
struct bpf_sock *sk;
__u32 user_ip4;
__u16 user_port;
sk = ctx->sk;
if (!sk)
return 0;
if (sk->family != AF_INET)
return 0;
if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM)
return 0;
if (ctx->user_ip4 != bpf_htonl(SERV4_IP) ||
ctx->user_port != bpf_htons(SERV4_PORT))
return 0;
// u8 narrow loads:
user_ip4 = 0;
user_ip4 |= load_byte(ctx->user_ip4, 0, sizeof(user_ip4));
user_ip4 |= load_byte(ctx->user_ip4, 1, sizeof(user_ip4));
user_ip4 |= load_byte(ctx->user_ip4, 2, sizeof(user_ip4));
user_ip4 |= load_byte(ctx->user_ip4, 3, sizeof(user_ip4));
if (ctx->user_ip4 != user_ip4)
return 0;
user_port = 0;
user_port |= load_byte(ctx->user_port, 0, sizeof(user_port));
user_port |= load_byte(ctx->user_port, 1, sizeof(user_port));
if (ctx->user_port != user_port)
return 0;
// u16 narrow loads:
user_ip4 = 0;
user_ip4 |= load_word(ctx->user_ip4, 0, sizeof(user_ip4));
user_ip4 |= load_word(ctx->user_ip4, 1, sizeof(user_ip4));
if (ctx->user_ip4 != user_ip4)
return 0;
/* Bind to device and unbind it. */
if (bind_to_device(ctx))
return 0;
/* Test for misc socket options. */
if (misc_opts(ctx, SO_MARK) || misc_opts(ctx, SO_PRIORITY))
return 0;
/* Set reuseport and unset */
if (bind_reuseport(ctx))
return 0;
ctx->user_ip4 = bpf_htonl(SERV4_REWRITE_IP);
ctx->user_port = bpf_htons(SERV4_REWRITE_PORT);
return 1;
}
SEC("cgroup/bind4")
int bind_v4_deny_prog(struct bpf_sock_addr *ctx)
{
return 0;
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/bind6_prog.c
+184
View File
@@ -0,0 +1,184 @@
// SPDX-License-Identifier: GPL-2.0
#include <string.h>
#include <linux/stddef.h>
#include <linux/bpf.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <linux/if.h>
#include <errno.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>
#include "bind_prog.h"
#define SERV6_IP_0 0xfaceb00c /* face:b00c:1234:5678::abcd */
#define SERV6_IP_1 0x12345678
#define SERV6_IP_2 0x00000000
#define SERV6_IP_3 0x0000abcd
#define SERV6_PORT 6060
#define SERV6_REWRITE_IP_0 0x00000000
#define SERV6_REWRITE_IP_1 0x00000000
#define SERV6_REWRITE_IP_2 0x00000000
#define SERV6_REWRITE_IP_3 0x00000001
#define SERV6_REWRITE_PORT 6666
#ifndef IFNAMSIZ
#define IFNAMSIZ 16
#endif
static __inline int bind_to_device(struct bpf_sock_addr *ctx)
{
char veth1[IFNAMSIZ] = "test_sock_addr1";
char veth2[IFNAMSIZ] = "test_sock_addr2";
char missing[IFNAMSIZ] = "nonexistent_dev";
char del_bind[IFNAMSIZ] = "";
int veth1_idx, veth2_idx;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE,
&veth1, sizeof(veth1)))
return 1;
if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
&veth1_idx, sizeof(veth1_idx)) || !veth1_idx)
return 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE,
&veth2, sizeof(veth2)))
return 1;
if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
&veth2_idx, sizeof(veth2_idx)) || !veth2_idx ||
veth1_idx == veth2_idx)
return 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE,
&missing, sizeof(missing)) != -ENODEV)
return 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
&veth1_idx, sizeof(veth1_idx)))
return 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE,
&del_bind, sizeof(del_bind)))
return 1;
return 0;
}
static __inline int bind_reuseport(struct bpf_sock_addr *ctx)
{
int val = 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_REUSEPORT,
&val, sizeof(val)))
return 1;
if (bpf_getsockopt(ctx, SOL_SOCKET, SO_REUSEPORT,
&val, sizeof(val)) || !val)
return 1;
val = 0;
if (bpf_setsockopt(ctx, SOL_SOCKET, SO_REUSEPORT,
&val, sizeof(val)))
return 1;
if (bpf_getsockopt(ctx, SOL_SOCKET, SO_REUSEPORT,
&val, sizeof(val)) || val)
return 1;
return 0;
}
static __inline int misc_opts(struct bpf_sock_addr *ctx, int opt)
{
int old, tmp, new = 0xeb9f;
/* Socket in test case has guarantee that old never equals to new. */
if (bpf_getsockopt(ctx, SOL_SOCKET, opt, &old, sizeof(old)) ||
old == new)
return 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, opt, &new, sizeof(new)))
return 1;
if (bpf_getsockopt(ctx, SOL_SOCKET, opt, &tmp, sizeof(tmp)) ||
tmp != new)
return 1;
if (bpf_setsockopt(ctx, SOL_SOCKET, opt, &old, sizeof(old)))
return 1;
return 0;
}
SEC("cgroup/bind6")
int bind_v6_prog(struct bpf_sock_addr *ctx)
{
struct bpf_sock *sk;
__u32 user_ip6;
__u16 user_port;
int i;
sk = ctx->sk;
if (!sk)
return 0;
if (sk->family != AF_INET6)
return 0;
if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM)
return 0;
if (ctx->user_ip6[0] != bpf_htonl(SERV6_IP_0) ||
ctx->user_ip6[1] != bpf_htonl(SERV6_IP_1) ||
ctx->user_ip6[2] != bpf_htonl(SERV6_IP_2) ||
ctx->user_ip6[3] != bpf_htonl(SERV6_IP_3) ||
ctx->user_port != bpf_htons(SERV6_PORT))
return 0;
// u8 narrow loads:
for (i = 0; i < 4; i++) {
user_ip6 = 0;
user_ip6 |= load_byte(ctx->user_ip6[i], 0, sizeof(user_ip6));
user_ip6 |= load_byte(ctx->user_ip6[i], 1, sizeof(user_ip6));
user_ip6 |= load_byte(ctx->user_ip6[i], 2, sizeof(user_ip6));
user_ip6 |= load_byte(ctx->user_ip6[i], 3, sizeof(user_ip6));
if (ctx->user_ip6[i] != user_ip6)
return 0;
}
user_port = 0;
user_port |= load_byte(ctx->user_port, 0, sizeof(user_port));
user_port |= load_byte(ctx->user_port, 1, sizeof(user_port));
if (ctx->user_port != user_port)
return 0;
// u16 narrow loads:
for (i = 0; i < 4; i++) {
user_ip6 = 0;
user_ip6 |= load_word(ctx->user_ip6[i], 0, sizeof(user_ip6));
user_ip6 |= load_word(ctx->user_ip6[i], 1, sizeof(user_ip6));
if (ctx->user_ip6[i] != user_ip6)
return 0;
}
/* Bind to device and unbind it. */
if (bind_to_device(ctx))
return 0;
/* Test for misc socket options. */
if (misc_opts(ctx, SO_MARK) || misc_opts(ctx, SO_PRIORITY))
return 0;
/* Set reuseport and unset */
if (bind_reuseport(ctx))
return 0;
ctx->user_ip6[0] = bpf_htonl(SERV6_REWRITE_IP_0);
ctx->user_ip6[1] = bpf_htonl(SERV6_REWRITE_IP_1);
ctx->user_ip6[2] = bpf_htonl(SERV6_REWRITE_IP_2);
ctx->user_ip6[3] = bpf_htonl(SERV6_REWRITE_IP_3);
ctx->user_port = bpf_htons(SERV6_REWRITE_PORT);
return 1;
}
SEC("cgroup/bind6")
int bind_v6_deny_prog(struct bpf_sock_addr *ctx)
{
return 0;
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/bind_perm.c
+45
View File
@@ -0,0 +1,45 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/stddef.h>
#include <linux/bpf.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>
static __always_inline int bind_prog(struct bpf_sock_addr *ctx, int family)
{
struct bpf_sock *sk;
sk = ctx->sk;
if (!sk)
return 0;
if (sk->family != family)
return 0;
if (ctx->type != SOCK_STREAM)
return 0;
/* Return 1 OR'ed with the first bit set to indicate
* that CAP_NET_BIND_SERVICE should be bypassed.
*/
if (ctx->user_port == bpf_htons(111))
return (1 | 2);
return 1;
}
SEC("cgroup/bind4")
int bind_v4_prog(struct bpf_sock_addr *ctx)
{
return bind_prog(ctx, AF_INET);
}
SEC("cgroup/bind6")
int bind_v6_prog(struct bpf_sock_addr *ctx)
{
return bind_prog(ctx, AF_INET6);
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/bind_prog.h
+19
View File
@@ -0,0 +1,19 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __BIND_PROG_H__
#define __BIND_PROG_H__
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define load_byte(src, b, s) \
(((volatile __u8 *)&(src))[b] << 8 * b)
#define load_word(src, w, s) \
(((volatile __u16 *)&(src))[w] << 16 * w)
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#define load_byte(src, b, s) \
(((volatile __u8 *)&(src))[(b) + (sizeof(src) - (s))] << 8 * ((s) - (b) - 1))
#define load_word(src, w, s) \
(((volatile __u16 *)&(src))[w] << 16 * (((s) / 2) - (w) - 1))
#else
# error "Fix your compiler's __BYTE_ORDER__?!"
#endif
#endif
linux-7.2.3/tools/testing/selftests/bpf/progs/bloom_filter_bench.c
+154
View File
@@ -0,0 +1,154 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2021 Facebook */
#include <errno.h>
#include <linux/bpf.h>
#include <stdbool.h>
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
char _license[] SEC("license") = "GPL";
struct bpf_map;
__u8 rand_vals[2500000];
const __u32 nr_rand_bytes = 2500000;
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(key_size, sizeof(__u32));
/* max entries and value_size will be set programmatically.
* They are configurable from the userspace bench program.
*/
} array_map SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_BLOOM_FILTER);
/* max entries, value_size, and # of hash functions will be set
* programmatically. They are configurable from the userspace
* bench program.
*/
__uint(map_extra, 3);
} bloom_map SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_HASH);
/* max entries, key_size, and value_size, will be set
* programmatically. They are configurable from the userspace
* bench program.
*/
} hashmap SEC(".maps");
struct callback_ctx {
struct bpf_map *map;
bool update;
};
/* Tracks the number of hits, drops, and false hits */
struct {
__u32 stats[3];
} __attribute__((__aligned__(256))) percpu_stats[256];
const __u32 hit_key = 0;
const __u32 drop_key = 1;
const __u32 false_hit_key = 2;
__u8 value_size;
const volatile bool hashmap_use_bloom;
const volatile bool count_false_hits;
int error = 0;
static __always_inline void log_result(__u32 key)
{
__u32 cpu = bpf_get_smp_processor_id();
percpu_stats[cpu & 255].stats[key]++;
}
static __u64
bloom_callback(struct bpf_map *map, __u32 *key, void *val,
struct callback_ctx *data)
{
int err;
if (data->update)
err = bpf_map_push_elem(data->map, val, 0);
else
err = bpf_map_peek_elem(data->map, val);
if (err) {
error |= 1;
return 1; /* stop the iteration */
}
log_result(hit_key);
return 0;
}
SEC("fentry/" SYS_PREFIX "sys_getpgid")
int bloom_lookup(void *ctx)
{
struct callback_ctx data;
data.map = (struct bpf_map *)&bloom_map;
data.update = false;
bpf_for_each_map_elem(&array_map, bloom_callback, &data, 0);
return 0;
}
SEC("fentry/" SYS_PREFIX "sys_getpgid")
int bloom_update(void *ctx)
{
struct callback_ctx data;
data.map = (struct bpf_map *)&bloom_map;
data.update = true;
bpf_for_each_map_elem(&array_map, bloom_callback, &data, 0);
return 0;
}
SEC("fentry/" SYS_PREFIX "sys_getpgid")
int bloom_hashmap_lookup(void *ctx)
{
__u64 *result;
int i, err;
__u32 index = bpf_get_prandom_u32();
__u32 bitmask = (1ULL << 21) - 1;
for (i = 0; i < 1024; i++, index += value_size) {
index = index & bitmask;
if (hashmap_use_bloom) {
err = bpf_map_peek_elem(&bloom_map,
rand_vals + index);
if (err) {
if (err != -ENOENT) {
error |= 2;
return 0;
}
log_result(hit_key);
continue;
}
}
result = bpf_map_lookup_elem(&hashmap,
rand_vals + index);
if (result) {
log_result(hit_key);
} else {
if (hashmap_use_bloom && count_false_hits)
log_result(false_hit_key);
log_result(drop_key);
}
}
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bloom_filter_map.c
+83
View File
@@ -0,0 +1,83 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2021 Facebook */
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
char _license[] SEC("license") = "GPL";
struct bpf_map;
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__type(key, __u32);
__type(value, __u32);
__uint(max_entries, 1000);
} map_random_data SEC(".maps");
struct map_bloom_type {
__uint(type, BPF_MAP_TYPE_BLOOM_FILTER);
__type(value, __u32);
__uint(max_entries, 10000);
__uint(map_extra, 5);
} map_bloom SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
__type(key, int);
__type(value, int);
__uint(max_entries, 1);
__array(values, struct map_bloom_type);
} outer_map SEC(".maps");
struct callback_ctx {
struct bpf_map *map;
};
int error = 0;
static __u64
check_elem(struct bpf_map *map, __u32 *key, __u32 *val,
struct callback_ctx *data)
{
int err;
err = bpf_map_peek_elem(data->map, val);
if (err) {
error |= 1;
return 1; /* stop the iteration */
}
return 0;
}
SEC("fentry/" SYS_PREFIX "sys_getpgid")
int inner_map(void *ctx)
{
struct bpf_map *inner_map;
struct callback_ctx data;
int key = 0;
inner_map = bpf_map_lookup_elem(&outer_map, &key);
if (!inner_map) {
error |= 2;
return 0;
}
data.map = inner_map;
bpf_for_each_map_elem(&map_random_data, check_elem, &data, 0);
return 0;
}
SEC("fentry/" SYS_PREFIX "sys_getpgid")
int check_bloom(void *ctx)
{
struct callback_ctx data;
data.map = (struct bpf_map *)&map_bloom;
bpf_for_each_map_elem(&map_random_data, check_elem, &data, 0);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_cc_cubic.c
+180
View File
@@ -0,0 +1,180 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Highlights:
* 1. The major difference between this bpf program and tcp_cubic.c
* is that this bpf program relies on `cong_control` rather than
* `cong_avoid` in the struct tcp_congestion_ops.
* 2. Logic such as tcp_cwnd_reduction, tcp_cong_avoid, and
* tcp_update_pacing_rate is bypassed when `cong_control` is
* defined, so moving these logic to `cong_control`.
* 3. WARNING: This bpf program is NOT the same as tcp_cubic.c.
* The main purpose is to show use cases of the arguments in
* `cong_control`. For simplicity's sake, it reuses tcp cubic's
* kernel functions.
*/
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#define USEC_PER_SEC 1000000UL
#define TCP_PACING_SS_RATIO (200)
#define TCP_PACING_CA_RATIO (120)
#define TCP_REORDERING (12)
extern void cubictcp_init(struct sock *sk) __ksym;
extern void cubictcp_cwnd_event_tx_start(struct sock *sk) __ksym;
extern __u32 cubictcp_recalc_ssthresh(struct sock *sk) __ksym;
extern void cubictcp_state(struct sock *sk, __u8 new_state) __ksym;
extern __u32 tcp_reno_undo_cwnd(struct sock *sk) __ksym;
extern void cubictcp_acked(struct sock *sk, const struct ack_sample *sample) __ksym;
extern void cubictcp_cong_avoid(struct sock *sk, __u32 ack, __u32 acked) __ksym;
static __u64 div64_u64(__u64 dividend, __u64 divisor)
{
return dividend / divisor;
}
static void tcp_update_pacing_rate(struct sock *sk)
{
const struct tcp_sock *tp = tcp_sk(sk);
__u64 rate;
/* set sk_pacing_rate to 200 % of current rate (mss * cwnd / srtt) */
rate = (__u64)tp->mss_cache * ((USEC_PER_SEC / 100) << 3);
/* current rate is (cwnd * mss) / srtt
* In Slow Start [1], set sk_pacing_rate to 200 % the current rate.
* In Congestion Avoidance phase, set it to 120 % the current rate.
*
* [1] : Normal Slow Start condition is (tp->snd_cwnd < tp->snd_ssthresh)
* If snd_cwnd >= (tp->snd_ssthresh / 2), we are approaching
* end of slow start and should slow down.
*/
if (tp->snd_cwnd < tp->snd_ssthresh / 2)
rate *= TCP_PACING_SS_RATIO;
else
rate *= TCP_PACING_CA_RATIO;
rate *= max(tp->snd_cwnd, tp->packets_out);
if (tp->srtt_us)
rate = div64_u64(rate, (__u64)tp->srtt_us);
sk->sk_pacing_rate = min(rate, sk->sk_max_pacing_rate);
}
static void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked,
int newly_lost, int flag)
{
struct tcp_sock *tp = tcp_sk(sk);
int sndcnt = 0;
__u32 pkts_in_flight = tp->packets_out - (tp->sacked_out + tp->lost_out) + tp->retrans_out;
int delta = tp->snd_ssthresh - pkts_in_flight;
if (newly_acked_sacked <= 0 || !tp->prior_cwnd)
return;
__u32 prr_delivered = tp->prr_delivered + newly_acked_sacked;
if (delta < 0) {
__u64 dividend =
(__u64)tp->snd_ssthresh * prr_delivered + tp->prior_cwnd - 1;
sndcnt = (__u32)div64_u64(dividend, (__u64)tp->prior_cwnd) - tp->prr_out;
} else {
sndcnt = max(prr_delivered - tp->prr_out, newly_acked_sacked);
if (flag & FLAG_SND_UNA_ADVANCED && !newly_lost)
sndcnt++;
sndcnt = min(delta, sndcnt);
}
/* Force a fast retransmit upon entering fast recovery */
sndcnt = max(sndcnt, (tp->prr_out ? 0 : 1));
tp->snd_cwnd = pkts_in_flight + sndcnt;
}
/* Decide whether to run the increase function of congestion control. */
static bool tcp_may_raise_cwnd(const struct sock *sk, const int flag)
{
if (tcp_sk(sk)->reordering > TCP_REORDERING)
return flag & FLAG_FORWARD_PROGRESS;
return flag & FLAG_DATA_ACKED;
}
SEC("struct_ops")
void BPF_PROG(bpf_cubic_init, struct sock *sk)
{
cubictcp_init(sk);
}
SEC("struct_ops")
void BPF_PROG(bpf_cubic_cwnd_event_tx_start, struct sock *sk)
{
cubictcp_cwnd_event_tx_start(sk);
}
SEC("struct_ops")
void BPF_PROG(bpf_cubic_cong_control, struct sock *sk, __u32 ack, int flag,
const struct rate_sample *rs)
{
struct tcp_sock *tp = tcp_sk(sk);
if (((1<<TCP_CA_CWR) | (1<<TCP_CA_Recovery)) &
(1 << inet_csk(sk)->icsk_ca_state)) {
/* Reduce cwnd if state mandates */
tcp_cwnd_reduction(sk, rs->acked_sacked, rs->losses, flag);
if (!before(tp->snd_una, tp->high_seq)) {
/* Reset cwnd to ssthresh in CWR or Recovery (unless it's undone) */
if (tp->snd_ssthresh < TCP_INFINITE_SSTHRESH &&
inet_csk(sk)->icsk_ca_state == TCP_CA_CWR) {
tp->snd_cwnd = tp->snd_ssthresh;
tp->snd_cwnd_stamp = tcp_jiffies32;
}
}
} else if (tcp_may_raise_cwnd(sk, flag)) {
/* Advance cwnd if state allows */
cubictcp_cong_avoid(sk, ack, rs->acked_sacked);
tp->snd_cwnd_stamp = tcp_jiffies32;
}
tcp_update_pacing_rate(sk);
}
SEC("struct_ops")
__u32 BPF_PROG(bpf_cubic_recalc_ssthresh, struct sock *sk)
{
return cubictcp_recalc_ssthresh(sk);
}
SEC("struct_ops")
void BPF_PROG(bpf_cubic_state, struct sock *sk, __u8 new_state)
{
cubictcp_state(sk, new_state);
}
SEC("struct_ops")
void BPF_PROG(bpf_cubic_acked, struct sock *sk, const struct ack_sample *sample)
{
cubictcp_acked(sk, sample);
}
SEC("struct_ops")
__u32 BPF_PROG(bpf_cubic_undo_cwnd, struct sock *sk)
{
return tcp_reno_undo_cwnd(sk);
}
SEC(".struct_ops")
struct tcp_congestion_ops cc_cubic = {
.init = (void *)bpf_cubic_init,
.ssthresh = (void *)bpf_cubic_recalc_ssthresh,
.cong_control = (void *)bpf_cubic_cong_control,
.set_state = (void *)bpf_cubic_state,
.undo_cwnd = (void *)bpf_cubic_undo_cwnd,
.cwnd_event_tx_start = (void *)bpf_cubic_cwnd_event_tx_start,
.pkts_acked = (void *)bpf_cubic_acked,
.name = "bpf_cc_cubic",
};
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_compiler.h
+33
View File
@@ -0,0 +1,33 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __BPF_COMPILER_H__
#define __BPF_COMPILER_H__
#define DO_PRAGMA_(X) _Pragma(#X)
#if __clang__
#define __pragma_loop_unroll DO_PRAGMA_(clang loop unroll(enable))
#else
/* In GCC -funroll-loops, which is enabled with -O2, should have the
same impact than the loop-unroll-enable pragma above. */
#define __pragma_loop_unroll
#endif
#if __clang__
#define __pragma_loop_unroll_count(N) DO_PRAGMA_(clang loop unroll_count(N))
#else
#define __pragma_loop_unroll_count(N) DO_PRAGMA_(GCC unroll N)
#endif
#if __clang__
#define __pragma_loop_unroll_full DO_PRAGMA_(clang loop unroll(full))
#else
#define __pragma_loop_unroll_full DO_PRAGMA_(GCC unroll 65534)
#endif
#if __clang__
#define __pragma_loop_no_unroll DO_PRAGMA_(clang loop unroll(disable))
#else
#define __pragma_loop_no_unroll DO_PRAGMA_(GCC unroll 1)
#endif
#endif
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_cubic.c
+554
View File
@@ -0,0 +1,554 @@
// SPDX-License-Identifier: GPL-2.0-only
/* WARNING: This implementation is not necessarily the same
* as the tcp_cubic.c. The purpose is mainly for testing
* the kernel BPF logic.
*
* Highlights:
* 1. CONFIG_HZ .kconfig map is used.
* 2. In bictcp_update(), calculation is changed to use usec
* resolution (i.e. USEC_PER_JIFFY) instead of using jiffies.
* Thus, usecs_to_jiffies() is not used in the bpf_cubic.c.
* 3. In bitctcp_update() [under tcp_friendliness], the original
* "while (ca->ack_cnt > delta)" loop is changed to the equivalent
* "ca->ack_cnt / delta" operation.
*/
#include "bpf_tracing_net.h"
#include <bpf/bpf_tracing.h>
#include <errno.h>
char _license[] SEC("license") = "GPL";
#define clamp(val, lo, hi) min((typeof(val))max(val, lo), hi)
extern __u32 tcp_slow_start(struct tcp_sock *tp, __u32 acked) __ksym;
extern void tcp_cong_avoid_ai(struct tcp_sock *tp, __u32 w, __u32 acked) __ksym;
#define BICTCP_BETA_SCALE 1024 /* Scale factor beta calculation
* max_cwnd = snd_cwnd * beta
*/
#define BICTCP_HZ 10 /* BIC HZ 2^10 = 1024 */
/* Two methods of hybrid slow start */
#define HYSTART_ACK_TRAIN 0x1
#define HYSTART_DELAY 0x2
/* Number of delay samples for detecting the increase of delay */
#define HYSTART_MIN_SAMPLES 8
#define HYSTART_DELAY_MIN (4000U) /* 4ms */
#define HYSTART_DELAY_MAX (16000U) /* 16 ms */
#define HYSTART_DELAY_THRESH(x) clamp(x, HYSTART_DELAY_MIN, HYSTART_DELAY_MAX)
static int fast_convergence = 1;
static const int beta = 717; /* = 717/1024 (BICTCP_BETA_SCALE) */
static int initial_ssthresh;
static const int bic_scale = 41;
static int tcp_friendliness = 1;
static int hystart = 1;
static int hystart_detect = HYSTART_ACK_TRAIN | HYSTART_DELAY;
static int hystart_low_window = 16;
static int hystart_ack_delta_us = 2000;
static const __u32 cube_rtt_scale = (bic_scale * 10); /* 1024*c/rtt */
static const __u32 beta_scale = 8*(BICTCP_BETA_SCALE+beta) / 3
/ (BICTCP_BETA_SCALE - beta);
/* calculate the "K" for (wmax-cwnd) = c/rtt * K^3
* so K = cubic_root( (wmax-cwnd)*rtt/c )
* the unit of K is bictcp_HZ=2^10, not HZ
*
* c = bic_scale >> 10
* rtt = 100ms
*
* the following code has been designed and tested for
* cwnd < 1 million packets
* RTT < 100 seconds
* HZ < 1,000,00 (corresponding to 10 nano-second)
*/
/* 1/c * 2^2*bictcp_HZ * srtt, 2^40 */
static const __u64 cube_factor = (__u64)(1ull << (10+3*BICTCP_HZ))
/ (bic_scale * 10);
/* BIC TCP Parameters */
struct bpf_bictcp {
__u32 cnt; /* increase cwnd by 1 after ACKs */
__u32 last_max_cwnd; /* last maximum snd_cwnd */
__u32 last_cwnd; /* the last snd_cwnd */
__u32 last_time; /* time when updated last_cwnd */
__u32 bic_origin_point;/* origin point of bic function */
__u32 bic_K; /* time to origin point
from the beginning of the current epoch */
__u32 delay_min; /* min delay (usec) */
__u32 epoch_start; /* beginning of an epoch */
__u32 ack_cnt; /* number of acks */
__u32 tcp_cwnd; /* estimated tcp cwnd */
__u16 unused;
__u8 sample_cnt; /* number of samples to decide curr_rtt */
__u8 found; /* the exit point is found? */
__u32 round_start; /* beginning of each round */
__u32 end_seq; /* end_seq of the round */
__u32 last_ack; /* last time when the ACK spacing is close */
__u32 curr_rtt; /* the minimum rtt of current round */
};
static void bictcp_reset(struct bpf_bictcp *ca)
{
ca->cnt = 0;
ca->last_max_cwnd = 0;
ca->last_cwnd = 0;
ca->last_time = 0;
ca->bic_origin_point = 0;
ca->bic_K = 0;
ca->delay_min = 0;
ca->epoch_start = 0;
ca->ack_cnt = 0;
ca->tcp_cwnd = 0;
ca->found = 0;
}
extern unsigned long CONFIG_HZ __kconfig;
#define HZ CONFIG_HZ
#define USEC_PER_MSEC 1000UL
#define USEC_PER_SEC 1000000UL
#define USEC_PER_JIFFY (USEC_PER_SEC / HZ)
static __u64 div64_u64(__u64 dividend, __u64 divisor)
{
return dividend / divisor;
}
#define div64_ul div64_u64
#define BITS_PER_U64 (sizeof(__u64) * 8)
static int fls64(__u64 x)
{
int num = BITS_PER_U64 - 1;
if (x == 0)
return 0;
if (!(x & (~0ull << (BITS_PER_U64-32)))) {
num -= 32;
x <<= 32;
}
if (!(x & (~0ull << (BITS_PER_U64-16)))) {
num -= 16;
x <<= 16;
}
if (!(x & (~0ull << (BITS_PER_U64-8)))) {
num -= 8;
x <<= 8;
}
if (!(x & (~0ull << (BITS_PER_U64-4)))) {
num -= 4;
x <<= 4;
}
if (!(x & (~0ull << (BITS_PER_U64-2)))) {
num -= 2;
x <<= 2;
}
if (!(x & (~0ull << (BITS_PER_U64-1))))
num -= 1;
return num + 1;
}
static __u32 bictcp_clock_us(const struct sock *sk)
{
return tcp_sk(sk)->tcp_mstamp;
}
static void bictcp_hystart_reset(struct sock *sk)
{
struct tcp_sock *tp = tcp_sk(sk);
struct bpf_bictcp *ca = inet_csk_ca(sk);
ca->round_start = ca->last_ack = bictcp_clock_us(sk);
ca->end_seq = tp->snd_nxt;
ca->curr_rtt = ~0U;
ca->sample_cnt = 0;
}
bool nodelay_init_reject = false;
bool nodelay_cwnd_event_tx_start_reject = false;
SEC("struct_ops")
void BPF_PROG(bpf_cubic_init, struct sock *sk)
{
struct bpf_bictcp *ca = inet_csk_ca(sk);
int true_val = 1, ret;
ret = bpf_setsockopt(sk, SOL_TCP, TCP_NODELAY, &true_val, sizeof(true_val));
if (ret == -EOPNOTSUPP)
nodelay_init_reject = true;
bictcp_reset(ca);
if (hystart)
bictcp_hystart_reset(sk);
if (!hystart && initial_ssthresh)
tcp_sk(sk)->snd_ssthresh = initial_ssthresh;
}
SEC("struct_ops")
void BPF_PROG(bpf_cubic_cwnd_event_tx_start, struct sock *sk)
{
struct bpf_bictcp *ca = inet_csk_ca(sk);
__u32 now = tcp_jiffies32;
int true_val = 1, ret;
__s32 delta;
ret = bpf_setsockopt(sk, SOL_TCP, TCP_NODELAY, &true_val, sizeof(true_val));
if (ret == -EOPNOTSUPP)
nodelay_cwnd_event_tx_start_reject = true;
delta = now - tcp_sk(sk)->lsndtime;
/* We were application limited (idle) for a while.
* Shift epoch_start to keep cwnd growth to cubic curve.
*/
if (ca->epoch_start && delta > 0) {
ca->epoch_start += delta;
if (after(ca->epoch_start, now))
ca->epoch_start = now;
}
}
/*
* cbrt(x) MSB values for x MSB values in [0..63].
* Precomputed then refined by hand - Willy Tarreau
*
* For x in [0..63],
* v = cbrt(x << 18) - 1
* cbrt(x) = (v[x] + 10) >> 6
*/
static const __u8 v[] = {
/* 0x00 */ 0, 54, 54, 54, 118, 118, 118, 118,
/* 0x08 */ 123, 129, 134, 138, 143, 147, 151, 156,
/* 0x10 */ 157, 161, 164, 168, 170, 173, 176, 179,
/* 0x18 */ 181, 185, 187, 190, 192, 194, 197, 199,
/* 0x20 */ 200, 202, 204, 206, 209, 211, 213, 215,
/* 0x28 */ 217, 219, 221, 222, 224, 225, 227, 229,
/* 0x30 */ 231, 232, 234, 236, 237, 239, 240, 242,
/* 0x38 */ 244, 245, 246, 248, 250, 251, 252, 254,
};
/* calculate the cubic root of x using a table lookup followed by one
* Newton-Raphson iteration.
* Avg err ~= 0.195%
*/
static __u32 cubic_root(__u64 a)
{
__u32 x, b, shift;
if (a < 64) {
/* a in [0..63] */
return ((__u32)v[(__u32)a] + 35) >> 6;
}
b = fls64(a);
b = ((b * 84) >> 8) - 1;
shift = (a >> (b * 3));
/* it is needed for verifier's bound check on v */
if (shift >= 64)
return 0;
x = ((__u32)(((__u32)v[shift] + 10) << b)) >> 6;
/*
* Newton-Raphson iteration
* 2
* x = ( 2 * x + a / x ) / 3
* k+1 k k
*/
x = (2 * x + (__u32)div64_u64(a, (__u64)x * (__u64)(x - 1)));
x = ((x * 341) >> 10);
return x;
}
/*
* Compute congestion window to use.
*/
static void bictcp_update(struct bpf_bictcp *ca, __u32 cwnd, __u32 acked)
{
__u32 delta, bic_target, max_cnt;
__u64 offs, t;
ca->ack_cnt += acked; /* count the number of ACKed packets */
if (ca->last_cwnd == cwnd &&
(__s32)(tcp_jiffies32 - ca->last_time) <= HZ / 32)
return;
/* The CUBIC function can update ca->cnt at most once per jiffy.
* On all cwnd reduction events, ca->epoch_start is set to 0,
* which will force a recalculation of ca->cnt.
*/
if (ca->epoch_start && tcp_jiffies32 == ca->last_time)
goto tcp_friendliness;
ca->last_cwnd = cwnd;
ca->last_time = tcp_jiffies32;
if (ca->epoch_start == 0) {
ca->epoch_start = tcp_jiffies32; /* record beginning */
ca->ack_cnt = acked; /* start counting */
ca->tcp_cwnd = cwnd; /* syn with cubic */
if (ca->last_max_cwnd <= cwnd) {
ca->bic_K = 0;
ca->bic_origin_point = cwnd;
} else {
/* Compute new K based on
* (wmax-cwnd) * (srtt>>3 / HZ) / c * 2^(3*bictcp_HZ)
*/
ca->bic_K = cubic_root(cube_factor
* (ca->last_max_cwnd - cwnd));
ca->bic_origin_point = ca->last_max_cwnd;
}
}
/* cubic function - calc*/
/* calculate c * time^3 / rtt,
* while considering overflow in calculation of time^3
* (so time^3 is done by using 64 bit)
* and without the support of division of 64bit numbers
* (so all divisions are done by using 32 bit)
* also NOTE the unit of those variables
* time = (t - K) / 2^bictcp_HZ
* c = bic_scale >> 10
* rtt = (srtt >> 3) / HZ
* !!! The following code does not have overflow problems,
* if the cwnd < 1 million packets !!!
*/
t = (__s32)(tcp_jiffies32 - ca->epoch_start) * USEC_PER_JIFFY;
t += ca->delay_min;
/* change the unit from usec to bictcp_HZ */
t <<= BICTCP_HZ;
t /= USEC_PER_SEC;
if (t < ca->bic_K) /* t - K */
offs = ca->bic_K - t;
else
offs = t - ca->bic_K;
/* c/rtt * (t-K)^3 */
delta = (cube_rtt_scale * offs * offs * offs) >> (10+3*BICTCP_HZ);
if (t < ca->bic_K) /* below origin*/
bic_target = ca->bic_origin_point - delta;
else /* above origin*/
bic_target = ca->bic_origin_point + delta;
/* cubic function - calc bictcp_cnt*/
if (bic_target > cwnd) {
ca->cnt = cwnd / (bic_target - cwnd);
} else {
ca->cnt = 100 * cwnd; /* very small increment*/
}
/*
* The initial growth of cubic function may be too conservative
* when the available bandwidth is still unknown.
*/
if (ca->last_max_cwnd == 0 && ca->cnt > 20)
ca->cnt = 20; /* increase cwnd 5% per RTT */
tcp_friendliness:
/* TCP Friendly */
if (tcp_friendliness) {
__u32 scale = beta_scale;
__u32 n;
/* update tcp cwnd */
delta = (cwnd * scale) >> 3;
if (ca->ack_cnt > delta && delta) {
n = ca->ack_cnt / delta;
ca->ack_cnt -= n * delta;
ca->tcp_cwnd += n;
}
if (ca->tcp_cwnd > cwnd) { /* if bic is slower than tcp */
delta = ca->tcp_cwnd - cwnd;
max_cnt = cwnd / delta;
if (ca->cnt > max_cnt)
ca->cnt = max_cnt;
}
}
/* The maximum rate of cwnd increase CUBIC allows is 1 packet per
* 2 packets ACKed, meaning cwnd grows at 1.5x per RTT.
*/
ca->cnt = max(ca->cnt, 2U);
}
SEC("struct_ops")
void BPF_PROG(bpf_cubic_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
{
struct tcp_sock *tp = tcp_sk(sk);
struct bpf_bictcp *ca = inet_csk_ca(sk);
if (!tcp_is_cwnd_limited(sk))
return;
if (tcp_in_slow_start(tp)) {
if (hystart && after(ack, ca->end_seq))
bictcp_hystart_reset(sk);
acked = tcp_slow_start(tp, acked);
if (!acked)
return;
}
bictcp_update(ca, tp->snd_cwnd, acked);
tcp_cong_avoid_ai(tp, ca->cnt, acked);
}
SEC("struct_ops")
__u32 BPF_PROG(bpf_cubic_recalc_ssthresh, struct sock *sk)
{
const struct tcp_sock *tp = tcp_sk(sk);
struct bpf_bictcp *ca = inet_csk_ca(sk);
ca->epoch_start = 0; /* end of epoch */
/* Wmax and fast convergence */
if (tp->snd_cwnd < ca->last_max_cwnd && fast_convergence)
ca->last_max_cwnd = (tp->snd_cwnd * (BICTCP_BETA_SCALE + beta))
/ (2 * BICTCP_BETA_SCALE);
else
ca->last_max_cwnd = tp->snd_cwnd;
return max((tp->snd_cwnd * beta) / BICTCP_BETA_SCALE, 2U);
}
SEC("struct_ops")
void BPF_PROG(bpf_cubic_state, struct sock *sk, __u8 new_state)
{
if (new_state == TCP_CA_Loss) {
bictcp_reset(inet_csk_ca(sk));
bictcp_hystart_reset(sk);
}
}
#define GSO_MAX_SIZE 65536
/* Account for TSO/GRO delays.
* Otherwise short RTT flows could get too small ssthresh, since during
* slow start we begin with small TSO packets and ca->delay_min would
* not account for long aggregation delay when TSO packets get bigger.
* Ideally even with a very small RTT we would like to have at least one
* TSO packet being sent and received by GRO, and another one in qdisc layer.
* We apply another 100% factor because @rate is doubled at this point.
* We cap the cushion to 1ms.
*/
static __u32 hystart_ack_delay(struct sock *sk)
{
unsigned long rate;
rate = sk->sk_pacing_rate;
if (!rate)
return 0;
return min((__u64)USEC_PER_MSEC,
div64_ul((__u64)GSO_MAX_SIZE * 4 * USEC_PER_SEC, rate));
}
static void hystart_update(struct sock *sk, __u32 delay)
{
struct tcp_sock *tp = tcp_sk(sk);
struct bpf_bictcp *ca = inet_csk_ca(sk);
__u32 threshold;
if (hystart_detect & HYSTART_ACK_TRAIN) {
__u32 now = bictcp_clock_us(sk);
/* first detection parameter - ack-train detection */
if ((__s32)(now - ca->last_ack) <= hystart_ack_delta_us) {
ca->last_ack = now;
threshold = ca->delay_min + hystart_ack_delay(sk);
/* Hystart ack train triggers if we get ack past
* ca->delay_min/2.
* Pacing might have delayed packets up to RTT/2
* during slow start.
*/
if (sk->sk_pacing_status == SK_PACING_NONE)
threshold >>= 1;
if ((__s32)(now - ca->round_start) > threshold) {
ca->found = 1;
tp->snd_ssthresh = tp->snd_cwnd;
}
}
}
if (hystart_detect & HYSTART_DELAY) {
/* obtain the minimum delay of more than sampling packets */
if (ca->curr_rtt > delay)
ca->curr_rtt = delay;
if (ca->sample_cnt < HYSTART_MIN_SAMPLES) {
ca->sample_cnt++;
} else {
if (ca->curr_rtt > ca->delay_min +
HYSTART_DELAY_THRESH(ca->delay_min >> 3)) {
ca->found = 1;
tp->snd_ssthresh = tp->snd_cwnd;
}
}
}
}
int bpf_cubic_acked_called = 0;
SEC("struct_ops")
void BPF_PROG(bpf_cubic_acked, struct sock *sk, const struct ack_sample *sample)
{
const struct tcp_sock *tp = tcp_sk(sk);
struct bpf_bictcp *ca = inet_csk_ca(sk);
__u32 delay;
bpf_cubic_acked_called = 1;
/* Some calls are for duplicates without timestamps */
if (sample->rtt_us < 0)
return;
/* Discard delay samples right after fast recovery */
if (ca->epoch_start && (__s32)(tcp_jiffies32 - ca->epoch_start) < HZ)
return;
delay = sample->rtt_us;
if (delay == 0)
delay = 1;
/* first time call or link delay decreases */
if (ca->delay_min == 0 || ca->delay_min > delay)
ca->delay_min = delay;
/* hystart triggers when cwnd is larger than some threshold */
if (!ca->found && tcp_in_slow_start(tp) && hystart &&
tp->snd_cwnd >= hystart_low_window)
hystart_update(sk, delay);
}
extern __u32 tcp_reno_undo_cwnd(struct sock *sk) __ksym;
SEC("struct_ops")
__u32 BPF_PROG(bpf_cubic_undo_cwnd, struct sock *sk)
{
return tcp_reno_undo_cwnd(sk);
}
SEC(".struct_ops")
struct tcp_congestion_ops cubic = {
.init = (void *)bpf_cubic_init,
.ssthresh = (void *)bpf_cubic_recalc_ssthresh,
.cong_avoid = (void *)bpf_cubic_cong_avoid,
.set_state = (void *)bpf_cubic_state,
.undo_cwnd = (void *)bpf_cubic_undo_cwnd,
.cwnd_event_tx_start = (void *)bpf_cubic_cwnd_event_tx_start,
.pkts_acked = (void *)bpf_cubic_acked,
.name = "bpf_cubic",
};
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_dctcp.c
+258
View File
@@ -0,0 +1,258 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019 Facebook */
/* WARNING: This implementation is not necessarily the same
* as the tcp_dctcp.c. The purpose is mainly for testing
* the kernel BPF logic.
*/
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#ifndef EBUSY
#define EBUSY 16
#endif
#define min_not_zero(x, y) ({ \
typeof(x) __x = (x); \
typeof(y) __y = (y); \
__x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); })
char _license[] SEC("license") = "GPL";
volatile const char fallback_cc[TCP_CA_NAME_MAX];
const char bpf_dctcp[] = "bpf_dctcp";
const char tcp_cdg[] = "cdg";
char cc_res[TCP_CA_NAME_MAX];
int tcp_cdg_res = 0;
int stg_result = 0;
int ebusy_cnt = 0;
struct {
__uint(type, BPF_MAP_TYPE_SK_STORAGE);
__uint(map_flags, BPF_F_NO_PREALLOC);
__type(key, int);
__type(value, int);
} sk_stg_map SEC(".maps");
#define DCTCP_MAX_ALPHA 1024U
struct bpf_dctcp {
__u32 old_delivered;
__u32 old_delivered_ce;
__u32 prior_rcv_nxt;
__u32 dctcp_alpha;
__u32 next_seq;
__u32 ce_state;
__u32 loss_cwnd;
};
static unsigned int dctcp_shift_g = 4; /* g = 1/2^4 */
static unsigned int dctcp_alpha_on_init = DCTCP_MAX_ALPHA;
static void dctcp_reset(const struct tcp_sock *tp, struct bpf_dctcp *ca)
{
ca->next_seq = tp->snd_nxt;
ca->old_delivered = tp->delivered;
ca->old_delivered_ce = tp->delivered_ce;
}
SEC("struct_ops")
void BPF_PROG(bpf_dctcp_init, struct sock *sk)
{
const struct tcp_sock *tp = tcp_sk(sk);
struct bpf_dctcp *ca = inet_csk_ca(sk);
int *stg;
if (!(tp->ecn_flags & TCP_ECN_OK) && fallback_cc[0]) {
/* Switch to fallback */
if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
(void *)fallback_cc, sizeof(fallback_cc)) == -EBUSY)
ebusy_cnt++;
/* Switch back to myself and the recurred bpf_dctcp_init()
* will get -EBUSY for all bpf_setsockopt(TCP_CONGESTION),
* except the last "cdg" one.
*/
if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
(void *)bpf_dctcp, sizeof(bpf_dctcp)) == -EBUSY)
ebusy_cnt++;
/* Switch back to fallback */
if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
(void *)fallback_cc, sizeof(fallback_cc)) == -EBUSY)
ebusy_cnt++;
/* Expecting -ENOTSUPP for tcp_cdg_res */
tcp_cdg_res = bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
(void *)tcp_cdg, sizeof(tcp_cdg));
bpf_getsockopt(sk, SOL_TCP, TCP_CONGESTION,
(void *)cc_res, sizeof(cc_res));
return;
}
ca->prior_rcv_nxt = tp->rcv_nxt;
ca->dctcp_alpha = min(dctcp_alpha_on_init, DCTCP_MAX_ALPHA);
ca->loss_cwnd = 0;
ca->ce_state = 0;
stg = bpf_sk_storage_get(&sk_stg_map, (void *)tp, NULL, 0);
if (stg) {
stg_result = *stg;
bpf_sk_storage_delete(&sk_stg_map, (void *)tp);
}
dctcp_reset(tp, ca);
}
SEC("struct_ops")
__u32 BPF_PROG(bpf_dctcp_ssthresh, struct sock *sk)
{
struct bpf_dctcp *ca = inet_csk_ca(sk);
struct tcp_sock *tp = tcp_sk(sk);
ca->loss_cwnd = tp->snd_cwnd;
return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->dctcp_alpha) >> 11U), 2U);
}
SEC("struct_ops")
void BPF_PROG(bpf_dctcp_update_alpha, struct sock *sk, __u32 flags)
{
const struct tcp_sock *tp = tcp_sk(sk);
struct bpf_dctcp *ca = inet_csk_ca(sk);
/* Expired RTT */
if (!before(tp->snd_una, ca->next_seq)) {
__u32 delivered_ce = tp->delivered_ce - ca->old_delivered_ce;
__u32 alpha = ca->dctcp_alpha;
/* alpha = (1 - g) * alpha + g * F */
alpha -= min_not_zero(alpha, alpha >> dctcp_shift_g);
if (delivered_ce) {
__u32 delivered = tp->delivered - ca->old_delivered;
/* If dctcp_shift_g == 1, a 32bit value would overflow
* after 8 M packets.
*/
delivered_ce <<= (10 - dctcp_shift_g);
delivered_ce /= max(1U, delivered);
alpha = min(alpha + delivered_ce, DCTCP_MAX_ALPHA);
}
ca->dctcp_alpha = alpha;
dctcp_reset(tp, ca);
}
}
static void dctcp_react_to_loss(struct sock *sk)
{
struct bpf_dctcp *ca = inet_csk_ca(sk);
struct tcp_sock *tp = tcp_sk(sk);
ca->loss_cwnd = tp->snd_cwnd;
tp->snd_ssthresh = max(tp->snd_cwnd >> 1U, 2U);
}
SEC("struct_ops")
void BPF_PROG(bpf_dctcp_state, struct sock *sk, __u8 new_state)
{
if (new_state == TCP_CA_Recovery &&
new_state != BPF_CORE_READ_BITFIELD(inet_csk(sk), icsk_ca_state))
dctcp_react_to_loss(sk);
/* We handle RTO in bpf_dctcp_cwnd_event to ensure that we perform only
* one loss-adjustment per RTT.
*/
}
static void dctcp_ece_ack_cwr(struct sock *sk, __u32 ce_state)
{
struct tcp_sock *tp = tcp_sk(sk);
if (ce_state == 1)
tp->ecn_flags |= TCP_ECN_DEMAND_CWR;
else
tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
}
/* Minimal DCTP CE state machine:
*
* S: 0 <- last pkt was non-CE
* 1 <- last pkt was CE
*/
static void dctcp_ece_ack_update(struct sock *sk, enum tcp_ca_event evt,
__u32 *prior_rcv_nxt, __u32 *ce_state)
{
__u32 new_ce_state = (evt == CA_EVENT_ECN_IS_CE) ? 1 : 0;
if (*ce_state != new_ce_state) {
/* CE state has changed, force an immediate ACK to
* reflect the new CE state. If an ACK was delayed,
* send that first to reflect the prior CE state.
*/
if (inet_csk(sk)->icsk_ack.pending & ICSK_ACK_TIMER) {
dctcp_ece_ack_cwr(sk, *ce_state);
bpf_tcp_send_ack(sk, *prior_rcv_nxt);
}
inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
}
*prior_rcv_nxt = tcp_sk(sk)->rcv_nxt;
*ce_state = new_ce_state;
dctcp_ece_ack_cwr(sk, new_ce_state);
}
SEC("struct_ops")
void BPF_PROG(bpf_dctcp_cwnd_event, struct sock *sk, enum tcp_ca_event ev)
{
struct bpf_dctcp *ca = inet_csk_ca(sk);
switch (ev) {
case CA_EVENT_ECN_IS_CE:
case CA_EVENT_ECN_NO_CE:
dctcp_ece_ack_update(sk, ev, &ca->prior_rcv_nxt, &ca->ce_state);
break;
case CA_EVENT_LOSS:
dctcp_react_to_loss(sk);
break;
default:
/* Don't care for the rest. */
break;
}
}
SEC("struct_ops")
__u32 BPF_PROG(bpf_dctcp_cwnd_undo, struct sock *sk)
{
const struct bpf_dctcp *ca = inet_csk_ca(sk);
return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd);
}
extern void tcp_reno_cong_avoid(struct sock *sk, __u32 ack, __u32 acked) __ksym;
SEC("struct_ops")
void BPF_PROG(bpf_dctcp_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
{
tcp_reno_cong_avoid(sk, ack, acked);
}
SEC(".struct_ops")
struct tcp_congestion_ops dctcp_nouse = {
.init = (void *)bpf_dctcp_init,
.set_state = (void *)bpf_dctcp_state,
.flags = TCP_CONG_NEEDS_ECN,
.name = "bpf_dctcp_nouse",
};
SEC(".struct_ops")
struct tcp_congestion_ops dctcp = {
.init = (void *)bpf_dctcp_init,
.in_ack_event = (void *)bpf_dctcp_update_alpha,
.cwnd_event = (void *)bpf_dctcp_cwnd_event,
.ssthresh = (void *)bpf_dctcp_ssthresh,
.cong_avoid = (void *)bpf_dctcp_cong_avoid,
.undo_cwnd = (void *)bpf_dctcp_cwnd_undo,
.set_state = (void *)bpf_dctcp_state,
.flags = TCP_CONG_NEEDS_ECN,
.name = "bpf_dctcp",
};
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_dctcp_release.c
+22
View File
@@ -0,0 +1,22 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2021 Facebook */
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
char _license[] SEC("license") = "GPL";
const char cubic[] = "cubic";
SEC("struct_ops")
void BPF_PROG(dctcp_nouse_release, struct sock *sk)
{
bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
(void *)cubic, sizeof(cubic));
}
SEC(".struct_ops")
struct tcp_congestion_ops dctcp_rel = {
.release = (void *)dctcp_nouse_release,
.name = "bpf_dctcp_rel",
};
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_flow.c
+437
View File
@@ -0,0 +1,437 @@
// SPDX-License-Identifier: GPL-2.0
#include <limits.h>
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include <linux/pkt_cls.h>
#include <linux/bpf.h>
#include <linux/in.h>
#include <linux/if_ether.h>
#include <linux/icmp.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/if_packet.h>
#include <sys/socket.h>
#include <linux/if_tunnel.h>
#include <linux/mpls.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>
#define PROG(F) PROG_(F, _##F)
#define PROG_(NUM, NAME) SEC("flow_dissector") int flow_dissector_##NUM
#define FLOW_CONTINUE_SADDR 0x7f00007f /* 127.0.0.127 */
/* These are the identifiers of the BPF programs that will be used in tail
* calls. Name is limited to 16 characters, with the terminating character and
* bpf_func_ above, we have only 6 to work with, anything after will be cropped.
*/
#define IP 0
#define IPV6 1
#define IPV6OP 2 /* Destination/Hop-by-Hop Options IPv6 Ext. Header */
#define IPV6FR 3 /* Fragmentation IPv6 Extension Header */
#define MPLS 4
#define VLAN 5
#define MAX_PROG 6
#define IP_MF 0x2000
#define IP_OFFSET 0x1FFF
#define IP6_MF 0x0001
#define IP6_OFFSET 0xFFF8
struct vlan_hdr {
__be16 h_vlan_TCI;
__be16 h_vlan_encapsulated_proto;
};
struct gre_hdr {
__be16 flags;
__be16 proto;
};
struct frag_hdr {
__u8 nexthdr;
__u8 reserved;
__be16 frag_off;
__be32 identification;
};
struct {
__uint(type, BPF_MAP_TYPE_PROG_ARRAY);
__uint(max_entries, MAX_PROG);
__uint(key_size, sizeof(__u32));
__uint(value_size, sizeof(__u32));
} jmp_table SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 1024);
__type(key, __u32);
__type(value, struct bpf_flow_keys);
} last_dissection SEC(".maps");
static __always_inline int export_flow_keys(struct bpf_flow_keys *keys,
int ret)
{
__u32 key = (__u32)(keys->sport) << 16 | keys->dport;
struct bpf_flow_keys val;
memcpy(&val, keys, sizeof(val));
bpf_map_update_elem(&last_dissection, &key, &val, BPF_ANY);
return ret;
}
#define IPV6_FLOWLABEL_MASK __bpf_constant_htonl(0x000FFFFF)
static inline __be32 ip6_flowlabel(const struct ipv6hdr *hdr)
{
return *(__be32 *)hdr & IPV6_FLOWLABEL_MASK;
}
static __always_inline void *bpf_flow_dissect_get_header(struct __sk_buff *skb,
__u16 hdr_size,
void *buffer)
{
void *data_end = (void *)(long)skb->data_end;
void *data = (void *)(long)skb->data;
__u16 thoff = skb->flow_keys->thoff;
__u8 *hdr;
/* Verifies this variable offset does not overflow */
if (thoff > (USHRT_MAX - hdr_size))
return NULL;
hdr = data + thoff;
if (hdr + hdr_size <= data_end)
return hdr;
if (bpf_skb_load_bytes(skb, thoff, buffer, hdr_size))
return NULL;
return buffer;
}
/* Dispatches on ETHERTYPE */
static __always_inline int parse_eth_proto(struct __sk_buff *skb, __be16 proto)
{
struct bpf_flow_keys *keys = skb->flow_keys;
switch (proto) {
case bpf_htons(ETH_P_IP):
bpf_tail_call_static(skb, &jmp_table, IP);
break;
case bpf_htons(ETH_P_IPV6):
bpf_tail_call_static(skb, &jmp_table, IPV6);
break;
case bpf_htons(ETH_P_MPLS_MC):
case bpf_htons(ETH_P_MPLS_UC):
bpf_tail_call_static(skb, &jmp_table, MPLS);
break;
case bpf_htons(ETH_P_8021Q):
case bpf_htons(ETH_P_8021AD):
bpf_tail_call_static(skb, &jmp_table, VLAN);
break;
default:
/* Protocol not supported */
return export_flow_keys(keys, BPF_DROP);
}
return export_flow_keys(keys, BPF_DROP);
}
SEC("flow_dissector")
int _dissect(struct __sk_buff *skb)
{
struct bpf_flow_keys *keys = skb->flow_keys;
if (keys->n_proto == bpf_htons(ETH_P_IP)) {
/* IP traffic from FLOW_CONTINUE_SADDR falls-back to
* standard dissector
*/
struct iphdr *iph, _iph;
iph = bpf_flow_dissect_get_header(skb, sizeof(*iph), &_iph);
if (iph && iph->ihl == 5 &&
iph->saddr == bpf_htonl(FLOW_CONTINUE_SADDR)) {
return BPF_FLOW_DISSECTOR_CONTINUE;
}
}
return parse_eth_proto(skb, keys->n_proto);
}
/* Parses on IPPROTO_* */
static __always_inline int parse_ip_proto(struct __sk_buff *skb, __u8 proto)
{
struct bpf_flow_keys *keys = skb->flow_keys;
void *data_end = (void *)(long)skb->data_end;
struct icmphdr *icmp, _icmp;
struct gre_hdr *gre, _gre;
struct ethhdr *eth, _eth;
struct tcphdr *tcp, _tcp;
struct udphdr *udp, _udp;
switch (proto) {
case IPPROTO_ICMP:
icmp = bpf_flow_dissect_get_header(skb, sizeof(*icmp), &_icmp);
if (!icmp)
return export_flow_keys(keys, BPF_DROP);
return export_flow_keys(keys, BPF_OK);
case IPPROTO_IPIP:
keys->is_encap = true;
if (keys->flags & BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP)
return export_flow_keys(keys, BPF_OK);
return parse_eth_proto(skb, bpf_htons(ETH_P_IP));
case IPPROTO_IPV6:
keys->is_encap = true;
if (keys->flags & BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP)
return export_flow_keys(keys, BPF_OK);
return parse_eth_proto(skb, bpf_htons(ETH_P_IPV6));
case IPPROTO_GRE:
gre = bpf_flow_dissect_get_header(skb, sizeof(*gre), &_gre);
if (!gre)
return export_flow_keys(keys, BPF_DROP);
if (bpf_htons(gre->flags & GRE_VERSION))
/* Only inspect standard GRE packets with version 0 */
return export_flow_keys(keys, BPF_OK);
keys->thoff += sizeof(*gre); /* Step over GRE Flags and Proto */
if (GRE_IS_CSUM(gre->flags))
keys->thoff += 4; /* Step over chksum and Padding */
if (GRE_IS_KEY(gre->flags))
keys->thoff += 4; /* Step over key */
if (GRE_IS_SEQ(gre->flags))
keys->thoff += 4; /* Step over sequence number */
keys->is_encap = true;
if (keys->flags & BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP)
return export_flow_keys(keys, BPF_OK);
if (gre->proto == bpf_htons(ETH_P_TEB)) {
eth = bpf_flow_dissect_get_header(skb, sizeof(*eth),
&_eth);
if (!eth)
return export_flow_keys(keys, BPF_DROP);
keys->thoff += sizeof(*eth);
return parse_eth_proto(skb, eth->h_proto);
} else {
return parse_eth_proto(skb, gre->proto);
}
case IPPROTO_TCP:
tcp = bpf_flow_dissect_get_header(skb, sizeof(*tcp), &_tcp);
if (!tcp)
return export_flow_keys(keys, BPF_DROP);
if (tcp->doff < 5)
return export_flow_keys(keys, BPF_DROP);
if ((__u8 *)tcp + (tcp->doff << 2) > data_end)
return export_flow_keys(keys, BPF_DROP);
keys->sport = tcp->source;
keys->dport = tcp->dest;
return export_flow_keys(keys, BPF_OK);
case IPPROTO_UDP:
case IPPROTO_UDPLITE:
udp = bpf_flow_dissect_get_header(skb, sizeof(*udp), &_udp);
if (!udp)
return export_flow_keys(keys, BPF_DROP);
keys->sport = udp->source;
keys->dport = udp->dest;
return export_flow_keys(keys, BPF_OK);
default:
return export_flow_keys(keys, BPF_DROP);
}
return export_flow_keys(keys, BPF_DROP);
}
static __always_inline int parse_ipv6_proto(struct __sk_buff *skb, __u8 nexthdr)
{
struct bpf_flow_keys *keys = skb->flow_keys;
switch (nexthdr) {
case IPPROTO_HOPOPTS:
case IPPROTO_DSTOPTS:
bpf_tail_call_static(skb, &jmp_table, IPV6OP);
break;
case IPPROTO_FRAGMENT:
bpf_tail_call_static(skb, &jmp_table, IPV6FR);
break;
default:
return parse_ip_proto(skb, nexthdr);
}
return export_flow_keys(keys, BPF_DROP);
}
PROG(IP)(struct __sk_buff *skb)
{
void *data_end = (void *)(long)skb->data_end;
struct bpf_flow_keys *keys = skb->flow_keys;
void *data = (void *)(long)skb->data;
struct iphdr *iph, _iph;
bool done = false;
iph = bpf_flow_dissect_get_header(skb, sizeof(*iph), &_iph);
if (!iph)
return export_flow_keys(keys, BPF_DROP);
/* IP header cannot be smaller than 20 bytes */
if (iph->ihl < 5)
return export_flow_keys(keys, BPF_DROP);
keys->addr_proto = ETH_P_IP;
keys->ipv4_src = iph->saddr;
keys->ipv4_dst = iph->daddr;
keys->ip_proto = iph->protocol;
keys->thoff += iph->ihl << 2;
if (data + keys->thoff > data_end)
return export_flow_keys(keys, BPF_DROP);
if (iph->frag_off & bpf_htons(IP_MF | IP_OFFSET)) {
keys->is_frag = true;
if (iph->frag_off & bpf_htons(IP_OFFSET)) {
/* From second fragment on, packets do not have headers
* we can parse.
*/
done = true;
} else {
keys->is_first_frag = true;
/* No need to parse fragmented packet unless
* explicitly asked for.
*/
if (!(keys->flags &
BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG))
done = true;
}
}
if (done)
return export_flow_keys(keys, BPF_OK);
return parse_ip_proto(skb, iph->protocol);
}
PROG(IPV6)(struct __sk_buff *skb)
{
struct bpf_flow_keys *keys = skb->flow_keys;
struct ipv6hdr *ip6h, _ip6h;
ip6h = bpf_flow_dissect_get_header(skb, sizeof(*ip6h), &_ip6h);
if (!ip6h)
return export_flow_keys(keys, BPF_DROP);
keys->addr_proto = ETH_P_IPV6;
memcpy(&keys->ipv6_src, &ip6h->saddr, 2*sizeof(ip6h->saddr));
keys->thoff += sizeof(struct ipv6hdr);
keys->ip_proto = ip6h->nexthdr;
keys->flow_label = ip6_flowlabel(ip6h);
if (keys->flow_label && keys->flags & BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL)
return export_flow_keys(keys, BPF_OK);
return parse_ipv6_proto(skb, ip6h->nexthdr);
}
PROG(IPV6OP)(struct __sk_buff *skb)
{
struct bpf_flow_keys *keys = skb->flow_keys;
struct ipv6_opt_hdr *ip6h, _ip6h;
ip6h = bpf_flow_dissect_get_header(skb, sizeof(*ip6h), &_ip6h);
if (!ip6h)
return export_flow_keys(keys, BPF_DROP);
/* hlen is in 8-octets and does not include the first 8 bytes
* of the header
*/
keys->thoff += (1 + ip6h->hdrlen) << 3;
keys->ip_proto = ip6h->nexthdr;
return parse_ipv6_proto(skb, ip6h->nexthdr);
}
PROG(IPV6FR)(struct __sk_buff *skb)
{
struct bpf_flow_keys *keys = skb->flow_keys;
struct frag_hdr *fragh, _fragh;
fragh = bpf_flow_dissect_get_header(skb, sizeof(*fragh), &_fragh);
if (!fragh)
return export_flow_keys(keys, BPF_DROP);
keys->thoff += sizeof(*fragh);
keys->is_frag = true;
keys->ip_proto = fragh->nexthdr;
if (!(fragh->frag_off & bpf_htons(IP6_OFFSET))) {
keys->is_first_frag = true;
/* No need to parse fragmented packet unless
* explicitly asked for.
*/
if (!(keys->flags & BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG))
return export_flow_keys(keys, BPF_OK);
} else {
return export_flow_keys(keys, BPF_OK);
}
return parse_ipv6_proto(skb, fragh->nexthdr);
}
PROG(MPLS)(struct __sk_buff *skb)
{
struct bpf_flow_keys *keys = skb->flow_keys;
struct mpls_label *mpls, _mpls;
mpls = bpf_flow_dissect_get_header(skb, sizeof(*mpls), &_mpls);
if (!mpls)
return export_flow_keys(keys, BPF_DROP);
return export_flow_keys(keys, BPF_OK);
}
PROG(VLAN)(struct __sk_buff *skb)
{
struct bpf_flow_keys *keys = skb->flow_keys;
struct vlan_hdr *vlan, _vlan;
/* Account for double-tagging */
if (keys->n_proto == bpf_htons(ETH_P_8021AD)) {
vlan = bpf_flow_dissect_get_header(skb, sizeof(*vlan), &_vlan);
if (!vlan)
return export_flow_keys(keys, BPF_DROP);
if (vlan->h_vlan_encapsulated_proto != bpf_htons(ETH_P_8021Q))
return export_flow_keys(keys, BPF_DROP);
keys->nhoff += sizeof(*vlan);
keys->thoff += sizeof(*vlan);
}
vlan = bpf_flow_dissect_get_header(skb, sizeof(*vlan), &_vlan);
if (!vlan)
return export_flow_keys(keys, BPF_DROP);
keys->nhoff += sizeof(*vlan);
keys->thoff += sizeof(*vlan);
/* Only allow 8021AD + 8021Q double tagging and no triple tagging.*/
if (vlan->h_vlan_encapsulated_proto == bpf_htons(ETH_P_8021AD) ||
vlan->h_vlan_encapsulated_proto == bpf_htons(ETH_P_8021Q))
return export_flow_keys(keys, BPF_DROP);
keys->n_proto = vlan->h_vlan_encapsulated_proto;
return parse_eth_proto(skb, vlan->h_vlan_encapsulated_proto);
}
char __license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_gotox.c
+479
View File
@@ -0,0 +1,479 @@
// SPDX-License-Identifier: GPL-2.0
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
#include "bpf_misc.h"
__u64 in_user;
__u64 ret_user;
int pid;
/*
* Skip all the tests if compiler doesn't support indirect jumps.
*
* If tests are skipped, then all functions below are compiled as
* dummy, such that the skeleton looks the same, and the userspace
* program can avoid any checks rather than if data->skip is set.
*/
#ifdef __BPF_FEATURE_GOTOX
__u64 skip SEC(".data") = 0;
#else
__u64 skip = 1;
#endif
struct simple_ctx {
__u64 x;
};
#ifdef __BPF_FEATURE_GOTOX
__u64 some_var;
/*
* This function adds code which will be replaced by a different
* number of instructions by the verifier. This adds additional
* stress on testing the insn_array maps corresponding to indirect jumps.
*/
static __always_inline void adjust_insns(__u64 x)
{
some_var ^= x + bpf_jiffies64();
}
SEC("syscall")
int one_switch(struct simple_ctx *ctx)
{
switch (ctx->x) {
case 0:
adjust_insns(ctx->x + 1);
ret_user = 2;
break;
case 1:
adjust_insns(ctx->x + 7);
ret_user = 3;
break;
case 2:
adjust_insns(ctx->x + 9);
ret_user = 4;
break;
case 3:
adjust_insns(ctx->x + 11);
ret_user = 5;
break;
case 4:
adjust_insns(ctx->x + 17);
ret_user = 7;
break;
default:
adjust_insns(ctx->x + 177);
ret_user = 19;
break;
}
return 0;
}
SEC("syscall")
int one_switch_non_zero_sec_off(struct simple_ctx *ctx)
{
switch (ctx->x) {
case 0:
adjust_insns(ctx->x + 1);
ret_user = 2;
break;
case 1:
adjust_insns(ctx->x + 7);
ret_user = 3;
break;
case 2:
adjust_insns(ctx->x + 9);
ret_user = 4;
break;
case 3:
adjust_insns(ctx->x + 11);
ret_user = 5;
break;
case 4:
adjust_insns(ctx->x + 17);
ret_user = 7;
break;
default:
adjust_insns(ctx->x + 177);
ret_user = 19;
break;
}
return 0;
}
SEC("fentry/" SYS_PREFIX "sys_nanosleep")
int simple_test_other_sec(struct pt_regs *ctx)
{
__u64 x = in_user;
if (bpf_get_current_pid_tgid() >> 32 != pid)
return 0;
switch (x) {
case 0:
adjust_insns(x + 1);
ret_user = 2;
break;
case 1:
adjust_insns(x + 7);
ret_user = 3;
break;
case 2:
adjust_insns(x + 9);
ret_user = 4;
break;
case 3:
adjust_insns(x + 11);
ret_user = 5;
break;
case 4:
adjust_insns(x + 17);
ret_user = 7;
break;
default:
adjust_insns(x + 177);
ret_user = 19;
break;
}
return 0;
}
SEC("syscall")
int two_switches(struct simple_ctx *ctx)
{
switch (ctx->x) {
case 0:
adjust_insns(ctx->x + 1);
ret_user = 2;
break;
case 1:
adjust_insns(ctx->x + 7);
ret_user = 3;
break;
case 2:
adjust_insns(ctx->x + 9);
ret_user = 4;
break;
case 3:
adjust_insns(ctx->x + 11);
ret_user = 5;
break;
case 4:
adjust_insns(ctx->x + 17);
ret_user = 7;
break;
default:
adjust_insns(ctx->x + 177);
ret_user = 19;
break;
}
switch (ctx->x + !!ret_user) {
case 1:
adjust_insns(ctx->x + 7);
ret_user = 103;
break;
case 2:
adjust_insns(ctx->x + 9);
ret_user = 104;
break;
case 3:
adjust_insns(ctx->x + 11);
ret_user = 107;
break;
case 4:
adjust_insns(ctx->x + 11);
ret_user = 205;
break;
case 5:
adjust_insns(ctx->x + 11);
ret_user = 115;
break;
default:
adjust_insns(ctx->x + 177);
ret_user = 1019;
break;
}
return 0;
}
SEC("syscall")
int big_jump_table(struct simple_ctx *ctx __attribute__((unused)))
{
const void *const jt[256] = {
[0 ... 255] = &&default_label,
[0] = &&l0,
[11] = &&l11,
[27] = &&l27,
[31] = &&l31,
};
goto *jt[ctx->x & 0xff];
l0:
adjust_insns(ctx->x + 1);
ret_user = 2;
return 0;
l11:
adjust_insns(ctx->x + 7);
ret_user = 3;
return 0;
l27:
adjust_insns(ctx->x + 9);
ret_user = 4;
return 0;
l31:
adjust_insns(ctx->x + 11);
ret_user = 5;
return 0;
default_label:
adjust_insns(ctx->x + 177);
ret_user = 19;
return 0;
}
SEC("syscall")
int one_jump_two_maps(struct simple_ctx *ctx __attribute__((unused)))
{
__label__ l1, l2, l3, l4;
void *jt1[2] = { &&l1, &&l2 };
void *jt2[2] = { &&l3, &&l4 };
unsigned int a = ctx->x % 2;
unsigned int b = (ctx->x / 2) % 2;
volatile int ret = 0;
if (!(a < 2 && b < 2))
return 19;
if (ctx->x % 2)
goto *jt1[a];
else
goto *jt2[b];
l1: ret += 1;
l2: ret += 3;
l3: ret += 5;
l4: ret += 7;
ret_user = ret;
return ret;
}
SEC("syscall")
int one_map_two_jumps(struct simple_ctx *ctx __attribute__((unused)))
{
__label__ l1, l2, l3;
void *jt[3] = { &&l1, &&l2, &&l3 };
unsigned int a = (ctx->x >> 2) & 1;
unsigned int b = (ctx->x >> 3) & 1;
volatile int ret = 0;
if (ctx->x % 2)
goto *jt[a];
if (ctx->x % 3)
goto *jt[a + b];
l1: ret += 3;
l2: ret += 5;
l3: ret += 7;
ret_user = ret;
return ret;
}
/* Just to introduce some non-zero offsets in .text */
static __noinline int f0(volatile struct simple_ctx *ctx __arg_ctx)
{
if (ctx)
return 1;
else
return 13;
}
SEC("syscall") int f1(struct simple_ctx *ctx)
{
ret_user = 0;
return f0(ctx);
}
static __noinline int __static_global(__u64 x)
{
switch (x) {
case 0:
adjust_insns(x + 1);
ret_user = 2;
break;
case 1:
adjust_insns(x + 7);
ret_user = 3;
break;
case 2:
adjust_insns(x + 9);
ret_user = 4;
break;
case 3:
adjust_insns(x + 11);
ret_user = 5;
break;
case 4:
adjust_insns(x + 17);
ret_user = 7;
break;
default:
adjust_insns(x + 177);
ret_user = 19;
break;
}
return 0;
}
SEC("syscall")
int use_static_global1(struct simple_ctx *ctx)
{
ret_user = 0;
return __static_global(ctx->x);
}
SEC("syscall")
int use_static_global2(struct simple_ctx *ctx)
{
ret_user = 0;
adjust_insns(ctx->x + 1);
return __static_global(ctx->x);
}
SEC("fentry/" SYS_PREFIX "sys_nanosleep")
int use_static_global_other_sec(void *ctx)
{
if (bpf_get_current_pid_tgid() >> 32 != pid)
return 0;
return __static_global(in_user);
}
__noinline int __nonstatic_global(__u64 x)
{
switch (x) {
case 0:
adjust_insns(x + 1);
ret_user = 2;
break;
case 1:
adjust_insns(x + 7);
ret_user = 3;
break;
case 2:
adjust_insns(x + 9);
ret_user = 4;
break;
case 3:
adjust_insns(x + 11);
ret_user = 5;
break;
case 4:
adjust_insns(x + 17);
ret_user = 7;
break;
default:
adjust_insns(x + 177);
ret_user = 19;
break;
}
return 0;
}
SEC("syscall")
int use_nonstatic_global1(struct simple_ctx *ctx)
{
ret_user = 0;
return __nonstatic_global(ctx->x);
}
SEC("syscall")
int use_nonstatic_global2(struct simple_ctx *ctx)
{
ret_user = 0;
adjust_insns(ctx->x + 1);
return __nonstatic_global(ctx->x);
}
SEC("fentry/" SYS_PREFIX "sys_nanosleep")
int use_nonstatic_global_other_sec(void *ctx)
{
if (bpf_get_current_pid_tgid() >> 32 != pid)
return 0;
return __nonstatic_global(in_user);
}
SEC("syscall")
int load_with_nonzero_offset(struct simple_ctx *ctx)
{
void *jj[] = { &&l1, &&l2, &&l3 };
/*
* This makes LLVM to generate a load from the jj map with an offset:
* r1 = 0x0 ll
* r1 = *(u64 *)(r1 + 0x10)
* gotox r1
*/
if (ctx->x == 2)
goto *jj[ctx->x];
ret_user = 1;
return 1;
l1:
/* never reached, but leave it here to outsmart LLVM */
ret_user = 0;
return 0;
l2:
/* never reached, but leave it here to outsmart LLVM */
ret_user = 3;
return 3;
l3:
ret_user = 5;
return 5;
}
#else /* __BPF_FEATURE_GOTOX */
#define SKIP_TEST(TEST_NAME) \
SEC("syscall") int TEST_NAME(void *ctx) \
{ \
return 0; \
}
SKIP_TEST(one_switch);
SKIP_TEST(one_switch_non_zero_sec_off);
SKIP_TEST(simple_test_other_sec);
SKIP_TEST(two_switches);
SKIP_TEST(big_jump_table);
SKIP_TEST(one_jump_two_maps);
SKIP_TEST(one_map_two_jumps);
SKIP_TEST(use_static_global1);
SKIP_TEST(use_static_global2);
SKIP_TEST(use_static_global_other_sec);
SKIP_TEST(use_nonstatic_global1);
SKIP_TEST(use_nonstatic_global2);
SKIP_TEST(use_nonstatic_global_other_sec);
SKIP_TEST(load_with_nonzero_offset);
#endif /* __BPF_FEATURE_GOTOX */
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_hashmap_full_update_bench.c
+40
View File
@@ -0,0 +1,40 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2022 Bytedance */
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
char _license[] SEC("license") = "GPL";
#define MAX_ENTRIES 1000
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__type(key, u32);
__type(value, u64);
__uint(max_entries, MAX_ENTRIES);
} hash_map_bench SEC(".maps");
u64 __attribute__((__aligned__(256))) percpu_time[256];
u64 nr_loops;
static int loop_update_callback(__u32 index, u32 *key)
{
u64 init_val = 1;
bpf_map_update_elem(&hash_map_bench, key, &init_val, BPF_ANY);
return 0;
}
SEC("fentry/" SYS_PREFIX "sys_getpgid")
int benchmark(void *ctx)
{
u32 cpu = bpf_get_smp_processor_id();
u32 key = cpu + MAX_ENTRIES;
u64 start_time = bpf_ktime_get_ns();
bpf_loop(nr_loops, loop_update_callback, &key, 0);
percpu_time[cpu & 255] = bpf_ktime_get_ns() - start_time;
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_hashmap_lookup.c
+63
View File
@@ -0,0 +1,63 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2023 Isovalent */
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
char _license[] SEC("license") = "GPL";
struct {
__uint(type, BPF_MAP_TYPE_HASH);
} hash_map_bench SEC(".maps");
/* The number of slots to store times */
#define NR_SLOTS 32
#define NR_CPUS 256
#define CPU_MASK (NR_CPUS-1)
/* Configured by userspace */
u64 nr_entries;
u64 nr_loops;
u32 __attribute__((__aligned__(8))) key[NR_CPUS];
/* Filled by us */
u64 __attribute__((__aligned__(256))) percpu_times_index[NR_CPUS];
u64 __attribute__((__aligned__(256))) percpu_times[NR_CPUS][NR_SLOTS];
static inline void patch_key(u32 i)
{
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
key[0] = i + 1;
#else
key[0] = __builtin_bswap32(i + 1);
#endif
/* the rest of key is random and is configured by userspace */
}
static int lookup_callback(__u32 index, u32 *unused)
{
patch_key(index);
return bpf_map_lookup_elem(&hash_map_bench, key) ? 0 : 1;
}
static int loop_lookup_callback(__u32 index, u32 *unused)
{
return bpf_loop(nr_entries, lookup_callback, NULL, 0) ? 0 : 1;
}
SEC("fentry/" SYS_PREFIX "sys_getpgid")
int benchmark(void *ctx)
{
u32 cpu = bpf_get_smp_processor_id();
u32 times_index;
u64 start_time;
times_index = percpu_times_index[cpu & CPU_MASK] % NR_SLOTS;
start_time = bpf_ktime_get_ns();
bpf_loop(nr_loops, loop_lookup_callback, NULL, 0);
percpu_times[cpu & CPU_MASK][times_index] = bpf_ktime_get_ns() - start_time;
percpu_times_index[cpu & CPU_MASK] += 1;
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_bpf_array_map.c
+53
View File
@@ -0,0 +1,53 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
char _license[] SEC("license") = "GPL";
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(max_entries, 3);
__type(key, __u32);
__type(value, __u64);
} arraymap1 SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 10);
__type(key, __u64);
__type(value, __u32);
} hashmap1 SEC(".maps");
__u32 key_sum = 0;
__u64 val_sum = 0;
SEC("iter/bpf_map_elem")
int dump_bpf_array_map(struct bpf_iter__bpf_map_elem *ctx)
{
__u32 *hmap_val, *key = ctx->key;
__u64 *val = ctx->value;
if (key == (void *)0 || val == (void *)0)
return 0;
bpf_seq_write(ctx->meta->seq, key, sizeof(__u32));
bpf_seq_write(ctx->meta->seq, val, sizeof(__u64));
key_sum += *key;
val_sum += *val;
/* workaround - It's necessary to do this convoluted (val, key)
* write into hashmap1, instead of simply doing
* bpf_map_update_elem(&hashmap1, val, key, BPF_ANY);
* because key has MEM_RDONLY flag and bpf_map_update elem expects
* types without this flag
*/
bpf_map_update_elem(&hashmap1, val, val, BPF_ANY);
hmap_val = bpf_map_lookup_elem(&hashmap1, val);
if (hmap_val)
*hmap_val = *key;
*val = *key;
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_bpf_hash_map.c
+123
View File
@@ -0,0 +1,123 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
struct key_t {
int a;
int b;
int c;
};
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 3);
__type(key, struct key_t);
__type(value, __u64);
} hashmap1 SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 3);
__type(key, __u64);
__type(value, __u64);
} hashmap2 SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 3);
__type(key, struct key_t);
__type(value, __u32);
} hashmap3 SEC(".maps");
/* will set before prog run */
bool in_test_mode = 0;
/* will collect results during prog run */
__u32 key_sum_a = 0, key_sum_b = 0, key_sum_c = 0;
__u64 val_sum = 0;
SEC("iter/bpf_map_elem")
int dump_bpf_hash_map(struct bpf_iter__bpf_map_elem *ctx)
{
struct seq_file *seq = ctx->meta->seq;
__u32 seq_num = ctx->meta->seq_num;
struct bpf_map *map = ctx->map;
struct key_t *key = ctx->key;
struct key_t tmp_key;
__u64 *val = ctx->value;
__u64 tmp_val = 0;
int ret;
if (in_test_mode) {
/* test mode is used by selftests to
* test functionality of bpf_hash_map iter.
*
* the above hashmap1 will have correct size
* and will be accepted, hashmap2 and hashmap3
* should be rejected due to smaller key/value
* size.
*/
if (key == (void *)0 || val == (void *)0)
return 0;
/* update the value and then delete the <key, value> pair.
* it should not impact the existing 'val' which is still
* accessible under rcu.
*/
__builtin_memcpy(&tmp_key, key, sizeof(struct key_t));
ret = bpf_map_update_elem(&hashmap1, &tmp_key, &tmp_val, 0);
if (ret)
return 0;
ret = bpf_map_delete_elem(&hashmap1, &tmp_key);
if (ret)
return 0;
key_sum_a += key->a;
key_sum_b += key->b;
key_sum_c += key->c;
val_sum += *val;
return 0;
}
/* non-test mode, the map is prepared with the
* below bpftool command sequence:
* bpftool map create /sys/fs/bpf/m1 type hash \
* key 12 value 8 entries 3 name map1
* bpftool map update id 77 key 0 0 0 1 0 0 0 0 0 0 0 1 \
* value 0 0 0 1 0 0 0 1
* bpftool map update id 77 key 0 0 0 1 0 0 0 0 0 0 0 2 \
* value 0 0 0 1 0 0 0 2
* The bpftool iter command line:
* bpftool iter pin ./bpf_iter_bpf_hash_map.o /sys/fs/bpf/p1 \
* map id 77
* The below output will be:
* map dump starts
* 77: (1000000 0 2000000) (200000001000000)
* 77: (1000000 0 1000000) (100000001000000)
* map dump ends
*/
if (seq_num == 0)
BPF_SEQ_PRINTF(seq, "map dump starts\n");
if (key == (void *)0 || val == (void *)0) {
BPF_SEQ_PRINTF(seq, "map dump ends\n");
return 0;
}
BPF_SEQ_PRINTF(seq, "%d: (%x %d %x) (%llx)\n", map->id,
key->a, key->b, key->c, *val);
return 0;
}
SEC("iter.s/bpf_map_elem")
int sleepable_dummy_dump(struct bpf_iter__bpf_map_elem *ctx)
{
if (ctx->meta->seq_num == 0)
BPF_SEQ_PRINTF(ctx->meta->seq, "map dump starts\n");
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_bpf_link.c
+21
View File
@@ -0,0 +1,21 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2022 Red Hat, Inc. */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
SEC("iter/bpf_link")
int dump_bpf_link(struct bpf_iter__bpf_link *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct bpf_link *link = ctx->link;
int link_id;
if (!link)
return 0;
link_id = link->id;
bpf_seq_write(seq, &link_id, sizeof(link_id));
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_bpf_map.c
+27
View File
@@ -0,0 +1,27 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
SEC("iter/bpf_map")
int dump_bpf_map(struct bpf_iter__bpf_map *ctx)
{
struct seq_file *seq = ctx->meta->seq;
__u64 seq_num = ctx->meta->seq_num;
struct bpf_map *map = ctx->map;
if (map == (void *)0) {
BPF_SEQ_PRINTF(seq, " %%%%%% END %%%%%%\n");
return 0;
}
if (seq_num == 0)
BPF_SEQ_PRINTF(seq, " id refcnt usercnt locked_vm\n");
BPF_SEQ_PRINTF(seq, "%8u %8ld %8ld %10lu\n", map->id, map->refcnt.counter,
map->usercnt.counter,
0LLU);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_bpf_percpu_array_map.c
+40
View File
@@ -0,0 +1,40 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
char _license[] SEC("license") = "GPL";
struct {
__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
__uint(max_entries, 3);
__type(key, __u32);
__type(value, __u32);
} arraymap1 SEC(".maps");
/* will set before prog run */
volatile const __u32 num_cpus = 0;
__u32 key_sum = 0, val_sum = 0;
SEC("iter/bpf_map_elem")
int dump_bpf_percpu_array_map(struct bpf_iter__bpf_map_elem *ctx)
{
__u32 *key = ctx->key;
void *pptr = ctx->value;
__u32 step;
int i;
if (key == (void *)0 || pptr == (void *)0)
return 0;
key_sum += *key;
step = 8;
for (i = 0; i < num_cpus; i++) {
val_sum += *(__u32 *)pptr;
pptr += step;
}
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_bpf_percpu_hash_map.c
+50
View File
@@ -0,0 +1,50 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
char _license[] SEC("license") = "GPL";
struct key_t {
int a;
int b;
int c;
};
struct {
__uint(type, BPF_MAP_TYPE_PERCPU_HASH);
__uint(max_entries, 3);
__type(key, struct key_t);
__type(value, __u32);
} hashmap1 SEC(".maps");
/* will set before prog run */
volatile const __s32 num_cpus = 0;
/* will collect results during prog run */
__u32 key_sum_a = 0, key_sum_b = 0, key_sum_c = 0;
__u32 val_sum = 0;
SEC("iter/bpf_map_elem")
int dump_bpf_percpu_hash_map(struct bpf_iter__bpf_map_elem *ctx)
{
struct key_t *key = ctx->key;
void *pptr = ctx->value;
__u32 step;
int i;
if (key == (void *)0 || pptr == (void *)0)
return 0;
key_sum_a += key->a;
key_sum_b += key->b;
key_sum_c += key->c;
step = 8;
for (i = 0; i < num_cpus; i++) {
val_sum += *(__u32 *)pptr;
pptr += step;
}
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_bpf_rhash_map.c
+34
View File
@@ -0,0 +1,34 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
struct {
__uint(type, BPF_MAP_TYPE_RHASH);
__uint(map_flags, BPF_F_NO_PREALLOC);
__uint(max_entries, 64);
__type(key, __u32);
__type(value, __u64);
} rhashmap SEC(".maps");
__u32 key_sum = 0;
__u64 val_sum = 0;
__u32 elem_count = 0;
__u32 err = 0;
SEC("iter/bpf_map_elem")
int dump_bpf_rhash_map(struct bpf_iter__bpf_map_elem *ctx)
{
__u32 *key = ctx->key;
__u64 *val = ctx->value;
if (!key || !val)
return 0;
key_sum += *key;
val_sum += *val;
elem_count++;
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_bpf_sk_storage_helpers.c
+65
View File
@@ -0,0 +1,65 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Google LLC. */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
char _license[] SEC("license") = "GPL";
struct {
__uint(type, BPF_MAP_TYPE_SK_STORAGE);
__uint(map_flags, BPF_F_NO_PREALLOC);
__type(key, int);
__type(value, int);
} sk_stg_map SEC(".maps");
SEC("iter/bpf_sk_storage_map")
int delete_bpf_sk_storage_map(struct bpf_iter__bpf_sk_storage_map *ctx)
{
if (ctx->sk)
bpf_sk_storage_delete(&sk_stg_map, ctx->sk);
return 0;
}
SEC("iter/task_file")
int fill_socket_owner(struct bpf_iter__task_file *ctx)
{
struct task_struct *task = ctx->task;
struct file *file = ctx->file;
struct socket *sock;
int *sock_tgid;
if (!task || !file)
return 0;
sock = bpf_sock_from_file(file);
if (!sock)
return 0;
sock_tgid = bpf_sk_storage_get(&sk_stg_map, sock->sk, 0, 0);
if (!sock_tgid)
return 0;
*sock_tgid = task->tgid;
return 0;
}
SEC("iter/tcp")
int negate_socket_local_storage(struct bpf_iter__tcp *ctx)
{
struct sock_common *sk_common = ctx->sk_common;
int *sock_tgid;
if (!sk_common)
return 0;
sock_tgid = bpf_sk_storage_get(&sk_stg_map, sk_common, 0, 0);
if (!sock_tgid)
return 0;
*sock_tgid = -*sock_tgid;
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_bpf_sk_storage_map.c
+52
View File
@@ -0,0 +1,52 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
char _license[] SEC("license") = "GPL";
struct {
__uint(type, BPF_MAP_TYPE_SK_STORAGE);
__uint(map_flags, BPF_F_NO_PREALLOC);
__type(key, int);
__type(value, int);
} sk_stg_map SEC(".maps");
__u32 val_sum = 0;
__u32 ipv6_sk_count = 0;
__u32 to_add_val = 0;
SEC("iter/bpf_sk_storage_map")
int rw_bpf_sk_storage_map(struct bpf_iter__bpf_sk_storage_map *ctx)
{
struct sock *sk = ctx->sk;
__u32 *val = ctx->value;
if (sk == NULL || val == NULL)
return 0;
if (sk->sk_family == AF_INET6)
ipv6_sk_count++;
val_sum += *val;
*val += to_add_val;
return 0;
}
SEC("iter/bpf_sk_storage_map")
int oob_write_bpf_sk_storage_map(struct bpf_iter__bpf_sk_storage_map *ctx)
{
struct sock *sk = ctx->sk;
__u32 *val = ctx->value;
if (sk == NULL || val == NULL)
return 0;
*(val + 1) = 0xdeadbeef;
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_ipv6_route.c
+56
View File
@@ -0,0 +1,56 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
extern bool CONFIG_IPV6_SUBTREES __kconfig __weak;
SEC("iter/ipv6_route")
int dump_ipv6_route(struct bpf_iter__ipv6_route *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct fib6_info *rt = ctx->rt;
const struct net_device *dev;
struct fib6_nh *fib6_nh;
unsigned int flags;
struct nexthop *nh;
if (rt == (void *)0)
return 0;
fib6_nh = &rt->fib6_nh[0];
flags = rt->fib6_flags;
/* FIXME: nexthop_is_multipath is not handled here. */
nh = rt->nh;
if (rt->nh)
fib6_nh = &nh->nh_info->fib6_nh;
BPF_SEQ_PRINTF(seq, "%pi6 %02x ", &rt->fib6_dst.addr, rt->fib6_dst.plen);
if (CONFIG_IPV6_SUBTREES)
BPF_SEQ_PRINTF(seq, "%pi6 %02x ", &rt->fib6_src.addr,
rt->fib6_src.plen);
else
BPF_SEQ_PRINTF(seq, "00000000000000000000000000000000 00 ");
if (fib6_nh->fib_nh_gw_family) {
flags |= RTF_GATEWAY;
BPF_SEQ_PRINTF(seq, "%pi6 ", &fib6_nh->fib_nh_gw6);
} else {
BPF_SEQ_PRINTF(seq, "00000000000000000000000000000000 ");
}
dev = fib6_nh->fib_nh_dev;
if (dev)
BPF_SEQ_PRINTF(seq, "%08x %08x %08x %08x %8s\n", rt->fib6_metric,
rt->fib6_ref.refs.counter, 0, flags, dev->name);
else
BPF_SEQ_PRINTF(seq, "%08x %08x %08x %08x\n", rt->fib6_metric,
rt->fib6_ref.refs.counter, 0, flags);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_ksym.c
+71
View File
@@ -0,0 +1,71 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2022, Oracle and/or its affiliates. */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
unsigned long last_sym_value = 0;
static inline char to_lower(char c)
{
if (c >= 'A' && c <= 'Z')
c += ('a' - 'A');
return c;
}
static inline char to_upper(char c)
{
if (c >= 'a' && c <= 'z')
c -= ('a' - 'A');
return c;
}
/* Dump symbols with max size; the latter is calculated by caching symbol N value
* and when iterating on symbol N+1, we can print max size of symbol N via
* address of N+1 - address of N.
*/
SEC("iter/ksym")
int dump_ksym(struct bpf_iter__ksym *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct kallsym_iter *iter = ctx->ksym;
__u32 seq_num = ctx->meta->seq_num;
unsigned long value;
char type;
if (!iter)
return 0;
if (seq_num == 0) {
BPF_SEQ_PRINTF(seq, "ADDR TYPE NAME MODULE_NAME KIND MAX_SIZE\n");
return 0;
}
if (last_sym_value)
BPF_SEQ_PRINTF(seq, "0x%x\n", iter->value - last_sym_value);
else
BPF_SEQ_PRINTF(seq, "\n");
value = iter->show_value ? iter->value : 0;
last_sym_value = value;
type = iter->type;
if (iter->module_name[0]) {
type = iter->exported ? to_upper(type) : to_lower(type);
BPF_SEQ_PRINTF(seq, "0x%llx %c %s [ %s ] ",
value, type, iter->name, iter->module_name);
} else {
BPF_SEQ_PRINTF(seq, "0x%llx %c %s ", value, type, iter->name);
}
if (!iter->pos_mod_end || iter->pos_mod_end > iter->pos)
BPF_SEQ_PRINTF(seq, "MOD ");
else if (!iter->pos_ftrace_mod_end || iter->pos_ftrace_mod_end > iter->pos)
BPF_SEQ_PRINTF(seq, "FTRACE_MOD ");
else if (!iter->pos_bpf_end || iter->pos_bpf_end > iter->pos)
BPF_SEQ_PRINTF(seq, "BPF ");
else
BPF_SEQ_PRINTF(seq, "KPROBE ");
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_map_elem.c
+22
View File
@@ -0,0 +1,22 @@
// SPDX-License-Identifier: GPL-2.0-only
#include "vmlinux.h"
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
__u32 value_sum = 0;
SEC("iter/bpf_map_elem")
int dump_bpf_map_values(struct bpf_iter__bpf_map_elem *ctx)
{
__u32 value = 0;
if (ctx->value == (void *)0)
return 0;
bpf_probe_read_kernel(&value, sizeof(value), ctx->value);
value_sum += value;
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_netlink.c
+63
View File
@@ -0,0 +1,63 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
static __attribute__((noinline)) struct inode *SOCK_INODE(struct socket *socket)
{
return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
}
SEC("iter/netlink")
int dump_netlink(struct bpf_iter__netlink *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct netlink_sock *nlk = ctx->sk;
unsigned long group, ino;
struct inode *inode;
struct socket *sk;
struct sock *s;
if (nlk == (void *)0)
return 0;
if (ctx->meta->seq_num == 0)
BPF_SEQ_PRINTF(seq, "sk Eth Pid Groups "
"Rmem Wmem Dump Locks Drops "
"Inode\n");
s = &nlk->sk;
BPF_SEQ_PRINTF(seq, "%pK %-3d ", s, s->sk_protocol);
if (!nlk->groups) {
group = 0;
} else {
/* FIXME: temporary use bpf_probe_read_kernel here, needs
* verifier support to do direct access.
*/
bpf_probe_read_kernel(&group, sizeof(group), &nlk->groups[0]);
}
BPF_SEQ_PRINTF(seq, "%-10u %08x %-8d %-8d %-5d %-8d ",
nlk->portid, (u32)group,
s->sk_rmem_alloc.counter,
s->sk_wmem_alloc.refs.counter - 1,
nlk->cb_running, s->sk_refcnt.refs.counter);
sk = s->sk_socket;
if (!sk) {
ino = 0;
} else {
/* FIXME: container_of inside SOCK_INODE has a forced
* type conversion, and direct access cannot be used
* with current verifier.
*/
inode = SOCK_INODE(sk);
bpf_probe_read_kernel(&ino, sizeof(ino), &inode->i_ino);
}
BPF_SEQ_PRINTF(seq, "%-8u %-8lu\n", s->sk_drops.counter, ino);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_setsockopt.c
+58
View File
@@ -0,0 +1,58 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2021 Facebook */
#include <vmlinux.h>
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>
#define bpf_tcp_sk(skc) ({ \
struct sock_common *_skc = skc; \
sk = NULL; \
tp = NULL; \
if (_skc) { \
tp = bpf_skc_to_tcp_sock(_skc); \
sk = (struct sock *)tp; \
} \
tp; \
})
unsigned short reuse_listen_hport = 0;
unsigned short listen_hport = 0;
const char cubic_cc[] = "bpf_cubic";
char dctcp_cc[TCP_CA_NAME_MAX] = "bpf_dctcp";
bool random_retry = false;
SEC("iter/tcp")
int change_tcp_cc(struct bpf_iter__tcp *ctx)
{
char cur_cc[TCP_CA_NAME_MAX];
struct tcp_sock *tp;
struct sock *sk;
if (!bpf_tcp_sk(ctx->sk_common))
return 0;
if (sk->sk_family != AF_INET6 ||
(sk->sk_state != TCP_LISTEN &&
sk->sk_state != TCP_ESTABLISHED) ||
(sk->sk_num != reuse_listen_hport &&
sk->sk_num != listen_hport &&
bpf_ntohs(sk->sk_dport) != listen_hport))
return 0;
if (bpf_getsockopt(tp, SOL_TCP, TCP_CONGESTION,
cur_cc, sizeof(cur_cc)))
return 0;
if (bpf_strncmp(cur_cc, TCP_CA_NAME_MAX, cubic_cc))
return 0;
if (random_retry && bpf_get_prandom_u32() % 4 == 1)
return 1;
bpf_setsockopt(tp, SOL_TCP, TCP_CONGESTION, dctcp_cc, sizeof(dctcp_cc));
return 0;
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_setsockopt_unix.c
+60
View File
@@ -0,0 +1,60 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright Amazon.com Inc. or its affiliates. */
#include <vmlinux.h>
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
#include <limits.h>
#define AUTOBIND_LEN 6
char sun_path[AUTOBIND_LEN];
#define NR_CASES 5
int sndbuf_setsockopt[NR_CASES] = {-1, 0, 8192, INT_MAX / 2, INT_MAX};
int sndbuf_getsockopt[NR_CASES] = {-1, -1, -1, -1, -1};
int sndbuf_getsockopt_expected[NR_CASES];
static inline int cmpname(struct unix_sock *unix_sk)
{
int i;
for (i = 0; i < AUTOBIND_LEN; i++) {
if (unix_sk->addr->name->sun_path[i] != sun_path[i])
return -1;
}
return 0;
}
SEC("iter/unix")
int change_sndbuf(struct bpf_iter__unix *ctx)
{
struct unix_sock *unix_sk = ctx->unix_sk;
int i, err;
if (!unix_sk || !unix_sk->addr)
return 0;
if (unix_sk->addr->name->sun_path[0])
return 0;
if (cmpname(unix_sk))
return 0;
for (i = 0; i < NR_CASES; i++) {
err = bpf_setsockopt(unix_sk, SOL_SOCKET, SO_SNDBUF,
&sndbuf_setsockopt[i],
sizeof(sndbuf_setsockopt[i]));
if (err)
break;
err = bpf_getsockopt(unix_sk, SOL_SOCKET, SO_SNDBUF,
&sndbuf_getsockopt[i],
sizeof(sndbuf_getsockopt[i]));
if (err)
break;
}
return 0;
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c
+59
View File
@@ -0,0 +1,59 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Cloudflare */
#include <vmlinux.h>
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <errno.h>
char _license[] SEC("license") = "GPL";
struct {
__uint(type, BPF_MAP_TYPE_SOCKMAP);
__uint(max_entries, 64);
__type(key, __u32);
__type(value, __u64);
} sockmap SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_SOCKHASH);
__uint(max_entries, 64);
__type(key, __u32);
__type(value, __u64);
} sockhash SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_SOCKHASH);
__uint(max_entries, 64);
__type(key, __u32);
__type(value, __u64);
} dst SEC(".maps");
__u32 elems = 0;
__u32 socks = 0;
SEC("iter/sockmap")
int copy(struct bpf_iter__sockmap *ctx)
{
struct sock *sk = ctx->sk;
__u32 tmp, *key = ctx->key;
int ret;
if (!key)
return 0;
elems++;
/* We need a temporary buffer on the stack, since the verifier doesn't
* let us use the pointer from the context as an argument to the helper.
*/
tmp = *key;
if (sk) {
socks++;
return bpf_map_update_elem(&dst, &tmp, sk, 0) != 0;
}
ret = bpf_map_delete_elem(&dst, &tmp);
return ret && ret != -ENOENT;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_task_btf.c
+49
View File
@@ -0,0 +1,49 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020, Oracle and/or its affiliates. */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
#include <errno.h>
char _license[] SEC("license") = "GPL";
long tasks = 0;
long seq_err = 0;
bool skip = false;
SEC("iter/task")
int dump_task_struct(struct bpf_iter__task *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct task_struct *task = ctx->task;
static struct btf_ptr ptr = { };
long ret;
#if __has_builtin(__builtin_btf_type_id)
ptr.type_id = bpf_core_type_id_kernel(struct task_struct);
ptr.ptr = task;
if (ctx->meta->seq_num == 0)
BPF_SEQ_PRINTF(seq, "Raw BTF task\n");
ret = bpf_seq_printf_btf(seq, &ptr, sizeof(ptr), 0);
switch (ret) {
case 0:
tasks++;
break;
case -ERANGE:
/* NULL task or task->fs, don't count it as an error. */
break;
case -E2BIG:
return 1;
default:
seq_err = ret;
break;
}
#else
skip = true;
#endif
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c
+40
View File
@@ -0,0 +1,40 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
int count = 0;
int tgid = 0;
int last_tgid = 0;
int unique_tgid_count = 0;
SEC("iter/task_file")
int dump_task_file(struct bpf_iter__task_file *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct task_struct *task = ctx->task;
struct file *file = ctx->file;
__u32 fd = ctx->fd;
if (task == (void *)0 || file == (void *)0)
return 0;
if (ctx->meta->seq_num == 0) {
count = 0;
BPF_SEQ_PRINTF(seq, " tgid gid fd file\n");
}
if (tgid == task->tgid && task->tgid != task->pid)
count++;
if (last_tgid != task->tgid) {
last_tgid = task->tgid;
unique_tgid_count++;
}
BPF_SEQ_PRINTF(seq, "%8d %8d %8d %lx\n", task->tgid, task->pid, fd,
(long)file->f_op);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_task_stack.c
+68
View File
@@ -0,0 +1,68 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
#define MAX_STACK_TRACE_DEPTH 64
unsigned long entries[MAX_STACK_TRACE_DEPTH] = {};
#define SIZE_OF_ULONG (sizeof(unsigned long))
SEC("iter/task")
int dump_task_stack(struct bpf_iter__task *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct task_struct *task = ctx->task;
long i, retlen;
if (task == (void *)0)
return 0;
retlen = bpf_get_task_stack(task, entries,
MAX_STACK_TRACE_DEPTH * SIZE_OF_ULONG, 0);
if (retlen < 0)
return 0;
BPF_SEQ_PRINTF(seq, "pid: %8u num_entries: %8u\n", task->pid,
retlen / SIZE_OF_ULONG);
for (i = 0; i < MAX_STACK_TRACE_DEPTH; i++) {
if (retlen > i * SIZE_OF_ULONG)
BPF_SEQ_PRINTF(seq, "[<0>] %pB\n", (void *)entries[i]);
}
BPF_SEQ_PRINTF(seq, "\n");
return 0;
}
int num_user_stacks = 0;
SEC("iter/task")
int get_task_user_stacks(struct bpf_iter__task *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct task_struct *task = ctx->task;
uint64_t buf_sz = 0;
int64_t res;
if (task == (void *)0)
return 0;
res = bpf_get_task_stack(task, entries,
MAX_STACK_TRACE_DEPTH * SIZE_OF_ULONG, BPF_F_USER_STACK);
if (res <= 0)
return 0;
/* Only one task, the current one, should succeed */
++num_user_stacks;
buf_sz += res;
/* If the verifier doesn't refine bpf_get_task_stack res, and instead
* assumes res is entirely unknown, this program will fail to load as
* the verifier will believe that max buf_sz value allows reading
* past the end of entries in bpf_seq_write call
*/
bpf_seq_write(seq, &entries, buf_sz);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c
+62
View File
@@ -0,0 +1,62 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
/* Copied from mm.h */
#define VM_READ 0x00000001
#define VM_WRITE 0x00000002
#define VM_EXEC 0x00000004
#define VM_MAYSHARE 0x00000080
/* Copied from kdev_t.h */
#define MINORBITS 20
#define MINORMASK ((1U << MINORBITS) - 1)
#define MAJOR(dev) ((unsigned int) ((dev) >> MINORBITS))
#define MINOR(dev) ((unsigned int) ((dev) & MINORMASK))
#define D_PATH_BUF_SIZE 1024
char d_path_buf[D_PATH_BUF_SIZE] = {};
__u32 pid = 0;
__u32 one_task = 0;
__u32 one_task_error = 0;
SEC("iter/task_vma") int proc_maps(struct bpf_iter__task_vma *ctx)
{
struct vm_area_struct *vma = ctx->vma;
struct seq_file *seq = ctx->meta->seq;
struct task_struct *task = ctx->task;
struct file *file;
char perm_str[] = "----";
if (task == (void *)0 || vma == (void *)0)
return 0;
file = vma->vm_file;
if (task->tgid != (pid_t)pid) {
if (one_task)
one_task_error = 1;
return 0;
}
perm_str[0] = (vma->vm_flags & VM_READ) ? 'r' : '-';
perm_str[1] = (vma->vm_flags & VM_WRITE) ? 'w' : '-';
perm_str[2] = (vma->vm_flags & VM_EXEC) ? 'x' : '-';
perm_str[3] = (vma->vm_flags & VM_MAYSHARE) ? 's' : 'p';
BPF_SEQ_PRINTF(seq, "%08llx-%08llx %s ", vma->vm_start, vma->vm_end, perm_str);
if (file) {
__u32 dev = file->f_inode->i_sb->s_dev;
bpf_d_path(&file->f_path, d_path_buf, D_PATH_BUF_SIZE);
BPF_SEQ_PRINTF(seq, "%08llx ", vma->vm_pgoff << 12);
BPF_SEQ_PRINTF(seq, "%02x:%02x %llu", MAJOR(dev), MINOR(dev),
file->f_inode->i_ino);
BPF_SEQ_PRINTF(seq, "\t%s\n", d_path_buf);
} else {
BPF_SEQ_PRINTF(seq, "%08llx 00:00 0\n", 0ULL);
}
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_tasks.c
+198
View File
@@ -0,0 +1,198 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
char _license[] SEC("license") = "GPL";
uint32_t tid = 0;
int num_unknown_tid = 0;
int num_known_tid = 0;
void *user_ptr = 0;
void *user_ptr_long = 0;
uint32_t pid = 0;
static char big_str1[5000];
static char big_str2[5005];
static char big_str3[4996];
SEC("iter/task")
int dump_task(struct bpf_iter__task *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct task_struct *task = ctx->task;
static char info[] = " === END ===";
if (task == (void *)0) {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
if (task->pid != (pid_t)tid)
num_unknown_tid++;
else
num_known_tid++;
if (ctx->meta->seq_num == 0)
BPF_SEQ_PRINTF(seq, " tgid gid\n");
BPF_SEQ_PRINTF(seq, "%8d %8d\n", task->tgid, task->pid);
return 0;
}
int num_expected_failure_copy_from_user_task = 0;
int num_expected_failure_copy_from_user_task_str = 0;
int num_success_copy_from_user_task = 0;
int num_success_copy_from_user_task_str = 0;
SEC("iter.s/task")
int dump_task_sleepable(struct bpf_iter__task *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct task_struct *task = ctx->task;
static const char info[] = " === END ===";
struct pt_regs *regs;
char task_str1[10] = "aaaaaaaaaa";
char task_str2[10], task_str3[10];
char task_str4[20] = "aaaaaaaaaaaaaaaaaaaa";
void *ptr;
uint32_t user_data = 0;
int ret;
if (task == (void *)0) {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
/* Read an invalid pointer and ensure we get an error */
ptr = NULL;
ret = bpf_copy_from_user_task(&user_data, sizeof(uint32_t), ptr, task, 0);
if (ret) {
++num_expected_failure_copy_from_user_task;
} else {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
/* Try to read the contents of the task's instruction pointer from the
* remote task's address space.
*/
regs = (struct pt_regs *)bpf_task_pt_regs(task);
if (regs == (void *)0) {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
ptr = (void *)PT_REGS_IP(regs);
ret = bpf_copy_from_user_task(&user_data, sizeof(uint32_t), ptr, task, 0);
if (ret) {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
++num_success_copy_from_user_task;
/* Read an invalid pointer and ensure we get an error */
ptr = NULL;
ret = bpf_copy_from_user_task_str((char *)task_str1, sizeof(task_str1), ptr, task, 0);
if (ret >= 0 || task_str1[9] != 'a' || task_str1[0] != '\0') {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
/* Read an invalid pointer and ensure we get error with pad zeros flag */
ptr = NULL;
ret = bpf_copy_from_user_task_str((char *)task_str1, sizeof(task_str1),
ptr, task, BPF_F_PAD_ZEROS);
if (ret >= 0 || task_str1[9] != '\0' || task_str1[0] != '\0') {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
++num_expected_failure_copy_from_user_task_str;
/* Same length as the string */
ret = bpf_copy_from_user_task_str((char *)task_str2, 10, user_ptr, task, 0);
/* only need to do the task pid check once */
if (bpf_strncmp(task_str2, 10, "test_data\0") != 0 || ret != 10 || task->tgid != pid) {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
/* Shorter length than the string */
ret = bpf_copy_from_user_task_str((char *)task_str3, 2, user_ptr, task, 0);
if (bpf_strncmp(task_str3, 2, "t\0") != 0 || ret != 2) {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
/* Longer length than the string */
ret = bpf_copy_from_user_task_str((char *)task_str4, 20, user_ptr, task, 0);
if (bpf_strncmp(task_str4, 10, "test_data\0") != 0 || ret != 10
|| task_str4[sizeof(task_str4) - 1] != 'a') {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
/* Longer length than the string with pad zeros flag */
ret = bpf_copy_from_user_task_str((char *)task_str4, 20, user_ptr, task, BPF_F_PAD_ZEROS);
if (bpf_strncmp(task_str4, 10, "test_data\0") != 0 || ret != 10
|| task_str4[sizeof(task_str4) - 1] != '\0') {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
/* Longer length than the string past a page boundary */
ret = bpf_copy_from_user_task_str(big_str1, 5000, user_ptr, task, 0);
if (bpf_strncmp(big_str1, 10, "test_data\0") != 0 || ret != 10) {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
/* String that crosses a page boundary */
ret = bpf_copy_from_user_task_str(big_str1, 5000, user_ptr_long, task, BPF_F_PAD_ZEROS);
if (bpf_strncmp(big_str1, 4, "baba") != 0 || ret != 5000
|| bpf_strncmp(big_str1 + 4996, 4, "bab\0") != 0) {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
for (int i = 0; i < 4999; ++i) {
if (i % 2 == 0) {
if (big_str1[i] != 'b') {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
} else {
if (big_str1[i] != 'a') {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
}
}
/* Longer length than the string that crosses a page boundary */
ret = bpf_copy_from_user_task_str(big_str2, 5005, user_ptr_long, task, BPF_F_PAD_ZEROS);
if (bpf_strncmp(big_str2, 4, "baba") != 0 || ret != 5000
|| bpf_strncmp(big_str2 + 4996, 5, "bab\0\0") != 0) {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
/* Shorter length than the string that crosses a page boundary */
ret = bpf_copy_from_user_task_str(big_str3, 4996, user_ptr_long, task, 0);
if (bpf_strncmp(big_str3, 4, "baba") != 0 || ret != 4996
|| bpf_strncmp(big_str3 + 4992, 4, "bab\0") != 0) {
BPF_SEQ_PRINTF(seq, "%s\n", info);
return 0;
}
++num_success_copy_from_user_task_str;
if (ctx->meta->seq_num == 0)
BPF_SEQ_PRINTF(seq, " tgid gid data\n");
BPF_SEQ_PRINTF(seq, "%8d %8d %8d\n", task->tgid, task->pid, user_data);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c
+233
View File
@@ -0,0 +1,233 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>
char _license[] SEC("license") = "GPL";
static int hlist_unhashed_lockless(const struct hlist_node *h)
{
return !(h->pprev);
}
static int timer_pending(const struct timer_list * timer)
{
return !hlist_unhashed_lockless(&timer->entry);
}
extern unsigned CONFIG_HZ __kconfig;
#define USER_HZ 100
#define NSEC_PER_SEC 1000000000ULL
static clock_t jiffies_to_clock_t(unsigned long x)
{
/* The implementation here tailored to a particular
* setting of USER_HZ.
*/
u64 tick_nsec = (NSEC_PER_SEC + CONFIG_HZ/2) / CONFIG_HZ;
u64 user_hz_nsec = NSEC_PER_SEC / USER_HZ;
if ((tick_nsec % user_hz_nsec) == 0) {
if (CONFIG_HZ < USER_HZ)
return x * (USER_HZ / CONFIG_HZ);
else
return x / (CONFIG_HZ / USER_HZ);
}
return x * tick_nsec/user_hz_nsec;
}
static clock_t jiffies_delta_to_clock_t(long delta)
{
if (delta <= 0)
return 0;
return jiffies_to_clock_t(delta);
}
static long sock_i_ino(const struct sock *sk)
{
const struct socket *sk_socket = sk->sk_socket;
const struct inode *inode;
unsigned long ino;
if (!sk_socket)
return 0;
inode = &container_of(sk_socket, struct socket_alloc, socket)->vfs_inode;
bpf_probe_read_kernel(&ino, sizeof(ino), &inode->i_ino);
return ino;
}
static bool
inet_csk_in_pingpong_mode(const struct inet_connection_sock *icsk)
{
return icsk->icsk_ack.pingpong >= TCP_PINGPONG_THRESH;
}
static bool tcp_in_initial_slowstart(const struct tcp_sock *tcp)
{
return tcp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
}
static int dump_tcp_sock(struct seq_file *seq, struct tcp_sock *tp,
uid_t uid, __u32 seq_num)
{
const struct inet_connection_sock *icsk;
const struct fastopen_queue *fastopenq;
const struct inet_sock *inet;
unsigned long timer_expires;
const struct sock *sp;
__u16 destp, srcp;
__be32 dest, src;
int timer_active;
int rx_queue;
int state;
icsk = &tp->inet_conn;
inet = &icsk->icsk_inet;
sp = &inet->sk;
fastopenq = &icsk->icsk_accept_queue.fastopenq;
dest = inet->inet_daddr;
src = inet->inet_rcv_saddr;
destp = bpf_ntohs(inet->inet_dport);
srcp = bpf_ntohs(inet->inet_sport);
if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
timer_active = 1;
timer_expires = sp->tcp_retransmit_timer.expires;
} else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
timer_active = 4;
timer_expires = sp->tcp_retransmit_timer.expires;
} else if (timer_pending(&icsk->icsk_keepalive_timer)) {
timer_active = 2;
timer_expires = icsk->icsk_keepalive_timer.expires;
} else {
timer_active = 0;
timer_expires = bpf_jiffies64();
}
state = sp->sk_state;
if (state == TCP_LISTEN) {
rx_queue = sp->sk_ack_backlog;
} else {
rx_queue = tp->rcv_nxt - tp->copied_seq;
if (rx_queue < 0)
rx_queue = 0;
}
BPF_SEQ_PRINTF(seq, "%4d: %08X:%04X %08X:%04X ",
seq_num, src, srcp, dest, destp);
BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d ",
state,
tp->write_seq - tp->snd_una, rx_queue,
timer_active,
jiffies_delta_to_clock_t(timer_expires - bpf_jiffies64()),
icsk->icsk_retransmits, uid,
icsk->icsk_probes_out,
sock_i_ino(sp),
sp->sk_refcnt.refs.counter);
BPF_SEQ_PRINTF(seq, "%pK %lu %lu %u %u %d\n",
tp,
jiffies_to_clock_t(icsk->icsk_rto),
jiffies_to_clock_t(icsk->icsk_ack.ato),
(icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(icsk),
tp->snd_cwnd,
state == TCP_LISTEN ? fastopenq->max_qlen
: (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)
);
return 0;
}
static int dump_tw_sock(struct seq_file *seq, struct tcp_timewait_sock *ttw,
uid_t uid, __u32 seq_num)
{
struct inet_timewait_sock *tw = &ttw->tw_sk;
__u16 destp, srcp;
__be32 dest, src;
long delta;
delta = tw->tw_timer.expires - bpf_jiffies64();
dest = tw->tw_daddr;
src = tw->tw_rcv_saddr;
destp = bpf_ntohs(tw->tw_dport);
srcp = bpf_ntohs(tw->tw_sport);
BPF_SEQ_PRINTF(seq, "%4d: %08X:%04X %08X:%04X ",
seq_num, src, srcp, dest, destp);
BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
tw->tw_substate, 0, 0,
3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
tw->tw_refcnt.refs.counter, tw);
return 0;
}
static int dump_req_sock(struct seq_file *seq, struct tcp_request_sock *treq,
uid_t uid, __u32 seq_num)
{
struct inet_request_sock *irsk = &treq->req;
struct request_sock *req = &irsk->req;
long ttd;
ttd = req->rsk_timer.expires - bpf_jiffies64();
if (ttd < 0)
ttd = 0;
BPF_SEQ_PRINTF(seq, "%4d: %08X:%04X %08X:%04X ",
seq_num, irsk->ir_loc_addr,
irsk->ir_num, irsk->ir_rmt_addr,
bpf_ntohs(irsk->ir_rmt_port));
BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
TCP_SYN_RECV, 0, 0, 1, jiffies_to_clock_t(ttd),
req->num_timeout, uid, 0, 0, 0, req);
return 0;
}
SEC("iter/tcp")
int dump_tcp4(struct bpf_iter__tcp *ctx)
{
struct sock_common *sk_common = ctx->sk_common;
struct seq_file *seq = ctx->meta->seq;
struct tcp_timewait_sock *tw;
struct tcp_request_sock *req;
struct tcp_sock *tp;
uid_t uid = ctx->uid;
__u32 seq_num;
if (sk_common == (void *)0)
return 0;
seq_num = ctx->meta->seq_num;
if (seq_num == 0)
BPF_SEQ_PRINTF(seq, " sl "
"local_address "
"rem_address "
"st tx_queue rx_queue tr tm->when retrnsmt"
" uid timeout inode\n");
if (sk_common->skc_family != AF_INET)
return 0;
tp = bpf_skc_to_tcp_sock(sk_common);
if (tp)
return dump_tcp_sock(seq, tp, uid, seq_num);
tw = bpf_skc_to_tcp_timewait_sock(sk_common);
if (tw)
return dump_tw_sock(seq, tw, uid, seq_num);
req = bpf_skc_to_tcp_request_sock(sk_common);
if (req)
return dump_req_sock(seq, req, uid, seq_num);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c
+249
View File
@@ -0,0 +1,249 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>
char _license[] SEC("license") = "GPL";
static int hlist_unhashed_lockless(const struct hlist_node *h)
{
return !(h->pprev);
}
static int timer_pending(const struct timer_list * timer)
{
return !hlist_unhashed_lockless(&timer->entry);
}
extern unsigned CONFIG_HZ __kconfig;
#define USER_HZ 100
#define NSEC_PER_SEC 1000000000ULL
static clock_t jiffies_to_clock_t(unsigned long x)
{
/* The implementation here tailored to a particular
* setting of USER_HZ.
*/
u64 tick_nsec = (NSEC_PER_SEC + CONFIG_HZ/2) / CONFIG_HZ;
u64 user_hz_nsec = NSEC_PER_SEC / USER_HZ;
if ((tick_nsec % user_hz_nsec) == 0) {
if (CONFIG_HZ < USER_HZ)
return x * (USER_HZ / CONFIG_HZ);
else
return x / (CONFIG_HZ / USER_HZ);
}
return x * tick_nsec/user_hz_nsec;
}
static clock_t jiffies_delta_to_clock_t(long delta)
{
if (delta <= 0)
return 0;
return jiffies_to_clock_t(delta);
}
static long sock_i_ino(const struct sock *sk)
{
const struct socket *sk_socket = sk->sk_socket;
const struct inode *inode;
unsigned long ino;
if (!sk_socket)
return 0;
inode = &container_of(sk_socket, struct socket_alloc, socket)->vfs_inode;
bpf_probe_read_kernel(&ino, sizeof(ino), &inode->i_ino);
return ino;
}
static bool
inet_csk_in_pingpong_mode(const struct inet_connection_sock *icsk)
{
return icsk->icsk_ack.pingpong >= TCP_PINGPONG_THRESH;
}
static bool tcp_in_initial_slowstart(const struct tcp_sock *tcp)
{
return tcp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
}
static int dump_tcp6_sock(struct seq_file *seq, struct tcp6_sock *tp,
uid_t uid, __u32 seq_num)
{
const struct inet_connection_sock *icsk;
const struct fastopen_queue *fastopenq;
const struct in6_addr *dest, *src;
const struct inet_sock *inet;
unsigned long timer_expires;
const struct sock *sp;
__u16 destp, srcp;
int timer_active;
int rx_queue;
int state;
icsk = &tp->tcp.inet_conn;
inet = &icsk->icsk_inet;
sp = &inet->sk;
fastopenq = &icsk->icsk_accept_queue.fastopenq;
dest = &sp->sk_v6_daddr;
src = &sp->sk_v6_rcv_saddr;
destp = bpf_ntohs(inet->inet_dport);
srcp = bpf_ntohs(inet->inet_sport);
if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
timer_active = 1;
timer_expires = sp->tcp_retransmit_timer.expires;
} else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
timer_active = 4;
timer_expires = sp->tcp_retransmit_timer.expires;
} else if (timer_pending(&icsk->icsk_keepalive_timer)) {
timer_active = 2;
timer_expires = icsk->icsk_keepalive_timer.expires;
} else {
timer_active = 0;
timer_expires = bpf_jiffies64();
}
state = sp->sk_state;
if (state == TCP_LISTEN) {
rx_queue = sp->sk_ack_backlog;
} else {
rx_queue = tp->tcp.rcv_nxt - tp->tcp.copied_seq;
if (rx_queue < 0)
rx_queue = 0;
}
BPF_SEQ_PRINTF(seq, "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X ",
seq_num,
src->s6_addr32[0], src->s6_addr32[1],
src->s6_addr32[2], src->s6_addr32[3], srcp,
dest->s6_addr32[0], dest->s6_addr32[1],
dest->s6_addr32[2], dest->s6_addr32[3], destp);
BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d ",
state,
tp->tcp.write_seq - tp->tcp.snd_una, rx_queue,
timer_active,
jiffies_delta_to_clock_t(timer_expires - bpf_jiffies64()),
icsk->icsk_retransmits, uid,
icsk->icsk_probes_out,
sock_i_ino(sp),
sp->sk_refcnt.refs.counter);
BPF_SEQ_PRINTF(seq, "%pK %lu %lu %u %u %d\n",
tp,
jiffies_to_clock_t(icsk->icsk_rto),
jiffies_to_clock_t(icsk->icsk_ack.ato),
(icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(icsk),
tp->tcp.snd_cwnd,
state == TCP_LISTEN ? fastopenq->max_qlen
: (tcp_in_initial_slowstart(&tp->tcp) ? -1
: tp->tcp.snd_ssthresh)
);
return 0;
}
static int dump_tw_sock(struct seq_file *seq, struct tcp_timewait_sock *ttw,
uid_t uid, __u32 seq_num)
{
struct inet_timewait_sock *tw = &ttw->tw_sk;
const struct in6_addr *dest, *src;
__u16 destp, srcp;
long delta;
delta = tw->tw_timer.expires - bpf_jiffies64();
dest = &tw->tw_v6_daddr;
src = &tw->tw_v6_rcv_saddr;
destp = bpf_ntohs(tw->tw_dport);
srcp = bpf_ntohs(tw->tw_sport);
BPF_SEQ_PRINTF(seq, "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X ",
seq_num,
src->s6_addr32[0], src->s6_addr32[1],
src->s6_addr32[2], src->s6_addr32[3], srcp,
dest->s6_addr32[0], dest->s6_addr32[1],
dest->s6_addr32[2], dest->s6_addr32[3], destp);
BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
tw->tw_substate, 0, 0,
3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
tw->tw_refcnt.refs.counter, tw);
return 0;
}
static int dump_req_sock(struct seq_file *seq, struct tcp_request_sock *treq,
uid_t uid, __u32 seq_num)
{
struct inet_request_sock *irsk = &treq->req;
struct request_sock *req = &irsk->req;
struct in6_addr *src, *dest;
long ttd;
ttd = req->rsk_timer.expires - bpf_jiffies64();
src = &irsk->ir_v6_loc_addr;
dest = &irsk->ir_v6_rmt_addr;
if (ttd < 0)
ttd = 0;
BPF_SEQ_PRINTF(seq, "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X ",
seq_num,
src->s6_addr32[0], src->s6_addr32[1],
src->s6_addr32[2], src->s6_addr32[3],
irsk->ir_num,
dest->s6_addr32[0], dest->s6_addr32[1],
dest->s6_addr32[2], dest->s6_addr32[3],
bpf_ntohs(irsk->ir_rmt_port));
BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
TCP_SYN_RECV, 0, 0, 1, jiffies_to_clock_t(ttd),
req->num_timeout, uid, 0, 0, 0, req);
return 0;
}
SEC("iter/tcp")
int dump_tcp6(struct bpf_iter__tcp *ctx)
{
struct sock_common *sk_common = ctx->sk_common;
struct seq_file *seq = ctx->meta->seq;
struct tcp_timewait_sock *tw;
struct tcp_request_sock *req;
struct tcp6_sock *tp;
uid_t uid = ctx->uid;
__u32 seq_num;
if (sk_common == (void *)0)
return 0;
seq_num = ctx->meta->seq_num;
if (seq_num == 0)
BPF_SEQ_PRINTF(seq, " sl "
"local_address "
"remote_address "
"st tx_queue rx_queue tr tm->when retrnsmt"
" uid timeout inode\n");
if (sk_common->skc_family != AF_INET6)
return 0;
tp = bpf_skc_to_tcp6_sock(sk_common);
if (tp)
return dump_tcp6_sock(seq, tp, uid, seq_num);
tw = bpf_skc_to_tcp_timewait_sock(sk_common);
if (tw)
return dump_tw_sock(seq, tw, uid, seq_num);
req = bpf_skc_to_tcp_request_sock(sk_common);
if (req)
return dump_req_sock(seq, req, uid, seq_num);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_test_kern1.c
+4
View File
@@ -0,0 +1,4 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#define START_CHAR 'a'
#include "bpf_iter_test_kern_common.h"
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_test_kern2.c
+4
View File
@@ -0,0 +1,4 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#define START_CHAR 'A'
#include "bpf_iter_test_kern_common.h"
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_test_kern3.c
+18
View File
@@ -0,0 +1,18 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
SEC("iter/task")
int dump_task(struct bpf_iter__task *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct task_struct *task = ctx->task;
int tgid;
tgid = task->tgid;
bpf_seq_write(seq, &tgid, sizeof(tgid));
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_test_kern4.c
+52
View File
@@ -0,0 +1,52 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
__u32 map1_id = 0, map2_id = 0;
__u32 map1_accessed = 0, map2_accessed = 0;
__u64 map1_seqnum = 0, map2_seqnum1 = 0, map2_seqnum2 = 0;
volatile const __u32 print_len;
volatile const __u32 ret1;
SEC("iter/bpf_map")
int dump_bpf_map(struct bpf_iter__bpf_map *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct bpf_map *map = ctx->map;
__u64 seq_num;
int i, ret = 0;
if (map == (void *)0)
return 0;
/* only dump map1_id and map2_id */
if (map->id != map1_id && map->id != map2_id)
return 0;
seq_num = ctx->meta->seq_num;
if (map->id == map1_id) {
map1_seqnum = seq_num;
map1_accessed++;
}
if (map->id == map2_id) {
if (map2_accessed == 0) {
map2_seqnum1 = seq_num;
if (ret1)
ret = 1;
} else {
map2_seqnum2 = seq_num;
}
map2_accessed++;
}
/* fill seq_file buffer */
for (i = 0; i < (int)print_len; i++)
bpf_seq_write(seq, &seq_num, sizeof(seq_num));
return ret;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_test_kern5.c
+35
View File
@@ -0,0 +1,35 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
char _license[] SEC("license") = "GPL";
struct key_t {
int a;
int b;
int c;
};
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 3);
__type(key, struct key_t);
__type(value, __u64);
} hashmap1 SEC(".maps");
__u32 key_sum = 0;
SEC("iter/bpf_map_elem")
int dump_bpf_hash_map(struct bpf_iter__bpf_map_elem *ctx)
{
void *key = ctx->key;
if (key == (void *)0)
return 0;
/* out of bound access w.r.t. hashmap1 */
key_sum += *(__u32 *)(key + sizeof(struct key_t));
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_test_kern6.c
+21
View File
@@ -0,0 +1,21 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
__u32 value_sum = 0;
SEC("iter/bpf_map_elem")
int dump_bpf_hash_map(struct bpf_iter__bpf_map_elem *ctx)
{
void *value = ctx->value;
if (value == (void *)0)
return 0;
/* negative offset, verifier failure. */
value_sum += *(__u32 *)(value - 4);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_test_kern_common.h
+22
View File
@@ -0,0 +1,22 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
int count = 0;
SEC("iter/task")
int dump_task(struct bpf_iter__task *ctx)
{
struct seq_file *seq = ctx->meta->seq;
char c;
if (count < 4) {
c = START_CHAR + count;
bpf_seq_write(seq, &c, sizeof(c));
count++;
}
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c
+71
View File
@@ -0,0 +1,71 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>
char _license[] SEC("license") = "GPL";
static long sock_i_ino(const struct sock *sk)
{
const struct socket *sk_socket = sk->sk_socket;
const struct inode *inode;
unsigned long ino;
if (!sk_socket)
return 0;
inode = &container_of(sk_socket, struct socket_alloc, socket)->vfs_inode;
bpf_probe_read_kernel(&ino, sizeof(ino), &inode->i_ino);
return ino;
}
SEC("iter/udp")
int dump_udp4(struct bpf_iter__udp *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct udp_sock *udp_sk = ctx->udp_sk;
struct inet_sock *inet;
__u16 srcp, destp;
__be32 dest, src;
__u32 seq_num;
int rqueue;
if (udp_sk == (void *)0)
return 0;
seq_num = ctx->meta->seq_num;
if (seq_num == 0)
BPF_SEQ_PRINTF(seq,
" sl local_address rem_address st tx_queue "
"rx_queue tr tm->when retrnsmt uid timeout "
"inode ref pointer drops\n");
/* filter out udp6 sockets */
inet = &udp_sk->inet;
if (inet->sk.sk_family == AF_INET6)
return 0;
inet = &udp_sk->inet;
dest = inet->inet_daddr;
src = inet->inet_rcv_saddr;
srcp = bpf_ntohs(inet->inet_sport);
destp = bpf_ntohs(inet->inet_dport);
rqueue = inet->sk.sk_rmem_alloc.counter - udp_sk->forward_deficit;
BPF_SEQ_PRINTF(seq, "%5d: %08X:%04X %08X:%04X ",
ctx->bucket, src, srcp, dest, destp);
BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u\n",
inet->sk.sk_state,
inet->sk.sk_wmem_alloc.refs.counter - 1,
rqueue,
0, 0L, 0, ctx->uid, 0,
sock_i_ino(&inet->sk),
inet->sk.sk_refcnt.refs.counter, udp_sk,
udp_sk->drop_counters.drops0.counter +
udp_sk->drop_counters.drops1.counter);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c
+78
View File
@@ -0,0 +1,78 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>
char _license[] SEC("license") = "GPL";
#define IPV6_SEQ_DGRAM_HEADER \
" sl " \
"local_address " \
"remote_address " \
"st tx_queue rx_queue tr tm->when retrnsmt" \
" uid timeout inode ref pointer drops\n"
static long sock_i_ino(const struct sock *sk)
{
const struct socket *sk_socket = sk->sk_socket;
const struct inode *inode;
unsigned long ino;
if (!sk_socket)
return 0;
inode = &container_of(sk_socket, struct socket_alloc, socket)->vfs_inode;
bpf_probe_read_kernel(&ino, sizeof(ino), &inode->i_ino);
return ino;
}
SEC("iter/udp")
int dump_udp6(struct bpf_iter__udp *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct udp_sock *udp_sk = ctx->udp_sk;
const struct in6_addr *dest, *src;
struct udp6_sock *udp6_sk;
struct inet_sock *inet;
__u16 srcp, destp;
__u32 seq_num;
int rqueue;
if (udp_sk == (void *)0)
return 0;
seq_num = ctx->meta->seq_num;
if (seq_num == 0)
BPF_SEQ_PRINTF(seq, IPV6_SEQ_DGRAM_HEADER);
udp6_sk = bpf_skc_to_udp6_sock(udp_sk);
if (udp6_sk == (void *)0)
return 0;
inet = &udp_sk->inet;
srcp = bpf_ntohs(inet->inet_sport);
destp = bpf_ntohs(inet->inet_dport);
rqueue = inet->sk.sk_rmem_alloc.counter - udp_sk->forward_deficit;
dest = &inet->sk.sk_v6_daddr;
src = &inet->sk.sk_v6_rcv_saddr;
BPF_SEQ_PRINTF(seq, "%5d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X ",
ctx->bucket,
src->s6_addr32[0], src->s6_addr32[1],
src->s6_addr32[2], src->s6_addr32[3], srcp,
dest->s6_addr32[0], dest->s6_addr32[1],
dest->s6_addr32[2], dest->s6_addr32[3], destp);
BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u\n",
inet->sk.sk_state,
inet->sk.sk_wmem_alloc.refs.counter - 1,
rqueue,
0, 0L, 0, ctx->uid, 0,
sock_i_ino(&inet->sk),
inet->sk.sk_refcnt.refs.counter, udp_sk,
udp_sk->drop_counters.drops0.counter +
udp_sk->drop_counters.drops1.counter);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_unix.c
+90
View File
@@ -0,0 +1,90 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright Amazon.com Inc. or its affiliates. */
#include <vmlinux.h>
#include "bpf_tracing_net.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>
char _license[] SEC("license") = "GPL";
SEC(".maps") struct {
__uint(type, BPF_MAP_TYPE_SOCKMAP);
__uint(max_entries, 1);
__type(key, __u32);
__type(value, __u64);
} sockmap;
static long sock_i_ino(const struct sock *sk)
{
const struct socket *sk_socket = sk->sk_socket;
const struct inode *inode;
unsigned long ino;
if (!sk_socket)
return 0;
inode = &container_of(sk_socket, struct socket_alloc, socket)->vfs_inode;
bpf_probe_read_kernel(&ino, sizeof(ino), &inode->i_ino);
return ino;
}
SEC("iter/unix")
int dump_unix(struct bpf_iter__unix *ctx)
{
struct unix_sock *unix_sk = ctx->unix_sk;
struct sock *sk = (struct sock *)unix_sk;
struct seq_file *seq;
__u32 seq_num;
if (!unix_sk)
return 0;
seq = ctx->meta->seq;
seq_num = ctx->meta->seq_num;
if (seq_num == 0)
BPF_SEQ_PRINTF(seq, "Num RefCount Protocol Flags Type St Inode Path\n");
BPF_SEQ_PRINTF(seq, "%pK: %08X %08X %08X %04X %02X %8lu",
unix_sk,
sk->sk_refcnt.refs.counter,
0,
sk->sk_state == TCP_LISTEN ? __SO_ACCEPTCON : 0,
sk->sk_type,
sk->sk_socket ?
(sk->sk_state == TCP_ESTABLISHED ? SS_CONNECTED : SS_UNCONNECTED) :
(sk->sk_state == TCP_ESTABLISHED ? SS_CONNECTING : SS_DISCONNECTING),
sock_i_ino(sk));
if (unix_sk->addr) {
if (unix_sk->addr->name->sun_path[0]) {
BPF_SEQ_PRINTF(seq, " %s", unix_sk->addr->name->sun_path);
} else {
/* The name of the abstract UNIX domain socket starts
* with '\0' and can contain '\0'. The null bytes
* should be escaped as done in unix_seq_show().
*/
__u64 i, len;
len = unix_sk->addr->len - sizeof(short);
BPF_SEQ_PRINTF(seq, " @");
for (i = 1; i < len; i++) {
/* unix_validate_addr() tests this upper bound. */
if (i >= sizeof(struct sockaddr_un))
break;
BPF_SEQ_PRINTF(seq, "%c",
unix_sk->addr->name->sun_path[i] ?:
'@');
}
}
}
BPF_SEQ_PRINTF(seq, "\n");
/* Test for deadlock. */
bpf_map_update_elem(&sockmap, &(int){0}, sk, 0);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_iter_vma_offset.c
+37
View File
@@ -0,0 +1,37 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
__u32 unique_tgid_cnt = 0;
uintptr_t address = 0;
uintptr_t offset = 0;
__u32 last_tgid = 0;
__u32 pid = 0;
__u32 page_shift = 0;
SEC("iter/task_vma")
int get_vma_offset(struct bpf_iter__task_vma *ctx)
{
struct vm_area_struct *vma = ctx->vma;
struct seq_file *seq = ctx->meta->seq;
struct task_struct *task = ctx->task;
if (task == NULL || vma == NULL)
return 0;
if (last_tgid != task->tgid)
unique_tgid_cnt++;
last_tgid = task->tgid;
if (task->tgid != pid)
return 0;
if (vma->vm_start <= address && vma->vm_end > address) {
offset = address - vma->vm_start + (vma->vm_pgoff << page_shift);
BPF_SEQ_PRINTF(seq, "OK\n");
}
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_loop.c
+225
View File
@@ -0,0 +1,225 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2021 Facebook */
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
char _license[] SEC("license") = "GPL";
struct callback_ctx {
int output;
};
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 32);
__type(key, int);
__type(value, int);
} map1 SEC(".maps");
/* These should be set by the user program */
u32 nested_callback_nr_loops;
u32 stop_index = -1;
u32 nr_loops;
int pid;
int callback_selector;
/* Making these global variables so that the userspace program
* can verify the output through the skeleton
*/
int nr_loops_returned;
int g_output;
int err;
static int callback(__u32 index, void *data)
{
struct callback_ctx *ctx = data;
if (index >= stop_index)
return 1;
ctx->output += index;
return 0;
}
static int empty_callback(__u32 index, void *data)
{
return 0;
}
static int nested_callback2(__u32 index, void *data)
{
nr_loops_returned += bpf_loop(nested_callback_nr_loops, callback, data, 0);
return 0;
}
static int nested_callback1(__u32 index, void *data)
{
bpf_loop(nested_callback_nr_loops, nested_callback2, data, 0);
return 0;
}
SEC("fentry/" SYS_PREFIX "sys_nanosleep")
int test_prog(void *ctx)
{
struct callback_ctx data = {};
if (bpf_get_current_pid_tgid() >> 32 != pid)
return 0;
nr_loops_returned = bpf_loop(nr_loops, callback, &data, 0);
if (nr_loops_returned < 0)
err = nr_loops_returned;
else
g_output = data.output;
return 0;
}
SEC("fentry/" SYS_PREFIX "sys_nanosleep")
int prog_null_ctx(void *ctx)
{
if (bpf_get_current_pid_tgid() >> 32 != pid)
return 0;
nr_loops_returned = bpf_loop(nr_loops, empty_callback, NULL, 0);
return 0;
}
SEC("fentry/" SYS_PREFIX "sys_nanosleep")
int prog_invalid_flags(void *ctx)
{
struct callback_ctx data = {};
if (bpf_get_current_pid_tgid() >> 32 != pid)
return 0;
err = bpf_loop(nr_loops, callback, &data, 1);
return 0;
}
SEC("fentry/" SYS_PREFIX "sys_nanosleep")
int prog_nested_calls(void *ctx)
{
struct callback_ctx data = {};
if (bpf_get_current_pid_tgid() >> 32 != pid)
return 0;
nr_loops_returned = 0;
bpf_loop(nr_loops, nested_callback1, &data, 0);
g_output = data.output;
return 0;
}
static int callback_set_f0(int i, void *ctx)
{
g_output = 0xF0;
return 0;
}
static int callback_set_0f(int i, void *ctx)
{
g_output = 0x0F;
return 0;
}
/*
* non-constant callback is a corner case for bpf_loop inline logic
*/
SEC("fentry/" SYS_PREFIX "sys_nanosleep")
int prog_non_constant_callback(void *ctx)
{
if (bpf_get_current_pid_tgid() >> 32 != pid)
return 0;
int (*callback)(int i, void *ctx);
g_output = 0;
if (callback_selector == 0x0F)
callback = callback_set_0f;
else
callback = callback_set_f0;
bpf_loop(1, callback, NULL, 0);
return 0;
}
static int stack_check_inner_callback(void *ctx)
{
return 0;
}
static int map1_lookup_elem(int key)
{
int *val = bpf_map_lookup_elem(&map1, &key);
return val ? *val : -1;
}
static void map1_update_elem(int key, int val)
{
bpf_map_update_elem(&map1, &key, &val, BPF_ANY);
}
static int stack_check_outer_callback(void *ctx)
{
int a = map1_lookup_elem(1);
int b = map1_lookup_elem(2);
int c = map1_lookup_elem(3);
int d = map1_lookup_elem(4);
int e = map1_lookup_elem(5);
int f = map1_lookup_elem(6);
bpf_loop(1, stack_check_inner_callback, NULL, 0);
map1_update_elem(1, a + 1);
map1_update_elem(2, b + 1);
map1_update_elem(3, c + 1);
map1_update_elem(4, d + 1);
map1_update_elem(5, e + 1);
map1_update_elem(6, f + 1);
return 0;
}
/* Some of the local variables in stack_check and
* stack_check_outer_callback would be allocated on stack by
* compiler. This test should verify that stack content for these
* variables is preserved between calls to bpf_loop (might be an issue
* if loop inlining allocates stack slots incorrectly).
*/
SEC("fentry/" SYS_PREFIX "sys_nanosleep")
int stack_check(void *ctx)
{
if (bpf_get_current_pid_tgid() >> 32 != pid)
return 0;
int a = map1_lookup_elem(7);
int b = map1_lookup_elem(8);
int c = map1_lookup_elem(9);
int d = map1_lookup_elem(10);
int e = map1_lookup_elem(11);
int f = map1_lookup_elem(12);
bpf_loop(1, stack_check_outer_callback, NULL, 0);
map1_update_elem(7, a + 1);
map1_update_elem(8, b + 1);
map1_update_elem(9, c + 1);
map1_update_elem(10, d + 1);
map1_update_elem(11, e + 1);
map1_update_elem(12, f + 1);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_loop_bench.c
+30
View File
@@ -0,0 +1,30 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2021 Facebook */
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
char _license[] SEC("license") = "GPL";
u32 nr_loops;
long hits;
static int empty_callback(__u32 index, void *data)
{
return 0;
}
static int outer_loop(__u32 index, void *data)
{
bpf_loop(nr_loops, empty_callback, NULL, 0);
__sync_add_and_fetch(&hits, nr_loops);
return 0;
}
SEC("fentry/" SYS_PREFIX "sys_getpgid")
int benchmark(void *ctx)
{
bpf_loop(1000, outer_loop, NULL, 0);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_misc.h
+281
View File
@@ -0,0 +1,281 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __BPF_MISC_H__
#define __BPF_MISC_H__
#define XSTR(s) STR(s)
#define STR(s) #s
/* Expand a macro and then stringize the expansion */
#define QUOTE(str) #str
#define EXPAND_QUOTE(str) QUOTE(str)
/* This set of attributes controls behavior of the
* test_loader.c:test_loader__run_subtests().
*
* The test_loader sequentially loads each program in a skeleton.
* Programs could be loaded in privileged and unprivileged modes.
* - __success, __failure, __msg, __regex imply privileged mode;
* - __success_unpriv, __failure_unpriv, __msg_unpriv, __regex_unpriv
* imply unprivileged mode.
* If combination of privileged and unprivileged attributes is present
* both modes are used. If none are present privileged mode is implied.
*
* See test_loader.c:drop_capabilities() for exact set of capabilities
* that differ between privileged and unprivileged modes.
*
* For test filtering purposes the name of the program loaded in
* unprivileged mode is derived from the usual program name by adding
* `@unpriv' suffix.
*
* __msg Message expected to be found in the verifier log.
* Multiple __msg attributes could be specified.
* To match a regular expression use "{{" "}}" brackets,
* e.g. "foo{{[0-9]+}}" matches strings like "foo007".
* Extended POSIX regular expression syntax is allowed
* inside the brackets.
* __not_msg Message not expected to be found in verifier log.
* If __msg_not is situated between __msg tags
* framework matches __msg tags first, and then
* checks that __msg_not is not present in a portion of
* a log between bracketing __msg tags.
* Same regex syntax as for __msg is supported.
* __msg_unpriv Same as __msg but for unprivileged mode.
* __not_msg_unpriv Same as __not_msg but for unprivileged mode.
*
* __stderr Message expected to be found in bpf stderr stream. The
* same regex rules apply like __msg.
* __stderr_unpriv Same as __stderr but for unpriveleged mode.
* __stdout Same as __stderr but for stdout stream.
* __stdout_unpriv Same as __stdout but for unpriveleged mode.
*
* __xlated Expect a line in a disassembly log after verifier applies rewrites.
* Multiple __xlated attributes could be specified.
* Regular expressions could be specified same way as in __msg.
* __xlated_unpriv Same as __xlated but for unprivileged mode.
*
* __jited Match a line in a disassembly of the jited BPF program.
* Has to be used after __arch_* macro.
* For example:
*
* __arch_x86_64
* __jited(" endbr64")
* __jited(" nopl (%rax,%rax)")
* __jited(" xorq %rax, %rax")
* ...
* __naked void some_test(void)
* {
* asm volatile (... ::: __clobber_all);
* }
*
* Regular expressions could be included in patterns same way
* as in __msg.
*
* By default assume that each pattern has to be matched on the
* next consecutive line of disassembly, e.g.:
*
* __jited(" endbr64") # matched on line N
* __jited(" nopl (%rax,%rax)") # matched on line N+1
*
* If match occurs on a wrong line an error is reported.
* To override this behaviour use literal "...", e.g.:
*
* __jited(" endbr64") # matched on line N
* __jited("...") # not matched
* __jited(" nopl (%rax,%rax)") # matched on any line >= N
*
* __jited_unpriv Same as __jited but for unprivileged mode.
*
*
* __success Expect program load success in privileged mode.
* __success_unpriv Expect program load success in unprivileged mode.
*
* __failure Expect program load failure in privileged mode.
* __failure_unpriv Expect program load failure in unprivileged mode.
*
* __retval Execute the program using BPF_PROG_TEST_RUN command,
* expect return value to match passed parameter:
* - a decimal number
* - a hexadecimal number, when starts from 0x
* - a macro which expands to one of the above
* - literal _INT_MIN (expands to INT_MIN)
* In addition, two special macros are defined below:
* - POINTER_VALUE
* - TEST_DATA_LEN
* __retval_unpriv Same, but load program in unprivileged mode.
*
* __description Text to be used for display and as an additional filter
* alias, while the original program name stays matchable.
*
* __log_level Log level to use for the program, numeric value expected.
*
* __flag Adds one flag use for the program, the following values are valid:
* - BPF_F_STRICT_ALIGNMENT;
* - BPF_F_TEST_RND_HI32;
* - BPF_F_TEST_STATE_FREQ;
* - BPF_F_SLEEPABLE;
* - BPF_F_XDP_HAS_FRAGS;
* - A numeric value.
* Multiple __flag attributes could be specified, the final flags
* value is derived by applying binary "or" to all specified values.
*
* __auxiliary Annotated program is not a separate test, but used as auxiliary
* for some other test cases and should always be loaded.
* __auxiliary_unpriv Same, but load program in unprivileged mode.
*
* __arch_* Specify on which architecture the test case should be tested.
* Several __arch_* annotations could be specified at once.
* When test case is not run on current arch it is marked as skipped.
* __caps_unpriv Specify the capabilities that should be set when running the test.
*
* __linear_size Specify the size of the linear area of non-linear skbs, or
* 0 for linear skbs.
*/
#define __test_tag(tag) __attribute__((btf_decl_tag("comment:" XSTR(__COUNTER__) ":" tag)))
#define __msg(msg) __test_tag("test_expect_msg=" msg)
#define __not_msg(msg) __test_tag("test_expect_not_msg=" msg)
#define __xlated(msg) __test_tag("test_expect_xlated=" msg)
#define __jited(msg) __test_tag("test_jited=" msg)
#define __failure __test_tag("test_expect_failure")
#define __success __test_tag("test_expect_success")
#define __description(desc) __test_tag("test_description=" desc)
#define __msg_unpriv(msg) __test_tag("test_expect_msg_unpriv=" msg)
#define __not_msg_unpriv(msg) __test_tag("test_expect_not_msg_unpriv=" msg)
#define __xlated_unpriv(msg) __test_tag("test_expect_xlated_unpriv=" msg)
#define __jited_unpriv(msg) __test_tag("test_jited_unpriv=" msg)
#define __failure_unpriv __test_tag("test_expect_failure_unpriv")
#define __success_unpriv __test_tag("test_expect_success_unpriv")
#define __log_level(lvl) __test_tag("test_log_level=" #lvl)
#define __flag(flag) __test_tag("test_prog_flags=" #flag)
#define __retval(val) __test_tag("test_retval=" XSTR(val))
#define __retval_unpriv(val) __test_tag("test_retval_unpriv=" XSTR(val))
#define __auxiliary __test_tag("test_auxiliary")
#define __auxiliary_unpriv __test_tag("test_auxiliary_unpriv")
#define __btf_path(path) __test_tag("test_btf_path=" path)
#define __btf_func_path(path) __test_tag("test_btf_func_path=" path)
#define __arch(arch) __test_tag("test_arch=" arch)
#define __arch_x86_64 __arch("X86_64")
#define __arch_arm64 __arch("ARM64")
#define __arch_riscv64 __arch("RISCV64")
#define __arch_s390x __arch("s390x")
#define __arch_loongarch __arch("LOONGARCH")
#define __caps_unpriv(caps) __test_tag("test_caps_unpriv=" EXPAND_QUOTE(caps))
#define __load_if_JITed() __test_tag("load_mode=jited")
#define __load_if_no_JITed() __test_tag("load_mode=no_jited")
#define __stderr(msg) __test_tag("test_expect_stderr=" msg)
#define __stderr_unpriv(msg) __test_tag("test_expect_stderr_unpriv=" msg)
#define __stdout(msg) __test_tag("test_expect_stdout=" msg)
#define __stdout_unpriv(msg) __test_tag("test_expect_stdout_unpriv=" msg)
#define __linear_size(sz) __test_tag("test_linear_size=" XSTR(sz))
/* Define common capabilities tested using __caps_unpriv */
#define CAP_NET_ADMIN 12
#define CAP_SYS_ADMIN 21
#define CAP_PERFMON 38
#define CAP_BPF 39
/* Convenience macro for use with 'asm volatile' blocks */
#define __naked __attribute__((naked))
#define __clobber_all "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "memory"
#define __clobber_common "r0", "r1", "r2", "r3", "r4", "r5", "memory"
#define __imm(name) [name]"i"(name)
#define __imm_const(name, expr) [name]"i"(expr)
#define __imm_addr(name) [name]"i"(&name)
#define __imm_ptr(name) [name]"r"(&name)
#define __imm_insn(name, expr) [name]"i"(*(long *)&(expr))
#define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
#define offsetofend(TYPE, MEMBER) \
(offsetof(TYPE, MEMBER) + sizeof_field(TYPE, MEMBER))
/* Magic constants used with __retval() */
#define POINTER_VALUE 0xbadcafe
#define TEST_DATA_LEN 64
#ifndef __aligned
#define __aligned(x) __attribute__((aligned(x)))
#endif
#ifndef __used
#define __used __attribute__((used))
#endif
#if defined(__TARGET_ARCH_x86)
#define SYSCALL_WRAPPER 1
#define SYS_PREFIX "__x64_"
#elif defined(__TARGET_ARCH_s390)
#define SYSCALL_WRAPPER 1
#define SYS_PREFIX "__s390x_"
#elif defined(__TARGET_ARCH_arm64)
#define SYSCALL_WRAPPER 1
#define SYS_PREFIX "__arm64_"
#elif defined(__TARGET_ARCH_riscv)
#define SYSCALL_WRAPPER 1
#define SYS_PREFIX "__riscv_"
#elif defined(__TARGET_ARCH_powerpc)
#define SYSCALL_WRAPPER 1
#define SYS_PREFIX ""
#else
#define SYSCALL_WRAPPER 0
#define SYS_PREFIX "__se_"
#endif
/* How many arguments are passed to function in register */
#if defined(__TARGET_ARCH_x86) || defined(__x86_64__)
#define FUNC_REG_ARG_CNT 6
#elif defined(__i386__)
#define FUNC_REG_ARG_CNT 3
#elif defined(__TARGET_ARCH_s390) || defined(__s390x__)
#define FUNC_REG_ARG_CNT 5
#elif defined(__TARGET_ARCH_arm) || defined(__arm__)
#define FUNC_REG_ARG_CNT 4
#elif defined(__TARGET_ARCH_arm64) || defined(__aarch64__)
#define FUNC_REG_ARG_CNT 8
#elif defined(__TARGET_ARCH_mips) || defined(__mips__)
#define FUNC_REG_ARG_CNT 8
#elif defined(__TARGET_ARCH_powerpc) || defined(__powerpc__) || defined(__powerpc64__)
#define FUNC_REG_ARG_CNT 8
#elif defined(__TARGET_ARCH_sparc) || defined(__sparc__)
#define FUNC_REG_ARG_CNT 6
#elif defined(__TARGET_ARCH_riscv) || defined(__riscv__)
#define FUNC_REG_ARG_CNT 8
#else
/* default to 5 for others */
#define FUNC_REG_ARG_CNT 5
#endif
/* make it look to compiler like value is read and written */
#define __sink(expr) asm volatile("" : "+g"(expr))
#ifndef ARRAY_SIZE
#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
#endif
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \
defined(__TARGET_ARCH_loongarch)) && \
__clang_major__ >= 18
#define CAN_USE_GOTOL
#endif
#if __clang_major__ >= 18
#define CAN_USE_BPF_ST
#endif
#if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \
(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
defined(__TARGET_ARCH_powerpc) || defined(__TARGET_ARCH_loongarch))
#define CAN_USE_LOAD_ACQ_STORE_REL
#endif
#if defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)
#define SPEC_V1
#endif
#if defined(__TARGET_ARCH_x86)
#define SPEC_V4
#endif
#endif
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_mod_race.c
+100
View File
@@ -0,0 +1,100 @@
// SPDX-License-Identifier: GPL-2.0
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
const volatile struct {
/* thread to activate trace programs for */
pid_t tgid;
/* return error from __init function */
int inject_error;
/* uffd monitored range start address */
void *fault_addr;
} bpf_mod_race_config = { -1 };
int bpf_blocking = 0;
int res_try_get_module = -1;
static __always_inline bool check_thread_id(void)
{
struct task_struct *task = bpf_get_current_task_btf();
return task->tgid == bpf_mod_race_config.tgid;
}
/* The trace of execution is something like this:
*
* finit_module()
* load_module()
* prepare_coming_module()
* notifier_call(MODULE_STATE_COMING)
* btf_parse_module()
* btf_alloc_id() // Visible to userspace at this point
* list_add(btf_mod->list, &btf_modules)
* do_init_module()
* freeinit = kmalloc()
* ret = mod->init()
* bpf_prog_widen_race()
* bpf_copy_from_user()
* ...<sleep>...
* if (ret < 0)
* ...
* free_module()
* return ret
*
* At this point, module loading thread is blocked, we now load the program:
*
* bpf_check
* add_kfunc_call/check_pseudo_btf_id
* btf_try_get_module
* try_get_module_live == false
* return -ENXIO
*
* Without the fix (try_get_module_live in btf_try_get_module):
*
* bpf_check
* add_kfunc_call/check_pseudo_btf_id
* btf_try_get_module
* try_get_module == true
* <store module reference in btf_kfunc_tab or used_btf array>
* ...
* return fd
*
* Now, if we inject an error in the blocked program, our module will be freed
* (going straight from MODULE_STATE_COMING to MODULE_STATE_GOING).
* Later, when bpf program is freed, it will try to module_put already freed
* module. This is why try_get_module_live returns false if mod->state is not
* MODULE_STATE_LIVE.
*/
SEC("fmod_ret.s/bpf_fentry_test1")
int BPF_PROG(widen_race, int a, int ret)
{
char dst;
if (!check_thread_id())
return 0;
/* Indicate that we will attempt to block */
bpf_blocking = 1;
bpf_copy_from_user(&dst, 1, bpf_mod_race_config.fault_addr);
return bpf_mod_race_config.inject_error;
}
SEC("fexit/do_init_module")
int BPF_PROG(fexit_init_module, struct module *mod, int ret)
{
if (!check_thread_id())
return 0;
/* Indicate that we finished blocking */
bpf_blocking = 2;
return 0;
}
SEC("fexit/btf_try_get_module")
int BPF_PROG(fexit_module_get, const struct btf *btf, struct module *mod)
{
res_try_get_module = !!mod;
return 0;
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_nop_bench.c
+14
View File
@@ -0,0 +1,14 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include "bench_bpf_timing.bpf.h"
SEC("syscall")
int bench_nop(void *ctx)
{
return BENCH_BPF_LOOP(0, ({}));
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_qdisc_common.h
+27
View File
@@ -0,0 +1,27 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BPF_QDISC_COMMON_H
#define _BPF_QDISC_COMMON_H
#define NET_XMIT_SUCCESS 0x00
#define NET_XMIT_DROP 0x01 /* skb dropped */
#define NET_XMIT_CN 0x02 /* congestion notification */
#define TC_PRIO_CONTROL 7
#define TC_PRIO_MAX 15
#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8)))
struct bpf_sk_buff_ptr;
static struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb)
{
return (struct qdisc_skb_cb *)skb->cb;
}
static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb)
{
return qdisc_skb_cb(skb)->pkt_len;
}
#endif
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_qdisc_dynptr_use_after_invalidate_clone.c
+74
View File
@@ -0,0 +1,74 @@
// SPDX-License-Identifier: GPL-2.0
#include <vmlinux.h>
#include "bpf_experimental.h"
#include "bpf_qdisc_common.h"
#include "bpf_misc.h"
char _license[] SEC("license") = "GPL";
int proto;
SEC("struct_ops")
__success
int BPF_PROG(dynptr_use_after_invalidate_clone, struct sk_buff *skb, struct Qdisc *sch,
struct bpf_sk_buff_ptr *to_free)
{
struct bpf_dynptr ptr, ptr_clone;
struct ethhdr *hdr;
bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &ptr);
bpf_dynptr_clone(&ptr, &ptr_clone);
hdr = bpf_dynptr_slice(&ptr_clone, 0, NULL, sizeof(*hdr));
if (!hdr) {
bpf_qdisc_skb_drop(skb, to_free);
return NET_XMIT_DROP;
}
*(int *)&ptr = 0;
proto = hdr->h_proto;
bpf_qdisc_skb_drop(skb, to_free);
return NET_XMIT_DROP;
}
SEC("struct_ops")
__auxiliary
struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch)
{
return NULL;
}
SEC("struct_ops")
__auxiliary
int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
return 0;
}
SEC("struct_ops")
__auxiliary
void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch)
{
}
SEC("struct_ops")
__auxiliary
void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch)
{
}
SEC(".struct_ops")
struct Qdisc_ops test = {
.enqueue = (void *)dynptr_use_after_invalidate_clone,
.dequeue = (void *)bpf_qdisc_test_dequeue,
.init = (void *)bpf_qdisc_test_init,
.reset = (void *)bpf_qdisc_test_reset,
.destroy = (void *)bpf_qdisc_test_destroy,
.id = "bpf_qdisc_test",
};
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__incompl_ops.c
+41
View File
@@ -0,0 +1,41 @@
// SPDX-License-Identifier: GPL-2.0
#include <vmlinux.h>
#include "bpf_experimental.h"
#include "bpf_qdisc_common.h"
char _license[] SEC("license") = "GPL";
SEC("struct_ops")
int BPF_PROG(bpf_qdisc_test_enqueue, struct sk_buff *skb, struct Qdisc *sch,
struct bpf_sk_buff_ptr *to_free)
{
bpf_qdisc_skb_drop(skb, to_free);
return NET_XMIT_DROP;
}
SEC("struct_ops")
struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch)
{
return NULL;
}
SEC("struct_ops")
void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch)
{
}
SEC("struct_ops")
void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch)
{
}
SEC(".struct_ops")
struct Qdisc_ops test = {
.enqueue = (void *)bpf_qdisc_test_enqueue,
.dequeue = (void *)bpf_qdisc_test_dequeue,
.reset = (void *)bpf_qdisc_test_reset,
.destroy = (void *)bpf_qdisc_test_destroy,
.id = "bpf_qdisc_test",
};
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr.c
+68
View File
@@ -0,0 +1,68 @@
// SPDX-License-Identifier: GPL-2.0
#include <vmlinux.h>
#include "bpf_experimental.h"
#include "bpf_qdisc_common.h"
#include "bpf_misc.h"
char _license[] SEC("license") = "GPL";
int proto;
SEC("struct_ops")
__failure __msg("Expected an initialized dynptr as R1")
int BPF_PROG(invalid_dynptr, struct sk_buff *skb, struct Qdisc *sch,
struct bpf_sk_buff_ptr *to_free)
{
struct bpf_dynptr ptr;
struct ethhdr *hdr;
bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &ptr);
bpf_qdisc_skb_drop(skb, to_free);
hdr = bpf_dynptr_slice(&ptr, 0, NULL, sizeof(*hdr));
if (!hdr)
return NET_XMIT_DROP;
proto = hdr->h_proto;
return NET_XMIT_DROP;
}
SEC("struct_ops")
__auxiliary
struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch)
{
return NULL;
}
SEC("struct_ops")
__auxiliary
int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
return 0;
}
SEC("struct_ops")
__auxiliary
void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch)
{
}
SEC("struct_ops")
__auxiliary
void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch)
{
}
SEC(".struct_ops")
struct Qdisc_ops test = {
.enqueue = (void *)invalid_dynptr,
.dequeue = (void *)bpf_qdisc_test_dequeue,
.init = (void *)bpf_qdisc_test_init,
.reset = (void *)bpf_qdisc_test_reset,
.destroy = (void *)bpf_qdisc_test_destroy,
.id = "bpf_qdisc_test",
};
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_cross_frame.c
+74
View File
@@ -0,0 +1,74 @@
// SPDX-License-Identifier: GPL-2.0
#include <vmlinux.h>
#include "bpf_experimental.h"
#include "bpf_qdisc_common.h"
#include "bpf_misc.h"
char _license[] SEC("license") = "GPL";
int proto;
static __noinline int free_skb(struct sk_buff *skb)
{
bpf_kfree_skb(skb);
return 0;
}
SEC("struct_ops")
__failure __msg("invalid mem access 'scalar'")
int BPF_PROG(invalid_dynptr_cross_frame, struct sk_buff *skb, struct Qdisc *sch,
struct bpf_sk_buff_ptr *to_free)
{
struct bpf_dynptr ptr;
struct ethhdr *hdr;
bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &ptr);
hdr = bpf_dynptr_slice(&ptr, 0, NULL, sizeof(*hdr));
if (!hdr)
return NET_XMIT_DROP;
free_skb(skb);
proto = hdr->h_proto;
return NET_XMIT_DROP;
}
SEC("struct_ops")
__auxiliary
struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch)
{
return NULL;
}
SEC("struct_ops")
__auxiliary
int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
return 0;
}
SEC("struct_ops")
__auxiliary
void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch)
{
}
SEC("struct_ops")
__auxiliary
void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch)
{
}
SEC(".struct_ops")
struct Qdisc_ops test = {
.enqueue = (void *)invalid_dynptr_cross_frame,
.dequeue = (void *)bpf_qdisc_test_dequeue,
.init = (void *)bpf_qdisc_test_init,
.reset = (void *)bpf_qdisc_test_reset,
.destroy = (void *)bpf_qdisc_test_destroy,
.id = "bpf_qdisc_test",
};
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_slice.c
+70
View File
@@ -0,0 +1,70 @@
// SPDX-License-Identifier: GPL-2.0
#include <vmlinux.h>
#include "bpf_experimental.h"
#include "bpf_qdisc_common.h"
#include "bpf_misc.h"
char _license[] SEC("license") = "GPL";
int proto;
SEC("struct_ops")
__failure __msg("invalid mem access 'scalar'")
int BPF_PROG(invalid_dynptr_slice, struct sk_buff *skb, struct Qdisc *sch,
struct bpf_sk_buff_ptr *to_free)
{
struct bpf_dynptr ptr;
struct ethhdr *hdr;
bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &ptr);
hdr = bpf_dynptr_slice(&ptr, 0, NULL, sizeof(*hdr));
if (!hdr) {
bpf_qdisc_skb_drop(skb, to_free);
return NET_XMIT_DROP;
}
bpf_qdisc_skb_drop(skb, to_free);
proto = hdr->h_proto;
return NET_XMIT_DROP;
}
SEC("struct_ops")
__auxiliary
struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch)
{
return NULL;
}
SEC("struct_ops")
__auxiliary
int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
return 0;
}
SEC("struct_ops")
__auxiliary
void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch)
{
}
SEC("struct_ops")
__auxiliary
void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch)
{
}
SEC(".struct_ops")
struct Qdisc_ops test = {
.enqueue = (void *)invalid_dynptr_slice,
.dequeue = (void *)bpf_qdisc_test_dequeue,
.init = (void *)bpf_qdisc_test_init,
.reset = (void *)bpf_qdisc_test_reset,
.destroy = (void *)bpf_qdisc_test_destroy,
.id = "bpf_qdisc_test",
};
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_qdisc_fifo.c
+126
View File
@@ -0,0 +1,126 @@
// SPDX-License-Identifier: GPL-2.0
#include <vmlinux.h>
#include "bpf_experimental.h"
#include "bpf_qdisc_common.h"
char _license[] SEC("license") = "GPL";
struct skb_node {
struct sk_buff __kptr * skb;
struct bpf_list_node node;
};
private(A) struct bpf_spin_lock q_fifo_lock;
private(A) struct bpf_list_head q_fifo __contains(skb_node, node);
bool init_called;
SEC("struct_ops/bpf_fifo_enqueue")
int BPF_PROG(bpf_fifo_enqueue, struct sk_buff *skb, struct Qdisc *sch,
struct bpf_sk_buff_ptr *to_free)
{
struct skb_node *skbn;
u32 pkt_len;
if (sch->q.qlen == sch->limit)
goto drop;
skbn = bpf_obj_new(typeof(*skbn));
if (!skbn)
goto drop;
pkt_len = qdisc_pkt_len(skb);
sch->q.qlen++;
skb = bpf_kptr_xchg(&skbn->skb, skb);
if (skb)
bpf_qdisc_skb_drop(skb, to_free);
bpf_spin_lock(&q_fifo_lock);
bpf_list_push_back(&q_fifo, &skbn->node);
bpf_spin_unlock(&q_fifo_lock);
sch->qstats.backlog += pkt_len;
return NET_XMIT_SUCCESS;
drop:
bpf_qdisc_skb_drop(skb, to_free);
return NET_XMIT_DROP;
}
SEC("struct_ops/bpf_fifo_dequeue")
struct sk_buff *BPF_PROG(bpf_fifo_dequeue, struct Qdisc *sch)
{
struct bpf_list_node *node;
struct sk_buff *skb = NULL;
struct skb_node *skbn;
bpf_spin_lock(&q_fifo_lock);
node = bpf_list_pop_front(&q_fifo);
bpf_spin_unlock(&q_fifo_lock);
if (!node)
return NULL;
skbn = container_of(node, struct skb_node, node);
skb = bpf_kptr_xchg(&skbn->skb, skb);
bpf_obj_drop(skbn);
if (!skb)
return NULL;
sch->qstats.backlog -= qdisc_pkt_len(skb);
bpf_qdisc_bstats_update(sch, skb);
sch->q.qlen--;
return skb;
}
SEC("struct_ops/bpf_fifo_init")
int BPF_PROG(bpf_fifo_init, struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
sch->limit = 1000;
init_called = true;
return 0;
}
SEC("struct_ops/bpf_fifo_reset")
void BPF_PROG(bpf_fifo_reset, struct Qdisc *sch)
{
struct bpf_list_node *node;
struct skb_node *skbn;
int i;
bpf_for(i, 0, sch->q.qlen) {
struct sk_buff *skb = NULL;
bpf_spin_lock(&q_fifo_lock);
node = bpf_list_pop_front(&q_fifo);
bpf_spin_unlock(&q_fifo_lock);
if (!node)
break;
skbn = container_of(node, struct skb_node, node);
skb = bpf_kptr_xchg(&skbn->skb, skb);
if (skb)
bpf_kfree_skb(skb);
bpf_obj_drop(skbn);
}
sch->q.qlen = 0;
}
SEC("struct_ops")
void BPF_PROG(bpf_fifo_destroy, struct Qdisc *sch)
{
}
SEC(".struct_ops")
struct Qdisc_ops fifo = {
.enqueue = (void *)bpf_fifo_enqueue,
.dequeue = (void *)bpf_fifo_dequeue,
.init = (void *)bpf_fifo_init,
.reset = (void *)bpf_fifo_reset,
.destroy = (void *)bpf_fifo_destroy,
.id = "bpf_fifo",
};
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_qdisc_fq.c
+751
View File
@@ -0,0 +1,751 @@
// SPDX-License-Identifier: GPL-2.0
/* bpf_fq is intended for testing the bpf qdisc infrastructure and not a direct
* copy of sch_fq. bpf_fq implements the scheduling algorithm of sch_fq before
* 29f834aa326e ("net_sched: sch_fq: add 3 bands and WRR scheduling") was
* introduced. It gives each flow a fair chance to transmit packets in a
* round-robin fashion. Note that for flow pacing, bpf_fq currently only
* respects skb->tstamp but not skb->sk->sk_pacing_rate. In addition, if there
* are multiple bpf_fq instances, they will have a shared view of flows and
* configuration since some key data structure such as fq_prio_flows,
* fq_nonprio_flows, and fq_bpf_data are global.
*
* To use bpf_fq alone without running selftests, use the following commands.
*
* 1. Register bpf_fq to the kernel
* bpftool struct_ops register bpf_qdisc_fq.bpf.o /sys/fs/bpf
* 2. Add bpf_fq to an interface
* tc qdisc add dev <interface name> root handle <handle> bpf_fq
* 3. Delete bpf_fq attached to the interface
* tc qdisc delete dev <interface name> root
* 4. Unregister bpf_fq
* bpftool struct_ops unregister name fq
*
* The qdisc name, bpf_fq, used in tc commands is defined by Qdisc_ops.id.
* The struct_ops_map_name, fq, used in the bpftool command is the name of the
* Qdisc_ops.
*
* SEC(".struct_ops")
* struct Qdisc_ops fq = {
* ...
* .id = "bpf_fq",
* };
*/
#include <vmlinux.h>
#include <errno.h>
#include <bpf/bpf_helpers.h>
#include "bpf_experimental.h"
#include "bpf_qdisc_common.h"
char _license[] SEC("license") = "GPL";
#define NSEC_PER_USEC 1000L
#define NSEC_PER_SEC 1000000000L
#define NUM_QUEUE (1 << 20)
struct fq_bpf_data {
u32 quantum;
u32 initial_quantum;
u32 flow_refill_delay;
u32 flow_plimit;
u64 horizon;
u32 orphan_mask;
u32 timer_slack;
u64 time_next_delayed_flow;
u64 unthrottle_latency_ns;
u8 horizon_drop;
u32 new_flow_cnt;
u32 old_flow_cnt;
u64 ktime_cache;
};
enum {
CLS_RET_PRIO = 0,
CLS_RET_NONPRIO = 1,
CLS_RET_ERR = 2,
};
struct skb_node {
u64 tstamp;
struct sk_buff __kptr * skb;
struct bpf_rb_node node;
};
struct fq_flow_node {
int credit;
u32 qlen;
u64 age;
u64 time_next_packet;
struct bpf_list_node list_node;
struct bpf_rb_node rb_node;
struct bpf_rb_root queue __contains(skb_node, node);
struct bpf_spin_lock lock;
struct bpf_refcount refcount;
};
struct dequeue_nonprio_ctx {
bool stop_iter;
u64 expire;
u64 now;
};
struct remove_flows_ctx {
bool gc_only;
u32 reset_cnt;
u32 reset_max;
};
struct unset_throttled_flows_ctx {
bool unset_all;
u64 now;
};
struct fq_stashed_flow {
struct fq_flow_node __kptr * flow;
};
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__type(key, __u64);
__type(value, struct fq_stashed_flow);
__uint(max_entries, NUM_QUEUE);
} fq_nonprio_flows SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__type(key, __u64);
__type(value, struct fq_stashed_flow);
__uint(max_entries, 1);
} fq_prio_flows SEC(".maps");
private(A) struct bpf_spin_lock fq_delayed_lock;
private(A) struct bpf_rb_root fq_delayed __contains(fq_flow_node, rb_node);
private(B) struct bpf_spin_lock fq_new_flows_lock;
private(B) struct bpf_list_head fq_new_flows __contains(fq_flow_node, list_node);
private(C) struct bpf_spin_lock fq_old_flows_lock;
private(C) struct bpf_list_head fq_old_flows __contains(fq_flow_node, list_node);
private(D) struct fq_bpf_data q;
/* Wrapper for bpf_kptr_xchg that expects NULL dst */
static void bpf_kptr_xchg_back(void *map_val, void *ptr)
{
void *ret;
ret = bpf_kptr_xchg(map_val, ptr);
if (ret)
bpf_obj_drop(ret);
}
static bool skbn_tstamp_less(struct bpf_rb_node *a, const struct bpf_rb_node *b)
{
struct skb_node *skbn_a;
struct skb_node *skbn_b;
skbn_a = container_of(a, struct skb_node, node);
skbn_b = container_of(b, struct skb_node, node);
return skbn_a->tstamp < skbn_b->tstamp;
}
static bool fn_time_next_packet_less(struct bpf_rb_node *a, const struct bpf_rb_node *b)
{
struct fq_flow_node *flow_a;
struct fq_flow_node *flow_b;
flow_a = container_of(a, struct fq_flow_node, rb_node);
flow_b = container_of(b, struct fq_flow_node, rb_node);
return flow_a->time_next_packet < flow_b->time_next_packet;
}
static void
fq_flows_add_head(struct bpf_list_head *head, struct bpf_spin_lock *lock,
struct fq_flow_node *flow, u32 *flow_cnt)
{
bpf_spin_lock(lock);
bpf_list_push_front(head, &flow->list_node);
bpf_spin_unlock(lock);
*flow_cnt += 1;
}
static void
fq_flows_add_tail(struct bpf_list_head *head, struct bpf_spin_lock *lock,
struct fq_flow_node *flow, u32 *flow_cnt)
{
bpf_spin_lock(lock);
bpf_list_push_back(head, &flow->list_node);
bpf_spin_unlock(lock);
*flow_cnt += 1;
}
static void
fq_flows_remove_front(struct bpf_list_head *head, struct bpf_spin_lock *lock,
struct bpf_list_node **node, u32 *flow_cnt)
{
bpf_spin_lock(lock);
*node = bpf_list_pop_front(head);
bpf_spin_unlock(lock);
*flow_cnt -= 1;
}
static bool
fq_flows_is_empty(struct bpf_list_head *head, struct bpf_spin_lock *lock)
{
bool empty;
bpf_spin_lock(lock);
empty = bpf_list_empty(head);
bpf_spin_unlock(lock);
return empty;
}
/* flow->age is used to denote the state of the flow (not-detached, detached, throttled)
* as well as the timestamp when the flow is detached.
*
* 0: not-detached
* 1 - (~0ULL-1): detached
* ~0ULL: throttled
*/
static void fq_flow_set_detached(struct fq_flow_node *flow)
{
flow->age = bpf_jiffies64();
}
static bool fq_flow_is_detached(struct fq_flow_node *flow)
{
return flow->age != 0 && flow->age != ~0ULL;
}
static bool sk_listener(struct sock *sk)
{
return (1 << sk->__sk_common.skc_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV);
}
static void fq_gc(void);
static int fq_new_flow(void *flow_map, struct fq_stashed_flow **sflow, u64 hash)
{
struct fq_stashed_flow tmp = {};
struct fq_flow_node *flow;
int ret;
flow = bpf_obj_new(typeof(*flow));
if (!flow)
return -ENOMEM;
flow->credit = q.initial_quantum,
flow->qlen = 0,
flow->age = 1,
flow->time_next_packet = 0,
ret = bpf_map_update_elem(flow_map, &hash, &tmp, 0);
if (ret == -ENOMEM || ret == -E2BIG) {
fq_gc();
bpf_map_update_elem(&fq_nonprio_flows, &hash, &tmp, 0);
}
*sflow = bpf_map_lookup_elem(flow_map, &hash);
if (!*sflow) {
bpf_obj_drop(flow);
return -ENOMEM;
}
bpf_kptr_xchg_back(&(*sflow)->flow, flow);
return 0;
}
static int
fq_classify(struct sk_buff *skb, struct fq_stashed_flow **sflow)
{
struct sock *sk = skb->sk;
int ret = CLS_RET_NONPRIO;
u64 hash = 0;
if ((skb->priority & TC_PRIO_MAX) == TC_PRIO_CONTROL) {
*sflow = bpf_map_lookup_elem(&fq_prio_flows, &hash);
ret = CLS_RET_PRIO;
} else {
if (!sk || sk_listener(sk)) {
hash = bpf_skb_get_hash(skb) & q.orphan_mask;
/* Avoid collision with an existing flow hash, which
* only uses the lower 32 bits of hash, by setting the
* upper half of hash to 1.
*/
hash |= (1ULL << 32);
} else if (sk->__sk_common.skc_state == TCP_CLOSE) {
hash = bpf_skb_get_hash(skb) & q.orphan_mask;
hash |= (1ULL << 32);
} else {
hash = sk->__sk_common.skc_hash;
}
*sflow = bpf_map_lookup_elem(&fq_nonprio_flows, &hash);
}
if (!*sflow)
ret = fq_new_flow(&fq_nonprio_flows, sflow, hash) < 0 ?
CLS_RET_ERR : CLS_RET_NONPRIO;
return ret;
}
static bool fq_packet_beyond_horizon(struct sk_buff *skb)
{
return (s64)skb->tstamp > (s64)(q.ktime_cache + q.horizon);
}
SEC("struct_ops/bpf_fq_enqueue")
int BPF_PROG(bpf_fq_enqueue, struct sk_buff *skb, struct Qdisc *sch,
struct bpf_sk_buff_ptr *to_free)
{
struct fq_flow_node *flow = NULL, *flow_copy;
struct fq_stashed_flow *sflow;
u64 time_to_send, jiffies;
struct skb_node *skbn;
int ret;
if (sch->q.qlen >= sch->limit)
goto drop;
if (!skb->tstamp) {
time_to_send = q.ktime_cache = bpf_ktime_get_ns();
} else {
if (fq_packet_beyond_horizon(skb)) {
q.ktime_cache = bpf_ktime_get_ns();
if (fq_packet_beyond_horizon(skb)) {
if (q.horizon_drop)
goto drop;
skb->tstamp = q.ktime_cache + q.horizon;
}
}
time_to_send = skb->tstamp;
}
ret = fq_classify(skb, &sflow);
if (ret == CLS_RET_ERR)
goto drop;
flow = bpf_kptr_xchg(&sflow->flow, flow);
if (!flow)
goto drop;
if (ret == CLS_RET_NONPRIO) {
if (flow->qlen >= q.flow_plimit) {
bpf_kptr_xchg_back(&sflow->flow, flow);
goto drop;
}
if (fq_flow_is_detached(flow)) {
flow_copy = bpf_refcount_acquire(flow);
jiffies = bpf_jiffies64();
if ((s64)(jiffies - (flow_copy->age + q.flow_refill_delay)) > 0) {
if (flow_copy->credit < q.quantum)
flow_copy->credit = q.quantum;
}
flow_copy->age = 0;
fq_flows_add_tail(&fq_new_flows, &fq_new_flows_lock, flow_copy,
&q.new_flow_cnt);
}
}
skbn = bpf_obj_new(typeof(*skbn));
if (!skbn) {
bpf_kptr_xchg_back(&sflow->flow, flow);
goto drop;
}
skbn->tstamp = skb->tstamp = time_to_send;
sch->qstats.backlog += qdisc_pkt_len(skb);
skb = bpf_kptr_xchg(&skbn->skb, skb);
if (skb)
bpf_qdisc_skb_drop(skb, to_free);
bpf_spin_lock(&flow->lock);
bpf_rbtree_add(&flow->queue, &skbn->node, skbn_tstamp_less);
bpf_spin_unlock(&flow->lock);
flow->qlen++;
bpf_kptr_xchg_back(&sflow->flow, flow);
sch->q.qlen++;
return NET_XMIT_SUCCESS;
drop:
bpf_qdisc_skb_drop(skb, to_free);
sch->qstats.drops++;
return NET_XMIT_DROP;
}
static int fq_unset_throttled_flows(u32 index, struct unset_throttled_flows_ctx *ctx)
{
struct bpf_rb_node *node = NULL;
struct fq_flow_node *flow;
bpf_spin_lock(&fq_delayed_lock);
node = bpf_rbtree_first(&fq_delayed);
if (!node) {
bpf_spin_unlock(&fq_delayed_lock);
return 1;
}
flow = container_of(node, struct fq_flow_node, rb_node);
if (!ctx->unset_all && flow->time_next_packet > ctx->now) {
q.time_next_delayed_flow = flow->time_next_packet;
bpf_spin_unlock(&fq_delayed_lock);
return 1;
}
node = bpf_rbtree_remove(&fq_delayed, &flow->rb_node);
bpf_spin_unlock(&fq_delayed_lock);
if (!node)
return 1;
flow = container_of(node, struct fq_flow_node, rb_node);
flow->age = 0;
fq_flows_add_tail(&fq_old_flows, &fq_old_flows_lock, flow, &q.old_flow_cnt);
return 0;
}
static void fq_flow_set_throttled(struct fq_flow_node *flow)
{
flow->age = ~0ULL;
if (q.time_next_delayed_flow > flow->time_next_packet)
q.time_next_delayed_flow = flow->time_next_packet;
bpf_spin_lock(&fq_delayed_lock);
bpf_rbtree_add(&fq_delayed, &flow->rb_node, fn_time_next_packet_less);
bpf_spin_unlock(&fq_delayed_lock);
}
static void fq_check_throttled(u64 now)
{
struct unset_throttled_flows_ctx ctx = {
.unset_all = false,
.now = now,
};
unsigned long sample;
if (q.time_next_delayed_flow > now)
return;
sample = (unsigned long)(now - q.time_next_delayed_flow);
q.unthrottle_latency_ns -= q.unthrottle_latency_ns >> 3;
q.unthrottle_latency_ns += sample >> 3;
q.time_next_delayed_flow = ~0ULL;
bpf_loop(NUM_QUEUE, fq_unset_throttled_flows, &ctx, 0);
}
static struct sk_buff*
fq_dequeue_nonprio_flows(u32 index, struct dequeue_nonprio_ctx *ctx)
{
u64 time_next_packet, time_to_send;
struct bpf_rb_node *rb_node;
struct sk_buff *skb = NULL;
struct bpf_list_head *head;
struct bpf_list_node *node;
struct bpf_spin_lock *lock;
struct fq_flow_node *flow;
struct skb_node *skbn;
bool is_empty;
u32 *cnt;
if (q.new_flow_cnt) {
head = &fq_new_flows;
lock = &fq_new_flows_lock;
cnt = &q.new_flow_cnt;
} else if (q.old_flow_cnt) {
head = &fq_old_flows;
lock = &fq_old_flows_lock;
cnt = &q.old_flow_cnt;
} else {
if (q.time_next_delayed_flow != ~0ULL)
ctx->expire = q.time_next_delayed_flow;
goto break_loop;
}
fq_flows_remove_front(head, lock, &node, cnt);
if (!node)
goto break_loop;
flow = container_of(node, struct fq_flow_node, list_node);
if (flow->credit <= 0) {
flow->credit += q.quantum;
fq_flows_add_tail(&fq_old_flows, &fq_old_flows_lock, flow, &q.old_flow_cnt);
return NULL;
}
bpf_spin_lock(&flow->lock);
rb_node = bpf_rbtree_first(&flow->queue);
if (!rb_node) {
bpf_spin_unlock(&flow->lock);
is_empty = fq_flows_is_empty(&fq_old_flows, &fq_old_flows_lock);
if (head == &fq_new_flows && !is_empty) {
fq_flows_add_tail(&fq_old_flows, &fq_old_flows_lock, flow, &q.old_flow_cnt);
} else {
fq_flow_set_detached(flow);
bpf_obj_drop(flow);
}
return NULL;
}
skbn = container_of(rb_node, struct skb_node, node);
time_to_send = skbn->tstamp;
time_next_packet = (time_to_send > flow->time_next_packet) ?
time_to_send : flow->time_next_packet;
if (ctx->now < time_next_packet) {
bpf_spin_unlock(&flow->lock);
flow->time_next_packet = time_next_packet;
fq_flow_set_throttled(flow);
return NULL;
}
rb_node = bpf_rbtree_remove(&flow->queue, rb_node);
bpf_spin_unlock(&flow->lock);
if (!rb_node)
goto add_flow_and_break;
skbn = container_of(rb_node, struct skb_node, node);
skb = bpf_kptr_xchg(&skbn->skb, skb);
bpf_obj_drop(skbn);
if (!skb)
goto add_flow_and_break;
flow->credit -= qdisc_skb_cb(skb)->pkt_len;
flow->qlen--;
add_flow_and_break:
fq_flows_add_head(head, lock, flow, cnt);
break_loop:
ctx->stop_iter = true;
return skb;
}
static struct sk_buff *fq_dequeue_prio(void)
{
struct fq_flow_node *flow = NULL;
struct fq_stashed_flow *sflow;
struct bpf_rb_node *rb_node;
struct sk_buff *skb = NULL;
struct skb_node *skbn;
u64 hash = 0;
sflow = bpf_map_lookup_elem(&fq_prio_flows, &hash);
if (!sflow)
return NULL;
flow = bpf_kptr_xchg(&sflow->flow, flow);
if (!flow)
return NULL;
bpf_spin_lock(&flow->lock);
rb_node = bpf_rbtree_first(&flow->queue);
if (!rb_node) {
bpf_spin_unlock(&flow->lock);
goto out;
}
skbn = container_of(rb_node, struct skb_node, node);
rb_node = bpf_rbtree_remove(&flow->queue, &skbn->node);
bpf_spin_unlock(&flow->lock);
if (!rb_node)
goto out;
skbn = container_of(rb_node, struct skb_node, node);
skb = bpf_kptr_xchg(&skbn->skb, skb);
bpf_obj_drop(skbn);
out:
bpf_kptr_xchg_back(&sflow->flow, flow);
return skb;
}
SEC("struct_ops/bpf_fq_dequeue")
struct sk_buff *BPF_PROG(bpf_fq_dequeue, struct Qdisc *sch)
{
struct dequeue_nonprio_ctx cb_ctx = {};
struct sk_buff *skb = NULL;
int i;
if (!sch->q.qlen)
goto out;
skb = fq_dequeue_prio();
if (skb)
goto dequeue;
q.ktime_cache = cb_ctx.now = bpf_ktime_get_ns();
fq_check_throttled(q.ktime_cache);
bpf_for(i, 0, sch->limit) {
skb = fq_dequeue_nonprio_flows(i, &cb_ctx);
if (cb_ctx.stop_iter)
break;
};
if (skb) {
dequeue:
sch->q.qlen--;
sch->qstats.backlog -= qdisc_pkt_len(skb);
bpf_qdisc_bstats_update(sch, skb);
return skb;
}
if (cb_ctx.expire)
bpf_qdisc_watchdog_schedule(sch, cb_ctx.expire, q.timer_slack);
out:
return NULL;
}
static int fq_remove_flows_in_list(u32 index, void *ctx)
{
struct bpf_list_node *node;
struct fq_flow_node *flow;
bpf_spin_lock(&fq_new_flows_lock);
node = bpf_list_pop_front(&fq_new_flows);
bpf_spin_unlock(&fq_new_flows_lock);
if (!node) {
bpf_spin_lock(&fq_old_flows_lock);
node = bpf_list_pop_front(&fq_old_flows);
bpf_spin_unlock(&fq_old_flows_lock);
if (!node)
return 1;
}
flow = container_of(node, struct fq_flow_node, list_node);
bpf_obj_drop(flow);
return 0;
}
extern unsigned CONFIG_HZ __kconfig;
/* limit number of collected flows per round */
#define FQ_GC_MAX 8
#define FQ_GC_AGE (3*CONFIG_HZ)
static bool fq_gc_candidate(struct fq_flow_node *flow)
{
u64 jiffies = bpf_jiffies64();
return fq_flow_is_detached(flow) &&
((s64)(jiffies - (flow->age + FQ_GC_AGE)) > 0);
}
static int
fq_remove_flows(struct bpf_map *flow_map, u64 *hash,
struct fq_stashed_flow *sflow, struct remove_flows_ctx *ctx)
{
if (sflow->flow &&
(!ctx->gc_only || fq_gc_candidate(sflow->flow))) {
bpf_map_delete_elem(flow_map, hash);
ctx->reset_cnt++;
}
return ctx->reset_cnt < ctx->reset_max ? 0 : 1;
}
static void fq_gc(void)
{
struct remove_flows_ctx cb_ctx = {
.gc_only = true,
.reset_cnt = 0,
.reset_max = FQ_GC_MAX,
};
bpf_for_each_map_elem(&fq_nonprio_flows, fq_remove_flows, &cb_ctx, 0);
}
SEC("struct_ops/bpf_fq_reset")
void BPF_PROG(bpf_fq_reset, struct Qdisc *sch)
{
struct unset_throttled_flows_ctx utf_ctx = {
.unset_all = true,
};
struct remove_flows_ctx rf_ctx = {
.gc_only = false,
.reset_cnt = 0,
.reset_max = NUM_QUEUE,
};
struct fq_stashed_flow *sflow;
u64 hash = 0;
sch->q.qlen = 0;
sch->qstats.backlog = 0;
bpf_for_each_map_elem(&fq_nonprio_flows, fq_remove_flows, &rf_ctx, 0);
rf_ctx.reset_cnt = 0;
bpf_for_each_map_elem(&fq_prio_flows, fq_remove_flows, &rf_ctx, 0);
fq_new_flow(&fq_prio_flows, &sflow, hash);
bpf_loop(NUM_QUEUE, fq_remove_flows_in_list, NULL, 0);
q.new_flow_cnt = 0;
q.old_flow_cnt = 0;
bpf_loop(NUM_QUEUE, fq_unset_throttled_flows, &utf_ctx, 0);
}
SEC("struct_ops/bpf_fq_init")
int BPF_PROG(bpf_fq_init, struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
struct net_device *dev = sch->dev_queue->dev;
u32 psched_mtu = dev->mtu + dev->hard_header_len;
struct fq_stashed_flow *sflow;
u64 hash = 0;
if (fq_new_flow(&fq_prio_flows, &sflow, hash) < 0)
return -ENOMEM;
sch->limit = 10000;
q.initial_quantum = 10 * psched_mtu;
q.quantum = 2 * psched_mtu;
q.flow_refill_delay = 40;
q.flow_plimit = 100;
q.horizon = 10ULL * NSEC_PER_SEC;
q.horizon_drop = 1;
q.orphan_mask = 1024 - 1;
q.timer_slack = 10 * NSEC_PER_USEC;
q.time_next_delayed_flow = ~0ULL;
q.unthrottle_latency_ns = 0ULL;
q.new_flow_cnt = 0;
q.old_flow_cnt = 0;
return 0;
}
SEC("struct_ops")
void BPF_PROG(bpf_fq_destroy, struct Qdisc *sch)
{
}
SEC(".struct_ops")
struct Qdisc_ops fq = {
.enqueue = (void *)bpf_fq_enqueue,
.dequeue = (void *)bpf_fq_dequeue,
.reset = (void *)bpf_fq_reset,
.init = (void *)bpf_fq_init,
.destroy = (void *)bpf_fq_destroy,
.id = "bpf_fq",
};
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_smc.c
+141
View File
@@ -0,0 +1,141 @@
// SPDX-License-Identifier: GPL-2.0
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include "bpf_tracing_net.h"
char _license[] SEC("license") = "GPL";
#ifndef SMC_HS_CTRL_NAME_MAX
#define SMC_HS_CTRL_NAME_MAX 16
#endif
enum {
BPF_SMC_LISTEN = 10,
};
struct smc_sock___local {
struct sock sk;
struct smc_sock *listen_smc;
bool use_fallback;
} __attribute__((preserve_access_index));
struct smc_hs_ctrl___local {
char name[SMC_HS_CTRL_NAME_MAX];
int (*syn_option)(struct tcp_sock *);
int (*synack_option)(const struct tcp_sock *, struct inet_request_sock *);
} __attribute__((preserve_access_index));
struct netns_smc___local {
struct smc_hs_ctrl___local *hs_ctrl;
} __attribute__((preserve_access_index));
struct net___local {
struct netns_smc___local smc;
} __attribute__((preserve_access_index));
int smc_cnt = 0;
int fallback_cnt = 0;
SEC("fentry/smc_release")
int BPF_PROG(bpf_smc_release, struct socket *sock)
{
/* only count from one side (client) */
if (sock->sk->__sk_common.skc_state == BPF_SMC_LISTEN)
return 0;
smc_cnt++;
return 0;
}
SEC("fentry/smc_switch_to_fallback")
int BPF_PROG(bpf_smc_switch_to_fallback, struct smc_sock___local *smc)
{
/* only count from one side (client) */
if (smc && !smc->listen_smc)
fallback_cnt++;
return 0;
}
/* go with default value if no strat was found */
bool default_ip_strat_value = true;
struct smc_policy_ip_key {
__u32 sip;
__u32 dip;
};
struct smc_policy_ip_value {
__u8 mode;
};
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(key_size, sizeof(struct smc_policy_ip_key));
__uint(value_size, sizeof(struct smc_policy_ip_value));
__uint(max_entries, 128);
__uint(map_flags, BPF_F_NO_PREALLOC);
} smc_policy_ip SEC(".maps");
static bool smc_check(__u32 src, __u32 dst)
{
struct smc_policy_ip_value *value;
struct smc_policy_ip_key key = {
.sip = src,
.dip = dst,
};
value = bpf_map_lookup_elem(&smc_policy_ip, &key);
return value ? value->mode : default_ip_strat_value;
}
SEC("fmod_ret/update_socket_protocol")
int BPF_PROG(smc_run, int family, int type, int protocol)
{
struct task_struct *task;
if (family != AF_INET && family != AF_INET6)
return protocol;
if ((type & 0xf) != SOCK_STREAM)
return protocol;
if (protocol != 0 && protocol != IPPROTO_TCP)
return protocol;
task = bpf_get_current_task_btf();
/* Prevent from affecting other tests */
if (!task) {
return protocol;
} else {
struct net___local *net = (struct net___local *)task->nsproxy->net_ns;
if (!bpf_core_field_exists(struct net___local, smc) || !net->smc.hs_ctrl)
return protocol;
}
return IPPROTO_SMC;
}
SEC("struct_ops")
int BPF_PROG(bpf_smc_set_tcp_option_cond, const struct tcp_sock *tp,
struct inet_request_sock *ireq)
{
return smc_check(ireq->req.__req_common.skc_daddr,
ireq->req.__req_common.skc_rcv_saddr);
}
SEC("struct_ops")
int BPF_PROG(bpf_smc_set_tcp_option, struct tcp_sock *tp)
{
return smc_check(tp->inet_conn.icsk_inet.sk.__sk_common.skc_rcv_saddr,
tp->inet_conn.icsk_inet.sk.__sk_common.skc_daddr);
}
SEC(".struct_ops")
struct smc_hs_ctrl___local linkcheck = {
.name = "linkcheck",
.syn_option = (void *)bpf_smc_set_tcp_option,
.synack_option = (void *)bpf_smc_set_tcp_option_cond,
};
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_syscall_macro.c
+108
View File
@@ -0,0 +1,108 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright 2022 Sony Group Corporation */
#include <vmlinux.h>
#include <bpf/bpf_core_read.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include "bpf_misc.h"
int arg1 = 0;
unsigned long arg2 = 0;
unsigned long arg3 = 0;
unsigned long arg4_cx = 0;
unsigned long arg4 = 0;
unsigned long arg5 = 0;
int arg1_core = 0;
unsigned long arg2_core = 0;
unsigned long arg3_core = 0;
unsigned long arg4_core_cx = 0;
unsigned long arg4_core = 0;
unsigned long arg5_core = 0;
int option_syscall = 0;
unsigned long arg2_syscall = 0;
unsigned long arg3_syscall = 0;
unsigned long arg4_syscall = 0;
unsigned long arg5_syscall = 0;
const volatile pid_t filter_pid = 0;
SEC("kprobe/" SYS_PREFIX "sys_prctl")
int BPF_KPROBE(handle_sys_prctl)
{
struct pt_regs *real_regs;
pid_t pid = bpf_get_current_pid_tgid() >> 32;
unsigned long tmp = 0;
if (pid != filter_pid)
return 0;
real_regs = PT_REGS_SYSCALL_REGS(ctx);
/* test for PT_REGS_PARM */
bpf_probe_read_kernel(&tmp, sizeof(tmp), &PT_REGS_PARM1_SYSCALL(real_regs));
arg1 = tmp;
bpf_probe_read_kernel(&arg2, sizeof(arg2), &PT_REGS_PARM2_SYSCALL(real_regs));
bpf_probe_read_kernel(&arg3, sizeof(arg3), &PT_REGS_PARM3_SYSCALL(real_regs));
bpf_probe_read_kernel(&arg4_cx, sizeof(arg4_cx), &PT_REGS_PARM4(real_regs));
bpf_probe_read_kernel(&arg4, sizeof(arg4), &PT_REGS_PARM4_SYSCALL(real_regs));
bpf_probe_read_kernel(&arg5, sizeof(arg5), &PT_REGS_PARM5_SYSCALL(real_regs));
/* test for the CORE variant of PT_REGS_PARM */
arg1_core = PT_REGS_PARM1_CORE_SYSCALL(real_regs);
arg2_core = PT_REGS_PARM2_CORE_SYSCALL(real_regs);
arg3_core = PT_REGS_PARM3_CORE_SYSCALL(real_regs);
arg4_core_cx = PT_REGS_PARM4_CORE(real_regs);
arg4_core = PT_REGS_PARM4_CORE_SYSCALL(real_regs);
arg5_core = PT_REGS_PARM5_CORE_SYSCALL(real_regs);
return 0;
}
SEC("ksyscall/prctl")
int BPF_KSYSCALL(prctl_enter, int option, unsigned long arg2,
unsigned long arg3, unsigned long arg4, unsigned long arg5)
{
pid_t pid = bpf_get_current_pid_tgid() >> 32;
if (pid != filter_pid)
return 0;
option_syscall = option;
arg2_syscall = arg2;
arg3_syscall = arg3;
arg4_syscall = arg4;
arg5_syscall = arg5;
return 0;
}
__u64 splice_fd_in;
__u64 splice_off_in;
__u64 splice_fd_out;
__u64 splice_off_out;
__u64 splice_len;
__u64 splice_flags;
SEC("ksyscall/splice")
int BPF_KSYSCALL(splice_enter, int fd_in, loff_t *off_in, int fd_out,
loff_t *off_out, size_t len, unsigned int flags)
{
pid_t pid = bpf_get_current_pid_tgid() >> 32;
if (pid != filter_pid)
return 0;
splice_fd_in = fd_in;
splice_off_in = (__u64)off_in;
splice_fd_out = fd_out;
splice_off_out = (__u64)off_out;
splice_len = len;
splice_flags = flags;
return 0;
}
char _license[] SEC("license") = "GPL";
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_tcp_nogpl.c
+17
View File
@@ -0,0 +1,17 @@
// SPDX-License-Identifier: GPL-2.0
#include "bpf_tracing_net.h"
#include <bpf/bpf_tracing.h>
char _license[] SEC("license") = "X";
SEC("struct_ops")
void BPF_PROG(nogpltcp_init, struct sock *sk)
{
}
SEC(".struct_ops")
struct tcp_congestion_ops bpf_nogpltcp = {
.init = (void *)nogpltcp_init,
.name = "bpf_nogpltcp",
};
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_test_utils.h
+18
View File
@@ -0,0 +1,18 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __BPF_TEST_UTILS_H__
#define __BPF_TEST_UTILS_H__
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
/* Clobber as many native registers and stack slots as possible. */
static __always_inline void clobber_regs_stack(void)
{
char tmp_str[] = "123456789";
unsigned long tmp;
bpf_strtoul(tmp_str, sizeof(tmp_str), 0, &tmp);
__sink(tmp);
}
#endif
linux-7.2.3/tools/testing/selftests/bpf/progs/bpf_tracing_net.h
+194
View File
@@ -0,0 +1,194 @@
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
#ifndef __BPF_TRACING_NET_H__
#define __BPF_TRACING_NET_H__
#include <vmlinux.h>
#include <bpf/bpf_core_read.h>
#define AF_INET 2
#define AF_INET6 10
#define SOL_SOCKET 1
#define SO_REUSEADDR 2
#define SO_SNDBUF 7
#define SO_RCVBUF 8
#define SO_KEEPALIVE 9
#define SO_PRIORITY 12
#define SO_REUSEPORT 15
#if defined(__TARGET_ARCH_powerpc)
#define SO_RCVLOWAT 16
#else
#define SO_RCVLOWAT 18
#endif
#define SO_BINDTODEVICE 25
#define SO_MARK 36
#define SO_MAX_PACING_RATE 47
#define SO_BINDTOIFINDEX 62
#define SO_TXREHASH 74
#define __SO_ACCEPTCON (1 << 16)
#define IP_TOS 1
#define SOL_IPV6 41
#define IPV6_TCLASS 67
#define IPV6_AUTOFLOWLABEL 70
#define TC_ACT_UNSPEC (-1)
#define TC_ACT_OK 0
#define TC_ACT_SHOT 2
#define SOL_TCP 6
#define TCP_NODELAY 1
#define TCP_MAXSEG 2
#define TCP_KEEPIDLE 4
#define TCP_KEEPINTVL 5
#define TCP_KEEPCNT 6
#define TCP_SYNCNT 7
#define TCP_WINDOW_CLAMP 10
#define TCP_CONGESTION 13
#define TCP_THIN_LINEAR_TIMEOUTS 16
#define TCP_USER_TIMEOUT 18
#define TCP_NOTSENT_LOWAT 25
#define TCP_SAVE_SYN 27
#define TCP_SAVED_SYN 28
#define TCP_CA_NAME_MAX 16
#define TCP_NAGLE_OFF 1
#define TCP_RTO_MAX_MS 44
#define TCP_ECN_OK 1
#define TCP_ECN_QUEUE_CWR 2
#define TCP_ECN_DEMAND_CWR 4
#define TCP_ECN_SEEN 8
#define TCP_CONG_NEEDS_ECN 0x2
#define ICSK_TIME_RETRANS 1
#define ICSK_TIME_PROBE0 3
#define ICSK_TIME_LOSS_PROBE 5
#define ICSK_TIME_REO_TIMEOUT 6
#define ETH_ALEN 6
#define ETH_HLEN 14
#define ETH_P_IP 0x0800
#define ETH_P_IPV6 0x86DD
#define NEXTHDR_TCP 6
#define TCPOPT_NOP 1
#define TCPOPT_EOL 0
#define TCPOPT_MSS 2
#define TCPOPT_WINDOW 3
#define TCPOPT_TIMESTAMP 8
#define TCPOPT_SACK_PERM 4
#define TCPOLEN_MSS 4
#define TCPOLEN_WINDOW 3
#define TCPOLEN_TIMESTAMP 10
#define TCPOLEN_SACK_PERM 2
#define CHECKSUM_NONE 0
#define CHECKSUM_PARTIAL 3
#define IFNAMSIZ 16
#define RTF_GATEWAY 0x0002
#define TCP_INFINITE_SSTHRESH 0x7fffffff
#define TCP_PINGPONG_THRESH 3
#define FLAG_DATA_ACKED 0x04 /* This ACK acknowledged new data. */
#define FLAG_SYN_ACKED 0x10 /* This ACK acknowledged SYN. */
#define FLAG_DATA_SACKED 0x20 /* New SACK. */
#define FLAG_SND_UNA_ADVANCED \
0x400 /* Snd_una was changed (!= FLAG_DATA_ACKED) */
#define FLAG_ACKED (FLAG_DATA_ACKED | FLAG_SYN_ACKED)
#define FLAG_FORWARD_PROGRESS (FLAG_ACKED | FLAG_DATA_SACKED)
#define fib_nh_dev nh_common.nhc_dev
#define fib_nh_gw_family nh_common.nhc_gw_family
#define fib_nh_gw6 nh_common.nhc_gw.ipv6
#define inet_daddr sk.__sk_common.skc_daddr
#define inet_rcv_saddr sk.__sk_common.skc_rcv_saddr
#define inet_dport sk.__sk_common.skc_dport
#define udp_portaddr_hash inet.sk.__sk_common.skc_u16hashes[1]
#define ir_loc_addr req.__req_common.skc_rcv_saddr
#define ir_num req.__req_common.skc_num
#define ir_rmt_addr req.__req_common.skc_daddr
#define ir_rmt_port req.__req_common.skc_dport
#define ir_v6_rmt_addr req.__req_common.skc_v6_daddr
#define ir_v6_loc_addr req.__req_common.skc_v6_rcv_saddr
#define sk_num __sk_common.skc_num
#define sk_dport __sk_common.skc_dport
#define sk_family __sk_common.skc_family
#define sk_rmem_alloc sk_backlog.rmem_alloc
#define sk_refcnt __sk_common.skc_refcnt
#define sk_state __sk_common.skc_state
#define sk_net __sk_common.skc_net
#define sk_rcv_saddr __sk_common.skc_rcv_saddr
#define sk_v6_daddr __sk_common.skc_v6_daddr
#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr
#define sk_flags __sk_common.skc_flags
#define sk_reuse __sk_common.skc_reuse
#define sk_cookie __sk_common.skc_cookie
#define s6_addr32 in6_u.u6_addr32
#define tw_daddr __tw_common.skc_daddr
#define tw_rcv_saddr __tw_common.skc_rcv_saddr
#define tw_dport __tw_common.skc_dport
#define tw_refcnt __tw_common.skc_refcnt
#define tw_v6_daddr __tw_common.skc_v6_daddr
#define tw_v6_rcv_saddr __tw_common.skc_v6_rcv_saddr
#define tcp_jiffies32 ((__u32)bpf_jiffies64())
#ifndef min
#define min(a, b) ((a) < (b) ? (a) : (b))
#endif
#ifndef max
#define max(a, b) ((a) > (b) ? (a) : (b))
#endif
static inline bool before(__u32 seq1, __u32 seq2)
{
return (__s32)(seq1 - seq2) < 0;
}
#define after(seq2, seq1) before(seq1, seq2)
static inline struct inet_connection_sock *inet_csk(const struct sock *sk)
{
return (struct inet_connection_sock *)sk;
}
static inline void *inet_csk_ca(const struct sock *sk)
{
return (void *)inet_csk(sk)->icsk_ca_priv;
}
static inline struct tcp_sock *tcp_sk(const struct sock *sk)
{
return (struct tcp_sock *)sk;
}
static inline bool tcp_in_slow_start(const struct tcp_sock *tp)
{
return tp->snd_cwnd < tp->snd_ssthresh;
}
static inline bool tcp_is_cwnd_limited(const struct sock *sk)
{
const struct tcp_sock *tp = tcp_sk(sk);
/* If in slow start, ensure cwnd grows to twice what was ACKed. */
if (tcp_in_slow_start(tp))
return tp->snd_cwnd < 2 * tp->max_packets_out;
return !!BPF_CORE_READ_BITFIELD(tp, is_cwnd_limited);
}
#endif
linux-7.2.3/tools/testing/selftests/bpf/progs/bprm_opts.c
+34
View File
@@ -0,0 +1,34 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright 2020 Google LLC.
*/
#include <linux/bpf.h>
#include <errno.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
char _license[] SEC("license") = "GPL";
struct {
__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
__uint(map_flags, BPF_F_NO_PREALLOC);
__type(key, int);
__type(value, int);
} secure_exec_task_map SEC(".maps");
SEC("lsm/bprm_creds_for_exec")
int BPF_PROG(secure_exec, struct linux_binprm *bprm)
{
int *secureexec;
secureexec = bpf_task_storage_get(&secure_exec_task_map,
bpf_get_current_task_btf(), 0,
BPF_LOCAL_STORAGE_GET_F_CREATE);
if (secureexec && *secureexec)
bpf_bprm_opts_set(bprm, BPF_F_BPRM_SECUREEXEC);
return 0;
}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_arrays x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___diff_arr_dim.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_arrays___diff_arr_dim x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___diff_arr_val_sz.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_arrays___diff_arr_val_sz x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___equiv_zero_sz_arr.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_arrays___equiv_zero_sz_arr x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_bad_signed_arr_elem_sz.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_arrays___err_bad_signed_arr_elem_sz x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_bad_zero_sz_arr.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_arrays___err_bad_zero_sz_arr x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_non_array.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_arrays___err_non_array x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_too_shallow.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_arrays___err_too_shallow x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_too_small.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_arrays___err_too_small x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_wrong_val_type.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_arrays___err_wrong_val_type x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___fixed_arr.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_arrays___fixed_arr x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_bitfields x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___bit_sz_change.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_bitfields___bit_sz_change x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___bitfield_vs_int.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_bitfields___bitfield_vs_int x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___err_too_big_bitfield.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_bitfields___err_too_big_bitfield x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___just_big_enough.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_bitfields___just_big_enough x) {}
linux-7.2.3/tools/testing/selftests/bpf/progs/btf__core_reloc_enum64val.c
+3
View File
@@ -0,0 +1,3 @@
#include "core_reloc_types.h"
void f(struct core_reloc_enum64val x) {}

Some files were not shown because too many files have changed in this diff Show More