lastdragon/linux-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resguardo inicial

Browse Source
This commit is contained in:
lastdragon
2026-09-06 12:53:21 -06:00
parent 662efe3c2a
commit ad2088d178
94853 changed files with 43200404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linux-7.2.3/rust/.gitignore
+10
View File
@@ -0,0 +1,10 @@
# SPDX-License-Identifier: GPL-2.0
bindings_generated.rs
bindings_helpers_generated.rs
doctests_kernel_generated.rs
doctests_kernel_generated_kunit.c
uapi_generated.rs
exports_*_generated.h
doc/
test/
linux-7.2.3/rust/.kunitconfig
+3
View File
@@ -0,0 +1,3 @@
CONFIG_KUNIT=y
CONFIG_RUST=y
CONFIG_RUST_KERNEL_DOCTESTS=y
linux-7.2.3/rust/Makefile
+797
View File
@@ -0,0 +1,797 @@
# SPDX-License-Identifier: GPL-2.0
# Where to place rustdoc generated documentation
rustdoc_output := $(objtree)/Documentation/output/rust/rustdoc
obj-$(CONFIG_RUST) += core.o compiler_builtins.o ffi.o
always-$(CONFIG_RUST) += exports_core_generated.h
obj-$(CONFIG_RUST) += zerocopy.o
ifdef CONFIG_RUST_INLINE_HELPERS
always-$(CONFIG_RUST) += helpers/helpers.bc helpers/helpers_module.bc
else
obj-$(CONFIG_RUST) += helpers/helpers.o
always-$(CONFIG_RUST) += exports_helpers_generated.h
endif
# Missing prototypes are expected in the helpers since these are exported
# for Rust only, thus there is no header nor prototypes.
CFLAGS_REMOVE_helpers/helpers.o = -Wmissing-prototypes -Wmissing-declarations
always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs
obj-$(CONFIG_RUST) += bindings.o pin_init.o kernel.o
always-$(CONFIG_RUST) += exports_bindings_generated.h exports_kernel_generated.h
always-$(CONFIG_RUST) += uapi/uapi_generated.rs
obj-$(CONFIG_RUST) += uapi.o
ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW
obj-$(CONFIG_RUST) += build_error.o
else
always-$(CONFIG_RUST) += build_error.o
endif
obj-$(CONFIG_RUST) += exports.o
always-$(CONFIG_RUST) += libproc_macro2.rlib libquote.rlib libsyn.rlib
always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.rs
always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.c
obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.o
obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.o
always-$(subst y,$(CONFIG_RUST),$(CONFIG_JUMP_LABEL)) += kernel/generated_arch_static_branch_asm.rs
ifndef CONFIG_UML
always-$(subst y,$(CONFIG_RUST),$(CONFIG_BUG)) += kernel/generated_arch_warn_asm.rs kernel/generated_arch_reachable_asm.rs
endif
# Avoids running `$(RUSTC)` when it may not be available.
ifdef CONFIG_RUST
procmacro-name = $(shell MAKEFLAGS= $(RUSTC) --print file-names --crate-name $(1) --crate-type proc-macro - </dev/null)
procmacro-extension := $(patsubst libname.%,%,$(call procmacro-name,name))
libzerocopy_derive_name := $(call procmacro-name,zerocopy_derive)
libmacros_name := $(call procmacro-name,macros)
libpin_init_internal_name := $(call procmacro-name,pin_init_internal)
always-$(CONFIG_RUST) += $(libzerocopy_derive_name) $(libmacros_name) $(libpin_init_internal_name)
# `$(rust_flags)` is passed in case the user added `--sysroot`.
rustc_sysroot := $(shell MAKEFLAGS= $(RUSTC) $(rust_flags) --print sysroot)
rustc_host_target := $(shell $(RUSTC) --version --verbose | grep -F 'host: ' | cut -d' ' -f2)
RUST_LIB_SRC ?= $(rustc_sysroot)/lib/rustlib/src/rust/library
ifneq ($(quiet),)
rust_test_quiet=-q
rustdoc_test_quiet=--test-args -q
rustdoc_test_kernel_quiet=>/dev/null
endif
cfgs-to-flags = $(patsubst %,--cfg='%',$1)
core-cfgs := \
no_fp_fmt_parse
core-edition := $(if $(call rustc-min-version,108700),2024,2021)
core-skip_flags := \
--edition=2021 \
-Wunreachable_pub
core-flags := \
--edition=$(core-edition) \
$(call cfgs-to-flags,$(core-cfgs))
zerocopy-cfgs := \
no_fp_fmt_parse
zerocopy-flags := \
--cap-lints=allow \
$(call cfgs-to-flags,$(zerocopy-cfgs))
zerocopy-envs := \
CARGO_PKG_VERSION=0.8.54
proc_macro2-cfgs := \
feature="proc-macro" \
wrap_proc_macro \
$(if $(call rustc-min-version,108800),proc_macro_span_file proc_macro_span_location)
proc_macro2-flags := \
--cap-lints=allow \
$(call cfgs-to-flags,$(proc_macro2-cfgs))
quote-cfgs := \
feature="proc-macro"
quote-skip_flags := \
--edition=2021
quote-flags := \
--edition=2018 \
--cap-lints=allow \
--extern proc_macro2 \
$(call cfgs-to-flags,$(quote-cfgs))
# `extra-traits`, `fold` and `visit` may be enabled if needed.
syn-cfgs := \
feature="clone-impls" \
feature="derive" \
feature="full" \
feature="parsing" \
feature="printing" \
feature="proc-macro" \
feature="visit" \
feature="visit-mut"
syn-flags := \
--cap-lints=allow \
--extern proc_macro2 \
--extern quote \
$(call cfgs-to-flags,$(syn-cfgs))
zerocopy_derive-cfgs := \
zerocopy_unstable_linux
zerocopy_derive-flags := \
--cap-lints=allow \
--extern proc_macro2 \
--extern quote \
--extern syn \
$(call cfgs-to-flags,$(zerocopy_derive-cfgs))
pin_init_internal-cfgs := \
kernel USE_RUSTC_FEATURES
pin_init_internal-flags := \
--extern proc_macro2 \
--extern quote \
--extern syn \
$(call cfgs-to-flags,$(pin_init_internal-cfgs))
pin_init-cfgs := \
kernel USE_RUSTC_FEATURES
pin_init-flags := \
--extern pin_init_internal \
--extern macros \
$(call cfgs-to-flags,$(pin_init-cfgs))
# `rustdoc` did not save the target modifiers, thus workaround for
# the time being (https://github.com/rust-lang/rust/issues/144521).
rustdoc_modifiers_workaround := $(if $(call rustc-min-version,108800),-Cunsafe-allow-abi-mismatch=fixed-x18)
# Similarly, for doctests (https://github.com/rust-lang/rust/issues/146465).
doctests_modifiers_workaround := $(rustdoc_modifiers_workaround)$(if $(call rustc-min-version,109100),$(comma)sanitizer)
quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $<
cmd_rustdoc = \
$(rustc_target_envs) \
OBJTREE=$(abspath $(objtree)) \
$(RUSTDOC) $(filter-out $(skip_flags) --remap-path-scope=%,$(if $(rustdoc_host),$(rust_common_flags),$(rust_flags))) \
$(rustc_target_flags) -L$(objtree)/$(obj) \
-Zunstable-options --generate-link-to-definition \
--output $(rustdoc_output) \
--crate-name $(subst rustdoc-,,$@) \
$(rustdoc_modifiers_workaround) \
$(if $(rustdoc_host),,--sysroot=/dev/null) \
@$(objtree)/include/generated/rustc_cfg $<
# The `html_logo_url` and `html_favicon_url` forms of the `doc` attribute
# can be used to specify a custom logo. However:
# - The given value is used as-is, thus it cannot be relative or a local file
# (unlike the non-custom case) since the generated docs have subfolders.
# - It requires adding it to every crate.
# - It requires changing `core` which comes from the sysroot.
#
# Using `-Zcrate-attr` would solve the last two points, but not the first.
# The https://github.com/rust-lang/rfcs/pull/3226 RFC suggests two new
# command-like flags to solve the issue. Meanwhile, we use the non-custom case
# and then retouch the generated files.
rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \
rustdoc-kernel rustdoc-pin_init rustdoc-zerocopy rustdoc-zerocopy_derive
$(Q)grep -Ehro '<a href="srctree/([^"]+)"' $(rustdoc_output) | \
cut -d'"' -f2 | cut -d/ -f2- | while read f; do \
if [ ! -e "$(srctree)/$$f" ]; then \
echo "warning: srctree/ link to $$f does not exist"; \
fi \
done
$(Q)cp $(srctree)/Documentation/images/logo.svg $(rustdoc_output)/static.files/
$(Q)cp $(srctree)/Documentation/images/COPYING-logo $(rustdoc_output)/static.files/
$(Q)find $(rustdoc_output) -name '*.html' -type f -print0 | xargs -0 sed -Ei \
-e 's:rust-logo-[0-9a-f]+\.svg:logo.svg:g' \
-e 's:favicon-[0-9a-f]+\.svg:logo.svg:g' \
-e 's:<link rel="alternate icon" type="image/png" href="[/.]+/static\.files/favicon-(16x16|32x32)-[0-9a-f]+\.png">::g' \
-e 's:<a href="srctree/([^"]+)">:<a href="$(realpath $(srctree))/\1">:g'
$(Q)for f in $(rustdoc_output)/static.files/rustdoc-*.css; do \
echo ".logo-container > img { object-fit: contain; }" >> $$f; done
rustdoc-proc_macro2: private rustdoc_host = yes
rustdoc-proc_macro2: private rustc_target_flags = $(proc_macro2-flags)
rustdoc-proc_macro2: $(src)/proc-macro2/lib.rs rustdoc-clean FORCE
+$(call if_changed,rustdoc)
rustdoc-quote: private rustdoc_host = yes
rustdoc-quote: private rustc_target_flags = $(quote-flags)
rustdoc-quote: private skip_flags = $(quote-skip_flags)
rustdoc-quote: $(src)/quote/lib.rs rustdoc-clean rustdoc-proc_macro2 FORCE
+$(call if_changed,rustdoc)
rustdoc-syn: private rustdoc_host = yes
rustdoc-syn: private rustc_target_flags = $(syn-flags)
rustdoc-syn: $(src)/syn/lib.rs rustdoc-clean rustdoc-quote FORCE
+$(call if_changed,rustdoc)
rustdoc-zerocopy_derive: private rustdoc_host = yes
rustdoc-zerocopy_derive: private rustc_target_flags = $(zerocopy_derive-flags) \
--extern proc_macro --crate-type proc-macro
rustdoc-zerocopy_derive: $(src)/zerocopy-derive/lib.rs rustdoc-clean rustdoc-syn FORCE
+$(call if_changed,rustdoc)
rustdoc-macros: private rustdoc_host = yes
rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \
--extern proc_macro --extern proc_macro2 --extern quote --extern syn
rustdoc-macros: $(src)/macros/lib.rs rustdoc-clean rustdoc-proc_macro2 \
rustdoc-quote rustdoc-syn FORCE
+$(call if_changed,rustdoc)
rustdoc-core: private skip_flags = $(core-skip_flags)
rustdoc-core: private rustc_target_flags = $(core-flags)
rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs rustdoc-clean FORCE
+$(call if_changed,rustdoc)
# Even if `rustdoc` targets are not kernel objects, they should still be
# treated as such so that we pass the same flags. Otherwise, for instance,
# `rustdoc` will complain about missing sanitizer flags causing an ABI mismatch.
rustdoc-compiler_builtins: private is-kernel-object := y
rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE
+$(call if_changed,rustdoc)
rustdoc-zerocopy: private rustc_target_envs := $(zerocopy-envs)
rustdoc-zerocopy: private is-kernel-object := y
rustdoc-zerocopy: private rustc_target_flags = $(zerocopy-flags) \
--extend-css $(src)/zerocopy/rustdoc/style.css
rustdoc-zerocopy: $(src)/zerocopy/src/lib.rs rustdoc-clean rustdoc-core FORCE
+$(call if_changed,rustdoc)
rustdoc-ffi: private is-kernel-object := y
rustdoc-ffi: $(src)/ffi.rs rustdoc-core FORCE
+$(call if_changed,rustdoc)
rustdoc-pin_init_internal: private rustdoc_host = yes
rustdoc-pin_init_internal: private rustc_target_flags = $(pin_init_internal-flags) \
--extern proc_macro --crate-type proc-macro
rustdoc-pin_init_internal: $(src)/pin-init/internal/src/lib.rs \
rustdoc-clean rustdoc-proc_macro2 rustdoc-quote rustdoc-syn FORCE
+$(call if_changed,rustdoc)
rustdoc-pin_init: private rustdoc_host = yes
rustdoc-pin_init: private rustc_target_flags = $(pin_init-flags) \
--extern alloc --cfg feature=\"alloc\"
rustdoc-pin_init: $(src)/pin-init/src/lib.rs rustdoc-pin_init_internal \
rustdoc-macros FORCE
+$(call if_changed,rustdoc)
rustdoc-kernel: private is-kernel-object := y
rustdoc-kernel: private rustc_target_flags = --extern ffi --extern pin_init \
--extern build_error --extern macros \
--extern bindings --extern uapi \
--extern zerocopy --extern zerocopy_derive
rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-ffi rustdoc-macros \
rustdoc-pin_init rustdoc-compiler_builtins $(obj)/$(libmacros_name) \
$(obj)/bindings.o FORCE
+$(call if_changed,rustdoc)
rustdoc-clean: FORCE
$(Q)rm -rf $(rustdoc_output)
quiet_cmd_rustc_test_library = $(RUSTC_OR_CLIPPY_QUIET) TL $<
cmd_rustc_test_library = \
$(rustc_target_envs) \
OBJTREE=$(abspath $(objtree)) \
$(RUSTC_OR_CLIPPY) $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \
@$(objtree)/include/generated/rustc_cfg \
--crate-type $(if $(rustc_test_library_proc),proc-macro,rlib) \
--out-dir $(objtree)/$(obj)/test --cfg testlib \
-L$(objtree)/$(obj)/test \
--crate-name $(subst rusttest-,,$(subst rusttestlib-,,$@)) $<
rusttestlib-zerocopy: private rustc_target_envs := $(zerocopy-envs)
rusttestlib-zerocopy: private rustc_target_flags = $(zerocopy-flags)
rusttestlib-zerocopy: $(src)/zerocopy/src/lib.rs FORCE
+$(call if_changed,rustc_test_library)
rusttestlib-build_error: $(src)/build_error.rs FORCE
+$(call if_changed,rustc_test_library)
rusttestlib-ffi: $(src)/ffi.rs FORCE
+$(call if_changed,rustc_test_library)
rusttestlib-proc_macro2: private rustc_target_flags = $(proc_macro2-flags)
rusttestlib-proc_macro2: $(src)/proc-macro2/lib.rs FORCE
+$(call if_changed,rustc_test_library)
rusttestlib-quote: private skip_flags = $(quote-skip_flags)
rusttestlib-quote: private rustc_target_flags = $(quote-flags)
rusttestlib-quote: $(src)/quote/lib.rs rusttestlib-proc_macro2 FORCE
+$(call if_changed,rustc_test_library)
rusttestlib-syn: private rustc_target_flags = $(syn-flags)
rusttestlib-syn: $(src)/syn/lib.rs rusttestlib-quote FORCE
+$(call if_changed,rustc_test_library)
rusttestlib-zerocopy_derive: private rustc_target_flags = $(zerocopy_derive-flags) \
--extern proc_macro
rusttestlib-zerocopy_derive: private rustc_test_library_proc = yes
rusttestlib-zerocopy_derive: $(src)/zerocopy-derive/lib.rs rusttestlib-syn FORCE
+$(call if_changed,rustc_test_library)
rusttestlib-macros: private rustc_target_flags = --extern proc_macro \
--extern proc_macro2 --extern quote --extern syn
rusttestlib-macros: private rustc_test_library_proc = yes
rusttestlib-macros: $(src)/macros/lib.rs \
rusttestlib-proc_macro2 rusttestlib-quote rusttestlib-syn FORCE
+$(call if_changed,rustc_test_library)
rusttestlib-pin_init_internal: private rustc_target_flags = $(pin_init_internal-flags) \
--extern proc_macro
rusttestlib-pin_init_internal: private rustc_test_library_proc = yes
rusttestlib-pin_init_internal: $(src)/pin-init/internal/src/lib.rs \
rusttestlib-proc_macro2 rusttestlib-quote rusttestlib-syn FORCE
+$(call if_changed,rustc_test_library)
rusttestlib-pin_init: private rustc_target_flags = $(pin_init-flags)
rusttestlib-pin_init: $(src)/pin-init/src/lib.rs rusttestlib-macros \
rusttestlib-pin_init_internal $(obj)/$(libpin_init_internal_name) FORCE
+$(call if_changed,rustc_test_library)
rusttestlib-kernel: private rustc_target_flags = --extern ffi \
--extern build_error --extern macros --extern pin_init \
--extern bindings --extern uapi \
--extern zerocopy --extern zerocopy_derive
rusttestlib-kernel: $(src)/kernel/lib.rs rusttestlib-bindings rusttestlib-uapi \
rusttestlib-build_error rusttestlib-pin_init $(obj)/$(libmacros_name) \
$(obj)/bindings.o rusttestlib-zerocopy rusttestlib-zerocopy_derive FORCE
+$(call if_changed,rustc_test_library)
rusttestlib-bindings: private rustc_target_flags = --extern ffi --extern pin_init
rusttestlib-bindings: $(src)/bindings/lib.rs rusttestlib-ffi rusttestlib-pin_init FORCE
+$(call if_changed,rustc_test_library)
rusttestlib-uapi: private rustc_target_flags = --extern ffi --extern pin_init
rusttestlib-uapi: $(src)/uapi/lib.rs rusttestlib-ffi rusttestlib-pin_init FORCE
+$(call if_changed,rustc_test_library)
quiet_cmd_rustdoc_test = RUSTDOC T $<
cmd_rustdoc_test = \
$(rustc_target_envs) \
RUST_MODFILE=test.rs \
OBJTREE=$(abspath $(objtree)) \
$(RUSTDOC) --test $(rust_common_flags) \
-Zcrate-attr='feature(used_with_arg)' \
@$(objtree)/include/generated/rustc_cfg \
$(rustc_target_flags) $(rustdoc_test_target_flags) \
$(rustdoc_test_quiet) \
-L$(objtree)/$(obj)/test --output $(rustdoc_output) \
--crate-name $(subst rusttest-,,$@) $<
quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $<
cmd_rustdoc_test_kernel = \
rm -rf $(objtree)/$(obj)/test/doctests/kernel; \
mkdir -p $(objtree)/$(obj)/test/doctests/kernel; \
$(rustc_target_envs) \
OBJTREE=$(abspath $(objtree)) \
$(RUSTDOC) --test $(filter-out --remap-path-scope=%,$(rust_flags)) \
-L$(objtree)/$(obj) --extern ffi --extern pin_init \
--extern kernel --extern build_error --extern macros \
--extern bindings --extern uapi \
--extern zerocopy --extern zerocopy_derive \
--no-run --crate-name kernel -Zunstable-options \
--sysroot=/dev/null \
$(doctests_modifiers_workaround) \
--test-builder $(objtree)/scripts/rustdoc_test_builder \
$< $(rustdoc_test_kernel_quiet); \
$(objtree)/scripts/rustdoc_test_gen
%/doctests_kernel_generated.rs %/doctests_kernel_generated_kunit.c: \
$(src)/kernel/lib.rs $(obj)/kernel.o \
$(objtree)/scripts/rustdoc_test_builder \
$(objtree)/scripts/rustdoc_test_gen FORCE
+$(call if_changed,rustdoc_test_kernel)
# We cannot use `-Zpanic-abort-tests` because some tests are dynamic,
# so for the moment we skip `-Cpanic=abort`.
quiet_cmd_rustc_test = $(RUSTC_OR_CLIPPY_QUIET) T $<
cmd_rustc_test = \
$(rustc_target_envs) \
OBJTREE=$(abspath $(objtree)) \
$(RUSTC_OR_CLIPPY) --test $(rust_common_flags) \
@$(objtree)/include/generated/rustc_cfg \
$(rustc_target_flags) --out-dir $(objtree)/$(obj)/test \
-L$(objtree)/$(obj)/test \
--crate-name $(subst rusttest-,,$@) $<; \
$(objtree)/$(obj)/test/$(subst rusttest-,,$@) $(rust_test_quiet) \
$(rustc_test_run_flags)
rusttest: rusttest-macros
rusttest-macros: private rustc_target_flags = --extern proc_macro \
--extern macros --extern kernel --extern pin_init \
--extern proc_macro2 --extern quote --extern syn
rusttest-macros: private rustdoc_test_target_flags = --crate-type proc-macro
rusttest-macros: $(src)/macros/lib.rs \
rusttestlib-macros rusttestlib-kernel rusttestlib-pin_init FORCE
+$(call if_changed,rustc_test)
+$(call if_changed,rustdoc_test)
ifdef CONFIG_CC_IS_CLANG
bindgen_c_flags = $(c_flags)
else
# bindgen relies on libclang to parse C. Ideally, bindgen would support a GCC
# plugin backend and/or the Clang driver would be perfectly compatible with GCC.
#
# For the moment, here we are tweaking the flags on the fly. This is a hack,
# and some kernel configurations may not work (e.g. `GCC_PLUGIN_RANDSTRUCT`
# if we end up using one of those structs).
bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \
-mskip-rax-setup -mgeneral-regs-only -msign-return-address=% \
-mindirect-branch=thunk-extern -mindirect-branch-register \
-mfunction-return=thunk-extern -mrecord-mcount -mabi=lp64 \
-mindirect-branch-cs-prefix -mstack-protector-guard% -mtraceback=no \
-mno-pointers-to-nested-functions -mno-string \
-mno-strict-align -mstrict-align -mdirect-extern-access \
-mexplicit-relocs -mno-check-zero-division \
-fconserve-stack -falign-jumps=% -falign-loops=% \
-femit-struct-debug-baseonly -fno-ipa-cp-clone -fno-ipa-sra \
-fno-partial-inlining -fplugin-arg-arm_ssp_per_task_plugin-% \
-fno-reorder-blocks -fno-allow-store-data-races -fasan-shadow-offset=% \
-fzero-call-used-regs=% -fno-stack-clash-protection \
-fno-inline-functions-called-once -fsanitize=bounds-strict \
-fstrict-flex-arrays=% -fmin-function-alignment=% \
-fzero-init-padding-bits=% -mno-fdpic \
-fdiagnostics-show-context -fdiagnostics-show-context=% \
--param=% --param asan-% -fno-isolate-erroneous-paths-dereference
# Derived from `scripts/Makefile.clang`.
BINDGEN_TARGET_x86 := x86_64-linux-gnu
BINDGEN_TARGET_arm64 := aarch64-linux-gnu
BINDGEN_TARGET_arm := arm-linux-gnueabi
BINDGEN_TARGET_loongarch := loongarch64-linux-gnusf
BINDGEN_TARGET_s390 := s390x-linux-gnu
# This is only for i386 UM builds, which need the 32-bit target not -m32
BINDGEN_TARGET_i386 := i386-linux-gnu
BINDGEN_TARGET_um := $(BINDGEN_TARGET_$(SUBARCH))
BINDGEN_TARGET := $(BINDGEN_TARGET_$(SRCARCH))
# All warnings are inhibited since GCC builds are very experimental,
# many GCC warnings are not supported by Clang, they may only appear in
# some configurations, with new GCC versions, etc.
bindgen_extra_c_flags = -w --target=$(BINDGEN_TARGET)
# Auto variable zero-initialization requires an additional special option with
# clang that is going to be removed sometime in the future (likely in
# clang-18), so make sure to pass this option only if clang supports it
# (libclang major version < 16).
#
# https://github.com/llvm/llvm-project/issues/44842
# https://github.com/llvm/llvm-project/blob/llvmorg-16.0.0-rc2/clang/docs/ReleaseNotes.rst#deprecated-compiler-flags
ifdef CONFIG_INIT_STACK_ALL_ZERO
libclang_maj_ver=$(shell $(BINDGEN) $(srctree)/scripts/rust_is_available_bindgen_libclang.h 2>&1 | sed -ne 's/.*clang version \([0-9]*\).*/\1/p')
ifeq ($(shell expr $(libclang_maj_ver) \< 16), 1)
bindgen_extra_c_flags += -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang
endif
endif
bindgen_c_flags = $(filter-out $(bindgen_skip_c_flags), $(c_flags)) \
$(bindgen_extra_c_flags)
endif
ifdef CONFIG_LTO
bindgen_c_flags_lto = $(filter-out $(CC_FLAGS_LTO), $(bindgen_c_flags))
else
bindgen_c_flags_lto = $(bindgen_c_flags)
endif
# `-fno-builtin` is passed to avoid `bindgen` from using `clang` builtin
# prototypes for functions like `memcpy` -- if this flag is not passed,
# `bindgen`-generated prototypes use `c_ulong` or `c_uint` depending on
# architecture instead of generating `usize`.
bindgen_c_flags_final = $(bindgen_c_flags_lto) -fno-builtin -D__BINDGEN__
# `--rust-target` points to our minimum supported Rust version.
quiet_cmd_bindgen = BINDGEN $@
cmd_bindgen = \
$(BINDGEN) $< $(bindgen_target_flags) --rust-target 1.85 \
--use-core --with-derive-default --ctypes-prefix ffi --no-layout-tests \
--no-debug '.*' --enable-function-attribute-detection \
-o $@ -- $(bindgen_c_flags_final) -DMODULE \
$(bindgen_target_cflags) $(bindgen_target_extra)
$(obj)/bindings/bindings_generated.rs: private bindgen_target_flags = \
$(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters)
$(obj)/bindings/bindings_generated.rs: private bindgen_target_extra = ; \
sed -Ei 's/pub const RUST_CONST_HELPER_([a-zA-Z0-9_]*)/pub const \1/g' $@
$(obj)/bindings/bindings_generated.rs: $(src)/bindings/bindings_helper.h \
$(src)/bindgen_parameters FORCE
$(call if_changed_dep,bindgen)
$(obj)/uapi/uapi_generated.rs: private bindgen_target_flags = \
$(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters)
$(obj)/uapi/uapi_generated.rs: $(src)/uapi/uapi_helper.h \
$(src)/bindgen_parameters FORCE
$(call if_changed_dep,bindgen)
# See `CFLAGS_REMOVE_helpers.o` above. In addition, Clang on C does not warn
# with `-Wmissing-declarations` (unlike GCC), so it is not strictly needed here
# given it is `libclang`; but for consistency, future Clang changes and/or
# a potential future GCC backend for `bindgen`, we disable it too.
$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_flags = \
--blocklist-type '.*' --allowlist-var '' \
--allowlist-function 'rust_helper_.*'
$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_cflags = \
-I$(objtree)/$(obj) -Wno-missing-prototypes -Wno-missing-declarations
$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ; \
sed -Ei 's/pub fn rust_helper_([a-zA-Z0-9_]*)/#[link_name="rust_helper_\1"]\n pub fn \1/g' $@
$(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers/helpers.c FORCE
$(call if_changed_dep,bindgen)
quiet_cmd_rust_helper = HELPER $@
cmd_rust_helper = \
$(CC) $(filter-out $(CFLAGS_REMOVE_helpers/helpers.o), $(c_flags)) \
-c -g0 $< -emit-llvm -o $@
$(obj)/helpers/helpers.bc: private part-of-builtin := y
$(obj)/helpers/helpers_module.bc: private part-of-module := y
$(obj)/helpers/helpers.bc $(obj)/helpers/helpers_module.bc: $(src)/helpers/helpers.c FORCE
+$(call if_changed_dep,rust_helper)
rust_exports = $(NM) -p --defined-only $(1) | awk '$$2~/(T|R|D|B)/ && $$3!~/__(pfx|cfi|odr_asan)/ { printf $(2),$$3 }'
quiet_cmd_exports = EXPORTS $@
cmd_exports = \
$(call rust_exports,$<,"EXPORT_SYMBOL_RUST_GPL(%s);\n") > $@
$(obj)/exports_core_generated.h: $(obj)/core.o FORCE
$(call if_changed,exports)
# Even though Rust kernel modules should never use the bindings directly,
# symbols from the `bindings` crate and the C helpers need to be exported
# because Rust generics and inlined functions may not get their code generated
# in the crate where they are defined. Other helpers, called from non-inline
# functions, may not be exported, in principle. However, in general, the Rust
# compiler does not guarantee codegen will be performed for a non-inline
# function either. Therefore, we export all symbols from helpers and bindings.
# In the future, this may be revisited to reduce the number of exports after
# the compiler is informed about the places codegen is required.
$(obj)/exports_helpers_generated.h: $(obj)/helpers/helpers.o FORCE
$(call if_changed,exports)
$(obj)/exports_bindings_generated.h: $(obj)/bindings.o FORCE
$(call if_changed,exports)
$(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE
$(call if_changed,exports)
quiet_cmd_rustc_procmacrolibrary = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) PL $@
cmd_rustc_procmacrolibrary = \
$(rustc_target_envs) \
$(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \
$(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \
--emit=dep-info=$(depfile) --emit=link=$@ --crate-type rlib -O \
--out-dir $(objtree)/$(obj) -L$(objtree)/$(obj) \
--crate-name $(patsubst lib%.rlib,%,$(notdir $@)) $<
$(obj)/libproc_macro2.rlib: private skip_clippy = 1
$(obj)/libproc_macro2.rlib: private rustc_target_flags = $(proc_macro2-flags)
$(obj)/libproc_macro2.rlib: $(src)/proc-macro2/lib.rs FORCE
+$(call if_changed_dep,rustc_procmacrolibrary)
$(obj)/libquote.rlib: private skip_clippy = 1
$(obj)/libquote.rlib: private skip_flags = $(quote-skip_flags)
$(obj)/libquote.rlib: private rustc_target_flags = $(quote-flags)
$(obj)/libquote.rlib: $(src)/quote/lib.rs $(obj)/libproc_macro2.rlib FORCE
+$(call if_changed_dep,rustc_procmacrolibrary)
$(obj)/libsyn.rlib: private skip_clippy = 1
$(obj)/libsyn.rlib: private rustc_target_flags = $(syn-flags)
$(obj)/libsyn.rlib: $(src)/syn/lib.rs $(obj)/libquote.rlib FORCE
+$(call if_changed_dep,rustc_procmacrolibrary)
quiet_cmd_rustc_procmacro = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) P $@
cmd_rustc_procmacro = \
$(rustc_target_envs) \
$(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) $(rust_common_flags) $(rustc_target_flags) \
-Clinker-flavor=gcc -Clinker=$(HOSTCC) \
-Clink-args='$(call escsq,$(KBUILD_PROCMACROLDFLAGS))' \
--emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \
--crate-type proc-macro -L$(objtree)/$(obj) \
--crate-name $(patsubst lib%.$(procmacro-extension),%,$(notdir $@)) \
@$(objtree)/include/generated/rustc_cfg $<
# Procedural macros can only be used with the `rustc` that compiled it.
$(obj)/$(libzerocopy_derive_name): private skip_clippy = 1
$(obj)/$(libzerocopy_derive_name): private rustc_target_flags = $(zerocopy_derive-flags)
$(obj)/$(libzerocopy_derive_name): $(src)/zerocopy-derive/lib.rs $(obj)/libproc_macro2.rlib \
$(obj)/libquote.rlib $(obj)/libsyn.rlib FORCE
+$(call if_changed_dep,rustc_procmacro)
$(obj)/$(libmacros_name): private rustc_target_flags = \
--extern proc_macro2 --extern quote --extern syn
$(obj)/$(libmacros_name): $(src)/macros/lib.rs $(obj)/libproc_macro2.rlib \
$(obj)/libquote.rlib $(obj)/libsyn.rlib FORCE
+$(call if_changed_dep,rustc_procmacro)
$(obj)/$(libpin_init_internal_name): private rustc_target_flags = $(pin_init_internal-flags)
$(obj)/$(libpin_init_internal_name): $(src)/pin-init/internal/src/lib.rs \
$(obj)/libproc_macro2.rlib $(obj)/libquote.rlib $(obj)/libsyn.rlib FORCE
+$(call if_changed_dep,rustc_procmacro)
# `rustc` requires `-Zunstable-options` to use custom target specifications
# since Rust 1.95.0 (https://github.com/rust-lang/rust/pull/151534).
quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@
cmd_rustc_library = \
$(rustc_target_envs) \
OBJTREE=$(abspath $(objtree)) \
$(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \
$(filter-out $(skip_flags),$(rust_flags)) $(rustc_target_flags) \
--emit=dep-info=$(depfile) --emit=$(if $(link_helper),llvm-bc=$(patsubst %.o,%.bc,$@),obj=$@) \
--emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \
--crate-type rlib -L$(objtree)/$(obj) \
--crate-name $(patsubst %.o,%,$(notdir $@)) $< \
--sysroot=/dev/null \
-Zunstable-options \
$(if $(link_helper),;$(LLVM_LINK) --internalize --suppress-warnings $(patsubst %.o,%.bc,$@) \
$(obj)/helpers/helpers$(if $(part-of-module),_module).bc -o $(patsubst %.o,%.m.bc,$@); \
$(CC) $(CLANG_FLAGS) $(KBUILD_CFLAGS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@ \
$(cmd_ld_single)) \
$(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) \
$(cmd_objtool)
rust-analyzer:
$(Q)MAKEFLAGS= $(srctree)/scripts/generate_rust_analyzer.py \
--cfgs='core=$(core-cfgs)' $(core-edition) \
--cfgs='zerocopy=$(zerocopy-cfgs)' \
--cfgs='proc_macro2=$(proc_macro2-cfgs)' \
--cfgs='quote=$(quote-cfgs)' \
--cfgs='syn=$(syn-cfgs)' \
--cfgs='zerocopy_derive=$(zerocopy_derive-cfgs)' \
--cfgs='pin_init_internal=$(pin_init_internal-cfgs)' \
--cfgs='pin_init=$(pin_init-cfgs)' \
--envs='zerocopy=$(zerocopy-envs)' \
$(realpath $(srctree)) $(realpath $(objtree)) \
$(rustc_sysroot) $(RUST_LIB_SRC) $(if $(KBUILD_EXTMOD),$(srcroot)) \
> rust-project.json
redirect-intrinsics = \
__addsf3 __eqsf2 __extendsfdf2 __gesf2 __lesf2 __ltsf2 __mulsf3 __nesf2 __truncdfsf2 __unordsf2 \
__adddf3 __eqdf2 __ledf2 __ltdf2 __muldf3 __unorddf2 \
__muloti4 __multi3 \
__udivmodti4 __udivti3 __umodti3
ifdef CONFIG_ARM
# Add eabi initrinsics for ARM 32-bit
redirect-intrinsics += \
__aeabi_fadd __aeabi_fmul __aeabi_fcmpeq __aeabi_fcmple __aeabi_fcmplt __aeabi_fcmpun \
__aeabi_dadd __aeabi_dmul __aeabi_dcmple __aeabi_dcmplt __aeabi_dcmpun \
__aeabi_uldivmod
endif
ifneq ($(or $(CONFIG_ARM64),$(and $(CONFIG_RISCV),$(CONFIG_64BIT))),)
# These intrinsics are defined for ARM64 and RISCV64
redirect-intrinsics += \
__ashrti3 \
__ashlti3 __lshrti3
endif
ifdef CONFIG_MODVERSIONS
cmd_gendwarfksyms = $(if $(skip_gendwarfksyms),, \
$(call rust_exports,$@,"%s\n") | \
scripts/gendwarfksyms/gendwarfksyms \
$(if $(KBUILD_GENDWARFKSYMS_STABLE), --stable) \
$(if $(KBUILD_SYMTYPES), --symtypes $(@:.o=.symtypes),) \
$@ >> $(dot-target).cmd)
endif
define rule_rustc_library
$(call cmd_and_fixdep,rustc_library)
$(call cmd,gen_objtooldep)
$(call cmd,gendwarfksyms)
endef
define rule_rust_cc_library
$(call if_changed_rule,cc_o_c)
$(call cmd,force_checksrc)
$(call cmd,gendwarfksyms)
endef
# helpers.o uses the same export mechanism as Rust libraries, so ensure symbol
# versions are calculated for the helpers too.
$(obj)/helpers/helpers.o: $(src)/helpers/helpers.c $(recordmcount_source) FORCE
+$(call if_changed_rule,rust_cc_library)
# Disable symbol versioning for exports.o to avoid conflicts with the actual
# symbol versions generated from Rust objects.
$(obj)/exports.o: private skip_gendwarfksyms = 1
$(obj)/core.o: private skip_clippy = 1
$(obj)/core.o: private skip_flags = $(core-skip_flags)
$(obj)/core.o: private rustc_objcopy = $(foreach sym,$(redirect-intrinsics),--redefine-sym $(sym)=__rust$(sym))
$(obj)/core.o: private rustc_target_flags = $(core-flags)
$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs \
$(wildcard $(objtree)/include/config/RUSTC_VERSION_TEXT) FORCE
+$(call if_changed_rule,rustc_library)
ifneq ($(or $(CONFIG_X86_64),$(CONFIG_X86_32)),)
$(obj)/core.o: scripts/target.json
endif
KCOV_INSTRUMENT_core.o := n
$(obj)/compiler_builtins.o: private skip_gendwarfksyms = 1
$(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*'
$(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE
+$(call if_changed_rule,rustc_library)
$(obj)/zerocopy.o: private skip_clippy = 1
$(obj)/zerocopy.o: private skip_gendwarfksyms = 1
$(obj)/zerocopy.o: private rustc_target_envs := $(zerocopy-envs)
$(obj)/zerocopy.o: private rustc_target_flags = $(zerocopy-flags)
$(obj)/zerocopy.o: $(src)/zerocopy/src/lib.rs $(obj)/compiler_builtins.o FORCE
+$(call if_changed_rule,rustc_library)
$(obj)/pin_init.o: private skip_gendwarfksyms = 1
$(obj)/pin_init.o: private rustc_target_flags = $(pin_init-flags)
$(obj)/pin_init.o: $(src)/pin-init/src/lib.rs $(obj)/compiler_builtins.o \
$(obj)/$(libpin_init_internal_name) $(obj)/$(libmacros_name) FORCE
+$(call if_changed_rule,rustc_library)
# Even if normally `build_error` is not a kernel object, it should still be
# treated as such so that we pass the same flags. Otherwise, for instance,
# `rustc` will complain about missing sanitizer flags causing an ABI mismatch.
$(obj)/build_error.o: private is-kernel-object := y
$(obj)/build_error.o: private skip_gendwarfksyms = 1
$(obj)/build_error.o: $(src)/build_error.rs $(obj)/compiler_builtins.o FORCE
+$(call if_changed_rule,rustc_library)
$(obj)/ffi.o: private skip_gendwarfksyms = 1
$(obj)/ffi.o: $(src)/ffi.rs $(obj)/compiler_builtins.o FORCE
+$(call if_changed_rule,rustc_library)
$(obj)/bindings.o: private rustc_target_flags = --extern ffi --extern pin_init
$(obj)/bindings.o: $(src)/bindings/lib.rs \
$(obj)/ffi.o \
$(obj)/pin_init.o \
$(obj)/bindings/bindings_generated.rs \
$(obj)/bindings/bindings_helpers_generated.rs FORCE
+$(call if_changed_rule,rustc_library)
$(obj)/uapi.o: private rustc_target_flags = --extern ffi --extern pin_init
$(obj)/uapi.o: private skip_gendwarfksyms = 1
$(obj)/uapi.o: $(src)/uapi/lib.rs \
$(obj)/ffi.o \
$(obj)/pin_init.o \
$(obj)/uapi/uapi_generated.rs FORCE
+$(call if_changed_rule,rustc_library)
$(obj)/kernel.o: private rustc_target_flags = --extern ffi --extern pin_init \
--extern build_error --extern macros --extern bindings --extern uapi \
--extern zerocopy --extern zerocopy_derive
$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/build_error.o $(obj)/pin_init.o \
$(obj)/$(libmacros_name) $(obj)/bindings.o $(obj)/uapi.o \
$(obj)/zerocopy.o $(obj)/$(libzerocopy_derive_name) FORCE
+$(call if_changed_rule,rustc_library)
ifdef CONFIG_JUMP_LABEL
$(obj)/kernel.o: $(obj)/kernel/generated_arch_static_branch_asm.rs
endif
ifndef CONFIG_UML
ifdef CONFIG_BUG
$(obj)/kernel.o: $(obj)/kernel/generated_arch_warn_asm.rs $(obj)/kernel/generated_arch_reachable_asm.rs
endif
endif
ifdef CONFIG_RUST_INLINE_HELPERS
$(obj)/kernel.o: private link_helper = 1
$(obj)/kernel.o: $(obj)/helpers/helpers.bc
endif
endif # CONFIG_RUST
linux-7.2.3/rust/bindgen_parameters
+73
View File
@@ -0,0 +1,73 @@
# SPDX-License-Identifier: GPL-2.0
# We want to map these types to `isize`/`usize` manually, instead of
# define them as `int`/`long` depending on platform bitwidth.
--blocklist-type __kernel_s?size_t
--blocklist-type __kernel_ptrdiff_t
--opaque-type xregs_state
--opaque-type desc_struct
--opaque-type arch_lbr_state
--opaque-type local_apic
# Packed type cannot transitively contain a `#[repr(align)]` type.
--opaque-type alt_instr
--opaque-type x86_msi_data
--opaque-type x86_msi_addr_lo
# s390-only: same packed/align issue as above (E0588).
--opaque-type lowcore
--opaque-type tod_clock
--opaque-type tpi_info
--opaque-type uv_cb.*
--opaque-type uv_secret.*
--opaque-type zpci_fib
# If SMP is disabled, `arch_spinlock_t` is defined as a ZST which triggers a Rust
# warning. We don't need to peek into it anyway.
--opaque-type spinlock
# enums that appear in indirect function calls should specify a cfi type
--newtype-enum lru_status
--with-attribute-custom-enum=lru_status='#[cfi_encoding="10lru_status"]'
# `seccomp`'s comment gets understood as a doctest
--no-doc-comments
# These functions use the `__preserve_most` calling convention, which neither bindgen
# nor Rust currently understand, and which Clang currently declares to be unstable.
--blocklist-function __list_.*_report
# These constants are sometimes not recognized by bindgen depending on config.
# We use const helpers to aid bindgen, to avoid conflicts when constants are
# recognized, block generation of the non-helper constants.
--blocklist-item ARCH_SLAB_MINALIGN
--blocklist-item ARCH_KMALLOC_MINALIGN
--blocklist-item VM_MERGEABLE
--blocklist-item VM_READ
--blocklist-item VM_WRITE
--blocklist-item VM_EXEC
--blocklist-item VM_SHARED
--blocklist-item VM_MAYREAD
--blocklist-item VM_MAYWRITE
--blocklist-item VM_MAYEXEC
--blocklist-item VM_MAYEXEC
--blocklist-item VM_PFNMAP
--blocklist-item VM_IO
--blocklist-item VM_DONTCOPY
--blocklist-item VM_DONTEXPAND
--blocklist-item VM_LOCKONFAULT
--blocklist-item VM_ACCOUNT
--blocklist-item VM_NORESERVE
--blocklist-item VM_HUGETLB
--blocklist-item VM_SYNC
--blocklist-item VM_ARCH_1
--blocklist-item VM_WIPEONFORK
--blocklist-item VM_DONTDUMP
--blocklist-item VM_SOFTDIRTY
--blocklist-item VM_MIXEDMAP
--blocklist-item VM_HUGEPAGE
--blocklist-item VM_NOHUGEPAGE
# Structs should implement `Zeroable` when all of their fields do.
--with-derive-custom-struct .*=MaybeZeroable
--with-derive-custom-union .*=MaybeZeroable
linux-7.2.3/rust/bindings/bindings_helper.h
+167
View File
@@ -0,0 +1,167 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Header that contains the code (mostly headers) for which Rust bindings
* will be automatically generated by `bindgen`.
*
* Sorted alphabetically.
*/
/*
* First, avoid forward references to `enum` types.
*
* This workarounds a `bindgen` issue with them:
* <https://github.com/rust-lang/rust-bindgen/issues/3179>.
*
* Without this, the generated Rust type may be the wrong one (`i32`) or
* the proper one (typically `c_uint`) depending on how the headers are
* included, which in turn may depend on the particular kernel configuration
* or the architecture.
*
* The alternative would be to use casts and likely an
* `#[allow(clippy::unnecessary_cast)]` in the Rust source files. Instead,
* this approach allows us to keep the correct code in the source files and
* simply remove this section when the issue is fixed upstream and we bump
* the minimum `bindgen` version.
*
* This workaround may not be possible in some cases, depending on how the C
* headers are set up.
*/
#include <linux/hrtimer_types.h>
#include <linux/acpi.h>
#include <linux/gpu_buddy.h>
#include <drm/drm_device.h>
#include <drm/drm_drv.h>
#include <drm/drm_file.h>
#include <drm/drm_gem.h>
#include <drm/drm_gem_shmem_helper.h>
#include <drm/drm_gpuvm.h>
#include <drm/drm_ioctl.h>
#include <kunit/test.h>
#include <linux/auxiliary_bus.h>
#include <linux/bitmap.h>
#include <linux/blk-mq.h>
#include <linux/blk_types.h>
#include <linux/blkdev.h>
#include <linux/clk.h>
#include <linux/completion.h>
#include <linux/configfs.h>
#include <linux/cpu.h>
#include <linux/cpufreq.h>
#include <linux/cpumask.h>
#include <linux/cred.h>
#include <linux/debugfs.h>
#include <linux/device/faux.h>
#include <linux/dma-direction.h>
#include <linux/dma-mapping.h>
#include <linux/dma-resv.h>
#include <linux/errname.h>
#include <linux/ethtool.h>
#include <linux/fdtable.h>
#include <linux/file.h>
#include <linux/firmware.h>
#include <linux/fs.h>
#include <linux/i2c.h>
#include <linux/interrupt.h>
#include <linux/io-pgtable.h>
#include <linux/ioport.h>
#include <linux/iosys-map.h>
#include <linux/jiffies.h>
#include <linux/jump_label.h>
#include <linux/mdio.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/of_device.h>
#include <linux/pci.h>
#include <linux/phy.h>
#include <linux/pid_namespace.h>
#include <linux/platform_device.h>
#include <linux/pm_opp.h>
#include <linux/poll.h>
#include <linux/property.h>
#include <linux/pwm.h>
#include <linux/random.h>
#include <linux/refcount.h>
#include <linux/regulator/consumer.h>
#include <linux/sched.h>
#include <linux/security.h>
#include <linux/slab.h>
#include <linux/sys_soc.h>
#include <linux/task_work.h>
#include <linux/tracepoint.h>
#include <linux/usb.h>
#include <linux/wait.h>
#include <linux/workqueue.h>
#include <linux/xarray.h>
#include <trace/events/rust_sample.h>
/*
* The driver-core Rust code needs to know about some C driver-core private
* structures.
*/
#include <../../drivers/base/base.h>
#if defined(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
// Used by `#[export]` in `drivers/gpu/drm/drm_panic_qr.rs`.
#include <drm/drm_panic.h>
#endif
/* `bindgen` gets confused at certain things. */
const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN;
const size_t RUST_CONST_HELPER_ARCH_KMALLOC_MINALIGN = ARCH_KMALLOC_MINALIGN;
const size_t RUST_CONST_HELPER_PAGE_SIZE = PAGE_SIZE;
const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC;
const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL;
const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT;
const gfp_t RUST_CONST_HELPER_GFP_NOWAIT = GFP_NOWAIT;
const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO;
const gfp_t RUST_CONST_HELPER___GFP_HIGHMEM = ___GFP_HIGHMEM;
const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN;
const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL;
const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET;
const xa_mark_t RUST_CONST_HELPER_XA_PRESENT = XA_PRESENT;
const gfp_t RUST_CONST_HELPER_XA_FLAGS_ALLOC = XA_FLAGS_ALLOC;
const gfp_t RUST_CONST_HELPER_XA_FLAGS_ALLOC1 = XA_FLAGS_ALLOC1;
const vm_flags_t RUST_CONST_HELPER_VM_MERGEABLE = VM_MERGEABLE;
const vm_flags_t RUST_CONST_HELPER_VM_READ = VM_READ;
const vm_flags_t RUST_CONST_HELPER_VM_WRITE = VM_WRITE;
const vm_flags_t RUST_CONST_HELPER_VM_EXEC = VM_EXEC;
const vm_flags_t RUST_CONST_HELPER_VM_SHARED = VM_SHARED;
const vm_flags_t RUST_CONST_HELPER_VM_MAYREAD = VM_MAYREAD;
const vm_flags_t RUST_CONST_HELPER_VM_MAYWRITE = VM_MAYWRITE;
const vm_flags_t RUST_CONST_HELPER_VM_MAYEXEC = VM_MAYEXEC;
const vm_flags_t RUST_CONST_HELPER_VM_MAYSHARE = VM_MAYEXEC;
const vm_flags_t RUST_CONST_HELPER_VM_PFNMAP = VM_PFNMAP;
const vm_flags_t RUST_CONST_HELPER_VM_IO = VM_IO;
const vm_flags_t RUST_CONST_HELPER_VM_DONTCOPY = VM_DONTCOPY;
const vm_flags_t RUST_CONST_HELPER_VM_DONTEXPAND = VM_DONTEXPAND;
const vm_flags_t RUST_CONST_HELPER_VM_LOCKONFAULT = VM_LOCKONFAULT;
const vm_flags_t RUST_CONST_HELPER_VM_ACCOUNT = VM_ACCOUNT;
const vm_flags_t RUST_CONST_HELPER_VM_NORESERVE = VM_NORESERVE;
const vm_flags_t RUST_CONST_HELPER_VM_HUGETLB = VM_HUGETLB;
const vm_flags_t RUST_CONST_HELPER_VM_SYNC = VM_SYNC;
const vm_flags_t RUST_CONST_HELPER_VM_ARCH_1 = VM_ARCH_1;
const vm_flags_t RUST_CONST_HELPER_VM_WIPEONFORK = VM_WIPEONFORK;
const vm_flags_t RUST_CONST_HELPER_VM_DONTDUMP = VM_DONTDUMP;
const vm_flags_t RUST_CONST_HELPER_VM_SOFTDIRTY = VM_SOFTDIRTY;
const vm_flags_t RUST_CONST_HELPER_VM_MIXEDMAP = VM_MIXEDMAP;
const vm_flags_t RUST_CONST_HELPER_VM_HUGEPAGE = VM_HUGEPAGE;
const vm_flags_t RUST_CONST_HELPER_VM_NOHUGEPAGE = VM_NOHUGEPAGE;
#if IS_ENABLED(CONFIG_GPU_BUDDY)
const unsigned long RUST_CONST_HELPER_GPU_BUDDY_RANGE_ALLOCATION = GPU_BUDDY_RANGE_ALLOCATION;
const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TOPDOWN_ALLOCATION = GPU_BUDDY_TOPDOWN_ALLOCATION;
const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CONTIGUOUS_ALLOCATION =
GPU_BUDDY_CONTIGUOUS_ALLOCATION;
const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CLEAR_ALLOCATION = GPU_BUDDY_CLEAR_ALLOCATION;
const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CLEARED = GPU_BUDDY_CLEARED;
const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TRIM_DISABLE = GPU_BUDDY_TRIM_DISABLE;
#endif
#if IS_ENABLED(CONFIG_ANDROID_BINDER_IPC_RUST)
#include "../../drivers/android/binder/rust_binder.h"
#include "../../drivers/android/binder/rust_binder_events.h"
#endif
linux-7.2.3/rust/bindings/lib.rs
+83
View File
@@ -0,0 +1,83 @@
// SPDX-License-Identifier: GPL-2.0
//! Bindings.
//!
//! Imports the generated bindings by `bindgen`.
//!
//! This crate may not be directly used. If you need a kernel C API that is
//! not ported or wrapped in the `kernel` crate, then do so first instead of
//! using this crate.
#![no_std]
#![allow(
clippy::all,
missing_docs,
non_camel_case_types,
non_upper_case_globals,
non_snake_case,
improper_ctypes,
unreachable_pub,
unsafe_op_in_unsafe_fn
)]
#![feature(cfi_encoding)]
#[allow(dead_code)]
#[allow(clippy::cast_lossless)]
#[allow(clippy::ptr_as_ptr)]
#[allow(clippy::ref_as_ptr)]
#[allow(clippy::undocumented_unsafe_blocks)]
#[cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))]
#[cfg_attr(
CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS,
allow(suspicious_runtime_symbol_definitions)
)]
mod bindings_raw {
use pin_init::{MaybeZeroable, Zeroable};
// Manual definition for blocklisted types.
type __kernel_size_t = usize;
type __kernel_ssize_t = isize;
type __kernel_ptrdiff_t = isize;
// `bindgen` doesn't automatically do this, see
// <https://github.com/rust-lang/rust-bindgen/issues/3196>
//
// SAFETY: `__BindgenBitfieldUnit<Storage>` is a newtype around `Storage`.
unsafe impl<Storage> Zeroable for __BindgenBitfieldUnit<Storage> where Storage: Zeroable {}
// Use glob import here to expose all helpers.
// Symbols defined within the module will take precedence to the glob import.
pub use super::bindings_helper::*;
include!(concat!(
env!("OBJTREE"),
"/rust/bindings/bindings_generated.rs"
));
}
// When both a directly exposed symbol and a helper exists for the same function,
// the directly exposed symbol is preferred and the helper becomes dead code, so
// ignore the warning here.
#[allow(dead_code)]
mod bindings_helper {
// Import the generated bindings for types.
use super::bindings_raw::*;
include!(concat!(
env!("OBJTREE"),
"/rust/bindings/bindings_helpers_generated.rs"
));
}
pub use bindings_raw::*;
pub const compat_ptr_ioctl: Option<
unsafe extern "C" fn(*mut file, ffi::c_uint, ffi::c_ulong) -> ffi::c_long,
> = {
#[cfg(CONFIG_COMPAT)]
{
Some(bindings_raw::compat_ptr_ioctl)
}
#[cfg(not(CONFIG_COMPAT))]
{
None
}
};
linux-7.2.3/rust/build_error.rs
+31
View File
@@ -0,0 +1,31 @@
// SPDX-License-Identifier: GPL-2.0
//! Build-time error.
//!
//! This crate provides a [const function][const-functions] `build_error`, which will panic in
//! compile-time if executed in [const context][const-context], and will cause a build error
//! if not executed at compile time and the optimizer does not optimise away the call.
//!
//! It is used by `build_assert!` in the kernel crate, allowing checking of
//! conditions that could be checked statically, but could not be enforced in
//! Rust yet (e.g. perform some checks in [const functions][const-functions], but those
//! functions could still be called in the runtime).
//!
//! For details on constant evaluation in Rust, please see the [Reference][const-eval].
//!
//! [const-eval]: https://doc.rust-lang.org/reference/const_eval.html
//! [const-functions]: https://doc.rust-lang.org/reference/const_eval.html#const-functions
//! [const-context]: https://doc.rust-lang.org/reference/const_eval.html#const-context
#![no_std]
/// Panics if executed in [const context][const-context], or triggers a build error if not.
///
/// [const-context]: https://doc.rust-lang.org/reference/const_eval.html#const-context
#[inline(never)]
#[cold]
#[export_name = "rust_build_error"]
#[track_caller]
pub const fn build_error(msg: &'static str) -> ! {
panic!("{}", msg);
}
linux-7.2.3/rust/compiler_builtins.rs
+101
View File
@@ -0,0 +1,101 @@
// SPDX-License-Identifier: GPL-2.0
//! Our own `compiler_builtins`.
//!
//! Rust provides [`compiler_builtins`] as a port of LLVM's [`compiler-rt`].
//! Since we do not need the vast majority of them, we avoid the dependency
//! by providing this file.
//!
//! At the moment, some builtins are required that should not be. For instance,
//! [`core`] has 128-bit integers functionality which we should not be compiling
//! in. We will work with upstream [`core`] to provide feature flags to disable
//! the parts we do not need. For the moment, we define them to [`panic!`] at
//! runtime for simplicity to catch mistakes, instead of performing surgery
//! on `core.o`.
//!
//! In any case, all these symbols are weakened to ensure we do not override
//! those that may be provided by the rest of the kernel.
//!
//! [`compiler_builtins`]: https://github.com/rust-lang/compiler-builtins
//! [`compiler-rt`]: https://compiler-rt.llvm.org/
#![allow(internal_features)]
#![feature(compiler_builtins)]
#![compiler_builtins]
#![no_builtins]
#![no_std]
macro_rules! define_panicking_intrinsics(
($reason: tt, { $($ident: ident, )* }) => {
$(
#[doc(hidden)]
#[export_name = concat!("__rust", stringify!($ident))]
pub extern "C" fn $ident() {
panic!($reason);
}
)*
}
);
define_panicking_intrinsics!("`f32` should not be used", {
__addsf3,
__eqsf2,
__extendsfdf2,
__gesf2,
__lesf2,
__ltsf2,
__mulsf3,
__nesf2,
__truncdfsf2,
__unordsf2,
});
define_panicking_intrinsics!("`f64` should not be used", {
__adddf3,
__eqdf2,
__ledf2,
__ltdf2,
__muldf3,
__unorddf2,
});
define_panicking_intrinsics!("`i128` should not be used", {
__ashrti3,
__muloti4,
__multi3,
});
define_panicking_intrinsics!("`u128` should not be used", {
__ashlti3,
__lshrti3,
__udivmodti4,
__udivti3,
__umodti3,
});
#[cfg(target_arch = "arm")]
define_panicking_intrinsics!("`f32` should not be used", {
__aeabi_fadd,
__aeabi_fmul,
__aeabi_fcmpeq,
__aeabi_fcmple,
__aeabi_fcmplt,
__aeabi_fcmpun,
});
#[cfg(target_arch = "arm")]
define_panicking_intrinsics!("`f64` should not be used", {
__aeabi_dadd,
__aeabi_dmul,
__aeabi_dcmple,
__aeabi_dcmplt,
__aeabi_dcmpun,
});
#[cfg(target_arch = "arm")]
define_panicking_intrinsics!("`u64` division/modulo should not be used", {
__aeabi_uldivmod,
});
// NOTE: if you are adding a new intrinsic here, you should also add it to
// `redirect-intrinsics` in `rust/Makefile`.
linux-7.2.3/rust/exports.c
+29
View File
@@ -0,0 +1,29 @@
// SPDX-License-Identifier: GPL-2.0
/*
* A hack to export Rust symbols for loadable modules without having to redo
* the entire `include/linux/export.h` logic in Rust.
*
* This requires Rust's new/future `v0` mangling scheme because the default one
* ("legacy") uses invalid characters for C identifiers (thus we cannot use the
* `EXPORT_SYMBOL_*` macros).
*
* All symbols are exported as GPL-only to guarantee no GPL-only feature is
* accidentally exposed.
*/
#include <linux/export.h>
#define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym)
#include "exports_core_generated.h"
#include "exports_bindings_generated.h"
#include "exports_kernel_generated.h"
#ifndef CONFIG_RUST_INLINE_HELPERS
#include "exports_helpers_generated.h"
#endif
// For modules using `rust/build_error.rs`.
#ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW
EXPORT_SYMBOL_RUST_GPL(rust_build_error);
#endif
linux-7.2.3/rust/ffi.rs
+50
View File
@@ -0,0 +1,50 @@
// SPDX-License-Identifier: GPL-2.0
//! Foreign function interface (FFI) types.
//!
//! This crate provides mapping from C primitive types to Rust ones.
//!
//! The Rust [`core`] crate provides [`core::ffi`], which maps integer types to the platform default
//! C ABI. The kernel does not use [`core::ffi`], so it can customise the mapping that deviates from
//! the platform default.
#![no_std]
macro_rules! alias {
($($name:ident = $ty:ty;)*) => {$(
#[allow(non_camel_case_types, missing_docs)]
pub type $name = $ty;
// Check size compatibility with `core`.
const _: () = assert!(
::core::mem::size_of::<$name>() == ::core::mem::size_of::<::core::ffi::$name>()
);
)*}
}
alias! {
// `core::ffi::c_char` is either `i8` or `u8` depending on architecture. In the kernel, we use
// `-funsigned-char` so it's always mapped to `u8`.
c_char = u8;
c_schar = i8;
c_uchar = u8;
c_short = i16;
c_ushort = u16;
c_int = i32;
c_uint = u32;
// In the kernel, `intptr_t` is defined to be `long` in all platforms, so we can map the type to
// `isize`.
c_long = isize;
c_ulong = usize;
c_longlong = i64;
c_ulonglong = u64;
}
pub use core::ffi::c_void;
pub use core::ffi::CStr;
linux-7.2.3/rust/helpers/acpi.c
+16
View File
@@ -0,0 +1,16 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/acpi.h>
#include <acpi/acpi_bus.h>
__rust_helper bool rust_helper_acpi_of_match_device(const struct acpi_device *adev,
const struct of_device_id *of_match_table,
const struct of_device_id **of_id)
{
return acpi_of_match_device(adev, of_match_table, of_id);
}
__rust_helper struct acpi_device *rust_helper_to_acpi_device_node(struct fwnode_handle *fwnode)
{
return to_acpi_device_node(fwnode);
}
linux-7.2.3/rust/helpers/atomic.c
+1035
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/rust/helpers/atomic_ext.c
+87
View File
@@ -0,0 +1,87 @@
// SPDX-License-Identifier: GPL-2.0
#include <asm/barrier.h>
#include <asm/rwonce.h>
#include <linux/atomic.h>
#define GEN_READ_HELPER(tname, type) \
__rust_helper type rust_helper_atomic_##tname##_read(type *ptr) \
{ \
return READ_ONCE(*ptr); \
}
#define GEN_SET_HELPER(tname, type) \
__rust_helper void rust_helper_atomic_##tname##_set(type *ptr, type val) \
{ \
WRITE_ONCE(*ptr, val); \
}
#define GEN_READ_ACQUIRE_HELPER(tname, type) \
__rust_helper type rust_helper_atomic_##tname##_read_acquire(type *ptr) \
{ \
return smp_load_acquire(ptr); \
}
#define GEN_SET_RELEASE_HELPER(tname, type) \
__rust_helper void rust_helper_atomic_##tname##_set_release(type *ptr, type val)\
{ \
smp_store_release(ptr, val); \
}
#define GEN_READ_SET_HELPERS(tname, type) \
GEN_READ_HELPER(tname, type) \
GEN_SET_HELPER(tname, type) \
GEN_READ_ACQUIRE_HELPER(tname, type) \
GEN_SET_RELEASE_HELPER(tname, type) \
GEN_READ_SET_HELPERS(i8, s8)
GEN_READ_SET_HELPERS(i16, s16)
GEN_READ_SET_HELPERS(ptr, const void *)
/*
* xchg helpers depend on ARCH_SUPPORTS_ATOMIC_RMW and on the
* architecture provding xchg() support for i8 and i16.
*
* The architectures that currently support Rust (x86_64, armv7,
* arm64, riscv, and loongarch) satisfy these requirements.
*/
#define GEN_XCHG_HELPER(tname, type, suffix) \
__rust_helper type \
rust_helper_atomic_##tname##_xchg##suffix(type *ptr, type new) \
{ \
return xchg##suffix(ptr, new); \
}
#define GEN_XCHG_HELPERS(tname, type) \
GEN_XCHG_HELPER(tname, type, ) \
GEN_XCHG_HELPER(tname, type, _acquire) \
GEN_XCHG_HELPER(tname, type, _release) \
GEN_XCHG_HELPER(tname, type, _relaxed) \
GEN_XCHG_HELPERS(i8, s8)
GEN_XCHG_HELPERS(i16, s16)
GEN_XCHG_HELPERS(ptr, const void *)
/*
* try_cmpxchg helpers depend on ARCH_SUPPORTS_ATOMIC_RMW and on the
* architecture provding try_cmpxchg() support for i8 and i16.
*
* The architectures that currently support Rust (x86_64, armv7,
* arm64, riscv, and loongarch) satisfy these requirements.
*/
#define GEN_TRY_CMPXCHG_HELPER(tname, type, suffix) \
__rust_helper bool \
rust_helper_atomic_##tname##_try_cmpxchg##suffix(type *ptr, type *old, type new)\
{ \
return try_cmpxchg##suffix(ptr, old, new); \
}
#define GEN_TRY_CMPXCHG_HELPERS(tname, type) \
GEN_TRY_CMPXCHG_HELPER(tname, type, ) \
GEN_TRY_CMPXCHG_HELPER(tname, type, _acquire) \
GEN_TRY_CMPXCHG_HELPER(tname, type, _release) \
GEN_TRY_CMPXCHG_HELPER(tname, type, _relaxed) \
GEN_TRY_CMPXCHG_HELPERS(i8, s8)
GEN_TRY_CMPXCHG_HELPERS(i16, s16)
GEN_TRY_CMPXCHG_HELPERS(ptr, const void *)
linux-7.2.3/rust/helpers/auxiliary.c
+15
View File
@@ -0,0 +1,15 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/auxiliary_bus.h>
__rust_helper void
rust_helper_auxiliary_device_uninit(struct auxiliary_device *adev)
{
return auxiliary_device_uninit(adev);
}
__rust_helper void
rust_helper_auxiliary_device_delete(struct auxiliary_device *adev)
{
return auxiliary_device_delete(adev);
}
linux-7.2.3/rust/helpers/barrier.c
+18
View File
@@ -0,0 +1,18 @@
// SPDX-License-Identifier: GPL-2.0
#include <asm/barrier.h>
__rust_helper void rust_helper_smp_mb(void)
{
smp_mb();
}
__rust_helper void rust_helper_smp_wmb(void)
{
smp_wmb();
}
__rust_helper void rust_helper_smp_rmb(void)
{
smp_rmb();
}
linux-7.2.3/rust/helpers/binder.c
+27
View File
@@ -0,0 +1,27 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2025 Google LLC.
*/
#include <linux/list_lru.h>
#include <linux/task_work.h>
__rust_helper unsigned long rust_helper_list_lru_count(struct list_lru *lru)
{
return list_lru_count(lru);
}
__rust_helper unsigned long rust_helper_list_lru_walk(struct list_lru *lru,
list_lru_walk_cb isolate,
void *cb_arg,
unsigned long nr_to_walk)
{
return list_lru_walk(lru, isolate, cb_arg, nr_to_walk);
}
__rust_helper void rust_helper_init_task_work(struct callback_head *twork,
task_work_func_t func)
{
init_task_work(twork, func);
}
linux-7.2.3/rust/helpers/bitmap.c
+10
View File
@@ -0,0 +1,10 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/bitmap.h>
__rust_helper
void rust_helper_bitmap_copy_and_extend(unsigned long *to, const unsigned long *from,
unsigned int count, unsigned int size)
{
bitmap_copy_and_extend(to, from, count, size);
}
linux-7.2.3/rust/helpers/bitops.c
+69
View File
@@ -0,0 +1,69 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/bitops.h>
#include <linux/find.h>
__rust_helper
void rust_helper___set_bit(unsigned long nr, unsigned long *addr)
{
__set_bit(nr, addr);
}
__rust_helper
void rust_helper___clear_bit(unsigned long nr, unsigned long *addr)
{
__clear_bit(nr, addr);
}
__rust_helper
void rust_helper_set_bit(unsigned long nr, volatile unsigned long *addr)
{
set_bit(nr, addr);
}
__rust_helper
void rust_helper_clear_bit(unsigned long nr, volatile unsigned long *addr)
{
clear_bit(nr, addr);
}
/*
* The rust_helper_ prefix is intentionally omitted below so that the
* declarations in include/linux/find.h are compatible with these helpers.
*
* Note that the below #ifdefs mean that the helper is only created if C does
* not provide a definition.
*/
#ifdef find_first_zero_bit
__rust_helper
unsigned long _find_first_zero_bit(const unsigned long *p, unsigned long size)
{
return find_first_zero_bit(p, size);
}
#endif /* find_first_zero_bit */
#ifdef find_next_zero_bit
__rust_helper
unsigned long _find_next_zero_bit(const unsigned long *addr,
unsigned long size, unsigned long offset)
{
return find_next_zero_bit(addr, size, offset);
}
#endif /* find_next_zero_bit */
#ifdef find_first_bit
__rust_helper
unsigned long _find_first_bit(const unsigned long *addr, unsigned long size)
{
return find_first_bit(addr, size);
}
#endif /* find_first_bit */
#ifdef find_next_bit
__rust_helper
unsigned long _find_next_bit(const unsigned long *addr, unsigned long size,
unsigned long offset)
{
return find_next_bit(addr, size, offset);
}
#endif /* find_next_bit */
linux-7.2.3/rust/helpers/blk.c
+14
View File
@@ -0,0 +1,14 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/blk-mq.h>
#include <linux/blkdev.h>
__rust_helper void *rust_helper_blk_mq_rq_to_pdu(struct request *rq)
{
return blk_mq_rq_to_pdu(rq);
}
__rust_helper struct request *rust_helper_blk_mq_rq_from_pdu(void *pdu)
{
return blk_mq_rq_from_pdu(pdu);
}
linux-7.2.3/rust/helpers/bug.c
+13
View File
@@ -0,0 +1,13 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/bug.h>
__rust_helper __noreturn void rust_helper_BUG(void)
{
BUG();
}
__rust_helper bool rust_helper_WARN_ON(bool cond)
{
return WARN_ON(cond);
}
linux-7.2.3/rust/helpers/build_assert.c
+25
View File
@@ -0,0 +1,25 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/build_bug.h>
/*
* `bindgen` binds the C `size_t` type as the Rust `usize` type, so we can
* use it in contexts where Rust expects a `usize` like slice (array) indices.
* `usize` is defined to be the same as C's `uintptr_t` type (can hold any
* pointer) but not necessarily the same as `size_t` (can hold the size of any
* single object). Most modern platforms use the same concrete integer type for
* both of them, but in case we find ourselves on a platform where
* that's not true, fail early instead of risking ABI or
* integer-overflow issues.
*
* If your platform fails this assertion, it means that you are in
* danger of integer-overflow bugs (even if you attempt to add
* `--no-size_t-is-usize`). It may be easiest to change the kernel ABI on
* your platform such that `size_t` matches `uintptr_t` (i.e., to increase
* `size_t`, because `uintptr_t` has to be at least as big as `size_t`).
*/
static_assert(
sizeof(size_t) == sizeof(uintptr_t) &&
__alignof__(size_t) == __alignof__(uintptr_t),
"Rust code expects C `size_t` to match Rust `usize`"
);
linux-7.2.3/rust/helpers/build_bug.c
+8
View File
@@ -0,0 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/errname.h>
__rust_helper const char *rust_helper_errname(int err)
{
return errname(err);
}
linux-7.2.3/rust/helpers/clk.c
+68
View File
@@ -0,0 +1,68 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/clk.h>
/*
* The "inline" implementation of below helpers are only available when
* CONFIG_HAVE_CLK or CONFIG_HAVE_CLK_PREPARE aren't set.
*/
#ifndef CONFIG_HAVE_CLK
__rust_helper struct clk *rust_helper_clk_get(struct device *dev,
const char *id)
{
return clk_get(dev, id);
}
__rust_helper void rust_helper_clk_put(struct clk *clk)
{
clk_put(clk);
}
__rust_helper int rust_helper_clk_enable(struct clk *clk)
{
return clk_enable(clk);
}
__rust_helper void rust_helper_clk_disable(struct clk *clk)
{
clk_disable(clk);
}
__rust_helper unsigned long rust_helper_clk_get_rate(struct clk *clk)
{
return clk_get_rate(clk);
}
__rust_helper int rust_helper_clk_set_rate(struct clk *clk, unsigned long rate)
{
return clk_set_rate(clk, rate);
}
#endif
#ifndef CONFIG_HAVE_CLK_PREPARE
__rust_helper int rust_helper_clk_prepare(struct clk *clk)
{
return clk_prepare(clk);
}
__rust_helper void rust_helper_clk_unprepare(struct clk *clk)
{
clk_unprepare(clk);
}
#endif
__rust_helper struct clk *rust_helper_clk_get_optional(struct device *dev,
const char *id)
{
return clk_get_optional(dev, id);
}
__rust_helper int rust_helper_clk_prepare_enable(struct clk *clk)
{
return clk_prepare_enable(clk);
}
__rust_helper void rust_helper_clk_disable_unprepare(struct clk *clk)
{
clk_disable_unprepare(clk);
}
linux-7.2.3/rust/helpers/completion.c
+8
View File
@@ -0,0 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/completion.h>
__rust_helper void rust_helper_init_completion(struct completion *x)
{
init_completion(x);
}
linux-7.2.3/rust/helpers/cpu.c
+8
View File
@@ -0,0 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/smp.h>
__rust_helper unsigned int rust_helper_raw_smp_processor_id(void)
{
return raw_smp_processor_id();
}
linux-7.2.3/rust/helpers/cpufreq.c
+11
View File
@@ -0,0 +1,11 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/cpufreq.h>
#ifdef CONFIG_CPU_FREQ
__rust_helper void
rust_helper_cpufreq_register_em_with_opp(struct cpufreq_policy *policy)
{
cpufreq_register_em_with_opp(policy);
}
#endif
linux-7.2.3/rust/helpers/cpumask.c
+83
View File
@@ -0,0 +1,83 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/cpumask.h>
__rust_helper
void rust_helper_cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
{
cpumask_set_cpu(cpu, dstp);
}
__rust_helper
void rust_helper___cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
{
__cpumask_set_cpu(cpu, dstp);
}
__rust_helper
void rust_helper_cpumask_clear_cpu(int cpu, struct cpumask *dstp)
{
cpumask_clear_cpu(cpu, dstp);
}
__rust_helper
void rust_helper___cpumask_clear_cpu(int cpu, struct cpumask *dstp)
{
__cpumask_clear_cpu(cpu, dstp);
}
__rust_helper
bool rust_helper_cpumask_test_cpu(int cpu, struct cpumask *srcp)
{
return cpumask_test_cpu(cpu, srcp);
}
__rust_helper
void rust_helper_cpumask_setall(struct cpumask *dstp)
{
cpumask_setall(dstp);
}
__rust_helper
bool rust_helper_cpumask_empty(struct cpumask *srcp)
{
return cpumask_empty(srcp);
}
__rust_helper
bool rust_helper_cpumask_full(struct cpumask *srcp)
{
return cpumask_full(srcp);
}
__rust_helper
unsigned int rust_helper_cpumask_weight(struct cpumask *srcp)
{
return cpumask_weight(srcp);
}
__rust_helper
void rust_helper_cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp)
{
cpumask_copy(dstp, srcp);
}
__rust_helper
bool rust_helper_alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
{
return alloc_cpumask_var(mask, flags);
}
__rust_helper
bool rust_helper_zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
{
return zalloc_cpumask_var(mask, flags);
}
#ifndef CONFIG_CPUMASK_OFFSTACK
__rust_helper
void rust_helper_free_cpumask_var(cpumask_var_t mask)
{
free_cpumask_var(mask);
}
#endif
linux-7.2.3/rust/helpers/cred.c
+13
View File
@@ -0,0 +1,13 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/cred.h>
__rust_helper const struct cred *rust_helper_get_cred(const struct cred *cred)
{
return get_cred(cred);
}
__rust_helper void rust_helper_put_cred(const struct cred *cred)
{
put_cred(cred);
}
linux-7.2.3/rust/helpers/device.c
+32
View File
@@ -0,0 +1,32 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/device.h>
__rust_helper int rust_helper_devm_add_action(struct device *dev,
void (*action)(void *),
void *data)
{
return devm_add_action(dev, action, data);
}
__rust_helper int rust_helper_devm_add_action_or_reset(struct device *dev,
void (*action)(void *),
void *data)
{
return devm_add_action_or_reset(dev, action, data);
}
__rust_helper void *rust_helper_dev_get_drvdata(const struct device *dev)
{
return dev_get_drvdata(dev);
}
__rust_helper void rust_helper_dev_set_drvdata(struct device *dev, void *data)
{
dev_set_drvdata(dev, data);
}
__rust_helper const char *rust_helper_dev_name(const struct device *dev)
{
return dev_name(dev);
}
linux-7.2.3/rust/helpers/dma-resv.c
+14
View File
@@ -0,0 +1,14 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/dma-resv.h>
__rust_helper
int rust_helper_dma_resv_lock(struct dma_resv *obj, struct ww_acquire_ctx *ctx)
{
return dma_resv_lock(obj, ctx);
}
__rust_helper void rust_helper_dma_resv_unlock(struct dma_resv *obj)
{
dma_resv_unlock(obj);
}
linux-7.2.3/rust/helpers/dma.c
+51
View File
@@ -0,0 +1,51 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/dma-mapping.h>
__rust_helper void *rust_helper_dma_alloc_attrs(struct device *dev, size_t size,
dma_addr_t *dma_handle,
gfp_t flag, unsigned long attrs)
{
return dma_alloc_attrs(dev, size, dma_handle, flag, attrs);
}
__rust_helper void rust_helper_dma_free_attrs(struct device *dev, size_t size,
void *cpu_addr,
dma_addr_t dma_handle,
unsigned long attrs)
{
dma_free_attrs(dev, size, cpu_addr, dma_handle, attrs);
}
__rust_helper int rust_helper_dma_set_mask_and_coherent(struct device *dev,
u64 mask)
{
return dma_set_mask_and_coherent(dev, mask);
}
__rust_helper int rust_helper_dma_set_mask(struct device *dev, u64 mask)
{
return dma_set_mask(dev, mask);
}
__rust_helper int rust_helper_dma_set_coherent_mask(struct device *dev, u64 mask)
{
return dma_set_coherent_mask(dev, mask);
}
__rust_helper int rust_helper_dma_map_sgtable(struct device *dev, struct sg_table *sgt,
enum dma_data_direction dir, unsigned long attrs)
{
return dma_map_sgtable(dev, sgt, dir, attrs);
}
__rust_helper size_t rust_helper_dma_max_mapping_size(struct device *dev)
{
return dma_max_mapping_size(dev);
}
__rust_helper void rust_helper_dma_set_max_seg_size(struct device *dev,
unsigned int size)
{
dma_set_max_seg_size(dev, size);
}
linux-7.2.3/rust/helpers/drm.c
+78
View File
@@ -0,0 +1,78 @@
// SPDX-License-Identifier: GPL-2.0
#include <drm/drm_gem.h>
#include <drm/drm_gem_shmem_helper.h>
#include <drm/drm_vma_manager.h>
#ifdef CONFIG_DRM
__rust_helper void rust_helper_drm_gem_object_get(struct drm_gem_object *obj)
{
drm_gem_object_get(obj);
}
__rust_helper void rust_helper_drm_gem_object_put(struct drm_gem_object *obj)
{
drm_gem_object_put(obj);
}
__rust_helper __u64
rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node)
{
return drm_vma_node_offset_addr(node);
}
#ifdef CONFIG_DRM_GEM_SHMEM_HELPER
__rust_helper void
rust_helper_drm_gem_shmem_object_free(struct drm_gem_object *obj)
{
return drm_gem_shmem_object_free(obj);
}
__rust_helper void
rust_helper_drm_gem_shmem_object_print_info(struct drm_printer *p, unsigned int indent,
const struct drm_gem_object *obj)
{
drm_gem_shmem_object_print_info(p, indent, obj);
}
__rust_helper int
rust_helper_drm_gem_shmem_object_pin(struct drm_gem_object *obj)
{
return drm_gem_shmem_object_pin(obj);
}
__rust_helper void
rust_helper_drm_gem_shmem_object_unpin(struct drm_gem_object *obj)
{
drm_gem_shmem_object_unpin(obj);
}
__rust_helper struct sg_table *
rust_helper_drm_gem_shmem_object_get_sg_table(struct drm_gem_object *obj)
{
return drm_gem_shmem_object_get_sg_table(obj);
}
__rust_helper int
rust_helper_drm_gem_shmem_object_vmap(struct drm_gem_object *obj,
struct iosys_map *map)
{
return drm_gem_shmem_object_vmap(obj, map);
}
__rust_helper void
rust_helper_drm_gem_shmem_object_vunmap(struct drm_gem_object *obj,
struct iosys_map *map)
{
drm_gem_shmem_object_vunmap(obj, map);
}
__rust_helper int
rust_helper_drm_gem_shmem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
{
return drm_gem_shmem_object_mmap(obj, vma);
}
#endif /* CONFIG_DRM_GEM_SHMEM_HELPER */
#endif /* CONFIG_DRM */
linux-7.2.3/rust/helpers/drm_gpuvm.c
+26
View File
@@ -0,0 +1,26 @@
// SPDX-License-Identifier: GPL-2.0 or MIT
#ifdef CONFIG_RUST_DRM_GPUVM
#include <drm/drm_gpuvm.h>
__rust_helper
struct drm_gpuvm_bo *rust_helper_drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo)
{
return drm_gpuvm_bo_get(vm_bo);
}
__rust_helper
struct drm_gpuvm *rust_helper_drm_gpuvm_get(struct drm_gpuvm *obj)
{
return drm_gpuvm_get(obj);
}
__rust_helper
bool rust_helper_drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm,
struct drm_gem_object *obj)
{
return drm_gpuvm_is_extobj(gpuvm, obj);
}
#endif // CONFIG_RUST_DRM_GPUVM
linux-7.2.3/rust/helpers/err.c
+18
View File
@@ -0,0 +1,18 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/err.h>
__rust_helper __force void *rust_helper_ERR_PTR(long err)
{
return ERR_PTR(err);
}
__rust_helper bool rust_helper_IS_ERR(__force const void *ptr)
{
return IS_ERR(ptr);
}
__rust_helper long rust_helper_PTR_ERR(__force const void *ptr)
{
return PTR_ERR(ptr);
}
linux-7.2.3/rust/helpers/fs.c
+12
View File
@@ -0,0 +1,12 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2024 Google LLC.
*/
#include <linux/fs.h>
__rust_helper struct file *rust_helper_get_file(struct file *f)
{
return get_file(f);
}
linux-7.2.3/rust/helpers/gpu.c
+17
View File
@@ -0,0 +1,17 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/gpu_buddy.h>
#ifdef CONFIG_GPU_BUDDY
__rust_helper u64 rust_helper_gpu_buddy_block_offset(const struct gpu_buddy_block *block)
{
return gpu_buddy_block_offset(block);
}
__rust_helper unsigned int rust_helper_gpu_buddy_block_order(struct gpu_buddy_block *block)
{
return gpu_buddy_block_order(block);
}
#endif /* CONFIG_GPU_BUDDY */
linux-7.2.3/rust/helpers/helpers.c
+102
View File
@@ -0,0 +1,102 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Non-trivial C macros cannot be used in Rust. Similarly, inlined C functions
* cannot be called either. This file explicitly creates functions ("helpers")
* that wrap those so that they can be called from Rust.
*
* Sorted alphabetically.
*/
#include <linux/compiler_types.h>
#ifdef __BINDGEN__
// Omit `inline` for bindgen as it ignores inline functions.
#define __rust_helper
#else
// The helper functions are all inline functions.
//
// We use `__always_inline` here to bypass LLVM inlining checks, in case the
// helpers are inlined directly into Rust CGUs.
//
// The LLVM inlining checks are false positives:
// * LLVM doesn't want to inline functions compiled with
// `-fno-delete-null-pointer-checks` with code compiled without.
// The C CGUs all have this enabled and Rust CGUs don't. Inlining is okay
// since this is one of the hardening features that does not change the ABI,
// and we shouldn't have null pointer dereferences in these helpers.
// * LLVM doesn't want to inline functions with different list of builtins. C
// side has `-fno-builtin-wcslen`; `wcslen` is not a Rust builtin, so they
// should be compatible, but LLVM does not perform inlining due to attributes
// mismatch.
// * clang and Rust doesn't have the exact target string. Clang generates
// `+cmov,+cx8,+fxsr` but Rust doesn't enable them (in fact, Rust will
// complain if `-Ctarget-feature=+cmov,+cx8,+fxsr` is used). x86-64 always
// enable these features, so they are in fact the same target string, but
// LLVM doesn't understand this and so inlining is inhibited. This can be
// bypassed with `--ignore-tti-inline-compatible`, but this is a hidden
// option.
#define __rust_helper __always_inline
#endif
#include "acpi.c"
#include "atomic.c"
#include "atomic_ext.c"
#include "auxiliary.c"
#include "barrier.c"
#include "binder.c"
#include "bitmap.c"
#include "bitops.c"
#include "blk.c"
#include "bug.c"
#include "build_assert.c"
#include "build_bug.c"
#include "clk.c"
#include "completion.c"
#include "cpu.c"
#include "cpufreq.c"
#include "cpumask.c"
#include "cred.c"
#include "device.c"
#include "dma.c"
#include "dma-resv.c"
#include "drm.c"
#include "drm_gpuvm.c"
#include "err.c"
#include "irq.c"
#include "fs.c"
#include "gpu.c"
#include "io.c"
#include "jump_label.c"
#include "kunit.c"
#include "list.c"
#include "maple_tree.c"
#include "mm.c"
#include "mutex.c"
#include "of.c"
#include "page.c"
#include "pci.c"
#include "pid_namespace.c"
#include "platform.c"
#include "poll.c"
#include "processor.c"
#include "property.c"
#include "pwm.c"
#include "rbtree.c"
#include "rcu.c"
#include "refcount.c"
#include "regulator.c"
#include "scatterlist.c"
#include "security.c"
#include "signal.c"
#include "slab.c"
#include "spinlock.c"
#include "string.c"
#include "sync.c"
#include "task.c"
#include "time.c"
#include "uaccess.c"
#include "usb.c"
#include "vmalloc.c"
#include "wait.c"
#include "workqueue.c"
#include "xarray.c"
linux-7.2.3/rust/helpers/io.c
+148
View File
@@ -0,0 +1,148 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/io.h>
#include <linux/ioport.h>
#ifdef CONFIG_HAS_IOMEM
__rust_helper void __iomem *rust_helper_ioremap(phys_addr_t offset, size_t size)
{
return ioremap(offset, size);
}
__rust_helper void __iomem *rust_helper_ioremap_np(phys_addr_t offset,
size_t size)
{
return ioremap_np(offset, size);
}
__rust_helper void rust_helper_iounmap(void __iomem *addr)
{
iounmap(addr);
}
#endif /* CONFIG_HAS_IOMEM */
__rust_helper u8 rust_helper_readb(const void __iomem *addr)
{
return readb(addr);
}
__rust_helper u16 rust_helper_readw(const void __iomem *addr)
{
return readw(addr);
}
__rust_helper u32 rust_helper_readl(const void __iomem *addr)
{
return readl(addr);
}
#ifdef CONFIG_64BIT
__rust_helper u64 rust_helper_readq(const void __iomem *addr)
{
return readq(addr);
}
#endif
__rust_helper void rust_helper_writeb(u8 value, void __iomem *addr)
{
writeb(value, addr);
}
__rust_helper void rust_helper_writew(u16 value, void __iomem *addr)
{
writew(value, addr);
}
__rust_helper void rust_helper_writel(u32 value, void __iomem *addr)
{
writel(value, addr);
}
#ifdef CONFIG_64BIT
__rust_helper void rust_helper_writeq(u64 value, void __iomem *addr)
{
writeq(value, addr);
}
#endif
__rust_helper u8 rust_helper_readb_relaxed(const void __iomem *addr)
{
return readb_relaxed(addr);
}
__rust_helper u16 rust_helper_readw_relaxed(const void __iomem *addr)
{
return readw_relaxed(addr);
}
__rust_helper u32 rust_helper_readl_relaxed(const void __iomem *addr)
{
return readl_relaxed(addr);
}
#ifdef CONFIG_64BIT
__rust_helper u64 rust_helper_readq_relaxed(const void __iomem *addr)
{
return readq_relaxed(addr);
}
#endif
__rust_helper void rust_helper_writeb_relaxed(u8 value, void __iomem *addr)
{
writeb_relaxed(value, addr);
}
__rust_helper void rust_helper_writew_relaxed(u16 value, void __iomem *addr)
{
writew_relaxed(value, addr);
}
__rust_helper void rust_helper_writel_relaxed(u32 value, void __iomem *addr)
{
writel_relaxed(value, addr);
}
#ifdef CONFIG_64BIT
__rust_helper void rust_helper_writeq_relaxed(u64 value, void __iomem *addr)
{
writeq_relaxed(value, addr);
}
#endif
__rust_helper resource_size_t rust_helper_resource_size(struct resource *res)
{
return resource_size(res);
}
__rust_helper struct resource *
rust_helper_request_mem_region(resource_size_t start, resource_size_t n,
const char *name)
{
return request_mem_region(start, n, name);
}
__rust_helper void rust_helper_release_mem_region(resource_size_t start,
resource_size_t n)
{
release_mem_region(start, n);
}
__rust_helper struct resource *rust_helper_request_region(resource_size_t start,
resource_size_t n,
const char *name)
{
return request_region(start, n, name);
}
__rust_helper struct resource *
rust_helper_request_muxed_region(resource_size_t start, resource_size_t n,
const char *name)
{
return request_muxed_region(start, n, name);
}
__rust_helper void rust_helper_release_region(resource_size_t start,
resource_size_t n)
{
release_region(start, n);
}
linux-7.2.3/rust/helpers/irq.c
+11
View File
@@ -0,0 +1,11 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/interrupt.h>
__rust_helper int rust_helper_request_irq(unsigned int irq,
irq_handler_t handler,
unsigned long flags, const char *name,
void *dev)
{
return request_irq(irq, handler, flags, name, dev);
}
linux-7.2.3/rust/helpers/jump_label.c
+14
View File
@@ -0,0 +1,14 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2024 Google LLC.
*/
#include <linux/jump_label.h>
#ifndef CONFIG_JUMP_LABEL
__rust_helper int rust_helper_static_key_count(struct static_key *key)
{
return static_key_count(key);
}
#endif
linux-7.2.3/rust/helpers/kunit.c
+8
View File
@@ -0,0 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
#include <kunit/test-bug.h>
__rust_helper struct kunit *rust_helper_kunit_get_current_test(void)
{
return kunit_get_current_test();
}
linux-7.2.3/rust/helpers/list.c
+17
View File
@@ -0,0 +1,17 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Helpers for C circular doubly linked list implementation.
*/
#include <linux/list.h>
__rust_helper void rust_helper_INIT_LIST_HEAD(struct list_head *list)
{
INIT_LIST_HEAD(list);
}
__rust_helper void rust_helper_list_add_tail(struct list_head *new, struct list_head *head)
{
list_add_tail(new, head);
}
linux-7.2.3/rust/helpers/maple_tree.c
+9
View File
@@ -0,0 +1,9 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/maple_tree.h>
__rust_helper void rust_helper_mt_init_flags(struct maple_tree *mt,
unsigned int flags)
{
mt_init_flags(mt, flags);
}
linux-7.2.3/rust/helpers/mm.c
+50
View File
@@ -0,0 +1,50 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/mm.h>
#include <linux/sched/mm.h>
__rust_helper void rust_helper_mmgrab(struct mm_struct *mm)
{
mmgrab(mm);
}
__rust_helper void rust_helper_mmdrop(struct mm_struct *mm)
{
mmdrop(mm);
}
__rust_helper void rust_helper_mmget(struct mm_struct *mm)
{
mmget(mm);
}
__rust_helper bool rust_helper_mmget_not_zero(struct mm_struct *mm)
{
return mmget_not_zero(mm);
}
__rust_helper void rust_helper_mmap_read_lock(struct mm_struct *mm)
{
mmap_read_lock(mm);
}
__rust_helper bool rust_helper_mmap_read_trylock(struct mm_struct *mm)
{
return mmap_read_trylock(mm);
}
__rust_helper void rust_helper_mmap_read_unlock(struct mm_struct *mm)
{
mmap_read_unlock(mm);
}
__rust_helper struct vm_area_struct *
rust_helper_vma_lookup(struct mm_struct *mm, unsigned long addr)
{
return vma_lookup(mm, addr);
}
__rust_helper void rust_helper_vma_end_read(struct vm_area_struct *vma)
{
vma_end_read(vma);
}
linux-7.2.3/rust/helpers/mutex.c
+30
View File
@@ -0,0 +1,30 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/mutex.h>
__rust_helper void rust_helper_mutex_lock(struct mutex *lock)
{
mutex_lock(lock);
}
__rust_helper int rust_helper_mutex_trylock(struct mutex *lock)
{
return mutex_trylock(lock);
}
__rust_helper void rust_helper___mutex_init(struct mutex *mutex,
const char *name,
struct lock_class_key *key)
{
__mutex_init(mutex, name, key);
}
__rust_helper void rust_helper_mutex_assert_is_held(struct mutex *mutex)
{
lockdep_assert_held(mutex);
}
__rust_helper void rust_helper_mutex_destroy(struct mutex *lock)
{
mutex_destroy(lock);
}
linux-7.2.3/rust/helpers/of.c
+8
View File
@@ -0,0 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/of.h>
__rust_helper bool rust_helper_is_of_node(const struct fwnode_handle *fwnode)
{
return is_of_node(fwnode);
}
linux-7.2.3/rust/helpers/page.c
+28
View File
@@ -0,0 +1,28 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/gfp.h>
#include <linux/highmem.h>
#include <linux/mm.h>
__rust_helper struct page *rust_helper_alloc_pages(gfp_t gfp_mask,
unsigned int order)
{
return alloc_pages(gfp_mask, order);
}
__rust_helper void *rust_helper_kmap_local_page(struct page *page)
{
return kmap_local_page(page);
}
__rust_helper void rust_helper_kunmap_local(const void *addr)
{
kunmap_local(addr);
}
#ifndef NODE_NOT_IN_PAGE_FLAGS
__rust_helper int rust_helper_page_to_nid(const struct page *page)
{
return page_to_nid(page);
}
#endif
linux-7.2.3/rust/helpers/pci.c
+46
View File
@@ -0,0 +1,46 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/pci.h>
__rust_helper u16 rust_helper_pci_dev_id(struct pci_dev *dev)
{
return PCI_DEVID(dev->bus->number, dev->devfn);
}
__rust_helper resource_size_t
rust_helper_pci_resource_start(struct pci_dev *pdev, int bar)
{
return pci_resource_start(pdev, bar);
}
__rust_helper resource_size_t rust_helper_pci_resource_len(struct pci_dev *pdev,
int bar)
{
return pci_resource_len(pdev, bar);
}
__rust_helper bool rust_helper_dev_is_pci(const struct device *dev)
{
return dev_is_pci(dev);
}
#ifndef CONFIG_PCI_MSI
__rust_helper int rust_helper_pci_alloc_irq_vectors(struct pci_dev *dev,
unsigned int min_vecs,
unsigned int max_vecs,
unsigned int flags)
{
return pci_alloc_irq_vectors(dev, min_vecs, max_vecs, flags);
}
__rust_helper void rust_helper_pci_free_irq_vectors(struct pci_dev *dev)
{
pci_free_irq_vectors(dev);
}
__rust_helper int rust_helper_pci_irq_vector(struct pci_dev *pdev,
unsigned int nvec)
{
return pci_irq_vector(pdev, nvec);
}
#endif
linux-7.2.3/rust/helpers/pid_namespace.c
+28
View File
@@ -0,0 +1,28 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/pid_namespace.h>
#include <linux/cleanup.h>
__rust_helper struct pid_namespace *
rust_helper_get_pid_ns(struct pid_namespace *ns)
{
return get_pid_ns(ns);
}
__rust_helper void rust_helper_put_pid_ns(struct pid_namespace *ns)
{
put_pid_ns(ns);
}
/* Get a reference on a task's pid namespace. */
__rust_helper struct pid_namespace *
rust_helper_task_get_pid_ns(struct task_struct *task)
{
struct pid_namespace *pid_ns;
guard(rcu)();
pid_ns = task_active_pid_ns(task);
if (pid_ns)
get_pid_ns(pid_ns);
return pid_ns;
}
linux-7.2.3/rust/helpers/platform.c
+8
View File
@@ -0,0 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/platform_device.h>
__rust_helper bool rust_helper_dev_is_platform(const struct device *dev)
{
return dev_is_platform(dev);
}
linux-7.2.3/rust/helpers/poll.c
+11
View File
@@ -0,0 +1,11 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/export.h>
#include <linux/poll.h>
__rust_helper void rust_helper_poll_wait(struct file *filp,
wait_queue_head_t *wait_address,
poll_table *p)
{
poll_wait(filp, wait_address, p);
}
linux-7.2.3/rust/helpers/processor.c
+8
View File
@@ -0,0 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/processor.h>
__rust_helper void rust_helper_cpu_relax(void)
{
cpu_relax();
}
linux-7.2.3/rust/helpers/property.c
+8
View File
@@ -0,0 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/property.h>
__rust_helper void rust_helper_fwnode_handle_put(struct fwnode_handle *fwnode)
{
fwnode_handle_put(fwnode);
}
linux-7.2.3/rust/helpers/pwm.c
+20
View File
@@ -0,0 +1,20 @@
// SPDX-License-Identifier: GPL-2.0
// Copyright (c) 2025 Samsung Electronics Co., Ltd.
// Author: Michal Wilczynski <m.wilczynski@samsung.com>
#include <linux/pwm.h>
__rust_helper struct device *rust_helper_pwmchip_parent(const struct pwm_chip *chip)
{
return pwmchip_parent(chip);
}
__rust_helper void *rust_helper_pwmchip_get_drvdata(struct pwm_chip *chip)
{
return pwmchip_get_drvdata(chip);
}
__rust_helper void rust_helper_pwmchip_set_drvdata(struct pwm_chip *chip, void *data)
{
pwmchip_set_drvdata(chip, data);
}
linux-7.2.3/rust/helpers/rbtree.c
+20
View File
@@ -0,0 +1,20 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/rbtree.h>
__rust_helper void rust_helper_rb_link_node(struct rb_node *node,
struct rb_node *parent,
struct rb_node **rb_link)
{
rb_link_node(node, parent, rb_link);
}
__rust_helper struct rb_node *rust_helper_rb_first(const struct rb_root *root)
{
return rb_first(root);
}
__rust_helper struct rb_node *rust_helper_rb_last(const struct rb_root *root)
{
return rb_last(root);
}
linux-7.2.3/rust/helpers/rcu.c
+13
View File
@@ -0,0 +1,13 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/rcupdate.h>
__rust_helper void rust_helper_rcu_read_lock(void)
{
rcu_read_lock();
}
__rust_helper void rust_helper_rcu_read_unlock(void)
{
rcu_read_unlock();
}
linux-7.2.3/rust/helpers/refcount.c
+28
View File
@@ -0,0 +1,28 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/refcount.h>
__rust_helper refcount_t rust_helper_REFCOUNT_INIT(int n)
{
return (refcount_t)REFCOUNT_INIT(n);
}
__rust_helper void rust_helper_refcount_set(refcount_t *r, int n)
{
refcount_set(r, n);
}
__rust_helper void rust_helper_refcount_inc(refcount_t *r)
{
refcount_inc(r);
}
__rust_helper void rust_helper_refcount_dec(refcount_t *r)
{
refcount_dec(r);
}
__rust_helper bool rust_helper_refcount_dec_and_test(refcount_t *r)
{
return refcount_dec_and_test(r);
}
linux-7.2.3/rust/helpers/regulator.c
+57
View File
@@ -0,0 +1,57 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/regulator/consumer.h>
#ifndef CONFIG_REGULATOR
__rust_helper void rust_helper_regulator_put(struct regulator *regulator)
{
regulator_put(regulator);
}
__rust_helper int rust_helper_regulator_set_voltage(struct regulator *regulator,
int min_uV, int max_uV)
{
return regulator_set_voltage(regulator, min_uV, max_uV);
}
__rust_helper int rust_helper_regulator_get_voltage(struct regulator *regulator)
{
return regulator_get_voltage(regulator);
}
__rust_helper struct regulator *rust_helper_regulator_get(struct device *dev,
const char *id)
{
return regulator_get(dev, id);
}
__rust_helper int rust_helper_regulator_enable(struct regulator *regulator)
{
return regulator_enable(regulator);
}
__rust_helper int rust_helper_regulator_disable(struct regulator *regulator)
{
return regulator_disable(regulator);
}
__rust_helper int rust_helper_regulator_is_enabled(struct regulator *regulator)
{
return regulator_is_enabled(regulator);
}
__rust_helper int rust_helper_devm_regulator_get_enable(struct device *dev,
const char *id)
{
return devm_regulator_get_enable(dev, id);
}
__rust_helper int
rust_helper_devm_regulator_get_enable_optional(struct device *dev,
const char *id)
{
return devm_regulator_get_enable_optional(dev, id);
}
#endif
linux-7.2.3/rust/helpers/scatterlist.c
+26
View File
@@ -0,0 +1,26 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/dma-direction.h>
__rust_helper dma_addr_t rust_helper_sg_dma_address(struct scatterlist *sg)
{
return sg_dma_address(sg);
}
__rust_helper unsigned int rust_helper_sg_dma_len(struct scatterlist *sg)
{
return sg_dma_len(sg);
}
__rust_helper struct scatterlist *rust_helper_sg_next(struct scatterlist *sg)
{
return sg_next(sg);
}
__rust_helper void rust_helper_dma_unmap_sgtable(struct device *dev,
struct sg_table *sgt,
enum dma_data_direction dir,
unsigned long attrs)
{
return dma_unmap_sgtable(dev, sgt, dir, attrs);
}
linux-7.2.3/rust/helpers/security.c
+48
View File
@@ -0,0 +1,48 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/security.h>
#ifndef CONFIG_SECURITY
__rust_helper void rust_helper_security_cred_getsecid(const struct cred *c,
u32 *secid)
{
security_cred_getsecid(c, secid);
}
__rust_helper int rust_helper_security_secid_to_secctx(u32 secid,
struct lsm_context *cp)
{
return security_secid_to_secctx(secid, cp);
}
__rust_helper void rust_helper_security_release_secctx(struct lsm_context *cp)
{
security_release_secctx(cp);
}
__rust_helper int
rust_helper_security_binder_set_context_mgr(const struct cred *mgr)
{
return security_binder_set_context_mgr(mgr);
}
__rust_helper int
rust_helper_security_binder_transaction(const struct cred *from,
const struct cred *to)
{
return security_binder_transaction(from, to);
}
__rust_helper int
rust_helper_security_binder_transfer_binder(const struct cred *from,
const struct cred *to)
{
return security_binder_transfer_binder(from, to);
}
__rust_helper int rust_helper_security_binder_transfer_file(
const struct cred *from, const struct cred *to, const struct file *file)
{
return security_binder_transfer_file(from, to, file);
}
#endif
linux-7.2.3/rust/helpers/signal.c
+8
View File
@@ -0,0 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/sched/signal.h>
__rust_helper int rust_helper_signal_pending(struct task_struct *t)
{
return signal_pending(t);
}
linux-7.2.3/rust/helpers/slab.c
+17
View File
@@ -0,0 +1,17 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/slab.h>
__rust_helper void *__must_check __realloc_size(2)
rust_helper_krealloc_node_align(const void *objp, size_t new_size, unsigned long align,
gfp_t flags, int node)
{
return krealloc_node_align(objp, new_size, align, flags, node);
}
__rust_helper void *__must_check __realloc_size(2)
rust_helper_kvrealloc_node_align(const void *p, size_t size, unsigned long align,
gfp_t flags, int node)
{
return kvrealloc_node_align(p, size, align, flags, node);
}
linux-7.2.3/rust/helpers/spinlock.c
+38
View File
@@ -0,0 +1,38 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/spinlock.h>
__rust_helper void rust_helper___spin_lock_init(spinlock_t *lock,
const char *name,
struct lock_class_key *key)
{
#ifdef CONFIG_DEBUG_SPINLOCK
# if defined(CONFIG_PREEMPT_RT)
__spin_lock_init(lock, name, key, false);
# else /*!CONFIG_PREEMPT_RT */
__raw_spin_lock_init(spinlock_check(lock), name, key, LD_WAIT_CONFIG);
# endif /* CONFIG_PREEMPT_RT */
#else /* !CONFIG_DEBUG_SPINLOCK */
spin_lock_init(lock);
#endif /* CONFIG_DEBUG_SPINLOCK */
}
__rust_helper void rust_helper_spin_lock(spinlock_t *lock)
{
spin_lock(lock);
}
__rust_helper void rust_helper_spin_unlock(spinlock_t *lock)
{
spin_unlock(lock);
}
__rust_helper int rust_helper_spin_trylock(spinlock_t *lock)
{
return spin_trylock(lock);
}
__rust_helper void rust_helper_spin_assert_is_held(spinlock_t *lock)
{
lockdep_assert_held(lock);
}
linux-7.2.3/rust/helpers/string.c
+8
View File
@@ -0,0 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/string.h>
__rust_helper void *rust_helper_memchr(const void *s, int c, size_t n)
{
return memchr(s, c, n);
}
linux-7.2.3/rust/helpers/sync.c
+13
View File
@@ -0,0 +1,13 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/lockdep.h>
__rust_helper void rust_helper_lockdep_register_key(struct lock_class_key *k)
{
lockdep_register_key(k);
}
__rust_helper void rust_helper_lockdep_unregister_key(struct lock_class_key *k)
{
lockdep_unregister_key(k);
}
linux-7.2.3/rust/helpers/task.c
+62
View File
@@ -0,0 +1,62 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/sched/task.h>
__rust_helper void rust_helper_might_resched(void)
{
might_resched();
}
__rust_helper struct task_struct *rust_helper_get_current(void)
{
return current;
}
__rust_helper void rust_helper_get_task_struct(struct task_struct *t)
{
get_task_struct(t);
}
__rust_helper void rust_helper_put_task_struct(struct task_struct *t)
{
put_task_struct(t);
}
__rust_helper kuid_t rust_helper_task_uid(struct task_struct *task)
{
return task_uid(task);
}
__rust_helper kuid_t rust_helper_task_euid(struct task_struct *task)
{
return task_euid(task);
}
#ifndef CONFIG_USER_NS
__rust_helper uid_t rust_helper_from_kuid(struct user_namespace *to, kuid_t uid)
{
return from_kuid(to, uid);
}
#endif /* CONFIG_USER_NS */
__rust_helper bool rust_helper_uid_eq(kuid_t left, kuid_t right)
{
return uid_eq(left, right);
}
__rust_helper kuid_t rust_helper_current_euid(void)
{
return current_euid();
}
__rust_helper struct user_namespace *rust_helper_current_user_ns(void)
{
return current_user_ns();
}
__rust_helper pid_t rust_helper_task_tgid_nr_ns(struct task_struct *tsk,
struct pid_namespace *ns)
{
return task_tgid_nr_ns(tsk, ns);
}
linux-7.2.3/rust/helpers/time.c
+40
View File
@@ -0,0 +1,40 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/delay.h>
#include <linux/ktime.h>
#include <linux/timekeeping.h>
__rust_helper void rust_helper_fsleep(unsigned long usecs)
{
fsleep(usecs);
}
__rust_helper ktime_t rust_helper_ktime_get_real(void)
{
return ktime_get_real();
}
__rust_helper ktime_t rust_helper_ktime_get_boottime(void)
{
return ktime_get_boottime();
}
__rust_helper ktime_t rust_helper_ktime_get_clocktai(void)
{
return ktime_get_clocktai();
}
__rust_helper s64 rust_helper_ktime_to_us(const ktime_t kt)
{
return ktime_to_us(kt);
}
__rust_helper s64 rust_helper_ktime_to_ms(const ktime_t kt)
{
return ktime_to_ms(kt);
}
__rust_helper void rust_helper_udelay(unsigned long usec)
{
udelay(usec);
}
linux-7.2.3/rust/helpers/uaccess.c
+29
View File
@@ -0,0 +1,29 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/uaccess.h>
__rust_helper unsigned long
rust_helper_copy_from_user(void *to, const void __user *from, unsigned long n)
{
return copy_from_user(to, from, n);
}
__rust_helper unsigned long
rust_helper_copy_to_user(void __user *to, const void *from, unsigned long n)
{
return copy_to_user(to, from, n);
}
#ifdef INLINE_COPY_USER
__rust_helper
unsigned long rust_helper__copy_from_user(void *to, const void __user *from, unsigned long n)
{
return _inline_copy_from_user(to, from, n);
}
__rust_helper
unsigned long rust_helper__copy_to_user(void __user *to, const void *from, unsigned long n)
{
return _inline_copy_to_user(to, from, n);
}
#endif
linux-7.2.3/rust/helpers/usb.c
+9
View File
@@ -0,0 +1,9 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/usb.h>
__rust_helper struct usb_device *
rust_helper_interface_to_usbdev(struct usb_interface *intf)
{
return interface_to_usbdev(intf);
}
linux-7.2.3/rust/helpers/vmalloc.c
+16
View File
@@ -0,0 +1,16 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/mm.h>
#include <linux/vmalloc.h>
__rust_helper void *__must_check __realloc_size(2)
rust_helper_vrealloc_node_align(const void *p, size_t size, unsigned long align,
gfp_t flags, int node)
{
return vrealloc_node_align(p, size, align, flags, node);
}
__rust_helper bool rust_helper_is_vmalloc_addr(const void *x)
{
return is_vmalloc_addr(x);
}
linux-7.2.3/rust/helpers/wait.c
+8
View File
@@ -0,0 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/wait.h>
__rust_helper void rust_helper_init_wait(struct wait_queue_entry *wq_entry)
{
init_wait(wq_entry);
}
linux-7.2.3/rust/helpers/workqueue.c
+16
View File
@@ -0,0 +1,16 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/workqueue.h>
__rust_helper void rust_helper_init_work_with_key(struct work_struct *work,
work_func_t func,
bool onstack,
const char *name,
struct lock_class_key *key)
{
__init_work(work, onstack);
work->data = (atomic_long_t)WORK_DATA_INIT();
lockdep_init_map(&work->lockdep_map, name, key, 0);
INIT_LIST_HEAD(&work->entry);
work->func = func;
}
linux-7.2.3/rust/helpers/xarray.c
+28
View File
@@ -0,0 +1,28 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/xarray.h>
__rust_helper int rust_helper_xa_err(void *entry)
{
return xa_err(entry);
}
__rust_helper void rust_helper_xa_init_flags(struct xarray *xa, gfp_t flags)
{
return xa_init_flags(xa, flags);
}
__rust_helper int rust_helper_xa_trylock(struct xarray *xa)
{
return xa_trylock(xa);
}
__rust_helper void rust_helper_xa_lock(struct xarray *xa)
{
return xa_lock(xa);
}
__rust_helper void rust_helper_xa_unlock(struct xarray *xa)
{
return xa_unlock(xa);
}
linux-7.2.3/rust/kernel/.gitignore
+5
View File
@@ -0,0 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
/generated_arch_static_branch_asm.rs
/generated_arch_warn_asm.rs
/generated_arch_reachable_asm.rs
linux-7.2.3/rust/kernel/Kconfig.test
+86
View File
@@ -0,0 +1,86 @@
# SPDX-License-Identifier: GPL-2.0-only
menuconfig RUST_KUNIT_TESTS
bool "Rust KUnit tests"
depends on KUNIT && RUST
default KUNIT_ALL_TESTS
help
This menu collects all options for Rust KUnit tests.
See Documentation/rust/testing.rst for how to protect
unit tests with these options.
Say Y here to enable Rust KUnit tests.
If unsure, say N.
if RUST_KUNIT_TESTS
config RUST_ALLOCATOR_KUNIT_TEST
bool "KUnit tests for Rust allocator API" if !KUNIT_ALL_TESTS
default KUNIT_ALL_TESTS
help
This option enables KUnit tests for the Rust allocator API.
These are only for development and testing, not for regular
kernel use cases.
If unsure, say N.
config RUST_KVEC_KUNIT_TEST
bool "KUnit tests for Rust KVec API" if !KUNIT_ALL_TESTS
default KUNIT_ALL_TESTS
help
This option enables KUnit tests for the Rust KVec API.
These are only for development and testing, not for
regular kernel use cases.
If unsure, say N.
config RUST_BITMAP_KUNIT_TEST
bool "KUnit tests for Rust bitmap API" if !KUNIT_ALL_TESTS
default KUNIT_ALL_TESTS
help
This option enables KUnit tests for the Rust bitmap API.
These are only for development and testing, not for regular
kernel use cases.
If unsure, say N.
config RUST_KUNIT_SELFTEST
bool "KUnit selftests for Rust" if !KUNIT_ALL_TESTS
default KUNIT_ALL_TESTS
help
This option enables KUnit selftests. These are only
for development and testing, not for regular kernel
use cases.
If unsure, say N.
config RUST_STR_KUNIT_TEST
bool "KUnit tests for Rust strings API" if !KUNIT_ALL_TESTS
default KUNIT_ALL_TESTS
help
This option enables KUnit tests for the Rust strings API.
These are only for development and testing, not for regular
kernel use cases.
If unsure, say N.
config RUST_ATOMICS_KUNIT_TEST
bool "KUnit tests for Rust atomics API" if !KUNIT_ALL_TESTS
default KUNIT_ALL_TESTS
help
This option enables KUnit tests for the Rust atomics API.
These are only for development and testing, not for regular
kernel use cases.
If unsure, say N.
config RUST_BITFIELD_KUNIT_TEST
bool "KUnit tests for the Rust `bitfield!` macro" if !KUNIT_ALL_TESTS
default KUNIT_ALL_TESTS
help
This option enables KUnit tests for the Rust `bitfield!` macro.
These are only for development and testing, not for regular
kernel use cases.
If unsure, say N.
endif
linux-7.2.3/rust/kernel/acpi.rs
+65
View File
@@ -0,0 +1,65 @@
// SPDX-License-Identifier: GPL-2.0
//! Advanced Configuration and Power Interface abstractions.
use crate::{
bindings,
device_id::{RawDeviceId, RawDeviceIdIndex},
prelude::*,
};
/// IdTable type for ACPI drivers.
pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>;
/// An ACPI device id.
#[repr(transparent)]
#[derive(Clone, Copy)]
pub struct DeviceId(bindings::acpi_device_id);
// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `acpi_device_id` and does not add
// additional invariants, so it's safe to transmute to `RawType`.
unsafe impl RawDeviceId for DeviceId {
type RawType = bindings::acpi_device_id;
}
// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field.
unsafe impl RawDeviceIdIndex for DeviceId {
const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::acpi_device_id, driver_data);
fn index(&self) -> usize {
self.0.driver_data
}
}
impl DeviceId {
const ACPI_ID_LEN: usize = 16;
/// Create a new device id from an ACPI 'id' string.
#[inline(always)]
pub const fn new(id: &'static CStr) -> Self {
let src = id.to_bytes_with_nul();
build_assert!(src.len() <= Self::ACPI_ID_LEN, "ID exceeds 16 bytes");
let mut acpi: bindings::acpi_device_id = pin_init::zeroed();
let mut i = 0;
while i < src.len() {
acpi.id[i] = src[i];
i += 1;
}
Self(acpi)
}
}
/// Create an ACPI `IdTable` with an "alias" for modpost.
#[macro_export]
macro_rules! acpi_device_table {
($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => {
const $table_name: $crate::device_id::IdArray<
$crate::acpi::DeviceId,
$id_info_type,
{ $table_data.len() },
> = $crate::device_id::IdArray::new($table_data);
$crate::module_device_table!("acpi", $module_table_name, $table_name);
};
}
linux-7.2.3/rust/kernel/alloc.rs
+273
View File
@@ -0,0 +1,273 @@
// SPDX-License-Identifier: GPL-2.0
//! Implementation of the kernel's memory allocation infrastructure.
pub mod allocator;
pub mod kbox;
pub mod kvec;
pub mod layout;
pub use self::kbox::Box;
pub use self::kbox::KBox;
pub use self::kbox::KVBox;
pub use self::kbox::VBox;
pub use self::kvec::IntoIter;
pub use self::kvec::KVVec;
pub use self::kvec::KVec;
pub use self::kvec::VVec;
pub use self::kvec::Vec;
/// Indicates an allocation error.
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub struct AllocError;
use crate::prelude::*;
use core::{
alloc::Layout,
ptr::NonNull, //
};
/// Flags to be used when allocating memory.
///
/// They can be combined with the operators `|`, `&`, and `!`.
///
/// Values can be used from the [`flags`] module.
#[derive(Clone, Copy, PartialEq)]
pub struct Flags(u32);
impl Flags {
/// Get the raw representation of this flag.
pub(crate) fn as_raw(self) -> u32 {
self.0
}
/// Check whether `flags` is contained in `self`.
pub fn contains(self, flags: Flags) -> bool {
(self & flags) == flags
}
}
impl core::ops::BitOr for Flags {
type Output = Self;
fn bitor(self, rhs: Self) -> Self::Output {
Self(self.0 | rhs.0)
}
}
impl core::ops::BitAnd for Flags {
type Output = Self;
fn bitand(self, rhs: Self) -> Self::Output {
Self(self.0 & rhs.0)
}
}
impl core::ops::Not for Flags {
type Output = Self;
fn not(self) -> Self::Output {
Self(!self.0)
}
}
/// Allocation flags.
///
/// These are meant to be used in functions that can allocate memory.
pub mod flags {
use super::Flags;
/// Zeroes out the allocated memory.
///
/// This is normally or'd with other flags.
pub const __GFP_ZERO: Flags = Flags(bindings::__GFP_ZERO);
/// Allow the allocation to be in high memory.
///
/// Allocations in high memory may not be mapped into the kernel's address space, so this can't
/// be used with `kmalloc` and other similar methods.
///
/// This is normally or'd with other flags.
pub const __GFP_HIGHMEM: Flags = Flags(bindings::__GFP_HIGHMEM);
/// Users can not sleep and need the allocation to succeed.
///
/// A lower watermark is applied to allow access to "atomic reserves". The current
/// implementation doesn't support NMI and few other strict non-preemptive contexts (e.g.
/// `raw_spin_lock`). The same applies to [`GFP_NOWAIT`].
pub const GFP_ATOMIC: Flags = Flags(bindings::GFP_ATOMIC);
/// Typical for kernel-internal allocations. The caller requires `ZONE_NORMAL` or a lower zone
/// for direct access but can direct reclaim.
pub const GFP_KERNEL: Flags = Flags(bindings::GFP_KERNEL);
/// The same as [`GFP_KERNEL`], except the allocation is accounted to kmemcg.
pub const GFP_KERNEL_ACCOUNT: Flags = Flags(bindings::GFP_KERNEL_ACCOUNT);
/// For kernel allocations that should not stall for direct reclaim, start physical IO or
/// use any filesystem callback. It is very likely to fail to allocate memory, even for very
/// small allocations.
pub const GFP_NOWAIT: Flags = Flags(bindings::GFP_NOWAIT);
/// Suppresses allocation failure reports.
///
/// This is normally or'd with other flags.
pub const __GFP_NOWARN: Flags = Flags(bindings::__GFP_NOWARN);
}
/// Non Uniform Memory Access (NUMA) node identifier.
#[derive(Clone, Copy, PartialEq)]
pub struct NumaNode(i32);
impl NumaNode {
/// Create a new NUMA node identifier (non-negative integer).
///
/// Returns [`EINVAL`] if a negative id or an id exceeding [`bindings::MAX_NUMNODES`] is
/// specified.
pub fn new(node: i32) -> Result<Self> {
// MAX_NUMNODES never exceeds 2**10 because NODES_SHIFT is 0..10.
if node < 0 || node >= bindings::MAX_NUMNODES as i32 {
return Err(EINVAL);
}
Ok(Self(node))
}
}
/// Specify necessary constant to pass the information to Allocator that the caller doesn't care
/// about the NUMA node to allocate memory from.
impl NumaNode {
/// No node preference.
pub const NO_NODE: NumaNode = NumaNode(bindings::NUMA_NO_NODE);
}
/// The kernel's [`Allocator`] trait.
///
/// An implementation of [`Allocator`] can allocate, re-allocate and free memory buffers described
/// via [`Layout`].
///
/// [`Allocator`] is designed to be implemented as a ZST; [`Allocator`] functions do not operate on
/// an object instance.
///
/// In order to be able to support `#[derive(CoercePointee)]` later on, we need to avoid a design
/// that requires an `Allocator` to be instantiated, hence its functions must not contain any kind
/// of `self` parameter.
///
/// # Safety
///
/// - A memory allocation returned from an allocator must remain valid until it is explicitly freed.
///
/// - Any pointer to a valid memory allocation must be valid to be passed to any other [`Allocator`]
/// function of the same type.
///
/// - Implementers must ensure that all trait functions abide by the guarantees documented in the
/// `# Guarantees` sections.
pub unsafe trait Allocator {
/// The minimum alignment satisfied by all allocations from this allocator.
///
/// # Guarantees
///
/// Any pointer allocated by this allocator is guaranteed to be aligned to `MIN_ALIGN` even if
/// the requested layout has a smaller alignment.
const MIN_ALIGN: usize;
/// Allocate memory based on `layout`, `flags` and `nid`.
///
/// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout
/// constraints (i.e. minimum size and alignment as specified by `layout`).
///
/// This function is equivalent to `realloc` when called with `None`.
///
/// # Guarantees
///
/// When the return value is `Ok(ptr)`, then `ptr` is
/// - valid for reads and writes for `layout.size()` bytes, until it is passed to
/// [`Allocator::free`] or [`Allocator::realloc`],
/// - aligned to `layout.align()`,
///
/// Additionally, `Flags` are honored as documented in
/// <https://docs.kernel.org/core-api/mm-api.html#mm-api-gfp-flags>.
fn alloc(layout: Layout, flags: Flags, nid: NumaNode) -> Result<NonNull<[u8]>, AllocError> {
// SAFETY: Passing `None` to `realloc` is valid by its safety requirements and asks for a
// new memory allocation.
unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags, nid) }
}
/// Re-allocate an existing memory allocation to satisfy the requested `layout` and
/// a specific NUMA node request to allocate the memory for.
///
/// Systems employing a Non Uniform Memory Access (NUMA) architecture contain collections of
/// hardware resources including processors, memory, and I/O buses, that comprise what is
/// commonly known as a NUMA node.
///
/// `nid` stands for NUMA id, i. e. NUMA node identifier, which is a non-negative integer
/// if a node needs to be specified, or [`NumaNode::NO_NODE`] if the caller doesn't care.
///
/// If the requested size is zero, `realloc` behaves equivalent to `free`.
///
/// If the requested size is larger than the size of the existing allocation, a successful call
/// to `realloc` guarantees that the new or grown buffer has at least `Layout::size` bytes, but
/// may also be larger.
///
/// If the requested size is smaller than the size of the existing allocation, `realloc` may or
/// may not shrink the buffer; this is implementation specific to the allocator.
///
/// On allocation failure, the existing buffer, if any, remains valid.
///
/// The buffer is represented as `NonNull<[u8]>`.
///
/// # Safety
///
/// - If `ptr == Some(p)`, then `p` must point to an existing and valid memory allocation
/// created by this [`Allocator`]; if `old_layout` is zero-sized `p` does not need to be a
/// pointer returned by this [`Allocator`].
/// - `ptr` is allowed to be `None`; in this case a new memory allocation is created and
/// `old_layout` is ignored.
/// - `old_layout` must match the `Layout` the allocation has been created with.
///
/// # Guarantees
///
/// This function has the same guarantees as [`Allocator::alloc`]. When `ptr == Some(p)`, then
/// it additionally guarantees that:
/// - the contents of the memory pointed to by `p` are preserved up to the lesser of the new
/// and old size, i.e. `ret_ptr[0..min(layout.size(), old_layout.size())] ==
/// p[0..min(layout.size(), old_layout.size())]`.
/// - when the return value is `Err(AllocError)`, then `ptr` is still valid.
unsafe fn realloc(
ptr: Option<NonNull<u8>>,
layout: Layout,
old_layout: Layout,
flags: Flags,
nid: NumaNode,
) -> Result<NonNull<[u8]>, AllocError>;
/// Free an existing memory allocation.
///
/// # Safety
///
/// - `ptr` must point to an existing and valid memory allocation created by this [`Allocator`];
/// if `old_layout` is zero-sized `p` does not need to be a pointer returned by this
/// [`Allocator`].
/// - `layout` must match the `Layout` the allocation has been created with.
/// - The memory allocation at `ptr` must never again be read from or written to.
unsafe fn free(ptr: NonNull<u8>, layout: Layout) {
// SAFETY: The caller guarantees that `ptr` points at a valid allocation created by this
// allocator. We are passing a `Layout` with the smallest possible alignment, so it is
// smaller than or equal to the alignment previously used with this allocation.
let _ = unsafe {
Self::realloc(
Some(ptr),
Layout::new::<()>(),
layout,
Flags(0),
NumaNode::NO_NODE,
)
};
}
}
/// Returns a properly aligned dangling pointer from the given `layout`.
pub(crate) fn dangling_from_layout(layout: Layout) -> NonNull<u8> {
let ptr = layout.align() as *mut u8;
// SAFETY: `layout.align()` (and hence `ptr`) is guaranteed to be non-zero.
unsafe { NonNull::new_unchecked(ptr) }
}
linux-7.2.3/rust/kernel/alloc/allocator.rs
+323
View File
@@ -0,0 +1,323 @@
// SPDX-License-Identifier: GPL-2.0
//! Allocator support.
//!
//! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide"
//! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the
//! typical application of the different kernel allocators.
//!
//! Reference: <https://docs.kernel.org/core-api/memory-allocation.html>
use super::{
AllocError,
Allocator,
Flags,
NumaNode, //
};
use crate::{
bindings,
page, //
};
use core::{
alloc::Layout,
ptr::{
self,
NonNull, //
}, //
};
const ARCH_KMALLOC_MINALIGN: usize = bindings::ARCH_KMALLOC_MINALIGN;
mod iter;
pub use self::iter::VmallocPageIter;
/// The contiguous kernel allocator.
///
/// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also
/// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific.
///
/// For more details see [self].
pub struct Kmalloc;
/// The virtually contiguous kernel allocator.
///
/// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel
/// virtual space. It is typically used for large allocations. The memory allocated with this
/// allocator is not physically contiguous.
///
/// For more details see [self].
pub struct Vmalloc;
/// The kvmalloc kernel allocator.
///
/// `KVmalloc` attempts to allocate memory with `Kmalloc` first, but falls back to `Vmalloc` upon
/// failure. This allocator is typically used when the size for the requested allocation is not
/// known and may exceed the capabilities of `Kmalloc`.
///
/// For more details see [self].
pub struct KVmalloc;
/// # Invariants
///
/// One of the following: `krealloc_node_align`, `vrealloc_node_align`, `kvrealloc_node_align`.
struct ReallocFunc(
unsafe extern "C" fn(
*const crate::ffi::c_void,
usize,
crate::ffi::c_ulong,
u32,
crate::ffi::c_int,
) -> *mut crate::ffi::c_void,
);
impl ReallocFunc {
// INVARIANT: `krealloc_node_align` satisfies the type invariants.
const KREALLOC: Self = Self(bindings::krealloc_node_align);
// INVARIANT: `vrealloc_node_align` satisfies the type invariants.
const VREALLOC: Self = Self(bindings::vrealloc_node_align);
// INVARIANT: `kvrealloc_node_align` satisfies the type invariants.
const KVREALLOC: Self = Self(bindings::kvrealloc_node_align);
/// # Safety
///
/// This method has the same safety requirements as [`Allocator::realloc`].
///
/// # Guarantees
///
/// This method has the same guarantees as `Allocator::realloc`. Additionally
/// - it accepts any pointer to a valid memory allocation allocated by this function.
/// - memory allocated by this function remains valid until it is passed to this function.
#[inline]
unsafe fn call(
&self,
ptr: Option<NonNull<u8>>,
layout: Layout,
old_layout: Layout,
flags: Flags,
nid: NumaNode,
) -> Result<NonNull<[u8]>, AllocError> {
let size = layout.size();
let ptr = match ptr {
Some(ptr) => {
if old_layout.size() == 0 {
ptr::null()
} else {
ptr.as_ptr()
}
}
None => ptr::null(),
};
// SAFETY:
// - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that
// `ptr` is NULL or valid.
// - `ptr` is either NULL or valid by the safety requirements of this function.
//
// GUARANTEE:
// - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`.
// - Those functions provide the guarantees of this function.
let raw_ptr = unsafe {
// If `size == 0` and `ptr != NULL` the memory behind the pointer is freed.
self.0(ptr.cast(), size, layout.align(), flags.0, nid.0).cast()
};
let ptr = if size == 0 {
crate::alloc::dangling_from_layout(layout)
} else {
NonNull::new(raw_ptr).ok_or(AllocError)?
};
Ok(NonNull::slice_from_raw_parts(ptr, size))
}
}
impl Kmalloc {
/// Returns a [`Layout`] that makes [`Kmalloc`] fulfill the requested size and alignment of
/// `layout`.
pub fn aligned_layout(layout: Layout) -> Layout {
// Note that `layout.size()` (after padding) is guaranteed to be a multiple of
// `layout.align()` which together with the slab guarantees means that `Kmalloc` will return
// a properly aligned object (see comments in `kmalloc()` for more information).
layout.pad_to_align()
}
}
// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that
// - memory remains valid until it is explicitly freed,
// - passing a pointer to a valid memory allocation is OK,
// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
unsafe impl Allocator for Kmalloc {
const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN;
#[inline]
unsafe fn realloc(
ptr: Option<NonNull<u8>>,
layout: Layout,
old_layout: Layout,
flags: Flags,
nid: NumaNode,
) -> Result<NonNull<[u8]>, AllocError> {
let layout = Kmalloc::aligned_layout(layout);
// SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`.
unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags, nid) }
}
}
impl Vmalloc {
/// Convert a pointer to a [`Vmalloc`] allocation to a [`page::BorrowedPage`].
///
/// # Examples
///
/// ```
/// # use core::ptr::{
/// # from_mut,
/// # NonNull, //
/// # };
/// # use kernel::page;
/// use kernel::alloc::allocator::Vmalloc;
///
/// let mut vbox = VBox::<[u8; page::PAGE_SIZE]>::new_uninit(GFP_KERNEL)?;
///
/// {
/// // SAFETY: By the type invariant of `Box` the inner pointer of `vbox` is non-null.
/// let ptr = unsafe { NonNull::new_unchecked(from_mut(&mut *vbox)) };
///
/// // SAFETY:
/// // `ptr` is a valid pointer to a `Vmalloc` allocation.
/// // `ptr` is valid for the entire lifetime of `page`.
/// let page = unsafe { Vmalloc::to_page(ptr.cast()) };
///
/// // SAFETY: There is no concurrent read or write to the same page.
/// unsafe { page.fill_zero_raw(0, page::PAGE_SIZE)? };
/// }
/// # Ok::<(), Error>(())
/// ```
///
/// # Safety
///
/// - `ptr` must be a valid pointer to a [`Vmalloc`] allocation.
/// - `ptr` must remain valid for the entire duration of `'a`.
pub unsafe fn to_page<'a>(ptr: NonNull<u8>) -> page::BorrowedPage<'a> {
// SAFETY: `ptr` is a valid pointer to `Vmalloc` memory.
let page = unsafe { bindings::vmalloc_to_page(ptr.as_ptr().cast()) };
// SAFETY: `vmalloc_to_page` returns a valid pointer to a `struct page` for a valid pointer
// to `Vmalloc` memory.
let page = unsafe { NonNull::new_unchecked(page) };
// SAFETY:
// - `page` is a valid pointer to a `struct page`, given that by the safety requirements of
// this function `ptr` is a valid pointer to a `Vmalloc` allocation.
// - By the safety requirements of this function `ptr` is valid for the entire lifetime of
// `'a`.
unsafe { page::BorrowedPage::from_raw(page) }
}
}
// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that
// - memory remains valid until it is explicitly freed,
// - passing a pointer to a valid memory allocation is OK,
// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
unsafe impl Allocator for Vmalloc {
const MIN_ALIGN: usize = kernel::page::PAGE_SIZE;
#[inline]
unsafe fn realloc(
ptr: Option<NonNull<u8>>,
layout: Layout,
old_layout: Layout,
flags: Flags,
nid: NumaNode,
) -> Result<NonNull<[u8]>, AllocError> {
// SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously
// allocated with this `Allocator`.
unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags, nid) }
}
}
// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that
// - memory remains valid until it is explicitly freed,
// - passing a pointer to a valid memory allocation is OK,
// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
unsafe impl Allocator for KVmalloc {
const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN;
#[inline]
unsafe fn realloc(
ptr: Option<NonNull<u8>>,
layout: Layout,
old_layout: Layout,
flags: Flags,
nid: NumaNode,
) -> Result<NonNull<[u8]>, AllocError> {
// `KVmalloc` may use the `Kmalloc` backend, hence we have to enforce a `Kmalloc`
// compatible layout.
let layout = Kmalloc::aligned_layout(layout);
// SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously
// allocated with this `Allocator`.
unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags, nid) }
}
}
#[cfg(CONFIG_RUST_ALLOCATOR_KUNIT_TEST)]
#[macros::kunit_tests(rust_allocator)]
mod tests {
use super::*;
use core::mem::MaybeUninit;
use kernel::prelude::*;
#[test]
fn test_alignment() -> Result {
const TEST_SIZE: usize = 1024;
const TEST_LARGE_ALIGN_SIZE: usize = kernel::page::PAGE_SIZE * 4;
// These two structs are used to test allocating aligned memory.
// they don't need to be accessed, so they're marked as dead_code.
#[expect(dead_code)]
#[repr(align(128))]
struct Blob([u8; TEST_SIZE]);
#[expect(dead_code)]
#[repr(align(8192))]
struct LargeAlignBlob([u8; TEST_LARGE_ALIGN_SIZE]);
struct TestAlign<T, A: Allocator>(Box<MaybeUninit<T>, A>);
impl<T, A: Allocator> TestAlign<T, A> {
fn new() -> Result<Self> {
Ok(Self(Box::<_, A>::new_uninit(GFP_KERNEL)?))
}
fn is_aligned_to(&self, align: usize) -> bool {
assert!(align.is_power_of_two());
let addr = self.0.as_ptr() as usize;
addr & (align - 1) == 0
}
}
let ta = TestAlign::<Blob, Kmalloc>::new()?;
assert!(ta.is_aligned_to(128));
let ta = TestAlign::<LargeAlignBlob, Kmalloc>::new()?;
assert!(ta.is_aligned_to(8192));
let ta = TestAlign::<Blob, Vmalloc>::new()?;
assert!(ta.is_aligned_to(128));
let ta = TestAlign::<LargeAlignBlob, Vmalloc>::new()?;
assert!(ta.is_aligned_to(8192));
let ta = TestAlign::<Blob, KVmalloc>::new()?;
assert!(ta.is_aligned_to(128));
let ta = TestAlign::<LargeAlignBlob, KVmalloc>::new()?;
assert!(ta.is_aligned_to(8192));
Ok(())
}
}
linux-7.2.3/rust/kernel/alloc/allocator/iter.rs
+100
View File
@@ -0,0 +1,100 @@
// SPDX-License-Identifier: GPL-2.0
use super::Vmalloc;
use crate::page;
use core::{
marker::PhantomData,
ptr::NonNull, //
};
/// An [`Iterator`] of [`page::BorrowedPage`] items owned by a [`Vmalloc`] allocation.
///
/// # Guarantees
///
/// The pages iterated by the [`Iterator`] appear in the order as they are mapped in the CPU's
/// virtual address space ascendingly.
///
/// # Invariants
///
/// - `buf` is a valid and [`page::PAGE_SIZE`] aligned pointer into a [`Vmalloc`] allocation.
/// - `size` is the number of bytes from `buf` until the end of the [`Vmalloc`] allocation `buf`
/// points to.
pub struct VmallocPageIter<'a> {
/// The base address of the [`Vmalloc`] buffer.
buf: NonNull<u8>,
/// The size of the buffer pointed to by `buf` in bytes.
size: usize,
/// The current page index of the [`Iterator`].
index: usize,
_p: PhantomData<page::BorrowedPage<'a>>,
}
impl<'a> Iterator for VmallocPageIter<'a> {
type Item = page::BorrowedPage<'a>;
fn next(&mut self) -> Option<Self::Item> {
let offset = self.index.checked_mul(page::PAGE_SIZE)?;
// Even though `self.size()` may be smaller than `Self::page_count() * page::PAGE_SIZE`, it
// is always a number between `(Self::page_count() - 1) * page::PAGE_SIZE` and
// `Self::page_count() * page::PAGE_SIZE`, hence the check below is sufficient.
if offset < self.size() {
self.index += 1;
} else {
return None;
}
// SAFETY: `offset` is in the interval `[0, (self.page_count() - 1) * page::PAGE_SIZE]`,
// hence the resulting pointer is guaranteed to be within the same allocation.
let ptr = unsafe { self.buf.add(offset) };
// SAFETY:
// - `ptr` is a valid pointer to a `Vmalloc` allocation.
// - `ptr` is valid for the duration of `'a`.
Some(unsafe { Vmalloc::to_page(ptr) })
}
fn size_hint(&self) -> (usize, Option<usize>) {
let remaining = self.page_count().saturating_sub(self.index);
(remaining, Some(remaining))
}
}
impl<'a> VmallocPageIter<'a> {
/// Creates a new [`VmallocPageIter`] instance.
///
/// # Safety
///
/// - `buf` must be a [`page::PAGE_SIZE`] aligned pointer into a [`Vmalloc`] allocation.
/// - `buf` must be valid for at least the lifetime of `'a`.
/// - `size` must be the number of bytes from `buf` until the end of the [`Vmalloc`] allocation
/// `buf` points to.
pub unsafe fn new(buf: NonNull<u8>, size: usize) -> Self {
// INVARIANT: By the safety requirements, `buf` is a valid and `page::PAGE_SIZE` aligned
// pointer into a [`Vmalloc`] allocation.
Self {
buf,
size,
index: 0,
_p: PhantomData,
}
}
/// Returns the size of the backing [`Vmalloc`] allocation in bytes.
///
/// Note that this is the size the [`Vmalloc`] allocation has been allocated with. Hence, this
/// number may be smaller than `[`Self::page_count`] * [`page::PAGE_SIZE`]`.
#[inline]
pub fn size(&self) -> usize {
self.size
}
/// Returns the number of pages owned by the backing [`Vmalloc`] allocation.
#[inline]
pub fn page_count(&self) -> usize {
self.size().div_ceil(page::PAGE_SIZE)
}
}
linux-7.2.3/rust/kernel/alloc/kbox.rs
+758
View File
@@ -0,0 +1,758 @@
// SPDX-License-Identifier: GPL-2.0
//! Implementation of [`Box`].
#[allow(unused_imports)] // Used in doc comments.
use super::allocator::{
KVmalloc,
Kmalloc,
Vmalloc,
VmallocPageIter, //
};
use super::{
AllocError,
Allocator,
Flags,
NumaNode, //
};
use crate::{
fmt,
page::AsPageIter,
prelude::*,
types::ForeignOwnable, //
};
use core::{
alloc::Layout,
borrow::{
Borrow,
BorrowMut, //
},
marker::PhantomData,
mem::{
ManuallyDrop,
MaybeUninit, //
},
ops::{
Deref,
DerefMut, //
},
ptr::NonNull,
result::Result, //
};
use pin_init::ZeroableOption;
/// The kernel's [`Box`] type -- a heap allocation for a single value of type `T`.
///
/// This is the kernel's version of the Rust stdlib's `Box`. There are several differences,
/// for example no `noalias` attribute is emitted and partially moving out of a `Box` is not
/// supported. There are also several API differences, e.g. `Box` always requires an [`Allocator`]
/// implementation to be passed as generic, page [`Flags`] when allocating memory and all functions
/// that may allocate memory are fallible.
///
/// `Box` works with any of the kernel's allocators, e.g. [`Kmalloc`], [`Vmalloc`] or [`KVmalloc`].
/// There are aliases for `Box` with these allocators ([`KBox`], [`VBox`], [`KVBox`]).
///
/// When dropping a [`Box`], the value is also dropped and the heap memory is automatically freed.
///
/// # Examples
///
/// ```
/// let b = KBox::<u64>::new(24_u64, GFP_KERNEL)?;
///
/// assert_eq!(*b, 24_u64);
/// # Ok::<(), Error>(())
/// ```
///
/// ```
/// # use kernel::bindings;
/// const SIZE: usize = bindings::KMALLOC_MAX_SIZE as usize + 1;
/// struct Huge([u8; SIZE]);
///
/// assert!(KBox::<Huge>::new_uninit(GFP_KERNEL | __GFP_NOWARN).is_err());
/// ```
///
/// ```
/// # use kernel::bindings;
/// const SIZE: usize = bindings::KMALLOC_MAX_SIZE as usize + 1;
/// struct Huge([u8; SIZE]);
///
/// assert!(KVBox::<Huge>::new_uninit(GFP_KERNEL).is_ok());
/// ```
///
/// [`Box`]es can also be used to store trait objects by coercing their type:
///
/// ```
/// trait FooTrait {}
///
/// struct FooStruct;
/// impl FooTrait for FooStruct {}
///
/// let _ = KBox::new(FooStruct, GFP_KERNEL)? as KBox<dyn FooTrait>;
/// # Ok::<(), Error>(())
/// ```
///
/// # Invariants
///
/// `self.0` is always properly aligned and either points to memory allocated with `A` or, for
/// zero-sized types, is a dangling, well aligned pointer.
#[repr(transparent)]
#[derive(core::marker::CoercePointee)]
pub struct Box<#[pointee] T: ?Sized, A: Allocator>(NonNull<T>, PhantomData<A>);
/// Type alias for [`Box`] with a [`Kmalloc`] allocator.
///
/// # Examples
///
/// ```
/// let b = KBox::new(24_u64, GFP_KERNEL)?;
///
/// assert_eq!(*b, 24_u64);
/// # Ok::<(), Error>(())
/// ```
pub type KBox<T> = Box<T, super::allocator::Kmalloc>;
/// Type alias for [`Box`] with a [`Vmalloc`] allocator.
///
/// # Examples
///
/// ```
/// let b = VBox::new(24_u64, GFP_KERNEL)?;
///
/// assert_eq!(*b, 24_u64);
/// # Ok::<(), Error>(())
/// ```
pub type VBox<T> = Box<T, super::allocator::Vmalloc>;
/// Type alias for [`Box`] with a [`KVmalloc`] allocator.
///
/// # Examples
///
/// ```
/// let b = KVBox::new(24_u64, GFP_KERNEL)?;
///
/// assert_eq!(*b, 24_u64);
/// # Ok::<(), Error>(())
/// ```
pub type KVBox<T> = Box<T, super::allocator::KVmalloc>;
// SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee:
// <https://doc.rust-lang.org/stable/std/option/index.html#representation>).
unsafe impl<T, A: Allocator> ZeroableOption for Box<T, A> {}
// SAFETY: `Box` is `Send` if `T` is `Send` because the `Box` owns a `T`.
unsafe impl<T, A> Send for Box<T, A>
where
T: Send + ?Sized,
A: Allocator,
{
}
// SAFETY: `Box` is `Sync` if `T` is `Sync` because the `Box` owns a `T`.
unsafe impl<T, A> Sync for Box<T, A>
where
T: Sync + ?Sized,
A: Allocator,
{
}
impl<T, A> Box<T, A>
where
T: ?Sized,
A: Allocator,
{
/// Creates a new `Box<T, A>` from a raw pointer.
///
/// # Safety
///
/// For non-ZSTs, `raw` must point at an allocation allocated with `A` that is sufficiently
/// aligned for and holds a valid `T`. The caller passes ownership of the allocation to the
/// `Box`.
///
/// For ZSTs, `raw` must be a dangling, well aligned pointer.
#[inline]
pub const unsafe fn from_raw(raw: *mut T) -> Self {
// INVARIANT: Validity of `raw` is guaranteed by the safety preconditions of this function.
// SAFETY: By the safety preconditions of this function, `raw` is not a NULL pointer.
Self(unsafe { NonNull::new_unchecked(raw) }, PhantomData)
}
/// Consumes the `Box<T, A>` and returns a raw pointer.
///
/// This will not run the destructor of `T` and for non-ZSTs the allocation will stay alive
/// indefinitely. Use [`Box::from_raw`] to recover the [`Box`], drop the value and free the
/// allocation, if any.
///
/// # Examples
///
/// ```
/// let x = KBox::new(24, GFP_KERNEL)?;
/// let ptr = KBox::into_raw(x);
/// // SAFETY: `ptr` comes from a previous call to `KBox::into_raw`.
/// let x = unsafe { KBox::from_raw(ptr) };
///
/// assert_eq!(*x, 24);
/// # Ok::<(), Error>(())
/// ```
#[inline]
pub fn into_raw(b: Self) -> *mut T {
ManuallyDrop::new(b).0.as_ptr()
}
/// Consumes and leaks the `Box<T, A>` and returns a mutable reference.
///
/// See [`Box::into_raw`] for more details.
#[inline]
pub fn leak<'a>(b: Self) -> &'a mut T {
// SAFETY: `Box::into_raw` always returns a properly aligned and dereferenceable pointer
// which points to an initialized instance of `T`.
unsafe { &mut *Box::into_raw(b) }
}
}
impl<T, A> Box<MaybeUninit<T>, A>
where
A: Allocator,
{
/// Converts a `Box<MaybeUninit<T>, A>` to a `Box<T, A>`.
///
/// It is undefined behavior to call this function while the value inside of `b` is not yet
/// fully initialized.
///
/// # Safety
///
/// Callers must ensure that the value inside of `b` is in an initialized state.
pub unsafe fn assume_init(self) -> Box<T, A> {
let raw = Self::into_raw(self);
// SAFETY: `raw` comes from a previous call to `Box::into_raw`. By the safety requirements
// of this function, the value inside the `Box` is in an initialized state. Hence, it is
// safe to reconstruct the `Box` as `Box<T, A>`.
unsafe { Box::from_raw(raw.cast()) }
}
/// Writes the value and converts to `Box<T, A>`.
pub fn write(mut self, value: T) -> Box<T, A> {
(*self).write(value);
// SAFETY: We've just initialized `b`'s value.
unsafe { self.assume_init() }
}
}
impl<T, A> Box<T, A>
where
A: Allocator,
{
/// Creates a new `Box<T, A>` and initializes its contents with `x`.
///
/// New memory is allocated with `A`. The allocation may fail, in which case an error is
/// returned. For ZSTs no memory is allocated.
pub fn new(x: T, flags: Flags) -> Result<Self, AllocError> {
let b = Self::new_uninit(flags)?;
Ok(Box::write(b, x))
}
/// Creates a new `Box<T, A>` with uninitialized contents.
///
/// New memory is allocated with `A`. The allocation may fail, in which case an error is
/// returned. For ZSTs no memory is allocated.
///
/// # Examples
///
/// ```
/// let b = KBox::<u64>::new_uninit(GFP_KERNEL)?;
/// let b = KBox::write(b, 24);
///
/// assert_eq!(*b, 24_u64);
/// # Ok::<(), Error>(())
/// ```
pub fn new_uninit(flags: Flags) -> Result<Box<MaybeUninit<T>, A>, AllocError> {
let layout = Layout::new::<MaybeUninit<T>>();
let ptr = A::alloc(layout, flags, NumaNode::NO_NODE)?;
// INVARIANT: `ptr` is either a dangling pointer or points to memory allocated with `A`,
// which is sufficient in size and alignment for storing a `T`.
Ok(Box(ptr.cast(), PhantomData))
}
/// Creates a new zero-initialized `Box<T, A>`.
///
/// New memory is allocated with `A` and the [`__GFP_ZERO`] flag. The allocation may fail, in
/// which case an error is returned. For ZSTs no memory is allocated.
///
/// # Examples
///
/// ```
/// let b = KBox::<[u8; 128]>::zeroed(GFP_KERNEL)?;
/// assert_eq!(*b, [0; 128]);
/// # Ok::<(), Error>(())
/// ```
pub fn zeroed(flags: Flags) -> Result<Self, AllocError>
where
T: Zeroable,
{
// SAFETY: `__GFP_ZERO` guarantees the memory is zeroed; `T: Zeroable` guarantees that
// all-zeroes is a valid bit pattern for `T`.
Ok(unsafe { Self::new_uninit(flags | __GFP_ZERO)?.assume_init() })
}
/// Constructs a new `Pin<Box<T, A>>`. If `T` does not implement [`Unpin`], then `x` will be
/// pinned in memory and can't be moved.
#[inline]
pub fn pin(x: T, flags: Flags) -> Result<Pin<Box<T, A>>, AllocError>
where
A: 'static,
{
Ok(Self::new(x, flags)?.into())
}
/// Construct a pinned slice of elements `Pin<Box<[T], A>>`.
///
/// This is a convenient means for creation of e.g. slices of structrures containing spinlocks
/// or mutexes.
///
/// # Examples
///
/// ```
/// use kernel::sync::{
/// new_spinlock,
/// SpinLock, //
/// };
///
/// struct Inner {
/// a: u32,
/// b: u32,
/// }
///
/// #[pin_data]
/// struct Example {
/// c: u32,
/// #[pin]
/// d: SpinLock<Inner>,
/// }
///
/// impl Example {
/// fn new() -> impl PinInit<Self, Error> {
/// try_pin_init!(Self {
/// c: 10,
/// d <- new_spinlock!(Inner { a: 20, b: 30 }),
/// })
/// }
/// }
///
/// // Allocate a boxed slice of 10 `Example`s.
/// let s = KBox::pin_slice(
/// | _i | Example::new(),
/// 10,
/// GFP_KERNEL
/// )?;
///
/// assert_eq!(s[5].c, 10);
/// assert_eq!(s[3].d.lock().a, 20);
/// # Ok::<(), Error>(())
/// ```
pub fn pin_slice<Func, Item, E>(
mut init: Func,
len: usize,
flags: Flags,
) -> Result<Pin<Box<[T], A>>, E>
where
Func: FnMut(usize) -> Item,
Item: PinInit<T, E>,
E: From<AllocError>,
{
let mut buffer = super::Vec::<T, A>::with_capacity(len, flags)?;
for i in 0..len {
let ptr = buffer.spare_capacity_mut().as_mut_ptr().cast();
// SAFETY:
// - `ptr` is a valid pointer to uninitialized memory.
// - `ptr` is not used if an error is returned.
// - `ptr` won't be moved until it is dropped, i.e. it is pinned.
unsafe { init(i).__pinned_init(ptr)? };
// SAFETY:
// - `i + 1 <= len`, hence we don't exceed the capacity, due to the call to
// `with_capacity()` above.
// - The new value at index buffer.len() + 1 is the only element being added here, and
// it has been initialized above by `init(i).__pinned_init(ptr)`.
unsafe { buffer.inc_len(1) };
}
let (ptr, _, _) = buffer.into_raw_parts();
let slice = core::ptr::slice_from_raw_parts_mut(ptr, len);
// SAFETY: `slice` points to an allocation allocated with `A` (`buffer`) and holds a valid
// `[T]`.
Ok(Pin::from(unsafe { Box::from_raw(slice) }))
}
/// Convert a [`Box<T,A>`] to a [`Pin<Box<T,A>>`]. If `T` does not implement
/// [`Unpin`], then `x` will be pinned in memory and can't be moved.
pub fn into_pin(this: Self) -> Pin<Self> {
this.into()
}
/// Forgets the contents (does not run the destructor), but keeps the allocation.
fn forget_contents(this: Self) -> Box<MaybeUninit<T>, A> {
let ptr = Self::into_raw(this);
// SAFETY: `ptr` is valid, because it came from `Box::into_raw`.
unsafe { Box::from_raw(ptr.cast()) }
}
/// Drops the contents, but keeps the allocation.
///
/// # Examples
///
/// ```
/// let value = KBox::new([0; 32], GFP_KERNEL)?;
/// assert_eq!(*value, [0; 32]);
/// let value = KBox::drop_contents(value);
/// // Now we can re-use `value`:
/// let value = KBox::write(value, [1; 32]);
/// assert_eq!(*value, [1; 32]);
/// # Ok::<(), Error>(())
/// ```
pub fn drop_contents(this: Self) -> Box<MaybeUninit<T>, A> {
let ptr = this.0.as_ptr();
// SAFETY: `ptr` is valid, because it came from `this`. After this call we never access the
// value stored in `this` again.
unsafe { core::ptr::drop_in_place(ptr) };
Self::forget_contents(this)
}
/// Moves the `Box`'s value out of the `Box` and consumes the `Box`.
pub fn into_inner(b: Self) -> T {
// SAFETY: By the type invariant `&*b` is valid for `read`.
let value = unsafe { core::ptr::read(&*b) };
let _ = Self::forget_contents(b);
value
}
}
impl<T, A> From<Box<T, A>> for Pin<Box<T, A>>
where
T: ?Sized,
A: Allocator,
{
/// Converts a `Box<T, A>` into a `Pin<Box<T, A>>`. If `T` does not implement [`Unpin`], then
/// `*b` will be pinned in memory and can't be moved.
///
/// This moves `b` into `Pin` without moving `*b` or allocating and copying any memory.
fn from(b: Box<T, A>) -> Self {
// SAFETY: The value wrapped inside a `Pin<Box<T, A>>` cannot be moved or replaced as long
// as `T` does not implement `Unpin`.
unsafe { Pin::new_unchecked(b) }
}
}
impl<T, A> InPlaceWrite<T> for Box<MaybeUninit<T>, A>
where
A: Allocator + 'static,
{
type Initialized = Box<T, A>;
#[inline]
fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid.
unsafe { init.__init(slot)? };
// SAFETY: All fields have been initialized.
Ok(unsafe { Box::assume_init(self) })
}
#[inline]
fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid and will not be moved, because we pin it later.
unsafe { init.__pinned_init(slot)? };
// SAFETY: All fields have been initialized.
Ok(unsafe { Box::assume_init(self) }.into())
}
}
impl<T, A> InPlaceInit<T> for Box<T, A>
where
A: Allocator + 'static,
{
type PinnedSelf = Pin<Self>;
#[inline]
fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E>
where
E: From<AllocError>,
{
Box::<_, A>::new_uninit(flags)?.write_pin_init(init)
}
#[inline]
fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
where
E: From<AllocError>,
{
Box::<_, A>::new_uninit(flags)?.write_init(init)
}
}
// SAFETY: The pointer returned by `into_foreign` comes from a well aligned
// pointer to `T` allocated by `A`.
unsafe impl<T, A> ForeignOwnable for Box<T, A>
where
A: Allocator,
{
const FOREIGN_ALIGN: usize = if core::mem::align_of::<T>() < A::MIN_ALIGN {
A::MIN_ALIGN
} else {
core::mem::align_of::<T>()
};
type Borrowed<'a>
= &'a T
where
Self: 'a;
type BorrowedMut<'a>
= &'a mut T
where
Self: 'a;
fn into_foreign(self) -> *mut c_void {
Box::into_raw(self).cast()
}
unsafe fn from_foreign(ptr: *mut c_void) -> Self {
// SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
// call to `Self::into_foreign`.
unsafe { Box::from_raw(ptr.cast()) }
}
unsafe fn borrow<'a>(ptr: *mut c_void) -> &'a T {
// SAFETY: The safety requirements of this method ensure that the object remains alive and
// immutable for the duration of 'a.
unsafe { &*ptr.cast() }
}
unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> &'a mut T {
let ptr = ptr.cast();
// SAFETY: The safety requirements of this method ensure that the pointer is valid and that
// nothing else will access the value for the duration of 'a.
unsafe { &mut *ptr }
}
}
// SAFETY: The pointer returned by `into_foreign` comes from a well aligned
// pointer to `T` allocated by `A`.
unsafe impl<T, A> ForeignOwnable for Pin<Box<T, A>>
where
A: Allocator,
{
const FOREIGN_ALIGN: usize = <Box<T, A> as ForeignOwnable>::FOREIGN_ALIGN;
type Borrowed<'a>
= Pin<&'a T>
where
Self: 'a;
type BorrowedMut<'a>
= Pin<&'a mut T>
where
Self: 'a;
fn into_foreign(self) -> *mut c_void {
// SAFETY: We are still treating the box as pinned.
Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }).cast()
}
unsafe fn from_foreign(ptr: *mut c_void) -> Self {
// SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
// call to `Self::into_foreign`.
unsafe { Pin::new_unchecked(Box::from_raw(ptr.cast())) }
}
unsafe fn borrow<'a>(ptr: *mut c_void) -> Pin<&'a T> {
// SAFETY: The safety requirements for this function ensure that the object is still alive,
// so it is safe to dereference the raw pointer.
// The safety requirements of `from_foreign` also ensure that the object remains alive for
// the lifetime of the returned value.
let r = unsafe { &*ptr.cast() };
// SAFETY: This pointer originates from a `Pin<Box<T>>`.
unsafe { Pin::new_unchecked(r) }
}
unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Pin<&'a mut T> {
let ptr = ptr.cast();
// SAFETY: The safety requirements for this function ensure that the object is still alive,
// so it is safe to dereference the raw pointer.
// The safety requirements of `from_foreign` also ensure that the object remains alive for
// the lifetime of the returned value.
let r = unsafe { &mut *ptr };
// SAFETY: This pointer originates from a `Pin<Box<T>>`.
unsafe { Pin::new_unchecked(r) }
}
}
impl<T, A> Deref for Box<T, A>
where
T: ?Sized,
A: Allocator,
{
type Target = T;
fn deref(&self) -> &T {
// SAFETY: `self.0` is always properly aligned, dereferenceable and points to an initialized
// instance of `T`.
unsafe { self.0.as_ref() }
}
}
impl<T, A> DerefMut for Box<T, A>
where
T: ?Sized,
A: Allocator,
{
fn deref_mut(&mut self) -> &mut T {
// SAFETY: `self.0` is always properly aligned, dereferenceable and points to an initialized
// instance of `T`.
unsafe { self.0.as_mut() }
}
}
/// # Examples
///
/// ```
/// # use core::borrow::Borrow;
/// struct Foo<B: Borrow<u32>>(B);
///
/// // Owned instance.
/// let owned = Foo(1);
///
/// // Owned instance using `KBox`.
/// let owned_kbox = Foo(KBox::new(1, GFP_KERNEL)?);
///
/// let i = 1;
/// // Borrowed from `i`.
/// let borrowed = Foo(&i);
/// # Ok::<(), Error>(())
/// ```
impl<T, A> Borrow<T> for Box<T, A>
where
T: ?Sized,
A: Allocator,
{
fn borrow(&self) -> &T {
self.deref()
}
}
/// # Examples
///
/// ```
/// # use core::borrow::BorrowMut;
/// struct Foo<B: BorrowMut<u32>>(B);
///
/// // Owned instance.
/// let owned = Foo(1);
///
/// // Owned instance using `KBox`.
/// let owned_kbox = Foo(KBox::new(1, GFP_KERNEL)?);
///
/// let mut i = 1;
/// // Borrowed from `i`.
/// let borrowed = Foo(&mut i);
/// # Ok::<(), Error>(())
/// ```
impl<T, A> BorrowMut<T> for Box<T, A>
where
T: ?Sized,
A: Allocator,
{
fn borrow_mut(&mut self) -> &mut T {
self.deref_mut()
}
}
impl<T, A> fmt::Display for Box<T, A>
where
T: ?Sized + fmt::Display,
A: Allocator,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
<T as fmt::Display>::fmt(&**self, f)
}
}
impl<T, A> fmt::Debug for Box<T, A>
where
T: ?Sized + fmt::Debug,
A: Allocator,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
<T as fmt::Debug>::fmt(&**self, f)
}
}
impl<T, A> Drop for Box<T, A>
where
T: ?Sized,
A: Allocator,
{
fn drop(&mut self) {
let layout = Layout::for_value::<T>(self);
// SAFETY: The pointer in `self.0` is guaranteed to be valid by the type invariant.
unsafe { core::ptr::drop_in_place::<T>(self.deref_mut()) };
// SAFETY:
// - `self.0` was previously allocated with `A`.
// - `layout` is equal to the `Layout´ `self.0` was allocated with.
unsafe { A::free(self.0.cast(), layout) };
}
}
/// # Examples
///
/// ```
/// use kernel::{
/// alloc::allocator::VmallocPageIter,
/// page::{
/// AsPageIter,
/// PAGE_SIZE, //
/// }, //
/// };
///
/// let mut vbox = VBox::new((), GFP_KERNEL)?;
///
/// assert!(vbox.page_iter().next().is_none());
///
/// let mut vbox = VBox::<[u8; PAGE_SIZE]>::new_uninit(GFP_KERNEL)?;
///
/// let page = vbox.page_iter().next().expect("At least one page should be available.\n");
///
/// // SAFETY: There is no concurrent read or write to the same page.
/// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? };
/// # Ok::<(), Error>(())
/// ```
impl<T> AsPageIter for VBox<T> {
type Iter<'a>
= VmallocPageIter<'a>
where
T: 'a;
fn page_iter(&mut self) -> Self::Iter<'_> {
let ptr = self.0.cast();
let size = core::mem::size_of::<T>();
// SAFETY:
// - `ptr` is a valid pointer to the beginning of a `Vmalloc` allocation.
// - `ptr` is guaranteed to be valid for the lifetime of `'a`.
// - `size` is the size of the `Vmalloc` allocation `ptr` points to.
unsafe { VmallocPageIter::new(ptr, size) }
}
}
linux-7.2.3/rust/kernel/alloc/kvec.rs
+1661
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/rust/kernel/alloc/kvec/errors.rs
+66
View File
@@ -0,0 +1,66 @@
// SPDX-License-Identifier: GPL-2.0
//! Errors for the [`Vec`] type.
use crate::{
fmt,
prelude::*, //
};
/// Error type for [`Vec::push_within_capacity`].
pub struct PushError<T>(pub T);
impl<T> fmt::Debug for PushError<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Not enough capacity")
}
}
impl<T> From<PushError<T>> for Error {
#[inline]
fn from(_: PushError<T>) -> Error {
// Returning ENOMEM isn't appropriate because the system is not out of memory. The vector
// is just full and we are refusing to resize it.
EINVAL
}
}
/// Error type for [`Vec::remove`].
pub struct RemoveError;
impl fmt::Debug for RemoveError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Index out of bounds")
}
}
impl From<RemoveError> for Error {
#[inline]
fn from(_: RemoveError) -> Error {
EINVAL
}
}
/// Error type for [`Vec::insert_within_capacity`].
pub enum InsertError<T> {
/// The value could not be inserted because the index is out of bounds.
IndexOutOfBounds(T),
/// The value could not be inserted because the vector is out of capacity.
OutOfCapacity(T),
}
impl<T> fmt::Debug for InsertError<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
InsertError::IndexOutOfBounds(_) => write!(f, "Index out of bounds"),
InsertError::OutOfCapacity(_) => write!(f, "Not enough capacity"),
}
}
}
impl<T> From<InsertError<T>> for Error {
#[inline]
fn from(_: InsertError<T>) -> Error {
EINVAL
}
}
linux-7.2.3/rust/kernel/alloc/layout.rs
+121
View File
@@ -0,0 +1,121 @@
// SPDX-License-Identifier: GPL-2.0
//! Memory layout.
//!
//! Custom layout types extending or improving [`Layout`].
use core::{
alloc::Layout,
marker::PhantomData, //
};
/// Error when constructing an [`ArrayLayout`].
pub struct LayoutError;
/// A layout for an array `[T; n]`.
///
/// # Invariants
///
/// - `len * size_of::<T>() <= isize::MAX`.
pub struct ArrayLayout<T> {
len: usize,
_phantom: PhantomData<fn() -> T>,
}
impl<T> Clone for ArrayLayout<T> {
fn clone(&self) -> Self {
*self
}
}
impl<T> Copy for ArrayLayout<T> {}
const ISIZE_MAX: usize = isize::MAX as usize;
impl<T> ArrayLayout<T> {
/// Creates a new layout for `[T; 0]`.
pub const fn empty() -> Self {
// INVARIANT: `0 * size_of::<T>() <= isize::MAX`.
Self {
len: 0,
_phantom: PhantomData,
}
}
/// Creates a new layout for `[T; len]`.
///
/// # Errors
///
/// When `len * size_of::<T>()` overflows or when `len * size_of::<T>() > isize::MAX`.
///
/// # Examples
///
/// ```
/// # use kernel::alloc::layout::{
/// # ArrayLayout,
/// # LayoutError, //
/// # };
/// let layout = ArrayLayout::<i32>::new(15)?;
/// assert_eq!(layout.len(), 15);
///
/// // Errors because `len * size_of::<T>()` overflows.
/// let layout = ArrayLayout::<i32>::new(isize::MAX as usize);
/// assert!(layout.is_err());
///
/// // Errors because `len * size_of::<i32>() > isize::MAX`,
/// // even though `len < isize::MAX`.
/// let layout = ArrayLayout::<i32>::new(isize::MAX as usize / 2);
/// assert!(layout.is_err());
///
/// # Ok::<(), Error>(())
/// ```
pub const fn new(len: usize) -> Result<Self, LayoutError> {
match len.checked_mul(core::mem::size_of::<T>()) {
Some(size) if size <= ISIZE_MAX => {
// INVARIANT: We checked above that `len * size_of::<T>() <= isize::MAX`.
Ok(Self {
len,
_phantom: PhantomData,
})
}
_ => Err(LayoutError),
}
}
/// Creates a new layout for `[T; len]`.
///
/// # Safety
///
/// `len` must be a value, for which `len * size_of::<T>() <= isize::MAX` is true.
pub const unsafe fn new_unchecked(len: usize) -> Self {
// INVARIANT: By the safety requirements of this function
// `len * size_of::<T>() <= isize::MAX`.
Self {
len,
_phantom: PhantomData,
}
}
/// Returns the number of array elements represented by this layout.
pub const fn len(&self) -> usize {
self.len
}
/// Returns `true` when no array elements are represented by this layout.
pub const fn is_empty(&self) -> bool {
self.len == 0
}
/// Returns the size of the [`ArrayLayout`] in bytes.
pub const fn size(&self) -> usize {
self.len() * core::mem::size_of::<T>()
}
}
impl<T> From<ArrayLayout<T>> for Layout {
fn from(value: ArrayLayout<T>) -> Self {
let res = Layout::array::<T>(value.len);
// SAFETY: By the type invariant of `ArrayLayout` we have
// `len * size_of::<T>() <= isize::MAX` and thus the result must be `Ok`.
unsafe { res.unwrap_unchecked() }
}
}
linux-7.2.3/rust/kernel/auxiliary.rs
+553
View File
@@ -0,0 +1,553 @@
// SPDX-License-Identifier: GPL-2.0
//! Abstractions for the auxiliary bus.
//!
//! C header: [`include/linux/auxiliary_bus.h`](srctree/include/linux/auxiliary_bus.h)
use crate::{
bindings,
container_of,
device,
device_id::{
RawDeviceId,
RawDeviceIdIndex, //
},
driver,
error::{
from_result,
to_result, //
},
prelude::*,
types::{
ForLt,
ForeignOwnable,
Opaque, //
},
ThisModule, //
};
use core::{
any::TypeId,
marker::PhantomData,
mem::offset_of,
pin::Pin,
ptr::{
addr_of_mut,
NonNull, //
},
};
/// An adapter for the registration of auxiliary drivers.
pub struct Adapter<T: Driver>(T);
// SAFETY:
// - `bindings::auxiliary_driver` is a C type declared as `repr(C)`.
// - `T::Data` is the type of the driver's device private data.
// - `struct auxiliary_driver` embeds a `struct device_driver`.
// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`.
unsafe impl<T: Driver> driver::DriverLayout for Adapter<T> {
type DriverType = bindings::auxiliary_driver;
type DriverData<'bound> = T::Data<'bound>;
const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver);
}
// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if
// a preceding call to `register` has been successful.
unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> {
unsafe fn register(
adrv: &Opaque<Self::DriverType>,
name: &'static CStr,
module: &'static ThisModule,
) -> Result {
// SAFETY: It's safe to set the fields of `struct auxiliary_driver` on initialization.
unsafe {
(*adrv.get()).name = name.as_char_ptr();
(*adrv.get()).probe = Some(Self::probe_callback);
(*adrv.get()).remove = Some(Self::remove_callback);
(*adrv.get()).id_table = T::ID_TABLE.as_ptr();
}
// SAFETY: `adrv` is guaranteed to be a valid `DriverType`.
to_result(unsafe {
bindings::__auxiliary_driver_register(adrv.get(), module.0, name.as_char_ptr())
})
}
unsafe fn unregister(adrv: &Opaque<Self::DriverType>) {
// SAFETY: `adrv` is guaranteed to be a valid `DriverType`.
unsafe { bindings::auxiliary_driver_unregister(adrv.get()) }
}
}
impl<T: Driver> Adapter<T> {
extern "C" fn probe_callback(
adev: *mut bindings::auxiliary_device,
id: *const bindings::auxiliary_device_id,
) -> c_int {
// SAFETY: The auxiliary bus only ever calls the probe callback with a valid pointer to a
// `struct auxiliary_device`.
//
// INVARIANT: `adev` is valid for the duration of `probe_callback()`.
let adev = unsafe { &*adev.cast::<Device<device::CoreInternal<'_>>>() };
// SAFETY: `DeviceId` is a `#[repr(transparent)`] wrapper of `struct auxiliary_device_id`
// and does not add additional invariants, so it's safe to transmute.
let id = unsafe { &*id.cast::<DeviceId>() };
let info = T::ID_TABLE.info(id.index());
from_result(|| {
let data = T::probe(adev, info);
adev.as_ref().set_drvdata(data)?;
Ok(0)
})
}
extern "C" fn remove_callback(adev: *mut bindings::auxiliary_device) {
// SAFETY: The auxiliary bus only ever calls the probe callback with a valid pointer to a
// `struct auxiliary_device`.
//
// INVARIANT: `adev` is valid for the duration of `remove_callback()`.
let adev = unsafe { &*adev.cast::<Device<device::CoreInternal<'_>>>() };
// SAFETY: `remove_callback` is only ever called after a successful call to
// `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called
// and stored a `Pin<KBox<T::Data<'_>>>`.
let data = unsafe { adev.as_ref().drvdata_borrow::<T::Data<'_>>() };
T::unbind(adev, data);
}
}
/// Declares a kernel module that exposes a single auxiliary driver.
#[macro_export]
macro_rules! module_auxiliary_driver {
($($f:tt)*) => {
$crate::module_driver!(<T>, $crate::auxiliary::Adapter<T>, { $($f)* });
};
}
/// Abstraction for `bindings::auxiliary_device_id`.
#[repr(transparent)]
#[derive(Clone, Copy)]
pub struct DeviceId(bindings::auxiliary_device_id);
impl DeviceId {
/// Create a new [`DeviceId`] from name.
pub const fn new(modname: &'static CStr, name: &'static CStr) -> Self {
let name = name.to_bytes_with_nul();
let modname = modname.to_bytes_with_nul();
let mut id: bindings::auxiliary_device_id = pin_init::zeroed();
let mut i = 0;
while i < modname.len() {
id.name[i] = modname[i];
i += 1;
}
// Reuse the space of the NULL terminator.
id.name[i - 1] = b'.';
let mut j = 0;
while j < name.len() {
id.name[i] = name[j];
i += 1;
j += 1;
}
Self(id)
}
}
// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `auxiliary_device_id` and does not add
// additional invariants, so it's safe to transmute to `RawType`.
unsafe impl RawDeviceId for DeviceId {
type RawType = bindings::auxiliary_device_id;
}
// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field.
unsafe impl RawDeviceIdIndex for DeviceId {
const DRIVER_DATA_OFFSET: usize =
core::mem::offset_of!(bindings::auxiliary_device_id, driver_data);
fn index(&self) -> usize {
self.0.driver_data
}
}
/// IdTable type for auxiliary drivers.
pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>;
/// Create a auxiliary `IdTable` with its alias for modpost.
#[macro_export]
macro_rules! auxiliary_device_table {
($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => {
const $table_name: $crate::device_id::IdArray<
$crate::auxiliary::DeviceId,
$id_info_type,
{ $table_data.len() },
> = $crate::device_id::IdArray::new($table_data);
$crate::module_device_table!("auxiliary", $module_table_name, $table_name);
};
}
/// The auxiliary driver trait.
///
/// Drivers must implement this trait in order to get an auxiliary driver registered.
pub trait Driver {
/// The type holding information about each device id supported by the driver.
///
/// TODO: Use associated_type_defaults once stabilized:
///
/// type IdInfo: 'static = ();
type IdInfo: 'static;
/// The type of the driver's bus device private data.
type Data<'bound>: Send + 'bound;
/// The table of device ids supported by the driver.
const ID_TABLE: IdTable<Self::IdInfo>;
/// Auxiliary driver probe.
///
/// Called when an auxiliary device is matches a corresponding driver.
fn probe<'bound>(
dev: &'bound Device<device::Core<'_>>,
id_info: &'bound Self::IdInfo,
) -> impl PinInit<Self::Data<'bound>, Error> + 'bound;
/// Auxiliary driver unbind.
///
/// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback
/// is optional.
///
/// This callback serves as a place for drivers to perform teardown operations that require a
/// `&Device<Core>` or `&Device<Bound>` reference. For instance, drivers may try to perform I/O
/// operations to gracefully tear down the device.
///
/// Otherwise, release operations for driver resources should be performed in `Drop`.
fn unbind<'bound>(dev: &'bound Device<device::Core<'_>>, this: Pin<&Self::Data<'bound>>) {
let _ = (dev, this);
}
}
/// The auxiliary device representation.
///
/// This structure represents the Rust abstraction for a C `struct auxiliary_device`. The
/// implementation abstracts the usage of an already existing C `struct auxiliary_device` within
/// Rust code that we get passed from the C side.
///
/// # Invariants
///
/// A [`Device`] instance represents a valid `struct auxiliary_device` created by the C portion of
/// the kernel.
#[repr(transparent)]
pub struct Device<Ctx: device::DeviceContext = device::Normal>(
Opaque<bindings::auxiliary_device>,
PhantomData<Ctx>,
);
impl<Ctx: device::DeviceContext> Device<Ctx> {
fn as_raw(&self) -> *mut bindings::auxiliary_device {
self.0.get()
}
/// Returns the auxiliary device' id.
pub fn id(&self) -> u32 {
// SAFETY: By the type invariant `self.as_raw()` is a valid pointer to a
// `struct auxiliary_device`.
unsafe { (*self.as_raw()).id }
}
}
impl Device<device::Bound> {
/// Returns a bound reference to the parent [`device::Device`].
pub fn parent(&self) -> &device::Device<device::Bound> {
let parent = (**self).parent();
// SAFETY: A bound auxiliary device always has a bound parent device.
unsafe { parent.as_bound() }
}
/// Returns a pinned reference to the registration data set by the registering (parent) driver.
///
/// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The returned
/// reference has its lifetime shortened from `'static` to `&self`'s borrow lifetime via
/// [`ForLt::cast_ref`].
///
/// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling
/// [`Registration::new()`].
///
/// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was
/// registered by a C driver.
pub fn registration_data<F: ForLt + 'static>(&self) -> Result<Pin<&F::Of<'_>>> {
// SAFETY: By the type invariant, `self.as_raw()` is a valid `struct auxiliary_device`.
let ptr = unsafe { (*self.as_raw()).registration_data_rust };
if ptr.is_null() {
dev_warn!(
self.as_ref(),
"No registration data set; parent is not a Rust driver.\n"
);
return Err(ENOENT);
}
// SAFETY: `ptr` is non-null and was set via `into_foreign()` in `Registration::new()`;
// `RegistrationData` is `#[repr(C)]` with `type_id` at offset 0, so reading a `TypeId`
// at the start of the allocation is valid regardless of `F`.
let type_id = unsafe { ptr.cast::<TypeId>().read() };
if type_id != TypeId::of::<F>() {
return Err(EINVAL);
}
// SAFETY: The `TypeId` check above confirms that the stored type matches
// `F::Of<'static>`; `ptr` remains valid until `Registration::drop()` calls
// `from_foreign()`.
let wrapper = unsafe { Pin::<KBox<RegistrationData<F::Of<'static>>>>::borrow(ptr) };
// SAFETY: `data` is a structurally pinned field of `RegistrationData`.
let pinned: Pin<&F::Of<'_>> = unsafe { wrapper.map_unchecked(|w| &w.data) };
// SAFETY: The data was pinned when stored; `cast_ref` only shortens
// the lifetime, so the pinning guarantee is preserved.
Ok(unsafe { Pin::new_unchecked(F::cast_ref(pinned.get_ref())) })
}
}
impl Device {
/// Returns a reference to the parent [`device::Device`].
pub fn parent(&self) -> &device::Device {
// SAFETY: A `struct auxiliary_device` always has a parent.
unsafe { self.as_ref().parent().unwrap_unchecked() }
}
extern "C" fn release(dev: *mut bindings::device) {
// SAFETY: By the type invariant `self.0.as_raw` is a pointer to the `struct device`
// embedded in `struct auxiliary_device`.
let adev = unsafe { container_of!(dev, bindings::auxiliary_device, dev) };
// SAFETY: `adev` points to the memory that has been allocated in `Registration::new`, via
// `KBox::new(Opaque::<bindings::auxiliary_device>::zeroed(), GFP_KERNEL)`.
let _ = unsafe { KBox::<Opaque<bindings::auxiliary_device>>::from_raw(adev.cast()) };
}
}
// SAFETY: `auxiliary::Device` is a transparent wrapper of `struct auxiliary_device`.
// The offset is guaranteed to point to a valid device field inside `auxiliary::Device`.
unsafe impl<Ctx: device::DeviceContext> device::AsBusDevice<Ctx> for Device<Ctx> {
const OFFSET: usize = offset_of!(bindings::auxiliary_device, dev);
}
// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic
// argument.
kernel::impl_device_context_deref!(unsafe { Device });
kernel::impl_device_context_into_aref!(Device);
// SAFETY: Instances of `Device` are always reference-counted.
unsafe impl crate::sync::aref::AlwaysRefCounted for Device {
fn inc_ref(&self) {
// SAFETY: The existence of a shared reference guarantees that the refcount is non-zero.
unsafe { bindings::get_device(self.as_ref().as_raw()) };
}
unsafe fn dec_ref(obj: NonNull<Self>) {
// CAST: `Self` a transparent wrapper of `bindings::auxiliary_device`.
let adev: *mut bindings::auxiliary_device = obj.cast().as_ptr();
// SAFETY: By the type invariant of `Self`, `adev` is a pointer to a valid
// `struct auxiliary_device`.
let dev = unsafe { addr_of_mut!((*adev).dev) };
// SAFETY: The safety requirements guarantee that the refcount is non-zero.
unsafe { bindings::put_device(dev) }
}
}
impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> {
fn as_ref(&self) -> &device::Device<Ctx> {
// SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid
// `struct auxiliary_device`.
let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) };
// SAFETY: `dev` points to a valid `struct device`.
unsafe { device::Device::from_raw(dev) }
}
}
// SAFETY: A `Device` is always reference-counted and can be released from any thread.
unsafe impl Send for Device {}
// SAFETY: `Device` can be shared among threads because all methods of `Device`
// (i.e. `Device<Normal>) are thread safe.
unsafe impl Sync for Device {}
// SAFETY: Same as `Device<Normal>` -- the underlying `struct auxiliary_device` is the same;
// `Bound` is a zero-sized type-state marker that does not affect thread safety.
unsafe impl Sync for Device<device::Bound> {}
/// Wrapper that stores a [`TypeId`] alongside the registration data for runtime type checking.
#[repr(C)]
#[pin_data]
struct RegistrationData<T> {
type_id: TypeId,
#[pin]
data: T,
}
/// The registration of an auxiliary device.
///
/// This type represents the registration of a [`struct auxiliary_device`]. When its parent device
/// is unbound, the corresponding auxiliary device will be unregistered from the system.
///
/// The type parameter `F` is a [`ForLt`](trait@ForLt) encoding of the registration
/// data type. For non-lifetime-parameterized types, use [`ForLt!(T)`](macro@ForLt).
/// The data can be accessed by the auxiliary driver through [`Device::registration_data()`].
///
/// # Invariants
///
/// `self.adev` always holds a valid pointer to an initialized and registered
/// [`struct auxiliary_device`] whose `registration_data_rust` field points to a
/// valid `Pin<KBox<RegistrationData<F::Of<'static>>>>`.
pub struct Registration<'a, F: ForLt + 'static> {
adev: NonNull<bindings::auxiliary_device>,
_phantom: PhantomData<F::Of<'a>>,
}
impl<'a, F: ForLt> Registration<'a, F>
where
for<'b> F::Of<'b>: Send + Sync,
{
/// Create and register a new auxiliary device with the given registration data.
///
/// The `data` is owned by the registration and can be accessed through the auxiliary device
/// via [`Device::registration_data()`].
///
/// # Safety
///
/// The caller must not `mem::forget()` the returned [`Registration`] or otherwise prevent its
/// [`Drop`] implementation from running, since the registration data may contain borrowed
/// references that become invalid after `'a` ends.
///
/// If the registration data is `'static`, use the safe [`Registration::new()`] instead.
pub unsafe fn new_with_lt<E>(
parent: &'a device::Device<device::Bound>,
name: &CStr,
id: u32,
modname: &CStr,
data: impl PinInit<F::Of<'a>, E>,
) -> Result<Self>
where
Error: From<E>,
{
let data = KBox::pin_init::<Error>(
try_pin_init!(RegistrationData {
type_id: TypeId::of::<F>(),
data <- data,
}),
GFP_KERNEL,
)?;
// SAFETY: `'a` is invariant (via `Registration`'s `PhantomData`). Lifetimes do not
// affect layout, so RegistrationData<F::Of<'a>> and RegistrationData<F::Of<'static>>
// have identical representation.
let data: Pin<KBox<RegistrationData<F::Of<'static>>>> =
unsafe { core::mem::transmute(data) };
let boxed: KBox<Opaque<bindings::auxiliary_device>> = KBox::zeroed(GFP_KERNEL)?;
let adev = boxed.get();
// SAFETY: It's safe to set the fields of `struct auxiliary_device` on initialization.
unsafe {
(*adev).dev.parent = parent.as_raw();
(*adev).dev.release = Some(Device::release);
(*adev).name = name.as_char_ptr();
(*adev).id = id;
(*adev).registration_data_rust = data.into_foreign();
}
// SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`,
// which has not been initialized yet.
unsafe { bindings::auxiliary_device_init(adev) };
// Now that `adev` is initialized, leak the `Box`; the corresponding memory will be
// freed by `Device::release` when the last reference to the `struct auxiliary_device`
// is dropped.
let _ = KBox::into_raw(boxed);
// SAFETY:
// - `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, which
// has been initialized,
// - `modname.as_char_ptr()` is a NULL terminated string.
let ret = unsafe { bindings::__auxiliary_device_add(adev, modname.as_char_ptr()) };
if ret != 0 {
// SAFETY: `registration_data` was set above via `into_foreign()`.
drop(unsafe {
Pin::<KBox<RegistrationData<F::Of<'static>>>>::from_foreign(
(*adev).registration_data_rust,
)
});
// SAFETY: `adev` is guaranteed to be a valid pointer to a
// `struct auxiliary_device`, which has been initialized.
unsafe { bindings::auxiliary_device_uninit(adev) };
return Err(Error::from_errno(ret));
}
// INVARIANT: The device will remain registered until `auxiliary_device_delete()` is
// called, which happens in `Self::drop()`.
Ok(Self {
// SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated
// successfully.
adev: unsafe { NonNull::new_unchecked(adev) },
_phantom: PhantomData,
})
}
/// Create and register a new auxiliary device with `'static` registration data.
///
/// Safe variant of [`Registration::new_with_lt()`] for registration data that does not contain
/// borrowed references.
pub fn new<E>(
parent: &'a device::Device<device::Bound>,
name: &CStr,
id: u32,
modname: &CStr,
data: impl PinInit<F::Of<'a>, E>,
) -> Result<Self>
where
F::Of<'a>: 'static,
Error: From<E>,
{
// SAFETY: `F::Of<'a>: 'static` guarantees the data contains no borrowed references,
// so forgetting the `Registration` cannot cause use-after-free.
unsafe { Self::new_with_lt(parent, name, id, modname, data) }
}
}
impl<F: ForLt> Drop for Registration<'_, F> {
fn drop(&mut self) {
// SAFETY: By the type invariant of `Self`, `self.adev.as_ptr()` is a valid registered
// `struct auxiliary_device`.
unsafe { bindings::auxiliary_device_delete(self.adev.as_ptr()) };
// SAFETY: `registration_data` was set in `new()` via `into_foreign()`.
drop(unsafe {
Pin::<KBox<RegistrationData<F::Of<'static>>>>::from_foreign(
(*self.adev.as_ptr()).registration_data_rust,
)
});
// This drops the reference we acquired through `auxiliary_device_init()`.
//
// SAFETY: By the type invariant of `Self`, `self.adev.as_ptr()` is a valid registered
// `struct auxiliary_device`.
unsafe { bindings::auxiliary_device_uninit(self.adev.as_ptr()) };
}
}
// SAFETY: A `Registration` of a `struct auxiliary_device` can be released from any thread.
unsafe impl<F: ForLt> Send for Registration<'_, F> where for<'a> F::Of<'a>: Send {}
// SAFETY: `Registration` does not expose any methods or fields that need synchronization.
unsafe impl<F: ForLt> Sync for Registration<'_, F> where for<'a> F::Of<'a>: Send {}
linux-7.2.3/rust/kernel/bitfield.rs
+863
View File
@@ -0,0 +1,863 @@
// SPDX-License-Identifier: GPL-2.0
//! Support for defining bitfields as Rust structures.
//!
//! The [`bitfield!`](kernel::bitfield!) macro declares integer types that are split into distinct
//! bit fields of arbitrary length. Each field is typed using [`Bounded`](kernel::num::Bounded) to
//! ensure values are properly validated and to avoid implicit data loss.
//!
//! # Example
//!
//! ```rust
//! use kernel::bitfield;
//! use kernel::num::Bounded;
//!
//! bitfield! {
//! pub struct Rgb(u16) {
//! 15:11 blue;
//! 10:5 green;
//! 4:0 red;
//! }
//! }
//!
//! // Valid value for the `blue` field.
//! let blue = Bounded::<u16, 5>::new::<0x18>();
//!
//! // Setters can be chained. Values ranges are checked at compile-time.
//! let color = Rgb::zeroed()
//! // Compile-time bounds check of constant value.
//! .with_const_red::<0x10>()
//! .with_const_green::<0x1f>()
//! // A `Bounded` can also be passed.
//! .with_blue(blue);
//!
//! assert_eq!(color.red(), 0x10);
//! assert_eq!(color.green(), 0x1f);
//! assert_eq!(color.blue(), 0x18);
//! assert_eq!(
//! color.into_raw(),
//! (0x18 << Rgb::BLUE_SHIFT) + (0x1f << Rgb::GREEN_SHIFT) + 0x10,
//! );
//!
//! // Convert to/from the backing storage type.
//! let raw: u16 = color.into();
//! assert_eq!(Rgb::from(raw), color);
//! ```
//!
//! # Syntax
//!
//! ```text
//! bitfield! {
//! #[attributes]
//! // Documentation for `Name`.
//! pub struct Name(storage_type) {
//! // `field_1` documentation.
//! hi:lo field_1;
//! // `field_2` documentation.
//! hi:lo field_2 => ConvertedType;
//! // `field_3` documentation.
//! hi:lo field_3 ?=> ConvertedType;
//! ...
//! }
//! }
//! ```
//!
//! - `storage_type`: The underlying unsigned integer type ([`u8`], [`u16`], [`u32`], [`u64`]).
//! Signed integer storage types are not supported.
//! - `hi:lo`: Bit range (inclusive), where `hi >= lo`.
//! - `=> Type`: Optional infallible conversion (see [below](#infallible-conversion-)).
//! - `?=> Type`: Optional fallible conversion (see [below](#fallible-conversion-)).
//! - Documentation strings and attributes are optional.
//!
//! # Generated code
//!
//! Each field is internally represented as a [`Bounded`] parameterized by its bit width. Field
//! values can either be set/retrieved directly, or converted from/to another type.
//!
//! The use of [`Bounded`] for each field enforces bounds-checking (at build time or runtime) of
//! every value assigned to a field. This ensures that data is never accidentally truncated.
//!
//! The macro generates the bitfield type, [`From`] and [`Into`] implementations for its storage
//! type, as well as [`Debug`] and [`Zeroable`](pin_init::Zeroable) implementations.
//!
//! For each field, it also generates:
//!
//! - `field()`: Getter method for the field value.
//! - `with_field(value)`: Infallible setter; the argument type must fit within the field's width.
//! - `with_const_field::<VALUE>()`: `const` setter; the value is validated at compile time.
//! Usually shorter to use than `with_field` for constant values as it doesn't require
//! constructing a [`Bounded`].
//! - `try_with_field(value)`: Fallible setter. Returns an error if the value is out of range.
//! - `FIELD_MASK`, `FIELD_SHIFT`, `FIELD_RANGE`: Constants for manual bit manipulation.
//!
//! # Reserved names for field identifiers
//!
//! Field identifiers are used to generate methods and associated constants on the bitfield type.
//! For a field named `field`, the macro may generate methods named `field`, `with_field`,
//! `with_const_field`, `try_with_field`, `__field` and `__with_field`, as well as constants named
//! `FIELD_MASK`, `FIELD_SHIFT` and `FIELD_RANGE`.
//!
//! Therefore, field identifiers must not use names that would collide with generated items for
//! any field in the same bitfield. The following prefixes are thus reserved for field identifiers:
//!
//! - `with_`
//! - `const_`
//! - `try_with_`
//! - `__`
//!
//! The field identifiers `from_raw`, `into_raw`, and `into` are also reserved.
//!
//! In addition, field identifiers should follow Rust `snake_case` conventions, since the associated
//! constants are generated by uppercasing the field name.
//!
//! # Implicit conversions
//!
//! Types that fit entirely within a field's bit width can be used directly with setters. For
//! example, [`bool`] works with single-bit fields, and [`u8`] works with 8-bit fields:
//!
//! ```rust
//! use kernel::bitfield;
//!
//! bitfield! {
//! pub struct Flags(u32) {
//! 15:8 byte_field;
//! 0:0 flag;
//! }
//! }
//!
//! let flags = Flags::zeroed()
//! .with_byte_field(0x42_u8)
//! .with_flag(true);
//!
//! assert_eq!(flags.into_raw(), (0x42 << Flags::BYTE_FIELD_SHIFT) | 1);
//! ```
//!
//! # Runtime bounds checking
//!
//! When a value is not known at compile time, use `try_with_field()` to check bounds at runtime:
//!
//! ```rust
//! use kernel::bitfield;
//!
//! bitfield! {
//! pub struct Config(u8) {
//! 3:0 nibble;
//! }
//! }
//!
//! fn set_nibble(config: Config, value: u8) -> Result<Config, Error> {
//! // Returns `EOVERFLOW` if `value > 0xf`.
//! config.try_with_nibble(value)
//! }
//! # Ok::<(), Error>(())
//! ```
//!
//! # Type conversion
//!
//! Fields can be automatically converted to/from a custom type using `=>` (infallible) or `?=>`
//! (fallible). The custom type must implement the appropriate [`From`] or [`TryFrom`] traits with
//! [`Bounded`].
//!
//! ## Infallible conversion (`=>`)
//!
//! Use this when all possible bit patterns of a field map to valid values:
//!
//! ```rust
//! use kernel::bitfield;
//! use kernel::num::Bounded;
//!
//! #[derive(Debug, Clone, Copy, PartialEq)]
//! enum Power {
//! Off,
//! On,
//! }
//!
//! impl From<Bounded<u32, 1>> for Power {
//! fn from(v: Bounded<u32, 1>) -> Self {
//! match *v {
//! 0 => Power::Off,
//! _ => Power::On,
//! }
//! }
//! }
//!
//! impl From<Power> for Bounded<u32, 1> {
//! fn from(p: Power) -> Self {
//! (p as u32 != 0).into()
//! }
//! }
//!
//! bitfield! {
//! pub struct Control(u32) {
//! 0:0 power => Power;
//! }
//! }
//!
//! let ctrl = Control::zeroed().with_power(Power::On);
//! assert_eq!(ctrl.power(), Power::On);
//! ```
//!
//! ## Fallible conversion (`?=>`)
//!
//! Use this when some bit patterns of a field are invalid. The getter returns a [`Result`]:
//!
//! ```rust
//! use kernel::bitfield;
//! use kernel::num::Bounded;
//!
//! #[derive(Debug, Clone, Copy, PartialEq)]
//! enum Mode {
//! Low = 0,
//! High = 1,
//! Auto = 2,
//! // 3 is invalid
//! }
//!
//! impl TryFrom<Bounded<u32, 2>> for Mode {
//! type Error = u32;
//!
//! fn try_from(v: Bounded<u32, 2>) -> Result<Self, u32> {
//! match *v {
//! 0 => Ok(Mode::Low),
//! 1 => Ok(Mode::High),
//! 2 => Ok(Mode::Auto),
//! n => Err(n),
//! }
//! }
//! }
//!
//! impl From<Mode> for Bounded<u32, 2> {
//! fn from(m: Mode) -> Self {
//! match m {
//! Mode::Low => Bounded::<u32, _>::new::<0>(),
//! Mode::High => Bounded::<u32, _>::new::<1>(),
//! Mode::Auto => Bounded::<u32, _>::new::<2>(),
//! }
//! }
//! }
//!
//! bitfield! {
//! pub struct Config(u32) {
//! 1:0 mode ?=> Mode;
//! }
//! }
//!
//! let cfg = Config::zeroed().with_mode(Mode::Auto);
//! assert_eq!(cfg.mode(), Ok(Mode::Auto));
//!
//! // Invalid bit pattern returns an error.
//! assert_eq!(Config::from(0b11).mode(), Err(3));
//! ```
//!
//! # Bits outside of declared fields
//!
//! Bits of the storage type that are not part of any declared field are preserved by the setter
//! methods, and can only be modified through `from_raw` or the [`From`] implementation from the
//! storage type.
//!
//! ```rust
//! use kernel::bitfield;
//!
//! bitfield! {
//! pub struct Sparse(u8) {
//! 7:6 high;
//! // Bits 5:1 are not covered by any field.
//! 0:0 low;
//! }
//! }
//!
//! // Set the gap bits via `from_raw`, then mutate the declared fields.
//! let val = Sparse::from_raw(0b0010_1010)
//! .with_const_high::<0b11>()
//! .with_low(true);
//!
//! // Bits 5:1 are unchanged.
//! assert_eq!(val.into_raw(), 0b1110_1011);
//! ```
//!
//! # Signed field values
//!
//! Bitfield storage types are unsigned. Since field getter methods return a [`Bounded`] of the
//! storage type, fields are also unsigned by default.
//!
//! If a field needs to encode a signed value, use a custom conversion type with `=>` or `?=>` to
//! perform the sign interpretation explicitly.
//!
//! [`Bounded`]: kernel::num::Bounded
/// Defines a bitfield struct with bounds-checked accessors for individual bit ranges.
///
/// See the [`mod@kernel::bitfield`] module for full documentation and examples.
#[macro_export]
macro_rules! bitfield {
// Entry point defining the bitfield struct, its implementations and its field accessors.
(
$(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* }
) => {
$crate::bitfield!(@core
#[allow(non_camel_case_types)]
$(#[$attr])* $vis $name $storage
);
$crate::bitfield!(@fields $vis $name $storage { $($fields)* });
};
// All rules below are helpers.
// Defines the wrapper `$name` type and its conversions from/to the storage type.
(@core $(#[$attr:meta])* $vis:vis $name:ident $storage:ty) => {
$(#[$attr])*
#[repr(transparent)]
#[derive(Clone, Copy, PartialEq, Eq)]
$vis struct $name {
inner: $storage,
}
#[allow(dead_code)]
impl $name {
/// Creates a bitfield from a raw value.
#[inline(always)]
$vis const fn from_raw(value: $storage) -> Self {
Self{ inner: value }
}
/// Turns this bitfield into its raw value.
///
/// This is similar to the [`From`] implementation, but is shorter to invoke in
/// most cases.
#[inline(always)]
$vis const fn into_raw(self) -> $storage {
self.inner
}
}
// SAFETY: `$storage` is `Zeroable` and `$name` is transparent.
unsafe impl ::pin_init::Zeroable for $name {}
impl ::core::convert::From<$name> for $storage {
#[inline(always)]
fn from(val: $name) -> $storage {
val.into_raw()
}
}
impl ::core::convert::From<$storage> for $name {
#[inline(always)]
fn from(val: $storage) -> $name {
Self::from_raw(val)
}
}
};
// Definitions requiring knowledge of individual fields: private and public field accessors,
// and `Debug` implementation.
(@fields $vis:vis $name:ident $storage:ty {
$($(#[doc = $doc:expr])* $hi:literal:$lo:literal $field:ident
$(?=> $try_into_type:ty)?
$(=> $into_type:ty)?
;
)*
}
) => {
#[allow(dead_code)]
impl $name {
$(
$crate::bitfield!(@private_field_accessors $vis $name $storage : $hi:$lo $field);
$crate::bitfield!(
@public_field_accessors $(#[doc = $doc])* $vis $name $storage : $hi:$lo $field
$(?=> $try_into_type)?
$(=> $into_type)?
);
)*
}
$crate::bitfield!(@debug $name { $($field;)* });
};
// Private field accessors working with the exact `Bounded` type for the field.
(
@private_field_accessors $vis:vis $name:ident $storage:ty : $hi:tt:$lo:tt $field:ident
) => {
::kernel::macros::paste!(
$vis const [<$field:upper _RANGE>]: ::core::ops::RangeInclusive<u8> = $lo..=$hi;
$vis const [<$field:upper _MASK>]: $storage =
((((1 << $hi) - 1) << 1) + 1) - ((1 << $lo) - 1);
$vis const [<$field:upper _SHIFT>]: u32 = $lo;
);
::kernel::macros::paste!(
#[inline(always)]
fn [<__ $field>](self) ->
::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> {
// Left shift to align the field's MSB with the storage MSB.
const ALIGN_TOP: u32 = $storage::BITS - ($hi + 1);
// Right shift to move the top-aligned field to bit 0 of the storage.
const ALIGN_BOTTOM: u32 = ALIGN_TOP + $lo;
// Extract the field using two shifts. `Bounded::shr` produces the correctly-sized
// output type.
let val = ::kernel::num::Bounded::<$storage, { $storage::BITS }>::from(
self.inner << ALIGN_TOP
);
val.shr::<ALIGN_BOTTOM, { $hi + 1 - $lo } >()
}
#[inline(always)]
const fn [<__with_ $field>](
mut self,
value: ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>,
) -> Self
{
const MASK: $storage = <$name>::[<$field:upper _MASK>];
const SHIFT: u32 = <$name>::[<$field:upper _SHIFT>];
let value = value.get() << SHIFT;
self.inner = (self.inner & !MASK) | value;
self
}
);
};
// Public accessors for fields infallibly (`=>`) converted to a type.
(
@public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty :
$hi:literal:$lo:literal $field:ident => $into_type:ty
) => {
::kernel::macros::paste!(
$(#[doc = $doc])*
#[doc = "Returns the value of this field."]
#[inline(always)]
$vis fn $field(self) -> $into_type
{
self.[<__ $field>]().into()
}
$(#[doc = $doc])*
#[doc = "Sets this field to the given `value`."]
#[inline(always)]
$vis fn [<with_ $field>](self, value: $into_type) -> Self
{
self.[<__with_ $field>](value.into())
}
);
};
// Public accessors for fields fallibly (`?=>`) converted to a type.
(
@public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty :
$hi:tt:$lo:tt $field:ident ?=> $try_into_type:ty
) => {
::kernel::macros::paste!(
$(#[doc = $doc])*
#[doc = "Returns the value of this field."]
#[inline(always)]
$vis fn $field(self) ->
::core::result::Result<
$try_into_type,
<$try_into_type as ::core::convert::TryFrom<
::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>
>>::Error
>
{
self.[<__ $field>]().try_into()
}
$(#[doc = $doc])*
#[doc = "Sets this field to the given `value`."]
#[inline(always)]
$vis fn [<with_ $field>](self, value: $try_into_type) -> Self
{
self.[<__with_ $field>](value.into())
}
);
};
// Public accessors for fields not converted to a type.
(
@public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty :
$hi:tt:$lo:tt $field:ident
) => {
::kernel::macros::paste!(
$(#[doc = $doc])*
#[doc = "Returns the value of this field."]
#[inline(always)]
$vis fn $field(self) ->
::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>
{
self.[<__ $field>]()
}
$(#[doc = $doc])*
#[doc = "Sets this field to the compile-time constant `VALUE`."]
#[inline(always)]
$vis const fn [<with_const_ $field>]<const VALUE: $storage>(self) -> Self {
self.[<__with_ $field>](
::kernel::num::Bounded::<$storage, { $hi + 1 - $lo }>::new::<VALUE>()
)
}
$(#[doc = $doc])*
#[doc = "Sets this field to the given `value`."]
#[inline(always)]
$vis fn [<with_ $field>]<T>(
self,
value: T,
) -> Self
where T: ::core::convert::Into<::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>>,
{
self.[<__with_ $field>](value.into())
}
$(#[doc = $doc])*
#[doc = "Tries to set this field to `value`, returning an error if it is out of range."]
#[inline(always)]
$vis fn [<try_with_ $field>]<T>(
self,
value: T,
) -> ::kernel::error::Result<Self>
where T: ::kernel::num::TryIntoBounded<$storage, { $hi + 1 - $lo }>,
{
Ok(
self.[<__with_ $field>](
value.try_into_bounded().ok_or(::kernel::error::code::EOVERFLOW)?
)
)
}
);
};
// `Debug` implementation.
(@debug $name:ident { $($field:ident;)* }) => {
impl ::kernel::fmt::Debug for $name {
#[inline]
fn fmt(&self, f: &mut ::kernel::fmt::Formatter<'_>) -> ::kernel::fmt::Result {
f.debug_struct(stringify!($name))
.field("<raw>", &::kernel::prelude::fmt!("{:#x}", self.inner))
$(
.field(stringify!($field), &self.$field())
)*
.finish()
}
}
};
}
#[cfg(CONFIG_RUST_BITFIELD_KUNIT_TEST)]
#[::kernel::macros::kunit_tests(rust_kernel_bitfield)]
mod tests {
use core::convert::TryFrom;
use pin_init::Zeroable;
use kernel::num::Bounded;
// Enum types for testing `=>` and `?=>` conversions.
#[derive(Debug, Clone, Copy, PartialEq)]
enum MemoryType {
Unmapped = 0,
Normal = 1,
Device = 2,
Reserved = 3,
}
impl TryFrom<Bounded<u64, 4>> for MemoryType {
type Error = u64;
fn try_from(value: Bounded<u64, 4>) -> Result<Self, Self::Error> {
match value.get() {
0 => Ok(MemoryType::Unmapped),
1 => Ok(MemoryType::Normal),
2 => Ok(MemoryType::Device),
3 => Ok(MemoryType::Reserved),
_ => Err(value.get()),
}
}
}
impl From<MemoryType> for Bounded<u64, 4> {
fn from(mt: MemoryType) -> Bounded<u64, 4> {
Bounded::from_expr(mt as u64)
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
enum Priority {
Low = 0,
Medium = 1,
High = 2,
Critical = 3,
}
impl From<Bounded<u16, 2>> for Priority {
fn from(value: Bounded<u16, 2>) -> Self {
match value & 0x3 {
0 => Priority::Low,
1 => Priority::Medium,
2 => Priority::High,
_ => Priority::Critical,
}
}
}
impl From<Priority> for Bounded<u16, 2> {
fn from(p: Priority) -> Bounded<u16, 2> {
Bounded::from_expr(p as u16)
}
}
bitfield! {
struct TestU64(u64) {
63:63 field_63;
61:52 field_61_52;
51:16 field_51_16;
15:12 field_15_12 ?=> MemoryType;
11:9 field_11_9;
1:1 field_1;
0:0 field_0;
}
}
bitfield! {
struct TestU16(u16) {
15:8 field_15_8;
7:4 field_7_4; // Partial overlap with `field_5_4`.
5:4 field_5_4 => Priority;
3:1 field_3_1;
0:0 field_0;
}
}
bitfield! {
struct TestU8(u8) {
7:0 field_7_0; // Full byte overlap.
7:4 field_7_4;
3:2 field_3_2;
1:1 field_1;
0:0 field_0;
}
}
// Single and multi-bit fields basic access.
#[test]
fn test_basic_access() {
// `TestU64`.
let mut val = TestU64::zeroed();
assert_eq!(val.into_raw(), 0x0);
val = val.with_field_0(true);
assert!(val.field_0().into_bool());
assert_eq!(val.into_raw(), 0x1);
val = val.with_field_1(true);
assert!(val.field_1().into_bool());
val = val.with_field_1(false);
assert!(!val.field_1().into_bool());
assert_eq!(val.into_raw(), 0x1);
val = val.with_const_field_11_9::<0x5>();
assert_eq!(val.field_11_9(), 0x5);
assert_eq!(val.into_raw(), 0xA01);
val = val.with_const_field_51_16::<0x123456>();
assert_eq!(val.field_51_16(), 0x123456);
assert_eq!(val.into_raw(), 0x0012_3456_0A01);
const MAX_FIELD_51_16: u64 = ::kernel::bits::genmask_u64(0..=35);
val = val.with_const_field_51_16::<{ MAX_FIELD_51_16 }>();
assert_eq!(val.field_51_16(), MAX_FIELD_51_16);
val = val.with_const_field_61_52::<0x3FF>();
assert_eq!(val.field_61_52(), 0x3FF);
val = val.with_field_63(true);
assert!(val.field_63().into_bool());
// `TestU16`.
let mut val = TestU16::zeroed();
assert_eq!(val.into_raw(), 0x0);
val = val.with_field_0(true);
assert!(val.field_0().into_bool());
assert_eq!(val.into_raw(), 0x1);
val = val.with_const_field_3_1::<0x5>();
assert_eq!(val.field_3_1(), 0x5);
assert_eq!(val.into_raw(), 0xB);
val = val.with_const_field_7_4::<0xA>();
assert_eq!(val.field_7_4(), 0xA);
assert_eq!(val.into_raw(), 0xAB);
val = val.with_const_field_15_8::<0x42>();
assert_eq!(val.field_15_8(), 0x42);
assert_eq!(val.into_raw(), 0x42AB);
// `TestU8`.
let mut val = TestU8::zeroed();
assert_eq!(val.into_raw(), 0x0);
val = val.with_field_0(true);
assert!(val.field_0().into_bool());
assert_eq!(val.into_raw(), 0x1);
val = val.with_field_1(true);
assert!(val.field_1().into_bool());
assert_eq!(val.into_raw(), 0x3);
val = val.with_const_field_3_2::<0x3>();
assert_eq!(val.field_3_2(), 0x3);
assert_eq!(val.into_raw(), 0xF);
val = val.with_const_field_7_4::<0xA>();
assert_eq!(val.field_7_4(), 0xA);
assert_eq!(val.into_raw(), 0xAF);
}
// `=>` infallible conversion.
#[test]
fn test_infallible_conversion() {
let mut val = TestU16::zeroed();
val = val.with_field_5_4(Priority::Low);
assert_eq!(val.field_5_4(), Priority::Low);
assert_eq!(val.into_raw() & 0x30, 0x00);
val = val.with_field_5_4(Priority::Medium);
assert_eq!(val.field_5_4(), Priority::Medium);
assert_eq!(val.into_raw() & 0x30, 0x10);
val = val.with_field_5_4(Priority::High);
assert_eq!(val.field_5_4(), Priority::High);
assert_eq!(val.into_raw() & 0x30, 0x20);
val = val.with_field_5_4(Priority::Critical);
assert_eq!(val.field_5_4(), Priority::Critical);
assert_eq!(val.into_raw() & 0x30, 0x30);
}
// `?=>` fallible conversion.
#[test]
fn test_fallible_conversion() {
let mut val = TestU64::zeroed();
val = val.with_field_15_12(MemoryType::Unmapped);
assert_eq!(val.field_15_12(), Ok(MemoryType::Unmapped));
val = val.with_field_15_12(MemoryType::Normal);
assert_eq!(val.field_15_12(), Ok(MemoryType::Normal));
val = val.with_field_15_12(MemoryType::Device);
assert_eq!(val.field_15_12(), Ok(MemoryType::Device));
val = val.with_field_15_12(MemoryType::Reserved);
assert_eq!(val.field_15_12(), Ok(MemoryType::Reserved));
// `field_15_12` is 4 bits wide (0-15); `MemoryType` only covers 0-3, so 4-15 return `Err`.
let raw = (val.into_raw() & !::kernel::bits::genmask_u64(12..=15)) | (0x7 << 12);
assert_eq!(TestU64::from_raw(raw).field_15_12(), Err(0x7));
}
// Test that setting an overlapping field affects the overlapped one as expected.
#[test]
fn test_overlapping_fields() {
let mut val = TestU16::zeroed();
val = val.with_field_5_4(Priority::High); // High == 2 == 0b10.
assert_eq!(val.field_5_4(), Priority::High);
assert_eq!(val.field_7_4(), 0x2); // Bits 7:6 == 0, bits 5:4 == 0b10.
val = val.with_const_field_7_4::<0xF>();
assert_eq!(val.field_7_4(), 0xF);
assert_eq!(val.field_5_4(), Priority::Critical); // Bits 5:4 == 0b11.
// `field_7_0` should encompass all other fields.
let mut val = TestU8::zeroed()
.with_field_0(true)
.with_field_1(true)
.with_const_field_3_2::<0x3>()
.with_const_field_7_4::<0xA>();
assert_eq!(val.into_raw(), 0xAF);
val = val.with_field_7_0(0x55);
assert_eq!(val.field_7_0(), 0x55);
assert!(val.field_0().into_bool());
assert!(!val.field_1().into_bool());
assert_eq!(val.field_3_2(), 0x1);
assert_eq!(val.field_7_4(), 0x5);
}
// Checks that bits not mapped to any field are left untouched.
#[test]
fn test_unallocated_bits() {
let gap_bits = (1u64 << 62) | 0x1FC;
let set_all_fields = |val: TestU64| {
val.with_field_63(true)
.with_const_field_61_52::<0x155>()
.with_const_field_51_16::<0x123456>()
.with_field_15_12(MemoryType::Device)
.with_const_field_11_9::<0x5>()
.with_field_1(true)
.with_field_0(true)
};
// Gap bits to 0.
let val = set_all_fields(TestU64::from_raw(0));
assert_eq!(val.into_raw() & gap_bits, 0);
// Gap bits to 1.
let val = set_all_fields(TestU64::from_raw(gap_bits));
assert_eq!(val.into_raw() & gap_bits, gap_bits);
}
#[test]
fn test_try_with() {
let val = TestU64::zeroed().try_with_field_51_16(0x123456).unwrap();
assert_eq!(val.field_51_16(), 0x123456);
let err = TestU64::zeroed().try_with_field_51_16(u64::MAX);
assert_eq!(err, Err(::kernel::error::code::EOVERFLOW));
let val = TestU64::zeroed()
.try_with_field_51_16(0xABCDEF)
.and_then(|p| p.try_with_field_0(1))
.unwrap();
assert_eq!(val.field_51_16(), 0xABCDEF);
assert!(val.field_0().into_bool());
}
// `from_raw`/`into_raw` and `From`/`Into` round-trips.
#[test]
fn test_raw() {
let raw: u64 = 0xBFF0_0000_3123_3E03;
let val = TestU64::from_raw(raw);
assert_eq!(u64::from(val), raw);
assert!(val.field_0().into_bool());
assert!(val.field_1().into_bool());
assert_eq!(val.field_11_9(), 0x7);
assert_eq!(val.field_51_16(), 0x3123);
assert_eq!(val.field_15_12(), Ok(MemoryType::Reserved));
assert_eq!(val.field_61_52(), 0x3FF);
assert!(val.field_63().into_bool());
let raw: u16 = 0x42AB;
let val = TestU16::from_raw(raw);
assert_eq!(u16::from(val), raw);
assert!(val.field_0().into_bool());
assert_eq!(val.field_3_1(), 0x5);
assert_eq!(val.field_7_4(), 0xA);
assert_eq!(val.field_15_8(), 0x42);
let raw: u8 = 0xAF;
let val = TestU8::from_raw(raw);
assert_eq!(u8::from(val), raw);
assert!(val.field_0().into_bool());
assert!(val.field_1().into_bool());
assert_eq!(val.field_3_2(), 0x3);
assert_eq!(val.field_7_4(), 0xA);
assert_eq!(val.field_7_0(), 0xAF);
}
}
linux-7.2.3/rust/kernel/bitmap.rs
+620
View File
@@ -0,0 +1,620 @@
// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2025 Google LLC.
//! Rust API for bitmap.
//!
//! C headers: [`include/linux/bitmap.h`](srctree/include/linux/bitmap.h).
use crate::alloc::{AllocError, Flags};
use crate::bindings;
#[cfg(not(CONFIG_RUST_BITMAP_HARDENED))]
use crate::pr_err;
use core::ptr::NonNull;
/// Represents a C bitmap. Wraps underlying C bitmap API.
///
/// # Invariants
///
/// Must reference a `[c_ulong]` long enough to fit `data.len()` bits.
#[cfg_attr(CONFIG_64BIT, repr(align(8)))]
#[cfg_attr(not(CONFIG_64BIT), repr(align(4)))]
pub struct Bitmap {
data: [()],
}
impl Bitmap {
/// Borrows a C bitmap.
///
/// # Safety
///
/// * `ptr` holds a non-null address of an initialized array of `unsigned long`
/// that is large enough to hold `nbits` bits.
/// * the array must not be freed for the lifetime of this [`Bitmap`]
/// * concurrent access only happens through atomic operations
pub unsafe fn from_raw<'a>(ptr: *const usize, nbits: usize) -> &'a Bitmap {
let data: *const [()] = core::ptr::slice_from_raw_parts(ptr.cast(), nbits);
// INVARIANT: `data` references an initialized array that can hold `nbits` bits.
// SAFETY:
// The caller guarantees that `data` (derived from `ptr` and `nbits`)
// points to a valid, initialized, and appropriately sized memory region
// that will not be freed for the lifetime 'a.
// We are casting `*const [()]` to `*const Bitmap`. The `Bitmap`
// struct is a ZST with a `data: [()]` field. This means its layout
// is compatible with a slice of `()`, and effectively it's a "thin pointer"
// (its size is 0 and alignment is 1). The `slice_from_raw_parts`
// function correctly encodes the length (number of bits, not elements)
// into the metadata of the fat pointer. Therefore, dereferencing this
// pointer as `&Bitmap` is safe given the caller's guarantees.
unsafe { &*(data as *const Bitmap) }
}
/// Borrows a C bitmap exclusively.
///
/// # Safety
///
/// * `ptr` holds a non-null address of an initialized array of `unsigned long`
/// that is large enough to hold `nbits` bits.
/// * the array must not be freed for the lifetime of this [`Bitmap`]
/// * no concurrent access may happen.
pub unsafe fn from_raw_mut<'a>(ptr: *mut usize, nbits: usize) -> &'a mut Bitmap {
let data: *mut [()] = core::ptr::slice_from_raw_parts_mut(ptr.cast(), nbits);
// INVARIANT: `data` references an initialized array that can hold `nbits` bits.
// SAFETY:
// The caller guarantees that `data` (derived from `ptr` and `nbits`)
// points to a valid, initialized, and appropriately sized memory region
// that will not be freed for the lifetime 'a.
// Furthermore, the caller guarantees no concurrent access will happen,
// which upholds the exclusivity requirement for a mutable reference.
// Similar to `from_raw`, casting `*mut [()]` to `*mut Bitmap` is
// safe because `Bitmap` is a ZST with a `data: [()]` field,
// making its layout compatible with a slice of `()`.
unsafe { &mut *(data as *mut Bitmap) }
}
/// Returns a raw pointer to the backing [`Bitmap`].
pub fn as_ptr(&self) -> *const usize {
core::ptr::from_ref::<Bitmap>(self).cast::<usize>()
}
/// Returns a mutable raw pointer to the backing [`Bitmap`].
pub fn as_mut_ptr(&mut self) -> *mut usize {
core::ptr::from_mut::<Bitmap>(self).cast::<usize>()
}
/// Returns length of this [`Bitmap`].
#[expect(clippy::len_without_is_empty)]
pub fn len(&self) -> usize {
self.data.len()
}
}
/// Holds either a pointer to array of `unsigned long` or a small bitmap.
#[repr(C)]
union BitmapRepr {
bitmap: usize,
ptr: NonNull<usize>,
}
macro_rules! bitmap_assert {
($cond:expr, $($arg:tt)+) => {
#[cfg(CONFIG_RUST_BITMAP_HARDENED)]
assert!($cond, $($arg)*);
}
}
macro_rules! bitmap_assert_return {
($cond:expr, $($arg:tt)+) => {
#[cfg(CONFIG_RUST_BITMAP_HARDENED)]
assert!($cond, $($arg)*);
#[cfg(not(CONFIG_RUST_BITMAP_HARDENED))]
if !($cond) {
pr_err!($($arg)*);
return
}
}
}
/// Represents an owned bitmap.
///
/// Wraps underlying C bitmap API. See [`Bitmap`] for available
/// methods.
///
/// # Examples
///
/// Basic usage
///
/// ```
/// use kernel::alloc::flags::GFP_KERNEL;
/// use kernel::bitmap::BitmapVec;
///
/// let mut b = BitmapVec::new(16, GFP_KERNEL)?;
///
/// assert_eq!(16, b.len());
/// for i in 0..16 {
/// if i % 4 == 0 {
/// b.set_bit(i);
/// }
/// }
/// assert_eq!(Some(0), b.next_bit(0));
/// assert_eq!(Some(1), b.next_zero_bit(0));
/// assert_eq!(Some(4), b.next_bit(1));
/// assert_eq!(Some(5), b.next_zero_bit(4));
/// assert_eq!(Some(12), b.last_bit());
/// # Ok::<(), Error>(())
/// ```
///
/// # Invariants
///
/// * `nbits` is `<= MAX_LEN`.
/// * if `nbits <= MAX_INLINE_LEN`, then `repr` is a `usize`.
/// * otherwise, `repr` holds a non-null pointer to an initialized
/// array of `unsigned long` that is large enough to hold `nbits` bits.
pub struct BitmapVec {
/// Representation of bitmap.
repr: BitmapRepr,
/// Length of this bitmap. Must be `<= MAX_LEN`.
nbits: usize,
}
impl core::ops::Deref for BitmapVec {
type Target = Bitmap;
fn deref(&self) -> &Bitmap {
let ptr = if self.nbits <= BitmapVec::MAX_INLINE_LEN {
// SAFETY: Bitmap is represented inline.
#[allow(unused_unsafe, reason = "Safe since Rust 1.92.0")]
unsafe {
core::ptr::addr_of!(self.repr.bitmap)
}
} else {
// SAFETY: Bitmap is represented as array of `unsigned long`.
unsafe { self.repr.ptr.as_ptr() }
};
// SAFETY: We got the right pointer and invariants of [`Bitmap`] hold.
// An inline bitmap is treated like an array with single element.
unsafe { Bitmap::from_raw(ptr, self.nbits) }
}
}
impl core::ops::DerefMut for BitmapVec {
fn deref_mut(&mut self) -> &mut Bitmap {
let ptr = if self.nbits <= BitmapVec::MAX_INLINE_LEN {
// SAFETY: Bitmap is represented inline.
#[allow(unused_unsafe, reason = "Safe since Rust 1.92.0")]
unsafe {
core::ptr::addr_of_mut!(self.repr.bitmap)
}
} else {
// SAFETY: Bitmap is represented as array of `unsigned long`.
unsafe { self.repr.ptr.as_ptr() }
};
// SAFETY: We got the right pointer and invariants of [`BitmapVec`] hold.
// An inline bitmap is treated like an array with single element.
unsafe { Bitmap::from_raw_mut(ptr, self.nbits) }
}
}
/// Enable ownership transfer to other threads.
///
/// SAFETY: We own the underlying bitmap representation.
unsafe impl Send for BitmapVec {}
/// Enable unsynchronized concurrent access to [`BitmapVec`] through shared references.
///
/// SAFETY: `deref()` will return a reference to a [`Bitmap`]. Its methods
/// take immutable references are either atomic or read-only.
unsafe impl Sync for BitmapVec {}
impl Drop for BitmapVec {
fn drop(&mut self) {
if self.nbits <= BitmapVec::MAX_INLINE_LEN {
return;
}
// SAFETY: `self.ptr` was returned by the C `bitmap_zalloc`.
//
// INVARIANT: there is no other use of the `self.ptr` after this
// call and the value is being dropped so the broken invariant is
// not observable on function exit.
unsafe { bindings::bitmap_free(self.repr.ptr.as_ptr()) };
}
}
impl BitmapVec {
/// The maximum possible length of a `BitmapVec`.
pub const MAX_LEN: usize = i32::MAX as usize;
/// The maximum length that uses the inline representation.
pub const MAX_INLINE_LEN: usize = usize::BITS as usize;
/// Construct a longest possible inline [`BitmapVec`].
#[inline]
pub fn new_inline() -> Self {
// INVARIANT: `nbits <= MAX_INLINE_LEN`, so an inline bitmap is the right repr.
BitmapVec {
repr: BitmapRepr { bitmap: 0 },
nbits: BitmapVec::MAX_INLINE_LEN,
}
}
/// Constructs a new [`BitmapVec`].
///
/// Fails with [`AllocError`] when the [`BitmapVec`] could not be allocated. This
/// includes the case when `nbits` is greater than `MAX_LEN`.
#[inline]
pub fn new(nbits: usize, flags: Flags) -> Result<Self, AllocError> {
if nbits <= BitmapVec::MAX_INLINE_LEN {
return Ok(BitmapVec {
repr: BitmapRepr { bitmap: 0 },
nbits,
});
}
if nbits > Self::MAX_LEN {
return Err(AllocError);
}
let nbits_u32 = u32::try_from(nbits).unwrap();
// SAFETY: `MAX_INLINE_LEN < nbits` and `nbits <= MAX_LEN`.
let ptr = unsafe { bindings::bitmap_zalloc(nbits_u32, flags.as_raw()) };
let ptr = NonNull::new(ptr).ok_or(AllocError)?;
// INVARIANT: `ptr` returned by C `bitmap_zalloc` and `nbits` checked.
Ok(BitmapVec {
repr: BitmapRepr { ptr },
nbits,
})
}
/// Returns length of this [`Bitmap`].
#[allow(clippy::len_without_is_empty)]
#[inline]
pub fn len(&self) -> usize {
self.nbits
}
/// Fills this `Bitmap` with random bits.
#[cfg(CONFIG_FIND_BIT_BENCHMARK_RUST)]
pub fn fill_random(&mut self) {
// SAFETY: `self.as_mut_ptr` points to either an array of the
// appropriate length or one usize.
unsafe {
bindings::get_random_bytes(
self.as_mut_ptr().cast::<ffi::c_void>(),
usize::div_ceil(self.nbits, bindings::BITS_PER_LONG as usize)
* bindings::BITS_PER_LONG as usize
/ 8,
);
}
}
}
impl Bitmap {
/// Set bit with index `index`.
///
/// ATTENTION: `set_bit` is non-atomic, which differs from the naming
/// convention in C code. The corresponding C function is `__set_bit`.
///
/// If CONFIG_RUST_BITMAP_HARDENED is not enabled and `index` is greater than
/// or equal to `self.nbits`, does nothing.
///
/// # Panics
///
/// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `index` is greater than
/// or equal to `self.nbits`.
#[inline]
pub fn set_bit(&mut self, index: usize) {
bitmap_assert_return!(
index < self.len(),
"Bit `index` must be < {}, was {}",
self.len(),
index
);
// SAFETY: Bit `index` is within bounds.
unsafe { bindings::__set_bit(index, self.as_mut_ptr()) };
}
/// Set bit with index `index`, atomically.
///
/// This is a relaxed atomic operation (no implied memory barriers).
///
/// ATTENTION: The naming convention differs from C, where the corresponding
/// function is called `set_bit`.
///
/// If CONFIG_RUST_BITMAP_HARDENED is not enabled and `index` is greater than
/// or equal to `self.len()`, does nothing.
///
/// # Panics
///
/// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `index` is greater than
/// or equal to `self.len()`.
#[inline]
pub fn set_bit_atomic(&self, index: usize) {
bitmap_assert_return!(
index < self.len(),
"Bit `index` must be < {}, was {}",
self.len(),
index
);
// SAFETY: `index` is within bounds and the caller has ensured that
// there is no mix of non-atomic and atomic operations.
unsafe { bindings::set_bit(index, self.as_ptr().cast_mut()) };
}
/// Clear `index` bit.
///
/// ATTENTION: `clear_bit` is non-atomic, which differs from the naming
/// convention in C code. The corresponding C function is `__clear_bit`.
///
/// If CONFIG_RUST_BITMAP_HARDENED is not enabled and `index` is greater than
/// or equal to `self.len()`, does nothing.
///
/// # Panics
///
/// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `index` is greater than
/// or equal to `self.len()`.
#[inline]
pub fn clear_bit(&mut self, index: usize) {
bitmap_assert_return!(
index < self.len(),
"Bit `index` must be < {}, was {}",
self.len(),
index
);
// SAFETY: `index` is within bounds.
unsafe { bindings::__clear_bit(index, self.as_mut_ptr()) };
}
/// Clear `index` bit, atomically.
///
/// This is a relaxed atomic operation (no implied memory barriers).
///
/// ATTENTION: The naming convention differs from C, where the corresponding
/// function is called `clear_bit`.
///
/// If CONFIG_RUST_BITMAP_HARDENED is not enabled and `index` is greater than
/// or equal to `self.len()`, does nothing.
///
/// # Panics
///
/// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `index` is greater than
/// or equal to `self.len()`.
#[inline]
pub fn clear_bit_atomic(&self, index: usize) {
bitmap_assert_return!(
index < self.len(),
"Bit `index` must be < {}, was {}",
self.len(),
index
);
// SAFETY: `index` is within bounds and the caller has ensured that
// there is no mix of non-atomic and atomic operations.
unsafe { bindings::clear_bit(index, self.as_ptr().cast_mut()) };
}
/// Copy `src` into this [`Bitmap`] and set any remaining bits to zero.
///
/// # Examples
///
/// ```
/// use kernel::alloc::{AllocError, flags::GFP_KERNEL};
/// use kernel::bitmap::BitmapVec;
///
/// let mut long_bitmap = BitmapVec::new(256, GFP_KERNEL)?;
///
/// assert_eq!(None, long_bitmap.last_bit());
///
/// let mut short_bitmap = BitmapVec::new(16, GFP_KERNEL)?;
///
/// short_bitmap.set_bit(7);
/// long_bitmap.copy_and_extend(&short_bitmap);
/// assert_eq!(Some(7), long_bitmap.last_bit());
///
/// # Ok::<(), AllocError>(())
/// ```
#[inline]
pub fn copy_and_extend(&mut self, src: &Bitmap) {
let len = core::cmp::min(src.len(), self.len());
// SAFETY: access to `self` and `src` is within bounds.
unsafe {
bindings::bitmap_copy_and_extend(
self.as_mut_ptr(),
src.as_ptr(),
len as u32,
self.len() as u32,
)
};
}
/// Finds last set bit.
///
/// # Examples
///
/// ```
/// use kernel::alloc::{AllocError, flags::GFP_KERNEL};
/// use kernel::bitmap::BitmapVec;
///
/// let bitmap = BitmapVec::new(64, GFP_KERNEL)?;
///
/// match bitmap.last_bit() {
/// Some(idx) => {
/// pr_info!("The last bit has index {idx}.\n");
/// }
/// None => {
/// pr_info!("All bits in this bitmap are 0.\n");
/// }
/// }
/// # Ok::<(), AllocError>(())
/// ```
#[inline]
pub fn last_bit(&self) -> Option<usize> {
// SAFETY: `_find_next_bit` access is within bounds due to invariant.
let index = unsafe { bindings::_find_last_bit(self.as_ptr(), self.len()) };
if index >= self.len() {
None
} else {
Some(index)
}
}
/// Finds next set bit, starting from `start`.
///
/// Returns `None` if `start` is greater or equal to `self.nbits`.
#[inline]
pub fn next_bit(&self, start: usize) -> Option<usize> {
bitmap_assert!(
start < self.len(),
"`start` must be < {} was {}",
self.len(),
start
);
// SAFETY: `_find_next_bit` tolerates out-of-bounds arguments and returns a
// value larger than or equal to `self.len()` in that case.
let index = unsafe { bindings::_find_next_bit(self.as_ptr(), self.len(), start) };
if index >= self.len() {
None
} else {
Some(index)
}
}
/// Finds next zero bit, starting from `start`.
/// Returns `None` if `start` is greater than or equal to `self.len()`.
#[inline]
pub fn next_zero_bit(&self, start: usize) -> Option<usize> {
bitmap_assert!(
start < self.len(),
"`start` must be < {} was {}",
self.len(),
start
);
// SAFETY: `_find_next_zero_bit` tolerates out-of-bounds arguments and returns a
// value larger than or equal to `self.len()` in that case.
let index = unsafe { bindings::_find_next_zero_bit(self.as_ptr(), self.len(), start) };
if index >= self.len() {
None
} else {
Some(index)
}
}
}
#[cfg(CONFIG_RUST_BITMAP_KUNIT_TEST)]
#[macros::kunit_tests(rust_kernel_bitmap)]
mod tests {
use super::*;
use kernel::alloc::flags::GFP_KERNEL;
#[test]
fn bitmap_borrow() {
let fake_bitmap: [usize; 2] = [0, 0];
let fake_bitmap_len = 2 * usize::BITS as usize;
// SAFETY: `fake_c_bitmap` is an array of expected length.
let b = unsafe { Bitmap::from_raw(fake_bitmap.as_ptr(), fake_bitmap_len) };
assert_eq!(fake_bitmap_len, b.len());
assert_eq!(None, b.next_bit(0));
}
#[test]
fn bitmap_copy() {
let fake_bitmap: usize = 0xFF;
// SAFETY: `fake_c_bitmap` can be used as one-element array of expected length.
let b = unsafe { Bitmap::from_raw(core::ptr::addr_of!(fake_bitmap), 8) };
assert_eq!(8, b.len());
assert_eq!(None, b.next_zero_bit(0));
}
#[test]
fn bitmap_vec_new() -> Result<(), AllocError> {
let b = BitmapVec::new(0, GFP_KERNEL)?;
assert_eq!(0, b.len());
let b = BitmapVec::new(3, GFP_KERNEL)?;
assert_eq!(3, b.len());
let b = BitmapVec::new(1024, GFP_KERNEL)?;
assert_eq!(1024, b.len());
// Requesting too large values results in [`AllocError`].
let res = BitmapVec::new(1 << 31, GFP_KERNEL);
assert!(res.is_err());
Ok(())
}
#[test]
fn bitmap_set_clear_find() -> Result<(), AllocError> {
let mut b = BitmapVec::new(128, GFP_KERNEL)?;
// Zero-initialized
assert_eq!(None, b.next_bit(0));
assert_eq!(Some(0), b.next_zero_bit(0));
assert_eq!(None, b.last_bit());
b.set_bit(17);
assert_eq!(Some(17), b.next_bit(0));
assert_eq!(Some(17), b.next_bit(17));
assert_eq!(None, b.next_bit(18));
assert_eq!(Some(17), b.last_bit());
b.set_bit(107);
assert_eq!(Some(17), b.next_bit(0));
assert_eq!(Some(17), b.next_bit(17));
assert_eq!(Some(107), b.next_bit(18));
assert_eq!(Some(107), b.last_bit());
b.clear_bit(17);
assert_eq!(Some(107), b.next_bit(0));
assert_eq!(Some(107), b.last_bit());
Ok(())
}
#[test]
fn owned_bitmap_out_of_bounds() -> Result<(), AllocError> {
// TODO: Kunit #[test]s do not support `cfg` yet,
// so we add it here in the body.
#[cfg(not(CONFIG_RUST_BITMAP_HARDENED))]
{
let mut b = BitmapVec::new(128, GFP_KERNEL)?;
b.set_bit(2048);
b.set_bit_atomic(2048);
b.clear_bit(2048);
b.clear_bit_atomic(2048);
assert_eq!(None, b.next_bit(2048));
assert_eq!(None, b.next_zero_bit(2048));
assert_eq!(None, b.last_bit());
}
Ok(())
}
// TODO: uncomment once kunit supports [should_panic] and `cfg`.
// #[cfg(CONFIG_RUST_BITMAP_HARDENED)]
// #[test]
// #[should_panic]
// fn owned_bitmap_out_of_bounds() -> Result<(), AllocError> {
// let mut b = BitmapVec::new(128, GFP_KERNEL)?;
//
// b.set_bit(2048);
// }
#[test]
fn bitmap_copy_and_extend() -> Result<(), AllocError> {
let mut long_bitmap = BitmapVec::new(256, GFP_KERNEL)?;
long_bitmap.set_bit(3);
long_bitmap.set_bit(200);
let mut short_bitmap = BitmapVec::new(32, GFP_KERNEL)?;
short_bitmap.set_bit(17);
long_bitmap.copy_and_extend(&short_bitmap);
// Previous bits have been cleared.
assert_eq!(Some(17), long_bitmap.next_bit(0));
assert_eq!(Some(17), long_bitmap.last_bit());
Ok(())
}
}
linux-7.2.3/rust/kernel/bits.rs
+205
View File
@@ -0,0 +1,205 @@
// SPDX-License-Identifier: GPL-2.0
//! Bit manipulation macros.
//!
//! C header: [`include/linux/bits.h`](srctree/include/linux/bits.h)
use crate::prelude::*;
use core::ops::RangeInclusive;
use macros::paste;
macro_rules! impl_bit_fn {
(
$ty:ty
) => {
paste! {
/// Computes `1 << n` if `n` is in bounds, i.e.: if `n` is smaller than
/// the maximum number of bits supported by the type.
///
/// Returns [`None`] otherwise.
#[inline]
pub fn [<checked_bit_ $ty>](n: u32) -> Option<$ty> {
(1 as $ty).checked_shl(n)
}
/// Computes `1 << n` by performing a compile-time assertion that `n` is
/// in bounds.
///
/// This version is the default and should be used if `n` is known at
/// compile time.
// Always inline to optimize out error path of `build_assert`.
#[inline(always)]
pub const fn [<bit_ $ty>](n: u32) -> $ty {
build_assert!(n < <$ty>::BITS);
(1 as $ty) << n
}
}
};
}
impl_bit_fn!(u64);
impl_bit_fn!(u32);
impl_bit_fn!(u16);
impl_bit_fn!(u8);
macro_rules! impl_genmask_fn {
(
$ty:ty,
$(#[$genmask_checked_ex:meta])*,
$(#[$genmask_ex:meta])*
) => {
paste! {
/// Creates a contiguous bitmask for the given range by validating
/// the range at runtime.
///
/// Returns [`None`] if the range is invalid, i.e.: if the start is
/// greater than the end or if the range is outside of the
/// representable range for the type.
$(#[$genmask_checked_ex])*
#[inline]
pub fn [<genmask_checked_ $ty>](range: RangeInclusive<u32>) -> Option<$ty> {
let start = *range.start();
let end = *range.end();
if start > end {
return None;
}
let high = [<checked_bit_ $ty>](end)?;
let low = [<checked_bit_ $ty>](start)?;
Some((high | (high - 1)) & !(low - 1))
}
/// Creates a compile-time contiguous bitmask for the given range by
/// performing a compile-time assertion that the range is valid.
///
/// This version is the default and should be used if the range is known
/// at compile time.
$(#[$genmask_ex])*
// Always inline to optimize out error path of `build_assert`.
#[inline(always)]
pub const fn [<genmask_ $ty>](range: RangeInclusive<u32>) -> $ty {
let start = *range.start();
let end = *range.end();
build_assert!(start <= end);
let high = [<bit_ $ty>](end);
let low = [<bit_ $ty>](start);
(high | (high - 1)) & !(low - 1)
}
}
};
}
impl_genmask_fn!(
u64,
/// # Examples
///
/// ```
/// # #![expect(clippy::reversed_empty_ranges)]
/// # use kernel::bits::genmask_checked_u64;
/// assert_eq!(genmask_checked_u64(0..=0), Some(0b1));
/// assert_eq!(genmask_checked_u64(0..=63), Some(u64::MAX));
/// assert_eq!(genmask_checked_u64(21..=39), Some(0x0000_00ff_ffe0_0000));
///
/// // `80` is out of the supported bit range.
/// assert_eq!(genmask_checked_u64(21..=80), None);
///
/// // Invalid range where the start is bigger than the end.
/// assert_eq!(genmask_checked_u64(15..=8), None);
/// ```
,
/// # Examples
///
/// ```
/// # use kernel::bits::genmask_u64;
/// assert_eq!(genmask_u64(21..=39), 0x0000_00ff_ffe0_0000);
/// assert_eq!(genmask_u64(0..=0), 0b1);
/// assert_eq!(genmask_u64(0..=63), u64::MAX);
/// ```
);
impl_genmask_fn!(
u32,
/// # Examples
///
/// ```
/// # #![expect(clippy::reversed_empty_ranges)]
/// # use kernel::bits::genmask_checked_u32;
/// assert_eq!(genmask_checked_u32(0..=0), Some(0b1));
/// assert_eq!(genmask_checked_u32(0..=31), Some(u32::MAX));
/// assert_eq!(genmask_checked_u32(21..=31), Some(0xffe0_0000));
///
/// // `40` is out of the supported bit range.
/// assert_eq!(genmask_checked_u32(21..=40), None);
///
/// // Invalid range where the start is bigger than the end.
/// assert_eq!(genmask_checked_u32(15..=8), None);
/// ```
,
/// # Examples
///
/// ```
/// # use kernel::bits::genmask_u32;
/// assert_eq!(genmask_u32(21..=31), 0xffe0_0000);
/// assert_eq!(genmask_u32(0..=0), 0b1);
/// assert_eq!(genmask_u32(0..=31), u32::MAX);
/// ```
);
impl_genmask_fn!(
u16,
/// # Examples
///
/// ```
/// # #![expect(clippy::reversed_empty_ranges)]
/// # use kernel::bits::genmask_checked_u16;
/// assert_eq!(genmask_checked_u16(0..=0), Some(0b1));
/// assert_eq!(genmask_checked_u16(0..=15), Some(u16::MAX));
/// assert_eq!(genmask_checked_u16(6..=15), Some(0xffc0));
///
/// // `20` is out of the supported bit range.
/// assert_eq!(genmask_checked_u16(6..=20), None);
///
/// // Invalid range where the start is bigger than the end.
/// assert_eq!(genmask_checked_u16(10..=5), None);
/// ```
,
/// # Examples
///
/// ```
/// # use kernel::bits::genmask_u16;
/// assert_eq!(genmask_u16(6..=15), 0xffc0);
/// assert_eq!(genmask_u16(0..=0), 0b1);
/// assert_eq!(genmask_u16(0..=15), u16::MAX);
/// ```
);
impl_genmask_fn!(
u8,
/// # Examples
///
/// ```
/// # #![expect(clippy::reversed_empty_ranges)]
/// # use kernel::bits::genmask_checked_u8;
/// assert_eq!(genmask_checked_u8(0..=0), Some(0b1));
/// assert_eq!(genmask_checked_u8(0..=7), Some(u8::MAX));
/// assert_eq!(genmask_checked_u8(6..=7), Some(0xc0));
///
/// // `10` is out of the supported bit range.
/// assert_eq!(genmask_checked_u8(6..=10), None);
///
/// // Invalid range where the start is bigger than the end.
/// assert_eq!(genmask_checked_u8(5..=2), None);
/// ```
,
/// # Examples
///
/// ```
/// # use kernel::bits::genmask_u8;
/// assert_eq!(genmask_u8(6..=7), 0xc0);
/// assert_eq!(genmask_u8(0..=0), 0b1);
/// assert_eq!(genmask_u8(0..=7), u8::MAX);
/// ```
);
linux-7.2.3/rust/kernel/block.rs
+18
View File
@@ -0,0 +1,18 @@
// SPDX-License-Identifier: GPL-2.0
//! Types for working with the block layer.
pub mod mq;
/// Bit mask for masking out [`SECTOR_SIZE`].
pub const SECTOR_MASK: u32 = bindings::SECTOR_MASK;
/// Sectors are size `1 << SECTOR_SHIFT`.
pub const SECTOR_SHIFT: u32 = bindings::SECTOR_SHIFT;
/// Size of a sector.
pub const SECTOR_SIZE: u32 = bindings::SECTOR_SIZE;
/// The difference between the size of a page and the size of a sector,
/// expressed as a power of two.
pub const PAGE_SECTORS_SHIFT: u32 = bindings::PAGE_SECTORS_SHIFT;
linux-7.2.3/rust/kernel/block/mq.rs
+103
View File
@@ -0,0 +1,103 @@
// SPDX-License-Identifier: GPL-2.0
//! This module provides types for implementing block drivers that interface the
//! blk-mq subsystem.
//!
//! To implement a block device driver, a Rust module must do the following:
//!
//! - Implement [`Operations`] for a type `T`.
//! - Create a [`TagSet<T>`].
//! - Create a [`GenDisk<T>`], via the [`GenDiskBuilder`].
//! - Add the disk to the system by calling [`GenDiskBuilder::build`] passing in
//! the `TagSet` reference.
//!
//! The types available in this module that have direct C counterparts are:
//!
//! - The [`TagSet`] type that abstracts the C type `struct tag_set`.
//! - The [`GenDisk`] type that abstracts the C type `struct gendisk`.
//! - The [`Request`] type that abstracts the C type `struct request`.
//!
//! The kernel will interface with the block device driver by calling the method
//! implementations of the `Operations` trait.
//!
//! IO requests are passed to the driver as [`kernel::sync::aref::ARef<Request>`]
//! instances. The `Request` type is a wrapper around the C `struct request`.
//! The driver must mark end of processing by calling one of the
//! `Request::end`, methods. Failure to do so can lead to deadlock or timeout
//! errors. Please note that the C function `blk_mq_start_request` is implicitly
//! called when the request is queued with the driver.
//!
//! The `TagSet` is responsible for creating and maintaining a mapping between
//! `Request`s and integer ids as well as carrying a pointer to the vtable
//! generated by `Operations`. This mapping is useful for associating
//! completions from hardware with the correct `Request` instance. The `TagSet`
//! determines the maximum queue depth by setting the number of `Request`
//! instances available to the driver, and it determines the number of queues to
//! instantiate for the driver. If possible, a driver should allocate one queue
//! per core, to keep queue data local to a core.
//!
//! One `TagSet` instance can be shared between multiple `GenDisk` instances.
//! This can be useful when implementing drivers where one piece of hardware
//! with one set of IO resources are represented to the user as multiple disks.
//!
//! One significant difference between block device drivers implemented with
//! these Rust abstractions and drivers implemented in C, is that the Rust
//! drivers have to own a reference count on the `Request` type when the IO is
//! in flight. This is to ensure that the C `struct request` instances backing
//! the Rust `Request` instances are live while the Rust driver holds a
//! reference to the `Request`. In addition, the conversion of an integer tag to
//! a `Request` via the `TagSet` would not be sound without this bookkeeping.
//!
//! [`GenDisk`]: gen_disk::GenDisk
//! [`GenDisk<T>`]: gen_disk::GenDisk
//! [`GenDiskBuilder`]: gen_disk::GenDiskBuilder
//! [`GenDiskBuilder::build`]: gen_disk::GenDiskBuilder::build
//!
//! # Examples
//!
//! ```rust
//! use kernel::{
//! alloc::flags,
//! block::mq::*,
//! new_mutex,
//! prelude::*,
//! sync::{aref::ARef, Arc, Mutex},
//! };
//!
//! struct MyBlkDevice;
//!
//! #[vtable]
//! impl Operations for MyBlkDevice {
//! type QueueData = ();
//!
//! fn queue_rq(_queue_data: (), rq: ARef<Request<Self>>, _is_last: bool) -> Result {
//! Request::end_ok(rq);
//! Ok(())
//! }
//!
//! fn commit_rqs(_queue_data: ()) {}
//!
//! fn complete(rq: ARef<Request<Self>>) {
//! Request::end_ok(rq)
//! .map_err(|_e| kernel::error::code::EIO)
//! .expect("Fatal error - expected to be able to end request");
//! }
//! }
//!
//! let tagset: Arc<TagSet<MyBlkDevice>> =
//! Arc::pin_init(TagSet::new(1, 256, 1), flags::GFP_KERNEL)?;
//! let mut disk = gen_disk::GenDiskBuilder::new()
//! .capacity_sectors(4096)
//! .build(fmt!("myblk"), tagset, ())?;
//!
//! # Ok::<(), kernel::error::Error>(())
//! ```
pub mod gen_disk;
mod operations;
mod request;
mod tag_set;
pub use operations::Operations;
pub use request::Request;
pub use tag_set::TagSet;
linux-7.2.3/rust/kernel/block/mq/gen_disk.rs
+241
View File
@@ -0,0 +1,241 @@
// SPDX-License-Identifier: GPL-2.0
//! Generic disk abstraction.
//!
//! C header: [`include/linux/blkdev.h`](srctree/include/linux/blkdev.h)
//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h)
use crate::{
bindings,
block::mq::{Operations, TagSet},
error::{self, from_err_ptr, Result},
fmt::{self, Write},
prelude::*,
static_lock_class,
str::NullTerminatedFormatter,
sync::Arc,
types::{ForeignOwnable, ScopeGuard},
};
/// A builder for [`GenDisk`].
///
/// Use this struct to configure and add new [`GenDisk`] to the VFS.
pub struct GenDiskBuilder {
rotational: bool,
logical_block_size: u32,
physical_block_size: u32,
capacity_sectors: u64,
}
impl Default for GenDiskBuilder {
fn default() -> Self {
Self {
rotational: false,
logical_block_size: bindings::PAGE_SIZE as u32,
physical_block_size: bindings::PAGE_SIZE as u32,
capacity_sectors: 0,
}
}
}
impl GenDiskBuilder {
/// Create a new instance.
pub fn new() -> Self {
Self::default()
}
/// Set the rotational media attribute for the device to be built.
pub fn rotational(mut self, rotational: bool) -> Self {
self.rotational = rotational;
self
}
/// Validate block size by verifying that it is between 512 and `PAGE_SIZE`,
/// and that it is a power of two.
pub fn validate_block_size(size: u32) -> Result {
if !(512..=bindings::PAGE_SIZE as u32).contains(&size) || !size.is_power_of_two() {
Err(error::code::EINVAL)
} else {
Ok(())
}
}
/// Set the logical block size of the device to be built.
///
/// This method will check that block size is a power of two and between 512
/// and 4096. If not, an error is returned and the block size is not set.
///
/// This is the smallest unit the storage device can address. It is
/// typically 4096 bytes.
pub fn logical_block_size(mut self, block_size: u32) -> Result<Self> {
Self::validate_block_size(block_size)?;
self.logical_block_size = block_size;
Ok(self)
}
/// Set the physical block size of the device to be built.
///
/// This method will check that block size is a power of two and between 512
/// and 4096. If not, an error is returned and the block size is not set.
///
/// This is the smallest unit a physical storage device can write
/// atomically. It is usually the same as the logical block size but may be
/// bigger. One example is SATA drives with 4096 byte physical block size
/// that expose a 512 byte logical block size to the operating system.
pub fn physical_block_size(mut self, block_size: u32) -> Result<Self> {
Self::validate_block_size(block_size)?;
self.physical_block_size = block_size;
Ok(self)
}
/// Set the capacity of the device to be built, in sectors (512 bytes).
pub fn capacity_sectors(mut self, capacity: u64) -> Self {
self.capacity_sectors = capacity;
self
}
/// Build a new `GenDisk` and add it to the VFS.
pub fn build<T: Operations>(
self,
name: fmt::Arguments<'_>,
tagset: Arc<TagSet<T>>,
queue_data: T::QueueData,
) -> Result<GenDisk<T>> {
let data = queue_data.into_foreign();
let recover_data = ScopeGuard::new(|| {
// SAFETY: T::QueueData was created by the call to `into_foreign()` above
drop(unsafe { T::QueueData::from_foreign(data) });
});
let mut lim: bindings::queue_limits = pin_init::zeroed();
lim.logical_block_size = self.logical_block_size;
lim.physical_block_size = self.physical_block_size;
if self.rotational {
lim.features = bindings::BLK_FEAT_ROTATIONAL;
}
// SAFETY: `tagset.raw_tag_set()` points to a valid and initialized tag set
let gendisk = from_err_ptr(unsafe {
bindings::__blk_mq_alloc_disk(
tagset.raw_tag_set(),
&mut lim,
data,
static_lock_class!().as_ptr(),
)
})?;
const TABLE: bindings::block_device_operations = bindings::block_device_operations {
submit_bio: None,
open: None,
release: None,
ioctl: None,
compat_ioctl: None,
check_events: None,
unlock_native_capacity: None,
getgeo: None,
set_read_only: None,
swap_slot_free_notify: None,
report_zones: None,
devnode: None,
alternative_gpt_sector: None,
get_unique_id: None,
// TODO: Set to `THIS_MODULE`.
owner: core::ptr::null_mut(),
pr_ops: core::ptr::null_mut(),
free_disk: None,
poll_bio: None,
};
// SAFETY: `gendisk` is a valid pointer as we initialized it above
unsafe { (*gendisk).fops = &TABLE };
let cleanup_failure = ScopeGuard::new_with_data((gendisk, data), |(gendisk, data)| {
// SAFETY: `gendisk` came from `__blk_mq_alloc_disk()` above and
// has not been added to the VFS on this cleanup path.
unsafe { bindings::put_disk(gendisk) };
// SAFETY: `data` came from `into_foreign()` above and has not been
// converted back on this cleanup path.
drop(unsafe { T::QueueData::from_foreign(data) });
});
// The failure guard now owns both pieces of cleanup; the early guard
// must not run on this path anymore.
recover_data.dismiss();
let mut writer = NullTerminatedFormatter::new(
// SAFETY: `gendisk` points to a valid and initialized instance. We
// have exclusive access, since the disk is not added to the VFS
// yet.
unsafe { &mut (*gendisk).disk_name },
)
.ok_or(EINVAL)?;
writer.write_fmt(name)?;
// SAFETY: `gendisk` points to a valid and initialized instance of
// `struct gendisk`. `set_capacity` takes a lock to synchronize this
// operation, so we will not race.
unsafe { bindings::set_capacity(gendisk, self.capacity_sectors) };
crate::error::to_result(
// SAFETY: `gendisk` points to a valid and initialized instance of
// `struct gendisk`.
unsafe {
bindings::device_add_disk(core::ptr::null_mut(), gendisk, core::ptr::null_mut())
},
)?;
cleanup_failure.dismiss();
// INVARIANT: `gendisk` was initialized above.
// INVARIANT: `gendisk` was added to the VFS via `device_add_disk` above.
// INVARIANT: `gendisk.queue.queue_data` is set to `data` in the call to
// `__blk_mq_alloc_disk` above.
Ok(GenDisk {
_tagset: tagset,
gendisk,
})
}
}
/// A generic block device.
///
/// # Invariants
///
/// - `gendisk` must always point to an initialized and valid `struct gendisk`.
/// - `gendisk` was added to the VFS through a call to
/// `bindings::device_add_disk`.
/// - `self.gendisk.queue.queuedata` is initialized by a call to `ForeignOwnable::into_foreign`.
pub struct GenDisk<T: Operations> {
_tagset: Arc<TagSet<T>>,
gendisk: *mut bindings::gendisk,
}
// SAFETY: `GenDisk` is an owned pointer to a `struct gendisk` and an `Arc` to a
// `TagSet` It is safe to send this to other threads as long as T is Send.
unsafe impl<T: Operations + Send> Send for GenDisk<T> {}
impl<T: Operations> Drop for GenDisk<T> {
fn drop(&mut self) {
// SAFETY: By type invariant of `Self`, `self.gendisk` points to a valid
// and initialized instance of `struct gendisk`, and, `queuedata` was
// initialized with the result of a call to
// `ForeignOwnable::into_foreign`.
let queue_data = unsafe { (*(*self.gendisk).queue).queuedata };
// SAFETY: By type invariant, `self.gendisk` points to a valid and
// initialized instance of `struct gendisk`, and it was previously added
// to the VFS.
unsafe { bindings::del_gendisk(self.gendisk) };
// SAFETY: By type invariant, `self.gendisk` was added to the VFS, so
// `put_disk()` must follow `del_gendisk()` to drop the final gendisk
// reference and trigger the remaining release path.
unsafe { bindings::put_disk(self.gendisk) };
// SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with
// a call to `ForeignOwnable::into_foreign` to create `queuedata`.
// `ForeignOwnable::from_foreign` is only called here.
drop(unsafe { T::QueueData::from_foreign(queue_data) });
}
}
linux-7.2.3/rust/kernel/block/mq/operations.rs
+286
View File
@@ -0,0 +1,286 @@
// SPDX-License-Identifier: GPL-2.0
//! This module provides an interface for blk-mq drivers to implement.
//!
//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h)
use crate::{
bindings,
block::mq::{request::RequestDataWrapper, Request},
error::{from_result, Result},
prelude::*,
sync::{aref::ARef, Refcount},
types::ForeignOwnable,
};
use core::marker::PhantomData;
type ForeignBorrowed<'a, T> = <T as ForeignOwnable>::Borrowed<'a>;
/// Implement this trait to interface blk-mq as block devices.
///
/// To implement a block device driver, implement this trait as described in the
/// [module level documentation]. The kernel will use the implementation of the
/// functions defined in this trait to interface a block device driver. Note:
/// There is no need for an exit_request() implementation, because the `drop`
/// implementation of the [`Request`] type will be invoked by automatically by
/// the C/Rust glue logic.
///
/// [module level documentation]: kernel::block::mq
#[macros::vtable]
pub trait Operations: Sized {
/// Data associated with the `struct request_queue` that is allocated for
/// the `GenDisk` associated with this `Operations` implementation.
type QueueData: ForeignOwnable;
/// Called by the kernel to queue a request with the driver. If `is_last` is
/// `false`, the driver is allowed to defer committing the request.
fn queue_rq(
queue_data: ForeignBorrowed<'_, Self::QueueData>,
rq: ARef<Request<Self>>,
is_last: bool,
) -> Result;
/// Called by the kernel to indicate that queued requests should be submitted.
fn commit_rqs(queue_data: ForeignBorrowed<'_, Self::QueueData>);
/// Called by the kernel when the request is completed.
fn complete(rq: ARef<Request<Self>>);
/// Called by the kernel to poll the device for completed requests. Only
/// used for poll queues.
fn poll() -> bool {
build_error!(crate::error::VTABLE_DEFAULT_ERROR)
}
}
/// A vtable for blk-mq to interact with a block device driver.
///
/// A `bindings::blk_mq_ops` vtable is constructed from pointers to the `extern
/// "C"` functions of this struct, exposed through the `OperationsVTable::VTABLE`.
///
/// For general documentation of these methods, see the kernel source
/// documentation related to `struct blk_mq_operations` in
/// [`include/linux/blk-mq.h`].
///
/// [`include/linux/blk-mq.h`]: srctree/include/linux/blk-mq.h
pub(crate) struct OperationsVTable<T: Operations>(PhantomData<T>);
impl<T: Operations> OperationsVTable<T> {
/// This function is called by the C kernel. A pointer to this function is
/// installed in the `blk_mq_ops` vtable for the driver.
///
/// # Safety
///
/// - The caller of this function must ensure that the pointee of `bd` is
/// valid for reads for the duration of this function.
/// - This function must be called for an initialized and live `hctx`. That
/// is, `Self::init_hctx_callback` was called and
/// `Self::exit_hctx_callback()` was not yet called.
/// - `(*bd).rq` must point to an initialized and live `bindings:request`.
/// That is, `Self::init_request_callback` was called but
/// `Self::exit_request_callback` was not yet called for the request.
/// - `(*bd).rq` must be owned by the driver. That is, the block layer must
/// promise to not access the request until the driver calls
/// `bindings::blk_mq_end_request` for the request.
unsafe extern "C" fn queue_rq_callback(
hctx: *mut bindings::blk_mq_hw_ctx,
bd: *const bindings::blk_mq_queue_data,
) -> bindings::blk_status_t {
// SAFETY: `bd.rq` is valid as required by the safety requirement for
// this function.
let request = unsafe { &*(*bd).rq.cast::<Request<T>>() };
// One refcount for the ARef, one for being in flight
request.wrapper_ref().refcount().set(2);
// SAFETY:
// - We own a refcount that we took above. We pass that to `ARef`.
// - By the safety requirements of this function, `request` is a valid
// `struct request` and the private data is properly initialized.
// - `rq` will be alive until `blk_mq_end_request` is called and is
// reference counted by `ARef` until then.
let rq = unsafe { Request::aref_from_raw((*bd).rq) };
// SAFETY: `hctx` is valid as required by this function.
let queue_data = unsafe { (*(*hctx).queue).queuedata };
// SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with
// a call to `ForeignOwnable::into_foreign` to create `queuedata`.
// `ForeignOwnable::from_foreign` is only called when the tagset is
// dropped, which happens after we are dropped.
let queue_data = unsafe { T::QueueData::borrow(queue_data) };
// SAFETY: We have exclusive access and we just set the refcount above.
unsafe { Request::start_unchecked(&rq) };
let ret = T::queue_rq(
queue_data,
rq,
// SAFETY: `bd` is valid as required by the safety requirement for
// this function.
unsafe { (*bd).last },
);
if let Err(e) = ret {
e.to_blk_status()
} else {
bindings::BLK_STS_OK as bindings::blk_status_t
}
}
/// This function is called by the C kernel. A pointer to this function is
/// installed in the `blk_mq_ops` vtable for the driver.
///
/// # Safety
///
/// This function may only be called by blk-mq C infrastructure. The caller
/// must ensure that `hctx` is valid.
unsafe extern "C" fn commit_rqs_callback(hctx: *mut bindings::blk_mq_hw_ctx) {
// SAFETY: `hctx` is valid as required by this function.
let queue_data = unsafe { (*(*hctx).queue).queuedata };
// SAFETY: `queue.queuedata` was created by `GenDisk::try_new()` with a
// call to `ForeignOwnable::into_foreign()` to create `queuedata`.
// `ForeignOwnable::from_foreign()` is only called when the tagset is
// dropped, which happens after we are dropped.
let queue_data = unsafe { T::QueueData::borrow(queue_data) };
T::commit_rqs(queue_data)
}
/// This function is called by the C kernel. A pointer to this function is
/// installed in the `blk_mq_ops` vtable for the driver.
///
/// # Safety
///
/// This function may only be called by blk-mq C infrastructure. `rq` must
/// point to a valid request that has been marked as completed. The pointee
/// of `rq` must be valid for write for the duration of this function.
unsafe extern "C" fn complete_callback(rq: *mut bindings::request) {
// SAFETY: This function can only be dispatched through
// `Request::complete`. We leaked a refcount then which we pick back up
// now.
let aref = unsafe { Request::aref_from_raw(rq) };
T::complete(aref);
}
/// This function is called by the C kernel. A pointer to this function is
/// installed in the `blk_mq_ops` vtable for the driver.
///
/// # Safety
///
/// This function may only be called by blk-mq C infrastructure.
unsafe extern "C" fn poll_callback(
_hctx: *mut bindings::blk_mq_hw_ctx,
_iob: *mut bindings::io_comp_batch,
) -> crate::ffi::c_int {
T::poll().into()
}
/// This function is called by the C kernel. A pointer to this function is
/// installed in the `blk_mq_ops` vtable for the driver.
///
/// # Safety
///
/// This function may only be called by blk-mq C infrastructure. This
/// function may only be called once before `exit_hctx_callback` is called
/// for the same context.
unsafe extern "C" fn init_hctx_callback(
_hctx: *mut bindings::blk_mq_hw_ctx,
_tagset_data: *mut crate::ffi::c_void,
_hctx_idx: crate::ffi::c_uint,
) -> crate::ffi::c_int {
from_result(|| Ok(0))
}
/// This function is called by the C kernel. A pointer to this function is
/// installed in the `blk_mq_ops` vtable for the driver.
///
/// # Safety
///
/// This function may only be called by blk-mq C infrastructure.
unsafe extern "C" fn exit_hctx_callback(
_hctx: *mut bindings::blk_mq_hw_ctx,
_hctx_idx: crate::ffi::c_uint,
) {
}
/// This function is called by the C kernel. A pointer to this function is
/// installed in the `blk_mq_ops` vtable for the driver.
///
/// # Safety
///
/// - This function may only be called by blk-mq C infrastructure.
/// - `_set` must point to an initialized `TagSet<T>`.
/// - `rq` must point to an initialized `bindings::request`.
/// - The allocation pointed to by `rq` must be at the size of `Request`
/// plus the size of `RequestDataWrapper`.
unsafe extern "C" fn init_request_callback(
_set: *mut bindings::blk_mq_tag_set,
rq: *mut bindings::request,
_hctx_idx: crate::ffi::c_uint,
_numa_node: crate::ffi::c_int,
) -> crate::ffi::c_int {
from_result(|| {
// SAFETY: By the safety requirements of this function, `rq` points
// to a valid allocation.
let pdu = unsafe { Request::wrapper_ptr(rq.cast::<Request<T>>()) };
// SAFETY: The refcount field is allocated but not initialized, so
// it is valid for writes.
unsafe { RequestDataWrapper::refcount_ptr(pdu.as_ptr()).write(Refcount::new(0)) };
Ok(0)
})
}
/// This function is called by the C kernel. A pointer to this function is
/// installed in the `blk_mq_ops` vtable for the driver.
///
/// # Safety
///
/// - This function may only be called by blk-mq C infrastructure.
/// - `_set` must point to an initialized `TagSet<T>`.
/// - `rq` must point to an initialized and valid `Request`.
unsafe extern "C" fn exit_request_callback(
_set: *mut bindings::blk_mq_tag_set,
rq: *mut bindings::request,
_hctx_idx: crate::ffi::c_uint,
) {
// SAFETY: The tagset invariants guarantee that all requests are allocated with extra memory
// for the request data.
let pdu = unsafe { bindings::blk_mq_rq_to_pdu(rq) }.cast::<RequestDataWrapper>();
// SAFETY: `pdu` is valid for read and write and is properly initialised.
unsafe { core::ptr::drop_in_place(pdu) };
}
const VTABLE: bindings::blk_mq_ops = bindings::blk_mq_ops {
queue_rq: Some(Self::queue_rq_callback),
queue_rqs: None,
commit_rqs: Some(Self::commit_rqs_callback),
get_budget: None,
put_budget: None,
set_rq_budget_token: None,
get_rq_budget_token: None,
timeout: None,
poll: if T::HAS_POLL {
Some(Self::poll_callback)
} else {
None
},
complete: Some(Self::complete_callback),
init_hctx: Some(Self::init_hctx_callback),
exit_hctx: Some(Self::exit_hctx_callback),
init_request: Some(Self::init_request_callback),
exit_request: Some(Self::exit_request_callback),
cleanup_rq: None,
busy: None,
map_queues: None,
#[cfg(CONFIG_BLK_DEBUG_FS)]
show_rq: None,
};
pub(crate) const fn build() -> &'static bindings::blk_mq_ops {
&Self::VTABLE
}
}
linux-7.2.3/rust/kernel/block/mq/request.rs
+257
View File
@@ -0,0 +1,257 @@
// SPDX-License-Identifier: GPL-2.0
//! This module provides a wrapper for the C `struct request` type.
//!
//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h)
use crate::{
bindings,
block::mq::Operations,
error::Result,
sync::{
aref::{ARef, AlwaysRefCounted},
atomic::Relaxed,
Refcount,
},
types::Opaque,
};
use core::{marker::PhantomData, ptr::NonNull};
/// A wrapper around a blk-mq [`struct request`]. This represents an IO request.
///
/// # Implementation details
///
/// There are four states for a request that the Rust bindings care about:
///
/// 1. Request is owned by block layer (refcount 0).
/// 2. Request is owned by driver but with zero [`ARef`]s in existence
/// (refcount 1).
/// 3. Request is owned by driver with exactly one [`ARef`] in existence
/// (refcount 2).
/// 4. Request is owned by driver with more than one [`ARef`] in existence
/// (refcount > 2).
///
///
/// We need to track 1 and 2 to ensure we fail tag to request conversions for
/// requests that are not owned by the driver.
///
/// We need to track 3 and 4 to ensure that it is safe to end the request and hand
/// back ownership to the block layer.
///
/// Note that the driver can still obtain new `ARef` even if there is no `ARef`s in existence by
/// using `tag_to_rq`, hence the need to distinguish B and C.
///
/// The states are tracked through the private `refcount` field of
/// `RequestDataWrapper`. This structure lives in the private data area of the C
/// [`struct request`].
///
/// # Invariants
///
/// * `self.0` is a valid [`struct request`] created by the C portion of the
/// kernel.
/// * The private data area associated with this request must be an initialized
/// and valid `RequestDataWrapper<T>`.
/// * `self` is reference counted by atomic modification of
/// `self.wrapper_ref().refcount()`.
///
/// [`struct request`]: srctree/include/linux/blk-mq.h
///
#[repr(transparent)]
pub struct Request<T>(Opaque<bindings::request>, PhantomData<T>);
impl<T: Operations> Request<T> {
/// Create an [`ARef<Request>`] from a [`struct request`] pointer.
///
/// # Safety
///
/// * The caller must own a refcount on `ptr` that is transferred to the
/// returned [`ARef`].
/// * The type invariants for [`Request`] must hold for the pointee of `ptr`.
///
/// [`struct request`]: srctree/include/linux/blk-mq.h
pub(crate) unsafe fn aref_from_raw(ptr: *mut bindings::request) -> ARef<Self> {
// INVARIANT: By the safety requirements of this function, invariants are upheld.
// SAFETY: By the safety requirement of this function, we own a
// reference count that we can pass to `ARef`.
unsafe { ARef::from_raw(NonNull::new_unchecked(ptr.cast())) }
}
/// Notify the block layer that a request is going to be processed now.
///
/// The block layer uses this hook to do proper initializations such as
/// starting the timeout timer. It is a requirement that block device
/// drivers call this function when starting to process a request.
///
/// # Safety
///
/// The caller must have exclusive ownership of `self`, that is
/// `self.wrapper_ref().refcount() == 2`.
pub(crate) unsafe fn start_unchecked(this: &ARef<Self>) {
// SAFETY: By type invariant, `self.0` is a valid `struct request` and
// we have exclusive access.
unsafe { bindings::blk_mq_start_request(this.0.get()) };
}
/// Try to take exclusive ownership of `this` by dropping the refcount to 0.
/// This fails if `this` is not the only [`ARef`] pointing to the underlying
/// [`Request`].
///
/// If the operation is successful, [`Ok`] is returned with a pointer to the
/// C [`struct request`]. If the operation fails, `this` is returned in the
/// [`Err`] variant.
///
/// [`struct request`]: srctree/include/linux/blk-mq.h
fn try_set_end(this: ARef<Self>) -> Result<*mut bindings::request, ARef<Self>> {
// To hand back the ownership, we need the current refcount to be 2.
// Since we can race with `TagSet::tag_to_rq`, this needs to atomically reduce
// refcount to 0. `Refcount` does not provide a way to do this, so use the underlying
// atomics directly.
if let Err(_old) = this
.wrapper_ref()
.refcount()
.as_atomic()
.cmpxchg(2, 0, Relaxed)
{
return Err(this);
}
let request_ptr = this.0.get();
core::mem::forget(this);
Ok(request_ptr)
}
/// Notify the block layer that the request has been completed without errors.
///
/// This function will return [`Err`] if `this` is not the only [`ARef`]
/// referencing the request.
pub fn end_ok(this: ARef<Self>) -> Result<(), ARef<Self>> {
let request_ptr = Self::try_set_end(this)?;
// SAFETY: By type invariant, `this.0` was a valid `struct request`. The
// success of the call to `try_set_end` guarantees that there are no
// `ARef`s pointing to this request. Therefore it is safe to hand it
// back to the block layer.
unsafe {
bindings::blk_mq_end_request(
request_ptr,
bindings::BLK_STS_OK as bindings::blk_status_t,
)
};
Ok(())
}
/// Complete the request by scheduling `Operations::complete` for
/// execution.
///
/// The function may be scheduled locally, via SoftIRQ or remotely via IPMI.
/// See `blk_mq_complete_request_remote` in [`blk-mq.c`] for details.
///
/// [`blk-mq.c`]: srctree/block/blk-mq.c
pub fn complete(this: ARef<Self>) {
let ptr = ARef::into_raw(this).cast::<bindings::request>().as_ptr();
// SAFETY: By type invariant, `self.0` is a valid `struct request`
if !unsafe { bindings::blk_mq_complete_request_remote(ptr) } {
// SAFETY: We released a refcount above that we can reclaim here.
let this = unsafe { Request::aref_from_raw(ptr) };
T::complete(this);
}
}
/// Return a pointer to the [`RequestDataWrapper`] stored in the private area
/// of the request structure.
///
/// # Safety
///
/// - `this` must point to a valid allocation of size at least size of
/// [`Self`] plus size of [`RequestDataWrapper`].
pub(crate) unsafe fn wrapper_ptr(this: *mut Self) -> NonNull<RequestDataWrapper> {
let request_ptr = this.cast::<bindings::request>();
// SAFETY: By safety requirements for this function, `this` is a
// valid allocation.
let wrapper_ptr =
unsafe { bindings::blk_mq_rq_to_pdu(request_ptr).cast::<RequestDataWrapper>() };
// SAFETY: By C API contract, `wrapper_ptr` points to a valid allocation
// and is not null.
unsafe { NonNull::new_unchecked(wrapper_ptr) }
}
/// Return a reference to the [`RequestDataWrapper`] stored in the private
/// area of the request structure.
pub(crate) fn wrapper_ref(&self) -> &RequestDataWrapper {
// SAFETY: By type invariant, `self.0` is a valid allocation. Further,
// the private data associated with this request is initialized and
// valid. The existence of `&self` guarantees that the private data is
// valid as a shared reference.
unsafe { Self::wrapper_ptr(core::ptr::from_ref(self).cast_mut()).as_ref() }
}
}
/// A wrapper around data stored in the private area of the C [`struct request`].
///
/// [`struct request`]: srctree/include/linux/blk-mq.h
pub(crate) struct RequestDataWrapper {
/// The Rust request refcount has the following states:
///
/// - 0: The request is owned by C block layer.
/// - 1: The request is owned by Rust abstractions but there are no [`ARef`] references to it.
/// - 2+: There are [`ARef`] references to the request.
refcount: Refcount,
}
impl RequestDataWrapper {
/// Return a reference to the refcount of the request that is embedding
/// `self`.
pub(crate) fn refcount(&self) -> &Refcount {
&self.refcount
}
/// Return a pointer to the refcount of the request that is embedding the
/// pointee of `this`.
///
/// # Safety
///
/// - `this` must point to a live allocation of at least the size of `Self`.
pub(crate) unsafe fn refcount_ptr(this: *mut Self) -> *mut Refcount {
// SAFETY: Because of the safety requirements of this function, the
// field projection is safe.
unsafe { &raw mut (*this).refcount }
}
}
// SAFETY: Exclusive access is thread-safe for `Request`. `Request` has no `&mut
// self` methods and `&self` methods that mutate `self` are internally
// synchronized.
unsafe impl<T: Operations> Send for Request<T> {}
// SAFETY: Shared access is thread-safe for `Request`. `&self` methods that
// mutate `self` are internally synchronized`
unsafe impl<T: Operations> Sync for Request<T> {}
// SAFETY: All instances of `Request<T>` are reference counted. This
// implementation of `AlwaysRefCounted` ensure that increments to the ref count
// keeps the object alive in memory at least until a matching reference count
// decrement is executed.
unsafe impl<T: Operations> AlwaysRefCounted for Request<T> {
fn inc_ref(&self) {
self.wrapper_ref().refcount().inc();
}
unsafe fn dec_ref(obj: core::ptr::NonNull<Self>) {
// SAFETY: The type invariants of `ARef` guarantee that `obj` is valid
// for read.
let wrapper_ptr = unsafe { Self::wrapper_ptr(obj.as_ptr()).as_ptr() };
// SAFETY: The type invariant of `Request` guarantees that the private
// data area is initialized and valid.
let refcount = unsafe { &*RequestDataWrapper::refcount_ptr(wrapper_ptr) };
#[cfg_attr(not(CONFIG_DEBUG_MISC), allow(unused_variables))]
let is_zero = refcount.dec_and_test();
#[cfg(CONFIG_DEBUG_MISC)]
if is_zero {
panic!("Request reached refcount zero in Rust abstractions");
}
}
}
linux-7.2.3/rust/kernel/block/mq/tag_set.rs
+85
View File
@@ -0,0 +1,85 @@
// SPDX-License-Identifier: GPL-2.0
//! This module provides the `TagSet` struct to wrap the C `struct blk_mq_tag_set`.
//!
//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h)
use core::pin::Pin;
use crate::{
bindings,
block::mq::{operations::OperationsVTable, request::RequestDataWrapper, Operations},
error::{self, Result},
prelude::try_pin_init,
types::Opaque,
};
use core::{convert::TryInto, marker::PhantomData};
use pin_init::{pin_data, pinned_drop, PinInit};
/// A wrapper for the C `struct blk_mq_tag_set`.
///
/// `struct blk_mq_tag_set` contains a `struct list_head` and so must be pinned.
///
/// # Invariants
///
/// - `inner` is initialized and valid.
#[pin_data(PinnedDrop)]
#[repr(transparent)]
pub struct TagSet<T: Operations> {
#[pin]
inner: Opaque<bindings::blk_mq_tag_set>,
_p: PhantomData<T>,
}
impl<T: Operations> TagSet<T> {
/// Try to create a new tag set
pub fn new(
nr_hw_queues: u32,
num_tags: u32,
num_maps: u32,
) -> impl PinInit<Self, error::Error> {
let tag_set: bindings::blk_mq_tag_set = pin_init::zeroed();
let tag_set: Result<_> = core::mem::size_of::<RequestDataWrapper>()
.try_into()
.map(|cmd_size| {
bindings::blk_mq_tag_set {
ops: OperationsVTable::<T>::build(),
nr_hw_queues,
timeout: 0, // 0 means default which is 30Hz in C
numa_node: bindings::NUMA_NO_NODE,
queue_depth: num_tags,
cmd_size,
flags: 0,
driver_data: core::ptr::null_mut::<crate::ffi::c_void>(),
nr_maps: num_maps,
..tag_set
}
})
.map(Opaque::new)
.map_err(|e| e.into());
try_pin_init!(TagSet {
inner <- tag_set.pin_chain(|tag_set| {
// SAFETY: we do not move out of `tag_set`.
let tag_set: &mut Opaque<_> = unsafe { Pin::get_unchecked_mut(tag_set) };
// SAFETY: `tag_set` is a reference to an initialized `blk_mq_tag_set`.
error::to_result( unsafe { bindings::blk_mq_alloc_tag_set(tag_set.get())})
}),
_p: PhantomData,
})
}
/// Return the pointer to the wrapped `struct blk_mq_tag_set`
pub(crate) fn raw_tag_set(&self) -> *mut bindings::blk_mq_tag_set {
self.inner.get()
}
}
#[pinned_drop]
impl<T: Operations> PinnedDrop for TagSet<T> {
fn drop(self: Pin<&mut Self>) {
// SAFETY: By type invariant `inner` is valid and has been properly
// initialized during construction.
unsafe { bindings::blk_mq_free_tag_set(self.inner.get()) };
}
}
linux-7.2.3/rust/kernel/bug.rs
+132
View File
@@ -0,0 +1,132 @@
// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2024, 2025 FUJITA Tomonori <fujita.tomonori@gmail.com>
//! Support for BUG and WARN functionality.
//!
//! C header: [`include/asm-generic/bug.h`](srctree/include/asm-generic/bug.h)
#[macro_export]
#[doc(hidden)]
#[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))]
#[cfg(CONFIG_DEBUG_BUGVERBOSE)]
macro_rules! warn_flags {
($file:expr, $flags:expr) => {
const FLAGS: u32 = $crate::bindings::BUGFLAG_WARNING | $flags;
const _FILE: &[u8] = $file.as_bytes();
// Plus one for null-terminator.
static FILE: [u8; _FILE.len() + 1] = {
let mut bytes = [0; _FILE.len() + 1];
let mut i = 0;
while i < _FILE.len() {
bytes[i] = _FILE[i];
i += 1;
}
bytes
};
// SAFETY:
// - `file`, `line`, `flags`, and `size` are all compile-time constants or
// symbols, preventing any invalid memory access.
// - The asm block has no side effects and does not modify any registers
// or memory. It is purely for embedding metadata into the ELF section.
unsafe {
$crate::asm!(
concat!(
"/* {size} */",
include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_warn_asm.rs")),
include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_reachable_asm.rs")));
file = sym FILE,
line = const line!(),
flags = const FLAGS,
size = const ::core::mem::size_of::<$crate::bindings::bug_entry>(),
);
}
}
}
#[macro_export]
#[doc(hidden)]
#[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))]
#[cfg(not(CONFIG_DEBUG_BUGVERBOSE))]
macro_rules! warn_flags {
($file:expr, $flags:expr) => {
const FLAGS: u32 = $crate::bindings::BUGFLAG_WARNING | $flags;
// SAFETY:
// - `flags` and `size` are all compile-time constants, preventing
// any invalid memory access.
// - The asm block has no side effects and does not modify any registers
// or memory. It is purely for embedding metadata into the ELF section.
unsafe {
$crate::asm!(
concat!(
"/* {size} */",
include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_warn_asm.rs")),
include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_reachable_asm.rs")));
flags = const FLAGS,
size = const ::core::mem::size_of::<$crate::bindings::bug_entry>(),
);
}
}
}
#[macro_export]
#[doc(hidden)]
#[cfg(all(CONFIG_BUG, CONFIG_UML))]
macro_rules! warn_flags {
($file:expr, $flags:expr) => {
// SAFETY: It is always safe to call `warn_slowpath_fmt()`
// with a valid null-terminated string.
unsafe {
$crate::bindings::warn_slowpath_fmt(
$crate::c_str!(::core::file!()).as_char_ptr(),
line!() as $crate::ffi::c_int,
$flags as $crate::ffi::c_uint,
::core::ptr::null(),
);
}
};
}
#[macro_export]
#[doc(hidden)]
#[cfg(all(CONFIG_BUG, any(CONFIG_LOONGARCH, CONFIG_ARM)))]
macro_rules! warn_flags {
($file:expr, $flags:expr) => {
// SAFETY: It is always safe to call `WARN_ON()`.
unsafe { $crate::bindings::WARN_ON(true) }
};
}
#[macro_export]
#[doc(hidden)]
#[cfg(not(CONFIG_BUG))]
macro_rules! warn_flags {
($file:expr, $flags:expr) => {};
}
#[doc(hidden)]
pub const fn bugflag_taint(value: u32) -> u32 {
value << 8
}
/// Report a warning if `cond` is true and return the condition's evaluation result.
#[macro_export]
macro_rules! warn_on {
($cond:expr) => {{
let cond = $cond;
#[cfg(CONFIG_DEBUG_BUGVERBOSE_DETAILED)]
const _COND_STR: &str = concat!("[", stringify!($cond), "] ", file!());
#[cfg(not(CONFIG_DEBUG_BUGVERBOSE_DETAILED))]
const _COND_STR: &str = file!();
if cond {
const WARN_ON_FLAGS: u32 = $crate::bug::bugflag_taint($crate::bindings::TAINT_WARN);
$crate::warn_flags!(_COND_STR, WARN_ON_FLAGS);
}
cond
}};
}
linux-7.2.3/rust/kernel/build_assert.rs
+219
View File
@@ -0,0 +1,219 @@
// SPDX-License-Identifier: GPL-2.0
//! Various assertions that happen during build-time.
//!
//! There are three types of build-time assertions that you can use:
//! - [`static_assert!`]
//! - [`const_assert!`]
//! - [`build_assert!`]
//!
//! The ones towards the bottom of the list are more expressive, while the ones towards the top of
//! the list are more robust and trigger earlier in the compilation pipeline. Therefore, you should
//! prefer the ones towards the top of the list wherever possible.
//!
//! # Choosing the correct assertion
//!
//! If you're asserting outside any bodies (e.g. initializers or function bodies), you should use
//! [`static_assert!`] as it is the only assertion that can be used in that context.
//!
//! Inside bodies, if your assertion condition does not depend on any variable or generics, you
//! should use [`static_assert!`]. If the condition depends on generics, but not variables
//! (including function arguments), you should use [`const_assert!`]. Otherwise, use
//! [`build_assert!`]. The same is true regardless if the function is `const fn`.
//!
//! ```
//! // Outside any bodies.
//! static_assert!(core::mem::size_of::<u8>() == 1);
//! // `const_assert!` and `build_assert!` cannot be used here, they will fail to compile.
//!
//! #[inline(always)]
//! fn foo<const N: usize>(v: usize) {
//! static_assert!(core::mem::size_of::<u8>() == 1); // Preferred.
//! const_assert!(core::mem::size_of::<u8>() == 1); // Discouraged.
//! build_assert!(core::mem::size_of::<u8>() == 1); // Discouraged.
//!
//! // `static_assert!(N > 1);` is not allowed.
//! const_assert!(N > 1); // Preferred.
//! build_assert!(N > 1); // Discouraged.
//!
//! // `static_assert!(v > 1);` is not allowed.
//! // `const_assert!(v > 1);` is not allowed.
//! build_assert!(v > 1); // Works.
//! }
//! ```
//!
//! # Detailed behavior
//!
//! `static_assert!()` is equivalent to `static_assert` in C. It requires `expr` to be a constant
//! expression. This expression cannot refer to any generics. A `static_assert!(expr)` in a program
//! is always evaluated, regardless if the function it appears in is used or not. This is also the
//! only usable assertion outside a body.
//!
//! `const_assert!()` has no direct C equivalence. It is a more powerful version of
//! `static_assert!()`, where it may refer to generics in a function. Note that due to the ability
//! to refer to generics, the assertion is tied to a specific instance of a function. So if it is
//! used in a generic function that is not instantiated, the assertion will not be checked. For this
//! reason, `static_assert!()` is preferred wherever possible.
//!
//! `build_assert!()` is equivalent to `BUILD_BUG_ON`. It is even more powerful than
//! `const_assert!()` because it can be used to check tautologies that depend on runtime value (this
//! is the same as `BUILD_BUG_ON`). However, the assertion failure mechanism can possibly be
//! undefined symbols and linker errors, it is not developer friendly to debug, so it is recommended
//! to avoid it and prefer other two assertions where possible.
#[doc(inline)]
pub use crate::{
build_assert_macro as build_assert,
build_error,
const_assert,
static_assert, //
};
#[doc(hidden)]
pub use build_error::build_error as build_error_fn;
/// Static assert (i.e. compile-time assert).
///
/// Similar to C11 [`_Static_assert`] and C++11 [`static_assert`].
///
/// An optional panic message can be supplied after the expression.
/// Currently only a string literal without formatting is supported
/// due to constness limitations of the [`assert!`] macro.
///
/// The feature may be added to Rust in the future: see [RFC 2790].
///
/// You cannot refer to generics or variables with [`static_assert!`]. If you need to refer to
/// generics, use [`const_assert!`]; if you need to refer to variables, use [`build_assert!`]. See
/// the [module documentation](self).
///
/// [`_Static_assert`]: https://en.cppreference.com/w/c/language/_Static_assert
/// [`static_assert`]: https://en.cppreference.com/w/cpp/language/static_assert
/// [RFC 2790]: https://github.com/rust-lang/rfcs/issues/2790
///
/// # Examples
///
/// ```
/// static_assert!(42 > 24);
/// static_assert!(core::mem::size_of::<u8>() == 1);
///
/// const X: &[u8] = b"bar";
/// static_assert!(X[1] == b'a');
///
/// const fn f(x: i32) -> i32 {
/// x + 2
/// }
/// static_assert!(f(40) == 42);
/// static_assert!(f(40) == 42, "f(x) must add 2 to the given input.");
/// ```
#[macro_export]
#[doc(hidden)]
macro_rules! static_assert {
($condition:expr $(,$arg:literal)?) => {
const _: () = ::core::assert!($condition $(,$arg)?);
};
}
/// Assertion during constant evaluation.
///
/// This is a more powerful version of [`static_assert!`] that can refer to generics inside
/// functions or implementation blocks. However, it also has a limitation where it can only appear
/// in places where statements can appear; for example, you cannot use it as an item in the module.
///
/// [`static_assert!`] should be preferred if no generics are referred to in the condition. You
/// cannot refer to variables with [`const_assert!`] (even inside `const fn`); if you need the
/// capability, use [`build_assert!`]. See the [module documentation](self).
///
/// # Examples
///
/// ```
/// fn foo<const N: usize>() {
/// const_assert!(N > 1);
/// }
///
/// fn bar<T>() {
/// const_assert!(size_of::<T>() > 0, "T cannot be ZST");
/// }
/// ```
#[macro_export]
#[doc(hidden)]
macro_rules! const_assert {
($condition:expr $(,$arg:literal)?) => {
const { ::core::assert!($condition $(,$arg)?) };
};
}
/// Fails the build if the code path calling `build_error!` can possibly be executed.
///
/// If the macro is executed in const context, `build_error!` will panic.
/// If the compiler or optimizer cannot guarantee that `build_error!` can never
/// be called, a build error will be triggered.
///
/// # Examples
///
/// ```
/// #[inline]
/// fn foo(a: usize) -> usize {
/// a.checked_add(1).unwrap_or_else(|| build_error!("overflow"))
/// }
///
/// assert_eq!(foo(usize::MAX - 1), usize::MAX); // OK.
/// // foo(usize::MAX); // Fails to compile.
/// ```
#[macro_export]
#[doc(hidden)]
macro_rules! build_error {
() => {{
$crate::build_assert::build_error_fn("")
}};
($msg:expr) => {{
$crate::build_assert::build_error_fn($msg)
}};
}
/// Asserts that a boolean expression is `true` at compile time.
///
/// If the condition is evaluated to `false` in const context, `build_assert!`
/// will panic. If the compiler or optimizer cannot guarantee the condition will
/// be evaluated to `true`, a build error will be triggered.
///
/// When a condition depends on a function argument, the function must be annotated with
/// `#[inline(always)]`. Without this attribute, the compiler may choose to not inline the
/// function, preventing it from optimizing out the error path.
///
/// If the assertion condition does not depend on any variables or generics, you should use
/// [`static_assert!`]. If the assertion condition does not depend on variables, but does depend on
/// generics, you should use [`const_assert!`]. See the [module documentation](self).
///
/// # Examples
///
/// ```
/// #[inline(always)] // Important.
/// fn bar(n: usize) {
/// build_assert!(n > 1);
/// }
///
/// fn foo() {
/// bar(2);
/// }
///
/// #[inline(always)] // Important.
/// const fn const_bar(n: usize) {
/// build_assert!(n > 1);
/// }
///
/// const _: () = const_bar(2);
/// ```
#[macro_export]
#[doc(hidden)]
macro_rules! build_assert_macro {
($cond:expr $(,)?) => {{
if !$cond {
$crate::build_assert::build_error_fn(concat!("assertion failed: ", stringify!($cond)));
}
}};
($cond:expr, $msg:expr) => {{
if !$cond {
$crate::build_assert::build_error_fn($msg);
}
}};
}
linux-7.2.3/rust/kernel/clk.rs
+335
View File
@@ -0,0 +1,335 @@
// SPDX-License-Identifier: GPL-2.0
//! Clock abstractions.
//!
//! C header: [`include/linux/clk.h`](srctree/include/linux/clk.h)
//!
//! Reference: <https://docs.kernel.org/driver-api/clk.html>
use crate::ffi::c_ulong;
/// The frequency unit.
///
/// Represents a frequency in hertz, wrapping a [`c_ulong`] value.
///
/// # Examples
///
/// ```
/// use kernel::clk::Hertz;
///
/// let hz = 1_000_000_000;
/// let rate = Hertz(hz);
///
/// assert_eq!(rate.as_hz(), hz);
/// assert_eq!(rate, Hertz(hz));
/// assert_eq!(rate, Hertz::from_khz(hz / 1_000));
/// assert_eq!(rate, Hertz::from_mhz(hz / 1_000_000));
/// assert_eq!(rate, Hertz::from_ghz(hz / 1_000_000_000));
/// ```
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub struct Hertz(pub c_ulong);
impl Hertz {
const KHZ_TO_HZ: c_ulong = 1_000;
const MHZ_TO_HZ: c_ulong = 1_000_000;
const GHZ_TO_HZ: c_ulong = 1_000_000_000;
/// Create a new instance from kilohertz (kHz)
pub const fn from_khz(khz: c_ulong) -> Self {
Self(khz * Self::KHZ_TO_HZ)
}
/// Create a new instance from megahertz (MHz)
pub const fn from_mhz(mhz: c_ulong) -> Self {
Self(mhz * Self::MHZ_TO_HZ)
}
/// Create a new instance from gigahertz (GHz)
pub const fn from_ghz(ghz: c_ulong) -> Self {
Self(ghz * Self::GHZ_TO_HZ)
}
/// Get the frequency in hertz
pub const fn as_hz(&self) -> c_ulong {
self.0
}
/// Get the frequency in kilohertz
pub const fn as_khz(&self) -> c_ulong {
self.0 / Self::KHZ_TO_HZ
}
/// Get the frequency in megahertz
pub const fn as_mhz(&self) -> c_ulong {
self.0 / Self::MHZ_TO_HZ
}
/// Get the frequency in gigahertz
pub const fn as_ghz(&self) -> c_ulong {
self.0 / Self::GHZ_TO_HZ
}
}
impl From<Hertz> for c_ulong {
fn from(freq: Hertz) -> Self {
freq.0
}
}
#[cfg(CONFIG_COMMON_CLK)]
mod common_clk {
use super::Hertz;
use crate::{
device::Device,
error::{from_err_ptr, to_result, Result},
prelude::*,
};
use core::{ops::Deref, ptr};
/// A reference-counted clock.
///
/// Rust abstraction for the C [`struct clk`].
///
/// # Invariants
///
/// A [`Clk`] instance holds either a pointer to a valid [`struct clk`] created by the C
/// portion of the kernel or a `NULL` pointer.
///
/// Instances of this type are reference-counted. Calling [`Clk::get`] ensures that the
/// allocation remains valid for the lifetime of the [`Clk`].
///
/// # Examples
///
/// The following example demonstrates how to obtain and configure a clock for a device.
///
/// ```
/// use kernel::clk::{Clk, Hertz};
/// use kernel::device::Device;
/// use kernel::error::Result;
///
/// fn configure_clk(dev: &Device) -> Result {
/// let clk = Clk::get(dev, Some(c"apb_clk"))?;
///
/// clk.prepare_enable()?;
///
/// let expected_rate = Hertz::from_ghz(1);
///
/// if clk.rate() != expected_rate {
/// clk.set_rate(expected_rate)?;
/// }
///
/// clk.disable_unprepare();
/// Ok(())
/// }
/// ```
///
/// [`struct clk`]: https://docs.kernel.org/driver-api/clk.html
#[repr(transparent)]
pub struct Clk(*mut bindings::clk);
// SAFETY: It is safe to call `clk_put` on another thread than where `clk_get` was called.
unsafe impl Send for Clk {}
// SAFETY: It is safe to call any combination of the `&self` methods in parallel, as the
// methods are synchronized internally.
unsafe impl Sync for Clk {}
impl Clk {
/// Gets [`Clk`] corresponding to a [`Device`] and a connection id.
///
/// Equivalent to the kernel's [`clk_get`] API.
///
/// [`clk_get`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_get
pub fn get(dev: &Device, name: Option<&CStr>) -> Result<Self> {
let con_id = name.map_or(ptr::null(), |n| n.as_char_ptr());
// SAFETY: It is safe to call [`clk_get`] for a valid device pointer.
//
// INVARIANT: The reference-count is decremented when [`Clk`] goes out of scope.
Ok(Self(from_err_ptr(unsafe {
bindings::clk_get(dev.as_raw(), con_id)
})?))
}
/// Obtain the raw [`struct clk`] pointer.
#[inline]
pub fn as_raw(&self) -> *mut bindings::clk {
self.0
}
/// Enable the clock.
///
/// Equivalent to the kernel's [`clk_enable`] API.
///
/// [`clk_enable`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_enable
#[inline]
pub fn enable(&self) -> Result {
// SAFETY: By the type invariants, self.as_raw() is a valid argument for
// [`clk_enable`].
to_result(unsafe { bindings::clk_enable(self.as_raw()) })
}
/// Disable the clock.
///
/// Equivalent to the kernel's [`clk_disable`] API.
///
/// [`clk_disable`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_disable
#[inline]
pub fn disable(&self) {
// SAFETY: By the type invariants, self.as_raw() is a valid argument for
// [`clk_disable`].
unsafe { bindings::clk_disable(self.as_raw()) };
}
/// Prepare the clock.
///
/// Equivalent to the kernel's [`clk_prepare`] API.
///
/// [`clk_prepare`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_prepare
#[inline]
pub fn prepare(&self) -> Result {
// SAFETY: By the type invariants, self.as_raw() is a valid argument for
// [`clk_prepare`].
to_result(unsafe { bindings::clk_prepare(self.as_raw()) })
}
/// Unprepare the clock.
///
/// Equivalent to the kernel's [`clk_unprepare`] API.
///
/// [`clk_unprepare`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_unprepare
#[inline]
pub fn unprepare(&self) {
// SAFETY: By the type invariants, self.as_raw() is a valid argument for
// [`clk_unprepare`].
unsafe { bindings::clk_unprepare(self.as_raw()) };
}
/// Prepare and enable the clock.
///
/// Equivalent to calling [`Clk::prepare`] followed by [`Clk::enable`].
#[inline]
pub fn prepare_enable(&self) -> Result {
// SAFETY: By the type invariants, self.as_raw() is a valid argument for
// [`clk_prepare_enable`].
to_result(unsafe { bindings::clk_prepare_enable(self.as_raw()) })
}
/// Disable and unprepare the clock.
///
/// Equivalent to calling [`Clk::disable`] followed by [`Clk::unprepare`].
#[inline]
pub fn disable_unprepare(&self) {
// SAFETY: By the type invariants, self.as_raw() is a valid argument for
// [`clk_disable_unprepare`].
unsafe { bindings::clk_disable_unprepare(self.as_raw()) };
}
/// Get clock's rate.
///
/// Equivalent to the kernel's [`clk_get_rate`] API.
///
/// [`clk_get_rate`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_get_rate
#[inline]
pub fn rate(&self) -> Hertz {
// SAFETY: By the type invariants, self.as_raw() is a valid argument for
// [`clk_get_rate`].
Hertz(unsafe { bindings::clk_get_rate(self.as_raw()) })
}
/// Set clock's rate.
///
/// Equivalent to the kernel's [`clk_set_rate`] API.
///
/// [`clk_set_rate`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_set_rate
#[inline]
pub fn set_rate(&self, rate: Hertz) -> Result {
// SAFETY: By the type invariants, self.as_raw() is a valid argument for
// [`clk_set_rate`].
to_result(unsafe { bindings::clk_set_rate(self.as_raw(), rate.as_hz()) })
}
}
impl Drop for Clk {
fn drop(&mut self) {
// SAFETY: By the type invariants, self.as_raw() is a valid argument for [`clk_put`].
unsafe { bindings::clk_put(self.as_raw()) };
}
}
/// A reference-counted optional clock.
///
/// A lightweight wrapper around an optional [`Clk`]. An [`OptionalClk`] represents a [`Clk`]
/// that a driver can function without but may improve performance or enable additional
/// features when available.
///
/// # Invariants
///
/// An [`OptionalClk`] instance encapsulates a [`Clk`] with either a valid [`struct clk`] or
/// `NULL` pointer.
///
/// Instances of this type are reference-counted. Calling [`OptionalClk::get`] ensures that the
/// allocation remains valid for the lifetime of the [`OptionalClk`].
///
/// # Examples
///
/// The following example demonstrates how to obtain and configure an optional clock for a
/// device. The code functions correctly whether or not the clock is available.
///
/// ```
/// use kernel::clk::{OptionalClk, Hertz};
/// use kernel::device::Device;
/// use kernel::error::Result;
///
/// fn configure_clk(dev: &Device) -> Result {
/// let clk = OptionalClk::get(dev, Some(c"apb_clk"))?;
///
/// clk.prepare_enable()?;
///
/// let expected_rate = Hertz::from_ghz(1);
///
/// if clk.rate() != expected_rate {
/// clk.set_rate(expected_rate)?;
/// }
///
/// clk.disable_unprepare();
/// Ok(())
/// }
/// ```
///
/// [`struct clk`]: https://docs.kernel.org/driver-api/clk.html
pub struct OptionalClk(Clk);
impl OptionalClk {
/// Gets [`OptionalClk`] corresponding to a [`Device`] and a connection id.
///
/// Equivalent to the kernel's [`clk_get_optional`] API.
///
/// [`clk_get_optional`]:
/// https://docs.kernel.org/core-api/kernel-api.html#c.clk_get_optional
pub fn get(dev: &Device, name: Option<&CStr>) -> Result<Self> {
let con_id = name.map_or(ptr::null(), |n| n.as_char_ptr());
// SAFETY: It is safe to call [`clk_get_optional`] for a valid device pointer.
//
// INVARIANT: The reference-count is decremented when [`OptionalClk`] goes out of
// scope.
Ok(Self(Clk(from_err_ptr(unsafe {
bindings::clk_get_optional(dev.as_raw(), con_id)
})?)))
}
}
// Make [`OptionalClk`] behave like [`Clk`].
impl Deref for OptionalClk {
type Target = Clk;
fn deref(&self) -> &Clk {
&self.0
}
}
}
#[cfg(CONFIG_COMMON_CLK)]
pub use common_clk::*;
linux-7.2.3/rust/kernel/configfs.rs
+1044
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/rust/kernel/cpu.rs
+152
View File
@@ -0,0 +1,152 @@
// SPDX-License-Identifier: GPL-2.0
//! Generic CPU definitions.
//!
//! C header: [`include/linux/cpu.h`](srctree/include/linux/cpu.h)
use crate::{bindings, device::Device, error::Result, prelude::ENODEV};
/// Returns the maximum number of possible CPUs in the current system configuration.
#[inline]
pub fn nr_cpu_ids() -> u32 {
#[cfg(any(NR_CPUS_1, CONFIG_FORCE_NR_CPUS))]
{
bindings::NR_CPUS
}
#[cfg(not(any(NR_CPUS_1, CONFIG_FORCE_NR_CPUS)))]
// SAFETY: `nr_cpu_ids` is a valid global provided by the kernel.
unsafe {
bindings::nr_cpu_ids
}
}
/// The CPU ID.
///
/// Represents a CPU identifier as a wrapper around an [`u32`].
///
/// # Invariants
///
/// The CPU ID lies within the range `[0, nr_cpu_ids())`.
///
/// # Examples
///
/// ```
/// use kernel::cpu::CpuId;
///
/// let cpu = 0;
///
/// // SAFETY: 0 is always a valid CPU number.
/// let id = unsafe { CpuId::from_u32_unchecked(cpu) };
///
/// assert_eq!(id.as_u32(), cpu);
/// assert!(CpuId::from_i32(0).is_some());
/// assert!(CpuId::from_i32(-1).is_none());
/// ```
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub struct CpuId(u32);
impl CpuId {
/// Creates a new [`CpuId`] from the given `id` without checking bounds.
///
/// # Safety
///
/// The caller must ensure that `id` is a valid CPU ID (i.e., `0 <= id < nr_cpu_ids()`).
#[inline]
pub unsafe fn from_i32_unchecked(id: i32) -> Self {
debug_assert!(id >= 0);
debug_assert!((id as u32) < nr_cpu_ids());
// INVARIANT: The function safety guarantees `id` is a valid CPU id.
Self(id as u32)
}
/// Creates a new [`CpuId`] from the given `id`, checking that it is valid.
pub fn from_i32(id: i32) -> Option<Self> {
if id < 0 || id as u32 >= nr_cpu_ids() {
None
} else {
// INVARIANT: `id` has just been checked as a valid CPU ID.
Some(Self(id as u32))
}
}
/// Creates a new [`CpuId`] from the given `id` without checking bounds.
///
/// # Safety
///
/// The caller must ensure that `id` is a valid CPU ID (i.e., `0 <= id < nr_cpu_ids()`).
#[inline]
pub unsafe fn from_u32_unchecked(id: u32) -> Self {
debug_assert!(id < nr_cpu_ids());
// Ensure the `id` fits in an [`i32`] as it's also representable that way.
debug_assert!(id <= i32::MAX as u32);
// INVARIANT: The function safety guarantees `id` is a valid CPU id.
Self(id)
}
/// Creates a new [`CpuId`] from the given `id`, checking that it is valid.
pub fn from_u32(id: u32) -> Option<Self> {
if id >= nr_cpu_ids() {
None
} else {
// INVARIANT: `id` has just been checked as a valid CPU ID.
Some(Self(id))
}
}
/// Returns CPU number.
#[inline]
pub fn as_u32(&self) -> u32 {
self.0
}
/// Returns the ID of the CPU the code is currently running on.
///
/// The returned value is considered unstable because it may change
/// unexpectedly due to preemption or CPU migration. It should only be
/// used when the context ensures that the task remains on the same CPU
/// or the users could use a stale (yet valid) CPU ID.
#[inline]
pub fn current() -> Self {
// SAFETY: raw_smp_processor_id() always returns a valid CPU ID.
unsafe { Self::from_u32_unchecked(bindings::raw_smp_processor_id()) }
}
}
impl From<CpuId> for u32 {
fn from(id: CpuId) -> Self {
id.as_u32()
}
}
impl From<CpuId> for i32 {
fn from(id: CpuId) -> Self {
id.as_u32() as i32
}
}
/// Creates a new instance of CPU's device.
///
/// # Safety
///
/// Reference counting is not implemented for the CPU device in the C code. When a CPU is
/// hot-unplugged, the corresponding CPU device is unregistered, but its associated memory
/// is not freed.
///
/// Callers must ensure that the CPU device is not used after it has been unregistered.
/// This can be achieved, for example, by registering a CPU hotplug notifier and removing
/// any references to the CPU device within the notifier's callback.
pub unsafe fn from_cpu(cpu: CpuId) -> Result<&'static Device> {
// SAFETY: It is safe to call `get_cpu_device()` for any CPU.
let ptr = unsafe { bindings::get_cpu_device(u32::from(cpu)) };
if ptr.is_null() {
return Err(ENODEV);
}
// SAFETY: The pointer returned by `get_cpu_device()`, if not `NULL`, is a valid pointer to
// a `struct device` and is never freed by the C code.
Ok(unsafe { Device::from_raw(ptr) })
}
linux-7.2.3/rust/kernel/cpufreq.rs
+1399
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/rust/kernel/cpumask.rs
+343
View File
@@ -0,0 +1,343 @@
// SPDX-License-Identifier: GPL-2.0
//! CPU Mask abstractions.
//!
//! C header: [`include/linux/cpumask.h`](srctree/include/linux/cpumask.h)
use crate::{
alloc::{AllocError, Flags},
cpu::CpuId,
prelude::*,
types::Opaque,
};
#[cfg(CONFIG_CPUMASK_OFFSTACK)]
use core::ptr::{self, NonNull};
use core::ops::{Deref, DerefMut};
/// A CPU Mask.
///
/// Rust abstraction for the C `struct cpumask`.
///
/// # Invariants
///
/// A [`Cpumask`] instance always corresponds to a valid C `struct cpumask`.
///
/// The callers must ensure that the `struct cpumask` is valid for access and
/// remains valid for the lifetime of the returned reference.
///
/// # Examples
///
/// The following example demonstrates how to update a [`Cpumask`].
///
/// ```
/// use kernel::bindings;
/// use kernel::cpu::CpuId;
/// use kernel::cpumask::Cpumask;
///
/// fn set_clear_cpu(ptr: *mut bindings::cpumask, set_cpu: CpuId, clear_cpu: CpuId) {
/// // SAFETY: The `ptr` is valid for writing and remains valid for the lifetime of the
/// // returned reference.
/// let mask = unsafe { Cpumask::from_raw_mut(ptr) };
///
/// mask.set(set_cpu);
/// mask.clear(clear_cpu);
/// }
/// ```
#[repr(transparent)]
pub struct Cpumask(Opaque<bindings::cpumask>);
impl Cpumask {
/// Creates a mutable reference from an existing `struct cpumask` pointer.
///
/// # Safety
///
/// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime
/// of the returned reference.
pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpumask) -> &'a mut Self {
// SAFETY: Guaranteed by the safety requirements of the function.
//
// INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the
// lifetime of the returned reference.
unsafe { &mut *ptr.cast() }
}
/// Creates a reference from an existing `struct cpumask` pointer.
///
/// # Safety
///
/// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime
/// of the returned reference.
pub unsafe fn from_raw<'a>(ptr: *const bindings::cpumask) -> &'a Self {
// SAFETY: Guaranteed by the safety requirements of the function.
//
// INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the
// lifetime of the returned reference.
unsafe { &*ptr.cast() }
}
/// Obtain the raw `struct cpumask` pointer.
pub fn as_raw(&self) -> *mut bindings::cpumask {
let this: *const Self = self;
this.cast_mut().cast()
}
/// Set `cpu` in the cpumask.
///
/// ATTENTION: Contrary to C, this Rust `set()` method is non-atomic.
/// This mismatches kernel naming convention and corresponds to the C
/// function `__cpumask_set_cpu()`.
#[inline]
pub fn set(&mut self, cpu: CpuId) {
// SAFETY: By the type invariant, `self.as_raw` is a valid argument to `__cpumask_set_cpu`.
unsafe { bindings::__cpumask_set_cpu(u32::from(cpu), self.as_raw()) };
}
/// Clear `cpu` in the cpumask.
///
/// ATTENTION: Contrary to C, this Rust `clear()` method is non-atomic.
/// This mismatches kernel naming convention and corresponds to the C
/// function `__cpumask_clear_cpu()`.
#[inline]
pub fn clear(&mut self, cpu: CpuId) {
// SAFETY: By the type invariant, `self.as_raw` is a valid argument to
// `__cpumask_clear_cpu`.
unsafe { bindings::__cpumask_clear_cpu(i32::from(cpu), self.as_raw()) };
}
/// Test `cpu` in the cpumask.
///
/// Equivalent to the kernel's `cpumask_test_cpu` API.
#[inline]
pub fn test(&self, cpu: CpuId) -> bool {
// SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_test_cpu`.
unsafe { bindings::cpumask_test_cpu(i32::from(cpu), self.as_raw()) }
}
/// Set all CPUs in the cpumask.
///
/// Equivalent to the kernel's `cpumask_setall` API.
#[inline]
pub fn setall(&mut self) {
// SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_setall`.
unsafe { bindings::cpumask_setall(self.as_raw()) };
}
/// Checks if cpumask is empty.
///
/// Equivalent to the kernel's `cpumask_empty` API.
#[inline]
pub fn empty(&self) -> bool {
// SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_empty`.
unsafe { bindings::cpumask_empty(self.as_raw()) }
}
/// Checks if cpumask is full.
///
/// Equivalent to the kernel's `cpumask_full` API.
#[inline]
pub fn full(&self) -> bool {
// SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_full`.
unsafe { bindings::cpumask_full(self.as_raw()) }
}
/// Get weight of the cpumask.
///
/// Equivalent to the kernel's `cpumask_weight` API.
#[inline]
pub fn weight(&self) -> u32 {
// SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_weight`.
unsafe { bindings::cpumask_weight(self.as_raw()) }
}
/// Copy cpumask.
///
/// Equivalent to the kernel's `cpumask_copy` API.
#[inline]
pub fn copy(&self, dstp: &mut Self) {
// SAFETY: By the type invariant, `Self::as_raw` is a valid argument to `cpumask_copy`.
unsafe { bindings::cpumask_copy(dstp.as_raw(), self.as_raw()) };
}
}
/// A CPU Mask pointer.
///
/// Rust abstraction for the C `struct cpumask_var_t`.
///
/// # Invariants
///
/// A [`CpumaskVar`] instance always corresponds to a valid C `struct cpumask_var_t`.
///
/// The callers must ensure that the `struct cpumask_var_t` is valid for access and remains valid
/// for the lifetime of [`CpumaskVar`].
///
/// # Examples
///
/// The following example demonstrates how to create and update a [`CpumaskVar`].
///
/// ```
/// use kernel::cpu::CpuId;
/// use kernel::cpumask::CpumaskVar;
///
/// let mut mask = CpumaskVar::new_zero(GFP_KERNEL).unwrap();
///
/// assert!(mask.empty());
/// let mut count = 0;
///
/// let cpu2 = CpuId::from_u32(2);
/// if let Some(cpu) = cpu2 {
/// mask.set(cpu);
/// assert!(mask.test(cpu));
/// count += 1;
/// }
///
/// let cpu3 = CpuId::from_u32(3);
/// if let Some(cpu) = cpu3 {
/// mask.set(cpu);
/// assert!(mask.test(cpu));
/// count += 1;
/// }
///
/// assert_eq!(mask.weight(), count);
///
/// let mask2 = CpumaskVar::try_clone(&mask).unwrap();
///
/// if let Some(cpu) = cpu2 {
/// assert!(mask2.test(cpu));
/// }
///
/// if let Some(cpu) = cpu3 {
/// assert!(mask2.test(cpu));
/// }
/// assert_eq!(mask2.weight(), count);
/// ```
#[repr(transparent)]
pub struct CpumaskVar {
#[cfg(CONFIG_CPUMASK_OFFSTACK)]
ptr: NonNull<Cpumask>,
#[cfg(not(CONFIG_CPUMASK_OFFSTACK))]
mask: Cpumask,
}
impl CpumaskVar {
/// Creates a zero-initialized instance of the [`CpumaskVar`].
pub fn new_zero(_flags: Flags) -> Result<Self, AllocError> {
Ok(Self {
#[cfg(CONFIG_CPUMASK_OFFSTACK)]
ptr: {
let mut ptr: *mut bindings::cpumask = ptr::null_mut();
// SAFETY: It is safe to call this method as the reference to `ptr` is valid.
//
// INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of
// scope.
unsafe { bindings::zalloc_cpumask_var(&mut ptr, _flags.as_raw()) };
NonNull::new(ptr.cast()).ok_or(AllocError)?
},
#[cfg(not(CONFIG_CPUMASK_OFFSTACK))]
mask: Cpumask(Opaque::zeroed()),
})
}
/// Creates an instance of the [`CpumaskVar`].
///
/// # Safety
///
/// The caller must ensure that the returned [`CpumaskVar`] is properly initialized before
/// getting used.
pub unsafe fn new(_flags: Flags) -> Result<Self, AllocError> {
Ok(Self {
#[cfg(CONFIG_CPUMASK_OFFSTACK)]
ptr: {
let mut ptr: *mut bindings::cpumask = ptr::null_mut();
// SAFETY: It is safe to call this method as the reference to `ptr` is valid.
//
// INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of
// scope.
unsafe { bindings::alloc_cpumask_var(&mut ptr, _flags.as_raw()) };
NonNull::new(ptr.cast()).ok_or(AllocError)?
},
#[cfg(not(CONFIG_CPUMASK_OFFSTACK))]
mask: Cpumask(Opaque::uninit()),
})
}
/// Creates a mutable reference to an existing `struct cpumask_var_t` pointer.
///
/// # Safety
///
/// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime
/// of the returned reference.
pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpumask_var_t) -> &'a mut Self {
// SAFETY: Guaranteed by the safety requirements of the function.
//
// INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the
// lifetime of the returned reference.
unsafe { &mut *ptr.cast() }
}
/// Creates a reference to an existing `struct cpumask_var_t` pointer.
///
/// # Safety
///
/// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime
/// of the returned reference.
pub unsafe fn from_raw<'a>(ptr: *const bindings::cpumask_var_t) -> &'a Self {
// SAFETY: Guaranteed by the safety requirements of the function.
//
// INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the
// lifetime of the returned reference.
unsafe { &*ptr.cast() }
}
/// Clones cpumask.
pub fn try_clone(cpumask: &Cpumask) -> Result<Self> {
// SAFETY: The returned cpumask_var is initialized right after this call.
let mut cpumask_var = unsafe { Self::new(GFP_KERNEL) }?;
cpumask.copy(&mut cpumask_var);
Ok(cpumask_var)
}
}
// Make [`CpumaskVar`] behave like a pointer to [`Cpumask`].
impl Deref for CpumaskVar {
type Target = Cpumask;
#[cfg(CONFIG_CPUMASK_OFFSTACK)]
fn deref(&self) -> &Self::Target {
// SAFETY: The caller owns CpumaskVar, so it is safe to deref the cpumask.
unsafe { &*self.ptr.as_ptr() }
}
#[cfg(not(CONFIG_CPUMASK_OFFSTACK))]
fn deref(&self) -> &Self::Target {
&self.mask
}
}
impl DerefMut for CpumaskVar {
#[cfg(CONFIG_CPUMASK_OFFSTACK)]
fn deref_mut(&mut self) -> &mut Cpumask {
// SAFETY: The caller owns CpumaskVar, so it is safe to deref the cpumask.
unsafe { self.ptr.as_mut() }
}
#[cfg(not(CONFIG_CPUMASK_OFFSTACK))]
fn deref_mut(&mut self) -> &mut Cpumask {
&mut self.mask
}
}
impl Drop for CpumaskVar {
fn drop(&mut self) {
#[cfg(CONFIG_CPUMASK_OFFSTACK)]
// SAFETY: By the type invariant, `self.as_raw` is a valid argument to `free_cpumask_var`.
unsafe {
bindings::free_cpumask_var(self.as_raw())
};
}
}

Some files were not shown because too many files have changed in this diff Show More