lastdragon/linux-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resguardo inicial

Browse Source
This commit is contained in:
lastdragon
2026-09-06 12:53:21 -06:00
parent 662efe3c2a
commit ad2088d178
94853 changed files with 43200404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linux-7.2.3/rust/kernel/sync/arc.rs
+918
View File
@@ -0,0 +1,918 @@
// SPDX-License-Identifier: GPL-2.0
//! A reference-counted pointer.
//!
//! This module implements a way for users to create reference-counted objects and pointers to
//! them. Such a pointer automatically increments and decrements the count, and drops the
//! underlying object when it reaches zero. It is also safe to use concurrently from multiple
//! threads.
//!
//! It is different from the standard library's [`Arc`] in a few ways:
//! 1. It is backed by the kernel's [`Refcount`] type.
//! 2. It does not support weak references, which allows it to be half the size.
//! 3. It saturates the reference count instead of aborting when it goes over a threshold.
//! 4. It does not provide a `get_mut` method, so the ref counted object is pinned.
//! 5. The object in [`Arc`] is pinned implicitly.
//!
//! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html
use crate::{
alloc::{AllocError, Flags, KBox},
ffi::c_void,
fmt,
init::InPlaceInit,
sync::Refcount,
try_init,
types::ForeignOwnable,
};
use core::{
alloc::Layout,
borrow::{Borrow, BorrowMut},
marker::PhantomData,
mem::{ManuallyDrop, MaybeUninit},
ops::{Deref, DerefMut},
pin::Pin,
ptr::NonNull,
};
use pin_init::{self, pin_data, InPlaceWrite, Init, PinInit};
mod std_vendor;
/// A reference-counted pointer to an instance of `T`.
///
/// The reference count is incremented when new instances of [`Arc`] are created, and decremented
/// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.
///
/// # Invariants
///
/// The reference count on an instance of [`Arc`] is always non-zero.
/// The object pointed to by [`Arc`] is always pinned.
///
/// # Examples
///
/// ```
/// use kernel::sync::Arc;
///
/// struct Example {
/// a: u32,
/// b: u32,
/// }
///
/// // Create a refcounted instance of `Example`.
/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
///
/// // Get a new pointer to `obj` and increment the refcount.
/// let cloned = obj.clone();
///
/// // Assert that both `obj` and `cloned` point to the same underlying object.
/// assert!(core::ptr::eq(&*obj, &*cloned));
///
/// // Destroy `obj` and decrement its refcount.
/// drop(obj);
///
/// // Check that the values are still accessible through `cloned`.
/// assert_eq!(cloned.a, 10);
/// assert_eq!(cloned.b, 20);
///
/// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed.
/// # Ok::<(), Error>(())
/// ```
///
/// Using `Arc<T>` as the type of `self`:
///
/// ```
/// use kernel::sync::Arc;
///
/// struct Example {
/// a: u32,
/// b: u32,
/// }
///
/// impl Example {
/// fn take_over(self: Arc<Self>) {
/// // ...
/// }
///
/// fn use_reference(self: &Arc<Self>) {
/// // ...
/// }
/// }
///
/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
/// obj.use_reference();
/// obj.take_over();
/// # Ok::<(), Error>(())
/// ```
///
/// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`:
///
/// ```
/// use kernel::sync::{Arc, ArcBorrow};
///
/// trait MyTrait {
/// // Trait has a function whose `self` type is `Arc<Self>`.
/// fn example1(self: Arc<Self>) {}
///
/// // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`.
/// fn example2(self: ArcBorrow<'_, Self>) {}
/// }
///
/// struct Example;
/// impl MyTrait for Example {}
///
/// // `obj` has type `Arc<Example>`.
/// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?;
///
/// // `coerced` has type `Arc<dyn MyTrait>`.
/// let coerced: Arc<dyn MyTrait> = obj;
/// # Ok::<(), Error>(())
/// ```
#[repr(transparent)]
#[derive(core::marker::CoercePointee)]
pub struct Arc<T: ?Sized> {
ptr: NonNull<ArcInner<T>>,
// NB: this informs dropck that objects of type `ArcInner<T>` may be used in `<Arc<T> as
// Drop>::drop`. Note that dropck already assumes that objects of type `T` may be used in
// `<Arc<T> as Drop>::drop` and the distinction between `T` and `ArcInner<T>` is not presently
// meaningful with respect to dropck - but this may change in the future so this is left here
// out of an abundance of caution.
//
// See <https://doc.rust-lang.org/nomicon/phantom-data.html#generic-parameters-and-drop-checking>
// for more detail on the semantics of dropck in the presence of `PhantomData`.
_p: PhantomData<ArcInner<T>>,
}
#[pin_data]
#[repr(C)]
struct ArcInner<T: ?Sized> {
refcount: Refcount,
data: T,
}
impl<T: ?Sized> ArcInner<T> {
/// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`].
///
/// # Safety
///
/// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must
/// not yet have been destroyed.
unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> {
let refcount_layout = Layout::new::<Refcount>();
// SAFETY: The caller guarantees that the pointer is valid.
let val_layout = Layout::for_value(unsafe { &*ptr });
// SAFETY: We're computing the layout of a real struct that existed when compiling this
// binary, so its layout is not so large that it can trigger arithmetic overflow.
let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 };
// Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and
// `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field.
//
// This is documented at:
// <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>.
let ptr = ptr as *const ArcInner<T>;
// SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the
// pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is
// still valid.
let ptr = unsafe { ptr.byte_sub(val_offset) };
// SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null
// address.
unsafe { NonNull::new_unchecked(ptr.cast_mut()) }
}
}
// SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because
// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
// `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a
// mutable reference when the reference count reaches zero and `T` is dropped.
unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
// SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync`
// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally,
// it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an
// `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when
// the reference count reaches zero and `T` is dropped.
unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
impl<T> InPlaceInit<T> for Arc<T> {
type PinnedSelf = Self;
#[inline]
fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
where
E: From<AllocError>,
{
UniqueArc::try_pin_init(init, flags).map(|u| u.into())
}
#[inline]
fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
where
E: From<AllocError>,
{
UniqueArc::try_init(init, flags).map(|u| u.into())
}
}
impl<T> Arc<T> {
/// Constructs a new reference counted instance of `T`.
pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> {
// INVARIANT: The refcount is initialised to a non-zero value.
let value = ArcInner {
refcount: Refcount::new(1),
data: contents,
};
let inner = KBox::new(value, flags)?;
let inner = KBox::leak(inner).into();
// SAFETY: We just created `inner` with a reference count of 1, which is owned by the new
// `Arc` object.
Ok(unsafe { Self::from_inner(inner) })
}
/// The offset that the value is stored at.
pub const DATA_OFFSET: usize = core::mem::offset_of!(ArcInner<T>, data);
}
impl<T: ?Sized> Arc<T> {
/// Constructs a new [`Arc`] from an existing [`ArcInner`].
///
/// # Safety
///
/// The caller must ensure that `inner` points to a valid location and has a non-zero reference
/// count, one of which will be owned by the new [`Arc`] instance.
unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self {
// INVARIANT: By the safety requirements, the invariants hold.
Arc {
ptr: inner,
_p: PhantomData,
}
}
/// Convert the [`Arc`] into a raw pointer.
///
/// The raw pointer has ownership of the refcount that this Arc object owned.
pub fn into_raw(self) -> *const T {
let ptr = self.ptr.as_ptr();
core::mem::forget(self);
// SAFETY: The pointer is valid.
unsafe { core::ptr::addr_of!((*ptr).data) }
}
/// Return a raw pointer to the data in this arc.
pub fn as_ptr(this: &Self) -> *const T {
let ptr = this.ptr.as_ptr();
// SAFETY: As `ptr` points to a valid allocation of type `ArcInner`,
// field projection to `data`is within bounds of the allocation.
unsafe { core::ptr::addr_of!((*ptr).data) }
}
/// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`].
///
/// # Safety
///
/// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it
/// must not be called more than once for each previous call to [`Arc::into_raw`].
pub unsafe fn from_raw(ptr: *const T) -> Self {
// SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
// `Arc` that is still valid.
let ptr = unsafe { ArcInner::container_of(ptr) };
// SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the
// reference count held then will be owned by the new `Arc` object.
unsafe { Self::from_inner(ptr) }
}
/// Returns an [`ArcBorrow`] from the given [`Arc`].
///
/// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method
/// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised.
#[inline]
pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> {
// SAFETY: The constraint that the lifetime of the shared reference must outlive that of
// the returned `ArcBorrow` ensures that the object remains alive and that no mutable
// reference can be created.
unsafe { ArcBorrow::new(self.ptr) }
}
/// Compare whether two [`Arc`] pointers reference the same underlying object.
pub fn ptr_eq(this: &Self, other: &Self) -> bool {
core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr())
}
/// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique.
///
/// When this destroys the `Arc`, it does so while properly avoiding races. This means that
/// this method will never call the destructor of the value.
///
/// # Examples
///
/// ```
/// use kernel::sync::{Arc, UniqueArc};
///
/// let arc = Arc::new(42, GFP_KERNEL)?;
/// let unique_arc = Arc::into_unique_or_drop(arc);
///
/// // The above conversion should succeed since refcount of `arc` is 1.
/// assert!(unique_arc.is_some());
///
/// assert_eq!(*(unique_arc.unwrap()), 42);
///
/// # Ok::<(), Error>(())
/// ```
///
/// ```
/// use kernel::sync::{Arc, UniqueArc};
///
/// let arc = Arc::new(42, GFP_KERNEL)?;
/// let another = arc.clone();
///
/// let unique_arc = Arc::into_unique_or_drop(arc);
///
/// // The above conversion should fail since refcount of `arc` is >1.
/// assert!(unique_arc.is_none());
///
/// # Ok::<(), Error>(())
/// ```
pub fn into_unique_or_drop(this: Self) -> Option<Pin<UniqueArc<T>>> {
// We will manually manage the refcount in this method, so we disable the destructor.
let this = ManuallyDrop::new(this);
// SAFETY: We own a refcount, so the pointer is still valid.
let refcount = unsafe { &this.ptr.as_ref().refcount };
// If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will
// return without further touching the `Arc`. If the refcount reaches zero, then there are
// no other arcs, and we can create a `UniqueArc`.
if refcount.dec_and_test() {
refcount.set(1);
// INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We
// must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin
// their values.
Some(Pin::from(UniqueArc {
inner: ManuallyDrop::into_inner(this),
}))
} else {
None
}
}
}
// SAFETY: The pointer returned by `into_foreign` was originally allocated as an
// `KBox<ArcInner<T>>`, so that type is what determines the alignment.
unsafe impl<T: 'static> ForeignOwnable for Arc<T> {
const FOREIGN_ALIGN: usize = <KBox<ArcInner<T>> as ForeignOwnable>::FOREIGN_ALIGN;
type Borrowed<'a> = ArcBorrow<'a, T>;
type BorrowedMut<'a> = Self::Borrowed<'a>;
fn into_foreign(self) -> *mut c_void {
ManuallyDrop::new(self).ptr.as_ptr().cast()
}
unsafe fn from_foreign(ptr: *mut c_void) -> Self {
// SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
// call to `Self::into_foreign`.
let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) };
// SAFETY: By the safety requirement of this function, we know that `ptr` came from
// a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and
// holds a reference count increment that is transferrable to us.
unsafe { Self::from_inner(inner) }
}
unsafe fn borrow<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> {
// SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
// call to `Self::into_foreign`.
let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) };
// SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive
// for the lifetime of the returned value.
unsafe { ArcBorrow::new(inner) }
}
unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> {
// SAFETY: The safety requirements for `borrow_mut` are a superset of the safety
// requirements for `borrow`.
unsafe { <Self as ForeignOwnable>::borrow(ptr) }
}
}
impl<T: ?Sized> Deref for Arc<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
// SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
// safe to dereference it.
unsafe { &self.ptr.as_ref().data }
}
}
impl<T: ?Sized> AsRef<T> for Arc<T> {
fn as_ref(&self) -> &T {
self.deref()
}
}
/// # Examples
///
/// ```
/// # use core::borrow::Borrow;
/// # use kernel::sync::Arc;
/// struct Foo<B: Borrow<u32>>(B);
///
/// // Owned instance.
/// let owned = Foo(1);
///
/// // Shared instance.
/// let arc = Arc::new(1, GFP_KERNEL)?;
/// let shared = Foo(arc.clone());
///
/// let i = 1;
/// // Borrowed from `i`.
/// let borrowed = Foo(&i);
/// # Ok::<(), Error>(())
/// ```
impl<T: ?Sized> Borrow<T> for Arc<T> {
fn borrow(&self) -> &T {
self.deref()
}
}
impl<T: ?Sized> Clone for Arc<T> {
fn clone(&self) -> Self {
// INVARIANT: `Refcount` saturates the refcount, so it cannot overflow to zero.
// SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
// safe to increment the refcount.
unsafe { self.ptr.as_ref() }.refcount.inc();
// SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`.
unsafe { Self::from_inner(self.ptr) }
}
}
impl<T: ?Sized> Drop for Arc<T> {
fn drop(&mut self) {
// INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and
// this instance is being dropped, so the broken invariant is not observable.
// SAFETY: By the type invariant, there is necessarily a reference to the object.
let is_zero = unsafe { self.ptr.as_ref() }.refcount.dec_and_test();
if is_zero {
// The count reached zero, we must free the memory.
//
// SAFETY: The pointer was initialised from the result of `KBox::leak`.
unsafe { drop(KBox::from_raw(self.ptr.as_ptr())) };
}
}
}
impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> {
fn from(item: UniqueArc<T>) -> Self {
item.inner
}
}
impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {
fn from(item: Pin<UniqueArc<T>>) -> Self {
// SAFETY: The type invariants of `Arc` guarantee that the data is pinned.
unsafe { Pin::into_inner_unchecked(item).inner }
}
}
/// A borrowed reference to an [`Arc`] instance.
///
/// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler
/// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance.
///
/// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>`
/// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference)
/// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double
/// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if
/// needed.
///
/// # Invariants
///
/// There are no mutable references to the underlying [`Arc`], and it remains valid for the
/// lifetime of the [`ArcBorrow`] instance.
///
/// # Examples
///
/// ```
/// use kernel::sync::{Arc, ArcBorrow};
///
/// struct Example;
///
/// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> {
/// e.into()
/// }
///
/// let obj = Arc::new(Example, GFP_KERNEL)?;
/// let cloned = do_something(obj.as_arc_borrow());
///
/// // Assert that both `obj` and `cloned` point to the same underlying object.
/// assert!(core::ptr::eq(&*obj, &*cloned));
/// # Ok::<(), Error>(())
/// ```
///
/// Using `ArcBorrow<T>` as the type of `self`:
///
/// ```
/// use kernel::sync::{Arc, ArcBorrow};
///
/// struct Example {
/// a: u32,
/// b: u32,
/// }
///
/// impl Example {
/// fn use_reference(self: ArcBorrow<'_, Self>) {
/// // ...
/// }
/// }
///
/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
/// obj.as_arc_borrow().use_reference();
/// # Ok::<(), Error>(())
/// ```
#[repr(transparent)]
#[derive(core::marker::CoercePointee)]
pub struct ArcBorrow<'a, T: ?Sized + 'a> {
inner: NonNull<ArcInner<T>>,
_p: PhantomData<&'a ()>,
}
impl<T: ?Sized> Clone for ArcBorrow<'_, T> {
fn clone(&self) -> Self {
*self
}
}
impl<T: ?Sized> Copy for ArcBorrow<'_, T> {}
impl<T: ?Sized> ArcBorrow<'_, T> {
/// Creates a new [`ArcBorrow`] instance.
///
/// # Safety
///
/// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance:
/// 1. That `inner` remains valid;
/// 2. That no mutable references to `inner` are created.
unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self {
// INVARIANT: The safety requirements guarantee the invariants.
Self {
inner,
_p: PhantomData,
}
}
/// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with
/// [`Arc::into_raw`] or [`Arc::as_ptr`].
///
/// # Safety
///
/// * The provided pointer must originate from a call to [`Arc::into_raw`] or [`Arc::as_ptr`].
/// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must
/// not hit zero.
/// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a
/// [`UniqueArc`] reference to this value.
pub unsafe fn from_raw(ptr: *const T) -> Self {
// SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
// `Arc` that is still valid.
let ptr = unsafe { ArcInner::container_of(ptr) };
// SAFETY: The caller promises that the value remains valid since the reference count must
// not hit zero, and no mutable reference will be created since that would involve a
// `UniqueArc`.
unsafe { Self::new(ptr) }
}
}
impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> {
fn from(b: ArcBorrow<'_, T>) -> Self {
// SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop`
// guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the
// increment.
ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) })
.deref()
.clone()
}
}
impl<T: ?Sized> Deref for ArcBorrow<'_, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
// SAFETY: By the type invariant, the underlying object is still alive with no mutable
// references to it, so it is safe to create a shared reference.
unsafe { &self.inner.as_ref().data }
}
}
/// A refcounted object that is known to have a refcount of 1.
///
/// It is mutable and can be converted to an [`Arc`] so that it can be shared.
///
/// # Invariants
///
/// `inner` always has a reference count of 1.
///
/// # Examples
///
/// In the following example, we make changes to the inner object before turning it into an
/// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()`
/// cannot fail.
///
/// ```
/// use kernel::sync::{Arc, UniqueArc};
///
/// struct Example {
/// a: u32,
/// b: u32,
/// }
///
/// fn test() -> Result<Arc<Example>> {
/// let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
/// x.a += 1;
/// x.b += 1;
/// Ok(x.into())
/// }
///
/// # test().unwrap();
/// ```
///
/// In the following example we first allocate memory for a refcounted `Example` but we don't
/// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`],
/// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens
/// in one context (e.g., sleepable) and initialisation in another (e.g., atomic):
///
/// ```
/// use kernel::sync::{Arc, UniqueArc};
///
/// struct Example {
/// a: u32,
/// b: u32,
/// }
///
/// fn test() -> Result<Arc<Example>> {
/// let x = UniqueArc::new_uninit(GFP_KERNEL)?;
/// Ok(x.write(Example { a: 10, b: 20 }).into())
/// }
///
/// # test().unwrap();
/// ```
///
/// In the last example below, the caller gets a pinned instance of `Example` while converting to
/// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during
/// initialisation, for example, when initialising fields that are wrapped in locks.
///
/// ```
/// use kernel::sync::{Arc, UniqueArc};
///
/// struct Example {
/// a: u32,
/// b: u32,
/// }
///
/// fn test() -> Result<Arc<Example>> {
/// let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?);
/// // We can modify `pinned` because it is `Unpin`.
/// pinned.as_mut().a += 1;
/// Ok(pinned.into())
/// }
///
/// # test().unwrap();
/// ```
pub struct UniqueArc<T: ?Sized> {
inner: Arc<T>,
}
impl<T> InPlaceInit<T> for UniqueArc<T> {
type PinnedSelf = Pin<Self>;
#[inline]
fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
where
E: From<AllocError>,
{
UniqueArc::new_uninit(flags)?.write_pin_init(init)
}
#[inline]
fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
where
E: From<AllocError>,
{
UniqueArc::new_uninit(flags)?.write_init(init)
}
}
impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> {
type Initialized = UniqueArc<T>;
#[inline]
fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid.
unsafe { init.__init(slot)? };
// SAFETY: All fields have been initialized.
Ok(unsafe { self.assume_init() })
}
#[inline]
fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid and will not be moved, because we pin it later.
unsafe { init.__pinned_init(slot)? };
// SAFETY: All fields have been initialized.
Ok(unsafe { self.assume_init() }.into())
}
}
impl<T> UniqueArc<T> {
/// Tries to allocate a new [`UniqueArc`] instance.
pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> {
Ok(Self {
// INVARIANT: The newly-created object has a refcount of 1.
inner: Arc::new(value, flags)?,
})
}
/// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet.
pub fn new_uninit(flags: Flags) -> Result<UniqueArc<MaybeUninit<T>>, AllocError> {
// INVARIANT: The refcount is initialised to a non-zero value.
let inner = KBox::try_init::<AllocError>(
try_init!(ArcInner {
refcount: Refcount::new(1),
data <- pin_init::uninit::<T, AllocError>(),
}? AllocError),
flags,
)?;
Ok(UniqueArc {
// INVARIANT: The newly-created object has a refcount of 1.
// SAFETY: The pointer from the `KBox` is valid.
inner: unsafe { Arc::from_inner(KBox::leak(inner).into()) },
})
}
}
impl<T: ?Sized> UniqueArc<T> {
/// Return a raw pointer to the data in this [`UniqueArc`].
#[inline]
pub fn as_ptr(this: &Self) -> *const T {
Arc::as_ptr(&this.inner)
}
}
impl<T> UniqueArc<MaybeUninit<T>> {
/// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it.
pub fn write(mut self, value: T) -> UniqueArc<T> {
self.deref_mut().write(value);
// SAFETY: We just wrote the value to be initialized.
unsafe { self.assume_init() }
}
/// Unsafely assume that `self` is initialized.
///
/// # Safety
///
/// The caller guarantees that the value behind this pointer has been initialized. It is
/// *immediate* UB to call this when the value is not initialized.
pub unsafe fn assume_init(self) -> UniqueArc<T> {
let inner = ManuallyDrop::new(self).inner.ptr;
UniqueArc {
// SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be
// dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`.
inner: unsafe { Arc::from_inner(inner.cast()) },
}
}
/// Initialize `self` using the given initializer.
#[inline]
pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> {
// SAFETY: The supplied pointer is valid for initialization.
match unsafe { init.__init(self.as_mut_ptr()) } {
// SAFETY: Initialization completed successfully.
Ok(()) => Ok(unsafe { self.assume_init() }),
Err(err) => Err(err),
}
}
/// Pin-initialize `self` using the given pin-initializer.
#[inline]
pub fn pin_init_with<E>(
mut self,
init: impl PinInit<T, E>,
) -> core::result::Result<Pin<UniqueArc<T>>, E> {
// SAFETY: The supplied pointer is valid for initialization and we will later pin the value
// to ensure it does not move.
match unsafe { init.__pinned_init(self.as_mut_ptr()) } {
// SAFETY: Initialization completed successfully.
Ok(()) => Ok(unsafe { self.assume_init() }.into()),
Err(err) => Err(err),
}
}
}
impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> {
fn from(obj: UniqueArc<T>) -> Self {
// SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T`
// is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`.
unsafe { Pin::new_unchecked(obj) }
}
}
impl<T: ?Sized> Deref for UniqueArc<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
self.inner.deref()
}
}
impl<T: ?Sized> DerefMut for UniqueArc<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
// SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so
// it is safe to dereference it. Additionally, we know there is only one reference when
// it's inside a `UniqueArc`, so it is safe to get a mutable reference.
unsafe { &mut self.inner.ptr.as_mut().data }
}
}
/// # Examples
///
/// ```
/// # use core::borrow::Borrow;
/// # use kernel::sync::UniqueArc;
/// struct Foo<B: Borrow<u32>>(B);
///
/// // Owned instance.
/// let owned = Foo(1);
///
/// // Owned instance using `UniqueArc`.
/// let arc = UniqueArc::new(1, GFP_KERNEL)?;
/// let shared = Foo(arc);
///
/// let i = 1;
/// // Borrowed from `i`.
/// let borrowed = Foo(&i);
/// # Ok::<(), Error>(())
/// ```
impl<T: ?Sized> Borrow<T> for UniqueArc<T> {
fn borrow(&self) -> &T {
self.deref()
}
}
/// # Examples
///
/// ```
/// # use core::borrow::BorrowMut;
/// # use kernel::sync::UniqueArc;
/// struct Foo<B: BorrowMut<u32>>(B);
///
/// // Owned instance.
/// let owned = Foo(1);
///
/// // Owned instance using `UniqueArc`.
/// let arc = UniqueArc::new(1, GFP_KERNEL)?;
/// let shared = Foo(arc);
///
/// let mut i = 1;
/// // Borrowed from `i`.
/// let borrowed = Foo(&mut i);
/// # Ok::<(), Error>(())
/// ```
impl<T: ?Sized> BorrowMut<T> for UniqueArc<T> {
fn borrow_mut(&mut self) -> &mut T {
self.deref_mut()
}
}
impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(self.deref(), f)
}
}
impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(self.deref(), f)
}
}
impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(self.deref(), f)
}
}
impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(self.deref(), f)
}
}
linux-7.2.3/rust/kernel/sync/arc/std_vendor.rs
+30
View File
@@ -0,0 +1,30 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
//! Rust standard library vendored code.
//!
//! The contents of this file come from the Rust standard library, hosted in
//! the <https://github.com/rust-lang/rust> repository, licensed under
//! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details,
//! see <https://github.com/rust-lang/rust/blob/master/COPYRIGHT>.
use crate::sync::{arc::ArcInner, Arc};
use core::any::Any;
impl Arc<dyn Any + Send + Sync> {
/// Attempt to downcast the `Arc<dyn Any + Send + Sync>` to a concrete type.
pub fn downcast<T>(self) -> core::result::Result<Arc<T>, Self>
where
T: Any + Send + Sync,
{
if (*self).is::<T>() {
// SAFETY: We have just checked that the type is correct, so we can cast the pointer.
unsafe {
let ptr = self.ptr.cast::<ArcInner<T>>();
core::mem::forget(self);
Ok(Arc::from_inner(ptr))
}
} else {
Err(self)
}
}
}
linux-7.2.3/rust/kernel/sync/aref.rs
+199
View File
@@ -0,0 +1,199 @@
// SPDX-License-Identifier: GPL-2.0
//! Internal reference counting support.
//!
//! Many C types already have their own reference counting mechanism (e.g. by storing a
//! `refcount_t`). This module provides support for directly using their internal reference count
//! from Rust; instead of making users have to use an additional Rust-reference count in the form of
//! [`Arc`].
//!
//! The smart pointer [`ARef<T>`] acts similarly to [`Arc<T>`] in that it holds a refcount on the
//! underlying object, but this refcount is internal to the object. It essentially is a Rust
//! implementation of the `get_` and `put_` pattern used in C for reference counting.
//!
//! To make use of [`ARef<MyType>`], `MyType` needs to implement [`AlwaysRefCounted`]. It is a trait
//! for accessing the internal reference count of an object of the `MyType` type.
//!
//! [`Arc`]: crate::sync::Arc
//! [`Arc<T>`]: crate::sync::Arc
use core::{
marker::PhantomData,
mem::ManuallyDrop,
ops::Deref,
ptr::NonNull, //
};
/// Types that are _always_ reference counted.
///
/// It allows such types to define their own custom ref increment and decrement functions.
/// Additionally, it allows users to convert from a shared reference `&T` to an owned reference
/// [`ARef<T>`].
///
/// This is usually implemented by wrappers to existing structures on the C side of the code. For
/// Rust code, the recommendation is to use [`Arc`](crate::sync::Arc) to create reference-counted
/// instances of a type.
///
/// # Safety
///
/// Implementers must ensure that increments to the reference count keep the object alive in memory
/// at least until matching decrements are performed.
///
/// Implementers must also ensure that all instances are reference-counted. (Otherwise they
/// won't be able to honour the requirement that [`AlwaysRefCounted::inc_ref`] keep the object
/// alive.)
pub unsafe trait AlwaysRefCounted {
/// Increments the reference count on the object.
fn inc_ref(&self);
/// Decrements the reference count on the object.
///
/// Frees the object when the count reaches zero.
///
/// # Safety
///
/// Callers must ensure that there was a previous matching increment to the reference count,
/// and that the object is no longer used after its reference count is decremented (as it may
/// result in the object being freed), unless the caller owns another increment on the refcount
/// (e.g., it calls [`AlwaysRefCounted::inc_ref`] twice, then calls
/// [`AlwaysRefCounted::dec_ref`] once).
unsafe fn dec_ref(obj: NonNull<Self>);
}
/// An owned reference to an always-reference-counted object.
///
/// The object's reference count is automatically decremented when an instance of [`ARef`] is
/// dropped. It is also automatically incremented when a new instance is created via
/// [`ARef::clone`].
///
/// # Invariants
///
/// The pointer stored in `ptr` is non-null and valid for the lifetime of the [`ARef`] instance. In
/// particular, the [`ARef`] instance owns an increment on the underlying object's reference count.
pub struct ARef<T: AlwaysRefCounted> {
ptr: NonNull<T>,
_p: PhantomData<T>,
}
// SAFETY: It is safe to send `ARef<T>` to another thread when the underlying `T` is `Sync` because
// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
// `T` to be `Send` because any thread that has an `ARef<T>` may ultimately access `T` using a
// mutable reference, for example, when the reference count reaches zero and `T` is dropped.
unsafe impl<T: AlwaysRefCounted + Sync + Send> Send for ARef<T> {}
// SAFETY: It is safe to send `&ARef<T>` to another thread when the underlying `T` is `Sync`
// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally,
// it needs `T` to be `Send` because any thread that has a `&ARef<T>` may clone it and get an
// `ARef<T>` on that thread, so the thread may ultimately access `T` using a mutable reference, for
// example, when the reference count reaches zero and `T` is dropped.
unsafe impl<T: AlwaysRefCounted + Sync + Send> Sync for ARef<T> {}
// Even if `T` is pinned, pointers to `T` can still move.
impl<T: AlwaysRefCounted> Unpin for ARef<T> {}
impl<T: AlwaysRefCounted> ARef<T> {
/// Creates a new instance of [`ARef`].
///
/// It takes over an increment of the reference count on the underlying object.
///
/// # Safety
///
/// Callers must ensure that the reference count was incremented at least once, and that they
/// are properly relinquishing one increment. That is, if there is only one increment, callers
/// must not use the underlying object anymore -- it is only safe to do so via the newly
/// created [`ARef`].
pub unsafe fn from_raw(ptr: NonNull<T>) -> Self {
// INVARIANT: The safety requirements guarantee that the new instance now owns the
// increment on the refcount.
Self {
ptr,
_p: PhantomData,
}
}
/// Consumes the `ARef`, returning a raw pointer.
///
/// This function does not change the refcount. After calling this function, the caller is
/// responsible for the refcount previously managed by the `ARef`.
///
/// # Examples
///
/// ```
/// use core::ptr::NonNull;
/// use kernel::sync::aref::{ARef, AlwaysRefCounted};
///
/// struct Empty {}
///
/// # // SAFETY: TODO.
/// unsafe impl AlwaysRefCounted for Empty {
/// fn inc_ref(&self) {}
/// unsafe fn dec_ref(_obj: NonNull<Self>) {}
/// }
///
/// let mut data = Empty {};
/// let ptr = NonNull::<Empty>::new(&mut data).unwrap();
/// # // SAFETY: TODO.
/// let data_ref: ARef<Empty> = unsafe { ARef::from_raw(ptr) };
/// let raw_ptr: NonNull<Empty> = ARef::into_raw(data_ref);
///
/// assert_eq!(ptr, raw_ptr);
/// ```
pub fn into_raw(me: Self) -> NonNull<T> {
ManuallyDrop::new(me).ptr
}
}
impl<T: AlwaysRefCounted> Clone for ARef<T> {
fn clone(&self) -> Self {
self.inc_ref();
// SAFETY: We just incremented the refcount above.
unsafe { Self::from_raw(self.ptr) }
}
}
impl<T: AlwaysRefCounted> Deref for ARef<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
// SAFETY: The type invariants guarantee that the object is valid.
unsafe { self.ptr.as_ref() }
}
}
impl<T: AlwaysRefCounted> From<&T> for ARef<T> {
fn from(b: &T) -> Self {
b.inc_ref();
// SAFETY: We just incremented the refcount above.
unsafe { Self::from_raw(NonNull::from(b)) }
}
}
impl<T: AlwaysRefCounted> Drop for ARef<T> {
fn drop(&mut self) {
// SAFETY: The type invariants guarantee that the `ARef` owns the reference we're about to
// decrement.
unsafe { T::dec_ref(self.ptr) };
}
}
impl<T, U> PartialEq<ARef<U>> for ARef<T>
where
T: AlwaysRefCounted + PartialEq<U>,
U: AlwaysRefCounted,
{
#[inline]
fn eq(&self, other: &ARef<U>) -> bool {
T::eq(&**self, &**other)
}
}
impl<T: AlwaysRefCounted + Eq> Eq for ARef<T> {}
impl<T, U> PartialEq<&'_ U> for ARef<T>
where
T: AlwaysRefCounted + PartialEq<U>,
{
#[inline]
fn eq(&self, other: &&U) -> bool {
T::eq(&**self, other)
}
}
linux-7.2.3/rust/kernel/sync/atomic.rs
+850
View File
@@ -0,0 +1,850 @@
// SPDX-License-Identifier: GPL-2.0
//! Atomic primitives.
//!
//! These primitives have the same semantics as their C counterparts: and the precise definitions of
//! semantics can be found at [`LKMM`]. Note that Linux Kernel Memory (Consistency) Model is the
//! only model for Rust code in kernel, and Rust's own atomics should be avoided.
//!
//! # Data races
//!
//! [`LKMM`] atomics have different rules regarding data races:
//!
//! - A normal write from C side is treated as an atomic write if
//! CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=y.
//! - Mixed-size atomic accesses don't cause data races.
//!
//! [`LKMM`]: srctree/tools/memory-model/
mod internal;
pub mod ordering;
mod predefine;
pub use internal::AtomicImpl;
pub use ordering::{Acquire, Full, Relaxed, Release};
pub(crate) use internal::{AtomicArithmeticOps, AtomicBasicOps, AtomicExchangeOps};
use crate::build_error;
use internal::AtomicRepr;
use ordering::OrderingType;
/// A memory location which can be safely modified from multiple execution contexts.
///
/// This has the same size, alignment and bit validity as the underlying type `T`. And it disables
/// niche optimization for the same reason as [`UnsafeCell`].
///
/// The atomic operations are implemented in a way that is fully compatible with the [Linux Kernel
/// Memory (Consistency) Model][LKMM], hence they should be modeled as the corresponding
/// [`LKMM`][LKMM] atomic primitives. With the help of [`Atomic::from_ptr()`] and
/// [`Atomic::as_ptr()`], this provides a way to interact with [C-side atomic operations]
/// (including those without the `atomic` prefix, e.g. `READ_ONCE()`, `WRITE_ONCE()`,
/// `smp_load_acquire()` and `smp_store_release()`).
///
/// # Invariants
///
/// `self.0` is a valid `T`.
///
/// [`UnsafeCell`]: core::cell::UnsafeCell
/// [LKMM]: srctree/tools/memory-model/
/// [C-side atomic operations]: srctree/Documentation/atomic_t.txt
#[repr(transparent)]
pub struct Atomic<T: AtomicType>(AtomicRepr<T::Repr>);
// SAFETY: `Atomic<T>` is safe to transfer between execution contexts because of the safety
// requirement of `AtomicType`.
unsafe impl<T: AtomicType> Send for Atomic<T> {}
// SAFETY: `Atomic<T>` is safe to share among execution contexts because all accesses are atomic.
unsafe impl<T: AtomicType> Sync for Atomic<T> {}
/// Types that support basic atomic operations.
///
/// # Round-trip transmutability
///
/// `T` is round-trip transmutable to `U` if and only if both of these properties hold:
///
/// - Any valid bit pattern for `T` is also a valid bit pattern for `U`.
/// - Transmuting (e.g. using [`transmute()`]) a value of type `T` to `U` and then to `T` again
/// yields a value that is in all aspects equivalent to the original value.
///
/// # Safety
///
/// - [`Self`] must have the same size and alignment as [`Self::Repr`].
/// - [`Self`] must be [round-trip transmutable] to [`Self::Repr`].
/// - [`Self`] must be safe to transfer between execution contexts, if it's [`Send`], this is
/// automatically satisfied. The exception is pointer types that are even though marked as
/// `!Send` (e.g. raw pointers and [`NonNull<T>`]) but requiring `unsafe` to do anything
/// meaningful on them. This is because transferring pointer values between execution contexts is
/// safe as long as the actual `unsafe` dereferencing is justified.
///
/// Note that this is more relaxed than requiring the bi-directional transmutability (i.e.
/// [`transmute()`] is always sound between `U` and `T`) because of the support for atomic
/// variables over unit-only enums, see [Examples].
///
/// # Limitations
///
/// Because C primitives are used to implement the atomic operations, and a C function requires a
/// valid object of a type to operate on (i.e. no `MaybeUninit<_>`), hence at the Rust <-> C
/// surface, only types with all the bits initialized can be passed. As a result, types like `(u8,
/// u16)` (padding bytes are uninitialized) are currently not supported.
///
/// # Examples
///
/// A unit-only enum that implements [`AtomicType`]:
///
/// ```
/// use kernel::sync::atomic::{AtomicType, Atomic, Relaxed};
///
/// #[derive(Clone, Copy, PartialEq, Eq)]
/// #[repr(i32)]
/// enum State {
/// Uninit = 0,
/// Working = 1,
/// Done = 2,
/// };
///
/// // SAFETY: `State` and `i32` has the same size and alignment, and it's round-trip
/// // transmutable to `i32`.
/// unsafe impl AtomicType for State {
/// type Repr = i32;
/// }
///
/// let s = Atomic::new(State::Uninit);
///
/// assert_eq!(State::Uninit, s.load(Relaxed));
/// ```
/// [`transmute()`]: core::mem::transmute
/// [round-trip transmutable]: AtomicType#round-trip-transmutability
/// [Examples]: AtomicType#examples
/// [`NonNull<T>`]: core::ptr::NonNull
pub unsafe trait AtomicType: Sized + Copy {
/// The backing atomic implementation type.
type Repr: AtomicImpl;
}
/// Types that support atomic add operations.
///
/// # Safety
///
// TODO: Properly defines `wrapping_add` in the following comment.
/// `wrapping_add` any value of type `Self::Repr::Delta` obtained by [`Self::rhs_into_delta()`] to
/// any value of type `Self::Repr` obtained through transmuting a value of type `Self` to must
/// yield a value with a bit pattern also valid for `Self`.
pub unsafe trait AtomicAdd<Rhs = Self>: AtomicType {
/// Converts `Rhs` into the `Delta` type of the atomic implementation.
fn rhs_into_delta(rhs: Rhs) -> <Self::Repr as AtomicImpl>::Delta;
}
#[inline(always)]
const fn into_repr<T: AtomicType>(v: T) -> T::Repr {
// SAFETY: Per the safety requirement of `AtomicType`, `T` is round-trip transmutable to
// `T::Repr`, therefore the transmute operation is sound.
unsafe { core::mem::transmute_copy(&v) }
}
/// # Safety
///
/// `r` must be a valid bit pattern of `T`.
#[inline(always)]
const unsafe fn from_repr<T: AtomicType>(r: T::Repr) -> T {
// SAFETY: Per the safety requirement of the function, the transmute operation is sound.
unsafe { core::mem::transmute_copy(&r) }
}
impl<T: AtomicType> Atomic<T> {
/// Creates a new atomic `T`.
pub const fn new(v: T) -> Self {
// INVARIANT: Per the safety requirement of `AtomicType`, `into_repr(v)` is a valid `T`.
Self(AtomicRepr::new(into_repr(v)))
}
/// Creates a reference to an atomic `T` from a pointer of `T`.
///
/// This usually is used when communicating with C side or manipulating a C struct, see
/// examples below.
///
/// # Safety
///
/// - `ptr` is aligned to `align_of::<T>()`.
/// - `ptr` is valid for reads and writes for `'a`.
/// - For the duration of `'a`, other accesses to `*ptr` must not cause data races (defined
/// by [`LKMM`]) against atomic operations on the returned reference. Note that if all other
/// accesses are atomic, then this safety requirement is trivially fulfilled.
///
/// [`LKMM`]: srctree/tools/memory-model
///
/// # Examples
///
/// Using [`Atomic::from_ptr()`] combined with [`Atomic::load()`] or [`Atomic::store()`] can
/// achieve the same functionality as `READ_ONCE()`/`smp_load_acquire()` or
/// `WRITE_ONCE()`/`smp_store_release()` in C side:
///
/// ```
/// # use kernel::types::Opaque;
/// use kernel::sync::atomic::{Atomic, Relaxed, Release};
///
/// // Assume there is a C struct `foo`.
/// mod cbindings {
/// #[repr(C)]
/// pub(crate) struct foo {
/// pub(crate) a: i32,
/// pub(crate) b: i32
/// }
/// }
///
/// let tmp = Opaque::new(cbindings::foo { a: 1, b: 2 });
///
/// // struct foo *foo_ptr = ..;
/// let foo_ptr = tmp.get();
///
/// // SAFETY: `foo_ptr` is valid, and `.a` is in bounds.
/// let foo_a_ptr = unsafe { &raw mut (*foo_ptr).a };
///
/// // a = READ_ONCE(foo_ptr->a);
/// //
/// // SAFETY: `foo_a_ptr` is valid for read, and all other accesses on it is atomic, so no
/// // data race.
/// let a = unsafe { Atomic::from_ptr(foo_a_ptr) }.load(Relaxed);
/// # assert_eq!(a, 1);
///
/// // smp_store_release(&foo_ptr->a, 2);
/// //
/// // SAFETY: `foo_a_ptr` is valid for writes, and all other accesses on it is atomic, so
/// // no data race.
/// unsafe { Atomic::from_ptr(foo_a_ptr) }.store(2, Release);
/// ```
pub unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Self {
// CAST: `T` and `Atomic<T>` have the same size, alignment and bit validity.
// SAFETY: Per function safety requirement, `ptr` is a valid pointer and the object will
// live long enough. It's safe to return a `&Atomic<T>` because function safety requirement
// guarantees other accesses won't cause data races.
unsafe { &*ptr.cast::<Self>() }
}
/// Returns a pointer to the underlying atomic `T`.
///
/// Note that use of the return pointer must not cause data races defined by [`LKMM`].
///
/// # Guarantees
///
/// The returned pointer is valid and properly aligned (i.e. aligned to [`align_of::<T>()`]).
///
/// [`LKMM`]: srctree/tools/memory-model
/// [`align_of::<T>()`]: core::mem::align_of
pub const fn as_ptr(&self) -> *mut T {
// GUARANTEE: Per the function guarantee of `AtomicRepr::as_ptr()`, the `self.0.as_ptr()`
// must be a valid and properly aligned pointer for `T::Repr`, and per the safety guarantee
// of `AtomicType`, it's a valid and properly aligned pointer of `T`.
self.0.as_ptr().cast()
}
/// Returns a mutable reference to the underlying atomic `T`.
///
/// This is safe because the mutable reference of the atomic `T` guarantees exclusive access.
///
/// # Examples
///
/// ```
/// use kernel::sync::atomic::{Atomic, Relaxed};
///
/// let mut atomic_val = Atomic::new(0u32);
/// let val_mut = atomic_val.get_mut();
/// *val_mut = 101;
/// assert_eq!(101, atomic_val.load(Relaxed));
/// ```
pub fn get_mut(&mut self) -> &mut T {
// CAST: `T` and `T::Repr` has the same size and alignment per the safety requirement of
// `AtomicType`, and per the type invariants `self.0` is a valid `T`, therefore the casting
// result is a valid pointer of `T`.
// SAFETY: The pointer is valid per the CAST comment above, and the mutable reference
// guarantees exclusive access.
unsafe { &mut *self.0.as_ptr().cast() }
}
}
impl<T: AtomicType> Atomic<T>
where
T::Repr: AtomicBasicOps,
{
/// Loads the value from the atomic `T`.
///
/// # Examples
///
/// ```
/// use kernel::sync::atomic::{Atomic, Relaxed};
///
/// let x = Atomic::new(42i32);
///
/// assert_eq!(42, x.load(Relaxed));
///
/// let x = Atomic::new(42i64);
///
/// assert_eq!(42, x.load(Relaxed));
/// ```
#[doc(alias("atomic_read", "atomic64_read"))]
#[inline(always)]
pub fn load<Ordering: ordering::AcquireOrRelaxed>(&self, _: Ordering) -> T {
let v = {
match Ordering::TYPE {
OrderingType::Relaxed => T::Repr::atomic_read(&self.0),
OrderingType::Acquire => T::Repr::atomic_read_acquire(&self.0),
_ => build_error!("Wrong ordering"),
}
};
// SAFETY: `v` comes from reading `self.0`, which is a valid `T` per the type invariants.
unsafe { from_repr(v) }
}
/// Stores a value to the atomic `T`.
///
/// # Examples
///
/// ```
/// use kernel::sync::atomic::{Atomic, Relaxed};
///
/// let x = Atomic::new(42i32);
///
/// assert_eq!(42, x.load(Relaxed));
///
/// x.store(43, Relaxed);
///
/// assert_eq!(43, x.load(Relaxed));
/// ```
#[doc(alias("atomic_set", "atomic64_set"))]
#[inline(always)]
pub fn store<Ordering: ordering::ReleaseOrRelaxed>(&self, v: T, _: Ordering) {
let v = into_repr(v);
// INVARIANT: `v` is a valid `T`, and is stored to `self.0` by `atomic_set*()`.
match Ordering::TYPE {
OrderingType::Relaxed => T::Repr::atomic_set(&self.0, v),
OrderingType::Release => T::Repr::atomic_set_release(&self.0, v),
_ => build_error!("Wrong ordering"),
}
}
}
impl<T: AtomicType + core::fmt::Debug> core::fmt::Debug for Atomic<T>
where
T::Repr: AtomicBasicOps,
{
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
core::fmt::Debug::fmt(&self.load(Relaxed), f)
}
}
impl<T: AtomicType> Atomic<T>
where
T::Repr: AtomicExchangeOps,
{
/// Atomic exchange.
///
/// Atomically updates `*self` to `v` and returns the old value of `*self`.
///
/// # Examples
///
/// ```
/// use kernel::sync::atomic::{Atomic, Acquire, Relaxed};
///
/// let x = Atomic::new(42);
///
/// assert_eq!(42, x.xchg(52, Acquire));
/// assert_eq!(52, x.load(Relaxed));
/// ```
#[doc(alias("atomic_xchg", "atomic64_xchg", "swap"))]
#[inline(always)]
pub fn xchg<Ordering: ordering::Ordering>(&self, v: T, _: Ordering) -> T {
let v = into_repr(v);
// INVARIANT: `self.0` is a valid `T` after `atomic_xchg*()` because `v` is transmutable to
// `T`.
let ret = {
match Ordering::TYPE {
OrderingType::Full => T::Repr::atomic_xchg(&self.0, v),
OrderingType::Acquire => T::Repr::atomic_xchg_acquire(&self.0, v),
OrderingType::Release => T::Repr::atomic_xchg_release(&self.0, v),
OrderingType::Relaxed => T::Repr::atomic_xchg_relaxed(&self.0, v),
}
};
// SAFETY: `ret` comes from reading `*self`, which is a valid `T` per type invariants.
unsafe { from_repr(ret) }
}
/// Atomic compare and exchange.
///
/// If `*self` == `old`, atomically updates `*self` to `new`. Otherwise, `*self` is not
/// modified.
///
/// Compare: The comparison is done via the byte level comparison between `*self` and `old`.
///
/// Ordering: When succeeds, provides the corresponding ordering as the `Ordering` type
/// parameter indicates, and a failed one doesn't provide any ordering, the load part of a
/// failed cmpxchg is a [`Relaxed`] load.
///
/// Returns `Ok(value)` if cmpxchg succeeds, and `value` is guaranteed to be equal to `old`,
/// otherwise returns `Err(value)`, and `value` is the current value of `*self`.
///
/// # Examples
///
/// ```
/// use kernel::sync::atomic::{Atomic, Full, Relaxed};
///
/// let x = Atomic::new(42);
///
/// // Checks whether cmpxchg succeeded.
/// let success = x.cmpxchg(52, 64, Relaxed).is_ok();
/// # assert!(!success);
///
/// // Checks whether cmpxchg failed.
/// let failure = x.cmpxchg(52, 64, Relaxed).is_err();
/// # assert!(failure);
///
/// // Uses the old value if failed, probably re-try cmpxchg.
/// match x.cmpxchg(52, 64, Relaxed) {
/// Ok(_) => { },
/// Err(old) => {
/// // do something with `old`.
/// # assert_eq!(old, 42);
/// }
/// }
///
/// // Uses the latest value regardlessly, same as atomic_cmpxchg() in C.
/// let latest = x.cmpxchg(42, 64, Full).unwrap_or_else(|old| old);
/// # assert_eq!(42, latest);
/// assert_eq!(64, x.load(Relaxed));
/// ```
///
/// [`Relaxed`]: ordering::Relaxed
#[doc(alias(
"atomic_cmpxchg",
"atomic64_cmpxchg",
"atomic_try_cmpxchg",
"atomic64_try_cmpxchg",
"compare_exchange"
))]
#[inline(always)]
pub fn cmpxchg<Ordering: ordering::Ordering>(
&self,
mut old: T,
new: T,
o: Ordering,
) -> Result<T, T> {
// Note on code generation:
//
// try_cmpxchg() is used to implement cmpxchg(), and if the helper functions are inlined,
// the compiler is able to figure out that branch is not needed if the users don't care
// about whether the operation succeeds or not. One exception is on x86, due to commit
// 44fe84459faf ("locking/atomic: Fix atomic_try_cmpxchg() semantics"), the
// atomic_try_cmpxchg() on x86 has a branch even if the caller doesn't care about the
// success of cmpxchg and only wants to use the old value. For example, for code like:
//
// let latest = x.cmpxchg(42, 64, Full).unwrap_or_else(|old| old);
//
// It will still generate code:
//
// movl $0x40, %ecx
// movl $0x34, %eax
// lock
// cmpxchgl %ecx, 0x4(%rsp)
// jne 1f
// 2:
// ...
// 1: movl %eax, %ecx
// jmp 2b
//
// This might be "fixed" by introducing a try_cmpxchg_exclusive() that knows the "*old"
// location in the C function is always safe to write.
if self.try_cmpxchg(&mut old, new, o) {
Ok(old)
} else {
Err(old)
}
}
/// Atomic compare and exchange and returns whether the operation succeeds.
///
/// If `*self` == `old`, atomically updates `*self` to `new`. Otherwise, `*self` is not
/// modified, `*old` is updated to the current value of `*self`.
///
/// "Compare" and "Ordering" part are the same as [`Atomic::cmpxchg()`].
///
/// Returns `true` means the cmpxchg succeeds otherwise returns `false`.
#[inline(always)]
fn try_cmpxchg<Ordering: ordering::Ordering>(&self, old: &mut T, new: T, _: Ordering) -> bool {
let mut tmp = into_repr(*old);
let new = into_repr(new);
// INVARIANT: `self.0` is a valid `T` after `atomic_try_cmpxchg*()` because `new` is
// transmutable to `T`.
let ret = {
match Ordering::TYPE {
OrderingType::Full => T::Repr::atomic_try_cmpxchg(&self.0, &mut tmp, new),
OrderingType::Acquire => {
T::Repr::atomic_try_cmpxchg_acquire(&self.0, &mut tmp, new)
}
OrderingType::Release => {
T::Repr::atomic_try_cmpxchg_release(&self.0, &mut tmp, new)
}
OrderingType::Relaxed => {
T::Repr::atomic_try_cmpxchg_relaxed(&self.0, &mut tmp, new)
}
}
};
// SAFETY: `tmp` comes from reading `*self`, which is a valid `T` per type invariants.
*old = unsafe { from_repr(tmp) };
ret
}
}
impl<T: AtomicType> Atomic<T>
where
T::Repr: AtomicArithmeticOps,
{
/// Atomic add.
///
/// Atomically updates `*self` to `(*self).wrapping_add(v)`.
///
/// # Examples
///
/// ```
/// use kernel::sync::atomic::{Atomic, Relaxed};
///
/// let x = Atomic::new(42);
///
/// assert_eq!(42, x.load(Relaxed));
///
/// x.add(12, Relaxed);
///
/// assert_eq!(54, x.load(Relaxed));
/// ```
#[inline(always)]
pub fn add<Rhs>(&self, v: Rhs, _: ordering::Relaxed)
where
T: AtomicAdd<Rhs>,
{
let v = T::rhs_into_delta(v);
// INVARIANT: `self.0` is a valid `T` after `atomic_add()` due to safety requirement of
// `AtomicAdd`.
T::Repr::atomic_add(&self.0, v);
}
/// Atomic fetch and add.
///
/// Atomically updates `*self` to `(*self).wrapping_add(v)`, and returns the value of `*self`
/// before the update.
///
/// # Examples
///
/// ```
/// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed};
///
/// let x = Atomic::new(42);
/// assert_eq!(42, x.load(Relaxed));
/// assert_eq!(42, x.fetch_add(12, Acquire));
/// assert_eq!(54, x.load(Relaxed));
///
/// let x = Atomic::new(42);
/// assert_eq!(42, x.load(Relaxed));
/// assert_eq!(42, x.fetch_add(12, Full));
/// assert_eq!(54, x.load(Relaxed));
/// ```
#[inline(always)]
pub fn fetch_add<Rhs, Ordering: ordering::Ordering>(&self, v: Rhs, _: Ordering) -> T
where
T: AtomicAdd<Rhs>,
{
let v = T::rhs_into_delta(v);
// INVARIANT: `self.0` is a valid `T` after `atomic_fetch_add*()` due to safety requirement
// of `AtomicAdd`.
let ret = {
match Ordering::TYPE {
OrderingType::Full => T::Repr::atomic_fetch_add(&self.0, v),
OrderingType::Acquire => T::Repr::atomic_fetch_add_acquire(&self.0, v),
OrderingType::Release => T::Repr::atomic_fetch_add_release(&self.0, v),
OrderingType::Relaxed => T::Repr::atomic_fetch_add_relaxed(&self.0, v),
}
};
// SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants.
unsafe { from_repr(ret) }
}
/// Atomic fetch and subtract.
///
/// Atomically updates `*self` to `(*self).wrapping_sub(v)`, and returns the value of `*self`
/// before the update.
///
/// # Examples
///
/// ```
/// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed};
///
/// let x = Atomic::new(42);
/// assert_eq!(42, x.load(Relaxed));
/// assert_eq!(42, x.fetch_sub(12, Acquire));
/// assert_eq!(30, x.load(Relaxed));
///
/// let x = Atomic::new(42);
/// assert_eq!(42, x.load(Relaxed));
/// assert_eq!(42, x.fetch_sub(12, Full));
/// assert_eq!(30, x.load(Relaxed));
/// ```
#[inline(always)]
pub fn fetch_sub<Rhs, Ordering: ordering::Ordering>(&self, v: Rhs, _: Ordering) -> T
where
// Types that support addition also support subtraction.
T: AtomicAdd<Rhs>,
{
let v = T::rhs_into_delta(v);
// INVARIANT: `self.0` is a valid `T` after `atomic_fetch_sub*()` due to safety requirement
// of `AtomicAdd`.
let ret = {
match Ordering::TYPE {
OrderingType::Full => T::Repr::atomic_fetch_sub(&self.0, v),
OrderingType::Acquire => T::Repr::atomic_fetch_sub_acquire(&self.0, v),
OrderingType::Release => T::Repr::atomic_fetch_sub_release(&self.0, v),
OrderingType::Relaxed => T::Repr::atomic_fetch_sub_relaxed(&self.0, v),
}
};
// SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants.
unsafe { from_repr(ret) }
}
}
#[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))]
#[repr(C)]
#[derive(Clone, Copy)]
struct Flag {
bool_field: bool,
}
/// # Invariants
///
/// `padding` must be all zeroes.
#[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))]
#[repr(C, align(4))]
#[derive(Clone, Copy)]
struct Flag {
#[cfg(target_endian = "big")]
padding: [u8; 3],
bool_field: bool,
#[cfg(target_endian = "little")]
padding: [u8; 3],
}
impl Flag {
#[inline(always)]
const fn new(b: bool) -> Self {
// INVARIANT: `padding` is all zeroes.
Self {
bool_field: b,
#[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))]
padding: [0; 3],
}
}
}
// SAFETY: `Flag` and `Repr` have the same size and alignment, and `Flag` is round-trip
// transmutable to the selected representation (`i8` or `i32`).
unsafe impl AtomicType for Flag {
#[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))]
type Repr = i8;
#[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))]
type Repr = i32;
}
/// An atomic flag type intended to be backed by performance-optimal integer type.
///
/// The backing integer type is an implementation detail; it may vary by architecture and change
/// in the future.
///
/// [`AtomicFlag`] is generally preferable to [`Atomic<bool>`] when you need read-modify-write
/// (RMW) operations (e.g. [`Atomic::xchg()`]/[`Atomic::cmpxchg()`]) or when [`Atomic<bool>`] does
/// not save memory due to padding. On some architectures that do not support byte-sized atomic
/// RMW operations, RMW operations on [`Atomic<bool>`] are slower.
///
/// If you only use [`Atomic::load()`]/[`Atomic::store()`], [`Atomic<bool>`] is fine.
///
/// # Examples
///
/// ```
/// use kernel::sync::atomic::{AtomicFlag, Relaxed};
///
/// let flag = AtomicFlag::new(false);
/// assert_eq!(false, flag.load(Relaxed));
/// flag.store(true, Relaxed);
/// assert_eq!(true, flag.load(Relaxed));
/// ```
pub struct AtomicFlag(Atomic<Flag>);
impl AtomicFlag {
/// Creates a new atomic flag.
#[inline(always)]
pub const fn new(b: bool) -> Self {
Self(Atomic::new(Flag::new(b)))
}
/// Returns a mutable reference to the underlying flag as a [`bool`].
///
/// This is safe because the mutable reference of the atomic flag guarantees exclusive access.
///
/// # Examples
///
/// ```
/// use kernel::sync::atomic::{AtomicFlag, Relaxed};
///
/// let mut atomic_flag = AtomicFlag::new(false);
/// assert_eq!(false, atomic_flag.load(Relaxed));
/// *atomic_flag.get_mut() = true;
/// assert_eq!(true, atomic_flag.load(Relaxed));
/// ```
#[inline(always)]
pub fn get_mut(&mut self) -> &mut bool {
&mut self.0.get_mut().bool_field
}
/// Loads the value from the atomic flag.
#[inline(always)]
pub fn load<Ordering: ordering::AcquireOrRelaxed>(&self, o: Ordering) -> bool {
self.0.load(o).bool_field
}
/// Stores a value to the atomic flag.
#[inline(always)]
pub fn store<Ordering: ordering::ReleaseOrRelaxed>(&self, v: bool, o: Ordering) {
self.0.store(Flag::new(v), o);
}
/// Stores a value to the atomic flag and returns the previous value.
#[inline(always)]
pub fn xchg<Ordering: ordering::Ordering>(&self, new: bool, o: Ordering) -> bool {
self.0.xchg(Flag::new(new), o).bool_field
}
/// Store a value to the atomic flag if the current value is equal to `old`.
#[inline(always)]
pub fn cmpxchg<Ordering: ordering::Ordering>(
&self,
old: bool,
new: bool,
o: Ordering,
) -> Result<bool, bool> {
match self.0.cmpxchg(Flag::new(old), Flag::new(new), o) {
Ok(_) => Ok(old),
Err(f) => Err(f.bool_field),
}
}
}
/// Atomic load over raw pointers.
///
/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work
/// with C side on synchronizations:
///
/// - `atomic_load(.., Relaxed)` maps to `READ_ONCE()` when used for inter-thread communication.
/// - `atomic_load(.., Acquire)` maps to `smp_load_acquire()`.
///
/// # Safety
///
/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
/// - If there is a concurrent store from kernel (C or Rust), it has to be atomic.
#[doc(alias("READ_ONCE", "smp_load_acquire"))]
#[inline(always)]
pub unsafe fn atomic_load<T: AtomicType, Ordering: ordering::AcquireOrRelaxed>(
ptr: *mut T,
o: Ordering,
) -> T
where
T::Repr: AtomicBasicOps,
{
// SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
// `align_of::<T>()`, and all concurrent stores from kernel are atomic, hence no data race per
// LKMM.
unsafe { Atomic::from_ptr(ptr) }.load(o)
}
/// Atomic store over raw pointers.
///
/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work
/// with C side on synchronizations:
///
/// - `atomic_store(.., Relaxed)` maps to `WRITE_ONCE()` when used for inter-thread communication.
/// - `atomic_load(.., Release)` maps to `smp_store_release()`.
///
/// # Safety
///
/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.
#[doc(alias("WRITE_ONCE", "smp_store_release"))]
#[inline(always)]
pub unsafe fn atomic_store<T: AtomicType, Ordering: ordering::ReleaseOrRelaxed>(
ptr: *mut T,
v: T,
o: Ordering,
) where
T::Repr: AtomicBasicOps,
{
// SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
// `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race
// per LKMM.
unsafe { Atomic::from_ptr(ptr) }.store(v, o);
}
/// Atomic exchange over raw pointers.
///
/// This function provides a short-cut of `Atomic::from_ptr().xchg(..)`, and can be used to work
/// with C side on synchronizations.
///
/// # Safety
///
/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.
#[inline(always)]
pub unsafe fn xchg<T: AtomicType, Ordering: ordering::Ordering>(
ptr: *mut T,
new: T,
o: Ordering,
) -> T
where
T::Repr: AtomicExchangeOps,
{
// SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
// `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race
// per LKMM.
unsafe { Atomic::from_ptr(ptr) }.xchg(new, o)
}
/// Atomic compare and exchange over raw pointers.
///
/// This function provides a short-cut of `Atomic::from_ptr().cmpxchg(..)`, and can be used to work
/// with C side on synchronizations.
///
/// # Safety
///
/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.
#[doc(alias("try_cmpxchg"))]
#[inline(always)]
pub unsafe fn cmpxchg<T: AtomicType, Ordering: ordering::Ordering>(
ptr: *mut T,
old: T,
new: T,
o: Ordering,
) -> Result<T, T>
where
T::Repr: AtomicExchangeOps,
{
// SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
// `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race
// per LKMM.
unsafe { Atomic::from_ptr(ptr) }.cmpxchg(old, new, o)
}
linux-7.2.3/rust/kernel/sync/atomic/internal.rs
+352
View File
@@ -0,0 +1,352 @@
// SPDX-License-Identifier: GPL-2.0
//! Atomic internal implementations.
//!
//! Provides 1:1 mapping to the C atomic operations.
use crate::{
bindings,
build_assert::static_assert,
macros::paste, //
};
use core::cell::UnsafeCell;
use ffi::c_void;
mod private {
/// Sealed trait marker to disable customized impls on atomic implementation traits.
pub trait Sealed {}
}
// The C side supports atomic primitives only for `i32` and `i64` (`atomic_t` and `atomic64_t`),
// while the Rust side also provides atomic support for `i8`, `i16` and `*const c_void` on top of
// lower-level C primitives.
impl private::Sealed for i8 {}
impl private::Sealed for i16 {}
impl private::Sealed for *const c_void {}
impl private::Sealed for i32 {}
impl private::Sealed for i64 {}
/// A marker trait for types that implement atomic operations with C side primitives.
///
/// This trait is sealed, and only types that map directly to the C side atomics
/// or can be implemented with lower-level C primitives are allowed to implement this:
///
/// - `i8`, `i16` and `*const c_void` are implemented with lower-level C primitives.
/// - `i32` map to `atomic_t`
/// - `i64` map to `atomic64_t`
pub trait AtomicImpl: Sized + Copy + private::Sealed {
/// The type of the delta in arithmetic or logical operations.
///
/// For example, in `atomic_add(ptr, v)`, it's the type of `v`. Usually it's the same type of
/// [`Self`], but it may be different for the atomic pointer type.
type Delta;
}
// The current helpers of load/store of atomic `i8`, `i16` and pointers use `{WRITE,READ}_ONCE()`
// hence the atomicity is only guaranteed against read-modify-write operations if the architecture
// supports native atomic RmW.
//
// In the future when a CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=n architecture plans to support Rust, the
// load/store helpers that guarantee atomicity against RmW operations (usually via a lock) need to
// be added.
static_assert!(
cfg!(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW),
"The current implementation of atomic i8/i16/ptr relies on the architecure being \
ARCH_SUPPORTS_ATOMIC_RMW"
);
impl AtomicImpl for i8 {
type Delta = Self;
}
impl AtomicImpl for i16 {
type Delta = Self;
}
impl AtomicImpl for *const c_void {
type Delta = isize;
}
// `atomic_t` implements atomic operations on `i32`.
impl AtomicImpl for i32 {
type Delta = Self;
}
// `atomic64_t` implements atomic operations on `i64`.
impl AtomicImpl for i64 {
type Delta = Self;
}
/// Atomic representation.
#[repr(transparent)]
pub struct AtomicRepr<T: AtomicImpl>(UnsafeCell<T>);
impl<T: AtomicImpl> AtomicRepr<T> {
/// Creates a new atomic representation `T`.
pub const fn new(v: T) -> Self {
Self(UnsafeCell::new(v))
}
/// Returns a pointer to the underlying `T`.
///
/// # Guarantees
///
/// The returned pointer is valid and properly aligned (i.e. aligned to [`align_of::<T>()`]).
pub const fn as_ptr(&self) -> *mut T {
// GUARANTEE: `self.0` is an `UnsafeCell<T>`, therefore the pointer returned by `.get()`
// must be valid and properly aligned.
self.0.get()
}
}
// This macro generates the function signature with given argument list and return type.
macro_rules! declare_atomic_method {
(
$(#[doc=$doc:expr])*
$func:ident($($arg:ident : $arg_type:ty),*) $(-> $ret:ty)?
) => {
paste!(
$(#[doc = $doc])*
fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)?;
);
};
(
$(#[doc=$doc:expr])*
$func:ident [$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)?
) => {
paste!(
declare_atomic_method!(
$(#[doc = $doc])*
[< $func _ $variant >]($($arg_sig)*) $(-> $ret)?
);
);
declare_atomic_method!(
$(#[doc = $doc])*
$func [$($rest)*]($($arg_sig)*) $(-> $ret)?
);
};
(
$(#[doc=$doc:expr])*
$func:ident []($($arg_sig:tt)*) $(-> $ret:ty)?
) => {
declare_atomic_method!(
$(#[doc = $doc])*
$func($($arg_sig)*) $(-> $ret)?
);
}
}
// This macro generates the function implementation with given argument list and return type, and it
// will replace "call(...)" expression with "$ctype _ $func" to call the real C function.
macro_rules! impl_atomic_method {
(
($ctype:ident) $func:ident($($arg:ident: $arg_type:ty),*) $(-> $ret:ty)? {
$unsafe:tt { call($($c_arg:expr),*) }
}
) => {
paste!(
#[inline(always)]
fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)? {
// TODO: Ideally we want to use the SAFETY comments written at the macro invocation
// (e.g. in `declare_and_impl_atomic_methods!()`, however, since SAFETY comments
// are just comments, and they are not passed to macros as tokens, therefore we
// cannot use them here. One potential improvement is that if we support using
// attributes as an alternative for SAFETY comments, then we can use that for macro
// generating code.
//
// SAFETY: specified on macro invocation.
$unsafe { bindings::[< $ctype _ $func >]($($c_arg,)*) }
}
);
};
(
($ctype:ident) $func:ident[$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)? {
$unsafe:tt { call($($arg:tt)*) }
}
) => {
paste!(
impl_atomic_method!(
($ctype) [< $func _ $variant >]($($arg_sig)*) $( -> $ret)? {
$unsafe { call($($arg)*) }
}
);
);
impl_atomic_method!(
($ctype) $func [$($rest)*]($($arg_sig)*) $( -> $ret)? {
$unsafe { call($($arg)*) }
}
);
};
(
($ctype:ident) $func:ident[]($($arg_sig:tt)*) $( -> $ret:ty)? {
$unsafe:tt { call($($arg:tt)*) }
}
) => {
impl_atomic_method!(
($ctype) $func($($arg_sig)*) $(-> $ret)? {
$unsafe { call($($arg)*) }
}
);
}
}
macro_rules! declare_atomic_ops_trait {
(
$(#[$attr:meta])* $pub:vis trait $ops:ident {
$(
$(#[doc=$doc:expr])*
fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? {
$unsafe:tt { bindings::#call($($arg:tt)*) }
}
)*
}
) => {
$(#[$attr])*
$pub trait $ops: AtomicImpl {
$(
declare_atomic_method!(
$(#[doc=$doc])*
$func[$($variant)*]($($arg_sig)*) $(-> $ret)?
);
)*
}
}
}
macro_rules! impl_atomic_ops_for_one {
(
$ty:ty => $ctype:ident,
$(#[$attr:meta])* $pub:vis trait $ops:ident {
$(
$(#[doc=$doc:expr])*
fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? {
$unsafe:tt { bindings::#call($($arg:tt)*) }
}
)*
}
) => {
impl $ops for $ty {
$(
impl_atomic_method!(
($ctype) $func[$($variant)*]($($arg_sig)*) $(-> $ret)? {
$unsafe { call($($arg)*) }
}
);
)*
}
}
}
// Declares $ops trait with methods and implements the trait.
macro_rules! declare_and_impl_atomic_methods {
(
[ $($map:tt)* ]
$(#[$attr:meta])* $pub:vis trait $ops:ident { $($body:tt)* }
) => {
declare_and_impl_atomic_methods!(
@with_ops_def
[ $($map)* ]
( $(#[$attr])* $pub trait $ops { $($body)* } )
);
};
(@with_ops_def [ $($map:tt)* ] ( $($ops_def:tt)* )) => {
declare_atomic_ops_trait!( $($ops_def)* );
declare_and_impl_atomic_methods!(
@munch
[ $($map)* ]
( $($ops_def)* )
);
};
(@munch [] ( $($ops_def:tt)* )) => {};
(@munch [ $ty:ty => $ctype:ident $(, $($rest:tt)*)? ] ( $($ops_def:tt)* )) => {
impl_atomic_ops_for_one!(
$ty => $ctype,
$($ops_def)*
);
declare_and_impl_atomic_methods!(
@munch
[ $($($rest)*)? ]
( $($ops_def)* )
);
};
}
declare_and_impl_atomic_methods!(
[ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ]
/// Basic atomic operations
pub trait AtomicBasicOps {
/// Atomic read (load).
fn read[acquire](a: &AtomicRepr<Self>) -> Self {
// SAFETY: `a.as_ptr()` is valid and properly aligned.
unsafe { bindings::#call(a.as_ptr().cast()) }
}
/// Atomic set (store).
fn set[release](a: &AtomicRepr<Self>, v: Self) {
// SAFETY: `a.as_ptr()` is valid and properly aligned.
unsafe { bindings::#call(a.as_ptr().cast(), v) }
}
}
);
declare_and_impl_atomic_methods!(
[ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ]
/// Exchange and compare-and-exchange atomic operations
pub trait AtomicExchangeOps {
/// Atomic exchange.
///
/// Atomically updates `*a` to `v` and returns the old value.
fn xchg[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self) -> Self {
// SAFETY: `a.as_ptr()` is valid and properly aligned.
unsafe { bindings::#call(a.as_ptr().cast(), v) }
}
/// Atomic compare and exchange.
///
/// If `*a` == `*old`, atomically updates `*a` to `new`. Otherwise, `*a` is not
/// modified, `*old` is updated to the current value of `*a`.
///
/// Return `true` if the update of `*a` occurred, `false` otherwise.
fn try_cmpxchg[acquire, release, relaxed](
a: &AtomicRepr<Self>, old: &mut Self, new: Self
) -> bool {
// SAFETY: `a.as_ptr()` is valid and properly aligned. `core::ptr::from_mut(old)`
// is valid and properly aligned.
unsafe { bindings::#call(a.as_ptr().cast(), core::ptr::from_mut(old), new) }
}
}
);
declare_and_impl_atomic_methods!(
[ i32 => atomic, i64 => atomic64 ]
/// Atomic arithmetic operations
pub trait AtomicArithmeticOps {
/// Atomic add (wrapping).
///
/// Atomically updates `*a` to `(*a).wrapping_add(v)`.
fn add[](a: &AtomicRepr<Self>, v: Self::Delta) {
// SAFETY: `a.as_ptr()` is valid and properly aligned.
unsafe { bindings::#call(v, a.as_ptr().cast()) }
}
/// Atomic fetch and add (wrapping).
///
/// Atomically updates `*a` to `(*a).wrapping_add(v)`, and returns the value of `*a`
/// before the update.
fn fetch_add[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self {
// SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned.
unsafe { bindings::#call(v, a.as_ptr().cast()) }
}
fn fetch_sub[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self {
// SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned.
unsafe { bindings::#call(v, a.as_ptr().cast()) }
}
}
);
linux-7.2.3/rust/kernel/sync/atomic/ordering.rs
+104
View File
@@ -0,0 +1,104 @@
// SPDX-License-Identifier: GPL-2.0
//! Memory orderings.
//!
//! The semantics of these orderings follows the [`LKMM`] definitions and rules.
//!
//! - [`Acquire`] provides ordering between the load part of the annotated operation and all the
//! following memory accesses, and if there is a store part, the store part has the [`Relaxed`]
//! ordering.
//! - [`Release`] provides ordering between all the preceding memory accesses and the store part of
//! the annotated operation, and if there is a load part, the load part has the [`Relaxed`]
//! ordering.
//! - [`Full`] means "fully-ordered", that is:
//! - It provides ordering between all the preceding memory accesses and the annotated operation.
//! - It provides ordering between the annotated operation and all the following memory accesses.
//! - It provides ordering between all the preceding memory accesses and all the following memory
//! accesses.
//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb()`).
//! - [`Relaxed`] provides no ordering except the dependency orderings. Dependency orderings are
//! described in "DEPENDENCY RELATIONS" in [`LKMM`]'s [`explanation`].
//!
//! [`LKMM`]: srctree/tools/memory-model/
//! [`explanation`]: srctree/tools/memory-model/Documentation/explanation.txt
/// The annotation type for relaxed memory ordering, for the description of relaxed memory
/// ordering, see [module-level documentation].
///
/// [module-level documentation]: crate::sync::atomic::ordering
pub struct Relaxed;
/// The annotation type for acquire memory ordering, for the description of acquire memory
/// ordering, see [module-level documentation].
///
/// [module-level documentation]: crate::sync::atomic::ordering
pub struct Acquire;
/// The annotation type for release memory ordering, for the description of release memory
/// ordering, see [module-level documentation].
///
/// [module-level documentation]: crate::sync::atomic::ordering
pub struct Release;
/// The annotation type for fully-ordered memory ordering, for the description fully-ordered memory
/// ordering, see [module-level documentation].
///
/// [module-level documentation]: crate::sync::atomic::ordering
pub struct Full;
/// Describes the exact memory ordering.
#[doc(hidden)]
pub enum OrderingType {
/// Relaxed ordering.
Relaxed,
/// Acquire ordering.
Acquire,
/// Release ordering.
Release,
/// Fully-ordered.
Full,
}
mod internal {
/// Sealed trait, can be only implemented inside atomic mod.
pub trait Sealed {}
impl Sealed for super::Relaxed {}
impl Sealed for super::Acquire {}
impl Sealed for super::Release {}
impl Sealed for super::Full {}
}
/// The trait bound for annotating operations that support any ordering.
pub trait Ordering: internal::Sealed {
/// Describes the exact memory ordering.
const TYPE: OrderingType;
}
impl Ordering for Relaxed {
const TYPE: OrderingType = OrderingType::Relaxed;
}
impl Ordering for Acquire {
const TYPE: OrderingType = OrderingType::Acquire;
}
impl Ordering for Release {
const TYPE: OrderingType = OrderingType::Release;
}
impl Ordering for Full {
const TYPE: OrderingType = OrderingType::Full;
}
/// The trait bound for operations that only support acquire or relaxed ordering.
pub trait AcquireOrRelaxed: Ordering {}
impl AcquireOrRelaxed for Acquire {}
impl AcquireOrRelaxed for Relaxed {}
/// The trait bound for operations that only support release or relaxed ordering.
pub trait ReleaseOrRelaxed: Ordering {}
impl ReleaseOrRelaxed for Release {}
impl ReleaseOrRelaxed for Relaxed {}
linux-7.2.3/rust/kernel/sync/atomic/predefine.rs
+335
View File
@@ -0,0 +1,335 @@
// SPDX-License-Identifier: GPL-2.0
//! Pre-defined atomic types
use crate::prelude::*;
// Ensure size and alignment requirements are checked.
static_assert!(size_of::<bool>() == size_of::<i8>());
static_assert!(align_of::<bool>() == align_of::<i8>());
// SAFETY: `bool` has the same size and alignment as `i8`, and Rust guarantees that `bool` has
// only two valid bit patterns: 0 (false) and 1 (true). Those are valid `i8` values, so `bool` is
// round-trip transmutable to `i8`.
unsafe impl super::AtomicType for bool {
type Repr = i8;
}
// SAFETY: `i8` has the same size and alignment with itself, and is round-trip transmutable to
// itself.
unsafe impl super::AtomicType for i8 {
type Repr = i8;
}
// SAFETY: `i16` has the same size and alignment with itself, and is round-trip transmutable to
// itself.
unsafe impl super::AtomicType for i16 {
type Repr = i16;
}
// SAFETY:
//
// - `*mut T` has the same size and alignment with `*const c_void`, and is round-trip
// transmutable to `*const c_void`.
// - `*mut T` is safe to transfer between execution contexts. See the safety requirement of
// [`AtomicType`].
unsafe impl<T: Sized> super::AtomicType for *mut T {
type Repr = *const c_void;
}
// SAFETY:
//
// - `*const T` has the same size and alignment with `*const c_void`, and is round-trip
// transmutable to `*const c_void`.
// - `*const T` is safe to transfer between execution contexts. See the safety requirement of
// [`AtomicType`].
unsafe impl<T: Sized> super::AtomicType for *const T {
type Repr = *const c_void;
}
// SAFETY: `i32` has the same size and alignment with itself, and is round-trip transmutable to
// itself.
unsafe impl super::AtomicType for i32 {
type Repr = i32;
}
// SAFETY: The wrapping add result of two `i32`s is a valid `i32`.
unsafe impl super::AtomicAdd<i32> for i32 {
fn rhs_into_delta(rhs: i32) -> i32 {
rhs
}
}
// SAFETY: `i64` has the same size and alignment with itself, and is round-trip transmutable to
// itself.
unsafe impl super::AtomicType for i64 {
type Repr = i64;
}
// SAFETY: The wrapping add result of two `i64`s is a valid `i64`.
unsafe impl super::AtomicAdd<i64> for i64 {
fn rhs_into_delta(rhs: i64) -> i64 {
rhs
}
}
// Defines an internal type that always maps to the integer type which has the same size alignment
// as `isize` and `usize`, and `isize` and `usize` are always bi-directional transmutable to
// `isize_atomic_repr`, which also always implements `AtomicImpl`.
#[allow(non_camel_case_types)]
#[cfg(not(testlib))]
#[cfg(not(CONFIG_64BIT))]
type isize_atomic_repr = i32;
#[allow(non_camel_case_types)]
#[cfg(not(testlib))]
#[cfg(CONFIG_64BIT)]
type isize_atomic_repr = i64;
#[allow(non_camel_case_types)]
#[cfg(testlib)]
#[cfg(target_pointer_width = "32")]
type isize_atomic_repr = i32;
#[allow(non_camel_case_types)]
#[cfg(testlib)]
#[cfg(target_pointer_width = "64")]
type isize_atomic_repr = i64;
// Ensure size and alignment requirements are checked.
static_assert!(size_of::<isize>() == size_of::<isize_atomic_repr>());
static_assert!(align_of::<isize>() == align_of::<isize_atomic_repr>());
static_assert!(size_of::<usize>() == size_of::<isize_atomic_repr>());
static_assert!(align_of::<usize>() == align_of::<isize_atomic_repr>());
// SAFETY: `isize` has the same size and alignment with `isize_atomic_repr`, and is round-trip
// transmutable to `isize_atomic_repr`.
unsafe impl super::AtomicType for isize {
type Repr = isize_atomic_repr;
}
// SAFETY: The wrapping add result of two `isize_atomic_repr`s is a valid `usize`.
unsafe impl super::AtomicAdd<isize> for isize {
fn rhs_into_delta(rhs: isize) -> isize_atomic_repr {
rhs as isize_atomic_repr
}
}
// SAFETY: `u32` and `i32` has the same size and alignment, and `u32` is round-trip transmutable to
// `i32`.
unsafe impl super::AtomicType for u32 {
type Repr = i32;
}
// SAFETY: The wrapping add result of two `i32`s is a valid `u32`.
unsafe impl super::AtomicAdd<u32> for u32 {
fn rhs_into_delta(rhs: u32) -> i32 {
rhs as i32
}
}
// SAFETY: `u64` and `i64` has the same size and alignment, and `u64` is round-trip transmutable to
// `i64`.
unsafe impl super::AtomicType for u64 {
type Repr = i64;
}
// SAFETY: The wrapping add result of two `i64`s is a valid `u64`.
unsafe impl super::AtomicAdd<u64> for u64 {
fn rhs_into_delta(rhs: u64) -> i64 {
rhs as i64
}
}
// SAFETY: `usize` has the same size and alignment with `isize_atomic_repr`, and is round-trip
// transmutable to `isize_atomic_repr`.
unsafe impl super::AtomicType for usize {
type Repr = isize_atomic_repr;
}
// SAFETY: The wrapping add result of two `isize_atomic_repr`s is a valid `usize`.
unsafe impl super::AtomicAdd<usize> for usize {
fn rhs_into_delta(rhs: usize) -> isize_atomic_repr {
rhs as isize_atomic_repr
}
}
#[cfg(CONFIG_RUST_ATOMICS_KUNIT_TEST)]
#[macros::kunit_tests(rust_atomics)]
mod tests {
use super::super::*;
// Call $fn($val) with each $type of $val.
macro_rules! for_each_type {
($val:literal in [$($type:ty),*] $fn:expr) => {
$({
let v: $type = $val;
$fn(v);
})*
}
}
#[test]
fn atomic_basic_tests() {
for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
let x = Atomic::new(v);
assert_eq!(v, x.load(Relaxed));
});
for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
let x = Atomic::new(v);
let ptr = x.as_ptr();
// SAFETY: `ptr` is a valid pointer and no concurrent access.
assert_eq!(v, unsafe { atomic_load(ptr, Relaxed) });
});
}
#[test]
fn atomic_acquire_release_tests() {
for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
let x = Atomic::new(0);
x.store(v, Release);
assert_eq!(v, x.load(Acquire));
});
for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
let x = Atomic::new(0);
let ptr = x.as_ptr();
// SAFETY: `ptr` is a valid pointer and no concurrent access.
unsafe { atomic_store(ptr, v, Release) };
// SAFETY: `ptr` is a valid pointer and no concurrent access.
assert_eq!(v, unsafe { atomic_load(ptr, Acquire) });
});
}
#[test]
fn atomic_xchg_tests() {
for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
let x = Atomic::new(v);
let old = v;
let new = v + 1;
assert_eq!(old, x.xchg(new, Full));
assert_eq!(new, x.load(Relaxed));
});
for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
let x = Atomic::new(v);
let ptr = x.as_ptr();
let old = v;
let new = v + 1;
// SAFETY: `ptr` is a valid pointer and no concurrent access.
assert_eq!(old, unsafe { xchg(ptr, new, Full) });
assert_eq!(new, x.load(Relaxed));
});
}
#[test]
fn atomic_cmpxchg_tests() {
for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
let x = Atomic::new(v);
let old = v;
let new = v + 1;
assert_eq!(Err(old), x.cmpxchg(new, new, Full));
assert_eq!(old, x.load(Relaxed));
assert_eq!(Ok(old), x.cmpxchg(old, new, Relaxed));
assert_eq!(new, x.load(Relaxed));
});
for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
let x = Atomic::new(v);
let ptr = x.as_ptr();
let old = v;
let new = v + 1;
// SAFETY: `ptr` is a valid pointer and no concurrent access.
assert_eq!(Err(old), unsafe { cmpxchg(ptr, new, new, Full) });
assert_eq!(old, x.load(Relaxed));
// SAFETY: `ptr` is a valid pointer and no concurrent access.
assert_eq!(Ok(old), unsafe { cmpxchg(ptr, old, new, Relaxed) });
assert_eq!(new, x.load(Relaxed));
});
}
#[test]
fn atomic_arithmetic_tests() {
for_each_type!(42 in [i32, i64, u32, u64, isize, usize] |v| {
let x = Atomic::new(v);
assert_eq!(v, x.fetch_add(12, Full));
assert_eq!(v + 12, x.load(Relaxed));
x.add(13, Relaxed);
assert_eq!(v + 25, x.load(Relaxed));
});
}
#[test]
fn atomic_bool_tests() {
let x = Atomic::new(false);
assert_eq!(false, x.load(Relaxed));
x.store(true, Relaxed);
assert_eq!(true, x.load(Relaxed));
assert_eq!(true, x.xchg(false, Relaxed));
assert_eq!(false, x.load(Relaxed));
assert_eq!(Err(false), x.cmpxchg(true, true, Relaxed));
assert_eq!(false, x.load(Relaxed));
assert_eq!(Ok(false), x.cmpxchg(false, true, Full));
}
#[test]
fn atomic_ptr_tests() {
let mut v = 42;
let mut u = 43;
let x = Atomic::new(&raw mut v);
assert_eq!(x.load(Acquire), &raw mut v);
assert_eq!(x.cmpxchg(&raw mut u, &raw mut u, Relaxed), Err(&raw mut v));
assert_eq!(x.cmpxchg(&raw mut v, &raw mut u, Relaxed), Ok(&raw mut v));
assert_eq!(x.load(Relaxed), &raw mut u);
let x = Atomic::new(&raw const v);
assert_eq!(x.load(Acquire), &raw const v);
assert_eq!(
x.cmpxchg(&raw const u, &raw const u, Relaxed),
Err(&raw const v)
);
assert_eq!(
x.cmpxchg(&raw const v, &raw const u, Relaxed),
Ok(&raw const v)
);
assert_eq!(x.load(Relaxed), &raw const u);
}
#[test]
fn atomic_flag_tests() {
let mut flag = AtomicFlag::new(false);
assert_eq!(false, flag.load(Relaxed));
*flag.get_mut() = true;
assert_eq!(true, flag.load(Relaxed));
assert_eq!(true, flag.xchg(false, Relaxed));
assert_eq!(false, flag.load(Relaxed));
*flag.get_mut() = true;
assert_eq!(Ok(true), flag.cmpxchg(true, false, Full));
assert_eq!(false, flag.load(Relaxed));
}
}
linux-7.2.3/rust/kernel/sync/barrier.rs
+61
View File
@@ -0,0 +1,61 @@
// SPDX-License-Identifier: GPL-2.0
//! Memory barriers.
//!
//! These primitives have the same semantics as their C counterparts: and the precise definitions
//! of semantics can be found at [`LKMM`].
//!
//! [`LKMM`]: srctree/tools/memory-model/
/// A compiler barrier.
///
/// A barrier that prevents compiler from reordering memory accesses across the barrier.
#[inline(always)]
pub(crate) fn barrier() {
// By default, Rust inline asms are treated as being able to access any memory or flags, hence
// it suffices as a compiler barrier.
//
// SAFETY: An empty asm block.
unsafe { core::arch::asm!("") };
}
/// A full memory barrier.
///
/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
#[inline(always)]
pub fn smp_mb() {
if cfg!(CONFIG_SMP) {
// SAFETY: `smp_mb()` is safe to call.
unsafe { bindings::smp_mb() };
} else {
barrier();
}
}
/// A write-write memory barrier.
///
/// A barrier that prevents compiler and CPU from reordering memory write accesses across the
/// barrier.
#[inline(always)]
pub fn smp_wmb() {
if cfg!(CONFIG_SMP) {
// SAFETY: `smp_wmb()` is safe to call.
unsafe { bindings::smp_wmb() };
} else {
barrier();
}
}
/// A read-read memory barrier.
///
/// A barrier that prevents compiler and CPU from reordering memory read accesses across the
/// barrier.
#[inline(always)]
pub fn smp_rmb() {
if cfg!(CONFIG_SMP) {
// SAFETY: `smp_rmb()` is safe to call.
unsafe { bindings::smp_rmb() };
} else {
barrier();
}
}
linux-7.2.3/rust/kernel/sync/completion.rs
+114
View File
@@ -0,0 +1,114 @@
// SPDX-License-Identifier: GPL-2.0
//! Completion support.
//!
//! Reference: <https://docs.kernel.org/scheduler/completion.html>
//!
//! C header: [`include/linux/completion.h`](srctree/include/linux/completion.h)
use crate::{bindings, prelude::*, types::Opaque};
/// Synchronization primitive to signal when a certain task has been completed.
///
/// The [`Completion`] synchronization primitive signals when a certain task has been completed by
/// waking up other tasks that have been queued up to wait for the [`Completion`] to be completed.
///
/// # Examples
///
/// ```
/// use kernel::sync::{Arc, Completion};
/// use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem};
///
/// #[pin_data]
/// struct MyTask {
/// #[pin]
/// work: Work<MyTask>,
/// #[pin]
/// done: Completion,
/// }
///
/// impl_has_work! {
/// impl HasWork<Self> for MyTask { self.work }
/// }
///
/// impl MyTask {
/// fn new() -> Result<Arc<Self>> {
/// let this = Arc::pin_init(pin_init!(MyTask {
/// work <- new_work!("MyTask::work"),
/// done <- Completion::new(),
/// }), GFP_KERNEL)?;
///
/// let _ = workqueue::system().enqueue(this.clone());
///
/// Ok(this)
/// }
///
/// fn wait_for_completion(&self) {
/// self.done.wait_for_completion();
///
/// pr_info!("Completion: task complete\n");
/// }
/// }
///
/// impl WorkItem for MyTask {
/// type Pointer = Arc<MyTask>;
///
/// fn run(this: Arc<MyTask>) {
/// // process this task
/// this.done.complete_all();
/// }
/// }
///
/// let task = MyTask::new()?;
/// task.wait_for_completion();
/// # Ok::<(), Error>(())
/// ```
#[pin_data]
pub struct Completion {
#[pin]
inner: Opaque<bindings::completion>,
}
// SAFETY: `Completion` is safe to be send to any task.
unsafe impl Send for Completion {}
// SAFETY: `Completion` is safe to be accessed concurrently.
unsafe impl Sync for Completion {}
impl Completion {
/// Create an initializer for a new [`Completion`].
pub fn new() -> impl PinInit<Self> {
pin_init!(Self {
inner <- Opaque::ffi_init(|slot: *mut bindings::completion| {
// SAFETY: `slot` is a valid pointer to an uninitialized `struct completion`.
unsafe { bindings::init_completion(slot) };
}),
})
}
fn as_raw(&self) -> *mut bindings::completion {
self.inner.get()
}
/// Signal all tasks waiting on this completion.
///
/// This method wakes up all tasks waiting on this completion; after this operation the
/// completion is permanently done, i.e. signals all current and future waiters.
#[inline]
pub fn complete_all(&self) {
// SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`.
unsafe { bindings::complete_all(self.as_raw()) };
}
/// Wait for completion of a task.
///
/// This method waits for the completion of a task; it is not interruptible and there is no
/// timeout.
///
/// See also [`Completion::complete_all`].
#[inline]
pub fn wait_for_completion(&self) {
// SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`.
unsafe { bindings::wait_for_completion(self.as_raw()) };
}
}
linux-7.2.3/rust/kernel/sync/condvar.rs
+258
View File
@@ -0,0 +1,258 @@
// SPDX-License-Identifier: GPL-2.0
//! A condition variable.
//!
//! This module allows Rust code to use the kernel's [`struct wait_queue_head`] as a condition
//! variable.
use super::{lock::Backend, lock::Guard, LockClassKey};
use crate::{
ffi::{c_int, c_long},
str::{CStr, CStrExt as _},
task::{
MAX_SCHEDULE_TIMEOUT, TASK_FREEZABLE, TASK_INTERRUPTIBLE, TASK_NORMAL, TASK_UNINTERRUPTIBLE,
},
time::Jiffies,
types::Opaque,
};
use core::{marker::PhantomPinned, pin::Pin, ptr};
use pin_init::{pin_data, pin_init, PinInit};
/// Creates a [`CondVar`] initialiser with the given name and a newly-created lock class.
#[macro_export]
macro_rules! new_condvar {
($($name:literal)?) => {
$crate::sync::CondVar::new($crate::optional_name!($($name)?), $crate::static_lock_class!())
};
}
pub use new_condvar;
/// A conditional variable.
///
/// Exposes the kernel's [`struct wait_queue_head`] as a condition variable. It allows the caller to
/// atomically release the given lock and go to sleep. It reacquires the lock when it wakes up. And
/// it wakes up when notified by another thread (via [`CondVar::notify_one`] or
/// [`CondVar::notify_all`]) or because the thread received a signal. It may also wake up
/// spuriously.
///
/// Instances of [`CondVar`] need a lock class and to be pinned. The recommended way to create such
/// instances is with the [`pin_init`](pin_init::pin_init!) and [`new_condvar`] macros.
///
/// # Examples
///
/// The following is an example of using a condvar with a mutex:
///
/// ```
/// use kernel::sync::{new_condvar, new_mutex, CondVar, Mutex};
///
/// #[pin_data]
/// pub struct Example {
/// #[pin]
/// value: Mutex<u32>,
///
/// #[pin]
/// value_changed: CondVar,
/// }
///
/// /// Waits for `e.value` to become `v`.
/// fn wait_for_value(e: &Example, v: u32) {
/// let mut guard = e.value.lock();
/// while *guard != v {
/// e.value_changed.wait(&mut guard);
/// }
/// }
///
/// /// Increments `e.value` and notifies all potential waiters.
/// fn increment(e: &Example) {
/// *e.value.lock() += 1;
/// e.value_changed.notify_all();
/// }
///
/// /// Allocates a new boxed `Example`.
/// fn new_example() -> Result<Pin<KBox<Example>>> {
/// KBox::pin_init(pin_init!(Example {
/// value <- new_mutex!(0),
/// value_changed <- new_condvar!(),
/// }), GFP_KERNEL)
/// }
/// ```
///
/// [`struct wait_queue_head`]: srctree/include/linux/wait.h
#[pin_data]
pub struct CondVar {
#[pin]
pub(crate) wait_queue_head: Opaque<bindings::wait_queue_head>,
/// A condvar needs to be pinned because it contains a [`struct list_head`] that is
/// self-referential, so it cannot be safely moved once it is initialised.
///
/// [`struct list_head`]: srctree/include/linux/types.h
#[pin]
_pin: PhantomPinned,
}
// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on any thread.
unsafe impl Send for CondVar {}
// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on multiple threads
// concurrently.
unsafe impl Sync for CondVar {}
impl CondVar {
/// Constructs a new condvar initialiser.
pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> {
pin_init!(Self {
_pin: PhantomPinned,
// SAFETY: `slot` is valid while the closure is called and both `name` and `key` have
// static lifetimes so they live indefinitely.
wait_queue_head <- Opaque::ffi_init(|slot| unsafe {
bindings::__init_waitqueue_head(slot, name.as_char_ptr(), key.as_ptr())
}),
})
}
fn wait_internal<T: ?Sized, B: Backend>(
&self,
wait_state: c_int,
guard: &mut Guard<'_, T, B>,
timeout_in_jiffies: c_long,
) -> c_long {
let wait = Opaque::<bindings::wait_queue_entry>::uninit();
// SAFETY: `wait` points to valid memory.
unsafe { bindings::init_wait(wait.get()) };
// SAFETY: Both `wait` and `wait_queue_head` point to valid memory.
unsafe {
bindings::prepare_to_wait_exclusive(self.wait_queue_head.get(), wait.get(), wait_state)
};
// SAFETY: Switches to another thread. The timeout can be any number.
let ret = guard.do_unlocked(|| unsafe { bindings::schedule_timeout(timeout_in_jiffies) });
// SAFETY: Both `wait` and `wait_queue_head` point to valid memory.
unsafe { bindings::finish_wait(self.wait_queue_head.get(), wait.get()) };
ret
}
/// Releases the lock and waits for a notification in uninterruptible mode.
///
/// Atomically releases the given lock (whose ownership is proven by the guard) and puts the
/// thread to sleep, reacquiring the lock on wake up. It wakes up when notified by
/// [`CondVar::notify_one`] or [`CondVar::notify_all`]. Note that it may also wake up
/// spuriously.
pub fn wait<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) {
self.wait_internal(TASK_UNINTERRUPTIBLE, guard, MAX_SCHEDULE_TIMEOUT);
}
/// Releases the lock and waits for a notification in interruptible mode.
///
/// Similar to [`CondVar::wait`], except that the wait is interruptible. That is, the thread may
/// wake up due to signals. It may also wake up spuriously.
///
/// Returns whether there is a signal pending.
#[must_use = "wait_interruptible returns if a signal is pending, so the caller must check the return value"]
pub fn wait_interruptible<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) -> bool {
self.wait_internal(TASK_INTERRUPTIBLE, guard, MAX_SCHEDULE_TIMEOUT);
crate::current!().signal_pending()
}
/// Releases the lock and waits for a notification in interruptible and freezable mode.
///
/// The process is allowed to be frozen during this sleep. No lock should be held when calling
/// this function, and there is a lockdep assertion for this. Freezing a task that holds a lock
/// can trivially deadlock vs another task that needs that lock to complete before it too can
/// hit freezable.
#[must_use = "wait_interruptible_freezable returns if a signal is pending, so the caller must check the return value"]
pub fn wait_interruptible_freezable<T: ?Sized, B: Backend>(
&self,
guard: &mut Guard<'_, T, B>,
) -> bool {
self.wait_internal(
TASK_INTERRUPTIBLE | TASK_FREEZABLE,
guard,
MAX_SCHEDULE_TIMEOUT,
);
crate::current!().signal_pending()
}
/// Releases the lock and waits for a notification in interruptible mode.
///
/// Atomically releases the given lock (whose ownership is proven by the guard) and puts the
/// thread to sleep. It wakes up when notified by [`CondVar::notify_one`] or
/// [`CondVar::notify_all`], or when a timeout occurs, or when the thread receives a signal.
#[must_use = "wait_interruptible_timeout returns if a signal is pending, so the caller must check the return value"]
pub fn wait_interruptible_timeout<T: ?Sized, B: Backend>(
&self,
guard: &mut Guard<'_, T, B>,
jiffies: Jiffies,
) -> CondVarTimeoutResult {
let jiffies = jiffies.try_into().unwrap_or(MAX_SCHEDULE_TIMEOUT);
let res = self.wait_internal(TASK_INTERRUPTIBLE, guard, jiffies);
match (res as Jiffies, crate::current!().signal_pending()) {
(jiffies, true) => CondVarTimeoutResult::Signal { jiffies },
(0, false) => CondVarTimeoutResult::Timeout,
(jiffies, false) => CondVarTimeoutResult::Woken { jiffies },
}
}
/// Calls the kernel function to notify the appropriate number of threads.
fn notify(&self, count: c_int) {
// SAFETY: `wait_queue_head` points to valid memory.
unsafe {
bindings::__wake_up(
self.wait_queue_head.get(),
TASK_NORMAL,
count,
ptr::null_mut(),
)
};
}
/// Calls the kernel function to notify one thread synchronously.
///
/// This method behaves like `notify_one`, except that it hints to the scheduler that the
/// current thread is about to go to sleep, so it should schedule the target thread on the same
/// CPU.
#[inline]
pub fn notify_sync(&self) {
// SAFETY: `wait_queue_head` points to valid memory.
unsafe { bindings::__wake_up_sync(self.wait_queue_head.get(), TASK_NORMAL) };
}
/// Wakes a single waiter up, if any.
///
/// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost
/// completely (as opposed to automatically waking up the next waiter).
#[inline]
pub fn notify_one(&self) {
self.notify(1);
}
/// Wakes all waiters up, if any.
///
/// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost
/// completely (as opposed to automatically waking up the next waiter).
#[inline]
pub fn notify_all(&self) {
self.notify(0);
}
}
/// The return type of `wait_timeout`.
pub enum CondVarTimeoutResult {
/// The timeout was reached.
Timeout,
/// Somebody woke us up.
Woken {
/// Remaining sleep duration.
jiffies: Jiffies,
},
/// A signal occurred.
Signal {
/// Remaining sleep duration.
jiffies: Jiffies,
},
}
linux-7.2.3/rust/kernel/sync/lock.rs
+324
View File
@@ -0,0 +1,324 @@
// SPDX-License-Identifier: GPL-2.0
//! Generic kernel lock and guard.
//!
//! It contains a generic Rust lock and guard that allow for different backends (e.g., mutexes,
//! spinlocks, raw spinlocks) to be provided with minimal effort.
use super::LockClassKey;
use crate::{
str::{CStr, CStrExt as _},
types::{NotThreadSafe, Opaque, ScopeGuard},
};
use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin};
use pin_init::{pin_data, pin_init, PinInit, Wrapper};
pub mod mutex;
pub mod spinlock;
pub(super) mod global;
pub use global::{GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy};
/// The "backend" of a lock.
///
/// It is the actual implementation of the lock, without the need to repeat patterns used in all
/// locks.
///
/// # Safety
///
/// - Implementers must ensure that only one thread/CPU may access the protected data once the lock
/// is owned, that is, between calls to [`lock`] and [`unlock`].
/// - Implementers must also ensure that [`relock`] uses the same locking method as the original
/// lock operation.
///
/// [`lock`]: Backend::lock
/// [`unlock`]: Backend::unlock
/// [`relock`]: Backend::relock
pub unsafe trait Backend {
/// The state required by the lock.
type State;
/// The state required to be kept between [`lock`] and [`unlock`].
///
/// [`lock`]: Backend::lock
/// [`unlock`]: Backend::unlock
type GuardState;
/// Initialises the lock.
///
/// # Safety
///
/// `ptr` must be valid for write for the duration of the call, while `name` and `key` must
/// remain valid for read indefinitely.
unsafe fn init(
ptr: *mut Self::State,
name: *const crate::ffi::c_char,
key: *mut bindings::lock_class_key,
);
/// Acquires the lock, making the caller its owner.
///
/// # Safety
///
/// Callers must ensure that [`Backend::init`] has been previously called.
#[must_use]
unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState;
/// Tries to acquire the lock.
///
/// # Safety
///
/// Callers must ensure that [`Backend::init`] has been previously called.
unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState>;
/// Releases the lock, giving up its ownership.
///
/// # Safety
///
/// It must only be called by the current owner of the lock.
unsafe fn unlock(ptr: *mut Self::State, guard_state: &Self::GuardState);
/// Reacquires the lock, making the caller its owner.
///
/// # Safety
///
/// Callers must ensure that `guard_state` comes from a previous call to [`Backend::lock`] (or
/// variant) that has been unlocked with [`Backend::unlock`] and will be relocked now.
unsafe fn relock(ptr: *mut Self::State, guard_state: &mut Self::GuardState) {
// SAFETY: The safety requirements ensure that the lock is initialised.
*guard_state = unsafe { Self::lock(ptr) };
}
/// Asserts that the lock is held using lockdep.
///
/// # Safety
///
/// Callers must ensure that [`Backend::init`] has been previously called.
unsafe fn assert_is_held(ptr: *mut Self::State);
}
/// A mutual exclusion primitive.
///
/// Exposes one of the kernel locking primitives. Which one is exposed depends on the lock
/// [`Backend`] specified as the generic parameter `B`.
#[repr(C)]
#[pin_data]
pub struct Lock<T: ?Sized, B: Backend> {
/// The kernel lock object.
#[pin]
state: Opaque<B::State>,
/// Some locks are known to be self-referential (e.g., mutexes), while others are architecture
/// or config defined (e.g., spinlocks). So we conservatively require them to be pinned in case
/// some architecture uses self-references now or in the future.
#[pin]
_pin: PhantomPinned,
/// The data protected by the lock.
#[pin]
pub(crate) data: UnsafeCell<T>,
}
// SAFETY: `Lock` can be transferred across thread boundaries iff the data it protects can.
unsafe impl<T: ?Sized + Send, B: Backend> Send for Lock<T, B> {}
// SAFETY: `Lock` serialises the interior mutability it provides, so it is `Sync` as long as the
// data it protects is `Send`.
unsafe impl<T: ?Sized + Send, B: Backend> Sync for Lock<T, B> {}
impl<T, B: Backend> Lock<T, B> {
/// Constructs a new lock initialiser.
pub fn new(
t: impl PinInit<T>,
name: &'static CStr,
key: Pin<&'static LockClassKey>,
) -> impl PinInit<Self> {
pin_init!(Self {
data <- UnsafeCell::pin_init(t),
_pin: PhantomPinned,
// SAFETY: `slot` is valid while the closure is called and both `name` and `key` have
// static lifetimes so they live indefinitely.
state <- Opaque::ffi_init(|slot| unsafe {
B::init(slot, name.as_char_ptr(), key.as_ptr())
}),
})
}
}
impl<B: Backend> Lock<(), B> {
/// Constructs a [`Lock`] from a raw pointer.
///
/// This can be useful for interacting with a lock which was initialised outside of Rust.
///
/// # Safety
///
/// The caller promises that `ptr` points to a valid initialised instance of [`State`] during
/// the whole lifetime of `'a`.
///
/// [`State`]: Backend::State
#[inline]
pub unsafe fn from_raw<'a>(ptr: *mut B::State) -> &'a Self {
// SAFETY:
// - By the safety contract `ptr` must point to a valid initialised instance of `B::State`
// - Since the lock data type is `()` which is a ZST, `state` is the only non-ZST member of
// the struct
// - Combined with `#[repr(C)]`, this guarantees `Self` has an equivalent data layout to
// `B::State`.
unsafe { &*ptr.cast() }
}
}
impl<T: ?Sized, B: Backend> Lock<T, B> {
/// Acquires the lock and gives the caller access to the data protected by it.
#[inline]
pub fn lock(&self) -> Guard<'_, T, B> {
// SAFETY: The constructor of the type calls `init`, so the existence of the object proves
// that `init` was called.
let state = unsafe { B::lock(self.state.get()) };
// SAFETY: The lock was just acquired.
unsafe { Guard::new(self, state) }
}
/// Tries to acquire the lock.
///
/// Returns a guard that can be used to access the data protected by the lock if successful.
// `Option<T>` is not `#[must_use]` even if `T` is, thus the attribute is needed here.
#[must_use = "if unused, the lock will be immediately unlocked"]
#[inline]
pub fn try_lock(&self) -> Option<Guard<'_, T, B>> {
// SAFETY: The constructor of the type calls `init`, so the existence of the object proves
// that `init` was called.
unsafe { B::try_lock(self.state.get()).map(|state| Guard::new(self, state)) }
}
}
/// A lock guard.
///
/// Allows mutual exclusion primitives that implement the [`Backend`] trait to automatically unlock
/// when a guard goes out of scope. It also provides a safe and convenient way to access the data
/// protected by the lock.
#[must_use = "the lock unlocks immediately when the guard is unused"]
pub struct Guard<'a, T: ?Sized, B: Backend> {
pub(crate) lock: &'a Lock<T, B>,
pub(crate) state: B::GuardState,
_not_send: NotThreadSafe,
}
// SAFETY: `Guard` is sync when the data protected by the lock is also sync.
unsafe impl<T: Sync + ?Sized, B: Backend> Sync for Guard<'_, T, B> {}
impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> {
/// Returns the lock that this guard originates from.
///
/// # Examples
///
/// The following example shows how to use [`Guard::lock_ref()`] to assert the corresponding
/// lock is held.
///
/// ```
/// # use kernel::{new_spinlock, sync::lock::{Backend, Guard, Lock}};
/// # use pin_init::stack_pin_init;
///
/// fn assert_held<T, B: Backend>(guard: &Guard<'_, T, B>, lock: &Lock<T, B>) {
/// // Address-equal means the same lock.
/// assert!(core::ptr::eq(guard.lock_ref(), lock));
/// }
///
/// // Creates a new lock on the stack.
/// stack_pin_init!{
/// let l = new_spinlock!(42)
/// }
///
/// let g = l.lock();
///
/// // `g` originates from `l`.
/// assert_held(&g, &l);
/// ```
pub fn lock_ref(&self) -> &'a Lock<T, B> {
self.lock
}
pub(crate) fn do_unlocked<U>(&mut self, cb: impl FnOnce() -> U) -> U {
// SAFETY: The caller owns the lock, so it is safe to unlock it.
unsafe { B::unlock(self.lock.state.get(), &self.state) };
let _relock = ScopeGuard::new(||
// SAFETY: The lock was just unlocked above and is being relocked now.
unsafe { B::relock(self.lock.state.get(), &mut self.state) });
cb()
}
/// Returns a pinned mutable reference to the protected data.
///
/// The guard implements [`DerefMut`] when `T: Unpin`, so for [`Unpin`]
/// types [`DerefMut`] should be used instead of this function.
///
/// [`DerefMut`]: core::ops::DerefMut
/// [`Unpin`]: core::marker::Unpin
///
/// # Examples
///
/// ```
/// # use kernel::sync::{Mutex, MutexGuard};
/// # use core::{pin::Pin, marker::PhantomPinned};
/// struct Data(PhantomPinned);
///
/// fn example(mutex: &Mutex<Data>) {
/// let mut data: MutexGuard<'_, Data> = mutex.lock();
/// let mut data: Pin<&mut Data> = data.as_mut();
/// }
/// ```
pub fn as_mut(&mut self) -> Pin<&mut T> {
// SAFETY: `self.lock.data` is structurally pinned.
unsafe { Pin::new_unchecked(&mut *self.lock.data.get()) }
}
}
impl<T: ?Sized, B: Backend> core::ops::Deref for Guard<'_, T, B> {
type Target = T;
#[inline]
fn deref(&self) -> &Self::Target {
// SAFETY: The caller owns the lock, so it is safe to deref the protected data.
unsafe { &*self.lock.data.get() }
}
}
impl<T: ?Sized, B: Backend> core::ops::DerefMut for Guard<'_, T, B>
where
T: Unpin,
{
#[inline]
fn deref_mut(&mut self) -> &mut Self::Target {
// SAFETY: The caller owns the lock, so it is safe to deref the protected data.
unsafe { &mut *self.lock.data.get() }
}
}
impl<T: ?Sized, B: Backend> Drop for Guard<'_, T, B> {
#[inline]
fn drop(&mut self) {
// SAFETY: The caller owns the lock, so it is safe to unlock it.
unsafe { B::unlock(self.lock.state.get(), &self.state) };
}
}
impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> {
/// Constructs a new immutable lock guard.
///
/// # Safety
///
/// The caller must ensure that it owns the lock.
#[inline]
pub unsafe fn new(lock: &'a Lock<T, B>, state: B::GuardState) -> Self {
// SAFETY: The caller can only hold the lock if `Backend::init` has already been called.
unsafe { B::assert_is_held(lock.state.get()) };
Self {
lock,
state,
_not_send: NotThreadSafe,
}
}
}
linux-7.2.3/rust/kernel/sync/lock/global.rs
+309
View File
@@ -0,0 +1,309 @@
// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2024 Google LLC.
//! Support for defining statics containing locks.
use crate::{
str::{CStr, CStrExt as _},
sync::lock::{Backend, Guard, Lock},
sync::{LockClassKey, LockedBy},
types::Opaque,
};
use core::{
cell::UnsafeCell,
marker::{PhantomData, PhantomPinned},
pin::Pin,
};
/// Trait implemented for marker types for global locks.
///
/// See [`global_lock!`] for examples.
pub trait GlobalLockBackend {
/// The name for this global lock.
const NAME: &'static CStr;
/// Item type stored in this global lock.
type Item: 'static;
/// The backend used for this global lock.
type Backend: Backend + 'static;
/// The class for this global lock.
fn get_lock_class() -> Pin<&'static LockClassKey>;
}
/// Type used for global locks.
///
/// See [`global_lock!`] for examples.
pub struct GlobalLock<B: GlobalLockBackend> {
inner: Lock<B::Item, B::Backend>,
}
impl<B: GlobalLockBackend> GlobalLock<B> {
/// Creates a global lock.
///
/// # Safety
///
/// * Before any other method on this lock is called, [`Self::init`] must be called.
/// * The type `B` must not be used with any other lock.
pub const unsafe fn new(data: B::Item) -> Self {
Self {
inner: Lock {
state: Opaque::uninit(),
data: UnsafeCell::new(data),
_pin: PhantomPinned,
},
}
}
/// Initializes a global lock.
///
/// # Safety
///
/// Must not be called more than once on a given lock.
pub unsafe fn init(&'static self) {
// SAFETY: The pointer to `state` is valid for the duration of this call, and both `name`
// and `key` are valid indefinitely. The `state` is pinned since we have a `'static`
// reference to `self`.
//
// We have exclusive access to the `state` since the caller of `new` promised to call
// `init` before using any other methods. As `init` can only be called once, all other
// uses of this lock must happen after this call.
unsafe {
B::Backend::init(
self.inner.state.get(),
B::NAME.as_char_ptr(),
B::get_lock_class().as_ptr(),
)
}
}
/// Lock this global lock.
#[inline]
pub fn lock(&'static self) -> GlobalGuard<B> {
GlobalGuard {
inner: self.inner.lock(),
}
}
/// Try to lock this global lock.
#[must_use = "if unused, the lock will be immediately unlocked"]
#[inline]
pub fn try_lock(&'static self) -> Option<GlobalGuard<B>> {
Some(GlobalGuard {
inner: self.inner.try_lock()?,
})
}
}
/// A guard for a [`GlobalLock`].
///
/// See [`global_lock!`] for examples.
#[must_use = "the lock unlocks immediately when the guard is unused"]
pub struct GlobalGuard<B: GlobalLockBackend> {
inner: Guard<'static, B::Item, B::Backend>,
}
impl<B: GlobalLockBackend> core::ops::Deref for GlobalGuard<B> {
type Target = B::Item;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl<B: GlobalLockBackend> core::ops::DerefMut for GlobalGuard<B>
where
B::Item: Unpin,
{
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner
}
}
/// A version of [`LockedBy`] for a [`GlobalLock`].
///
/// See [`global_lock!`] for examples.
pub struct GlobalLockedBy<T: ?Sized, B: GlobalLockBackend> {
_backend: PhantomData<B>,
value: UnsafeCell<T>,
}
// SAFETY: The same thread-safety rules as `LockedBy` apply to `GlobalLockedBy`.
unsafe impl<T, B> Send for GlobalLockedBy<T, B>
where
T: ?Sized,
B: GlobalLockBackend,
LockedBy<T, B::Item>: Send,
{
}
// SAFETY: The same thread-safety rules as `LockedBy` apply to `GlobalLockedBy`.
unsafe impl<T, B> Sync for GlobalLockedBy<T, B>
where
T: ?Sized,
B: GlobalLockBackend,
LockedBy<T, B::Item>: Sync,
{
}
impl<T, B: GlobalLockBackend> GlobalLockedBy<T, B> {
/// Create a new [`GlobalLockedBy`].
///
/// The provided value will be protected by the global lock indicated by `B`.
pub fn new(val: T) -> Self {
Self {
value: UnsafeCell::new(val),
_backend: PhantomData,
}
}
}
impl<T: ?Sized, B: GlobalLockBackend> GlobalLockedBy<T, B> {
/// Access the value immutably.
///
/// The caller must prove shared access to the lock.
pub fn as_ref<'a>(&'a self, _guard: &'a GlobalGuard<B>) -> &'a T {
// SAFETY: The lock is globally unique, so there can only be one guard.
unsafe { &*self.value.get() }
}
/// Access the value mutably.
///
/// The caller must prove shared exclusive to the lock.
pub fn as_mut<'a>(&'a self, _guard: &'a mut GlobalGuard<B>) -> &'a mut T {
// SAFETY: The lock is globally unique, so there can only be one guard.
unsafe { &mut *self.value.get() }
}
/// Access the value mutably directly.
///
/// The caller has exclusive access to this `GlobalLockedBy`, so they do not need to hold the
/// lock.
pub fn get_mut(&mut self) -> &mut T {
self.value.get_mut()
}
}
/// Defines a global lock.
///
/// The global mutex must be initialized before first use. Usually this is done by calling
/// [`GlobalLock::init`] in the module initializer.
///
/// # Examples
///
/// A global counter:
///
/// ```
/// # mod ex {
/// # use kernel::prelude::*;
/// kernel::sync::global_lock! {
/// // SAFETY: Initialized in module initializer before first use.
/// unsafe(uninit) static MY_COUNTER: Mutex<u32> = 0;
/// }
///
/// fn increment_counter() -> u32 {
/// let mut guard = MY_COUNTER.lock();
/// *guard += 1;
/// *guard
/// }
///
/// impl kernel::Module for MyModule {
/// fn init(_module: &'static ThisModule) -> Result<Self> {
/// // SAFETY: Called exactly once.
/// unsafe { MY_COUNTER.init() };
///
/// Ok(MyModule {})
/// }
/// }
/// # struct MyModule {}
/// # }
/// ```
///
/// A global mutex used to protect all instances of a given struct:
///
/// ```
/// # mod ex {
/// # use kernel::prelude::*;
/// use kernel::sync::{GlobalGuard, GlobalLockedBy};
///
/// kernel::sync::global_lock! {
/// // SAFETY: Initialized in module initializer before first use.
/// unsafe(uninit) static MY_MUTEX: Mutex<()> = ();
/// }
///
/// /// All instances of this struct are protected by `MY_MUTEX`.
/// struct MyStruct {
/// my_counter: GlobalLockedBy<u32, MY_MUTEX>,
/// }
///
/// impl MyStruct {
/// /// Increment the counter in this instance.
/// ///
/// /// The caller must hold the `MY_MUTEX` mutex.
/// fn increment(&self, guard: &mut GlobalGuard<MY_MUTEX>) -> u32 {
/// let my_counter = self.my_counter.as_mut(guard);
/// *my_counter += 1;
/// *my_counter
/// }
/// }
///
/// impl kernel::Module for MyModule {
/// fn init(_module: &'static ThisModule) -> Result<Self> {
/// // SAFETY: Called exactly once.
/// unsafe { MY_MUTEX.init() };
///
/// Ok(MyModule {})
/// }
/// }
/// # struct MyModule {}
/// # }
/// ```
#[macro_export]
macro_rules! global_lock {
{
$(#[$meta:meta])* $pub:vis
unsafe(uninit) static $name:ident: $kind:ident<$valuety:ty> = $value:expr;
} => {
#[doc = ::core::concat!(
"Backend type used by [`",
::core::stringify!($name),
"`](static@",
::core::stringify!($name),
")."
)]
#[allow(non_camel_case_types, unreachable_pub)]
$pub enum $name {}
impl $crate::sync::lock::GlobalLockBackend for $name {
const NAME: &'static $crate::str::CStr = $crate::c_str!(::core::stringify!($name));
type Item = $valuety;
type Backend = $crate::global_lock_inner!(backend $kind);
fn get_lock_class() -> Pin<&'static $crate::sync::LockClassKey> {
$crate::static_lock_class!()
}
}
$(#[$meta])*
$pub static $name: $crate::sync::lock::GlobalLock<$name> = {
// Defined here to be outside the unsafe scope.
let init: $valuety = $value;
// SAFETY:
// * The user of this macro promises to initialize the macro before use.
// * We are only generating one static with this backend type.
unsafe { $crate::sync::lock::GlobalLock::new(init) }
};
};
}
pub use global_lock;
#[doc(hidden)]
#[macro_export]
macro_rules! global_lock_inner {
(backend Mutex) => {
$crate::sync::lock::mutex::MutexBackend
};
(backend SpinLock) => {
$crate::sync::lock::spinlock::SpinLockBackend
};
}
linux-7.2.3/rust/kernel/sync/lock/mutex.rs
+147
View File
@@ -0,0 +1,147 @@
// SPDX-License-Identifier: GPL-2.0
//! A kernel mutex.
//!
//! This module allows Rust code to use the kernel's `struct mutex`.
/// Creates a [`Mutex`] initialiser with the given name and a newly-created lock class.
///
/// It uses the name if one is given, otherwise it generates one based on the file name and line
/// number.
#[macro_export]
macro_rules! new_mutex {
($inner:expr $(, $name:literal)? $(,)?) => {
$crate::sync::Mutex::new(
$inner, $crate::optional_name!($($name)?), $crate::static_lock_class!())
};
}
pub use new_mutex;
/// A mutual exclusion primitive.
///
/// Exposes the kernel's [`struct mutex`]. When multiple threads attempt to lock the same mutex,
/// only one at a time is allowed to progress, the others will block (sleep) until the mutex is
/// unlocked, at which point another thread will be allowed to wake up and make progress.
///
/// Since it may block, [`Mutex`] needs to be used with care in atomic contexts.
///
/// Instances of [`Mutex`] need a lock class and to be pinned. The recommended way to create such
/// instances is with the [`pin_init`](pin_init::pin_init) and [`new_mutex`] macros.
///
/// # Examples
///
/// The following example shows how to declare, allocate and initialise a struct (`Example`) that
/// contains an inner struct (`Inner`) that is protected by a mutex.
///
/// ```
/// use kernel::sync::{new_mutex, Mutex};
///
/// struct Inner {
/// a: u32,
/// b: u32,
/// }
///
/// #[pin_data]
/// struct Example {
/// c: u32,
/// #[pin]
/// d: Mutex<Inner>,
/// }
///
/// impl Example {
/// fn new() -> impl PinInit<Self> {
/// pin_init!(Self {
/// c: 10,
/// d <- new_mutex!(Inner { a: 20, b: 30 }),
/// })
/// }
/// }
///
/// // Allocate a boxed `Example`.
/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
/// assert_eq!(e.c, 10);
/// assert_eq!(e.d.lock().a, 20);
/// assert_eq!(e.d.lock().b, 30);
/// # Ok::<(), Error>(())
/// ```
///
/// The following example shows how to use interior mutability to modify the contents of a struct
/// protected by a mutex despite only having a shared reference:
///
/// ```
/// use kernel::sync::Mutex;
///
/// struct Example {
/// a: u32,
/// b: u32,
/// }
///
/// fn example(m: &Mutex<Example>) {
/// let mut guard = m.lock();
/// guard.a += 10;
/// guard.b += 20;
/// }
/// ```
///
/// [`struct mutex`]: srctree/include/linux/mutex.h
pub type Mutex<T> = super::Lock<T, MutexBackend>;
/// A [`Guard`] acquired from locking a [`Mutex`].
///
/// This is simply a type alias for a [`Guard`] returned from locking a [`Mutex`]. It will unlock
/// the [`Mutex`] upon being dropped.
///
/// [`Guard`]: super::Guard
pub type MutexGuard<'a, T> = super::Guard<'a, T, MutexBackend>;
/// A kernel `struct mutex` lock backend.
pub struct MutexBackend;
// SAFETY: The underlying kernel `struct mutex` object ensures mutual exclusion.
unsafe impl super::Backend for MutexBackend {
type State = bindings::mutex;
type GuardState = ();
#[inline]
unsafe fn init(
ptr: *mut Self::State,
name: *const crate::ffi::c_char,
key: *mut bindings::lock_class_key,
) {
// SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and
// `key` are valid for read indefinitely.
unsafe { bindings::__mutex_init(ptr, name, key) }
}
#[inline]
unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState {
// SAFETY: The safety requirements of this function ensure that `ptr` points to valid
// memory, and that it has been initialised before.
unsafe { bindings::mutex_lock(ptr) };
}
#[inline]
unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) {
// SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the
// caller is the owner of the mutex.
unsafe { bindings::mutex_unlock(ptr) };
}
#[inline]
unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState> {
// SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
let result = unsafe { bindings::mutex_trylock(ptr) };
if result != 0 {
Some(())
} else {
None
}
}
#[inline]
unsafe fn assert_is_held(ptr: *mut Self::State) {
// SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
unsafe { bindings::mutex_assert_is_held(ptr) }
}
}
linux-7.2.3/rust/kernel/sync/lock/spinlock.rs
+146
View File
@@ -0,0 +1,146 @@
// SPDX-License-Identifier: GPL-2.0
//! A kernel spinlock.
//!
//! This module allows Rust code to use the kernel's `spinlock_t`.
/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class.
///
/// It uses the name if one is given, otherwise it generates one based on the file name and line
/// number.
#[macro_export]
macro_rules! new_spinlock {
($inner:expr $(, $name:literal)? $(,)?) => {
$crate::sync::SpinLock::new(
$inner, $crate::optional_name!($($name)?), $crate::static_lock_class!())
};
}
pub use new_spinlock;
/// A spinlock.
///
/// Exposes the kernel's [`spinlock_t`]. When multiple CPUs attempt to lock the same spinlock, only
/// one at a time is allowed to progress, the others will block (spinning) until the spinlock is
/// unlocked, at which point another CPU will be allowed to make progress.
///
/// Instances of [`SpinLock`] need a lock class and to be pinned. The recommended way to create such
/// instances is with the [`pin_init`](pin_init::pin_init) and [`new_spinlock`] macros.
///
/// # Examples
///
/// The following example shows how to declare, allocate and initialise a struct (`Example`) that
/// contains an inner struct (`Inner`) that is protected by a spinlock.
///
/// ```
/// use kernel::sync::{new_spinlock, SpinLock};
///
/// struct Inner {
/// a: u32,
/// b: u32,
/// }
///
/// #[pin_data]
/// struct Example {
/// c: u32,
/// #[pin]
/// d: SpinLock<Inner>,
/// }
///
/// impl Example {
/// fn new() -> impl PinInit<Self> {
/// pin_init!(Self {
/// c: 10,
/// d <- new_spinlock!(Inner { a: 20, b: 30 }),
/// })
/// }
/// }
///
/// // Allocate a boxed `Example`.
/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
/// assert_eq!(e.c, 10);
/// assert_eq!(e.d.lock().a, 20);
/// assert_eq!(e.d.lock().b, 30);
/// # Ok::<(), Error>(())
/// ```
///
/// The following example shows how to use interior mutability to modify the contents of a struct
/// protected by a spinlock despite only having a shared reference:
///
/// ```
/// use kernel::sync::SpinLock;
///
/// struct Example {
/// a: u32,
/// b: u32,
/// }
///
/// fn example(m: &SpinLock<Example>) {
/// let mut guard = m.lock();
/// guard.a += 10;
/// guard.b += 20;
/// }
/// ```
///
/// [`spinlock_t`]: srctree/include/linux/spinlock.h
pub type SpinLock<T> = super::Lock<T, SpinLockBackend>;
/// A kernel `spinlock_t` lock backend.
pub struct SpinLockBackend;
/// A [`Guard`] acquired from locking a [`SpinLock`].
///
/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock
/// the [`SpinLock`] upon being dropped.
///
/// [`Guard`]: super::Guard
pub type SpinLockGuard<'a, T> = super::Guard<'a, T, SpinLockBackend>;
// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the
// default implementation that always calls the same locking method.
unsafe impl super::Backend for SpinLockBackend {
type State = bindings::spinlock_t;
type GuardState = ();
#[inline]
unsafe fn init(
ptr: *mut Self::State,
name: *const crate::ffi::c_char,
key: *mut bindings::lock_class_key,
) {
// SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and
// `key` are valid for read indefinitely.
unsafe { bindings::__spin_lock_init(ptr, name, key) }
}
#[inline]
unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState {
// SAFETY: The safety requirements of this function ensure that `ptr` points to valid
// memory, and that it has been initialised before.
unsafe { bindings::spin_lock(ptr) }
}
#[inline]
unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) {
// SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the
// caller is the owner of the spinlock.
unsafe { bindings::spin_unlock(ptr) }
}
#[inline]
unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState> {
// SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
let result = unsafe { bindings::spin_trylock(ptr) };
if result != 0 {
Some(())
} else {
None
}
}
#[inline]
unsafe fn assert_is_held(ptr: *mut Self::State) {
// SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
unsafe { bindings::spin_assert_is_held(ptr) }
}
}
linux-7.2.3/rust/kernel/sync/locked_by.rs
+169
View File
@@ -0,0 +1,169 @@
// SPDX-License-Identifier: GPL-2.0
//! A wrapper for data protected by a lock that does not wrap it.
use super::{lock::Backend, lock::Lock};
use crate::build_assert::build_assert;
use core::{cell::UnsafeCell, mem::size_of, ptr};
/// Allows access to some data to be serialised by a lock that does not wrap it.
///
/// In most cases, data protected by a lock is wrapped by the appropriate lock type, e.g.,
/// [`Mutex`] or [`SpinLock`]. [`LockedBy`] is meant for cases when this is not possible.
/// For example, if a container has a lock and some data in the contained elements needs
/// to be protected by the same lock.
///
/// [`LockedBy`] wraps the data in lieu of another locking primitive, and only allows access to it
/// when the caller shows evidence that the 'external' lock is locked. It panics if the evidence
/// refers to the wrong instance of the lock.
///
/// [`Mutex`]: super::Mutex
/// [`SpinLock`]: super::SpinLock
///
/// # Examples
///
/// The following is an example for illustrative purposes: `InnerDirectory::bytes_used` is an
/// aggregate of all `InnerFile::bytes_used` and must be kept consistent; so we wrap `InnerFile` in
/// a `LockedBy` so that it shares a lock with `InnerDirectory`. This allows us to enforce at
/// compile-time that access to `InnerFile` is only granted when an `InnerDirectory` is also
/// locked; we enforce at run time that the right `InnerDirectory` is locked.
///
/// ```
/// use kernel::sync::{LockedBy, Mutex};
///
/// struct InnerFile {
/// bytes_used: u64,
/// }
///
/// struct File {
/// _ino: u32,
/// inner: LockedBy<InnerFile, InnerDirectory>,
/// }
///
/// struct InnerDirectory {
/// /// The sum of the bytes used by all files.
/// bytes_used: u64,
/// _files: KVec<File>,
/// }
///
/// struct Directory {
/// _ino: u32,
/// inner: Mutex<InnerDirectory>,
/// }
///
/// /// Prints `bytes_used` from both the directory and file.
/// fn print_bytes_used(dir: &Directory, file: &File) {
/// let guard = dir.inner.lock();
/// let inner_file = file.inner.access(&guard);
/// pr_info!("{} {}\n", guard.bytes_used, inner_file.bytes_used);
/// }
///
/// /// Increments `bytes_used` for both the directory and file.
/// fn inc_bytes_used(dir: &Directory, file: &File) {
/// let mut guard = dir.inner.lock();
/// guard.bytes_used += 10;
///
/// let file_inner = file.inner.access_mut(&mut guard);
/// file_inner.bytes_used += 10;
/// }
///
/// /// Creates a new file.
/// fn new_file(ino: u32, dir: &Directory) -> File {
/// File {
/// _ino: ino,
/// inner: LockedBy::new(&dir.inner, InnerFile { bytes_used: 0 }),
/// }
/// }
/// ```
pub struct LockedBy<T: ?Sized, U: ?Sized> {
owner: *const U,
data: UnsafeCell<T>,
}
// SAFETY: `LockedBy` can be transferred across thread boundaries iff the data it protects can.
unsafe impl<T: ?Sized + Send, U: ?Sized> Send for LockedBy<T, U> {}
// SAFETY: If `T` is not `Sync`, then parallel shared access to this `LockedBy` allows you to use
// `access_mut` to hand out `&mut T` on one thread at the time. The requirement that `T: Send` is
// sufficient to allow that.
//
// If `T` is `Sync`, then the `access` method also becomes available, which allows you to obtain
// several `&T` from several threads at once. However, this is okay as `T` is `Sync`.
unsafe impl<T: ?Sized + Send, U: ?Sized> Sync for LockedBy<T, U> {}
impl<T, U> LockedBy<T, U> {
/// Constructs a new instance of [`LockedBy`].
///
/// It stores a raw pointer to the owner that is never dereferenced. It is only used to ensure
/// that the right owner is being used to access the protected data. If the owner is freed, the
/// data becomes inaccessible; if another instance of the owner is allocated *on the same
/// memory location*, the data becomes accessible again: none of this affects memory safety
/// because in any case at most one thread (or CPU) can access the protected data at a time.
pub fn new<B: Backend>(owner: &Lock<U, B>, data: T) -> Self {
build_assert!(
size_of::<Lock<U, B>>() > 0,
"The lock type cannot be a ZST because it may be impossible to distinguish instances"
);
Self {
owner: owner.data.get(),
data: UnsafeCell::new(data),
}
}
}
impl<T: ?Sized, U> LockedBy<T, U> {
/// Returns a reference to the protected data when the caller provides evidence (via a
/// reference) that the owner is locked.
///
/// `U` cannot be a zero-sized type (ZST) because there are ways to get an `&U` that matches
/// the data protected by the lock without actually holding it.
///
/// # Panics
///
/// Panics if `owner` is different from the data protected by the lock used in
/// [`new`](LockedBy::new).
pub fn access<'a>(&'a self, owner: &'a U) -> &'a T
where
T: Sync,
{
build_assert!(
size_of::<U>() > 0,
"`U` cannot be a ZST because `owner` wouldn't be unique"
);
if !ptr::eq(owner, self.owner) {
panic!("mismatched owners");
}
// SAFETY: `owner` is evidence that there are only shared references to the owner for the
// duration of 'a, so it's not possible to use `Self::access_mut` to obtain a mutable
// reference to the inner value that aliases with this shared reference. The type is `Sync`
// so there are no other requirements.
unsafe { &*self.data.get() }
}
/// Returns a mutable reference to the protected data when the caller provides evidence (via a
/// mutable owner) that the owner is locked mutably.
///
/// `U` cannot be a zero-sized type (ZST) because there are ways to get an `&mut U` that
/// matches the data protected by the lock without actually holding it.
///
/// Showing a mutable reference to the owner is sufficient because we know no other references
/// can exist to it.
///
/// # Panics
///
/// Panics if `owner` is different from the data protected by the lock used in
/// [`new`](LockedBy::new).
pub fn access_mut<'a>(&'a self, owner: &'a mut U) -> &'a mut T {
build_assert!(
size_of::<U>() > 0,
"`U` cannot be a ZST because `owner` wouldn't be unique"
);
if !ptr::eq(owner, self.owner) {
panic!("mismatched owners");
}
// SAFETY: `owner` is evidence that there is only one reference to the owner.
unsafe { &mut *self.data.get() }
}
}
linux-7.2.3/rust/kernel/sync/poll.rs
+106
View File
@@ -0,0 +1,106 @@
// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2024 Google LLC.
//! Utilities for working with `struct poll_table`.
use crate::{
bindings,
fs::File,
prelude::*,
sync::{CondVar, LockClassKey},
};
use core::{marker::PhantomData, ops::Deref};
/// Creates a [`PollCondVar`] initialiser with the given name and a newly-created lock class.
#[macro_export]
macro_rules! new_poll_condvar {
($($name:literal)?) => {
$crate::sync::poll::PollCondVar::new(
$crate::optional_name!($($name)?), $crate::static_lock_class!()
)
};
}
/// Wraps the kernel's `poll_table`.
///
/// # Invariants
///
/// The pointer must be null or reference a valid `poll_table`.
#[repr(transparent)]
pub struct PollTable<'a> {
table: *mut bindings::poll_table,
_lifetime: PhantomData<&'a bindings::poll_table>,
}
impl<'a> PollTable<'a> {
/// Creates a [`PollTable`] from a valid pointer.
///
/// # Safety
///
/// The pointer must be null or reference a valid `poll_table` for the duration of `'a`.
pub unsafe fn from_raw(table: *mut bindings::poll_table) -> Self {
// INVARIANTS: The safety requirements are the same as the struct invariants.
PollTable {
table,
_lifetime: PhantomData,
}
}
/// Register this [`PollTable`] with the provided [`PollCondVar`], so that it can be notified
/// using the condition variable.
pub fn register_wait(&self, file: &File, cv: &PollCondVar) {
// SAFETY:
// * `file.as_ptr()` references a valid file for the duration of this call.
// * `self.table` is null or references a valid poll_table for the duration of this call.
// * Since `PollCondVar` is pinned, its destructor is guaranteed to run before the memory
// containing `cv.wait_queue_head` is invalidated. Since the destructor clears all
// waiters and then waits for an rcu grace period, it's guaranteed that
// `cv.wait_queue_head` remains valid for at least an rcu grace period after the removal
// of the last waiter.
unsafe { bindings::poll_wait(file.as_ptr(), cv.wait_queue_head.get(), self.table) }
}
}
/// A wrapper around [`CondVar`] that makes it usable with [`PollTable`].
///
/// [`CondVar`]: crate::sync::CondVar
#[pin_data(PinnedDrop)]
pub struct PollCondVar {
#[pin]
inner: CondVar,
}
impl PollCondVar {
/// Constructs a new condvar initialiser.
pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> {
pin_init!(Self {
inner <- CondVar::new(name, key),
})
}
}
// Make the `CondVar` methods callable on `PollCondVar`.
impl Deref for PollCondVar {
type Target = CondVar;
fn deref(&self) -> &CondVar {
&self.inner
}
}
#[pinned_drop]
impl PinnedDrop for PollCondVar {
#[inline]
fn drop(self: Pin<&mut Self>) {
// Clear anything registered using `register_wait`.
//
// SAFETY: The pointer points at a valid `wait_queue_head`.
unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.get()) };
// Wait for epoll items to be properly removed.
//
// SAFETY: Just an FFI call.
unsafe { bindings::synchronize_rcu() };
}
}
linux-7.2.3/rust/kernel/sync/rcu.rs
+52
View File
@@ -0,0 +1,52 @@
// SPDX-License-Identifier: GPL-2.0
//! RCU support.
//!
//! C header: [`include/linux/rcupdate.h`](srctree/include/linux/rcupdate.h)
use crate::{bindings, types::NotThreadSafe};
/// Evidence that the RCU read side lock is held on the current thread/CPU.
///
/// The type is explicitly not `Send` because this property is per-thread/CPU.
///
/// # Invariants
///
/// The RCU read side lock is actually held while instances of this guard exist.
pub struct Guard(NotThreadSafe);
impl Guard {
/// Acquires the RCU read side lock and returns a guard.
#[inline]
pub fn new() -> Self {
// SAFETY: An FFI call with no additional requirements.
unsafe { bindings::rcu_read_lock() };
// INVARIANT: The RCU read side lock was just acquired above.
Self(NotThreadSafe)
}
/// Explicitly releases the RCU read side lock.
#[inline]
pub fn unlock(self) {}
}
impl Default for Guard {
#[inline]
fn default() -> Self {
Self::new()
}
}
impl Drop for Guard {
#[inline]
fn drop(&mut self) {
// SAFETY: By the type invariants, the RCU read side is locked, so it is ok to unlock it.
unsafe { bindings::rcu_read_unlock() };
}
}
/// Acquires the RCU read side lock.
#[inline]
pub fn read_lock() -> Guard {
Guard::new()
}
linux-7.2.3/rust/kernel/sync/refcount.rs
+116
View File
@@ -0,0 +1,116 @@
// SPDX-License-Identifier: GPL-2.0
//! Atomic reference counting.
//!
//! C header: [`include/linux/refcount.h`](srctree/include/linux/refcount.h)
use crate::{
build_assert::build_assert,
sync::atomic::Atomic,
types::Opaque, //
};
/// Atomic reference counter.
///
/// This type is conceptually an atomic integer, but provides saturation semantics compared to
/// normal atomic integers. Values in the negative range when viewed as a signed integer are
/// saturation (bad) values. For details about the saturation semantics, please refer to top of
/// [`include/linux/refcount.h`](srctree/include/linux/refcount.h).
///
/// Wraps the kernel's C `refcount_t`.
#[repr(transparent)]
pub struct Refcount(Opaque<bindings::refcount_t>);
impl Refcount {
/// Construct a new [`Refcount`] from an initial value.
///
/// The initial value should be non-saturated.
// Always inline to optimize out error path of `build_assert`.
#[inline(always)]
pub fn new(value: i32) -> Self {
build_assert!(value >= 0, "initial value saturated");
// SAFETY: There are no safety requirements for this FFI call.
Self(Opaque::new(unsafe { bindings::REFCOUNT_INIT(value) }))
}
#[inline]
fn as_ptr(&self) -> *mut bindings::refcount_t {
self.0.get()
}
/// Get the underlying atomic counter that backs the refcount.
///
/// NOTE: Usage of this function is discouraged as it can circumvent the protections offered by
/// `refcount.h`. If there is no way to achieve the result using APIs in `refcount.h`, then
/// this function can be used. Otherwise consider adding a binding for the required API.
#[inline]
pub fn as_atomic(&self) -> &Atomic<i32> {
let ptr = self.0.get().cast();
// SAFETY: `refcount_t` is a transparent wrapper of `atomic_t`, which is an atomic 32-bit
// integer that is layout-wise compatible with `Atomic<i32>`. All values are valid for
// `refcount_t`, despite some of the values being considered saturated and "bad".
unsafe { &*ptr }
}
/// Set a refcount's value.
#[inline]
pub fn set(&self, value: i32) {
// SAFETY: `self.as_ptr()` is valid.
unsafe { bindings::refcount_set(self.as_ptr(), value) }
}
/// Increment a refcount.
///
/// It will saturate if overflows and `WARN`. It will also `WARN` if the refcount is 0, as this
/// represents a possible use-after-free condition.
///
/// Provides no memory ordering, it is assumed that caller already has a reference on the
/// object.
#[inline]
pub fn inc(&self) {
// SAFETY: self is valid.
unsafe { bindings::refcount_inc(self.as_ptr()) }
}
/// Decrement a refcount.
///
/// It will `WARN` on underflow and fail to decrement when saturated.
///
/// Provides release memory ordering, such that prior loads and stores are done
/// before.
#[inline]
pub fn dec(&self) {
// SAFETY: `self.as_ptr()` is valid.
unsafe { bindings::refcount_dec(self.as_ptr()) }
}
/// Decrement a refcount and test if it is 0.
///
/// It will `WARN` on underflow and fail to decrement when saturated.
///
/// Provides release memory ordering, such that prior loads and stores are done
/// before, and provides an acquire ordering on success such that memory deallocation
/// must come after.
///
/// Returns true if the resulting refcount is 0, false otherwise.
///
/// # Notes
///
/// A common pattern of using `Refcount` is to free memory when the reference count reaches
/// zero. This means that the reference to `Refcount` could become invalid after calling this
/// function. This is fine as long as the reference to `Refcount` is no longer used when this
/// function returns `false`. It is not necessary to use raw pointers in this scenario, see
/// <https://github.com/rust-lang/rust/issues/55005>.
#[inline]
#[must_use = "use `dec` instead if you do not need to test if it is 0"]
pub fn dec_and_test(&self) -> bool {
// SAFETY: `self.as_ptr()` is valid.
unsafe { bindings::refcount_dec_and_test(self.as_ptr()) }
}
}
// SAFETY: `refcount_t` is thread-safe.
unsafe impl Send for Refcount {}
// SAFETY: `refcount_t` is thread-safe.
unsafe impl Sync for Refcount {}
linux-7.2.3/rust/kernel/sync/set_once.rs
+133
View File
@@ -0,0 +1,133 @@
// SPDX-License-Identifier: GPL-2.0
//! A container that can be initialized at most once.
use super::atomic::{
ordering::{Acquire, Relaxed, Release},
Atomic,
};
use core::{cell::UnsafeCell, mem::MaybeUninit};
/// A container that can be populated at most once. Thread safe.
///
/// Once the a [`SetOnce`] is populated, it remains populated by the same object for the
/// lifetime `Self`.
///
/// # Invariants
///
/// - `init` may only increase in value.
/// - `init` may only assume values in the range `0..=2`.
/// - `init == 0` if and only if `value` is uninitialized.
/// - `init == 1` if and only if there is exactly one thread with exclusive
/// access to `self.value`.
/// - `init == 2` if and only if `value` is initialized and valid for shared
/// access.
///
/// # Example
///
/// ```
/// # use kernel::sync::SetOnce;
/// let value = SetOnce::new();
/// assert_eq!(None, value.as_ref());
///
/// let status = value.populate(42u8);
/// assert_eq!(true, status);
/// assert_eq!(Some(&42u8), value.as_ref());
/// assert_eq!(Some(42u8), value.copy());
///
/// let status = value.populate(101u8);
/// assert_eq!(false, status);
/// assert_eq!(Some(&42u8), value.as_ref());
/// assert_eq!(Some(42u8), value.copy());
/// ```
pub struct SetOnce<T> {
init: Atomic<u32>,
value: UnsafeCell<MaybeUninit<T>>,
}
impl<T> Default for SetOnce<T> {
fn default() -> Self {
Self::new()
}
}
impl<T> SetOnce<T> {
/// Create a new [`SetOnce`].
///
/// The returned instance will be empty.
pub const fn new() -> Self {
// INVARIANT: The container is empty and we initialize `init` to `0`.
Self {
value: UnsafeCell::new(MaybeUninit::uninit()),
init: Atomic::new(0),
}
}
/// Get a reference to the contained object.
///
/// Returns [`None`] if this [`SetOnce`] is empty.
pub fn as_ref(&self) -> Option<&T> {
if self.init.load(Acquire) == 2 {
// SAFETY: By the type invariants of `Self`, `self.init == 2` means that `self.value`
// is initialized and valid for shared access.
Some(unsafe { &*self.value.get().cast() })
} else {
None
}
}
/// Populate the [`SetOnce`].
///
/// Returns `true` if the [`SetOnce`] was successfully populated.
pub fn populate(&self, value: T) -> bool {
// INVARIANT: If the swap succeeds:
// - We increase `init`.
// - We write the valid value `1` to `init`.
// - Only one thread can succeed in this write, so we have exclusive access after the
// write.
if let Ok(0) = self.init.cmpxchg(0, 1, Relaxed) {
// SAFETY: By the type invariants of `Self`, the fact that we succeeded in writing `1`
// to `self.init` means we obtained exclusive access to `self.value`.
unsafe { core::ptr::write(self.value.get().cast(), value) };
// INVARIANT:
// - We increase `init`.
// - We write the valid value `2` to `init`.
// - We release our exclusive access to `self.value` and it is now valid for shared
// access.
self.init.store(2, Release);
true
} else {
false
}
}
/// Get a copy of the contained object.
///
/// Returns [`None`] if the [`SetOnce`] is empty.
pub fn copy(&self) -> Option<T>
where
T: Copy,
{
self.as_ref().copied()
}
}
impl<T> Drop for SetOnce<T> {
fn drop(&mut self) {
if *self.init.get_mut() == 2 {
let value = self.value.get_mut();
// SAFETY: By the type invariants of `Self`, `self.init == 2` means that `self.value`
// contains a valid value. We have exclusive access, as we hold a `mut` reference to
// `self`.
unsafe { value.assume_init_drop() };
}
}
}
// SAFETY: `SetOnce` can be transferred across thread boundaries iff the data it contains can.
unsafe impl<T: Send> Send for SetOnce<T> {}
// SAFETY: `SetOnce` synchronises access to the inner value via atomic operations,
// so shared references are safe when `T: Sync`. Since the inner `T` may be dropped
// on any thread, we also require `T: Send`.
unsafe impl<T: Send + Sync> Sync for SetOnce<T> {}