lastdragon/linux-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resguardo inicial

Browse Source
This commit is contained in:
lastdragon
2026-09-06 12:53:21 -06:00
parent 662efe3c2a
commit ad2088d178
94853 changed files with 43200404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linux-7.2.3/arch/x86/entry/Makefile
+26
View File
@@ -0,0 +1,26 @@
# SPDX-License-Identifier: GPL-2.0
#
# Makefile for the x86 low level entry code
#
KASAN_SANITIZE := n
UBSAN_SANITIZE := n
KCOV_INSTRUMENT := n
CFLAGS_REMOVE_syscall_32.o = $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_syscall_64.o = $(CC_FLAGS_FTRACE)
CFLAGS_syscall_32.o += -fno-stack-protector
CFLAGS_syscall_64.o += -fno-stack-protector
obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o common.o
obj-y += vdso/
obj-y += vsyscall/
obj-$(CONFIG_PREEMPTION) += thunk.o
CFLAGS_entry_fred.o += -fno-stack-protector
CFLAGS_REMOVE_entry_fred.o += -pg $(CC_FLAGS_FTRACE)
obj-$(CONFIG_X86_FRED) += entry_64_fred.o entry_fred.o
obj-$(CONFIG_IA32_EMULATION) += entry_64_compat.o syscall_32.o
linux-7.2.3/arch/x86/entry/calling.h
+489
View File
@@ -0,0 +1,489 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/jump_label.h>
#include <asm/unwind_hints.h>
#include <asm/cpufeatures.h>
#include <asm/page_types.h>
#include <asm/percpu.h>
#include <asm/asm-offsets.h>
#include <asm/processor-flags.h>
#include <asm/ptrace-abi.h>
#include <asm/msr.h>
#include <asm/nospec-branch.h>
/*
x86 function call convention, 64-bit:
-------------------------------------
arguments | callee-saved | extra caller-saved | return
[callee-clobbered] | | [callee-clobbered] |
---------------------------------------------------------------------------
rdi rsi rdx rcx r8-9 | rbx rbp [*] r12-15 | r10-11 | rax, rdx [**]
( rsp is obviously invariant across normal function calls. (gcc can 'merge'
functions when it sees tail-call optimization possibilities) rflags is
clobbered. Leftover arguments are passed over the stack frame.)
[*] In the frame-pointers case rbp is fixed to the stack frame.
[**] for struct return values wider than 64 bits the return convention is a
bit more complex: up to 128 bits width we return small structures
straight in rax, rdx. For structures larger than that (3 words or
larger) the caller puts a pointer to an on-stack return struct
[allocated in the caller's stack frame] into the first argument - i.e.
into rdi. All other arguments shift up by one in this case.
Fortunately this case is rare in the kernel.
For 32-bit we have the following conventions - kernel is built with
-mregparm=3 and -freg-struct-return:
x86 function calling convention, 32-bit:
----------------------------------------
arguments | callee-saved | extra caller-saved | return
[callee-clobbered] | | [callee-clobbered] |
-------------------------------------------------------------------------
eax edx ecx | ebx edi esi ebp [*] | <none> | eax, edx [**]
( here too esp is obviously invariant across normal function calls. eflags
is clobbered. Leftover arguments are passed over the stack frame. )
[*] In the frame-pointers case ebp is fixed to the stack frame.
[**] We build with -freg-struct-return, which on 32-bit means similar
semantics as on 64-bit: edx can be used for a second return value
(i.e. covering integer and structure sizes up to 64 bits) - after that
it gets more complex and more expensive: 3-word or larger struct returns
get done in the caller's frame and the pointer to the return struct goes
into regparm0, i.e. eax - the other arguments shift up and the
function's register parameters degenerate to regparm=2 in essence.
*/
#ifdef CONFIG_X86_64
/*
* 64-bit system call stack frame layout defines and helpers,
* for assembly code:
*/
.macro PUSH_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 unwind_hint=1
.if \save_ret
pushq %rsi /* pt_regs->si */
movq 8(%rsp), %rsi /* temporarily store the return address in %rsi */
movq %rdi, 8(%rsp) /* pt_regs->di (overwriting original return address) */
/* We just clobbered the return address - use the IRET frame for unwinding: */
UNWIND_HINT_IRET_REGS offset=3*8
.else
pushq %rdi /* pt_regs->di */
pushq %rsi /* pt_regs->si */
.endif
pushq \rdx /* pt_regs->dx */
pushq \rcx /* pt_regs->cx */
pushq \rax /* pt_regs->ax */
pushq %r8 /* pt_regs->r8 */
pushq %r9 /* pt_regs->r9 */
pushq %r10 /* pt_regs->r10 */
pushq %r11 /* pt_regs->r11 */
pushq %rbx /* pt_regs->rbx */
pushq %rbp /* pt_regs->rbp */
pushq %r12 /* pt_regs->r12 */
pushq %r13 /* pt_regs->r13 */
pushq %r14 /* pt_regs->r14 */
pushq %r15 /* pt_regs->r15 */
.if \unwind_hint
UNWIND_HINT_REGS
.endif
.if \save_ret
pushq %rsi /* return address on top of stack */
.endif
.endm
.macro CLEAR_REGS clear_callee=1
/*
* Sanitize registers of values that a speculation attack might
* otherwise want to exploit. The lower registers are likely clobbered
* well before they could be put to use in a speculative execution
* gadget.
*/
xorl %esi, %esi /* nospec si */
xorl %edx, %edx /* nospec dx */
xorl %ecx, %ecx /* nospec cx */
xorl %r8d, %r8d /* nospec r8 */
xorl %r9d, %r9d /* nospec r9 */
xorl %r10d, %r10d /* nospec r10 */
xorl %r11d, %r11d /* nospec r11 */
.if \clear_callee
xorl %ebx, %ebx /* nospec rbx */
xorl %ebp, %ebp /* nospec rbp */
xorl %r12d, %r12d /* nospec r12 */
xorl %r13d, %r13d /* nospec r13 */
xorl %r14d, %r14d /* nospec r14 */
xorl %r15d, %r15d /* nospec r15 */
.endif
.endm
.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_callee=1 unwind_hint=1
PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret unwind_hint=\unwind_hint
CLEAR_REGS clear_callee=\clear_callee
.endm
.macro POP_REGS pop_rdi=1
popq %r15
popq %r14
popq %r13
popq %r12
popq %rbp
popq %rbx
popq %r11
popq %r10
popq %r9
popq %r8
popq %rax
popq %rcx
popq %rdx
popq %rsi
.if \pop_rdi
popq %rdi
.endif
.endm
#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
/*
* MITIGATION_PAGE_TABLE_ISOLATION PGDs are 8k. Flip bit 12 to switch between the two
* halves:
*/
#define PTI_USER_PGTABLE_BIT PAGE_SHIFT
#define PTI_USER_PGTABLE_MASK (1 << PTI_USER_PGTABLE_BIT)
#define PTI_USER_PCID_BIT X86_CR3_PTI_PCID_USER_BIT
#define PTI_USER_PCID_MASK (1 << PTI_USER_PCID_BIT)
#define PTI_USER_PGTABLE_AND_PCID_MASK (PTI_USER_PCID_MASK | PTI_USER_PGTABLE_MASK)
.macro SET_NOFLUSH_BIT reg:req
bts $X86_CR3_PCID_NOFLUSH_BIT, \reg
.endm
.macro ADJUST_KERNEL_CR3 reg:req
ALTERNATIVE "", "SET_NOFLUSH_BIT \reg", X86_FEATURE_PCID
/* Clear PCID and "MITIGATION_PAGE_TABLE_ISOLATION bit", point CR3 at kernel pagetables: */
andq $(~PTI_USER_PGTABLE_AND_PCID_MASK), \reg
.endm
.macro SWITCH_TO_KERNEL_CR3 scratch_reg:req
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
mov %cr3, \scratch_reg
ADJUST_KERNEL_CR3 \scratch_reg
mov \scratch_reg, %cr3
.Lend_\@:
.endm
#define THIS_CPU_user_pcid_flush_mask \
PER_CPU_VAR(cpu_tlbstate + TLB_STATE_user_pcid_flush_mask)
.macro SWITCH_TO_USER_CR3 scratch_reg:req scratch_reg2:req
mov %cr3, \scratch_reg
ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
/*
* Test if the ASID needs a flush.
*/
movq \scratch_reg, \scratch_reg2
andq $(0x7FF), \scratch_reg /* mask ASID */
bt \scratch_reg, THIS_CPU_user_pcid_flush_mask
jnc .Lnoflush_\@
/* Flush needed, clear the bit */
btr \scratch_reg, THIS_CPU_user_pcid_flush_mask
movq \scratch_reg2, \scratch_reg
jmp .Lwrcr3_pcid_\@
.Lnoflush_\@:
movq \scratch_reg2, \scratch_reg
SET_NOFLUSH_BIT \scratch_reg
.Lwrcr3_pcid_\@:
/* Flip the ASID to the user version */
orq $(PTI_USER_PCID_MASK), \scratch_reg
.Lwrcr3_\@:
/* Flip the PGD to the user version */
orq $(PTI_USER_PGTABLE_MASK), \scratch_reg
mov \scratch_reg, %cr3
.endm
.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
SWITCH_TO_USER_CR3 \scratch_reg \scratch_reg2
.Lend_\@:
.endm
.macro SWITCH_TO_USER_CR3_STACK scratch_reg:req
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
pushq %rax
SWITCH_TO_USER_CR3 scratch_reg=\scratch_reg scratch_reg2=%rax
popq %rax
.Lend_\@:
.endm
.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
ALTERNATIVE "jmp .Ldone_\@", "", X86_FEATURE_PTI
movq %cr3, \scratch_reg
movq \scratch_reg, \save_reg
/*
* Test the user pagetable bit. If set, then the user page tables
* are active. If clear CR3 already has the kernel page table
* active.
*/
bt $PTI_USER_PGTABLE_BIT, \scratch_reg
jnc .Ldone_\@
ADJUST_KERNEL_CR3 \scratch_reg
movq \scratch_reg, %cr3
.Ldone_\@:
.endm
/* Restore CR3 from a kernel context. May restore a user CR3 value. */
.macro PARANOID_RESTORE_CR3 scratch_reg:req save_reg:req
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
/*
* If CR3 contained the kernel page tables at the paranoid exception
* entry, then there is nothing to restore as CR3 is not modified while
* handling the exception.
*/
bt $PTI_USER_PGTABLE_BIT, \save_reg
jnc .Lend_\@
ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
/*
* Check if there's a pending flush for the user ASID we're
* about to set.
*/
movq \save_reg, \scratch_reg
andq $(0x7FF), \scratch_reg
btr \scratch_reg, THIS_CPU_user_pcid_flush_mask
jc .Lwrcr3_\@
SET_NOFLUSH_BIT \save_reg
.Lwrcr3_\@:
movq \save_reg, %cr3
.Lend_\@:
.endm
#else /* CONFIG_MITIGATION_PAGE_TABLE_ISOLATION=n: */
.macro SWITCH_TO_KERNEL_CR3 scratch_reg:req
.endm
.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req
.endm
.macro SWITCH_TO_USER_CR3_STACK scratch_reg:req
.endm
.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
.endm
.macro PARANOID_RESTORE_CR3 scratch_reg:req save_reg:req
.endm
#endif
/*
* IBRS kernel mitigation for Spectre_v2.
*
* Assumes full context is established (PUSH_REGS, CR3 and GS) and it clobbers
* the regs it uses (AX, CX, DX). Must be called before the first RET
* instruction (NOTE! UNTRAIN_RET includes a RET instruction)
*
* The optional argument is used to save/restore the current value,
* which is used on the paranoid paths.
*
* Assumes x86_spec_ctrl_{base,current} to have SPEC_CTRL_IBRS set.
*/
.macro IBRS_ENTER save_reg
#ifdef CONFIG_MITIGATION_IBRS_ENTRY
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_KERNEL_IBRS
movl $MSR_IA32_SPEC_CTRL, %ecx
.ifnb \save_reg
rdmsr
shl $32, %rdx
or %rdx, %rax
mov %rax, \save_reg
test $SPEC_CTRL_IBRS, %eax
jz .Ldo_wrmsr_\@
lfence
jmp .Lend_\@
.Ldo_wrmsr_\@:
.endif
movq PER_CPU_VAR(x86_spec_ctrl_current), %rdx
movl %edx, %eax
shr $32, %rdx
wrmsr
.Lend_\@:
#endif
.endm
/*
* Similar to IBRS_ENTER, requires KERNEL GS,CR3 and clobbers (AX, CX, DX)
* regs. Must be called after the last RET.
*/
.macro IBRS_EXIT save_reg
#ifdef CONFIG_MITIGATION_IBRS_ENTRY
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_KERNEL_IBRS
movl $MSR_IA32_SPEC_CTRL, %ecx
.ifnb \save_reg
mov \save_reg, %rdx
.else
movq PER_CPU_VAR(x86_spec_ctrl_current), %rdx
andl $(~SPEC_CTRL_IBRS), %edx
.endif
movl %edx, %eax
shr $32, %rdx
wrmsr
.Lend_\@:
#endif
.endm
/*
* Mitigate Spectre v1 for conditional swapgs code paths.
*
* FENCE_SWAPGS_USER_ENTRY is used in the user entry swapgs code path, to
* prevent a speculative swapgs when coming from kernel space.
*
* FENCE_SWAPGS_KERNEL_ENTRY is used in the kernel entry non-swapgs code path,
* to prevent the swapgs from getting speculatively skipped when coming from
* user space.
*/
.macro FENCE_SWAPGS_USER_ENTRY
ALTERNATIVE "", "lfence", X86_FEATURE_FENCE_SWAPGS_USER
.endm
.macro FENCE_SWAPGS_KERNEL_ENTRY
ALTERNATIVE "", "lfence", X86_FEATURE_FENCE_SWAPGS_KERNEL
.endm
.macro STACKLEAK_ERASE_NOCLOBBER
#ifdef CONFIG_KSTACK_ERASE
PUSH_AND_CLEAR_REGS
call stackleak_erase
POP_REGS
#endif
.endm
.macro SAVE_AND_SET_GSBASE scratch_reg:req save_reg:req
rdgsbase \save_reg
GET_PERCPU_BASE \scratch_reg
wrgsbase \scratch_reg
.endm
#else /* CONFIG_X86_64 */
# undef UNWIND_HINT_IRET_REGS
# define UNWIND_HINT_IRET_REGS
#endif /* !CONFIG_X86_64 */
.macro STACKLEAK_ERASE
#ifdef CONFIG_KSTACK_ERASE
call stackleak_erase
#endif
.endm
#ifdef CONFIG_SMP
/*
* CPU/node NR is loaded from the limit (size) field of a special segment
* descriptor entry in GDT.
*/
.macro LOAD_CPU_AND_NODE_SEG_LIMIT reg:req
movq $__CPUNODE_SEG, \reg
lsl \reg, \reg
.endm
/*
* Fetch the per-CPU GSBASE value for this processor and put it in @reg.
* We normally use %gs for accessing per-CPU data, but we are setting up
* %gs here and obviously can not use %gs itself to access per-CPU data.
*
* Do not use RDPID, because KVM loads guest's TSC_AUX on vm-entry and
* may not restore the host's value until the CPU returns to userspace.
* Thus the kernel would consume a guest's TSC_AUX if an NMI arrives
* while running KVM's run loop.
*/
.macro GET_PERCPU_BASE reg:req
LOAD_CPU_AND_NODE_SEG_LIMIT \reg
andq $VDSO_CPUNODE_MASK, \reg
movq __per_cpu_offset(, \reg, 8), \reg
.endm
#else
.macro GET_PERCPU_BASE reg:req
movq pcpu_unit_offsets(%rip), \reg
.endm
#endif /* CONFIG_SMP */
#ifdef CONFIG_X86_64
/* rdi: arg1 ... normal C conventions. rax is saved/restored. */
.macro THUNK name, func
SYM_FUNC_START(\name)
ANNOTATE_NOENDBR
pushq %rbp
movq %rsp, %rbp
pushq %rdi
pushq %rsi
pushq %rdx
pushq %rcx
pushq %rax
pushq %r8
pushq %r9
pushq %r10
pushq %r11
call \func
popq %r11
popq %r10
popq %r9
popq %r8
popq %rax
popq %rcx
popq %rdx
popq %rsi
popq %rdi
popq %rbp
RET
SYM_FUNC_END(\name)
_ASM_NOKPROBE(\name)
.endm
#else /* CONFIG_X86_32 */
/* put return address in eax (arg1) */
.macro THUNK name, func, put_ret_addr_in_eax=0
SYM_CODE_START_NOALIGN(\name)
pushl %eax
pushl %ecx
pushl %edx
.if \put_ret_addr_in_eax
/* Place EIP in the arg1 */
movl 3*4(%esp), %eax
.endif
call \func
popl %edx
popl %ecx
popl %eax
RET
_ASM_NOKPROBE(\name)
SYM_CODE_END(\name)
.endm
#endif
linux-7.2.3/arch/x86/entry/common.c
+61
View File
@@ -0,0 +1,61 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/entry-common.h>
#include <linux/kvm_types.h>
#include <linux/hrtimer_rearm.h>
#include <asm/fred.h>
#include <asm/desc.h>
#if IS_ENABLED(CONFIG_KVM_INTEL)
/*
* On VMX, NMIs and IRQs (as configured by KVM) are acknowledged by hardware as
* part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit.
* x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs
* to the kernel for servicing. On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is
* purely a signal that an NMI/IRQ is pending, i.e. the event that triggered
* the VM-Exit is held pending until it's unblocked in the host.
*/
noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)
{
if (event_type == EVENT_TYPE_EXTINT) {
#ifdef CONFIG_X86_64
/*
* Use FRED dispatch, even when running IDT. The dispatch
* tables are kept in sync between FRED and IDT, and the FRED
* dispatch works well with CFI.
*/
fred_entry_from_kvm(event_type, vector);
#else
idt_entry_from_kvm(vector);
#endif
/*
* Strictly speaking, only the NMI path requires noinstr.
*/
instrumentation_begin();
/*
* KVM/VMX will dispatch from IRQ-disabled but for a context
* that will have IRQs-enabled. This confuses the entry code
* and it will not have reprogrammed the timer. Do so now.
*/
hrtimer_rearm_deferred();
instrumentation_end();
return;
}
WARN_ON_ONCE(event_type != EVENT_TYPE_NMI);
#ifdef CONFIG_X86_64
if (cpu_feature_enabled(X86_FEATURE_FRED))
return fred_entry_from_kvm(event_type, vector);
#endif
/*
* Notably, we must use IDT dispatch for NMI when running in IDT mode.
* The FRED NMI context is significantly different and will not work
* right (specifically FRED fixed the NMI recursion issue).
*/
idt_do_nmi_irqoff();
}
EXPORT_SYMBOL_FOR_KVM(x86_entry_from_kvm);
#endif
linux-7.2.3/arch/x86/entry/entry.S
+125
View File
@@ -0,0 +1,125 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Common place for both 32- and 64-bit entry routines.
*/
#include <linux/export.h>
#include <linux/kvm_types.h>
#include <linux/linkage.h>
#include <linux/objtool.h>
#include <asm/msr-index.h>
#include <asm/unwind_hints.h>
#include <asm/segment.h>
#include <asm/cache.h>
#include <asm/cpufeatures.h>
#include <asm/nospec-branch.h>
#include "calling.h"
.pushsection .noinstr.text, "ax"
/* Clobbers AX, CX, DX */
SYM_FUNC_START(write_ibpb)
ANNOTATE_NOENDBR
movl $MSR_IA32_PRED_CMD, %ecx
movl _ASM_RIP(x86_pred_cmd), %eax
xorl %edx, %edx
wrmsr
/* Make sure IBPB clears return stack preductions too. */
FILL_RETURN_BUFFER %rax, RSB_CLEAR_LOOPS, X86_BUG_IBPB_NO_RET
RET
SYM_FUNC_END(write_ibpb)
EXPORT_SYMBOL_FOR_KVM(write_ibpb);
SYM_FUNC_START(__WARN_trap)
ANNOTATE_NOENDBR
ANNOTATE_REACHABLE
ud1 (%edx), %_ASM_ARG1
RET
SYM_FUNC_END(__WARN_trap)
EXPORT_SYMBOL(__WARN_trap)
.popsection
/*
* Define the VERW operand that is disguised as entry code so that
* it can be referenced with KPTI enabled. This ensures VERW can be
* used late in exit-to-user path after page tables are switched.
*/
.pushsection .entry.text, "ax"
.align L1_CACHE_BYTES, 0xcc
SYM_CODE_START_NOALIGN(x86_verw_sel)
UNWIND_HINT_UNDEFINED
ANNOTATE_NOENDBR
.word __KERNEL_DS
.align L1_CACHE_BYTES, 0xcc
SYM_CODE_END(x86_verw_sel);
EXPORT_SYMBOL_FOR_KVM(x86_verw_sel);
.popsection
THUNK warn_thunk_thunk, __warn_thunk
/*
* Clang's implementation of TLS stack cookies requires the variable in
* question to be a TLS variable. If the variable happens to be defined as an
* ordinary variable with external linkage in the same compilation unit (which
* amounts to the whole of vmlinux with LTO enabled), Clang will drop the
* segment register prefix from the references, resulting in broken code. Work
* around this by avoiding the symbol used in -mstack-protector-guard-symbol=
* entirely in the C code, and use an alias emitted by the linker script
* instead.
*/
#if defined(CONFIG_STACKPROTECTOR) && defined(CONFIG_SMP)
EXPORT_SYMBOL(__ref_stack_chk_guard);
#endif
#if IS_ENABLED(CONFIG_KVM_INTEL)
.macro IDT_DO_EVENT_IRQOFF call_insn call_target
/*
* Unconditionally create a stack frame, getting the correct RSP on the
* stack (for x86-64) would take two instructions anyways, and RBP can
* be used to restore RSP to make objtool happy (see below).
*/
push %_ASM_BP
mov %_ASM_SP, %_ASM_BP
#ifdef CONFIG_X86_64
/*
* Align RSP to a 16-byte boundary (to emulate CPU behavior) before
* creating the synthetic interrupt stack frame for the IRQ/NMI.
*/
and $-16, %rsp
push $__KERNEL_DS
push %rbp
#endif
pushf
push $__KERNEL_CS
\call_insn \call_target
/*
* "Restore" RSP from RBP, even though IRET has already unwound RSP to
* the correct value. objtool doesn't know the callee will IRET and,
* without the explicit restore, thinks the stack is getting walloped.
* Using an unwind hint is problematic due to x86-64's dynamic alignment.
*/
leave
RET
.endm
#ifndef CONFIG_X86_64
.pushsection .text, "ax"
SYM_FUNC_START(idt_do_interrupt_irqoff)
IDT_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1
SYM_FUNC_END(idt_do_interrupt_irqoff)
.popsection
#endif
.pushsection .noinstr.text, "ax"
SYM_FUNC_START(idt_do_nmi_irqoff)
IDT_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx
SYM_FUNC_END(idt_do_nmi_irqoff)
.popsection
#endif
linux-7.2.3/arch/x86/entry/entry_32.S
+1226
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/arch/x86/entry/entry_64.S
+1576
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/arch/x86/entry/entry_64_compat.S
+299
View File
@@ -0,0 +1,299 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Compatibility mode system call entry point for x86-64.
*
* Copyright 2000-2002 Andi Kleen, SuSE Labs.
*/
#include <asm/asm-offsets.h>
#include <asm/current.h>
#include <asm/errno.h>
#include <asm/thread_info.h>
#include <asm/segment.h>
#include <asm/irqflags.h>
#include <asm/asm.h>
#include <asm/smap.h>
#include <asm/nospec-branch.h>
#include <linux/linkage.h>
#include <linux/err.h>
#include "calling.h"
.section .entry.text, "ax"
/*
* 32-bit SYSENTER entry.
*
* 32-bit system calls through the vDSO's __kernel_vsyscall enter here
* on 64-bit kernels running on Intel CPUs.
*
* The SYSENTER instruction, in principle, should *only* occur in the
* vDSO. In practice, a small number of Android devices were shipped
* with a copy of Bionic that inlined a SYSENTER instruction. This
* never happened in any of Google's Bionic versions -- it only happened
* in a narrow range of Intel-provided versions.
*
* SYSENTER loads SS, RSP, CS, and RIP from previously programmed MSRs.
* IF and VM in RFLAGS are cleared (IOW: interrupts are off).
* SYSENTER does not save anything on the stack,
* and does not save old RIP (!!!), RSP, or RFLAGS.
*
* Arguments:
* eax system call number
* ebx arg1
* ecx arg2
* edx arg3
* esi arg4
* edi arg5
* ebp user stack
* 0(%ebp) arg6
*/
SYM_CODE_START(entry_SYSENTER_compat)
UNWIND_HINT_ENTRY
ENDBR
/* Interrupts are off on entry. */
swapgs
pushq %rax
SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
popq %rax
movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
/* Construct struct pt_regs on stack */
pushq $__USER_DS /* pt_regs->ss */
pushq $0 /* pt_regs->sp = 0 (placeholder) */
/*
* Push flags. This is nasty. First, interrupts are currently
* off, but we need pt_regs->flags to have IF set. Second, if TS
* was set in usermode, it's still set, and we're singlestepping
* through this code. do_SYSENTER_32() will fix up IF.
*/
pushfq /* pt_regs->flags (except IF = 0) */
pushq $__USER32_CS /* pt_regs->cs */
pushq $0 /* pt_regs->ip = 0 (placeholder) */
SYM_INNER_LABEL(entry_SYSENTER_compat_after_hwframe, SYM_L_GLOBAL)
/*
* User tracing code (ptrace or signal handlers) might assume that
* the saved RAX contains a 32-bit number when we're invoking a 32-bit
* syscall. Just in case the high bits are nonzero, zero-extend
* the syscall number. (This could almost certainly be deleted
* with no ill effects.)
*/
movl %eax, %eax
pushq %rax /* pt_regs->orig_ax */
PUSH_AND_CLEAR_REGS rax=$-ENOSYS
UNWIND_HINT_REGS
cld
/*
* SYSENTER doesn't filter flags, so we need to clear NT and AC
* ourselves. To save a few cycles, we can check whether
* either was set instead of doing an unconditional popfq.
* This needs to happen before enabling interrupts so that
* we don't get preempted with NT set.
*
* If TF is set, we will single-step all the way to here -- do_debug
* will ignore all the traps. (Yes, this is slow, but so is
* single-stepping in general. This allows us to avoid having
* a more complicated code to handle the case where a user program
* forces us to single-step through the SYSENTER entry code.)
*
* NB.: .Lsysenter_fix_flags is a label with the code under it moved
* out-of-line as an optimization: NT is unlikely to be set in the
* majority of the cases and instead of polluting the I$ unnecessarily,
* we're keeping that code behind a branch which will predict as
* not-taken and therefore its instructions won't be fetched.
*/
testl $X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, EFLAGS(%rsp)
jnz .Lsysenter_fix_flags
.Lsysenter_flags_fixed:
/*
* CPU bugs mitigations mechanisms can call other functions. They
* should be invoked after making sure TF is cleared because
* single-step is ignored only for instructions inside the
* entry_SYSENTER_compat function.
*/
IBRS_ENTER
UNTRAIN_RET
CLEAR_BRANCH_HISTORY
movq %rsp, %rdi
call do_SYSENTER_32
jmp sysret32_from_system_call
.Lsysenter_fix_flags:
pushq $X86_EFLAGS_FIXED
popfq
jmp .Lsysenter_flags_fixed
SYM_INNER_LABEL(__end_entry_SYSENTER_compat, SYM_L_GLOBAL)
SYM_CODE_END(entry_SYSENTER_compat)
/*
* 32-bit SYSCALL entry.
*
* 32-bit system calls through the vDSO's __kernel_vsyscall enter here
* on 64-bit kernels running on AMD CPUs.
*
* The SYSCALL instruction, in principle, should *only* occur in the
* vDSO. In practice, it appears that this really is the case.
* As evidence:
*
* - The calling convention for SYSCALL has changed several times without
* anyone noticing.
*
* - Prior to the in-kernel X86_BUG_SYSRET_SS_ATTRS fixup, anything
* user task that did SYSCALL without immediately reloading SS
* would randomly crash.
*
* - Most programmers do not directly target AMD CPUs, and the 32-bit
* SYSCALL instruction does not exist on Intel CPUs. Even on AMD
* CPUs, Linux disables the SYSCALL instruction on 32-bit kernels
* because the SYSCALL instruction in legacy/native 32-bit mode (as
* opposed to compat mode) is sufficiently poorly designed as to be
* essentially unusable.
*
* 32-bit SYSCALL saves RIP to RCX, clears RFLAGS.RF, then saves
* RFLAGS to R11, then loads new SS, CS, and RIP from previously
* programmed MSRs. RFLAGS gets masked by a value from another MSR
* (so CLD and CLAC are not needed). SYSCALL does not save anything on
* the stack and does not change RSP.
*
* Note: RFLAGS saving+masking-with-MSR happens only in Long mode
* (in legacy 32-bit mode, IF, RF and VM bits are cleared and that's it).
* Don't get confused: RFLAGS saving+masking depends on Long Mode Active bit
* (EFER.LMA=1), NOT on bitness of userspace where SYSCALL executes
* or target CS descriptor's L bit (SYSCALL does not read segment descriptors).
*
* Arguments:
* eax system call number
* ecx return address
* ebx arg1
* ebp arg2 (note: not saved in the stack frame, should not be touched)
* edx arg3
* esi arg4
* edi arg5
* esp user stack
* 0(%esp) arg6
*/
SYM_CODE_START(entry_SYSCALL_compat)
UNWIND_HINT_ENTRY
ENDBR
/* Interrupts are off on entry. */
swapgs
/* Stash user ESP */
movl %esp, %r8d
/* Use %rsp as scratch reg. User ESP is stashed in r8 */
SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
/* Switch to the kernel stack */
movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
/* Construct struct pt_regs on stack */
pushq $__USER_DS /* pt_regs->ss */
pushq %r8 /* pt_regs->sp */
pushq %r11 /* pt_regs->flags */
pushq $__USER32_CS /* pt_regs->cs */
pushq %rcx /* pt_regs->ip */
SYM_INNER_LABEL(entry_SYSCALL_compat_after_hwframe, SYM_L_GLOBAL)
movl %eax, %eax /* discard orig_ax high bits */
pushq %rax /* pt_regs->orig_ax */
PUSH_AND_CLEAR_REGS rcx=%rbp rax=$-ENOSYS
UNWIND_HINT_REGS
IBRS_ENTER
UNTRAIN_RET
CLEAR_BRANCH_HISTORY
movq %rsp, %rdi
call do_fast_syscall_32
sysret32_from_system_call:
/* XEN PV guests always use IRET path */
ALTERNATIVE "testb %al, %al; jz swapgs_restore_regs_and_return_to_usermode", \
"jmp swapgs_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV
/*
* Opportunistic SYSRET
*
* We are not going to return to userspace from the trampoline
* stack. So let's erase the thread stack right now.
*/
STACKLEAK_ERASE
IBRS_EXIT
movq RBX(%rsp), %rbx /* pt_regs->rbx */
movq RBP(%rsp), %rbp /* pt_regs->rbp */
movq EFLAGS(%rsp), %r11 /* pt_regs->flags (in r11) */
movq RIP(%rsp), %rcx /* pt_regs->ip (in rcx) */
addq $RAX, %rsp /* Skip r8-r15 */
popq %rax /* pt_regs->rax */
popq %rdx /* Skip pt_regs->cx */
popq %rdx /* pt_regs->dx */
popq %rsi /* pt_regs->si */
popq %rdi /* pt_regs->di */
/*
* USERGS_SYSRET32 does:
* GSBASE = user's GS base
* EIP = ECX
* RFLAGS = R11
* CS = __USER32_CS
* SS = __USER_DS
*
* ECX will not match pt_regs->cx, but we're returning to a vDSO
* trampoline that will fix up RCX, so this is okay.
*
* R12-R15 are callee-saved, so they contain whatever was in them
* when the system call started, which is already known to user
* code. We zero R8-R10 to avoid info leaks.
*/
movq RSP-ORIG_RAX(%rsp), %rsp
SYM_INNER_LABEL(entry_SYSRETL_compat_unsafe_stack, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
/*
* The original userspace %rsp (RSP-ORIG_RAX(%rsp)) is stored
* on the process stack which is not mapped to userspace and
* not readable after we SWITCH_TO_USER_CR3. Delay the CR3
* switch until after after the last reference to the process
* stack.
*
* %r8/%r9 are zeroed before the sysret, thus safe to clobber.
*/
SWITCH_TO_USER_CR3_NOSTACK scratch_reg=%r8 scratch_reg2=%r9
xorl %r8d, %r8d
xorl %r9d, %r9d
xorl %r10d, %r10d
swapgs
CLEAR_CPU_BUFFERS
sysretl
SYM_INNER_LABEL(entry_SYSRETL_compat_end, SYM_L_GLOBAL)
ANNOTATE_NOENDBR
int3
SYM_CODE_END(entry_SYSCALL_compat)
/*
* int 0x80 is used by 32 bit mode as a system call entry. Normally idt entries
* point to C routines, however since this is a system call interface the branch
* history needs to be scrubbed to protect against BHI attacks, and that
* scrubbing needs to take place in assembly code prior to entering any C
* routines.
*/
SYM_CODE_START(int80_emulation)
ANNOTATE_NOENDBR
UNWIND_HINT_FUNC
CLEAR_BRANCH_HISTORY
jmp do_int80_emulation
SYM_CODE_END(int80_emulation)
linux-7.2.3/arch/x86/entry/entry_64_fred.S
+150
View File
@@ -0,0 +1,150 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* The actual FRED entry points.
*/
#include <linux/export.h>
#include <linux/kvm_types.h>
#include <asm/asm.h>
#include <asm/fred.h>
#include <asm/segment.h>
#include "calling.h"
.code64
.section .noinstr.text, "ax"
.macro FRED_ENTER
UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR
PUSH_AND_CLEAR_REGS
movq %rsp, %rdi /* %rdi -> pt_regs */
.endm
.macro FRED_EXIT
UNWIND_HINT_REGS
POP_REGS
.endm
/*
* The new RIP value that FRED event delivery establishes is
* IA32_FRED_CONFIG & ~FFFH for events that occur in ring 3.
* Thus the FRED ring 3 entry point must be 4K page aligned.
*/
.align 4096
SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user)
FRED_ENTER
call fred_entry_from_user
SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL)
FRED_EXIT
1: ERETU
_ASM_EXTABLE_TYPE(1b, asm_fred_entrypoint_user, EX_TYPE_ERETU)
SYM_CODE_END(asm_fred_entrypoint_user)
/*
* The new RIP value that FRED event delivery establishes is
* (IA32_FRED_CONFIG & ~FFFH) + 256 for events that occur in
* ring 0, i.e., asm_fred_entrypoint_user + 256.
*/
.org asm_fred_entrypoint_user + 256, 0xcc
SYM_CODE_START_NOALIGN(asm_fred_entrypoint_kernel)
FRED_ENTER
call fred_entry_from_kernel
FRED_EXIT
ERETS
SYM_CODE_END(asm_fred_entrypoint_kernel)
#if IS_ENABLED(CONFIG_KVM_INTEL)
SYM_FUNC_START(asm_fred_entry_from_kvm)
ANNOTATE_NOENDBR
push %rbp
mov %rsp, %rbp
UNWIND_HINT_SAVE
/*
* Both IRQ and NMI from VMX can be handled on current task stack
* because there is no need to protect from reentrancy and the call
* stack leading to this helper is effectively constant and shallow
* (relatively speaking). Do the same when FRED is active, i.e., no
* need to check current stack level for a stack switch.
*
* Emulate the FRED-defined redzone and stack alignment.
*/
sub $(FRED_CONFIG_REDZONE_AMOUNT << 6), %rsp
and $FRED_STACK_FRAME_RSP_MASK, %rsp
/*
* Start to push a FRED stack frame, which is always 64 bytes:
*
* +--------+-----------------+
* | Bytes | Usage |
* +--------+-----------------+
* | 63:56 | Reserved |
* | 55:48 | Event Data |
* | 47:40 | SS + Event Info |
* | 39:32 | RSP |
* | 31:24 | RFLAGS |
* | 23:16 | CS + Aux Info |
* | 15:8 | RIP |
* | 7:0 | Error Code |
* +--------+-----------------+
*/
push $0 /* Reserved, must be 0 */
push $0 /* Event data, 0 for IRQ/NMI */
push %rdi /* fred_ss handed in by the caller */
push %rbp
pushf
push $__KERNEL_CS
/*
* Unlike the IDT event delivery, FRED _always_ pushes an error code
* after pushing the return RIP, thus the CALL instruction CANNOT be
* used here to push the return RIP, otherwise there is no chance to
* push an error code before invoking the IRQ/NMI handler.
*
* Use LEA to get the return RIP and push it, then push an error code.
*/
lea 1f(%rip), %rax
push %rax /* Return RIP */
push $0 /* Error code, 0 for IRQ/NMI */
PUSH_AND_CLEAR_REGS clear_callee=0 unwind_hint=0
movq %rsp, %rdi /* %rdi -> pt_regs */
/*
* At this point: {rdi, rsi, rdx, rcx, r8, r9}, {r10, r11}, {rax, rdx}
* are clobbered, which corresponds to: arguments, extra caller-saved
* and return. All registers a C function is allowed to clobber.
*
* Notably, the callee-saved registers: {rbx, r12, r13, r14, r15}
* are untouched, with the exception of rbp, which carries the stack
* frame and will be restored before exit.
*
* Further calling another C function will not alter this state.
*/
call __fred_entry_from_kvm /* Call the C entry point */
/*
* When FRED, use ERETS to potentially clear NMIs, otherwise simply
* restore the stack pointer.
*/
ALTERNATIVE "nop; nop; mov %rbp, %rsp", \
__stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \
X86_FEATURE_FRED
1: /*
* Objtool doesn't understand ERETS, and the cfi register state is
* different from initial_func_cfi due to PUSH_REGS. Tell it the state
* is similar to where UNWIND_HINT_SAVE is.
*/
UNWIND_HINT_RESTORE
pop %rbp
RET
SYM_FUNC_END(asm_fred_entry_from_kvm)
#endif
linux-7.2.3/arch/x86/entry/entry_fred.c
+300
View File
@@ -0,0 +1,300 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* The FRED specific kernel/user entry functions which are invoked from
* assembly code and dispatch to the associated handlers.
*/
#include <linux/kernel.h>
#include <linux/kdebug.h>
#include <linux/nospec.h>
#include <asm/desc.h>
#include <asm/fred.h>
#include <asm/idtentry.h>
#include <asm/syscall.h>
#include <asm/trapnr.h>
#include <asm/traps.h>
/* FRED EVENT_TYPE_OTHER vector numbers */
#define FRED_SYSCALL 1
#define FRED_SYSENTER 2
static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code)
{
irqentry_state_t irq_state = irqentry_nmi_enter(regs);
instrumentation_begin();
/* Panic on events from a high stack level */
if (regs->fred_cs.sl > 0) {
pr_emerg("PANIC: invalid or fatal FRED event; event type %u "
"vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n",
regs->fred_ss.type, regs->fred_ss.vector, error_code,
fred_event_data(regs), regs->cs, regs->ip);
die("invalid or fatal FRED event", regs, error_code);
panic("invalid or fatal FRED event");
} else {
unsigned long flags = oops_begin();
int sig = SIGKILL;
pr_alert("BUG: invalid or fatal FRED event; event type %u "
"vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n",
regs->fred_ss.type, regs->fred_ss.vector, error_code,
fred_event_data(regs), regs->cs, regs->ip);
if (__die("Invalid or fatal FRED event", regs, error_code))
sig = 0;
oops_end(flags, regs, sig);
}
instrumentation_end();
irqentry_nmi_exit(regs, irq_state);
}
static noinstr void fred_intx(struct pt_regs *regs)
{
switch (regs->fred_ss.vector) {
/* Opcode 0xcd, 0x3, NOT INT3 (opcode 0xcc) */
case X86_TRAP_BP:
return exc_int3(regs);
/* Opcode 0xcd, 0x4, NOT INTO (opcode 0xce) */
case X86_TRAP_OF:
return exc_overflow(regs);
#ifdef CONFIG_IA32_EMULATION
/* INT80 */
case IA32_SYSCALL_VECTOR:
if (ia32_enabled())
return fred_int80_emulation(regs);
fallthrough;
#endif
default:
return exc_general_protection(regs, 0);
}
}
static __always_inline void fred_other(struct pt_regs *regs)
{
/* The compiler can fold these conditions into a single test */
if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.l)) {
regs->orig_ax = regs->ax;
regs->ax = -ENOSYS;
do_syscall_64(regs, regs->orig_ax);
return;
} else if (ia32_enabled() &&
likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.l)) {
regs->orig_ax = regs->ax;
regs->ax = -ENOSYS;
do_fast_syscall_32(regs);
return;
} else {
exc_invalid_op(regs);
return;
}
}
#define SYSVEC(_vector, _function) [_vector - FIRST_SYSTEM_VECTOR] = fred_sysvec_##_function
static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = {
SYSVEC(ERROR_APIC_VECTOR, error_interrupt),
SYSVEC(SPURIOUS_APIC_VECTOR, spurious_apic_interrupt),
SYSVEC(LOCAL_TIMER_VECTOR, apic_timer_interrupt),
SYSVEC(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi),
SYSVEC(RESCHEDULE_VECTOR, reschedule_ipi),
SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, call_function_single),
SYSVEC(CALL_FUNCTION_VECTOR, call_function),
SYSVEC(REBOOT_VECTOR, reboot),
SYSVEC(THRESHOLD_APIC_VECTOR, threshold),
SYSVEC(DEFERRED_ERROR_VECTOR, deferred_error),
SYSVEC(THERMAL_APIC_VECTOR, thermal),
SYSVEC(IRQ_WORK_VECTOR, irq_work),
SYSVEC(PERF_GUEST_MEDIATED_PMI_VECTOR, perf_guest_mediated_pmi_handler),
SYSVEC(POSTED_INTR_VECTOR, kvm_posted_intr_ipi),
SYSVEC(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi),
SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi),
SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, posted_msi_notification),
};
static bool fred_setup_done __initdata;
void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler)
{
if (WARN_ON_ONCE(sysvec < FIRST_SYSTEM_VECTOR))
return;
if (WARN_ON_ONCE(fred_setup_done))
return;
if (!WARN_ON_ONCE(sysvec_table[sysvec - FIRST_SYSTEM_VECTOR]))
sysvec_table[sysvec - FIRST_SYSTEM_VECTOR] = handler;
}
static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs)
{
spurious_interrupt(regs, regs->fred_ss.vector);
}
void __init fred_complete_exception_setup(void)
{
unsigned int vector;
for (vector = 0; vector < FIRST_EXTERNAL_VECTOR; vector++)
set_bit(vector, system_vectors);
for (vector = 0; vector < NR_SYSTEM_VECTORS; vector++) {
if (sysvec_table[vector])
set_bit(vector + FIRST_SYSTEM_VECTOR, system_vectors);
else
sysvec_table[vector] = fred_handle_spurious_interrupt;
}
fred_setup_done = true;
}
static noinstr void fred_extint(struct pt_regs *regs)
{
unsigned int vector = regs->fred_ss.vector;
if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR))
return;
if (likely(vector >= FIRST_SYSTEM_VECTOR)) {
irqentry_state_t state = irqentry_enter(regs);
instrumentation_begin();
sysvec_table[array_index_nospec(vector - FIRST_SYSTEM_VECTOR,
NR_SYSTEM_VECTORS)](regs);
instrumentation_end();
irqentry_exit(regs, state);
} else {
common_interrupt(regs, vector);
}
}
static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code)
{
/* Optimize for #PF. That's the only exception which matters performance wise */
if (likely(regs->fred_ss.vector == X86_TRAP_PF))
return exc_page_fault(regs, error_code);
switch (regs->fred_ss.vector) {
case X86_TRAP_DE: return exc_divide_error(regs);
case X86_TRAP_DB: return fred_exc_debug(regs);
case X86_TRAP_BR: return exc_bounds(regs);
case X86_TRAP_UD: return exc_invalid_op(regs);
case X86_TRAP_NM: return exc_device_not_available(regs);
case X86_TRAP_DF: return exc_double_fault(regs, error_code);
case X86_TRAP_TS: return exc_invalid_tss(regs, error_code);
case X86_TRAP_NP: return exc_segment_not_present(regs, error_code);
case X86_TRAP_SS: return exc_stack_segment(regs, error_code);
case X86_TRAP_GP: return exc_general_protection(regs, error_code);
case X86_TRAP_MF: return exc_coprocessor_error(regs);
case X86_TRAP_AC: return exc_alignment_check(regs, error_code);
case X86_TRAP_XF: return exc_simd_coprocessor_error(regs);
#ifdef CONFIG_X86_MCE
case X86_TRAP_MC: return fred_exc_machine_check(regs);
#endif
#ifdef CONFIG_INTEL_TDX_GUEST
case X86_TRAP_VE: return exc_virtualization_exception(regs);
#endif
#ifdef CONFIG_X86_CET
case X86_TRAP_CP: return exc_control_protection(regs, error_code);
#endif
#ifdef CONFIG_AMD_MEM_ENCRYPT
case X86_TRAP_VC: return exc_vmm_communication(regs, error_code);
#endif
default: return fred_bad_type(regs, error_code);
}
}
static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code)
{
switch (regs->fred_ss.vector) {
case X86_TRAP_BP: return exc_int3(regs);
case X86_TRAP_OF: return exc_overflow(regs);
default: return fred_bad_type(regs, error_code);
}
}
__visible noinstr void fred_entry_from_user(struct pt_regs *regs)
{
unsigned long error_code = regs->orig_ax;
/* Invalidate orig_ax so that syscall_get_nr() works correctly */
regs->orig_ax = -1;
switch (regs->fred_ss.type) {
case EVENT_TYPE_EXTINT:
return fred_extint(regs);
case EVENT_TYPE_NMI:
if (likely(regs->fred_ss.vector == X86_TRAP_NMI))
return fred_exc_nmi(regs);
break;
case EVENT_TYPE_HWEXC:
return fred_hwexc(regs, error_code);
case EVENT_TYPE_SWINT:
return fred_intx(regs);
case EVENT_TYPE_PRIV_SWEXC:
if (likely(regs->fred_ss.vector == X86_TRAP_DB))
return fred_exc_debug(regs);
break;
case EVENT_TYPE_SWEXC:
return fred_swexc(regs, error_code);
case EVENT_TYPE_OTHER:
return fred_other(regs);
default: break;
}
return fred_bad_type(regs, error_code);
}
__visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)
{
unsigned long error_code = regs->orig_ax;
/* Invalidate orig_ax so that syscall_get_nr() works correctly */
regs->orig_ax = -1;
switch (regs->fred_ss.type) {
case EVENT_TYPE_EXTINT:
return fred_extint(regs);
case EVENT_TYPE_NMI:
if (likely(regs->fred_ss.vector == X86_TRAP_NMI))
return fred_exc_nmi(regs);
break;
case EVENT_TYPE_HWEXC:
return fred_hwexc(regs, error_code);
case EVENT_TYPE_PRIV_SWEXC:
if (likely(regs->fred_ss.vector == X86_TRAP_DB))
return fred_exc_debug(regs);
break;
case EVENT_TYPE_SWEXC:
return fred_swexc(regs, error_code);
default: break;
}
return fred_bad_type(regs, error_code);
}
#if IS_ENABLED(CONFIG_KVM_INTEL)
__visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs)
{
switch (regs->fred_ss.type) {
case EVENT_TYPE_EXTINT:
return fred_extint(regs);
case EVENT_TYPE_NMI:
return fred_exc_nmi(regs);
default:
WARN_ON_ONCE(1);
}
}
#endif
linux-7.2.3/arch/x86/entry/syscall_32.c
+371
View File
@@ -0,0 +1,371 @@
// SPDX-License-Identifier: GPL-2.0-only
/* 32-bit system call dispatch */
#include <linux/linkage.h>
#include <linux/sys.h>
#include <linux/cache.h>
#include <linux/syscalls.h>
#include <linux/entry-common.h>
#include <linux/nospec.h>
#include <linux/uaccess.h>
#include <asm/apic.h>
#include <asm/traps.h>
#include <asm/cpufeature.h>
#include <asm/syscall.h>
#ifdef CONFIG_IA32_EMULATION
#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, compat)
#else
#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native)
#endif
#define __SYSCALL(nr, sym) extern long __ia32_##sym(const struct pt_regs *);
#define __SYSCALL_NORETURN(nr, sym) extern long __noreturn __ia32_##sym(const struct pt_regs *);
#include <asm/syscalls_32.h>
#undef __SYSCALL
#undef __SYSCALL_NORETURN
#define __SYSCALL_NORETURN __SYSCALL
/*
* The sys_call_table[] is no longer used for system calls, but
* kernel/trace/trace_syscalls.c still wants to know the system
* call address.
*/
#ifdef CONFIG_X86_32
#define __SYSCALL(nr, sym) __ia32_##sym,
const sys_call_ptr_t sys_call_table[] = {
#include <asm/syscalls_32.h>
};
#undef __SYSCALL
#endif
#define __SYSCALL(nr, sym) case nr: return __ia32_##sym(regs);
long ia32_sys_call(const struct pt_regs *regs, unsigned int nr)
{
switch (nr) {
#include <asm/syscalls_32.h>
default: return __ia32_sys_ni_syscall(regs);
}
}
static __always_inline int syscall_32_enter(struct pt_regs *regs)
{
if (IS_ENABLED(CONFIG_IA32_EMULATION))
current_thread_info()->status |= TS_COMPAT;
return (int)regs->orig_ax;
}
#ifdef CONFIG_IA32_EMULATION
bool __ia32_enabled __ro_after_init = !IS_ENABLED(CONFIG_IA32_EMULATION_DEFAULT_DISABLED);
static int __init ia32_emulation_override_cmdline(char *arg)
{
return kstrtobool(arg, &__ia32_enabled);
}
early_param("ia32_emulation", ia32_emulation_override_cmdline);
#endif
/*
* Invoke a 32-bit syscall. Called with IRQs on in CT_STATE_KERNEL.
*/
static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs, int nr)
{
/*
* Convert negative numbers to very high and thus out of range
* numbers for comparisons.
*/
unsigned int unr = nr;
if (likely(unr < IA32_NR_syscalls)) {
unr = array_index_nospec(unr, IA32_NR_syscalls);
regs->ax = ia32_sys_call(regs, unr);
} else if (nr != -1) {
regs->ax = __ia32_sys_ni_syscall(regs);
}
}
#ifdef CONFIG_IA32_EMULATION
static __always_inline bool int80_is_external(void)
{
const unsigned int offs = (0x80 / 32) * 0x10;
const u32 bit = BIT(0x80 % 32);
/* The local APIC on XENPV guests is fake */
if (cpu_feature_enabled(X86_FEATURE_XENPV))
return false;
/*
* If vector 0x80 is set in the APIC ISR then this is an external
* interrupt. Either from broken hardware or injected by a VMM.
*
* Note: In guest mode this is only valid for secure guests where
* the secure module fully controls the vAPIC exposed to the guest.
*/
return apic_read(APIC_ISR + offs) & bit;
}
/**
* do_int80_emulation - 32-bit legacy syscall C entry from asm
* @regs: syscall arguments in struct pt_args on the stack.
*
* This entry point can be used by 32-bit and 64-bit programs to perform
* 32-bit system calls. Instances of INT $0x80 can be found inline in
* various programs and libraries. It is also used by the vDSO's
* __kernel_vsyscall fallback for hardware that doesn't support a faster
* entry method. Restarted 32-bit system calls also fall back to INT
* $0x80 regardless of what instruction was originally used to do the
* system call.
*
* This is considered a slow path. It is not used by most libc
* implementations on modern hardware except during process startup.
*
* The arguments for the INT $0x80 based syscall are on stack in the
* pt_regs structure:
* eax: system call number
* ebx, ecx, edx, esi, edi, ebp: arg1 - arg 6
*/
__visible noinstr void do_int80_emulation(struct pt_regs *regs)
{
int nr;
/* Kernel does not use INT $0x80! */
if (unlikely(!user_mode(regs))) {
irqentry_enter(regs);
instrumentation_begin();
panic("Unexpected external interrupt 0x80\n");
}
/*
* Establish kernel context for instrumentation, including for
* int80_is_external() below which calls into the APIC driver.
* Identical for soft and external interrupts.
*/
enter_from_user_mode(regs);
instrumentation_begin();
add_random_kstack_offset();
/* Validate that this is a soft interrupt to the extent possible */
if (unlikely(int80_is_external()))
panic("Unexpected external interrupt 0x80\n");
/*
* The low level idtentry code pushed -1 into regs::orig_ax
* and regs::ax contains the syscall number.
*
* User tracing code (ptrace or signal handlers) might assume
* that the regs::orig_ax contains a 32-bit number on invoking
* a 32-bit syscall.
*
* Establish the syscall convention by saving the 32bit truncated
* syscall number in regs::orig_ax and by invalidating regs::ax.
*/
regs->orig_ax = regs->ax & GENMASK(31, 0);
regs->ax = -ENOSYS;
nr = syscall_32_enter(regs);
local_irq_enable();
nr = syscall_enter_from_user_mode_work(regs, nr);
do_syscall_32_irqs_on(regs, nr);
instrumentation_end();
syscall_exit_to_user_mode(regs);
}
#ifdef CONFIG_X86_FRED
/*
* A FRED-specific INT80 handler is warranted for the follwing reasons:
*
* 1) As INT instructions and hardware interrupts are separate event
* types, FRED does not preclude the use of vector 0x80 for external
* interrupts. As a result, the FRED setup code does not reserve
* vector 0x80 and calling int80_is_external() is not merely
* suboptimal but actively incorrect: it could cause a system call
* to be incorrectly ignored.
*
* 2) It is called only for handling vector 0x80 of event type
* EVENT_TYPE_SWINT and will never be called to handle any external
* interrupt (event type EVENT_TYPE_EXTINT).
*
* 3) FRED has separate entry flows depending on if the event came from
* user space or kernel space, and because the kernel does not use
* INT insns, the FRED kernel entry handler fred_entry_from_kernel()
* falls through to fred_bad_type() if the event type is
* EVENT_TYPE_SWINT, i.e., INT insns. So if the kernel is handling
* an INT insn, it can only be from a user level.
*
* 4) int80_emulation() does a CLEAR_BRANCH_HISTORY. While FRED will
* likely take a different approach if it is ever needed: it
* probably belongs in either fred_intx()/ fred_other() or
* asm_fred_entrypoint_user(), depending on if this ought to be done
* for all entries from userspace or only system
* calls.
*
* 5) INT $0x80 is the fast path for 32-bit system calls under FRED.
*/
DEFINE_FREDENTRY_RAW(int80_emulation)
{
int nr;
enter_from_user_mode(regs);
instrumentation_begin();
add_random_kstack_offset();
/*
* FRED pushed 0 into regs::orig_ax and regs::ax contains the
* syscall number.
*
* User tracing code (ptrace or signal handlers) might assume
* that the regs::orig_ax contains a 32-bit number on invoking
* a 32-bit syscall.
*
* Establish the syscall convention by saving the 32bit truncated
* syscall number in regs::orig_ax and by invalidating regs::ax.
*/
regs->orig_ax = regs->ax & GENMASK(31, 0);
regs->ax = -ENOSYS;
nr = syscall_32_enter(regs);
local_irq_enable();
nr = syscall_enter_from_user_mode_work(regs, nr);
do_syscall_32_irqs_on(regs, nr);
instrumentation_end();
syscall_exit_to_user_mode(regs);
}
#endif /* CONFIG_X86_FRED */
#else /* CONFIG_IA32_EMULATION */
/* Handles int $0x80 on a 32bit kernel */
__visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
{
int nr = syscall_32_enter(regs);
/*
* Subtlety here: if ptrace pokes something larger than 2^31-1 into
* orig_ax, the int return value truncates it. This matches
* the semantics of syscall_get_nr().
*/
nr = syscall_enter_from_user_mode(regs, nr);
instrumentation_begin();
add_random_kstack_offset();
do_syscall_32_irqs_on(regs, nr);
instrumentation_end();
syscall_exit_to_user_mode(regs);
}
#endif /* !CONFIG_IA32_EMULATION */
static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
{
int nr = syscall_32_enter(regs);
int res;
/*
* This cannot use syscall_enter_from_user_mode() as it has to
* fetch EBP before invoking any of the syscall entry work
* functions.
*/
enter_from_user_mode(regs);
instrumentation_begin();
add_random_kstack_offset();
local_irq_enable();
/* Fetch EBP from where the vDSO stashed it. */
if (IS_ENABLED(CONFIG_X86_64)) {
/*
* Micro-optimization: the pointer we're following is
* explicitly 32 bits, so it can't be out of range.
*/
res = __get_user(*(u32 *)&regs->bp,
(u32 __user __force *)(unsigned long)(u32)regs->sp);
} else {
res = get_user(*(u32 *)&regs->bp,
(u32 __user __force *)(unsigned long)(u32)regs->sp);
}
if (res) {
/* User code screwed up. */
regs->ax = -EFAULT;
local_irq_disable();
instrumentation_end();
irqentry_exit_to_user_mode(regs);
return false;
}
nr = syscall_enter_from_user_mode_work(regs, nr);
/* Now this is just like a normal syscall. */
do_syscall_32_irqs_on(regs, nr);
instrumentation_end();
syscall_exit_to_user_mode(regs);
return true;
}
/* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */
__visible noinstr bool do_fast_syscall_32(struct pt_regs *regs)
{
/*
* Called using the internal vDSO SYSENTER/SYSCALL32 calling
* convention. Adjust regs so it looks like we entered using int80.
*/
unsigned long landing_pad = (unsigned long)current->mm->context.vdso +
vdso32_image.sym_int80_landing_pad;
/*
* SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward
* so that 'regs->ip -= 2' lands back on an int $0x80 instruction.
* Fix it up.
*/
regs->ip = landing_pad;
/* Invoke the syscall. If it failed, keep it simple: use IRET. */
if (!__do_fast_syscall_32(regs))
return false;
/*
* Check that the register state is valid for using SYSRETL/SYSEXIT
* to exit to userspace. Otherwise use the slower but fully capable
* IRET exit path.
*/
/* XEN PV guests always use the IRET path */
if (cpu_feature_enabled(X86_FEATURE_XENPV))
return false;
/* EIP must point to the VDSO landing pad */
if (unlikely(regs->ip != landing_pad))
return false;
/* CS and SS must match the values set in MSR_STAR */
if (unlikely(regs->cs != __USER32_CS || regs->ss != __USER_DS))
return false;
/* If the TF, RF, or VM flags are set, use IRET */
if (unlikely(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF | X86_EFLAGS_VM)))
return false;
/* Use SYSRETL/SYSEXIT to exit to userspace */
return true;
}
/* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */
__visible noinstr bool do_SYSENTER_32(struct pt_regs *regs)
{
/* SYSENTER loses RSP, but the vDSO saved it in RBP. */
regs->sp = regs->bp;
/* SYSENTER clobbers EFLAGS.IF. Assume it was set in usermode. */
regs->flags |= X86_EFLAGS_IF;
return do_fast_syscall_32(regs);
}
linux-7.2.3/arch/x86/entry/syscall_64.c
+141
View File
@@ -0,0 +1,141 @@
// SPDX-License-Identifier: GPL-2.0-only
/* 64-bit system call dispatch */
#include <linux/linkage.h>
#include <linux/sys.h>
#include <linux/cache.h>
#include <linux/syscalls.h>
#include <linux/entry-common.h>
#include <linux/nospec.h>
#include <asm/syscall.h>
#define __SYSCALL(nr, sym) extern long __x64_##sym(const struct pt_regs *);
#define __SYSCALL_NORETURN(nr, sym) extern long __noreturn __x64_##sym(const struct pt_regs *);
#include <asm/syscalls_64.h>
#ifdef CONFIG_X86_X32_ABI
#include <asm/syscalls_x32.h>
#endif
#undef __SYSCALL
#undef __SYSCALL_NORETURN
#define __SYSCALL_NORETURN __SYSCALL
/*
* The sys_call_table[] is no longer used for system calls, but
* kernel/trace/trace_syscalls.c still wants to know the system
* call address.
*/
#define __SYSCALL(nr, sym) __x64_##sym,
const sys_call_ptr_t sys_call_table[] = {
#include <asm/syscalls_64.h>
};
#undef __SYSCALL
#define __SYSCALL(nr, sym) case nr: return __x64_##sym(regs);
long x64_sys_call(const struct pt_regs *regs, unsigned int nr)
{
switch (nr) {
#include <asm/syscalls_64.h>
default: return __x64_sys_ni_syscall(regs);
}
}
#ifdef CONFIG_X86_X32_ABI
long x32_sys_call(const struct pt_regs *regs, unsigned int nr)
{
switch (nr) {
#include <asm/syscalls_x32.h>
default: return __x64_sys_ni_syscall(regs);
}
}
#endif
static __always_inline bool do_syscall_x64(struct pt_regs *regs, int nr)
{
/*
* Convert negative numbers to very high and thus out of range
* numbers for comparisons.
*/
unsigned int unr = nr;
if (likely(unr < NR_syscalls)) {
unr = array_index_nospec(unr, NR_syscalls);
regs->ax = x64_sys_call(regs, unr);
return true;
}
return false;
}
static __always_inline bool do_syscall_x32(struct pt_regs *regs, int nr)
{
/*
* Adjust the starting offset of the table, and convert numbers
* < __X32_SYSCALL_BIT to very high and thus out of range
* numbers for comparisons.
*/
unsigned int xnr = nr - __X32_SYSCALL_BIT;
if (IS_ENABLED(CONFIG_X86_X32_ABI) && likely(xnr < X32_NR_syscalls)) {
xnr = array_index_nospec(xnr, X32_NR_syscalls);
regs->ax = x32_sys_call(regs, xnr);
return true;
}
return false;
}
/* Returns true to return using SYSRET, or false to use IRET */
__visible noinstr bool do_syscall_64(struct pt_regs *regs, int nr)
{
nr = syscall_enter_from_user_mode(regs, nr);
instrumentation_begin();
add_random_kstack_offset();
if (!do_syscall_x64(regs, nr) && !do_syscall_x32(regs, nr) && nr != -1) {
/* Invalid system call, but still a system call. */
regs->ax = __x64_sys_ni_syscall(regs);
}
instrumentation_end();
syscall_exit_to_user_mode(regs);
/*
* Check that the register state is valid for using SYSRET to exit
* to userspace. Otherwise use the slower but fully capable IRET
* exit path.
*/
/* XEN PV guests always use the IRET path */
if (cpu_feature_enabled(X86_FEATURE_XENPV))
return false;
/* SYSRET requires RCX == RIP and R11 == EFLAGS */
if (unlikely(regs->cx != regs->ip || regs->r11 != regs->flags))
return false;
/* CS and SS must match the values set in MSR_STAR */
if (unlikely(regs->cs != __USER_CS || regs->ss != __USER_DS))
return false;
/*
* On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
* in kernel space. This essentially lets the user take over
* the kernel, since userspace controls RSP.
*
* TASK_SIZE_MAX covers all user-accessible addresses other than
* the deprecated vsyscall page.
*/
if (unlikely(regs->ip >= TASK_SIZE_MAX))
return false;
/*
* SYSRET cannot restore RF. It can restore TF, but unlike IRET,
* restoring TF results in a trap from userspace immediately after
* SYSRET.
*/
if (unlikely(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF)))
return false;
/* Use SYSRET to exit to userspace */
return true;
}
linux-7.2.3/arch/x86/entry/syscalls/Makefile
+78
View File
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: GPL-2.0
out := arch/$(SRCARCH)/include/generated/asm
uapi := arch/$(SRCARCH)/include/generated/uapi/asm
# Create output directory if not already present
$(shell mkdir -p $(out) $(uapi))
syscall32 := $(src)/syscall_32.tbl
syscall64 := $(src)/syscall_64.tbl
syshdr := $(srctree)/scripts/syscallhdr.sh
systbl := $(srctree)/scripts/syscalltbl.sh
offset :=
prefix :=
quiet_cmd_syshdr = SYSHDR $@
cmd_syshdr = $(CONFIG_SHELL) $(syshdr) --abis $(abis) --emit-nr \
$(if $(offset),--offset $(offset)) \
$(if $(prefix),--prefix $(prefix)) \
$< $@
quiet_cmd_systbl = SYSTBL $@
cmd_systbl = $(CONFIG_SHELL) $(systbl) --abis $(abis) $< $@
quiet_cmd_hypercalls = HYPERCALLS $@
cmd_hypercalls = $(CONFIG_SHELL) '$<' $@ $(filter-out $<, $(real-prereqs))
$(uapi)/unistd_32.h: abis := i386
$(uapi)/unistd_32.h: $(syscall32) $(syshdr) FORCE
$(call if_changed,syshdr)
$(out)/unistd_32_ia32.h: abis := i386
$(out)/unistd_32_ia32.h: prefix := ia32_
$(out)/unistd_32_ia32.h: $(syscall32) $(syshdr) FORCE
$(call if_changed,syshdr)
$(uapi)/unistd_x32.h: abis := common,x32
$(uapi)/unistd_x32.h: offset := __X32_SYSCALL_BIT
$(uapi)/unistd_x32.h: $(syscall64) $(syshdr) FORCE
$(call if_changed,syshdr)
$(uapi)/unistd_64.h: abis := common,64
$(uapi)/unistd_64.h: $(syscall64) $(syshdr) FORCE
$(call if_changed,syshdr)
$(out)/unistd_64_x32.h: abis := x32
$(out)/unistd_64_x32.h: prefix := x32_
$(out)/unistd_64_x32.h: $(syscall64) $(syshdr) FORCE
$(call if_changed,syshdr)
$(out)/syscalls_32.h: abis := i386
$(out)/syscalls_32.h: $(syscall32) $(systbl) FORCE
$(call if_changed,systbl)
$(out)/syscalls_64.h: abis := common,64
$(out)/syscalls_64.h: $(syscall64) $(systbl) FORCE
$(call if_changed,systbl)
$(out)/syscalls_x32.h: abis := common,x32
$(out)/syscalls_x32.h: $(syscall64) $(systbl) FORCE
$(call if_changed,systbl)
$(out)/xen-hypercalls.h: $(srctree)/scripts/xen-hypercalls.sh FORCE
$(call if_changed,hypercalls)
$(out)/xen-hypercalls.h: $(srctree)/include/xen/interface/xen*.h
uapisyshdr-y += unistd_32.h unistd_64.h unistd_x32.h
syshdr-y += syscalls_32.h
syshdr-$(CONFIG_X86_64) += unistd_32_ia32.h unistd_64_x32.h
syshdr-$(CONFIG_X86_64) += syscalls_64.h
syshdr-$(CONFIG_X86_X32_ABI) += syscalls_x32.h
syshdr-$(CONFIG_XEN) += xen-hypercalls.h
uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y))
syshdr-y := $(addprefix $(out)/, $(syshdr-y))
targets += $(addprefix ../../../../, $(uapisyshdr-y) $(syshdr-y))
PHONY += all
all: $(uapisyshdr-y) $(syshdr-y)
@:
linux-7.2.3/arch/x86/entry/syscalls/syscall_32.tbl
+479
View File
@@ -0,0 +1,479 @@
# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
#
# 32-bit system call numbers and entry vectors
#
# The format is:
# <number> <abi> <name> <entry point> [<compat entry point> [noreturn]]
#
# The __ia32_sys and __ia32_compat_sys stubs are created on-the-fly for
# sys_*() system calls and compat_sys_*() compat system calls if
# IA32_EMULATION is defined, and expect struct pt_regs *regs as their only
# parameter.
#
# The abi is always "i386" for this file.
#
0 i386 restart_syscall sys_restart_syscall
1 i386 exit sys_exit - noreturn
2 i386 fork sys_fork
3 i386 read sys_read
4 i386 write sys_write
5 i386 open sys_open compat_sys_open
6 i386 close sys_close
7 i386 waitpid sys_waitpid
8 i386 creat sys_creat
9 i386 link sys_link
10 i386 unlink sys_unlink
11 i386 execve sys_execve compat_sys_execve
12 i386 chdir sys_chdir
13 i386 time sys_time32
14 i386 mknod sys_mknod
15 i386 chmod sys_chmod
16 i386 lchown sys_lchown16
17 i386 break
18 i386 oldstat sys_stat
19 i386 lseek sys_lseek compat_sys_lseek
20 i386 getpid sys_getpid
21 i386 mount sys_mount
22 i386 umount sys_oldumount
23 i386 setuid sys_setuid16
24 i386 getuid sys_getuid16
25 i386 stime sys_stime32
26 i386 ptrace sys_ptrace compat_sys_ptrace
27 i386 alarm sys_alarm
28 i386 oldfstat sys_fstat
29 i386 pause sys_pause
30 i386 utime sys_utime32
31 i386 stty
32 i386 gtty
33 i386 access sys_access
34 i386 nice sys_nice
35 i386 ftime
36 i386 sync sys_sync
37 i386 kill sys_kill
38 i386 rename sys_rename
39 i386 mkdir sys_mkdir
40 i386 rmdir sys_rmdir
41 i386 dup sys_dup
42 i386 pipe sys_pipe
43 i386 times sys_times compat_sys_times
44 i386 prof
45 i386 brk sys_brk
46 i386 setgid sys_setgid16
47 i386 getgid sys_getgid16
48 i386 signal sys_signal
49 i386 geteuid sys_geteuid16
50 i386 getegid sys_getegid16
51 i386 acct sys_acct
52 i386 umount2 sys_umount
53 i386 lock
54 i386 ioctl sys_ioctl compat_sys_ioctl
55 i386 fcntl sys_fcntl compat_sys_fcntl64
56 i386 mpx
57 i386 setpgid sys_setpgid
58 i386 ulimit
59 i386 oldolduname sys_olduname
60 i386 umask sys_umask
61 i386 chroot sys_chroot
62 i386 ustat sys_ustat compat_sys_ustat
63 i386 dup2 sys_dup2
64 i386 getppid sys_getppid
65 i386 getpgrp sys_getpgrp
66 i386 setsid sys_setsid
67 i386 sigaction sys_sigaction compat_sys_sigaction
68 i386 sgetmask sys_sgetmask
69 i386 ssetmask sys_ssetmask
70 i386 setreuid sys_setreuid16
71 i386 setregid sys_setregid16
72 i386 sigsuspend sys_sigsuspend
73 i386 sigpending sys_sigpending compat_sys_sigpending
74 i386 sethostname sys_sethostname
75 i386 setrlimit sys_setrlimit compat_sys_setrlimit
76 i386 getrlimit sys_old_getrlimit compat_sys_old_getrlimit
77 i386 getrusage sys_getrusage compat_sys_getrusage
78 i386 gettimeofday sys_gettimeofday compat_sys_gettimeofday
79 i386 settimeofday sys_settimeofday compat_sys_settimeofday
80 i386 getgroups sys_getgroups16
81 i386 setgroups sys_setgroups16
82 i386 select sys_old_select compat_sys_old_select
83 i386 symlink sys_symlink
84 i386 oldlstat sys_lstat
85 i386 readlink sys_readlink
86 i386 uselib sys_uselib
87 i386 swapon sys_swapon
88 i386 reboot sys_reboot
89 i386 readdir sys_old_readdir compat_sys_old_readdir
90 i386 mmap sys_old_mmap compat_sys_ia32_mmap
91 i386 munmap sys_munmap
92 i386 truncate sys_truncate compat_sys_truncate
93 i386 ftruncate sys_ftruncate compat_sys_ftruncate
94 i386 fchmod sys_fchmod
95 i386 fchown sys_fchown16
96 i386 getpriority sys_getpriority
97 i386 setpriority sys_setpriority
98 i386 profil
99 i386 statfs sys_statfs compat_sys_statfs
100 i386 fstatfs sys_fstatfs compat_sys_fstatfs
101 i386 ioperm sys_ioperm
102 i386 socketcall sys_socketcall compat_sys_socketcall
103 i386 syslog sys_syslog
104 i386 setitimer sys_setitimer compat_sys_setitimer
105 i386 getitimer sys_getitimer compat_sys_getitimer
106 i386 stat sys_newstat compat_sys_newstat
107 i386 lstat sys_newlstat compat_sys_newlstat
108 i386 fstat sys_newfstat compat_sys_newfstat
109 i386 olduname sys_uname
110 i386 iopl sys_iopl
111 i386 vhangup sys_vhangup
112 i386 idle
113 i386 vm86old sys_vm86old sys_ni_syscall
114 i386 wait4 sys_wait4 compat_sys_wait4
115 i386 swapoff sys_swapoff
116 i386 sysinfo sys_sysinfo compat_sys_sysinfo
117 i386 ipc sys_ipc compat_sys_ipc
118 i386 fsync sys_fsync
119 i386 sigreturn sys_sigreturn compat_sys_sigreturn
120 i386 clone sys_clone compat_sys_ia32_clone
121 i386 setdomainname sys_setdomainname
122 i386 uname sys_newuname
123 i386 modify_ldt sys_modify_ldt
124 i386 adjtimex sys_adjtimex_time32
125 i386 mprotect sys_mprotect
126 i386 sigprocmask sys_sigprocmask compat_sys_sigprocmask
127 i386 create_module
128 i386 init_module sys_init_module
129 i386 delete_module sys_delete_module
130 i386 get_kernel_syms
131 i386 quotactl sys_quotactl
132 i386 getpgid sys_getpgid
133 i386 fchdir sys_fchdir
134 i386 bdflush sys_ni_syscall
135 i386 sysfs sys_sysfs
136 i386 personality sys_personality
137 i386 afs_syscall
138 i386 setfsuid sys_setfsuid16
139 i386 setfsgid sys_setfsgid16
140 i386 _llseek sys_llseek
141 i386 getdents sys_getdents compat_sys_getdents
142 i386 _newselect sys_select compat_sys_select
143 i386 flock sys_flock
144 i386 msync sys_msync
145 i386 readv sys_readv
146 i386 writev sys_writev
147 i386 getsid sys_getsid
148 i386 fdatasync sys_fdatasync
149 i386 _sysctl sys_ni_syscall
150 i386 mlock sys_mlock
151 i386 munlock sys_munlock
152 i386 mlockall sys_mlockall
153 i386 munlockall sys_munlockall
154 i386 sched_setparam sys_sched_setparam
155 i386 sched_getparam sys_sched_getparam
156 i386 sched_setscheduler sys_sched_setscheduler
157 i386 sched_getscheduler sys_sched_getscheduler
158 i386 sched_yield sys_sched_yield
159 i386 sched_get_priority_max sys_sched_get_priority_max
160 i386 sched_get_priority_min sys_sched_get_priority_min
161 i386 sched_rr_get_interval sys_sched_rr_get_interval_time32
162 i386 nanosleep sys_nanosleep_time32
163 i386 mremap sys_mremap
164 i386 setresuid sys_setresuid16
165 i386 getresuid sys_getresuid16
166 i386 vm86 sys_vm86 sys_ni_syscall
167 i386 query_module
168 i386 poll sys_poll
169 i386 nfsservctl
170 i386 setresgid sys_setresgid16
171 i386 getresgid sys_getresgid16
172 i386 prctl sys_prctl
173 i386 rt_sigreturn sys_rt_sigreturn compat_sys_rt_sigreturn
174 i386 rt_sigaction sys_rt_sigaction compat_sys_rt_sigaction
175 i386 rt_sigprocmask sys_rt_sigprocmask compat_sys_rt_sigprocmask
176 i386 rt_sigpending sys_rt_sigpending compat_sys_rt_sigpending
177 i386 rt_sigtimedwait sys_rt_sigtimedwait_time32 compat_sys_rt_sigtimedwait_time32
178 i386 rt_sigqueueinfo sys_rt_sigqueueinfo compat_sys_rt_sigqueueinfo
179 i386 rt_sigsuspend sys_rt_sigsuspend compat_sys_rt_sigsuspend
180 i386 pread64 sys_ia32_pread64
181 i386 pwrite64 sys_ia32_pwrite64
182 i386 chown sys_chown16
183 i386 getcwd sys_getcwd
184 i386 capget sys_capget
185 i386 capset sys_capset
186 i386 sigaltstack sys_sigaltstack compat_sys_sigaltstack
187 i386 sendfile sys_sendfile compat_sys_sendfile
188 i386 getpmsg
189 i386 putpmsg
190 i386 vfork sys_vfork
191 i386 ugetrlimit sys_getrlimit compat_sys_getrlimit
192 i386 mmap2 sys_mmap_pgoff
193 i386 truncate64 sys_ia32_truncate64
194 i386 ftruncate64 sys_ia32_ftruncate64
195 i386 stat64 sys_stat64 compat_sys_ia32_stat64
196 i386 lstat64 sys_lstat64 compat_sys_ia32_lstat64
197 i386 fstat64 sys_fstat64 compat_sys_ia32_fstat64
198 i386 lchown32 sys_lchown
199 i386 getuid32 sys_getuid
200 i386 getgid32 sys_getgid
201 i386 geteuid32 sys_geteuid
202 i386 getegid32 sys_getegid
203 i386 setreuid32 sys_setreuid
204 i386 setregid32 sys_setregid
205 i386 getgroups32 sys_getgroups
206 i386 setgroups32 sys_setgroups
207 i386 fchown32 sys_fchown
208 i386 setresuid32 sys_setresuid
209 i386 getresuid32 sys_getresuid
210 i386 setresgid32 sys_setresgid
211 i386 getresgid32 sys_getresgid
212 i386 chown32 sys_chown
213 i386 setuid32 sys_setuid
214 i386 setgid32 sys_setgid
215 i386 setfsuid32 sys_setfsuid
216 i386 setfsgid32 sys_setfsgid
217 i386 pivot_root sys_pivot_root
218 i386 mincore sys_mincore
219 i386 madvise sys_madvise
220 i386 getdents64 sys_getdents64
221 i386 fcntl64 sys_fcntl64 compat_sys_fcntl64
# 222 is unused
# 223 is unused
224 i386 gettid sys_gettid
225 i386 readahead sys_ia32_readahead
226 i386 setxattr sys_setxattr
227 i386 lsetxattr sys_lsetxattr
228 i386 fsetxattr sys_fsetxattr
229 i386 getxattr sys_getxattr
230 i386 lgetxattr sys_lgetxattr
231 i386 fgetxattr sys_fgetxattr
232 i386 listxattr sys_listxattr
233 i386 llistxattr sys_llistxattr
234 i386 flistxattr sys_flistxattr
235 i386 removexattr sys_removexattr
236 i386 lremovexattr sys_lremovexattr
237 i386 fremovexattr sys_fremovexattr
238 i386 tkill sys_tkill
239 i386 sendfile64 sys_sendfile64
240 i386 futex sys_futex_time32
241 i386 sched_setaffinity sys_sched_setaffinity compat_sys_sched_setaffinity
242 i386 sched_getaffinity sys_sched_getaffinity compat_sys_sched_getaffinity
243 i386 set_thread_area sys_set_thread_area
244 i386 get_thread_area sys_get_thread_area
245 i386 io_setup sys_io_setup compat_sys_io_setup
246 i386 io_destroy sys_io_destroy
247 i386 io_getevents sys_io_getevents_time32
248 i386 io_submit sys_io_submit compat_sys_io_submit
249 i386 io_cancel sys_io_cancel
250 i386 fadvise64 sys_ia32_fadvise64
# 251 is available for reuse (was briefly sys_set_zone_reclaim)
252 i386 exit_group sys_exit_group - noreturn
253 i386 lookup_dcookie
254 i386 epoll_create sys_epoll_create
255 i386 epoll_ctl sys_epoll_ctl
256 i386 epoll_wait sys_epoll_wait
257 i386 remap_file_pages sys_remap_file_pages
258 i386 set_tid_address sys_set_tid_address
259 i386 timer_create sys_timer_create compat_sys_timer_create
260 i386 timer_settime sys_timer_settime32
261 i386 timer_gettime sys_timer_gettime32
262 i386 timer_getoverrun sys_timer_getoverrun
263 i386 timer_delete sys_timer_delete
264 i386 clock_settime sys_clock_settime32
265 i386 clock_gettime sys_clock_gettime32
266 i386 clock_getres sys_clock_getres_time32
267 i386 clock_nanosleep sys_clock_nanosleep_time32
268 i386 statfs64 sys_statfs64 compat_sys_statfs64
269 i386 fstatfs64 sys_fstatfs64 compat_sys_fstatfs64
270 i386 tgkill sys_tgkill
271 i386 utimes sys_utimes_time32
272 i386 fadvise64_64 sys_ia32_fadvise64_64
273 i386 vserver
274 i386 mbind sys_mbind
275 i386 get_mempolicy sys_get_mempolicy
276 i386 set_mempolicy sys_set_mempolicy
277 i386 mq_open sys_mq_open compat_sys_mq_open
278 i386 mq_unlink sys_mq_unlink
279 i386 mq_timedsend sys_mq_timedsend_time32
280 i386 mq_timedreceive sys_mq_timedreceive_time32
281 i386 mq_notify sys_mq_notify compat_sys_mq_notify
282 i386 mq_getsetattr sys_mq_getsetattr compat_sys_mq_getsetattr
283 i386 kexec_load sys_kexec_load compat_sys_kexec_load
284 i386 waitid sys_waitid compat_sys_waitid
# 285 sys_setaltroot
286 i386 add_key sys_add_key
287 i386 request_key sys_request_key
288 i386 keyctl sys_keyctl compat_sys_keyctl
289 i386 ioprio_set sys_ioprio_set
290 i386 ioprio_get sys_ioprio_get
291 i386 inotify_init sys_inotify_init
292 i386 inotify_add_watch sys_inotify_add_watch
293 i386 inotify_rm_watch sys_inotify_rm_watch
294 i386 migrate_pages sys_migrate_pages
295 i386 openat sys_openat compat_sys_openat
296 i386 mkdirat sys_mkdirat
297 i386 mknodat sys_mknodat
298 i386 fchownat sys_fchownat
299 i386 futimesat sys_futimesat_time32
300 i386 fstatat64 sys_fstatat64 compat_sys_ia32_fstatat64
301 i386 unlinkat sys_unlinkat
302 i386 renameat sys_renameat
303 i386 linkat sys_linkat
304 i386 symlinkat sys_symlinkat
305 i386 readlinkat sys_readlinkat
306 i386 fchmodat sys_fchmodat
307 i386 faccessat sys_faccessat
308 i386 pselect6 sys_pselect6_time32 compat_sys_pselect6_time32
309 i386 ppoll sys_ppoll_time32 compat_sys_ppoll_time32
310 i386 unshare sys_unshare
311 i386 set_robust_list sys_set_robust_list compat_sys_set_robust_list
312 i386 get_robust_list sys_get_robust_list compat_sys_get_robust_list
313 i386 splice sys_splice
314 i386 sync_file_range sys_ia32_sync_file_range
315 i386 tee sys_tee
316 i386 vmsplice sys_vmsplice
317 i386 move_pages sys_move_pages
318 i386 getcpu sys_getcpu
319 i386 epoll_pwait sys_epoll_pwait
320 i386 utimensat sys_utimensat_time32
321 i386 signalfd sys_signalfd compat_sys_signalfd
322 i386 timerfd_create sys_timerfd_create
323 i386 eventfd sys_eventfd
324 i386 fallocate sys_ia32_fallocate
325 i386 timerfd_settime sys_timerfd_settime32
326 i386 timerfd_gettime sys_timerfd_gettime32
327 i386 signalfd4 sys_signalfd4 compat_sys_signalfd4
328 i386 eventfd2 sys_eventfd2
329 i386 epoll_create1 sys_epoll_create1
330 i386 dup3 sys_dup3
331 i386 pipe2 sys_pipe2
332 i386 inotify_init1 sys_inotify_init1
333 i386 preadv sys_preadv compat_sys_preadv
334 i386 pwritev sys_pwritev compat_sys_pwritev
335 i386 rt_tgsigqueueinfo sys_rt_tgsigqueueinfo compat_sys_rt_tgsigqueueinfo
336 i386 perf_event_open sys_perf_event_open
337 i386 recvmmsg sys_recvmmsg_time32 compat_sys_recvmmsg_time32
338 i386 fanotify_init sys_fanotify_init
339 i386 fanotify_mark sys_fanotify_mark compat_sys_fanotify_mark
340 i386 prlimit64 sys_prlimit64
341 i386 name_to_handle_at sys_name_to_handle_at
342 i386 open_by_handle_at sys_open_by_handle_at compat_sys_open_by_handle_at
343 i386 clock_adjtime sys_clock_adjtime32
344 i386 syncfs sys_syncfs
345 i386 sendmmsg sys_sendmmsg compat_sys_sendmmsg
346 i386 setns sys_setns
347 i386 process_vm_readv sys_process_vm_readv
348 i386 process_vm_writev sys_process_vm_writev
349 i386 kcmp sys_kcmp
350 i386 finit_module sys_finit_module
351 i386 sched_setattr sys_sched_setattr
352 i386 sched_getattr sys_sched_getattr
353 i386 renameat2 sys_renameat2
354 i386 seccomp sys_seccomp
355 i386 getrandom sys_getrandom
356 i386 memfd_create sys_memfd_create
357 i386 bpf sys_bpf
358 i386 execveat sys_execveat compat_sys_execveat
359 i386 socket sys_socket
360 i386 socketpair sys_socketpair
361 i386 bind sys_bind
362 i386 connect sys_connect
363 i386 listen sys_listen
364 i386 accept4 sys_accept4
365 i386 getsockopt sys_getsockopt sys_getsockopt
366 i386 setsockopt sys_setsockopt sys_setsockopt
367 i386 getsockname sys_getsockname
368 i386 getpeername sys_getpeername
369 i386 sendto sys_sendto
370 i386 sendmsg sys_sendmsg compat_sys_sendmsg
371 i386 recvfrom sys_recvfrom compat_sys_recvfrom
372 i386 recvmsg sys_recvmsg compat_sys_recvmsg
373 i386 shutdown sys_shutdown
374 i386 userfaultfd sys_userfaultfd
375 i386 membarrier sys_membarrier
376 i386 mlock2 sys_mlock2
377 i386 copy_file_range sys_copy_file_range
378 i386 preadv2 sys_preadv2 compat_sys_preadv2
379 i386 pwritev2 sys_pwritev2 compat_sys_pwritev2
380 i386 pkey_mprotect sys_pkey_mprotect
381 i386 pkey_alloc sys_pkey_alloc
382 i386 pkey_free sys_pkey_free
383 i386 statx sys_statx
384 i386 arch_prctl sys_arch_prctl
385 i386 io_pgetevents sys_io_pgetevents_time32 compat_sys_io_pgetevents
386 i386 rseq sys_rseq
393 i386 semget sys_semget
394 i386 semctl sys_semctl compat_sys_semctl
395 i386 shmget sys_shmget
396 i386 shmctl sys_shmctl compat_sys_shmctl
397 i386 shmat sys_shmat compat_sys_shmat
398 i386 shmdt sys_shmdt
399 i386 msgget sys_msgget
400 i386 msgsnd sys_msgsnd compat_sys_msgsnd
401 i386 msgrcv sys_msgrcv compat_sys_msgrcv
402 i386 msgctl sys_msgctl compat_sys_msgctl
403 i386 clock_gettime64 sys_clock_gettime
404 i386 clock_settime64 sys_clock_settime
405 i386 clock_adjtime64 sys_clock_adjtime
406 i386 clock_getres_time64 sys_clock_getres
407 i386 clock_nanosleep_time64 sys_clock_nanosleep
408 i386 timer_gettime64 sys_timer_gettime
409 i386 timer_settime64 sys_timer_settime
410 i386 timerfd_gettime64 sys_timerfd_gettime
411 i386 timerfd_settime64 sys_timerfd_settime
412 i386 utimensat_time64 sys_utimensat
413 i386 pselect6_time64 sys_pselect6 compat_sys_pselect6_time64
414 i386 ppoll_time64 sys_ppoll compat_sys_ppoll_time64
416 i386 io_pgetevents_time64 sys_io_pgetevents compat_sys_io_pgetevents_time64
417 i386 recvmmsg_time64 sys_recvmmsg compat_sys_recvmmsg_time64
418 i386 mq_timedsend_time64 sys_mq_timedsend
419 i386 mq_timedreceive_time64 sys_mq_timedreceive
420 i386 semtimedop_time64 sys_semtimedop
421 i386 rt_sigtimedwait_time64 sys_rt_sigtimedwait compat_sys_rt_sigtimedwait_time64
422 i386 futex_time64 sys_futex
423 i386 sched_rr_get_interval_time64 sys_sched_rr_get_interval
424 i386 pidfd_send_signal sys_pidfd_send_signal
425 i386 io_uring_setup sys_io_uring_setup
426 i386 io_uring_enter sys_io_uring_enter
427 i386 io_uring_register sys_io_uring_register
428 i386 open_tree sys_open_tree
429 i386 move_mount sys_move_mount
430 i386 fsopen sys_fsopen
431 i386 fsconfig sys_fsconfig
432 i386 fsmount sys_fsmount
433 i386 fspick sys_fspick
434 i386 pidfd_open sys_pidfd_open
435 i386 clone3 sys_clone3
436 i386 close_range sys_close_range
437 i386 openat2 sys_openat2
438 i386 pidfd_getfd sys_pidfd_getfd
439 i386 faccessat2 sys_faccessat2
440 i386 process_madvise sys_process_madvise
441 i386 epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2
442 i386 mount_setattr sys_mount_setattr
443 i386 quotactl_fd sys_quotactl_fd
444 i386 landlock_create_ruleset sys_landlock_create_ruleset
445 i386 landlock_add_rule sys_landlock_add_rule
446 i386 landlock_restrict_self sys_landlock_restrict_self
447 i386 memfd_secret sys_memfd_secret
448 i386 process_mrelease sys_process_mrelease
449 i386 futex_waitv sys_futex_waitv
450 i386 set_mempolicy_home_node sys_set_mempolicy_home_node
451 i386 cachestat sys_cachestat
452 i386 fchmodat2 sys_fchmodat2
453 i386 map_shadow_stack sys_map_shadow_stack
454 i386 futex_wake sys_futex_wake
455 i386 futex_wait sys_futex_wait
456 i386 futex_requeue sys_futex_requeue
457 i386 statmount sys_statmount
458 i386 listmount sys_listmount
459 i386 lsm_get_self_attr sys_lsm_get_self_attr
460 i386 lsm_set_self_attr sys_lsm_set_self_attr
461 i386 lsm_list_modules sys_lsm_list_modules
462 i386 mseal sys_mseal
463 i386 setxattrat sys_setxattrat
464 i386 getxattrat sys_getxattrat
465 i386 listxattrat sys_listxattrat
466 i386 removexattrat sys_removexattrat
467 i386 open_tree_attr sys_open_tree_attr
468 i386 file_getattr sys_file_getattr
469 i386 file_setattr sys_file_setattr
470 i386 listns sys_listns
471 i386 rseq_slice_yield sys_rseq_slice_yield
linux-7.2.3/arch/x86/entry/syscalls/syscall_64.tbl
+443
View File
@@ -0,0 +1,443 @@
# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
#
# 64-bit system call numbers and entry vectors
#
# The format is:
# <number> <abi> <name> <entry point> [<compat entry point> [noreturn]]
#
# The __x64_sys_*() stubs are created on-the-fly for sys_*() system calls
#
# The abi is "common", "64" or "x32" for this file.
#
0 common read sys_read
1 common write sys_write
2 common open sys_open
3 common close sys_close
4 common stat sys_newstat
5 common fstat sys_newfstat
6 common lstat sys_newlstat
7 common poll sys_poll
8 common lseek sys_lseek
9 common mmap sys_mmap
10 common mprotect sys_mprotect
11 common munmap sys_munmap
12 common brk sys_brk
13 64 rt_sigaction sys_rt_sigaction
14 common rt_sigprocmask sys_rt_sigprocmask
15 64 rt_sigreturn sys_rt_sigreturn
16 64 ioctl sys_ioctl
17 common pread64 sys_pread64
18 common pwrite64 sys_pwrite64
19 64 readv sys_readv
20 64 writev sys_writev
21 common access sys_access
22 common pipe sys_pipe
23 common select sys_select
24 common sched_yield sys_sched_yield
25 common mremap sys_mremap
26 common msync sys_msync
27 common mincore sys_mincore
28 common madvise sys_madvise
29 common shmget sys_shmget
30 common shmat sys_shmat
31 common shmctl sys_shmctl
32 common dup sys_dup
33 common dup2 sys_dup2
34 common pause sys_pause
35 common nanosleep sys_nanosleep
36 common getitimer sys_getitimer
37 common alarm sys_alarm
38 common setitimer sys_setitimer
39 common getpid sys_getpid
40 common sendfile sys_sendfile64
41 common socket sys_socket
42 common connect sys_connect
43 common accept sys_accept
44 common sendto sys_sendto
45 64 recvfrom sys_recvfrom
46 64 sendmsg sys_sendmsg
47 64 recvmsg sys_recvmsg
48 common shutdown sys_shutdown
49 common bind sys_bind
50 common listen sys_listen
51 common getsockname sys_getsockname
52 common getpeername sys_getpeername
53 common socketpair sys_socketpair
54 64 setsockopt sys_setsockopt
55 64 getsockopt sys_getsockopt
56 common clone sys_clone
57 common fork sys_fork
58 common vfork sys_vfork
59 64 execve sys_execve
60 common exit sys_exit - noreturn
61 common wait4 sys_wait4
62 common kill sys_kill
63 common uname sys_newuname
64 common semget sys_semget
65 common semop sys_semop
66 common semctl sys_semctl
67 common shmdt sys_shmdt
68 common msgget sys_msgget
69 common msgsnd sys_msgsnd
70 common msgrcv sys_msgrcv
71 common msgctl sys_msgctl
72 common fcntl sys_fcntl
73 common flock sys_flock
74 common fsync sys_fsync
75 common fdatasync sys_fdatasync
76 common truncate sys_truncate
77 common ftruncate sys_ftruncate
78 common getdents sys_getdents
79 common getcwd sys_getcwd
80 common chdir sys_chdir
81 common fchdir sys_fchdir
82 common rename sys_rename
83 common mkdir sys_mkdir
84 common rmdir sys_rmdir
85 common creat sys_creat
86 common link sys_link
87 common unlink sys_unlink
88 common symlink sys_symlink
89 common readlink sys_readlink
90 common chmod sys_chmod
91 common fchmod sys_fchmod
92 common chown sys_chown
93 common fchown sys_fchown
94 common lchown sys_lchown
95 common umask sys_umask
96 common gettimeofday sys_gettimeofday
97 common getrlimit sys_getrlimit
98 common getrusage sys_getrusage
99 common sysinfo sys_sysinfo
100 common times sys_times
101 64 ptrace sys_ptrace
102 common getuid sys_getuid
103 common syslog sys_syslog
104 common getgid sys_getgid
105 common setuid sys_setuid
106 common setgid sys_setgid
107 common geteuid sys_geteuid
108 common getegid sys_getegid
109 common setpgid sys_setpgid
110 common getppid sys_getppid
111 common getpgrp sys_getpgrp
112 common setsid sys_setsid
113 common setreuid sys_setreuid
114 common setregid sys_setregid
115 common getgroups sys_getgroups
116 common setgroups sys_setgroups
117 common setresuid sys_setresuid
118 common getresuid sys_getresuid
119 common setresgid sys_setresgid
120 common getresgid sys_getresgid
121 common getpgid sys_getpgid
122 common setfsuid sys_setfsuid
123 common setfsgid sys_setfsgid
124 common getsid sys_getsid
125 common capget sys_capget
126 common capset sys_capset
127 64 rt_sigpending sys_rt_sigpending
128 64 rt_sigtimedwait sys_rt_sigtimedwait
129 64 rt_sigqueueinfo sys_rt_sigqueueinfo
130 common rt_sigsuspend sys_rt_sigsuspend
131 64 sigaltstack sys_sigaltstack
132 common utime sys_utime
133 common mknod sys_mknod
134 64 uselib
135 common personality sys_personality
136 common ustat sys_ustat
137 common statfs sys_statfs
138 common fstatfs sys_fstatfs
139 common sysfs sys_sysfs
140 common getpriority sys_getpriority
141 common setpriority sys_setpriority
142 common sched_setparam sys_sched_setparam
143 common sched_getparam sys_sched_getparam
144 common sched_setscheduler sys_sched_setscheduler
145 common sched_getscheduler sys_sched_getscheduler
146 common sched_get_priority_max sys_sched_get_priority_max
147 common sched_get_priority_min sys_sched_get_priority_min
148 common sched_rr_get_interval sys_sched_rr_get_interval
149 common mlock sys_mlock
150 common munlock sys_munlock
151 common mlockall sys_mlockall
152 common munlockall sys_munlockall
153 common vhangup sys_vhangup
154 common modify_ldt sys_modify_ldt
155 common pivot_root sys_pivot_root
156 64 _sysctl sys_ni_syscall
157 common prctl sys_prctl
158 common arch_prctl sys_arch_prctl
159 common adjtimex sys_adjtimex
160 common setrlimit sys_setrlimit
161 common chroot sys_chroot
162 common sync sys_sync
163 common acct sys_acct
164 common settimeofday sys_settimeofday
165 common mount sys_mount
166 common umount2 sys_umount
167 common swapon sys_swapon
168 common swapoff sys_swapoff
169 common reboot sys_reboot
170 common sethostname sys_sethostname
171 common setdomainname sys_setdomainname
172 common iopl sys_iopl
173 common ioperm sys_ioperm
174 64 create_module
175 common init_module sys_init_module
176 common delete_module sys_delete_module
177 64 get_kernel_syms
178 64 query_module
179 common quotactl sys_quotactl
180 64 nfsservctl
181 common getpmsg
182 common putpmsg
183 common afs_syscall
184 common tuxcall
185 common security
186 common gettid sys_gettid
187 common readahead sys_readahead
188 common setxattr sys_setxattr
189 common lsetxattr sys_lsetxattr
190 common fsetxattr sys_fsetxattr
191 common getxattr sys_getxattr
192 common lgetxattr sys_lgetxattr
193 common fgetxattr sys_fgetxattr
194 common listxattr sys_listxattr
195 common llistxattr sys_llistxattr
196 common flistxattr sys_flistxattr
197 common removexattr sys_removexattr
198 common lremovexattr sys_lremovexattr
199 common fremovexattr sys_fremovexattr
200 common tkill sys_tkill
201 common time sys_time
202 common futex sys_futex
203 common sched_setaffinity sys_sched_setaffinity
204 common sched_getaffinity sys_sched_getaffinity
205 64 set_thread_area
206 64 io_setup sys_io_setup
207 common io_destroy sys_io_destroy
208 common io_getevents sys_io_getevents
209 64 io_submit sys_io_submit
210 common io_cancel sys_io_cancel
211 64 get_thread_area
212 common lookup_dcookie
213 common epoll_create sys_epoll_create
214 64 epoll_ctl_old
215 64 epoll_wait_old
216 common remap_file_pages sys_remap_file_pages
217 common getdents64 sys_getdents64
218 common set_tid_address sys_set_tid_address
219 common restart_syscall sys_restart_syscall
220 common semtimedop sys_semtimedop
221 common fadvise64 sys_fadvise64
222 64 timer_create sys_timer_create
223 common timer_settime sys_timer_settime
224 common timer_gettime sys_timer_gettime
225 common timer_getoverrun sys_timer_getoverrun
226 common timer_delete sys_timer_delete
227 common clock_settime sys_clock_settime
228 common clock_gettime sys_clock_gettime
229 common clock_getres sys_clock_getres
230 common clock_nanosleep sys_clock_nanosleep
231 common exit_group sys_exit_group - noreturn
232 common epoll_wait sys_epoll_wait
233 common epoll_ctl sys_epoll_ctl
234 common tgkill sys_tgkill
235 common utimes sys_utimes
236 64 vserver
237 common mbind sys_mbind
238 common set_mempolicy sys_set_mempolicy
239 common get_mempolicy sys_get_mempolicy
240 common mq_open sys_mq_open
241 common mq_unlink sys_mq_unlink
242 common mq_timedsend sys_mq_timedsend
243 common mq_timedreceive sys_mq_timedreceive
244 64 mq_notify sys_mq_notify
245 common mq_getsetattr sys_mq_getsetattr
246 64 kexec_load sys_kexec_load
247 64 waitid sys_waitid
248 common add_key sys_add_key
249 common request_key sys_request_key
250 common keyctl sys_keyctl
251 common ioprio_set sys_ioprio_set
252 common ioprio_get sys_ioprio_get
253 common inotify_init sys_inotify_init
254 common inotify_add_watch sys_inotify_add_watch
255 common inotify_rm_watch sys_inotify_rm_watch
256 common migrate_pages sys_migrate_pages
257 common openat sys_openat
258 common mkdirat sys_mkdirat
259 common mknodat sys_mknodat
260 common fchownat sys_fchownat
261 common futimesat sys_futimesat
262 common newfstatat sys_newfstatat
263 common unlinkat sys_unlinkat
264 common renameat sys_renameat
265 common linkat sys_linkat
266 common symlinkat sys_symlinkat
267 common readlinkat sys_readlinkat
268 common fchmodat sys_fchmodat
269 common faccessat sys_faccessat
270 common pselect6 sys_pselect6
271 common ppoll sys_ppoll
272 common unshare sys_unshare
273 64 set_robust_list sys_set_robust_list
274 64 get_robust_list sys_get_robust_list
275 common splice sys_splice
276 common tee sys_tee
277 common sync_file_range sys_sync_file_range
278 64 vmsplice sys_vmsplice
279 64 move_pages sys_move_pages
280 common utimensat sys_utimensat
281 common epoll_pwait sys_epoll_pwait
282 common signalfd sys_signalfd
283 common timerfd_create sys_timerfd_create
284 common eventfd sys_eventfd
285 common fallocate sys_fallocate
286 common timerfd_settime sys_timerfd_settime
287 common timerfd_gettime sys_timerfd_gettime
288 common accept4 sys_accept4
289 common signalfd4 sys_signalfd4
290 common eventfd2 sys_eventfd2
291 common epoll_create1 sys_epoll_create1
292 common dup3 sys_dup3
293 common pipe2 sys_pipe2
294 common inotify_init1 sys_inotify_init1
295 64 preadv sys_preadv
296 64 pwritev sys_pwritev
297 64 rt_tgsigqueueinfo sys_rt_tgsigqueueinfo
298 common perf_event_open sys_perf_event_open
299 64 recvmmsg sys_recvmmsg
300 common fanotify_init sys_fanotify_init
301 common fanotify_mark sys_fanotify_mark
302 common prlimit64 sys_prlimit64
303 common name_to_handle_at sys_name_to_handle_at
304 common open_by_handle_at sys_open_by_handle_at
305 common clock_adjtime sys_clock_adjtime
306 common syncfs sys_syncfs
307 64 sendmmsg sys_sendmmsg
308 common setns sys_setns
309 common getcpu sys_getcpu
310 64 process_vm_readv sys_process_vm_readv
311 64 process_vm_writev sys_process_vm_writev
312 common kcmp sys_kcmp
313 common finit_module sys_finit_module
314 common sched_setattr sys_sched_setattr
315 common sched_getattr sys_sched_getattr
316 common renameat2 sys_renameat2
317 common seccomp sys_seccomp
318 common getrandom sys_getrandom
319 common memfd_create sys_memfd_create
320 common kexec_file_load sys_kexec_file_load
321 common bpf sys_bpf
322 64 execveat sys_execveat
323 common userfaultfd sys_userfaultfd
324 common membarrier sys_membarrier
325 common mlock2 sys_mlock2
326 common copy_file_range sys_copy_file_range
327 64 preadv2 sys_preadv2
328 64 pwritev2 sys_pwritev2
329 common pkey_mprotect sys_pkey_mprotect
330 common pkey_alloc sys_pkey_alloc
331 common pkey_free sys_pkey_free
332 common statx sys_statx
333 common io_pgetevents sys_io_pgetevents
334 common rseq sys_rseq
335 common uretprobe sys_uretprobe
336 common uprobe sys_uprobe
# don't use numbers 387 through 423, add new calls after the last
# 'common' entry
424 common pidfd_send_signal sys_pidfd_send_signal
425 common io_uring_setup sys_io_uring_setup
426 common io_uring_enter sys_io_uring_enter
427 common io_uring_register sys_io_uring_register
428 common open_tree sys_open_tree
429 common move_mount sys_move_mount
430 common fsopen sys_fsopen
431 common fsconfig sys_fsconfig
432 common fsmount sys_fsmount
433 common fspick sys_fspick
434 common pidfd_open sys_pidfd_open
435 common clone3 sys_clone3
436 common close_range sys_close_range
437 common openat2 sys_openat2
438 common pidfd_getfd sys_pidfd_getfd
439 common faccessat2 sys_faccessat2
440 common process_madvise sys_process_madvise
441 common epoll_pwait2 sys_epoll_pwait2
442 common mount_setattr sys_mount_setattr
443 common quotactl_fd sys_quotactl_fd
444 common landlock_create_ruleset sys_landlock_create_ruleset
445 common landlock_add_rule sys_landlock_add_rule
446 common landlock_restrict_self sys_landlock_restrict_self
447 common memfd_secret sys_memfd_secret
448 common process_mrelease sys_process_mrelease
449 common futex_waitv sys_futex_waitv
450 common set_mempolicy_home_node sys_set_mempolicy_home_node
451 common cachestat sys_cachestat
452 common fchmodat2 sys_fchmodat2
453 common map_shadow_stack sys_map_shadow_stack
454 common futex_wake sys_futex_wake
455 common futex_wait sys_futex_wait
456 common futex_requeue sys_futex_requeue
457 common statmount sys_statmount
458 common listmount sys_listmount
459 common lsm_get_self_attr sys_lsm_get_self_attr
460 common lsm_set_self_attr sys_lsm_set_self_attr
461 common lsm_list_modules sys_lsm_list_modules
462 common mseal sys_mseal
463 common setxattrat sys_setxattrat
464 common getxattrat sys_getxattrat
465 common listxattrat sys_listxattrat
466 common removexattrat sys_removexattrat
467 common open_tree_attr sys_open_tree_attr
468 common file_getattr sys_file_getattr
469 common file_setattr sys_file_setattr
470 common listns sys_listns
471 common rseq_slice_yield sys_rseq_slice_yield
#
# Due to a historical design error, certain syscalls are numbered differently
# in x32 as compared to native x86_64. These syscalls have numbers 512-547.
# Do not add new syscalls to this range. Numbers 548 and above are available
# for non-x32 use.
#
512 x32 rt_sigaction compat_sys_rt_sigaction
513 x32 rt_sigreturn compat_sys_x32_rt_sigreturn
514 x32 ioctl compat_sys_ioctl
515 x32 readv sys_readv
516 x32 writev sys_writev
517 x32 recvfrom compat_sys_recvfrom
518 x32 sendmsg compat_sys_sendmsg
519 x32 recvmsg compat_sys_recvmsg
520 x32 execve compat_sys_execve
521 x32 ptrace compat_sys_ptrace
522 x32 rt_sigpending compat_sys_rt_sigpending
523 x32 rt_sigtimedwait compat_sys_rt_sigtimedwait_time64
524 x32 rt_sigqueueinfo compat_sys_rt_sigqueueinfo
525 x32 sigaltstack compat_sys_sigaltstack
526 x32 timer_create compat_sys_timer_create
527 x32 mq_notify compat_sys_mq_notify
528 x32 kexec_load compat_sys_kexec_load
529 x32 waitid compat_sys_waitid
530 x32 set_robust_list compat_sys_set_robust_list
531 x32 get_robust_list compat_sys_get_robust_list
532 x32 vmsplice sys_vmsplice
533 x32 move_pages sys_move_pages
534 x32 preadv compat_sys_preadv64
535 x32 pwritev compat_sys_pwritev64
536 x32 rt_tgsigqueueinfo compat_sys_rt_tgsigqueueinfo
537 x32 recvmmsg compat_sys_recvmmsg_time64
538 x32 sendmmsg compat_sys_sendmmsg
539 x32 process_vm_readv sys_process_vm_readv
540 x32 process_vm_writev sys_process_vm_writev
541 x32 setsockopt sys_setsockopt
542 x32 getsockopt sys_getsockopt
543 x32 io_setup compat_sys_io_setup
544 x32 io_submit compat_sys_io_submit
545 x32 execveat compat_sys_execveat
546 x32 preadv2 compat_sys_preadv64v2
547 x32 pwritev2 compat_sys_pwritev64v2
# This is the end of the legacy x32 range. Numbers 548 and above are
# not special and are not to be used for x32-specific syscalls.
linux-7.2.3/arch/x86/entry/thunk.S
+15
View File
@@ -0,0 +1,15 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Save registers before calling assembly functions. This avoids
* disturbance of register allocation in some inline assembly constructs.
* Copyright 2001,2002 by Andi Kleen, SuSE Labs.
*/
#include <linux/export.h>
#include <linux/linkage.h>
#include "calling.h"
#include <asm/asm.h>
THUNK preempt_schedule_thunk, preempt_schedule
THUNK preempt_schedule_notrace_thunk, preempt_schedule_notrace
EXPORT_SYMBOL(preempt_schedule_thunk)
EXPORT_SYMBOL(preempt_schedule_notrace_thunk)
linux-7.2.3/arch/x86/entry/vdso/.gitignore
+6
View File
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
*.lds
*.so
*.so.dbg
vdso*-image*.c
vdso2c
linux-7.2.3/arch/x86/entry/vdso/Makefile
+12
View File
@@ -0,0 +1,12 @@
# SPDX-License-Identifier: GPL-2.0
#
# Building vDSO images for x86.
#
# Regular kernel objects
obj-y := vma.o extable.o
obj-$(CONFIG_COMPAT_32) += vdso32-setup.o
# vDSO directories
obj-$(CONFIG_X86_64) += vdso64/
obj-$(CONFIG_COMPAT_32) += vdso32/
linux-7.2.3/arch/x86/entry/vdso/common/Makefile.include
+89
View File
@@ -0,0 +1,89 @@
# SPDX-License-Identifier: GPL-2.0
#
# Building vDSO images for x86.
#
# Include the generic Makefile to check the built vDSO:
include $(srctree)/lib/vdso/Makefile.include
obj-y += $(foreach x,$(vdsos-y),vdso$(x)-image.o)
targets += $(foreach x,$(vdsos-y),vdso$(x)-image.c vdso$(x).so vdso$(x).so.dbg vdso$(x).lds)
targets += $(vobjs-y)
# vobjs-y with $(obj)/ prepended
vobjs := $(addprefix $(obj)/,$(vobjs-y))
# Options for vdso*.lds
CPPFLAGS_VDSO_LDS := -P -C -I$(src)/..
$(obj)/%.lds : KBUILD_CPPFLAGS += $(CPPFLAGS_VDSO_LDS)
#
# Options from KBUILD_[AC]FLAGS that should *NOT* be kept
#
flags-remove-y += \
-D__KERNEL__ -mcmodel=kernel -mregparm=3 \
-fno-pic -fno-PIC -fno-pie -fno-PIE \
-mfentry -pg \
$(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(KSTACK_ERASE_CFLAGS) \
$(RETPOLINE_CFLAGS) $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) \
$(PADDING_CFLAGS)
#
# Don't omit frame pointers for ease of userspace debugging, but do
# optimize sibling calls.
#
flags-y += -D__DISABLE_EXPORTS
flags-y += -DDISABLE_BRANCH_PROFILING
flags-y += -DBUILD_VDSO
flags-y += -I$(src)/.. -I$(srctree)
flags-y += -O2 -fpic
flags-y += -fno-stack-protector
flags-y += -fno-omit-frame-pointer
flags-y += -foptimize-sibling-calls
flags-y += -fasynchronous-unwind-tables
# Reset cf protections enabled by compiler default
flags-y += $(call cc-option, -fcf-protection=none)
flags-$(X86_USER_SHADOW_STACK) += $(call cc-option, -fcf-protection=return)
# When user space IBT is supported, enable this.
# flags-$(CONFIG_USER_IBT) += $(call cc-option, -fcf-protection=branch)
flags-$(CONFIG_MITIGATION_RETPOLINE) += $(RETPOLINE_VDSO_CFLAGS)
# These need to be conditional on $(vobjs) as they do not apply to
# the output vdso*-image.o files which are standard kernel objects.
$(vobjs) : KBUILD_AFLAGS := \
$(filter-out $(flags-remove-y),$(KBUILD_AFLAGS)) $(flags-y)
$(vobjs) : KBUILD_CFLAGS := \
$(filter-out $(flags-remove-y),$(KBUILD_CFLAGS)) $(flags-y)
#
# The VDSO images are built using a special linker script.
#
VDSO_LDFLAGS := -shared --hash-style=both --build-id=sha1 --no-undefined \
$(call ld-option, --eh-frame-hdr) -Bsymbolic -z noexecstack
quiet_cmd_vdso = VDSO $@
cmd_vdso = $(LD) -o $@ \
$(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$*) \
-T $(filter %.lds,$^) $(filter %.o,$^)
quiet_cmd_vdso_and_check = VDSO $@
cmd_vdso_and_check = $(cmd_vdso); $(cmd_vdso_check)
$(obj)/vdso%.so.dbg: $(obj)/vdso%.lds FORCE
$(call if_changed,vdso_and_check)
$(obj)/%.so: OBJCOPYFLAGS := -S --remove-section __ex_table
$(obj)/%.so: $(obj)/%.so.dbg FORCE
$(call if_changed,objcopy)
VDSO2C = $(objtree)/arch/x86/tools/vdso2c
quiet_cmd_vdso2c = VDSO2C $@
cmd_vdso2c = $(VDSO2C) $< $(<:%.dbg=%) $@
$(obj)/%-image.c: $(obj)/%.so.dbg $(obj)/%.so $(VDSO2C) FORCE
$(call if_changed,vdso2c)
$(obj)/%-image.o: $(obj)/%-image.c
linux-7.2.3/arch/x86/entry/vdso/common/note.S
+18
View File
@@ -0,0 +1,18 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* This supplies .note.* sections to go into the PT_NOTE inside the vDSO text.
* Here we can supply some information useful to userland.
*/
#include <linux/build-salt.h>
#include <linux/version.h>
#include <linux/elfnote.h>
/* Ideally this would use UTS_NAME, but using a quoted string here
doesn't work. Remember to change this when changing the
kernel's name. */
ELFNOTE_START(Linux, 0, "a")
.long LINUX_VERSION_CODE
ELFNOTE_END
BUILD_SALT
linux-7.2.3/arch/x86/entry/vdso/common/vclock_gettime.c
+85
View File
@@ -0,0 +1,85 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Fast user context implementation of clock_gettime, gettimeofday, and time.
*
* Copyright 2006 Andi Kleen, SUSE Labs.
* Copyright 2019 ARM Limited
*
* 32 Bit compat layer by Stefani Seibold <stefani@seibold.net>
* sponsored by Rohde & Schwarz GmbH & Co. KG Munich/Germany
*/
#include <linux/time.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <vdso/gettime.h>
#include "lib/vdso/gettimeofday.c"
int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
{
return __cvdso_gettimeofday(tv, tz);
}
int gettimeofday(struct __kernel_old_timeval *, struct timezone *)
__attribute__((weak, alias("__vdso_gettimeofday")));
__kernel_old_time_t __vdso_time(__kernel_old_time_t *t)
{
return __cvdso_time(t);
}
__kernel_old_time_t time(__kernel_old_time_t *t) __attribute__((weak, alias("__vdso_time")));
#if defined(CONFIG_X86_64) && !defined(BUILD_VDSO32_64)
/* both 64-bit and x32 use these */
int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
{
return __cvdso_clock_gettime(clock, ts);
}
int clock_gettime(clockid_t, struct __kernel_timespec *)
__attribute__((weak, alias("__vdso_clock_gettime")));
int __vdso_clock_getres(clockid_t clock,
struct __kernel_timespec *res)
{
return __cvdso_clock_getres(clock, res);
}
int clock_getres(clockid_t, struct __kernel_timespec *)
__attribute__((weak, alias("__vdso_clock_getres")));
#else
/* i386 only */
int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts)
{
return __cvdso_clock_gettime32(clock, ts);
}
int clock_gettime(clockid_t, struct old_timespec32 *)
__attribute__((weak, alias("__vdso_clock_gettime")));
int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts)
{
return __cvdso_clock_gettime(clock, ts);
}
int clock_gettime64(clockid_t, struct __kernel_timespec *)
__attribute__((weak, alias("__vdso_clock_gettime64")));
int __vdso_clock_getres(clockid_t clock, struct old_timespec32 *res)
{
return __cvdso_clock_getres_time32(clock, res);
}
int clock_getres(clockid_t, struct old_timespec32 *)
__attribute__((weak, alias("__vdso_clock_getres")));
int __vdso_clock_getres_time64(clockid_t clock, struct __kernel_timespec *ts)
{
return __cvdso_clock_getres(clock, ts);
}
int clock_getres_time64(clockid_t, struct __kernel_timespec *)
__attribute__((weak, alias("__vdso_clock_getres_time64")));
#endif
linux-7.2.3/arch/x86/entry/vdso/common/vdso-layout.lds.S
+109
View File
@@ -0,0 +1,109 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include <asm/vdso.h>
#include <asm/vdso/vsyscall.h>
#include <vdso/datapage.h>
/*
* Linker script for vDSO. This is an ELF shared object prelinked to
* its virtual address, and with only one read-only segment.
* This script controls its layout.
*/
SECTIONS
{
/*
* User/kernel shared data is before the vDSO. This may be a little
* uglier than putting it after the vDSO, but it avoids issues with
* non-allocatable things that dangle past the end of the PT_LOAD
* segment.
*/
VDSO_VVAR_SYMS
vclock_pages = VDSO_VCLOCK_PAGES_START(vdso_u_data);
pvclock_page = vclock_pages + VDSO_PAGE_PVCLOCK_OFFSET * PAGE_SIZE;
hvclock_page = vclock_pages + VDSO_PAGE_HVCLOCK_OFFSET * PAGE_SIZE;
. = SIZEOF_HEADERS;
.hash : { *(.hash) } :text
.gnu.hash : { *(.gnu.hash) }
.dynsym : { *(.dynsym) }
.dynstr : { *(.dynstr) }
.gnu.version : { *(.gnu.version) }
.gnu.version_d : { *(.gnu.version_d) }
.gnu.version_r : { *(.gnu.version_r) }
.dynamic : { *(.dynamic) } :text :dynamic
.rodata : {
*(.rodata*)
*(.data*)
*(.sdata*)
*(.got.plt) *(.got)
*(.gnu.linkonce.d.*)
*(.bss*)
*(.dynbss*)
*(.gnu.linkonce.b.*)
} :text
.note.gnu.property : {
*(.note.gnu.property)
} :text :note :gnu_property
.note : {
*(.note*)
} :text :note
.eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr
.eh_frame : {
KEEP (*(.eh_frame))
*(.eh_frame.*)
} :text
/*
* Text is well-separated from actual data: there's plenty of
* stuff that isn't used at runtime in between.
*/
.text : {
*(.text*)
} :text =0x90909090,
.altinstructions : { *(.altinstructions) } :text
.altinstr_replacement : { *(.altinstr_replacement) } :text
__ex_table : { *(__ex_table) } :text
/DISCARD/ : {
*(.discard)
*(.discard.*)
*(__bug_table)
}
}
/*
* Very old versions of ld do not recognize this name token; use the constant.
*/
#define PT_GNU_EH_FRAME 0x6474e550
#define PT_GNU_STACK 0x6474e551
#define PT_GNU_PROPERTY 0x6474e553
/*
* We must supply the ELF program headers explicitly to get just one
* PT_LOAD segment, and set the flags explicitly to make segments read-only.
*/
#define PF_R FLAGS(4)
#define PF_RW FLAGS(6)
#define PF_RX FLAGS(5)
PHDRS
{
text PT_LOAD PF_RX FILEHDR PHDRS;
dynamic PT_DYNAMIC PF_R;
note PT_NOTE PF_R;
eh_frame_hdr PT_GNU_EH_FRAME PF_R;
gnu_stack PT_GNU_STACK PF_RW;
gnu_property PT_GNU_PROPERTY PF_R;
}
linux-7.2.3/arch/x86/entry/vdso/common/vfutex.c
+71
View File
@@ -0,0 +1,71 @@
// SPDX-License-Identifier: GPL-2.0-only
#include <vdso/futex.h>
/*
* Assembly template for the try unlock functions. The basic functionality is:
*
* mov esi, %eax Move the TID into EAX
* xor %ecx, %ecx Clear ECX
* lock_cmpxchgl %ecx, (%rdi) Attempt the TID -> 0 transition
* .Lcs_start: Start of the critical section
* jnz .Lcs_end If cmpxchl failed jump to the end
* .Lcs_success: Start of the success section
* movq %rcx, (%rdx) Set the pending op pointer to 0
* .Lcs_end: End of the critical section
*
* .Lcs_start and .Lcs_end establish the critical section range. .Lcs_success is
* technically not required, but there for illustration, debugging and testing.
*
* When CONFIG_COMPAT is enabled then the 64-bit VDSO provides two functions.
* One for the regular 64-bit sized pending operation pointer and one for a
* 32-bit sized pointer to support gaming emulators.
*
* The 32-bit VDSO provides only the one for 32-bit sized pointers.
*/
#define __stringify_1(x...) #x
#define __stringify(x...) __stringify_1(x)
#define LABEL(prefix, which) __stringify(prefix##_try_unlock_cs_##which:)
#define JNZ_END(prefix) "jnz " __stringify(prefix) "_try_unlock_cs_end\n"
#define CLEAR_POPQ "movq %[zero], %a[pop]\n"
#define CLEAR_POPL "movl %k[zero], %a[pop]\n"
#define futex_robust_try_unlock(prefix, clear_pop, __lock, __tid, __pop)\
({ \
asm volatile ( \
" \n" \
" lock cmpxchgl %k[zero], %a[lock] \n" \
" \n" \
LABEL(prefix, start) \
" \n" \
JNZ_END(prefix) \
" \n" \
LABEL(prefix, success) \
" \n" \
clear_pop \
" \n" \
LABEL(prefix, end) \
: [tid] "+&a" (__tid) \
: [lock] "D" (__lock), \
[pop] "d" (__pop), \
[zero] "r" (0UL) \
: "memory" \
); \
__tid; \
})
#ifdef __x86_64__
__u32 __vdso_futex_robust_list64_try_unlock(__u32 *lock, __u32 tid, __u64 *pop)
{
return futex_robust_try_unlock(__futex_list64, CLEAR_POPQ, lock, tid, pop);
}
#endif /* __x86_64__ */
#if defined(CONFIG_X86_32) || defined(CONFIG_COMPAT)
__u32 __vdso_futex_robust_list32_try_unlock(__u32 *lock, __u32 tid, __u32 *pop)
{
return futex_robust_try_unlock(__futex_list32, CLEAR_POPL, lock, tid, pop);
}
#endif /* CONFIG_X86_32 || CONFIG_COMPAT */
linux-7.2.3/arch/x86/entry/vdso/common/vgetcpu.c
+21
View File
@@ -0,0 +1,21 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright 2006 Andi Kleen, SUSE Labs.
*
* Fast user context implementation of getcpu()
*/
#include <linux/kernel.h>
#include <asm/segment.h>
#include <vdso/processor.h>
notrace long
__vdso_getcpu(unsigned *cpu, unsigned *node, void *unused)
{
vdso_read_cpunode(cpu, node);
return 0;
}
long getcpu(unsigned *cpu, unsigned *node, void *tcache)
__attribute__((weak, alias("__vdso_getcpu")));
linux-7.2.3/arch/x86/entry/vdso/extable.c
+46
View File
@@ -0,0 +1,46 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/err.h>
#include <linux/mm.h>
#include <asm/current.h>
#include <asm/traps.h>
#include <asm/vdso.h>
struct vdso_exception_table_entry {
int insn, fixup;
};
bool fixup_vdso_exception(struct pt_regs *regs, int trapnr,
unsigned long error_code, unsigned long fault_addr)
{
const struct vdso_image *image = current->mm->context.vdso_image;
const struct vdso_exception_table_entry *extable;
unsigned int nr_entries, i;
unsigned long base;
/*
* Do not attempt to fixup #DB or #BP. It's impossible to identify
* whether or not a #DB/#BP originated from within an SGX enclave and
* SGX enclaves are currently the only use case for vDSO fixup.
*/
if (trapnr == X86_TRAP_DB || trapnr == X86_TRAP_BP)
return false;
if (!current->mm->context.vdso)
return false;
base = (unsigned long)current->mm->context.vdso + image->extable_base;
nr_entries = image->extable_len / (sizeof(*extable));
extable = image->extable;
for (i = 0; i < nr_entries; i++) {
if (regs->ip == base + extable[i].insn) {
regs->ip = base + extable[i].fixup;
regs->di = trapnr;
regs->si = error_code;
regs->dx = fault_addr;
return true;
}
}
return false;
}
linux-7.2.3/arch/x86/entry/vdso/extable.h
+28
View File
@@ -0,0 +1,28 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __VDSO_EXTABLE_H
#define __VDSO_EXTABLE_H
/*
* Inject exception fixup for vDSO code. Unlike normal exception fixup,
* vDSO uses a dedicated handler the addresses are relative to the overall
* exception table, not each individual entry.
*/
#ifdef __ASSEMBLER__
#define _ASM_VDSO_EXTABLE_HANDLE(from, to) \
ASM_VDSO_EXTABLE_HANDLE from to
.macro ASM_VDSO_EXTABLE_HANDLE from:req to:req
.pushsection __ex_table, "a"
.long (\from) - __ex_table
.long (\to) - __ex_table
.popsection
.endm
#else
#define _ASM_VDSO_EXTABLE_HANDLE(from, to) \
".pushsection __ex_table, \"a\"\n" \
".long (" #from ") - __ex_table\n" \
".long (" #to ") - __ex_table\n" \
".popsection\n"
#endif
#endif /* __VDSO_EXTABLE_H */
linux-7.2.3/arch/x86/entry/vdso/vdso32-setup.c
+86
View File
@@ -0,0 +1,86 @@
// SPDX-License-Identifier: GPL-2.0
/*
* (C) Copyright 2002 Linus Torvalds
* Portions based on the vdso-randomization code from exec-shield:
* Copyright(C) 2005-2006, Red Hat, Inc., Ingo Molnar
*
* This file contains the needed initializations to support sysenter.
*/
#include <linux/init.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/mm_types.h>
#include <linux/elf.h>
#include <asm/processor.h>
#include <asm/vdso.h>
#ifdef CONFIG_COMPAT_VDSO
#define VDSO_DEFAULT 0
#else
#define VDSO_DEFAULT 1
#endif
/*
* Should the kernel map a VDSO page into processes and pass its
* address down to glibc upon exec()?
*/
unsigned int __read_mostly vdso32_enabled = VDSO_DEFAULT;
static int __init vdso32_setup(char *s)
{
vdso32_enabled = simple_strtoul(s, NULL, 0);
if (vdso32_enabled > 1) {
pr_warn("vdso32 values other than 0 and 1 are no longer allowed; vdso disabled\n");
vdso32_enabled = 0;
}
return 1;
}
/*
* For consistency, the argument vdso32=[012] affects the 32-bit vDSO
* behavior on both 64-bit and 32-bit kernels.
* On 32-bit kernels, vdso=[012] means the same thing.
*/
__setup("vdso32=", vdso32_setup);
#ifdef CONFIG_X86_32
__setup_param("vdso=", vdso_setup, vdso32_setup, 0);
#endif
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
static const struct ctl_table vdso_table[] = {
{
#ifdef CONFIG_X86_64
.procname = "vsyscall32",
#else
.procname = "vdso_enabled",
#endif
.data = &vdso32_enabled,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
};
static __init int ia32_binfmt_init(void)
{
#ifdef CONFIG_X86_64
/* Register vsyscall32 into the ABI table */
register_sysctl("abi", vdso_table);
#else
register_sysctl_init("vm", vdso_table);
#endif
return 0;
}
__initcall(ia32_binfmt_init);
#endif /* CONFIG_SYSCTL */
linux-7.2.3/arch/x86/entry/vdso/vdso32/.gitignore
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: GPL-2.0-only
vdso32.lds
linux-7.2.3/arch/x86/entry/vdso/vdso32/Makefile
+29
View File
@@ -0,0 +1,29 @@
# SPDX-License-Identifier: GPL-2.0
#
# 32-bit vDSO images for x86.
#
# The vDSOs built in this directory
vdsos-y := 32
# Files to link into the vDSO:
vobjs-y := note.o vclock_gettime.o vgetcpu.o
vobjs-y += system_call.o sigreturn.o
vobjs-$(CONFIG_FUTEX_ROBUST_UNLOCK) += vfutex.o
# Compilation flags
flags-y := -DBUILD_VDSO32 -m32 -mregparm=0
flags-$(CONFIG_X86_64) += -include $(src)/fake_32bit_build.h
flags-remove-y := -m64
# Checker flags
CHECKFLAGS := $(subst -m64,-m32,$(CHECKFLAGS))
CHECKFLAGS := $(subst -D__x86_64__,-D__i386__,$(CHECKFLAGS))
# The location of this include matters!
include $(src)/../common/Makefile.include
# Linker options for the vdso
VDSO_LDFLAGS_32 := -m elf_i386 -soname linux-gate.so.1
$(obj)/vdso32.so.dbg: $(vobjs)
linux-7.2.3/arch/x86/entry/vdso/vdso32/fake_32bit_build.h
+26
View File
@@ -0,0 +1,26 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifdef CONFIG_X86_64
/*
* in case of a 32 bit VDSO for a 64 bit kernel fake a 32 bit kernel
* configuration
*/
#undef CONFIG_64BIT
#undef CONFIG_X86_64
#undef CONFIG_COMPAT
#undef CONFIG_PGTABLE_LEVELS
#undef CONFIG_ILLEGAL_POINTER_VALUE
#undef CONFIG_SPARSEMEM_VMEMMAP
#undef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
#undef CONFIG_NR_CPUS
#undef CONFIG_PARAVIRT_XXL
#define CONFIG_X86_32 1
#define CONFIG_PGTABLE_LEVELS 2
#define CONFIG_PAGE_OFFSET 0
#define CONFIG_ILLEGAL_POINTER_VALUE 0
#define CONFIG_NR_CPUS 1
#define BUILD_VDSO32_64
#endif
linux-7.2.3/arch/x86/entry/vdso/vdso32/note.S
+1
View File
@@ -0,0 +1 @@
#include "common/note.S"
linux-7.2.3/arch/x86/entry/vdso/vdso32/sigreturn.S
+84
View File
@@ -0,0 +1,84 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/linkage.h>
#include <asm/unistd_32.h>
#include <asm/dwarf2.h>
#include <asm/asm-offsets.h>
.macro STARTPROC_SIGNAL_FRAME sc
CFI_STARTPROC simple
CFI_SIGNAL_FRAME
/* -4 as pretcode has already been popped */
CFI_DEF_CFA esp, \sc - 4
CFI_OFFSET eip, IA32_SIGCONTEXT_ip
CFI_OFFSET eax, IA32_SIGCONTEXT_ax
CFI_OFFSET ebx, IA32_SIGCONTEXT_bx
CFI_OFFSET ecx, IA32_SIGCONTEXT_cx
CFI_OFFSET edx, IA32_SIGCONTEXT_dx
CFI_OFFSET esp, IA32_SIGCONTEXT_sp
CFI_OFFSET ebp, IA32_SIGCONTEXT_bp
CFI_OFFSET esi, IA32_SIGCONTEXT_si
CFI_OFFSET edi, IA32_SIGCONTEXT_di
CFI_OFFSET es, IA32_SIGCONTEXT_es
CFI_OFFSET cs, IA32_SIGCONTEXT_cs
CFI_OFFSET ss, IA32_SIGCONTEXT_ss
CFI_OFFSET ds, IA32_SIGCONTEXT_ds
CFI_OFFSET eflags, IA32_SIGCONTEXT_flags
.endm
/*
* WARNING:
*
* A bug in the libgcc unwinder as of at least gcc 15.2 (2026) means that
* the unwinder fails to recognize the signal frame flag.
*
* There is a hacky legacy fallback path in libgcc which ends up
* getting invoked instead. It happens to work as long as BOTH of the
* following conditions are true:
*
* 1. There is at least one byte before the each of the sigreturn
* functions which falls outside any function. This is enforced by
* an explicit nop instruction before the ALIGN.
* 2. The code sequences between the entry point up to and including
* the int $0x80 below need to match EXACTLY. Do not change them
* in any way. The exact byte sequences are:
*
* __kernel_sigreturn:
* 0: 58 pop %eax
* 1: b8 77 00 00 00 mov $0x77,%eax
* 6: cd 80 int $0x80
*
* __kernel_rt_sigreturn:
* 0: b8 ad 00 00 00 mov $0xad,%eax
* 5: cd 80 int $0x80
*
* For details, see: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=124050
*/
.text
.globl __kernel_sigreturn
.type __kernel_sigreturn,@function
nop /* libgcc hack: see comment above */
ALIGN
__kernel_sigreturn:
STARTPROC_SIGNAL_FRAME IA32_SIGFRAME_sigcontext
popl %eax
CFI_ADJUST_CFA_OFFSET -4
movl $__NR_sigreturn, %eax
int $0x80
SYM_INNER_LABEL(vdso32_sigreturn_landing_pad, SYM_L_GLOBAL)
ud2a
CFI_ENDPROC
.size __kernel_sigreturn,.-__kernel_sigreturn
.globl __kernel_rt_sigreturn
.type __kernel_rt_sigreturn,@function
nop /* libgcc hack: see comment above */
ALIGN
__kernel_rt_sigreturn:
STARTPROC_SIGNAL_FRAME IA32_RT_SIGFRAME_sigcontext
movl $__NR_rt_sigreturn, %eax
int $0x80
SYM_INNER_LABEL(vdso32_rt_sigreturn_landing_pad, SYM_L_GLOBAL)
ud2a
CFI_ENDPROC
.size __kernel_rt_sigreturn,.-__kernel_rt_sigreturn
.previous
linux-7.2.3/arch/x86/entry/vdso/vdso32/system_call.S
+91
View File
@@ -0,0 +1,91 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* AT_SYSINFO entry point
*/
#include <linux/linkage.h>
#include <asm/dwarf2.h>
#include <asm/cpufeatures.h>
#include <asm/alternative.h>
.text
.globl __kernel_vsyscall
.type __kernel_vsyscall,@function
ALIGN
__kernel_vsyscall:
CFI_STARTPROC
/*
* If using int $0x80, there is no reason to muck about with the
* stack here. Unfortunately just overwriting the push instructions
* would mess up the CFI annotations, but it is only a 3-byte
* NOP in that case. This could be avoided by patching the
* vdso symbol table (not the code) and entry point, but that
* would a fair bit of tooling work or by simply compiling
* two different vDSO images, but that doesn't seem worth it.
*/
ALTERNATIVE "int $0x80; ret", "", X86_FEATURE_SYSFAST32
/*
* Reshuffle regs so that all of any of the entry instructions
* will preserve enough state.
*
* A really nice entry sequence would be:
* pushl %edx
* pushl %ecx
* movl %esp, %ecx
*
* Unfortunately, naughty Android versions between July and December
* 2015 actually hardcode the traditional Linux SYSENTER entry
* sequence. That is severely broken for a number of reasons (ask
* anyone with an AMD CPU, for example). Nonetheless, we try to keep
* it working approximately as well as it ever worked.
*
* This link may elucidate some of the history:
* https://android-review.googlesource.com/#/q/Iac3295376d61ef83e713ac9b528f3b50aa780cd7
* personally, I find it hard to understand what's going on there.
*
* Note to future user developers: DO NOT USE SYSENTER IN YOUR CODE.
* Execute an indirect call to the address in the AT_SYSINFO auxv
* entry. That is the ONLY correct way to make a fast 32-bit system
* call on Linux. (Open-coding int $0x80 is also fine, but it's
* slow.)
*/
pushl %ecx
CFI_ADJUST_CFA_OFFSET 4
CFI_REL_OFFSET ecx, 0
pushl %edx
CFI_ADJUST_CFA_OFFSET 4
CFI_REL_OFFSET edx, 0
pushl %ebp
CFI_ADJUST_CFA_OFFSET 4
CFI_REL_OFFSET ebp, 0
#define SYSENTER_SEQUENCE "movl %esp, %ebp; sysenter"
#define SYSCALL_SEQUENCE "movl %ecx, %ebp; syscall"
ALTERNATIVE SYSENTER_SEQUENCE, SYSCALL_SEQUENCE, X86_FEATURE_SYSCALL32
/* Re-enter using int $0x80 */
int $0x80
SYM_INNER_LABEL(int80_landing_pad, SYM_L_GLOBAL)
/*
* Restore EDX and ECX in case they were clobbered. EBP is not
* clobbered (the kernel restores it), but it's cleaner and
* probably faster to pop it than to adjust ESP using addl.
*/
popl %ebp
CFI_RESTORE ebp
CFI_ADJUST_CFA_OFFSET -4
popl %edx
CFI_RESTORE edx
CFI_ADJUST_CFA_OFFSET -4
popl %ecx
CFI_RESTORE ecx
CFI_ADJUST_CFA_OFFSET -4
RET
CFI_ENDPROC
.size __kernel_vsyscall,.-__kernel_vsyscall
.previous
linux-7.2.3/arch/x86/entry/vdso/vdso32/vclock_gettime.c
+1
View File
@@ -0,0 +1 @@
#include "common/vclock_gettime.c"
linux-7.2.3/arch/x86/entry/vdso/vdso32/vdso32.lds.S
+45
View File
@@ -0,0 +1,45 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Linker script for 32-bit vDSO.
* We #include the file to define the layout details.
*
* This file defines the version script giving the user-exported symbols in
* the DSO.
*/
#include <asm/page.h>
#define BUILD_VDSO32
#include "common/vdso-layout.lds.S"
/* The ELF entry point can be used to set the AT_SYSINFO value. */
ENTRY(__kernel_vsyscall);
/*
* This controls what userland symbols we export from the vDSO.
*/
VERSION
{
LINUX_2.6 {
global:
__vdso_clock_gettime;
__vdso_gettimeofday;
__vdso_time;
__vdso_clock_getres;
__vdso_clock_gettime64;
__vdso_clock_getres_time64;
__vdso_getcpu;
#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
__vdso_futex_robust_list32_try_unlock;
#endif
};
LINUX_2.5 {
global:
__kernel_vsyscall;
__kernel_sigreturn;
__kernel_rt_sigreturn;
local: *;
};
}
linux-7.2.3/arch/x86/entry/vdso/vdso32/vfutex.c
+1
View File
@@ -0,0 +1 @@
#include "common/vfutex.c"
linux-7.2.3/arch/x86/entry/vdso/vdso32/vgetcpu.c
+1
View File
@@ -0,0 +1 @@
#include "common/vgetcpu.c"
linux-7.2.3/arch/x86/entry/vdso/vdso64/Makefile
+47
View File
@@ -0,0 +1,47 @@
# SPDX-License-Identifier: GPL-2.0
#
# 64-bit vDSO images for x86.
#
# The vDSOs built in this directory
vdsos-y := 64
vdsos-$(CONFIG_X86_X32_ABI) += x32
# Files to link into the vDSO:
vobjs-y := note.o vclock_gettime.o vgetcpu.o
vobjs-y += vgetrandom.o vgetrandom-chacha.o
vobjs-$(CONFIG_X86_SGX) += vsgx.o
vobjs-$(CONFIG_FUTEX_ROBUST_UNLOCK) += vfutex.o
# Compilation flags
flags-y := -DBUILD_VDSO64 -m64 -mcmodel=small
# The location of this include matters!
include $(src)/../common/Makefile.include
#
# X32 processes use x32 vDSO to access 64bit kernel data.
#
# Build x32 vDSO image:
# 1. Compile x32 vDSO as 64bit.
# 2. Convert object files to x32.
# 3. Build x32 VDSO image with x32 objects, which contains 64bit codes
# so that it can reach 64bit address space with 64bit pointers.
#
# Convert 64bit object file to x32 for x32 vDSO.
quiet_cmd_x32 = X32 $@
cmd_x32 = $(OBJCOPY) -O elf32-x86-64 $< $@
$(obj)/%-x32.o: $(obj)/%.o FORCE
$(call if_changed,x32)
vobjsx32 = $(patsubst %.o,%-x32.o,$(vobjs))
targets += $(patsubst %.o,%-x32.o,$(vobjs-y))
# Linker options for the vdso
VDSO_LDFLAGS_64 := -m elf_x86_64 -soname linux-vdso.so.1 -z max-page-size=4096
VDSO_LDFLAGS_x32 := $(subst elf_x86_64,elf32_x86_64,$(VDSO_LDFLAGS_64))
$(obj)/vdso64.so.dbg: $(vobjs)
$(obj)/vdsox32.so.dbg: $(vobjsx32)
linux-7.2.3/arch/x86/entry/vdso/vdso64/note.S
+1
View File
@@ -0,0 +1 @@
#include "common/note.S"
linux-7.2.3/arch/x86/entry/vdso/vdso64/vclock_gettime.c
+1
View File
@@ -0,0 +1 @@
#include "common/vclock_gettime.c"
linux-7.2.3/arch/x86/entry/vdso/vdso64/vdso64.lds.S
+44
View File
@@ -0,0 +1,44 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Linker script for 64-bit vDSO.
* We #include the file to define the layout details.
*
* This file defines the version script giving the user-exported symbols in
* the DSO.
*/
#define BUILD_VDSO64
#include "common/vdso-layout.lds.S"
/*
* This controls what userland symbols we export from the vDSO.
*/
VERSION {
LINUX_2.6 {
global:
clock_gettime;
__vdso_clock_gettime;
gettimeofday;
__vdso_gettimeofday;
getcpu;
__vdso_getcpu;
time;
__vdso_time;
clock_getres;
__vdso_clock_getres;
#ifdef CONFIG_X86_SGX
__vdso_sgx_enter_enclave;
#endif
getrandom;
__vdso_getrandom;
#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
__vdso_futex_robust_list64_try_unlock;
#ifdef CONFIG_COMPAT
__vdso_futex_robust_list32_try_unlock;
#endif
#endif
local: *;
};
}
linux-7.2.3/arch/x86/entry/vdso/vdso64/vdsox32.lds.S
+34
View File
@@ -0,0 +1,34 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Linker script for x32 vDSO.
* We #include the file to define the layout details.
*
* This file defines the version script giving the user-exported symbols in
* the DSO.
*/
#define BUILD_VDSOX32
#include "common/vdso-layout.lds.S"
/*
* This controls what userland symbols we export from the vDSO.
*/
VERSION {
LINUX_2.6 {
global:
__vdso_clock_gettime;
__vdso_gettimeofday;
__vdso_getcpu;
__vdso_time;
__vdso_clock_getres;
#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
__vdso_futex_robust_list64_try_unlock;
#ifdef CONFIG_COMPAT
__vdso_futex_robust_list32_try_unlock;
#endif
#endif
local: *;
};
}
linux-7.2.3/arch/x86/entry/vdso/vdso64/vfutex.c
+1
View File
@@ -0,0 +1 @@
#include "common/vfutex.c"
linux-7.2.3/arch/x86/entry/vdso/vdso64/vgetcpu.c
+1
View File
@@ -0,0 +1 @@
#include "common/vgetcpu.c"
linux-7.2.3/arch/x86/entry/vdso/vdso64/vgetrandom-chacha.S
+178
View File
@@ -0,0 +1,178 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2022-2024 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*/
#include <linux/linkage.h>
#include <asm/frame.h>
.section .rodata, "a"
.align 16
CONSTANTS: .octa 0x6b20657479622d323320646e61707865
.text
/*
* Very basic SSE2 implementation of ChaCha20. Produces a given positive number
* of blocks of output with a nonce of 0, taking an input key and 8-byte
* counter. Importantly does not spill to the stack. Its arguments are:
*
* rdi: output bytes
* rsi: 32-byte key input
* rdx: 8-byte counter input/output
* rcx: number of 64-byte blocks to write to output
*/
SYM_FUNC_START(__arch_chacha20_blocks_nostack)
.set output, %rdi
.set key, %rsi
.set counter, %rdx
.set nblocks, %rcx
.set i, %al
/* xmm registers are *not* callee-save. */
.set temp, %xmm0
.set state0, %xmm1
.set state1, %xmm2
.set state2, %xmm3
.set state3, %xmm4
.set copy0, %xmm5
.set copy1, %xmm6
.set copy2, %xmm7
.set copy3, %xmm8
.set one, %xmm9
/* copy0 = "expand 32-byte k" */
movaps CONSTANTS(%rip),copy0
/* copy1,copy2 = key */
movups 0x00(key),copy1
movups 0x10(key),copy2
/* copy3 = counter || zero nonce */
movq 0x00(counter),copy3
/* one = 1 || 0 */
movq $1,%rax
movq %rax,one
.Lblock:
/* state0,state1,state2,state3 = copy0,copy1,copy2,copy3 */
movdqa copy0,state0
movdqa copy1,state1
movdqa copy2,state2
movdqa copy3,state3
movb $10,i
.Lpermute:
/* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
paddd state1,state0
pxor state0,state3
movdqa state3,temp
pslld $16,temp
psrld $16,state3
por temp,state3
/* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
paddd state3,state2
pxor state2,state1
movdqa state1,temp
pslld $12,temp
psrld $20,state1
por temp,state1
/* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
paddd state1,state0
pxor state0,state3
movdqa state3,temp
pslld $8,temp
psrld $24,state3
por temp,state3
/* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
paddd state3,state2
pxor state2,state1
movdqa state1,temp
pslld $7,temp
psrld $25,state1
por temp,state1
/* state1[0,1,2,3] = state1[1,2,3,0] */
pshufd $0x39,state1,state1
/* state2[0,1,2,3] = state2[2,3,0,1] */
pshufd $0x4e,state2,state2
/* state3[0,1,2,3] = state3[3,0,1,2] */
pshufd $0x93,state3,state3
/* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
paddd state1,state0
pxor state0,state3
movdqa state3,temp
pslld $16,temp
psrld $16,state3
por temp,state3
/* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
paddd state3,state2
pxor state2,state1
movdqa state1,temp
pslld $12,temp
psrld $20,state1
por temp,state1
/* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
paddd state1,state0
pxor state0,state3
movdqa state3,temp
pslld $8,temp
psrld $24,state3
por temp,state3
/* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
paddd state3,state2
pxor state2,state1
movdqa state1,temp
pslld $7,temp
psrld $25,state1
por temp,state1
/* state1[0,1,2,3] = state1[3,0,1,2] */
pshufd $0x93,state1,state1
/* state2[0,1,2,3] = state2[2,3,0,1] */
pshufd $0x4e,state2,state2
/* state3[0,1,2,3] = state3[1,2,3,0] */
pshufd $0x39,state3,state3
decb i
jnz .Lpermute
/* output0 = state0 + copy0 */
paddd copy0,state0
movups state0,0x00(output)
/* output1 = state1 + copy1 */
paddd copy1,state1
movups state1,0x10(output)
/* output2 = state2 + copy2 */
paddd copy2,state2
movups state2,0x20(output)
/* output3 = state3 + copy3 */
paddd copy3,state3
movups state3,0x30(output)
/* ++copy3.counter */
paddq one,copy3
/* output += 64, --nblocks */
addq $64,output
decq nblocks
jnz .Lblock
/* counter = copy3.counter */
movq copy3,0x00(counter)
/* Zero out the potentially sensitive regs, in case nothing uses these again. */
pxor state0,state0
pxor state1,state1
pxor state2,state2
pxor state3,state3
pxor copy1,copy1
pxor copy2,copy2
pxor temp,temp
ret
SYM_FUNC_END(__arch_chacha20_blocks_nostack)
linux-7.2.3/arch/x86/entry/vdso/vdso64/vgetrandom.c
+15
View File
@@ -0,0 +1,15 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2022-2024 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*/
#include <linux/types.h>
#include "lib/vdso/getrandom.c"
ssize_t __vdso_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len)
{
return __cvdso_getrandom(buffer, len, flags, opaque_state, opaque_len);
}
ssize_t getrandom(void *, size_t, unsigned int, void *, size_t)
__attribute__((weak, alias("__vdso_getrandom")));
linux-7.2.3/arch/x86/entry/vdso/vdso64/vsgx.S
+150
View File
@@ -0,0 +1,150 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/linkage.h>
#include <asm/errno.h>
#include <asm/enclu.h>
#include "extable.h"
/* Relative to %rbp. */
#define SGX_ENCLAVE_OFFSET_OF_RUN 16
/* The offsets relative to struct sgx_enclave_run. */
#define SGX_ENCLAVE_RUN_TCS 0
#define SGX_ENCLAVE_RUN_LEAF 8
#define SGX_ENCLAVE_RUN_EXCEPTION_VECTOR 12
#define SGX_ENCLAVE_RUN_EXCEPTION_ERROR_CODE 14
#define SGX_ENCLAVE_RUN_EXCEPTION_ADDR 16
#define SGX_ENCLAVE_RUN_USER_HANDLER 24
#define SGX_ENCLAVE_RUN_USER_DATA 32 /* not used */
#define SGX_ENCLAVE_RUN_RESERVED_START 40
#define SGX_ENCLAVE_RUN_RESERVED_END 256
.code64
.section .text, "ax"
SYM_FUNC_START(__vdso_sgx_enter_enclave)
/* Prolog */
.cfi_startproc
push %rbp
.cfi_adjust_cfa_offset 8
.cfi_rel_offset %rbp, 0
mov %rsp, %rbp
.cfi_def_cfa_register %rbp
push %rbx
.cfi_rel_offset %rbx, -8
mov %ecx, %eax
.Lenter_enclave:
/* EENTER <= function <= ERESUME */
cmp $EENTER, %eax
jb .Linvalid_input
cmp $ERESUME, %eax
ja .Linvalid_input
mov SGX_ENCLAVE_OFFSET_OF_RUN(%rbp), %rcx
/* Validate that the reserved area contains only zeros. */
mov $SGX_ENCLAVE_RUN_RESERVED_START, %rbx
1:
cmpq $0, (%rcx, %rbx)
jne .Linvalid_input
add $8, %rbx
cmpq $SGX_ENCLAVE_RUN_RESERVED_END, %rbx
jne 1b
/* Load TCS and AEP */
mov SGX_ENCLAVE_RUN_TCS(%rcx), %rbx
lea .Lasync_exit_pointer(%rip), %rcx
/* Single ENCLU serving as both EENTER and AEP (ERESUME) */
.Lasync_exit_pointer:
.Lenclu_eenter_eresume:
enclu
/* EEXIT jumps here unless the enclave is doing something fancy. */
mov SGX_ENCLAVE_OFFSET_OF_RUN(%rbp), %rbx
/* Set exit_reason. */
movl $EEXIT, SGX_ENCLAVE_RUN_LEAF(%rbx)
/* Invoke userspace's exit handler if one was provided. */
.Lhandle_exit:
cmpq $0, SGX_ENCLAVE_RUN_USER_HANDLER(%rbx)
jne .Linvoke_userspace_handler
/* Success, in the sense that ENCLU was attempted. */
xor %eax, %eax
.Lout:
pop %rbx
leave
.cfi_def_cfa %rsp, 8
RET
/* The out-of-line code runs with the pre-leave stack frame. */
.cfi_def_cfa %rbp, 16
.Linvalid_input:
mov $(-EINVAL), %eax
jmp .Lout
.Lhandle_exception:
mov SGX_ENCLAVE_OFFSET_OF_RUN(%rbp), %rbx
/* Set the exception info. */
mov %eax, (SGX_ENCLAVE_RUN_LEAF)(%rbx)
mov %di, (SGX_ENCLAVE_RUN_EXCEPTION_VECTOR)(%rbx)
mov %si, (SGX_ENCLAVE_RUN_EXCEPTION_ERROR_CODE)(%rbx)
mov %rdx, (SGX_ENCLAVE_RUN_EXCEPTION_ADDR)(%rbx)
jmp .Lhandle_exit
.Linvoke_userspace_handler:
/* Pass the untrusted RSP (at exit) to the callback via %rcx. */
mov %rsp, %rcx
/* Save struct sgx_enclave_exception %rbx is about to be clobbered. */
mov %rbx, %rax
/* Save the untrusted RSP offset in %rbx (non-volatile register). */
mov %rsp, %rbx
and $0xf, %rbx
/*
* Align stack per x86_64 ABI. Note, %rsp needs to be 16-byte aligned
* _after_ pushing the parameters on the stack, hence the bonus push.
*/
and $-0x10, %rsp
push %rax
/* Push struct sgx_enclave_exception as a param to the callback. */
push %rax
/* Clear RFLAGS.DF per x86_64 ABI */
cld
/*
* Load the callback pointer to %rax and lfence for LVI (load value
* injection) protection before making the call.
*/
mov SGX_ENCLAVE_RUN_USER_HANDLER(%rax), %rax
lfence
call *%rax
/* Undo the post-exit %rsp adjustment. */
lea 0x10(%rsp, %rbx), %rsp
/*
* If the return from callback is zero or negative, return immediately,
* else re-execute ENCLU with the positive return value interpreted as
* the requested ENCLU function.
*/
cmp $0, %eax
jle .Lout
jmp .Lenter_enclave
.cfi_endproc
_ASM_VDSO_EXTABLE_HANDLE(.Lenclu_eenter_eresume, .Lhandle_exception)
SYM_FUNC_END(__vdso_sgx_enter_enclave)
linux-7.2.3/arch/x86/entry/vdso/vma.c
+306
View File
@@ -0,0 +1,306 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright 2007 Andi Kleen, SUSE Labs.
*
* This contains most of the x86 vDSO kernel-side code.
*/
#include <linux/mm.h>
#include <linux/err.h>
#include <linux/futex.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/random.h>
#include <linux/elf.h>
#include <linux/cpu.h>
#include <linux/ptrace.h>
#include <linux/vdso_datastore.h>
#include <asm/pvclock.h>
#include <asm/vgtod.h>
#include <asm/proto.h>
#include <asm/vdso.h>
#include <asm/tlb.h>
#include <asm/page.h>
#include <asm/desc.h>
#include <asm/cpufeature.h>
#include <asm/vdso/vsyscall.h>
#include <clocksource/hyperv_timer.h>
static_assert(VDSO_NR_PAGES + VDSO_NR_VCLOCK_PAGES == __VDSO_PAGES);
unsigned int vclocks_used __read_mostly;
#if defined(CONFIG_X86_64)
unsigned int __read_mostly vdso64_enabled = 1;
#endif
int __init init_vdso_image(const struct vdso_image *image)
{
BUILD_BUG_ON(VDSO_CLOCKMODE_MAX >= 32);
BUG_ON(image->size % PAGE_SIZE != 0);
apply_alternatives((struct alt_instr *)(image->data + image->alt),
(struct alt_instr *)(image->data + image->alt +
image->alt_len));
return 0;
}
struct linux_binprm;
static vm_fault_t vdso_fault(const struct vm_special_mapping *sm,
struct vm_area_struct *vma, struct vm_fault *vmf)
{
const struct vdso_image *image = vma->vm_mm->context.vdso_image;
if (!image || (vmf->pgoff << PAGE_SHIFT) >= image->size)
return VM_FAULT_SIGBUS;
vmf->page = virt_to_page(image->data + (vmf->pgoff << PAGE_SHIFT));
get_page(vmf->page);
return 0;
}
static void vdso_fix_landing(const struct vdso_image *image,
struct vm_area_struct *new_vma)
{
struct pt_regs *regs = current_pt_regs();
unsigned long ipoffset = regs->ip -
(unsigned long)current->mm->context.vdso;
if (ipoffset < image->size)
regs->ip = new_vma->vm_start + ipoffset;
}
#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
static void vdso_futex_robust_unlock_update_ips(void)
{
const struct vdso_image *image = current->mm->context.vdso_image;
unsigned long vdso = (unsigned long) current->mm->context.vdso;
struct futex_mm_data *fd = &current->mm->futex;
unsigned int idx = 0;
futex_reset_cs_ranges(fd);
#ifdef CONFIG_X86_64
futex_set_vdso_cs_range(fd, idx, vdso + image->sym___futex_list64_try_unlock_cs_start,
vdso + image->sym___futex_list64_try_unlock_cs_end, false);
idx++;
#endif /* CONFIG_X86_64 */
#if defined(CONFIG_X86_32) || defined(CONFIG_COMPAT)
futex_set_vdso_cs_range(fd, idx, vdso + image->sym___futex_list32_try_unlock_cs_start,
vdso + image->sym___futex_list32_try_unlock_cs_end, true);
#endif /* CONFIG_X86_32 || CONFIG_COMPAT */
}
#else
static inline void vdso_futex_robust_unlock_update_ips(void) { }
#endif
static int vdso_mremap(const struct vm_special_mapping *sm,
struct vm_area_struct *new_vma)
{
const struct vdso_image *image = current->mm->context.vdso_image;
vdso_fix_landing(image, new_vma);
current->mm->context.vdso = (void __user *)new_vma->vm_start;
vdso_futex_robust_unlock_update_ips();
return 0;
}
static vm_fault_t vvar_vclock_fault(const struct vm_special_mapping *sm,
struct vm_area_struct *vma, struct vm_fault *vmf)
{
switch (vmf->pgoff) {
case VDSO_PAGE_PVCLOCK_OFFSET:
{
struct pvclock_vsyscall_time_info *pvti =
pvclock_get_pvti_cpu0_va();
if (pvti && vclock_was_used(VDSO_CLOCKMODE_PVCLOCK))
return vmf_insert_pfn_prot(vma, vmf->address,
__pa(pvti) >> PAGE_SHIFT,
pgprot_decrypted(vma->vm_page_prot));
break;
}
case VDSO_PAGE_HVCLOCK_OFFSET:
{
unsigned long pfn = hv_get_tsc_pfn();
if (pfn && vclock_was_used(VDSO_CLOCKMODE_HVCLOCK))
return vmf_insert_pfn(vma, vmf->address, pfn);
break;
}
}
return VM_FAULT_SIGBUS;
}
static const struct vm_special_mapping vdso_mapping = {
.name = "[vdso]",
.fault = vdso_fault,
.mremap = vdso_mremap,
};
static const struct vm_special_mapping vvar_vclock_mapping = {
.name = "[vvar_vclock]",
.fault = vvar_vclock_fault,
};
/*
* Add vdso and vvar mappings to current process.
* @image - blob to map
* @addr - request a specific address (zero to map at free addr)
*/
static int map_vdso(const struct vdso_image *image, unsigned long addr)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned long text_start;
int ret = 0;
if (mmap_write_lock_killable(mm))
return -EINTR;
addr = get_unmapped_area(NULL, addr,
image->size + __VDSO_PAGES * PAGE_SIZE, 0, 0);
if (IS_ERR_VALUE(addr)) {
ret = addr;
goto up_fail;
}
text_start = addr + __VDSO_PAGES * PAGE_SIZE;
/*
* MAYWRITE to allow gdb to COW and set breakpoints
*/
vma = _install_special_mapping(mm,
text_start,
image->size,
VM_READ|VM_EXEC|
VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC|
VM_SEALED_SYSMAP,
&vdso_mapping);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
goto up_fail;
}
vma = vdso_install_vvar_mapping(mm, addr);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
do_munmap(mm, text_start, image->size, NULL);
goto up_fail;
}
vma = _install_special_mapping(mm,
VDSO_VCLOCK_PAGES_START(addr),
VDSO_NR_VCLOCK_PAGES * PAGE_SIZE,
VM_READ|VM_MAYREAD|VM_IO|VM_DONTDUMP|
VM_PFNMAP|VM_SEALED_SYSMAP,
&vvar_vclock_mapping);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
do_munmap(mm, text_start, image->size, NULL);
do_munmap(mm, addr, VDSO_NR_PAGES * PAGE_SIZE, NULL);
goto up_fail;
}
current->mm->context.vdso = (void __user *)text_start;
current->mm->context.vdso_image = image;
vdso_futex_robust_unlock_update_ips();
up_fail:
mmap_write_unlock(mm);
return ret;
}
int map_vdso_once(const struct vdso_image *image, unsigned long addr)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
VMA_ITERATOR(vmi, mm, 0);
mmap_write_lock(mm);
/*
* Check if we have already mapped vdso blob - fail to prevent
* abusing from userspace install_special_mapping, which may
* not do accounting and rlimit right.
* We could search vma near context.vdso, but it's a slowpath,
* so let's explicitly check all VMAs to be completely sure.
*/
for_each_vma(vmi, vma) {
if (vma_is_special_mapping(vma, &vdso_mapping) ||
vma_is_special_mapping(vma, &vdso_vvar_mapping) ||
vma_is_special_mapping(vma, &vvar_vclock_mapping)) {
mmap_write_unlock(mm);
return -EEXIST;
}
}
mmap_write_unlock(mm);
return map_vdso(image, addr);
}
static int load_vdso32(void)
{
if (vdso32_enabled != 1) /* Other values all mean "disabled" */
return 0;
return map_vdso(&vdso32_image, 0);
}
int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
{
if (IS_ENABLED(CONFIG_X86_64)) {
if (!vdso64_enabled)
return 0;
return map_vdso(&vdso64_image, 0);
}
return load_vdso32();
}
#ifdef CONFIG_COMPAT
int compat_arch_setup_additional_pages(struct linux_binprm *bprm,
int uses_interp, bool x32)
{
if (IS_ENABLED(CONFIG_X86_X32_ABI) && x32) {
if (!vdso64_enabled)
return 0;
return map_vdso(&vdsox32_image, 0);
}
if (IS_ENABLED(CONFIG_IA32_EMULATION))
return load_vdso32();
return 0;
}
#endif
bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs)
{
const struct vdso_image *image = current->mm->context.vdso_image;
unsigned long vdso = (unsigned long) current->mm->context.vdso;
if (in_ia32_syscall() && image == &vdso32_image) {
if (regs->ip == vdso + image->sym_vdso32_sigreturn_landing_pad ||
regs->ip == vdso + image->sym_vdso32_rt_sigreturn_landing_pad)
return true;
}
return false;
}
#ifdef CONFIG_X86_64
static __init int vdso_setup(char *s)
{
vdso64_enabled = simple_strtoul(s, NULL, 0);
return 1;
}
__setup("vdso=", vdso_setup);
#endif /* CONFIG_X86_64 */
linux-7.2.3/arch/x86/entry/vsyscall/Makefile
+6
View File
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# Makefile for the x86 low level vsyscall code
#
obj-$(CONFIG_X86_VSYSCALL_EMULATION) += vsyscall_64.o vsyscall_emu_64.o
linux-7.2.3/arch/x86/entry/vsyscall/vsyscall_64.c
+403
View File
@@ -0,0 +1,403 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2012-2014 Andy Lutomirski <luto@amacapital.net>
*
* Based on the original implementation which is:
* Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
* Copyright 2003 Andi Kleen, SuSE Labs.
*
* Parts of the original code have been moved to arch/x86/vdso/vma.c
*
* This file implements vsyscall emulation. vsyscalls are a legacy ABI:
* Userspace can request certain kernel services by calling fixed
* addresses. This concept is problematic:
*
* - It interferes with ASLR.
* - It's awkward to write code that lives in kernel addresses but is
* callable by userspace at fixed addresses.
* - The whole concept is impossible for 32-bit compat userspace.
* - UML cannot easily virtualize a vsyscall.
*
* As of mid-2014, I believe that there is no new userspace code that
* will use a vsyscall if the vDSO is present. I hope that there will
* soon be no new userspace code that will ever use a vsyscall.
*
* The code in this file emulates vsyscalls when notified of a page
* fault or a general protection fault to a vsyscall address.
*/
#include <linux/kernel.h>
#include <linux/timer.h>
#include <linux/sched/signal.h>
#include <linux/mm_types.h>
#include <linux/syscalls.h>
#include <linux/ratelimit.h>
#include <asm/vsyscall.h>
#include <asm/unistd.h>
#include <asm/fixmap.h>
#include <asm/traps.h>
#define CREATE_TRACE_POINTS
#include "vsyscall_trace.h"
static enum { EMULATE, XONLY, NONE } vsyscall_mode __ro_after_init =
#ifdef CONFIG_LEGACY_VSYSCALL_NONE
NONE;
#elif defined(CONFIG_LEGACY_VSYSCALL_XONLY)
XONLY;
#else
#error VSYSCALL config is broken
#endif
static int __init vsyscall_setup(char *str)
{
if (str) {
if (!strcmp("emulate", str))
vsyscall_mode = EMULATE;
else if (!strcmp("xonly", str))
vsyscall_mode = XONLY;
else if (!strcmp("none", str))
vsyscall_mode = NONE;
else
return -EINVAL;
if (cpu_feature_enabled(X86_FEATURE_LASS) && vsyscall_mode == EMULATE) {
setup_clear_cpu_cap(X86_FEATURE_LASS);
pr_warn_once("x86/cpu: Disabling LASS due to vsyscall=emulate\n");
}
return 0;
}
return -EINVAL;
}
early_param("vsyscall", vsyscall_setup);
static void warn_bad_vsyscall(const char *level, struct pt_regs *regs,
const char *message)
{
if (!show_unhandled_signals)
return;
printk_ratelimited("%s%s[%d] %s ip:%lx cs:%x sp:%lx ax:%lx si:%lx di:%lx\n",
level, current->comm, task_pid_nr(current),
message, regs->ip, regs->cs,
regs->sp, regs->ax, regs->si, regs->di);
}
static int addr_to_vsyscall_nr(unsigned long addr)
{
int nr;
if ((addr & ~0xC00UL) != VSYSCALL_ADDR)
return -EINVAL;
nr = (addr & 0xC00UL) >> 10;
if (nr >= 3)
return -EINVAL;
return nr;
}
static bool write_ok_or_segv(unsigned long ptr, size_t size)
{
if (!access_ok((void __user *)ptr, size)) {
struct thread_struct *thread = &current->thread;
thread->error_code = X86_PF_USER | X86_PF_WRITE;
thread->cr2 = ptr;
thread->trap_nr = X86_TRAP_PF;
force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)ptr);
return false;
} else {
return true;
}
}
static bool __emulate_vsyscall(struct pt_regs *regs, unsigned long address)
{
unsigned long caller;
int vsyscall_nr, syscall_nr, tmp;
long ret;
unsigned long orig_dx;
/* Confirm that the fault happened in 64-bit user mode */
if (!user_64bit_mode(regs))
return false;
if (vsyscall_mode == NONE) {
warn_bad_vsyscall(KERN_INFO, regs,
"vsyscall attempted with vsyscall=none");
return false;
}
vsyscall_nr = addr_to_vsyscall_nr(address);
trace_emulate_vsyscall(vsyscall_nr);
if (vsyscall_nr < 0) {
warn_bad_vsyscall(KERN_WARNING, regs,
"misaligned vsyscall (exploit attempt or buggy program) -- look up the vsyscall kernel parameter if you need a workaround");
goto sigsegv;
}
if (get_user(caller, (unsigned long __user *)regs->sp) != 0) {
warn_bad_vsyscall(KERN_WARNING, regs,
"vsyscall with bad stack (exploit attempt?)");
goto sigsegv;
}
/*
* Check for access_ok violations and find the syscall nr.
*
* NULL is a valid user pointer (in the access_ok sense) on 32-bit and
* 64-bit, so we don't need to special-case it here. For all the
* vsyscalls, NULL means "don't write anything" not "write it at
* address 0".
*/
switch (vsyscall_nr) {
case 0:
if (!write_ok_or_segv(regs->di, sizeof(struct __kernel_old_timeval)) ||
!write_ok_or_segv(regs->si, sizeof(struct timezone))) {
ret = -EFAULT;
goto check_fault;
}
syscall_nr = __NR_gettimeofday;
break;
case 1:
if (!write_ok_or_segv(regs->di, sizeof(__kernel_old_time_t))) {
ret = -EFAULT;
goto check_fault;
}
syscall_nr = __NR_time;
break;
case 2:
if (!write_ok_or_segv(regs->di, sizeof(unsigned)) ||
!write_ok_or_segv(regs->si, sizeof(unsigned))) {
ret = -EFAULT;
goto check_fault;
}
syscall_nr = __NR_getcpu;
break;
}
/*
* Handle seccomp. regs->ip must be the original value.
* See seccomp_send_sigsys and Documentation/userspace-api/seccomp_filter.rst.
*
* We could optimize the seccomp disabled case, but performance
* here doesn't matter.
*/
regs->orig_ax = syscall_nr;
regs->ax = -ENOSYS;
tmp = secure_computing();
if ((!tmp && regs->orig_ax != syscall_nr) || regs->ip != address) {
warn_bad_vsyscall(KERN_DEBUG, regs,
"seccomp tried to change syscall nr or ip");
force_exit_sig(SIGSYS);
return true;
}
regs->orig_ax = -1;
if (tmp)
goto do_ret; /* skip requested */
/*
* With a real vsyscall, page faults cause SIGSEGV.
*/
ret = -EFAULT;
switch (vsyscall_nr) {
case 0:
/* this decodes regs->di and regs->si on its own */
ret = __x64_sys_gettimeofday(regs);
break;
case 1:
/* this decodes regs->di on its own */
ret = __x64_sys_time(regs);
break;
case 2:
/* while we could clobber regs->dx, we didn't in the past... */
orig_dx = regs->dx;
regs->dx = 0;
/* this decodes regs->di, regs->si and regs->dx on its own */
ret = __x64_sys_getcpu(regs);
regs->dx = orig_dx;
break;
}
check_fault:
if (ret == -EFAULT) {
/* Bad news -- userspace fed a bad pointer to a vsyscall. */
warn_bad_vsyscall(KERN_INFO, regs,
"vsyscall fault (exploit attempt?)");
goto sigsegv;
}
regs->ax = ret;
do_ret:
/* Emulate a ret instruction. */
regs->ip = caller;
regs->sp += 8;
return true;
sigsegv:
force_sig(SIGSEGV);
return true;
}
bool emulate_vsyscall_pf(unsigned long error_code, struct pt_regs *regs,
unsigned long address)
{
/* Write faults or kernel-privilege faults never get fixed up. */
if ((error_code & (X86_PF_WRITE | X86_PF_USER)) != X86_PF_USER)
return false;
/*
* Assume that faults at regs->ip are because of an instruction
* fetch. Return early and avoid emulation for faults during
* data accesses:
*/
if (address != regs->ip) {
/* Failed vsyscall read */
if (vsyscall_mode == EMULATE)
return false;
/* User code tried and failed to read the vsyscall page. */
warn_bad_vsyscall(KERN_INFO, regs,
"vsyscall read attempt denied -- look up the vsyscall kernel parameter if you need a workaround");
return false;
}
/*
* X86_PF_INSTR is only set when NX is supported. When
* available, use it to double-check that the emulation code
* is only being used for instruction fetches:
*/
if (cpu_feature_enabled(X86_FEATURE_NX))
WARN_ON_ONCE(!(error_code & X86_PF_INSTR));
return __emulate_vsyscall(regs, address);
}
bool emulate_vsyscall_gp(struct pt_regs *regs)
{
/* Without LASS, vsyscall accesses are expected to generate a #PF */
if (!cpu_feature_enabled(X86_FEATURE_LASS))
return false;
/* Emulate only if the RIP points to the vsyscall address */
if (!is_vsyscall_vaddr(regs->ip))
return false;
return __emulate_vsyscall(regs, regs->ip);
}
/*
* A pseudo VMA to allow ptrace access for the vsyscall page. This only
* covers the 64bit vsyscall page now. 32bit has a real VMA now and does
* not need special handling anymore:
*/
static const char *gate_vma_name(struct vm_area_struct *vma)
{
return "[vsyscall]";
}
static const struct vm_operations_struct gate_vma_ops = {
.name = gate_vma_name,
};
static struct vm_area_struct gate_vma __ro_after_init = {
.vm_start = VSYSCALL_ADDR,
.vm_end = VSYSCALL_ADDR + PAGE_SIZE,
.vm_page_prot = PAGE_READONLY_EXEC,
.vm_flags = VM_READ | VM_EXEC,
.vm_ops = &gate_vma_ops,
};
struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
{
#ifdef CONFIG_COMPAT
if (!mm || !test_bit(MM_CONTEXT_HAS_VSYSCALL, &mm->context.flags))
return NULL;
#endif
if (vsyscall_mode == NONE)
return NULL;
return &gate_vma;
}
int in_gate_area(struct mm_struct *mm, unsigned long addr)
{
struct vm_area_struct *vma = get_gate_vma(mm);
if (!vma)
return 0;
return (addr >= vma->vm_start) && (addr < vma->vm_end);
}
/*
* Use this when you have no reliable mm, typically from interrupt
* context. It is less reliable than using a task's mm and may give
* false positives.
*/
int in_gate_area_no_mm(unsigned long addr)
{
return vsyscall_mode != NONE && (addr & PAGE_MASK) == VSYSCALL_ADDR;
}
/*
* The VSYSCALL page is the only user-accessible page in the kernel address
* range. Normally, the kernel page tables can have _PAGE_USER clear, but
* the tables covering VSYSCALL_ADDR need _PAGE_USER set if vsyscalls
* are enabled.
*
* Some day we may create a "minimal" vsyscall mode in which we emulate
* vsyscalls but leave the page not present. If so, we skip calling
* this.
*/
void __init set_vsyscall_pgtable_user_bits(pgd_t *root)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pgd = pgd_offset_pgd(root, VSYSCALL_ADDR);
set_pgd(pgd, __pgd(pgd_val(*pgd) | _PAGE_USER));
p4d = p4d_offset(pgd, VSYSCALL_ADDR);
set_p4d(p4d, __p4d(p4d_val(*p4d) | _PAGE_USER));
pud = pud_offset(p4d, VSYSCALL_ADDR);
set_pud(pud, __pud(pud_val(*pud) | _PAGE_USER));
pmd = pmd_offset(pud, VSYSCALL_ADDR);
set_pmd(pmd, __pmd(pmd_val(*pmd) | _PAGE_USER));
}
void __init map_vsyscall(void)
{
extern char __vsyscall_page;
unsigned long physaddr_vsyscall = __pa_symbol(&__vsyscall_page);
/*
* For full emulation, the page needs to exist for real. In
* execute-only mode, there is no PTE at all backing the vsyscall
* page.
*/
if (vsyscall_mode == EMULATE) {
__set_fixmap(VSYSCALL_PAGE, physaddr_vsyscall,
PAGE_KERNEL_VVAR);
set_vsyscall_pgtable_user_bits(swapper_pg_dir);
}
if (vsyscall_mode == XONLY)
vm_flags_init(&gate_vma, VM_EXEC);
BUILD_BUG_ON((unsigned long)__fix_to_virt(VSYSCALL_PAGE) !=
(unsigned long)VSYSCALL_ADDR);
}
linux-7.2.3/arch/x86/entry/vsyscall/vsyscall_emu_64.S
+39
View File
@@ -0,0 +1,39 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* vsyscall_emu_64.S: Vsyscall emulation page
*
* Copyright (c) 2011 Andy Lutomirski
*/
#include <linux/linkage.h>
#include <asm/irq_vectors.h>
#include <asm/page_types.h>
#include <asm/unistd_64.h>
__PAGE_ALIGNED_DATA
.globl __vsyscall_page
.balign PAGE_SIZE, 0xcc
.type __vsyscall_page, @object
__vsyscall_page:
mov $__NR_gettimeofday, %rax
syscall
ret
int3
.balign 1024, 0xcc
mov $__NR_time, %rax
syscall
ret
int3
.balign 1024, 0xcc
mov $__NR_getcpu, %rax
syscall
ret
int3
.balign 4096, 0xcc
.size __vsyscall_page, 4096
linux-7.2.3/arch/x86/entry/vsyscall/vsyscall_trace.h
+30
View File
@@ -0,0 +1,30 @@
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM vsyscall
#if !defined(__VSYSCALL_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
#define __VSYSCALL_TRACE_H
#include <linux/tracepoint.h>
TRACE_EVENT(emulate_vsyscall,
TP_PROTO(int nr),
TP_ARGS(nr),
TP_STRUCT__entry(__field(int, nr)),
TP_fast_assign(
__entry->nr = nr;
),
TP_printk("nr = %d", __entry->nr)
);
#endif
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH ../../arch/x86/entry/vsyscall/
#define TRACE_INCLUDE_FILE vsyscall_trace
#include <trace/define_trace.h>