lastdragon/linux-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resguardo inicial

Browse Source
This commit is contained in:
lastdragon
2026-09-06 12:53:21 -06:00
parent 662efe3c2a
commit ad2088d178
94853 changed files with 43200404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linux-7.2.3/arch/powerpc/include/asm/book3s/64/hash-4k.h
+173
View File
@@ -0,0 +1,173 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_BOOK3S_64_HASH_4K_H
#define _ASM_POWERPC_BOOK3S_64_HASH_4K_H
#define H_PTE_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps: 2^9 x 4KB = 2MB
#define H_PMD_INDEX_SIZE 7 // size: 8B << 7 = 1KB, maps: 2^7 x 2MB = 256MB
#define H_PUD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps: 2^9 x 256MB = 128GB
#define H_PGD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps: 2^9 x 128GB = 64TB
/*
* Each context is 512TB. But on 4k we restrict our max TASK size to 64TB
* Hence also limit max EA bits to 64TB.
*/
#define MAX_EA_BITS_PER_CONTEXT 46
/*
* Our page table limit us to 64TB. For 64TB physical memory, we only need 64GB
* of vmemmap space. To better support sparse memory layout, we use 61TB
* linear map range, 1TB of vmalloc, 1TB of I/O and 1TB of vmememmap.
*/
#define REGION_SHIFT (40)
#define H_KERN_MAP_SIZE (ASM_CONST(1) << REGION_SHIFT)
/*
* Limits the linear mapping range
*/
#define H_MAX_PHYSMEM_BITS 46
/*
* Define the address range of the kernel non-linear virtual area (61TB)
*/
#define H_KERN_VIRT_START ASM_CONST(0xc0003d0000000000)
#ifndef __ASSEMBLER__
#define H_PTE_TABLE_SIZE (sizeof(pte_t) << H_PTE_INDEX_SIZE)
#define H_PMD_TABLE_SIZE (sizeof(pmd_t) << H_PMD_INDEX_SIZE)
#define H_PUD_TABLE_SIZE (sizeof(pud_t) << H_PUD_INDEX_SIZE)
#define H_PGD_TABLE_SIZE (sizeof(pgd_t) << H_PGD_INDEX_SIZE)
#define H_PAGE_F_GIX_SHIFT _PAGE_PA_MAX
#define H_PAGE_F_SECOND _RPAGE_PKEY_BIT0 /* HPTE is in 2ndary HPTEG */
#define H_PAGE_F_GIX (_RPAGE_RPN43 | _RPAGE_RPN42 | _RPAGE_RPN41)
#define H_PAGE_BUSY _RPAGE_RSV1
#define H_PAGE_HASHPTE _RPAGE_PKEY_BIT4
/* PTE flags to conserve for HPTE identification */
#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | \
H_PAGE_F_SECOND | H_PAGE_F_GIX)
/*
* Not supported by 4k linux page size
*/
#define H_PAGE_4K_PFN 0x0
#define H_PAGE_THP_HUGE 0x0
#define H_PAGE_COMBO 0x0
/* 8 bytes per each pte entry */
#define H_PTE_FRAG_SIZE_SHIFT (H_PTE_INDEX_SIZE + 3)
#define H_PTE_FRAG_NR (PAGE_SIZE >> H_PTE_FRAG_SIZE_SHIFT)
#define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3)
#define H_PMD_FRAG_NR (PAGE_SIZE >> H_PMD_FRAG_SIZE_SHIFT)
/* memory key bits, only 8 keys supported */
#define H_PTE_PKEY_BIT4 0
#define H_PTE_PKEY_BIT3 0
#define H_PTE_PKEY_BIT2 _RPAGE_PKEY_BIT3
#define H_PTE_PKEY_BIT1 _RPAGE_PKEY_BIT2
#define H_PTE_PKEY_BIT0 _RPAGE_PKEY_BIT1
/*
* On all 4K setups, remap_4k_pfn() equates to remap_pfn_range()
*/
#define remap_4k_pfn(vma, addr, pfn, prot) \
remap_pfn_range((vma), (addr), (pfn), PAGE_SIZE, (prot))
/*
* With 4K page size the real_pte machinery is all nops.
*/
static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep, int offset)
{
return (real_pte_t){pte};
}
#define __rpte_to_pte(r) ((r).pte)
static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index)
{
return pte_val(__rpte_to_pte(rpte)) >> H_PAGE_F_GIX_SHIFT;
}
#define pte_iterate_hashed_subpages(rpte, psize, va, index, shift) \
do { \
index = 0; \
shift = mmu_psize_defs[psize].shift; \
#define pte_iterate_hashed_end() } while(0)
/*
* We expect this to be called only for user addresses or kernel virtual
* addresses other than the linear mapping.
*/
#define pte_pagesize_index(mm, addr, pte) MMU_PAGE_4K
/*
* 4K PTE format is different from 64K PTE format. Saving the hash_slot is just
* a matter of returning the PTE bits that need to be modified. On 64K PTE,
* things are a little more involved and hence needs many more parameters to
* accomplish the same. However we want to abstract this out from the caller by
* keeping the prototype consistent across the two formats.
*/
static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte,
unsigned int subpg_index, unsigned long hidx,
int offset)
{
return (hidx << H_PAGE_F_GIX_SHIFT) &
(H_PAGE_F_SECOND | H_PAGE_F_GIX);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline char *get_hpte_slot_array(pmd_t *pmdp)
{
BUG();
return NULL;
}
static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index)
{
BUG();
return 0;
}
static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array,
int index)
{
BUG();
return 0;
}
static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array,
unsigned int index, unsigned int hidx)
{
BUG();
}
static inline int hash__pmd_trans_huge(pmd_t pmd)
{
return 0;
}
static inline pmd_t hash__pmd_mkhuge(pmd_t pmd)
{
BUG();
return pmd;
}
extern unsigned long hash__pmd_hugepage_update(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp,
unsigned long clr, unsigned long set);
extern pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp);
extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
pgtable_t pgtable);
extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp);
extern int hash__has_transparent_hugepage(void);
#endif
#endif /* !__ASSEMBLER__ */
#endif /* _ASM_POWERPC_BOOK3S_64_HASH_4K_H */
linux-7.2.3/arch/powerpc/include/asm/book3s/64/hash-64k.h
+286
View File
@@ -0,0 +1,286 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_BOOK3S_64_HASH_64K_H
#define _ASM_POWERPC_BOOK3S_64_HASH_64K_H
#define H_PTE_INDEX_SIZE 8 // size: 8B << 8 = 2KB, maps 2^8 x 64KB = 16MB
#define H_PMD_INDEX_SIZE 10 // size: 8B << 10 = 8KB, maps 2^10 x 16MB = 16GB
#define H_PUD_INDEX_SIZE 10 // size: 8B << 10 = 8KB, maps 2^10 x 16GB = 16TB
#define H_PGD_INDEX_SIZE 8 // size: 8B << 8 = 2KB, maps 2^8 x 16TB = 4PB
/*
* If we store section details in page->flags we can't increase the MAX_PHYSMEM_BITS
* if we increase SECTIONS_WIDTH we will not store node details in page->flags and
* page_to_nid does a page->section->node lookup
* Hence only increase for VMEMMAP. Further depending on SPARSEMEM_EXTREME reduce
* memory requirements with large number of sections.
* 51 bits is the max physical real address on POWER9
*/
#if defined(CONFIG_SPARSEMEM_VMEMMAP) && defined(CONFIG_SPARSEMEM_EXTREME)
#define H_MAX_PHYSMEM_BITS 51
#else
#define H_MAX_PHYSMEM_BITS 46
#endif
/*
* Each context is 512TB size. SLB miss for first context/default context
* is handled in the hotpath.
*/
#define MAX_EA_BITS_PER_CONTEXT 49
#define REGION_SHIFT MAX_EA_BITS_PER_CONTEXT
/*
* We use one context for each MAP area.
*/
#define H_KERN_MAP_SIZE (1UL << MAX_EA_BITS_PER_CONTEXT)
/*
* Define the address range of the kernel non-linear virtual area
* 2PB
*/
#define H_KERN_VIRT_START ASM_CONST(0xc008000000000000)
/*
* 64k aligned address free up few of the lower bits of RPN for us
* We steal that here. For more deatils look at pte_pfn/pfn_pte()
*/
#define H_PAGE_COMBO _RPAGE_RPN0 /* this is a combo 4k page */
#define H_PAGE_4K_PFN _RPAGE_RPN1 /* PFN is for a single 4k page */
#define H_PAGE_BUSY _RPAGE_RSV1 /* software: PTE & hash are busy */
#define H_PAGE_HASHPTE _RPAGE_RPN43 /* PTE has associated HPTE */
/* memory key bits. */
#define H_PTE_PKEY_BIT4 _RPAGE_PKEY_BIT4
#define H_PTE_PKEY_BIT3 _RPAGE_PKEY_BIT3
#define H_PTE_PKEY_BIT2 _RPAGE_PKEY_BIT2
#define H_PTE_PKEY_BIT1 _RPAGE_PKEY_BIT1
#define H_PTE_PKEY_BIT0 _RPAGE_PKEY_BIT0
/*
* We need to differentiate between explicit huge page and THP huge
* page, since THP huge page also need to track real subpage details
*/
#define H_PAGE_THP_HUGE H_PAGE_4K_PFN
/* PTE flags to conserve for HPTE identification */
#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | H_PAGE_COMBO)
/*
* We use a 2K PTE page fragment and another 2K for storing
* real_pte_t hash index
* 8 bytes per each pte entry and another 8 bytes for storing
* slot details.
*/
#define H_PTE_FRAG_SIZE_SHIFT (H_PTE_INDEX_SIZE + 3 + 1)
#define H_PTE_FRAG_NR (PAGE_SIZE >> H_PTE_FRAG_SIZE_SHIFT)
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
#define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3 + 1)
#else
#define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3)
#endif
#define H_PMD_FRAG_NR (PAGE_SIZE >> H_PMD_FRAG_SIZE_SHIFT)
#ifndef __ASSEMBLER__
#include <asm/errno.h>
/*
* With 64K pages on hash table, we have a special PTE format that
* uses a second "half" of the page table to encode sub-page information
* in order to deal with 64K made of 4K HW pages. Thus we override the
* generic accessors and iterators here
*/
#define __real_pte __real_pte
static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep, int offset)
{
real_pte_t rpte;
unsigned long *hidxp;
rpte.pte = pte;
/*
* Ensure that we do not read the hidx before we read the PTE. Because
* the writer side is expected to finish writing the hidx first followed
* by the PTE, by using smp_wmb(). pte_set_hash_slot() ensures that.
*/
smp_rmb();
hidxp = (unsigned long *)(ptep + offset);
rpte.hidx = *hidxp;
return rpte;
}
/*
* shift the hidx representation by one-modulo-0xf; i.e hidx 0 is respresented
* as 1, 1 as 2,... , and 0xf as 0. This convention lets us represent a
* invalid hidx 0xf with a 0x0 bit value. PTEs are anyway zero'd when
* allocated. We dont have to zero them gain; thus save on the initialization.
*/
#define HIDX_UNSHIFT_BY_ONE(x) ((x + 0xfUL) & 0xfUL) /* shift backward by one */
#define HIDX_SHIFT_BY_ONE(x) ((x + 0x1UL) & 0xfUL) /* shift forward by one */
#define HIDX_BITS(x, index) (x << (index << 2))
#define BITS_TO_HIDX(x, index) ((x >> (index << 2)) & 0xfUL)
#define INVALID_RPTE_HIDX 0x0UL
static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index)
{
return HIDX_UNSHIFT_BY_ONE(BITS_TO_HIDX(rpte.hidx, index));
}
/*
* Commit the hidx and return PTE bits that needs to be modified. The caller is
* expected to modify the PTE bits accordingly and commit the PTE to memory.
*/
static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte,
unsigned int subpg_index,
unsigned long hidx, int offset)
{
unsigned long *hidxp = (unsigned long *)(ptep + offset);
rpte.hidx &= ~HIDX_BITS(0xfUL, subpg_index);
*hidxp = rpte.hidx | HIDX_BITS(HIDX_SHIFT_BY_ONE(hidx), subpg_index);
/*
* Anyone reading PTE must ensure hidx bits are read after reading the
* PTE by using the read-side barrier smp_rmb(). __real_pte() can be
* used for that.
*/
smp_wmb();
/* No PTE bits to be modified, return 0x0UL */
return 0x0UL;
}
#define __rpte_to_pte(r) ((r).pte)
extern bool __rpte_sub_valid(real_pte_t rpte, unsigned long index);
/*
* Trick: we set __end to va + 64k, which happens works for
* a 16M page as well as we want only one iteration
*/
#define pte_iterate_hashed_subpages(rpte, psize, vpn, index, shift) \
do { \
unsigned long __end = vpn + (1UL << (PAGE_SHIFT - VPN_SHIFT)); \
unsigned __split = (psize == MMU_PAGE_4K || \
psize == MMU_PAGE_64K_AP); \
shift = mmu_psize_defs[psize].shift; \
for (index = 0; vpn < __end; index++, \
vpn += (1L << (shift - VPN_SHIFT))) { \
if (!__split || __rpte_sub_valid(rpte, index))
#define pte_iterate_hashed_end() } } while(0)
#define pte_pagesize_index(mm, addr, pte) \
(((pte) & H_PAGE_COMBO)? MMU_PAGE_4K: MMU_PAGE_64K)
extern int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
unsigned long pfn, unsigned long size, pgprot_t);
static inline int hash__remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr,
unsigned long pfn, pgprot_t prot)
{
if (pfn > (PTE_RPN_MASK >> PAGE_SHIFT)) {
WARN(1, "remap_4k_pfn called with wrong pfn value\n");
return -EINVAL;
}
return remap_pfn_range(vma, addr, pfn, PAGE_SIZE,
__pgprot(pgprot_val(prot) | H_PAGE_4K_PFN));
}
#define H_PTE_TABLE_SIZE PTE_FRAG_SIZE
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined (CONFIG_HUGETLB_PAGE)
#define H_PMD_TABLE_SIZE ((sizeof(pmd_t) << PMD_INDEX_SIZE) + \
(sizeof(unsigned long) << PMD_INDEX_SIZE))
#else
#define H_PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE)
#endif
#ifdef CONFIG_HUGETLB_PAGE
#define H_PUD_TABLE_SIZE ((sizeof(pud_t) << PUD_INDEX_SIZE) + \
(sizeof(unsigned long) << PUD_INDEX_SIZE))
#else
#define H_PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE)
#endif
#define H_PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE)
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline char *get_hpte_slot_array(pmd_t *pmdp)
{
/*
* The hpte hindex is stored in the pgtable whose address is in the
* second half of the PMD
*
* Order this load with the test for pmd_trans_huge in the caller
*/
smp_rmb();
return *(char **)(pmdp + PTRS_PER_PMD);
}
/*
* The linux hugepage PMD now include the pmd entries followed by the address
* to the stashed pgtable_t. The stashed pgtable_t contains the hpte bits.
* [ 000 | 1 bit secondary | 3 bit hidx | 1 bit valid]. We use one byte per
* each HPTE entry. With 16MB hugepage and 64K HPTE we need 256 entries and
* with 4K HPTE we need 4096 entries. Both will fit in a 4K pgtable_t.
*
* The top three bits are intentionally left as zero. This memory location
* are also used as normal page PTE pointers. So if we have any pointers
* left around while we collapse a hugepage, we need to make sure
* _PAGE_PRESENT bit of that is zero when we look at them
*/
static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index)
{
return hpte_slot_array[index] & 0x1;
}
static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array,
int index)
{
return hpte_slot_array[index] >> 1;
}
static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array,
unsigned int index, unsigned int hidx)
{
hpte_slot_array[index] = (hidx << 1) | 0x1;
}
/*
*
* For core kernel code by design pmd_trans_huge is never run on any hugetlbfs
* page. The hugetlbfs page table walking and mangling paths are totally
* separated form the core VM paths and they're differentiated by
* VM_HUGETLB being set on vm_flags well before any pmd_trans_huge could run.
*
* pmd_trans_huge() is defined as false at build time if
* CONFIG_TRANSPARENT_HUGEPAGE=n to optimize away code blocks at build
* time in such case.
*
* For ppc64 we need to differntiate from explicit hugepages from THP, because
* for THP we also track the subpage details at the pmd level. We don't do
* that for explicit huge pages.
*
*/
static inline int hash__pmd_trans_huge(pmd_t pmd)
{
return !!((pmd_val(pmd) & (_PAGE_PTE | H_PAGE_THP_HUGE)) ==
(_PAGE_PTE | H_PAGE_THP_HUGE));
}
static inline pmd_t hash__pmd_mkhuge(pmd_t pmd)
{
return __pmd(pmd_val(pmd) | (_PAGE_PTE | H_PAGE_THP_HUGE));
}
extern unsigned long hash__pmd_hugepage_update(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp,
unsigned long clr, unsigned long set);
extern pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp);
extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
pgtable_t pgtable);
extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp);
extern int hash__has_transparent_hugepage(void);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif /* __ASSEMBLER__ */
#endif /* _ASM_POWERPC_BOOK3S_64_HASH_64K_H */
linux-7.2.3/arch/powerpc/include/asm/book3s/64/hash-pkey.h
+45
View File
@@ -0,0 +1,45 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_BOOK3S_64_HASH_PKEY_H
#define _ASM_POWERPC_BOOK3S_64_HASH_PKEY_H
/* We use key 3 for KERNEL */
#define HASH_DEFAULT_KERNEL_KEY (HPTE_R_KEY_BIT0 | HPTE_R_KEY_BIT1)
static inline u64 hash__vmflag_to_pte_pkey_bits(u64 vm_flags)
{
return (((vm_flags & VM_PKEY_BIT0) ? H_PTE_PKEY_BIT0 : 0x0UL) |
((vm_flags & VM_PKEY_BIT1) ? H_PTE_PKEY_BIT1 : 0x0UL) |
((vm_flags & VM_PKEY_BIT2) ? H_PTE_PKEY_BIT2 : 0x0UL) |
((vm_flags & VM_PKEY_BIT3) ? H_PTE_PKEY_BIT3 : 0x0UL) |
((vm_flags & VM_PKEY_BIT4) ? H_PTE_PKEY_BIT4 : 0x0UL));
}
static inline u64 pte_to_hpte_pkey_bits(u64 pteflags, unsigned long flags)
{
unsigned long pte_pkey;
pte_pkey = (((pteflags & H_PTE_PKEY_BIT4) ? HPTE_R_KEY_BIT4 : 0x0UL) |
((pteflags & H_PTE_PKEY_BIT3) ? HPTE_R_KEY_BIT3 : 0x0UL) |
((pteflags & H_PTE_PKEY_BIT2) ? HPTE_R_KEY_BIT2 : 0x0UL) |
((pteflags & H_PTE_PKEY_BIT1) ? HPTE_R_KEY_BIT1 : 0x0UL) |
((pteflags & H_PTE_PKEY_BIT0) ? HPTE_R_KEY_BIT0 : 0x0UL));
if (mmu_has_feature(MMU_FTR_KUAP) ||
mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) {
if ((pte_pkey == 0) && (flags & HPTE_USE_KERNEL_KEY))
return HASH_DEFAULT_KERNEL_KEY;
}
return pte_pkey;
}
static inline u16 hash__pte_to_pkey_bits(u64 pteflags)
{
return (((pteflags & H_PTE_PKEY_BIT4) ? 0x10 : 0x0UL) |
((pteflags & H_PTE_PKEY_BIT3) ? 0x8 : 0x0UL) |
((pteflags & H_PTE_PKEY_BIT2) ? 0x4 : 0x0UL) |
((pteflags & H_PTE_PKEY_BIT1) ? 0x2 : 0x0UL) |
((pteflags & H_PTE_PKEY_BIT0) ? 0x1 : 0x0UL));
}
#endif
linux-7.2.3/arch/powerpc/include/asm/book3s/64/hash.h
+300
View File
@@ -0,0 +1,300 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_BOOK3S_64_HASH_H
#define _ASM_POWERPC_BOOK3S_64_HASH_H
#ifdef __KERNEL__
#include <asm/asm-const.h>
#include <asm/book3s/64/slice.h>
/*
* Common bits between 4K and 64K pages in a linux-style PTE.
* Additional bits may be defined in pgtable-hash64-*.h
*
*/
#define H_PTE_NONE_MASK _PAGE_HPTEFLAGS
#ifdef CONFIG_PPC_64K_PAGES
#include <asm/book3s/64/hash-64k.h>
#else
#include <asm/book3s/64/hash-4k.h>
#endif
#define H_PTRS_PER_PTE (1 << H_PTE_INDEX_SIZE)
#define H_PTRS_PER_PMD (1 << H_PMD_INDEX_SIZE)
#define H_PTRS_PER_PUD (1 << H_PUD_INDEX_SIZE)
/* Bits to set in a PMD/PUD/PGD entry valid bit*/
#define HASH_PMD_VAL_BITS (0x8000000000000000UL)
#define HASH_PUD_VAL_BITS (0x8000000000000000UL)
#define HASH_PGD_VAL_BITS (0x8000000000000000UL)
/*
* Size of EA range mapped by our pagetables.
*/
#define H_PGTABLE_EADDR_SIZE (H_PTE_INDEX_SIZE + H_PMD_INDEX_SIZE + \
H_PUD_INDEX_SIZE + H_PGD_INDEX_SIZE + PAGE_SHIFT)
#define H_PGTABLE_RANGE (ASM_CONST(1) << H_PGTABLE_EADDR_SIZE)
/*
* Top 2 bits are ignored in page table walk.
*/
#define EA_MASK (~(0xcUL << 60))
/*
* We store the slot details in the second half of page table.
* Increase the pud level table so that hugetlb ptes can be stored
* at pud level.
*/
#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_PPC_64K_PAGES)
#define H_PUD_CACHE_INDEX (H_PUD_INDEX_SIZE + 1)
#else
#define H_PUD_CACHE_INDEX (H_PUD_INDEX_SIZE)
#endif
/*
* +------------------------------+
* | |
* | |
* | |
* +------------------------------+ Kernel virtual map end (0xc00e000000000000)
* | |
* | |
* | 512TB/16TB of vmemmap |
* | |
* | |
* +------------------------------+ Kernel vmemmap start
* | |
* | 512TB/16TB of IO map |
* | |
* +------------------------------+ Kernel IO map start
* | |
* | 512TB/16TB of vmap |
* | |
* +------------------------------+ Kernel virt start (0xc008000000000000)
* | |
* | |
* | |
* +------------------------------+ Kernel linear (0xc.....)
*/
#define H_VMALLOC_START H_KERN_VIRT_START
#define H_VMALLOC_SIZE H_KERN_MAP_SIZE
#define H_VMALLOC_END (H_VMALLOC_START + H_VMALLOC_SIZE)
#define H_KERN_IO_START H_VMALLOC_END
#define H_KERN_IO_SIZE H_KERN_MAP_SIZE
#define H_KERN_IO_END (H_KERN_IO_START + H_KERN_IO_SIZE)
#define H_VMEMMAP_START H_KERN_IO_END
#define H_VMEMMAP_SIZE H_KERN_MAP_SIZE
#define H_VMEMMAP_END (H_VMEMMAP_START + H_VMEMMAP_SIZE)
#define NON_LINEAR_REGION_ID(ea) ((((unsigned long)ea - H_KERN_VIRT_START) >> REGION_SHIFT) + 2)
/*
* Region IDs
*/
#define USER_REGION_ID 0
#define LINEAR_MAP_REGION_ID 1
#define VMALLOC_REGION_ID NON_LINEAR_REGION_ID(H_VMALLOC_START)
#define IO_REGION_ID NON_LINEAR_REGION_ID(H_KERN_IO_START)
#define VMEMMAP_REGION_ID NON_LINEAR_REGION_ID(H_VMEMMAP_START)
#define INVALID_REGION_ID (VMEMMAP_REGION_ID + 1)
/*
* Defines the address of the vmemap area, in its own region on
* hash table CPUs.
*/
/* PTEIDX nibble */
#define _PTEIDX_SECONDARY 0x8
#define _PTEIDX_GROUP_IX 0x7
#define H_PMD_BAD_BITS (PTE_TABLE_SIZE-1)
#define H_PUD_BAD_BITS (PMD_TABLE_SIZE-1)
#ifndef __ASSEMBLER__
static inline int get_region_id(unsigned long ea)
{
int region_id;
int id = (ea >> 60UL);
if (id == 0)
return USER_REGION_ID;
if (id != (PAGE_OFFSET >> 60))
return INVALID_REGION_ID;
if (ea < H_KERN_VIRT_START)
return LINEAR_MAP_REGION_ID;
BUILD_BUG_ON(NON_LINEAR_REGION_ID(H_VMALLOC_START) != 2);
region_id = NON_LINEAR_REGION_ID(ea);
return region_id;
}
static inline int hash__pmd_same(pmd_t pmd_a, pmd_t pmd_b)
{
return (((pmd_raw(pmd_a) ^ pmd_raw(pmd_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0);
}
#define hash__pmd_bad(pmd) (pmd_val(pmd) & H_PMD_BAD_BITS)
/*
* pud comparison that will work with both pte and page table pointer.
*/
static inline int hash__pud_same(pud_t pud_a, pud_t pud_b)
{
return (((pud_raw(pud_a) ^ pud_raw(pud_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0);
}
#define hash__pud_bad(pud) (pud_val(pud) & H_PUD_BAD_BITS)
static inline int hash__p4d_bad(p4d_t p4d)
{
return (p4d_val(p4d) == 0);
}
#ifdef CONFIG_STRICT_KERNEL_RWX
extern void hash__mark_rodata_ro(void);
extern void hash__mark_initmem_nx(void);
#endif
extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, unsigned long pte, int huge);
unsigned long htab_convert_pte_flags(unsigned long pteflags, unsigned long flags);
/* Atomic PTE updates */
static inline unsigned long hash__pte_update_one(pte_t *ptep, unsigned long clr,
unsigned long set)
{
__be64 old_be, tmp_be;
__asm__ __volatile__(
"1: ldarx %0,0,%3 # pte_update\n\
and. %1,%0,%6\n\
bne- 1b \n\
andc %1,%0,%4 \n\
or %1,%1,%7\n\
stdcx. %1,0,%3 \n\
bne- 1b"
: "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep)
: "r" (ptep), "r" (cpu_to_be64(clr)), "m" (*ptep),
"r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set))
: "cc" );
return be64_to_cpu(old_be);
}
static inline unsigned long hash__pte_update(struct mm_struct *mm,
unsigned long addr,
pte_t *ptep, unsigned long clr,
unsigned long set,
int huge)
{
unsigned long old;
old = hash__pte_update_one(ptep, clr, set);
if (IS_ENABLED(CONFIG_PPC_4K_PAGES) && huge) {
unsigned int psize = get_slice_psize(mm, addr);
int nb, i;
if (psize == MMU_PAGE_16M)
nb = SZ_16M / PMD_SIZE;
else if (psize == MMU_PAGE_16G)
nb = SZ_16G / PUD_SIZE;
else
nb = 1;
WARN_ON_ONCE(nb == 1); /* Should never happen */
for (i = 1; i < nb; i++)
hash__pte_update_one(ptep + i, clr, set);
}
/* huge pages use the old page table lock */
if (!huge)
assert_pte_locked(mm, addr);
if (old & H_PAGE_HASHPTE)
hpte_need_flush(mm, addr, ptep, old, huge);
return old;
}
/* Set the dirty and/or accessed bits atomically in a linux PTE, this
* function doesn't need to flush the hash entry
*/
static inline void hash__ptep_set_access_flags(pte_t *ptep, pte_t entry)
{
__be64 old, tmp, val, mask;
mask = cpu_to_be64(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_READ | _PAGE_WRITE |
_PAGE_EXEC | _PAGE_SOFT_DIRTY);
val = pte_raw(entry) & mask;
__asm__ __volatile__(
"1: ldarx %0,0,%4\n\
and. %1,%0,%6\n\
bne- 1b \n\
or %0,%3,%0\n\
stdcx. %0,0,%4\n\
bne- 1b"
:"=&r" (old), "=&r" (tmp), "=m" (*ptep)
:"r" (val), "r" (ptep), "m" (*ptep), "r" (cpu_to_be64(H_PAGE_BUSY))
:"cc");
}
static inline int hash__pte_same(pte_t pte_a, pte_t pte_b)
{
return (((pte_raw(pte_a) ^ pte_raw(pte_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0);
}
static inline int hash__pte_none(pte_t pte)
{
return (pte_val(pte) & ~H_PTE_NONE_MASK) == 0;
}
unsigned long pte_get_hash_gslot(unsigned long vpn, unsigned long shift,
int ssize, real_pte_t rpte, unsigned int subpg_index);
/* This low level function performs the actual PTE insertion
* Setting the PTE depends on the MMU type and other factors. It's
* an horrible mess that I'm not going to try to clean up now but
* I'm keeping it in one place rather than spread around
*/
static inline void hash__set_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t pte, int percpu)
{
/*
* Anything else just stores the PTE normally. That covers all 64-bit
* cases, and 32-bit non-hash with 32-bit PTEs.
*/
*ptep = pte;
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
extern void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp, unsigned long old_pmd);
#else
static inline void hpte_do_hugepage_flush(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp,
unsigned long old_pmd)
{
WARN(1, "%s called with THP disabled\n", __func__);
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
int hash__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot);
extern int __meminit hash__vmemmap_create_mapping(unsigned long start,
unsigned long page_size,
unsigned long phys);
extern void hash__vmemmap_remove_mapping(unsigned long start,
unsigned long page_size);
int hash__create_section_mapping(unsigned long start, unsigned long end,
int nid, pgprot_t prot);
int hash__remove_section_mapping(unsigned long start, unsigned long end);
#endif /* !__ASSEMBLER__ */
#endif /* __KERNEL__ */
#endif /* _ASM_POWERPC_BOOK3S_64_HASH_H */
linux-7.2.3/arch/powerpc/include/asm/book3s/64/hugetlb.h
+103
View File
@@ -0,0 +1,103 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_BOOK3S_64_HUGETLB_H
#define _ASM_POWERPC_BOOK3S_64_HUGETLB_H
#include <asm/firmware.h>
/*
* For radix we want generic code to handle hugetlb. But then if we want
* both hash and radix to be enabled together we need to workaround the
* limitations.
*/
void radix__flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
extern void radix__huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep,
pte_t old_pte, pte_t pte);
static inline int hstate_get_psize(struct hstate *hstate)
{
unsigned long shift;
shift = huge_page_shift(hstate);
if (shift == mmu_psize_defs[MMU_PAGE_2M].shift)
return MMU_PAGE_2M;
else if (shift == mmu_psize_defs[MMU_PAGE_1G].shift)
return MMU_PAGE_1G;
else if (shift == mmu_psize_defs[MMU_PAGE_16M].shift)
return MMU_PAGE_16M;
else if (shift == mmu_psize_defs[MMU_PAGE_16G].shift)
return MMU_PAGE_16G;
else {
WARN(1, "Wrong huge page shift\n");
return mmu_virtual_psize;
}
}
#define __HAVE_ARCH_GIGANTIC_PAGE_RUNTIME_SUPPORTED
static inline bool gigantic_page_runtime_supported(void)
{
/*
* We used gigantic page reservation with hypervisor assist in some case.
* We cannot use runtime allocation of gigantic pages in those platforms
* This is hash translation mode LPARs.
*/
if (firmware_has_feature(FW_FEATURE_LPAR) && !radix_enabled())
return false;
return true;
}
#define huge_ptep_modify_prot_start huge_ptep_modify_prot_start
extern pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep);
#define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit
extern void huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep,
pte_t old_pte, pte_t new_pte);
static inline void flush_hugetlb_page(struct vm_area_struct *vma,
unsigned long vmaddr)
{
if (radix_enabled())
return radix__flush_hugetlb_page(vma, vmaddr);
}
void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
static inline int check_and_get_huge_psize(int shift)
{
int mmu_psize;
if (shift > SLICE_HIGH_SHIFT)
return -EINVAL;
mmu_psize = shift_to_mmu_psize(shift);
/*
* We need to make sure that for different page sizes reported by
* firmware we only add hugetlb support for page sizes that can be
* supported by linux page table layout.
* For now we have
* Radix: 2M and 1G
* Hash: 16M and 16G
*/
if (radix_enabled()) {
if (mmu_psize != MMU_PAGE_2M && mmu_psize != MMU_PAGE_1G)
return -EINVAL;
} else {
if (mmu_psize != MMU_PAGE_16M && mmu_psize != MMU_PAGE_16G)
return -EINVAL;
}
return mmu_psize;
}
#define arch_has_huge_bootmem_alloc arch_has_huge_bootmem_alloc
static inline bool arch_has_huge_bootmem_alloc(void)
{
return (firmware_has_feature(FW_FEATURE_LPAR) && !radix_enabled());
}
#endif
linux-7.2.3/arch/powerpc/include/asm/book3s/64/kexec.h
+33
View File
@@ -0,0 +1,33 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_BOOK3S_64_KEXEC_H_
#define _ASM_POWERPC_BOOK3S_64_KEXEC_H_
#include <asm/plpar_wrappers.h>
#define reset_sprs reset_sprs
static inline void reset_sprs(void)
{
if (cpu_has_feature(CPU_FTR_ARCH_206)) {
mtspr(SPRN_AMR, 0);
mtspr(SPRN_UAMOR, 0);
}
if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
mtspr(SPRN_IAMR, 0);
if (cpu_has_feature(CPU_FTR_HVMODE))
mtspr(SPRN_CIABR, 0);
else
plpar_set_ciabr(0);
}
if (cpu_has_feature(CPU_FTR_ARCH_31)) {
mtspr(SPRN_DEXCR, 0);
mtspr(SPRN_HASHKEYR, 0);
}
/* Do we need isync()? We are going via a kexec reset */
isync();
}
#endif
linux-7.2.3/arch/powerpc/include/asm/book3s/64/kup.h
+416
View File
@@ -0,0 +1,416 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_BOOK3S_64_KUP_H
#define _ASM_POWERPC_BOOK3S_64_KUP_H
#include <linux/const.h>
#include <asm/reg.h>
#define AMR_KUAP_BLOCK_READ UL(0x5455555555555555)
#define AMR_KUAP_BLOCK_WRITE UL(0xa8aaaaaaaaaaaaaa)
#define AMR_KUEP_BLOCKED UL(0x5455555555555555)
#define AMR_KUAP_BLOCKED (AMR_KUAP_BLOCK_READ | AMR_KUAP_BLOCK_WRITE)
#ifdef __ASSEMBLER__
.macro kuap_user_restore gpr1, gpr2
#if defined(CONFIG_PPC_PKEY)
BEGIN_MMU_FTR_SECTION_NESTED(67)
b 100f // skip_restore_amr
END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67)
/*
* AMR and IAMR are going to be different when
* returning to userspace.
*/
ld \gpr1, STACK_REGS_AMR(r1)
/*
* If kuap feature is not enabled, do the mtspr
* only if AMR value is different.
*/
BEGIN_MMU_FTR_SECTION_NESTED(68)
mfspr \gpr2, SPRN_AMR
cmpd \gpr1, \gpr2
beq 99f
END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_KUAP, 68)
isync
mtspr SPRN_AMR, \gpr1
99:
/*
* Restore IAMR only when returning to userspace
*/
ld \gpr1, STACK_REGS_IAMR(r1)
/*
* If kuep feature is not enabled, do the mtspr
* only if IAMR value is different.
*/
BEGIN_MMU_FTR_SECTION_NESTED(69)
mfspr \gpr2, SPRN_IAMR
cmpd \gpr1, \gpr2
beq 100f
END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUEP, 69)
isync
mtspr SPRN_IAMR, \gpr1
100: //skip_restore_amr
/* No isync required, see kuap_user_restore() */
#endif
.endm
.macro kuap_kernel_restore gpr1, gpr2
#if defined(CONFIG_PPC_PKEY)
BEGIN_MMU_FTR_SECTION_NESTED(67)
/*
* AMR is going to be mostly the same since we are
* returning to the kernel. Compare and do a mtspr.
*/
ld \gpr2, STACK_REGS_AMR(r1)
mfspr \gpr1, SPRN_AMR
cmpd \gpr1, \gpr2
beq 100f
isync
mtspr SPRN_AMR, \gpr2
/*
* No isync required, see kuap_restore_amr()
* No need to restore IAMR when returning to kernel space.
*/
100:
END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 67)
#endif
.endm
#ifdef CONFIG_PPC_KUAP
.macro kuap_check_amr gpr1, gpr2
#ifdef CONFIG_PPC_KUAP_DEBUG
BEGIN_MMU_FTR_SECTION_NESTED(67)
mfspr \gpr1, SPRN_AMR
/* Prevent access to userspace using any key values */
LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED)
999: tdne \gpr1, \gpr2
EMIT_WARN_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING | BUGFLAG_ONCE)
END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 67)
#endif
.endm
#endif
/*
* if (pkey) {
*
* save AMR -> stack;
* if (kuap) {
* if (AMR != BLOCKED)
* KUAP_BLOCKED -> AMR;
* }
* if (from_user) {
* save IAMR -> stack;
* if (kuep) {
* KUEP_BLOCKED ->IAMR
* }
* }
* return;
* }
*
* if (kuap) {
* if (from_kernel) {
* save AMR -> stack;
* if (AMR != BLOCKED)
* KUAP_BLOCKED -> AMR;
* }
*
* }
*/
.macro kuap_save_amr_and_lock gpr1, gpr2, use_cr, msr_pr_cr
#if defined(CONFIG_PPC_PKEY)
/*
* if both pkey and kuap is disabled, nothing to do
*/
BEGIN_MMU_FTR_SECTION_NESTED(68)
b 100f // skip_save_amr
END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY | MMU_FTR_KUAP, 68)
/*
* if pkey is disabled and we are entering from userspace
* don't do anything.
*/
BEGIN_MMU_FTR_SECTION_NESTED(67)
.ifnb \msr_pr_cr
/*
* Without pkey we are not changing AMR outside the kernel
* hence skip this completely.
*/
bne \msr_pr_cr, 100f // from userspace
.endif
END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67)
/*
* pkey is enabled or pkey is disabled but entering from kernel
*/
mfspr \gpr1, SPRN_AMR
std \gpr1, STACK_REGS_AMR(r1)
/*
* update kernel AMR with AMR_KUAP_BLOCKED only
* if KUAP feature is enabled
*/
BEGIN_MMU_FTR_SECTION_NESTED(69)
LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED)
cmpd \use_cr, \gpr1, \gpr2
beq \use_cr, 102f
/*
* We don't isync here because we very recently entered via an interrupt
*/
mtspr SPRN_AMR, \gpr2
isync
102:
END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 69)
/*
* if entering from kernel we don't need save IAMR
*/
.ifnb \msr_pr_cr
beq \msr_pr_cr, 100f // from kernel space
mfspr \gpr1, SPRN_IAMR
std \gpr1, STACK_REGS_IAMR(r1)
/*
* update kernel IAMR with AMR_KUEP_BLOCKED only
* if KUEP feature is enabled
*/
BEGIN_MMU_FTR_SECTION_NESTED(70)
LOAD_REG_IMMEDIATE(\gpr2, AMR_KUEP_BLOCKED)
mtspr SPRN_IAMR, \gpr2
isync
END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUEP, 70)
.endif
100: // skip_save_amr
#endif
.endm
#else /* !__ASSEMBLER__ */
#include <linux/jump_label.h>
#include <linux/sched.h>
DECLARE_STATIC_KEY_FALSE(uaccess_flush_key);
#ifdef CONFIG_PPC_PKEY
extern u64 __ro_after_init default_uamor;
extern u64 __ro_after_init default_amr;
extern u64 __ro_after_init default_iamr;
#include <asm/mmu.h>
#include <asm/ptrace.h>
/* usage of kthread_use_mm() should inherit the
* AMR value of the operating address space. But, the AMR value is
* thread-specific and we inherit the address space and not thread
* access restrictions. Because of this ignore AMR value when accessing
* userspace via kernel thread.
*/
static __always_inline u64 current_thread_amr(void)
{
if (current->thread.regs)
return current->thread.regs->amr;
return default_amr;
}
static __always_inline u64 current_thread_iamr(void)
{
if (current->thread.regs)
return current->thread.regs->iamr;
return default_iamr;
}
#endif /* CONFIG_PPC_PKEY */
#ifdef CONFIG_PPC_KUAP
static __always_inline void kuap_user_restore(struct pt_regs *regs)
{
bool restore_amr = false, restore_iamr = false;
unsigned long amr, iamr;
if (!mmu_has_feature(MMU_FTR_PKEY))
return;
if (!mmu_has_feature(MMU_FTR_KUAP)) {
amr = mfspr(SPRN_AMR);
if (amr != regs->amr)
restore_amr = true;
} else {
restore_amr = true;
}
if (!mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) {
iamr = mfspr(SPRN_IAMR);
if (iamr != regs->iamr)
restore_iamr = true;
} else {
restore_iamr = true;
}
if (restore_amr || restore_iamr) {
isync();
if (restore_amr)
mtspr(SPRN_AMR, regs->amr);
if (restore_iamr)
mtspr(SPRN_IAMR, regs->iamr);
}
/*
* No isync required here because we are about to rfi
* back to previous context before any user accesses
* would be made, which is a CSI.
*/
}
static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long amr)
{
if (likely(regs->amr == amr))
return;
isync();
mtspr(SPRN_AMR, regs->amr);
/*
* No isync required here because we are about to rfi
* back to previous context before any user accesses
* would be made, which is a CSI.
*
* No need to restore IAMR when returning to kernel space.
*/
}
static __always_inline unsigned long __kuap_get_and_assert_locked(void)
{
unsigned long amr = mfspr(SPRN_AMR);
if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG)) /* kuap_check_amr() */
WARN_ON_ONCE(amr != AMR_KUAP_BLOCKED);
return amr;
}
#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked
/* __kuap_lock() not required, book3s/64 does that in ASM */
/*
* We support individually allowing read or write, but we don't support nesting
* because that would require an expensive read/modify write of the AMR.
*/
static __always_inline unsigned long get_kuap(void)
{
/*
* We return AMR_KUAP_BLOCKED when we don't support KUAP because
* prevent_user_access_return needs to return AMR_KUAP_BLOCKED to
* cause restore_user_access to do a flush.
*
* This has no effect in terms of actually blocking things on hash,
* so it doesn't break anything.
*/
if (!mmu_has_feature(MMU_FTR_KUAP))
return AMR_KUAP_BLOCKED;
return mfspr(SPRN_AMR);
}
static __always_inline void set_kuap(unsigned long value)
{
if (!mmu_has_feature(MMU_FTR_KUAP))
return;
/*
* ISA v3.0B says we need a CSI (Context Synchronising Instruction) both
* before and after the move to AMR. See table 6 on page 1134.
*/
isync();
mtspr(SPRN_AMR, value);
isync();
}
static __always_inline bool
__bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
{
/*
* For radix this will be a storage protection fault (DSISR_PROTFAULT).
* For hash this will be a key fault (DSISR_KEYFAULT)
*/
/*
* We do have exception table entry, but accessing the
* userspace results in fault. This could be because we
* didn't unlock the AMR or access is denied by userspace
* using a key value that blocks access. We are only interested
* in catching the use case of accessing without unlocking
* the AMR. Hence check for BLOCK_WRITE/READ against AMR.
*/
if (is_write) {
return (regs->amr & AMR_KUAP_BLOCK_WRITE) == AMR_KUAP_BLOCK_WRITE;
}
return (regs->amr & AMR_KUAP_BLOCK_READ) == AMR_KUAP_BLOCK_READ;
}
static __always_inline void allow_user_access(void __user *to, unsigned long dir)
{
unsigned long thread_amr = 0;
// This is written so we can resolve to a single case at build time
BUILD_BUG_ON(!__builtin_constant_p(dir));
if (mmu_has_feature(MMU_FTR_PKEY))
thread_amr = current_thread_amr();
if (dir == KUAP_READ)
set_kuap(thread_amr | AMR_KUAP_BLOCK_WRITE);
else if (dir == KUAP_WRITE)
set_kuap(thread_amr | AMR_KUAP_BLOCK_READ);
else if (dir == KUAP_READ_WRITE)
set_kuap(thread_amr);
else
BUILD_BUG();
}
#else /* CONFIG_PPC_KUAP */
static __always_inline unsigned long get_kuap(void)
{
return AMR_KUAP_BLOCKED;
}
static __always_inline void set_kuap(unsigned long value) { }
static __always_inline void allow_user_access(void __user *to, unsigned long dir)
{ }
#endif /* !CONFIG_PPC_KUAP */
static __always_inline void prevent_user_access(unsigned long dir)
{
set_kuap(AMR_KUAP_BLOCKED);
if (static_branch_unlikely(&uaccess_flush_key))
do_uaccess_flush();
}
static __always_inline unsigned long prevent_user_access_return(void)
{
unsigned long flags = get_kuap();
set_kuap(AMR_KUAP_BLOCKED);
if (static_branch_unlikely(&uaccess_flush_key))
do_uaccess_flush();
return flags;
}
static __always_inline void restore_user_access(unsigned long flags)
{
set_kuap(flags);
if (static_branch_unlikely(&uaccess_flush_key) && flags == AMR_KUAP_BLOCKED)
do_uaccess_flush();
}
#endif /* __ASSEMBLER__ */
#endif /* _ASM_POWERPC_BOOK3S_64_KUP_H */
linux-7.2.3/arch/powerpc/include/asm/book3s/64/mmu-hash.h
+884
View File
@@ -0,0 +1,884 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
#ifndef _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_
#define _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_
/*
* PowerPC64 memory management structures
*
* Dave Engebretsen & Mike Corrigan <{engebret|mikejc}@us.ibm.com>
* PPC64 rework.
*/
#include <asm/page.h>
#include <asm/bug.h>
#include <asm/asm-const.h>
/*
* This is necessary to get the definition of PGTABLE_RANGE which we
* need for various slices related matters. Note that this isn't the
* complete pgtable.h but only a portion of it.
*/
#include <asm/book3s/64/pgtable.h>
#include <asm/book3s/64/slice.h>
#include <asm/task_size_64.h>
#include <asm/cpu_has_feature.h>
/*
* SLB
*/
#define SLB_NUM_BOLTED 2
#define SLB_CACHE_ENTRIES 8
#define SLB_MIN_SIZE 32
/* Bits in the SLB ESID word */
#define SLB_ESID_V ASM_CONST(0x0000000008000000) /* valid */
/* Bits in the SLB VSID word */
#define SLB_VSID_SHIFT 12
#define SLB_VSID_SHIFT_256M SLB_VSID_SHIFT
#define SLB_VSID_SHIFT_1T 24
#define SLB_VSID_SSIZE_SHIFT 62
#define SLB_VSID_B ASM_CONST(0xc000000000000000)
#define SLB_VSID_B_256M ASM_CONST(0x0000000000000000)
#define SLB_VSID_B_1T ASM_CONST(0x4000000000000000)
#define SLB_VSID_KS ASM_CONST(0x0000000000000800)
#define SLB_VSID_KP ASM_CONST(0x0000000000000400)
#define SLB_VSID_N ASM_CONST(0x0000000000000200) /* no-execute */
#define SLB_VSID_L ASM_CONST(0x0000000000000100)
#define SLB_VSID_C ASM_CONST(0x0000000000000080) /* class */
#define SLB_VSID_LP ASM_CONST(0x0000000000000030)
#define SLB_VSID_LP_00 ASM_CONST(0x0000000000000000)
#define SLB_VSID_LP_01 ASM_CONST(0x0000000000000010)
#define SLB_VSID_LP_10 ASM_CONST(0x0000000000000020)
#define SLB_VSID_LP_11 ASM_CONST(0x0000000000000030)
#define SLB_VSID_LLP (SLB_VSID_L|SLB_VSID_LP)
#define SLB_VSID_KERNEL (SLB_VSID_KP)
#define SLB_VSID_USER (SLB_VSID_KP|SLB_VSID_KS|SLB_VSID_C)
#define SLBIE_C (0x08000000)
#define SLBIE_SSIZE_SHIFT 25
/*
* Hash table
*/
#define HPTES_PER_GROUP 8
#define HPTE_V_SSIZE_SHIFT 62
#define HPTE_V_AVPN_SHIFT 7
#define HPTE_V_COMMON_BITS ASM_CONST(0x000fffffffffffff)
#define HPTE_V_AVPN ASM_CONST(0x3fffffffffffff80)
#define HPTE_V_AVPN_3_0 ASM_CONST(0x000fffffffffff80)
#define HPTE_V_AVPN_VAL(x) (((x) & HPTE_V_AVPN) >> HPTE_V_AVPN_SHIFT)
#define HPTE_V_COMPARE(x,y) (!(((x) ^ (y)) & 0xffffffffffffff80UL))
#define HPTE_V_BOLTED ASM_CONST(0x0000000000000010)
#define HPTE_V_LOCK ASM_CONST(0x0000000000000008)
#define HPTE_V_LARGE ASM_CONST(0x0000000000000004)
#define HPTE_V_SECONDARY ASM_CONST(0x0000000000000002)
#define HPTE_V_VALID ASM_CONST(0x0000000000000001)
/*
* ISA 3.0 has a different HPTE format.
*/
#define HPTE_R_3_0_SSIZE_SHIFT 58
#define HPTE_R_3_0_SSIZE_MASK (3ull << HPTE_R_3_0_SSIZE_SHIFT)
#define HPTE_R_PP0 ASM_CONST(0x8000000000000000)
#define HPTE_R_TS ASM_CONST(0x4000000000000000)
#define HPTE_R_KEY_HI ASM_CONST(0x3000000000000000)
#define HPTE_R_KEY_BIT4 ASM_CONST(0x2000000000000000)
#define HPTE_R_KEY_BIT3 ASM_CONST(0x1000000000000000)
#define HPTE_R_RPN_SHIFT 12
#define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000)
#define HPTE_R_RPN_3_0 ASM_CONST(0x01fffffffffff000)
#define HPTE_R_PP ASM_CONST(0x0000000000000003)
#define HPTE_R_PPP ASM_CONST(0x8000000000000003)
#define HPTE_R_N ASM_CONST(0x0000000000000004)
#define HPTE_R_G ASM_CONST(0x0000000000000008)
#define HPTE_R_M ASM_CONST(0x0000000000000010)
#define HPTE_R_I ASM_CONST(0x0000000000000020)
#define HPTE_R_W ASM_CONST(0x0000000000000040)
#define HPTE_R_WIMG ASM_CONST(0x0000000000000078)
#define HPTE_R_C ASM_CONST(0x0000000000000080)
#define HPTE_R_R ASM_CONST(0x0000000000000100)
#define HPTE_R_KEY_LO ASM_CONST(0x0000000000000e00)
#define HPTE_R_KEY_BIT2 ASM_CONST(0x0000000000000800)
#define HPTE_R_KEY_BIT1 ASM_CONST(0x0000000000000400)
#define HPTE_R_KEY_BIT0 ASM_CONST(0x0000000000000200)
#define HPTE_R_KEY (HPTE_R_KEY_LO | HPTE_R_KEY_HI)
#define HPTE_V_1TB_SEG ASM_CONST(0x4000000000000000)
#define HPTE_V_VRMA_MASK ASM_CONST(0x4001ffffff000000)
/* Values for PP (assumes Ks=0, Kp=1) */
#define PP_RWXX 0 /* Supervisor read/write, User none */
#define PP_RWRX 1 /* Supervisor read/write, User read */
#define PP_RWRW 2 /* Supervisor read/write, User read/write */
#define PP_RXRX 3 /* Supervisor read, User read */
#define PP_RXXX (HPTE_R_PP0 | 2) /* Supervisor read, user none */
/* Fields for tlbiel instruction in architecture 2.06 */
#define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */
#define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */
#define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */
#define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */
#define TLBIEL_INVAL_SET_MASK 0xfff000 /* set number to inval. */
#define TLBIEL_INVAL_SET_SHIFT 12
#define POWER7_TLB_SETS 128 /* # sets in POWER7 TLB */
#define POWER8_TLB_SETS 512 /* # sets in POWER8 TLB */
#define POWER9_TLB_SETS_HASH 256 /* # sets in POWER9 TLB Hash mode */
#define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */
#ifndef __ASSEMBLER__
struct mmu_hash_ops {
void (*hpte_invalidate)(unsigned long slot,
unsigned long vpn,
int bpsize, int apsize,
int ssize, int local);
long (*hpte_updatepp)(unsigned long slot,
unsigned long newpp,
unsigned long vpn,
int bpsize, int apsize,
int ssize, unsigned long flags);
void (*hpte_updateboltedpp)(unsigned long newpp,
unsigned long ea,
int psize, int ssize);
long (*hpte_insert)(unsigned long hpte_group,
unsigned long vpn,
unsigned long prpn,
unsigned long rflags,
unsigned long vflags,
int psize, int apsize,
int ssize);
long (*hpte_remove)(unsigned long hpte_group);
int (*hpte_removebolted)(unsigned long ea,
int psize, int ssize);
void (*flush_hash_range)(unsigned long number, int local);
void (*hugepage_invalidate)(unsigned long vsid,
unsigned long addr,
unsigned char *hpte_slot_array,
int psize, int ssize, int local);
int (*resize_hpt)(unsigned long shift);
/*
* Special for kexec.
* To be called in real mode with interrupts disabled. No locks are
* taken as such, concurrent access on pre POWER5 hardware could result
* in a deadlock.
* The linear mapping is destroyed as well.
*/
void (*hpte_clear_all)(void);
};
extern struct mmu_hash_ops mmu_hash_ops;
struct hash_pte {
__be64 v;
__be64 r;
};
extern struct hash_pte *htab_address;
extern unsigned long htab_size_bytes;
extern unsigned long htab_hash_mask;
static inline int shift_to_mmu_psize(unsigned int shift)
{
int psize;
for (psize = 0; psize < MMU_PAGE_COUNT; ++psize)
if (mmu_psize_defs[psize].shift == shift)
return psize;
return -1;
}
static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize)
{
if (mmu_psize_defs[mmu_psize].shift)
return mmu_psize_defs[mmu_psize].shift;
BUG();
}
static inline unsigned int ap_to_shift(unsigned long ap)
{
int psize;
for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
if (mmu_psize_defs[psize].ap == ap)
return mmu_psize_defs[psize].shift;
}
return -1;
}
static inline unsigned long get_sllp_encoding(int psize)
{
unsigned long sllp;
sllp = ((mmu_psize_defs[psize].sllp & SLB_VSID_L) >> 6) |
((mmu_psize_defs[psize].sllp & SLB_VSID_LP) >> 4);
return sllp;
}
#endif /* __ASSEMBLER__ */
/*
* Segment sizes.
* These are the values used by hardware in the B field of
* SLB entries and the first dword of MMU hashtable entries.
* The B field is 2 bits; the values 2 and 3 are unused and reserved.
*/
#define MMU_SEGSIZE_256M 0
#define MMU_SEGSIZE_1T 1
/*
* encode page number shift.
* in order to fit the 78 bit va in a 64 bit variable we shift the va by
* 12 bits. This enable us to address upto 76 bit va.
* For hpt hash from a va we can ignore the page size bits of va and for
* hpte encoding we ignore up to 23 bits of va. So ignoring lower 12 bits ensure
* we work in all cases including 4k page size.
*/
#define VPN_SHIFT 12
/*
* HPTE Large Page (LP) details
*/
#define LP_SHIFT 12
#define LP_BITS 8
#define LP_MASK(i) ((0xFF >> (i)) << LP_SHIFT)
#ifndef __ASSEMBLER__
static inline int slb_vsid_shift(int ssize)
{
if (ssize == MMU_SEGSIZE_256M)
return SLB_VSID_SHIFT;
return SLB_VSID_SHIFT_1T;
}
static inline int segment_shift(int ssize)
{
if (ssize == MMU_SEGSIZE_256M)
return SID_SHIFT;
return SID_SHIFT_1T;
}
/*
* This array is indexed by the LP field of the HPTE second dword.
* Since this field may contain some RPN bits, some entries are
* replicated so that we get the same value irrespective of RPN.
* The top 4 bits are the page size index (MMU_PAGE_*) for the
* actual page size, the bottom 4 bits are the base page size.
*/
extern u8 hpte_page_sizes[1 << LP_BITS];
static inline unsigned long __hpte_page_size(unsigned long h, unsigned long l,
bool is_base_size)
{
unsigned int i, lp;
if (!(h & HPTE_V_LARGE))
return 1ul << 12;
/* Look at the 8 bit LP value */
lp = (l >> LP_SHIFT) & ((1 << LP_BITS) - 1);
i = hpte_page_sizes[lp];
if (!i)
return 0;
if (!is_base_size)
i >>= 4;
return 1ul << mmu_psize_defs[i & 0xf].shift;
}
static inline unsigned long hpte_page_size(unsigned long h, unsigned long l)
{
return __hpte_page_size(h, l, 0);
}
static inline unsigned long hpte_base_page_size(unsigned long h, unsigned long l)
{
return __hpte_page_size(h, l, 1);
}
/*
* The current system page and segment sizes
*/
extern int mmu_kernel_ssize;
extern int mmu_highuser_ssize;
extern u16 mmu_slb_size;
extern unsigned long tce_alloc_start, tce_alloc_end;
/*
* If the processor supports 64k normal pages but not 64k cache
* inhibited pages, we have to be prepared to switch processes
* to use 4k pages when they create cache-inhibited mappings.
* If this is the case, mmu_ci_restrictions will be set to 1.
*/
extern int mmu_ci_restrictions;
/*
* This computes the AVPN and B fields of the first dword of a HPTE,
* for use when we want to match an existing PTE. The bottom 7 bits
* of the returned value are zero.
*/
static inline unsigned long hpte_encode_avpn(unsigned long vpn, int psize,
int ssize)
{
unsigned long v;
/*
* The AVA field omits the low-order 23 bits of the 78 bits VA.
* These bits are not needed in the PTE, because the
* low-order b of these bits are part of the byte offset
* into the virtual page and, if b < 23, the high-order
* 23-b of these bits are always used in selecting the
* PTEGs to be searched
*/
v = (vpn >> (23 - VPN_SHIFT)) & ~(mmu_psize_defs[psize].avpnm);
v <<= HPTE_V_AVPN_SHIFT;
v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT;
return v;
}
/*
* ISA v3.0 defines a new HPTE format, which differs from the old
* format in having smaller AVPN and ARPN fields, and the B field
* in the second dword instead of the first.
*/
static inline unsigned long hpte_old_to_new_v(unsigned long v)
{
/* trim AVPN, drop B */
return v & HPTE_V_COMMON_BITS;
}
static inline unsigned long hpte_old_to_new_r(unsigned long v, unsigned long r)
{
/* move B field from 1st to 2nd dword, trim ARPN */
return (r & ~HPTE_R_3_0_SSIZE_MASK) |
(((v) >> HPTE_V_SSIZE_SHIFT) << HPTE_R_3_0_SSIZE_SHIFT);
}
static inline unsigned long hpte_new_to_old_v(unsigned long v, unsigned long r)
{
/* insert B field */
return (v & HPTE_V_COMMON_BITS) |
((r & HPTE_R_3_0_SSIZE_MASK) <<
(HPTE_V_SSIZE_SHIFT - HPTE_R_3_0_SSIZE_SHIFT));
}
static inline unsigned long hpte_new_to_old_r(unsigned long r)
{
/* clear out B field */
return r & ~HPTE_R_3_0_SSIZE_MASK;
}
static inline unsigned long hpte_get_old_v(struct hash_pte *hptep)
{
unsigned long hpte_v;
hpte_v = be64_to_cpu(hptep->v);
if (cpu_has_feature(CPU_FTR_ARCH_300))
hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r));
return hpte_v;
}
/*
* This function sets the AVPN and L fields of the HPTE appropriately
* using the base page size and actual page size.
*/
static inline unsigned long hpte_encode_v(unsigned long vpn, int base_psize,
int actual_psize, int ssize)
{
unsigned long v;
v = hpte_encode_avpn(vpn, base_psize, ssize);
if (actual_psize != MMU_PAGE_4K)
v |= HPTE_V_LARGE;
return v;
}
/*
* This function sets the ARPN, and LP fields of the HPTE appropriately
* for the page size. We assume the pa is already "clean" that is properly
* aligned for the requested page size
*/
static inline unsigned long hpte_encode_r(unsigned long pa, int base_psize,
int actual_psize)
{
/* A 4K page needs no special encoding */
if (actual_psize == MMU_PAGE_4K)
return pa & HPTE_R_RPN;
else {
unsigned int penc = mmu_psize_defs[base_psize].penc[actual_psize];
unsigned int shift = mmu_psize_defs[actual_psize].shift;
return (pa & ~((1ul << shift) - 1)) | (penc << LP_SHIFT);
}
}
/*
* Build a VPN_SHIFT bit shifted va given VSID, EA and segment size.
*/
static inline unsigned long hpt_vpn(unsigned long ea,
unsigned long vsid, int ssize)
{
unsigned long mask;
int s_shift = segment_shift(ssize);
mask = (1ul << (s_shift - VPN_SHIFT)) - 1;
return (vsid << (s_shift - VPN_SHIFT)) | ((ea >> VPN_SHIFT) & mask);
}
/*
* This hashes a virtual address
*/
static inline unsigned long hpt_hash(unsigned long vpn,
unsigned int shift, int ssize)
{
unsigned long mask;
unsigned long hash, vsid;
/* VPN_SHIFT can be atmost 12 */
if (ssize == MMU_SEGSIZE_256M) {
mask = (1ul << (SID_SHIFT - VPN_SHIFT)) - 1;
hash = (vpn >> (SID_SHIFT - VPN_SHIFT)) ^
((vpn & mask) >> (shift - VPN_SHIFT));
} else {
mask = (1ul << (SID_SHIFT_1T - VPN_SHIFT)) - 1;
vsid = vpn >> (SID_SHIFT_1T - VPN_SHIFT);
hash = vsid ^ (vsid << 25) ^
((vpn & mask) >> (shift - VPN_SHIFT)) ;
}
return hash & 0x7fffffffffUL;
}
#define HPTE_LOCAL_UPDATE 0x1
#define HPTE_NOHPTE_UPDATE 0x2
#define HPTE_USE_KERNEL_KEY 0x4
long hpte_insert_repeating(unsigned long hash, unsigned long vpn, unsigned long pa,
unsigned long rlags, unsigned long vflags, int psize, int ssize);
extern int __hash_page_4K(unsigned long ea, unsigned long access,
unsigned long vsid, pte_t *ptep, unsigned long trap,
unsigned long flags, int ssize, int subpage_prot);
extern int __hash_page_64K(unsigned long ea, unsigned long access,
unsigned long vsid, pte_t *ptep, unsigned long trap,
unsigned long flags, int ssize);
struct mm_struct;
unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap);
extern int hash_page_mm(struct mm_struct *mm, unsigned long ea,
unsigned long access, unsigned long trap,
unsigned long flags);
extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap,
unsigned long dsisr);
void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc);
int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr, unsigned long msr);
int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
pte_t *ptep, unsigned long trap, unsigned long flags,
int ssize, unsigned int shift, unsigned int mmu_psize);
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
extern int __hash_page_thp(unsigned long ea, unsigned long access,
unsigned long vsid, pmd_t *pmdp, unsigned long trap,
unsigned long flags, int ssize, unsigned int psize);
#else
static inline int __hash_page_thp(unsigned long ea, unsigned long access,
unsigned long vsid, pmd_t *pmdp,
unsigned long trap, unsigned long flags,
int ssize, unsigned int psize)
{
BUG();
return -1;
}
#endif
extern void hash_failure_debug(unsigned long ea, unsigned long access,
unsigned long vsid, unsigned long trap,
int ssize, int psize, int lpsize,
unsigned long pte);
extern int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
unsigned long pstart, unsigned long prot,
int psize, int ssize);
int htab_remove_mapping(unsigned long vstart, unsigned long vend,
int psize, int ssize);
extern void pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages);
extern void demote_segment_4k(struct mm_struct *mm, unsigned long addr);
extern void hash__setup_new_exec(void);
#ifdef CONFIG_PPC_PSERIES
void hpte_init_pseries(void);
#else
static inline void hpte_init_pseries(void) { }
#endif
extern void hpte_init_native(void);
struct slb_entry {
u64 esid;
u64 vsid;
};
extern void slb_initialize(void);
void slb_flush_and_restore_bolted(void);
void slb_flush_all_realmode(void);
void __slb_restore_bolted_realmode(void);
void slb_restore_bolted_realmode(void);
void slb_save_contents(struct slb_entry *slb_ptr);
void slb_dump_contents(struct slb_entry *slb_ptr);
extern void slb_vmalloc_update(void);
#ifdef CONFIG_PPC_64S_HASH_MMU
void slb_set_size(u16 size);
#else
static inline void slb_set_size(u16 size) { }
#endif
#endif /* __ASSEMBLER__ */
/*
* VSID allocation (256MB segment)
*
* We first generate a 37-bit "proto-VSID". Proto-VSIDs are generated
* from mmu context id and effective segment id of the address.
*
* For user processes max context id is limited to MAX_USER_CONTEXT.
* more details in get_user_context
*
* For kernel space get_kernel_context
*
* The proto-VSIDs are then scrambled into real VSIDs with the
* multiplicative hash:
*
* VSID = (proto-VSID * VSID_MULTIPLIER) % VSID_MODULUS
*
* VSID_MULTIPLIER is prime, so in particular it is
* co-prime to VSID_MODULUS, making this a 1:1 scrambling function.
* Because the modulus is 2^n-1 we can compute it efficiently without
* a divide or extra multiply (see below). The scramble function gives
* robust scattering in the hash table (at least based on some initial
* results).
*
* We use VSID 0 to indicate an invalid VSID. The means we can't use context id
* 0, because a context id of 0 and an EA of 0 gives a proto-VSID of 0, which
* will produce a VSID of 0.
*
* We also need to avoid the last segment of the last context, because that
* would give a protovsid of 0x1fffffffff. That will result in a VSID 0
* because of the modulo operation in vsid scramble.
*/
/*
* Max Va bits we support as of now is 68 bits. We want 19 bit
* context ID.
* Restrictions:
* GPU has restrictions of not able to access beyond 128TB
* (47 bit effective address). We also cannot do more than 20bit PID.
* For p4 and p5 which can only do 65 bit VA, we restrict our CONTEXT_BITS
* to 16 bits (ie, we can only have 2^16 pids at the same time).
*/
#define VA_BITS 68
#define CONTEXT_BITS 19
#define ESID_BITS (VA_BITS - (SID_SHIFT + CONTEXT_BITS))
#define ESID_BITS_1T (VA_BITS - (SID_SHIFT_1T + CONTEXT_BITS))
#define ESID_BITS_MASK ((1 << ESID_BITS) - 1)
#define ESID_BITS_1T_MASK ((1 << ESID_BITS_1T) - 1)
/*
* Now certain config support MAX_PHYSMEM more than 512TB. Hence we will need
* to use more than one context for linear mapping the kernel.
* For vmalloc and memmap, we use just one context with 512TB. With 64 byte
* struct page size, we need ony 32 TB in memmap for 2PB (51 bits (MAX_PHYSMEM_BITS)).
*/
#if (H_MAX_PHYSMEM_BITS > MAX_EA_BITS_PER_CONTEXT)
#define MAX_KERNEL_CTX_CNT (1UL << (H_MAX_PHYSMEM_BITS - MAX_EA_BITS_PER_CONTEXT))
#else
#define MAX_KERNEL_CTX_CNT 1
#endif
#define MAX_VMALLOC_CTX_CNT 1
#define MAX_IO_CTX_CNT 1
#define MAX_VMEMMAP_CTX_CNT 1
/*
* 256MB segment
* The proto-VSID space has 2^(CONTEX_BITS + ESID_BITS) - 1 segments
* available for user + kernel mapping. VSID 0 is reserved as invalid, contexts
* 1-4 are used for kernel mapping. Each segment contains 2^28 bytes. Each
* context maps 2^49 bytes (512TB).
*
* We also need to avoid the last segment of the last context, because that
* would give a protovsid of 0x1fffffffff. That will result in a VSID 0
* because of the modulo operation in vsid scramble.
*
*/
#define MAX_USER_CONTEXT ((ASM_CONST(1) << CONTEXT_BITS) - 2)
// The + 2 accounts for INVALID_REGION and 1 more to avoid overlap with kernel
#define MIN_USER_CONTEXT (MAX_KERNEL_CTX_CNT + MAX_VMALLOC_CTX_CNT + \
MAX_IO_CTX_CNT + MAX_VMEMMAP_CTX_CNT + 2)
/*
* For platforms that support on 65bit VA we limit the context bits
*/
#define MAX_USER_CONTEXT_65BIT_VA ((ASM_CONST(1) << (65 - (SID_SHIFT + ESID_BITS))) - 2)
/*
* This should be computed such that protovosid * vsid_mulitplier
* doesn't overflow 64 bits. The vsid_mutliplier should also be
* co-prime to vsid_modulus. We also need to make sure that number
* of bits in multiplied result (dividend) is less than twice the number of
* protovsid bits for our modulus optmization to work.
*
* The below table shows the current values used.
* |-------+------------+----------------------+------------+-------------------|
* | | Prime Bits | proto VSID_BITS_65VA | Total Bits | 2* prot VSID_BITS |
* |-------+------------+----------------------+------------+-------------------|
* | 1T | 24 | 25 | 49 | 50 |
* |-------+------------+----------------------+------------+-------------------|
* | 256MB | 24 | 37 | 61 | 74 |
* |-------+------------+----------------------+------------+-------------------|
*
* |-------+------------+----------------------+------------+--------------------|
* | | Prime Bits | proto VSID_BITS_68VA | Total Bits | 2* proto VSID_BITS |
* |-------+------------+----------------------+------------+--------------------|
* | 1T | 24 | 28 | 52 | 56 |
* |-------+------------+----------------------+------------+--------------------|
* | 256MB | 24 | 40 | 64 | 80 |
* |-------+------------+----------------------+------------+--------------------|
*
*/
#define VSID_MULTIPLIER_256M ASM_CONST(12538073) /* 24-bit prime */
#define VSID_BITS_256M (VA_BITS - SID_SHIFT)
#define VSID_BITS_65_256M (65 - SID_SHIFT)
/*
* Modular multiplicative inverse of VSID_MULTIPLIER under modulo VSID_MODULUS
*/
#define VSID_MULINV_256M ASM_CONST(665548017062)
#define VSID_MULTIPLIER_1T ASM_CONST(12538073) /* 24-bit prime */
#define VSID_BITS_1T (VA_BITS - SID_SHIFT_1T)
#define VSID_BITS_65_1T (65 - SID_SHIFT_1T)
#define VSID_MULINV_1T ASM_CONST(209034062)
/* 1TB VSID reserved for VRMA */
#define VRMA_VSID 0x1ffffffUL
#define USER_VSID_RANGE (1UL << (ESID_BITS + SID_SHIFT))
/* 4 bits per slice and we have one slice per 1TB */
#define SLICE_ARRAY_SIZE (H_PGTABLE_RANGE >> 41)
#define LOW_SLICE_ARRAY_SZ (BITS_PER_LONG / BITS_PER_BYTE)
#define TASK_SLICE_ARRAY_SZ(x) ((x)->hash_context->slb_addr_limit >> 41)
#ifndef __ASSEMBLER__
#ifdef CONFIG_PPC_SUBPAGE_PROT
/*
* For the sub-page protection option, we extend the PGD with one of
* these. Basically we have a 3-level tree, with the top level being
* the protptrs array. To optimize speed and memory consumption when
* only addresses < 4GB are being protected, pointers to the first
* four pages of sub-page protection words are stored in the low_prot
* array.
* Each page of sub-page protection words protects 1GB (4 bytes
* protects 64k). For the 3-level tree, each page of pointers then
* protects 8TB.
*/
struct subpage_prot_table {
unsigned long maxaddr; /* only addresses < this are protected */
unsigned int **protptrs[(TASK_SIZE_USER64 >> 43)];
unsigned int *low_prot[4];
};
#define SBP_L1_BITS (PAGE_SHIFT - 2)
#define SBP_L2_BITS (PAGE_SHIFT - 3)
#define SBP_L1_COUNT (1 << SBP_L1_BITS)
#define SBP_L2_COUNT (1 << SBP_L2_BITS)
#define SBP_L2_SHIFT (PAGE_SHIFT + SBP_L1_BITS)
#define SBP_L3_SHIFT (SBP_L2_SHIFT + SBP_L2_BITS)
extern void subpage_prot_free(struct mm_struct *mm);
#else
static inline void subpage_prot_free(struct mm_struct *mm) {}
#endif /* CONFIG_PPC_SUBPAGE_PROT */
/*
* One bit per slice. We have lower slices which cover 256MB segments
* upto 4G range. That gets us 16 low slices. For the rest we track slices
* in 1TB size.
*/
struct slice_mask {
u64 low_slices;
DECLARE_BITMAP(high_slices, SLICE_NUM_HIGH);
};
struct hash_mm_context {
u16 user_psize; /* page size index */
/* SLB page size encodings*/
unsigned char low_slices_psize[LOW_SLICE_ARRAY_SZ];
unsigned char high_slices_psize[SLICE_ARRAY_SIZE];
unsigned long slb_addr_limit;
#ifdef CONFIG_PPC_64K_PAGES
struct slice_mask mask_64k;
#endif
struct slice_mask mask_4k;
#ifdef CONFIG_HUGETLB_PAGE
struct slice_mask mask_16m;
struct slice_mask mask_16g;
#endif
#ifdef CONFIG_PPC_SUBPAGE_PROT
struct subpage_prot_table *spt;
#endif /* CONFIG_PPC_SUBPAGE_PROT */
};
#if 0
/*
* The code below is equivalent to this function for arguments
* < 2^VSID_BITS, which is all this should ever be called
* with. However gcc is not clever enough to compute the
* modulus (2^n-1) without a second multiply.
*/
#define vsid_scramble(protovsid, size) \
((((protovsid) * VSID_MULTIPLIER_##size) % VSID_MODULUS_##size))
/* simplified form avoiding mod operation */
#define vsid_scramble(protovsid, size) \
({ \
unsigned long x; \
x = (protovsid) * VSID_MULTIPLIER_##size; \
x = (x >> VSID_BITS_##size) + (x & VSID_MODULUS_##size); \
(x + ((x+1) >> VSID_BITS_##size)) & VSID_MODULUS_##size; \
})
#else /* 1 */
static inline unsigned long vsid_scramble(unsigned long protovsid,
unsigned long vsid_multiplier, int vsid_bits)
{
unsigned long vsid;
unsigned long vsid_modulus = ((1UL << vsid_bits) - 1);
/*
* We have same multipler for both 256 and 1T segements now
*/
vsid = protovsid * vsid_multiplier;
vsid = (vsid >> vsid_bits) + (vsid & vsid_modulus);
return (vsid + ((vsid + 1) >> vsid_bits)) & vsid_modulus;
}
#endif /* 1 */
/* Returns the segment size indicator for a user address */
static inline int user_segment_size(unsigned long addr)
{
/* Use 1T segments if possible for addresses >= 1T */
if (addr >= (1UL << SID_SHIFT_1T))
return mmu_highuser_ssize;
return MMU_SEGSIZE_256M;
}
static inline unsigned long get_vsid(unsigned long context, unsigned long ea,
int ssize)
{
unsigned long va_bits = VA_BITS;
unsigned long vsid_bits;
unsigned long protovsid;
/*
* Bad address. We return VSID 0 for that
*/
if ((ea & EA_MASK) >= H_PGTABLE_RANGE)
return 0;
if (!mmu_has_feature(MMU_FTR_68_BIT_VA))
va_bits = 65;
if (ssize == MMU_SEGSIZE_256M) {
vsid_bits = va_bits - SID_SHIFT;
protovsid = (context << ESID_BITS) |
((ea >> SID_SHIFT) & ESID_BITS_MASK);
return vsid_scramble(protovsid, VSID_MULTIPLIER_256M, vsid_bits);
}
/* 1T segment */
vsid_bits = va_bits - SID_SHIFT_1T;
protovsid = (context << ESID_BITS_1T) |
((ea >> SID_SHIFT_1T) & ESID_BITS_1T_MASK);
return vsid_scramble(protovsid, VSID_MULTIPLIER_1T, vsid_bits);
}
/*
* For kernel space, we use context ids as
* below. Range is 512TB per context.
*
* 0x00001 - [ 0xc000000000000000 - 0xc001ffffffffffff]
* 0x00002 - [ 0xc002000000000000 - 0xc003ffffffffffff]
* 0x00003 - [ 0xc004000000000000 - 0xc005ffffffffffff]
* 0x00004 - [ 0xc006000000000000 - 0xc007ffffffffffff]
*
* vmap, IO, vmemap
*
* 0x00005 - [ 0xc008000000000000 - 0xc009ffffffffffff]
* 0x00006 - [ 0xc00a000000000000 - 0xc00bffffffffffff]
* 0x00007 - [ 0xc00c000000000000 - 0xc00dffffffffffff]
*
*/
static inline unsigned long get_kernel_context(unsigned long ea)
{
unsigned long region_id = get_region_id(ea);
unsigned long ctx;
/*
* Depending on Kernel config, kernel region can have one context
* or more.
*/
if (region_id == LINEAR_MAP_REGION_ID) {
/*
* We already verified ea to be not beyond the addr limit.
*/
ctx = 1 + ((ea & EA_MASK) >> MAX_EA_BITS_PER_CONTEXT);
} else
ctx = region_id + MAX_KERNEL_CTX_CNT - 1;
return ctx;
}
/*
* This is only valid for addresses >= PAGE_OFFSET
*/
static inline unsigned long get_kernel_vsid(unsigned long ea, int ssize)
{
unsigned long context;
if (!is_kernel_addr(ea))
return 0;
context = get_kernel_context(ea);
return get_vsid(context, ea, ssize);
}
unsigned htab_shift_for_mem_size(unsigned long mem_size);
enum slb_index {
LINEAR_INDEX = 0, /* Kernel linear map (0xc000000000000000) */
KSTACK_INDEX = 1, /* Kernel stack map */
};
#define slb_esid_mask(ssize) \
(((ssize) == MMU_SEGSIZE_256M) ? ESID_MASK : ESID_MASK_1T)
static inline unsigned long mk_esid_data(unsigned long ea, int ssize,
enum slb_index index)
{
return (ea & slb_esid_mask(ssize)) | SLB_ESID_V | index;
}
static inline unsigned long __mk_vsid_data(unsigned long vsid, int ssize,
unsigned long flags)
{
return (vsid << slb_vsid_shift(ssize)) | flags |
((unsigned long)ssize << SLB_VSID_SSIZE_SHIFT);
}
static inline unsigned long mk_vsid_data(unsigned long ea, int ssize,
unsigned long flags)
{
return __mk_vsid_data(get_kernel_vsid(ea, ssize), ssize, flags);
}
#endif /* __ASSEMBLER__ */
#endif /* _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_ */
linux-7.2.3/arch/powerpc/include/asm/book3s/64/mmu.h
+292
View File
@@ -0,0 +1,292 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_BOOK3S_64_MMU_H_
#define _ASM_POWERPC_BOOK3S_64_MMU_H_
#include <asm/page.h>
#ifndef __ASSEMBLER__
/*
* Page size definition
*
* shift : is the "PAGE_SHIFT" value for that page size
* sllp : is a bit mask with the value of SLB L || LP to be or'ed
* directly to a slbmte "vsid" value
* penc : is the HPTE encoding mask for the "LP" field:
*
*/
struct mmu_psize_def {
unsigned int shift; /* number of bits */
int penc[MMU_PAGE_COUNT]; /* HPTE encoding */
unsigned int tlbiel; /* tlbiel supported for that page size */
unsigned long avpnm; /* bits to mask out in AVPN in the HPTE */
unsigned long h_rpt_pgsize; /* H_RPT_INVALIDATE page size encoding */
union {
unsigned long sllp; /* SLB L||LP (exact mask to use in slbmte) */
unsigned long ap; /* Ap encoding used by PowerISA 3.0 */
};
};
extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT];
#endif /* __ASSEMBLER__ */
/* 64-bit classic hash table MMU */
#include <asm/book3s/64/mmu-hash.h>
#ifndef __ASSEMBLER__
/*
* ISA 3.0 partition and process table entry format
*/
struct prtb_entry {
__be64 prtb0;
__be64 prtb1;
};
extern struct prtb_entry *process_tb;
struct patb_entry {
__be64 patb0;
__be64 patb1;
};
extern struct patb_entry *partition_tb;
/* Bits in patb0 field */
#define PATB_HR (1UL << 63)
#define RPDB_MASK 0x0fffffffffffff00UL
#define RPDB_SHIFT (1UL << 8)
#define RTS1_SHIFT 61 /* top 2 bits of radix tree size */
#define RTS1_MASK (3UL << RTS1_SHIFT)
#define RTS2_SHIFT 5 /* bottom 3 bits of radix tree size */
#define RTS2_MASK (7UL << RTS2_SHIFT)
#define RPDS_MASK 0x1f /* root page dir. size field */
/* Bits in patb1 field */
#define PATB_GR (1UL << 63) /* guest uses radix; must match HR */
#define PRTS_MASK 0x1f /* process table size field */
#define PRTB_MASK 0x0ffffffffffff000UL
/* Number of supported LPID bits */
extern unsigned int mmu_lpid_bits;
/* Number of supported PID bits */
extern unsigned int mmu_pid_bits;
/* Base PID to allocate from */
extern unsigned int mmu_base_pid;
extern unsigned long __ro_after_init memory_block_size;
#define PRTB_SIZE_SHIFT (mmu_pid_bits + 4)
#define PRTB_ENTRIES (1ul << mmu_pid_bits)
#define PATB_SIZE_SHIFT (mmu_lpid_bits + 4)
#define PATB_ENTRIES (1ul << mmu_lpid_bits)
typedef unsigned long mm_context_id_t;
struct spinlock;
/* Maximum possible number of NPUs in a system. */
#define NV_MAX_NPUS 8
typedef struct {
union {
/*
* We use id as the PIDR content for radix. On hash we can use
* more than one id. The extended ids are used when we start
* having address above 512TB. We allocate one extended id
* for each 512TB. The new id is then used with the 49 bit
* EA to build a new VA. We always use ESID_BITS_1T_MASK bits
* from EA and new context ids to build the new VAs.
*/
mm_context_id_t id;
#ifdef CONFIG_PPC_64S_HASH_MMU
mm_context_id_t extended_id[TASK_SIZE_USER64/TASK_CONTEXT_SIZE];
#endif
};
/* Number of bits in the mm_cpumask */
atomic_t active_cpus;
/* Number of users of the external (Nest) MMU */
atomic_t copros;
/* Number of user space windows opened in process mm_context */
atomic_t vas_windows;
#ifdef CONFIG_PPC_64S_HASH_MMU
struct hash_mm_context *hash_context;
#endif
void __user *vdso;
/*
* pagetable fragment support
*/
void *pte_frag;
void *pmd_frag;
#ifdef CONFIG_SPAPR_TCE_IOMMU
struct list_head iommu_group_mem_list;
#endif
#ifdef CONFIG_PPC_MEM_KEYS
/*
* Each bit represents one protection key.
* bit set -> key allocated
* bit unset -> key available for allocation
*/
u32 pkey_allocation_map;
s16 execute_only_pkey; /* key holding execute-only protection */
#endif
} mm_context_t;
#ifdef CONFIG_PPC_64S_HASH_MMU
static inline u16 mm_ctx_user_psize(mm_context_t *ctx)
{
return ctx->hash_context->user_psize;
}
static inline void mm_ctx_set_user_psize(mm_context_t *ctx, u16 user_psize)
{
ctx->hash_context->user_psize = user_psize;
}
static inline unsigned char *mm_ctx_low_slices(mm_context_t *ctx)
{
return ctx->hash_context->low_slices_psize;
}
static inline unsigned char *mm_ctx_high_slices(mm_context_t *ctx)
{
return ctx->hash_context->high_slices_psize;
}
static inline unsigned long mm_ctx_slb_addr_limit(mm_context_t *ctx)
{
return ctx->hash_context->slb_addr_limit;
}
static inline void mm_ctx_set_slb_addr_limit(mm_context_t *ctx, unsigned long limit)
{
ctx->hash_context->slb_addr_limit = limit;
}
static inline struct slice_mask *slice_mask_for_size(mm_context_t *ctx, int psize)
{
#ifdef CONFIG_PPC_64K_PAGES
if (psize == MMU_PAGE_64K)
return &ctx->hash_context->mask_64k;
#endif
#ifdef CONFIG_HUGETLB_PAGE
if (psize == MMU_PAGE_16M)
return &ctx->hash_context->mask_16m;
if (psize == MMU_PAGE_16G)
return &ctx->hash_context->mask_16g;
#endif
BUG_ON(psize != MMU_PAGE_4K);
return &ctx->hash_context->mask_4k;
}
#ifdef CONFIG_PPC_SUBPAGE_PROT
static inline struct subpage_prot_table *mm_ctx_subpage_prot(mm_context_t *ctx)
{
return ctx->hash_context->spt;
}
#endif
/*
* The current system page and segment sizes
*/
extern int mmu_virtual_psize;
extern int mmu_vmalloc_psize;
extern int mmu_io_psize;
#else /* CONFIG_PPC_64S_HASH_MMU */
#ifdef CONFIG_PPC_64K_PAGES
#define mmu_virtual_psize MMU_PAGE_64K
#else
#define mmu_virtual_psize MMU_PAGE_4K
#endif
#endif
extern int mmu_linear_psize;
extern int mmu_vmemmap_psize;
/* MMU initialization */
void mmu_early_init_devtree(void);
void hash__early_init_devtree(void);
void radix__early_init_devtree(void);
#ifdef CONFIG_PPC_PKEY
void pkey_early_init_devtree(void);
#else
static inline void pkey_early_init_devtree(void) {}
#endif
extern void hash__early_init_mmu(void);
extern void radix__early_init_mmu(void);
static inline void __init early_init_mmu(void)
{
if (radix_enabled())
return radix__early_init_mmu();
return hash__early_init_mmu();
}
extern void hash__early_init_mmu_secondary(void);
extern void radix__early_init_mmu_secondary(void);
static inline void early_init_mmu_secondary(void)
{
if (radix_enabled())
return radix__early_init_mmu_secondary();
return hash__early_init_mmu_secondary();
}
extern void hash__setup_initial_memory_limit(phys_addr_t first_memblock_base,
phys_addr_t first_memblock_size);
static inline void setup_initial_memory_limit(phys_addr_t first_memblock_base,
phys_addr_t first_memblock_size)
{
/*
* Hash has more strict restrictions. At this point we don't
* know which translations we will pick. Hence go with hash
* restrictions.
*/
if (!early_radix_enabled())
hash__setup_initial_memory_limit(first_memblock_base,
first_memblock_size);
}
#ifdef CONFIG_PPC_PSERIES
void __init radix_init_pseries(void);
#else
static inline void radix_init_pseries(void) { }
#endif
#ifdef CONFIG_HOTPLUG_CPU
#define arch_clear_mm_cpumask_cpu(cpu, mm) \
do { \
if (cpumask_test_cpu(cpu, mm_cpumask(mm))) { \
dec_mm_active_cpus(mm); \
cpumask_clear_cpu(cpu, mm_cpumask(mm)); \
} \
} while (0)
void cleanup_cpu_mmu_context(void);
#endif
#ifdef CONFIG_PPC_64S_HASH_MMU
static inline int get_user_context(mm_context_t *ctx, unsigned long ea)
{
int index = ea >> MAX_EA_BITS_PER_CONTEXT;
if (likely(index < ARRAY_SIZE(ctx->extended_id)))
return ctx->extended_id[index];
/* should never happen */
WARN_ON(1);
return 0;
}
static inline unsigned long get_user_vsid(mm_context_t *ctx,
unsigned long ea, int ssize)
{
unsigned long context = get_user_context(ctx, ea);
return get_vsid(context, ea, ssize);
}
#endif
#endif /* __ASSEMBLER__ */
#endif /* _ASM_POWERPC_BOOK3S_64_MMU_H_ */
linux-7.2.3/arch/powerpc/include/asm/book3s/64/pgalloc.h
+183
View File
@@ -0,0 +1,183 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
#ifndef _ASM_POWERPC_BOOK3S_64_PGALLOC_H
#define _ASM_POWERPC_BOOK3S_64_PGALLOC_H
/*
*/
#include <linux/slab.h>
#include <linux/cpumask.h>
#include <linux/kmemleak.h>
#include <linux/percpu.h>
struct vmemmap_backing {
struct vmemmap_backing *list;
unsigned long phys;
unsigned long virt_addr;
};
extern struct vmemmap_backing *vmemmap_list;
extern pmd_t *pmd_fragment_alloc(struct mm_struct *, unsigned long);
extern void pmd_fragment_free(unsigned long *);
extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
extern void __tlb_remove_table(void *_table);
void pte_frag_destroy(void *pte_frag);
static inline pgd_t *radix__pgd_alloc(struct mm_struct *mm)
{
#ifdef CONFIG_PPC_64K_PAGES
return (pgd_t *)__get_free_page(pgtable_gfp_flags(mm, PGALLOC_GFP));
#else
struct page *page;
page = alloc_pages(pgtable_gfp_flags(mm, PGALLOC_GFP | __GFP_RETRY_MAYFAIL),
4);
if (!page)
return NULL;
return (pgd_t *) page_address(page);
#endif
}
static inline void radix__pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
#ifdef CONFIG_PPC_64K_PAGES
free_page((unsigned long)pgd);
#else
free_pages((unsigned long)pgd, 4);
#endif
}
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
pgd_t *pgd;
if (radix_enabled())
return radix__pgd_alloc(mm);
pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE),
pgtable_gfp_flags(mm, GFP_KERNEL));
if (unlikely(!pgd))
return pgd;
/*
* Don't scan the PGD for pointers, it contains references to PUDs but
* those references are not full pointers and so can't be recognised by
* kmemleak.
*/
kmemleak_no_scan(pgd);
/*
* With hugetlb, we don't clear the second half of the page table.
* If we share the same slab cache with the pmd or pud level table,
* we need to make sure we zero out the full table on alloc.
* With 4K we don't store slot in the second half. Hence we don't
* need to do this for 4k.
*/
#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_PPC_64K_PAGES) && \
(H_PGD_INDEX_SIZE == H_PUD_CACHE_INDEX)
memset(pgd, 0, PGD_TABLE_SIZE);
#endif
return pgd;
}
static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
if (radix_enabled())
return radix__pgd_free(mm, pgd);
kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd);
}
static inline void p4d_populate(struct mm_struct *mm, p4d_t *pgd, pud_t *pud)
{
*pgd = __p4d(__pgtable_ptr_val(pud) | PGD_VAL_BITS);
}
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
pud_t *pud;
pud = kmem_cache_alloc(PGT_CACHE(PUD_CACHE_INDEX),
pgtable_gfp_flags(mm, GFP_KERNEL));
/*
* Tell kmemleak to ignore the PUD, that means don't scan it for
* pointers and don't consider it a leak. PUDs are typically only
* referred to by their PGD, but kmemleak is not able to recognise those
* as pointers, leading to false leak reports.
*/
kmemleak_ignore(pud);
return pud;
}
static inline void __pud_free(pud_t *pud)
{
struct page *page = virt_to_page(pud);
/*
* Early pud pages allocated via memblock allocator
* can't be directly freed to slab. KFENCE pages have
* both reserved and slab flags set so need to be freed
* kmem_cache_free.
*/
if (PageReserved(page) && !PageSlab(page))
free_reserved_page(page);
else
kmem_cache_free(PGT_CACHE(PUD_CACHE_INDEX), pud);
}
static inline void pud_free(struct mm_struct *mm, pud_t *pud)
{
return __pud_free(pud);
}
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
*pud = __pud(__pgtable_ptr_val(pmd) | PUD_VAL_BITS);
}
static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
unsigned long address)
{
pgtable_free_tlb(tlb, pud, PUD_INDEX);
}
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
return pmd_fragment_alloc(mm, addr);
}
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{
pmd_fragment_free((unsigned long *)pmd);
}
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
unsigned long address)
{
return pgtable_free_tlb(tlb, pmd, PMD_INDEX);
}
static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
pte_t *pte)
{
*pmd = __pmd(__pgtable_ptr_val(pte) | PMD_VAL_BITS);
}
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
pgtable_t pte_page)
{
*pmd = __pmd(__pgtable_ptr_val(pte_page) | PMD_VAL_BITS);
}
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
unsigned long address)
{
pgtable_free_tlb(tlb, table, PTE_INDEX);
}
extern atomic_long_t direct_pages_count[MMU_PAGE_COUNT];
static inline void update_page_count(int psize, long count)
{
if (IS_ENABLED(CONFIG_PROC_FS))
atomic_long_add(count, &direct_pages_count[psize]);
}
#endif /* _ASM_POWERPC_BOOK3S_64_PGALLOC_H */
linux-7.2.3/arch/powerpc/include/asm/book3s/64/pgtable-64k.h
+18
View File
@@ -0,0 +1,18 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_64K_H
#define _ASM_POWERPC_BOOK3S_64_PGTABLE_64K_H
#ifndef __ASSEMBLER__
#ifdef CONFIG_HUGETLB_PAGE
#endif /* CONFIG_HUGETLB_PAGE */
static inline int remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr,
unsigned long pfn, pgprot_t prot)
{
if (radix_enabled())
BUG();
return hash__remap_4k_pfn(vma, addr, pfn, prot);
}
#endif /* __ASSEMBLER__ */
#endif /*_ASM_POWERPC_BOOK3S_64_PGTABLE_64K_H */
linux-7.2.3/arch/powerpc/include/asm/book3s/64/pgtable.h
+1464
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/arch/powerpc/include/asm/book3s/64/pkeys.h
+25
View File
@@ -0,0 +1,25 @@
/* SPDX-License-Identifier: GPL-2.0+ */
#ifndef _ASM_POWERPC_BOOK3S_64_PKEYS_H
#define _ASM_POWERPC_BOOK3S_64_PKEYS_H
#include <asm/book3s/64/hash-pkey.h>
static inline u64 vmflag_to_pte_pkey_bits(vm_flags_t vm_flags)
{
if (!mmu_has_feature(MMU_FTR_PKEY))
return 0x0UL;
if (radix_enabled())
BUG();
return hash__vmflag_to_pte_pkey_bits(vm_flags);
}
static inline u16 pte_to_pkey_bits(u64 pteflags)
{
if (radix_enabled())
BUG();
return hash__pte_to_pkey_bits(pteflags);
}
#endif /*_ASM_POWERPC_KEYS_H */
linux-7.2.3/arch/powerpc/include/asm/book3s/64/radix-4k.h
+22
View File
@@ -0,0 +1,22 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_PGTABLE_RADIX_4K_H
#define _ASM_POWERPC_PGTABLE_RADIX_4K_H
/*
* For 4K page size supported index is 13/9/9/9
*/
#define RADIX_PTE_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 4K = 2MB
#define RADIX_PMD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 2MB = 1GB
#define RADIX_PUD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 1GB = 512GB
#define RADIX_PGD_INDEX_SIZE 13 // size: 8B << 13 = 64KB, maps 2^13 x 512GB = 4PB
/*
* One fragment per page
*/
#define RADIX_PTE_FRAG_SIZE_SHIFT (RADIX_PTE_INDEX_SIZE + 3)
#define RADIX_PTE_FRAG_NR (PAGE_SIZE >> RADIX_PTE_FRAG_SIZE_SHIFT)
#define RADIX_PMD_FRAG_SIZE_SHIFT (RADIX_PMD_INDEX_SIZE + 3)
#define RADIX_PMD_FRAG_NR (PAGE_SIZE >> RADIX_PMD_FRAG_SIZE_SHIFT)
#endif /* _ASM_POWERPC_PGTABLE_RADIX_4K_H */
linux-7.2.3/arch/powerpc/include/asm/book3s/64/radix-64k.h
+23
View File
@@ -0,0 +1,23 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_PGTABLE_RADIX_64K_H
#define _ASM_POWERPC_PGTABLE_RADIX_64K_H
/*
* For 64K page size supported index is 13/9/9/5
*/
#define RADIX_PTE_INDEX_SIZE 5 // size: 8B << 5 = 256B, maps 2^5 x 64K = 2MB
#define RADIX_PMD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 2MB = 1GB
#define RADIX_PUD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 1GB = 512GB
#define RADIX_PGD_INDEX_SIZE 13 // size: 8B << 13 = 64KB, maps 2^13 x 512GB = 4PB
/*
* We use a 256 byte PTE page fragment in radix
* 8 bytes per each PTE entry.
*/
#define RADIX_PTE_FRAG_SIZE_SHIFT (RADIX_PTE_INDEX_SIZE + 3)
#define RADIX_PTE_FRAG_NR (PAGE_SIZE >> RADIX_PTE_FRAG_SIZE_SHIFT)
#define RADIX_PMD_FRAG_SIZE_SHIFT (RADIX_PMD_INDEX_SIZE + 3)
#define RADIX_PMD_FRAG_NR (PAGE_SIZE >> RADIX_PMD_FRAG_SIZE_SHIFT)
#endif /* _ASM_POWERPC_PGTABLE_RADIX_64K_H */
linux-7.2.3/arch/powerpc/include/asm/book3s/64/radix.h
+366
View File
@@ -0,0 +1,366 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_PGTABLE_RADIX_H
#define _ASM_POWERPC_PGTABLE_RADIX_H
#include <asm/asm-const.h>
#ifndef __ASSEMBLER__
#include <asm/cmpxchg.h>
#endif
#ifdef CONFIG_PPC_64K_PAGES
#include <asm/book3s/64/radix-64k.h>
#else
#include <asm/book3s/64/radix-4k.h>
#endif
#ifndef __ASSEMBLER__
#include <asm/book3s/64/tlbflush-radix.h>
#include <asm/cpu_has_feature.h>
#endif
/* An empty PTE can still have a R or C writeback */
#define RADIX_PTE_NONE_MASK (_PAGE_DIRTY | _PAGE_ACCESSED)
/* Bits to set in a RPMD/RPUD/RPGD */
#define RADIX_PMD_VAL_BITS (0x8000000000000000UL | RADIX_PTE_INDEX_SIZE)
#define RADIX_PUD_VAL_BITS (0x8000000000000000UL | RADIX_PMD_INDEX_SIZE)
#define RADIX_PGD_VAL_BITS (0x8000000000000000UL | RADIX_PUD_INDEX_SIZE)
/* Don't have anything in the reserved bits and leaf bits */
#define RADIX_PMD_BAD_BITS 0x60000000000000e0UL
#define RADIX_PUD_BAD_BITS 0x60000000000000e0UL
#define RADIX_P4D_BAD_BITS 0x60000000000000e0UL
#define RADIX_PMD_SHIFT (PAGE_SHIFT + RADIX_PTE_INDEX_SIZE)
#define RADIX_PUD_SHIFT (RADIX_PMD_SHIFT + RADIX_PMD_INDEX_SIZE)
#define RADIX_PGD_SHIFT (RADIX_PUD_SHIFT + RADIX_PUD_INDEX_SIZE)
#define R_PTRS_PER_PTE (1 << RADIX_PTE_INDEX_SIZE)
#define R_PTRS_PER_PMD (1 << RADIX_PMD_INDEX_SIZE)
#define R_PTRS_PER_PUD (1 << RADIX_PUD_INDEX_SIZE)
/*
* Size of EA range mapped by our pagetables.
*/
#define RADIX_PGTABLE_EADDR_SIZE (RADIX_PTE_INDEX_SIZE + RADIX_PMD_INDEX_SIZE + \
RADIX_PUD_INDEX_SIZE + RADIX_PGD_INDEX_SIZE + PAGE_SHIFT)
#define RADIX_PGTABLE_RANGE (ASM_CONST(1) << RADIX_PGTABLE_EADDR_SIZE)
/*
* We support 52 bit address space, Use top bit for kernel
* virtual mapping. Also make sure kernel fit in the top
* quadrant.
*
* +------------------+
* +------------------+ Kernel virtual map (0xc008000000000000)
* | |
* | |
* | |
* 0b11......+------------------+ Kernel linear map (0xc....)
* | |
* | 2 quadrant |
* | |
* 0b10......+------------------+
* | |
* | 1 quadrant |
* | |
* 0b01......+------------------+
* | |
* | 0 quadrant |
* | |
* 0b00......+------------------+
*
*
* 3rd quadrant expanded:
* +------------------------------+ Highest address (0xc010000000000000)
* +------------------------------+ KASAN shadow end (0xc00fc00000000000)
* | |
* | |
* +------------------------------+ Kernel vmemmap end/shadow start (0xc00e000000000000)
* | |
* | 512TB |
* | |
* +------------------------------+ Kernel IO map end/vmemap start
* | |
* | 512TB |
* | |
* +------------------------------+ Kernel vmap end/ IO map start
* | |
* | 512TB |
* | |
* +------------------------------+ Kernel virt start (0xc008000000000000)
* | |
* | |
* | |
* +------------------------------+ Kernel linear (0xc.....)
*/
/* For the sizes of the shadow area, see kasan.h */
/*
* If we store section details in page->flags we can't increase the MAX_PHYSMEM_BITS
* if we increase SECTIONS_WIDTH we will not store node details in page->flags and
* page_to_nid does a page->section->node lookup
* Hence only increase for VMEMMAP. Further depending on SPARSEMEM_EXTREME reduce
* memory requirements with large number of sections.
* 51 bits is the max physical real address on POWER9
*/
#if defined(CONFIG_SPARSEMEM_VMEMMAP) && defined(CONFIG_SPARSEMEM_EXTREME)
#define R_MAX_PHYSMEM_BITS 51
#else
#define R_MAX_PHYSMEM_BITS 46
#endif
#define RADIX_KERN_VIRT_START ASM_CONST(0xc008000000000000)
/*
* 49 = MAX_EA_BITS_PER_CONTEXT (hash specific). To make sure we pick
* the same value as hash.
*/
#define RADIX_KERN_MAP_SIZE (1UL << 49)
#define RADIX_VMALLOC_START RADIX_KERN_VIRT_START
#define RADIX_VMALLOC_SIZE RADIX_KERN_MAP_SIZE
#define RADIX_VMALLOC_END (RADIX_VMALLOC_START + RADIX_VMALLOC_SIZE)
#define RADIX_KERN_IO_START RADIX_VMALLOC_END
#define RADIX_KERN_IO_SIZE RADIX_KERN_MAP_SIZE
#define RADIX_KERN_IO_END (RADIX_KERN_IO_START + RADIX_KERN_IO_SIZE)
#define RADIX_VMEMMAP_START RADIX_KERN_IO_END
#define RADIX_VMEMMAP_SIZE RADIX_KERN_MAP_SIZE
#define RADIX_VMEMMAP_END (RADIX_VMEMMAP_START + RADIX_VMEMMAP_SIZE)
#ifndef __ASSEMBLER__
#define RADIX_PTE_TABLE_SIZE (sizeof(pte_t) << RADIX_PTE_INDEX_SIZE)
#define RADIX_PMD_TABLE_SIZE (sizeof(pmd_t) << RADIX_PMD_INDEX_SIZE)
#define RADIX_PUD_TABLE_SIZE (sizeof(pud_t) << RADIX_PUD_INDEX_SIZE)
#define RADIX_PGD_TABLE_SIZE (sizeof(pgd_t) << RADIX_PGD_INDEX_SIZE)
#ifdef CONFIG_STRICT_KERNEL_RWX
extern void radix__mark_rodata_ro(void);
extern void radix__mark_initmem_nx(void);
#endif
extern void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
pte_t entry, unsigned long address,
int psize);
extern void radix__ptep_modify_prot_commit(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep,
pte_t old_pte, pte_t pte);
static inline unsigned long __radix_pte_update(pte_t *ptep, unsigned long clr,
unsigned long set)
{
__be64 old_be, tmp_be;
__asm__ __volatile__(
"1: ldarx %0,0,%3 # pte_update\n"
" andc %1,%0,%5 \n"
" or %1,%1,%4 \n"
" stdcx. %1,0,%3 \n"
" bne- 1b"
: "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep)
: "r" (ptep), "r" (cpu_to_be64(set)), "r" (cpu_to_be64(clr))
: "cc" );
return be64_to_cpu(old_be);
}
static inline unsigned long radix__pte_update(struct mm_struct *mm,
unsigned long addr,
pte_t *ptep, unsigned long clr,
unsigned long set,
int huge)
{
unsigned long old_pte;
old_pte = __radix_pte_update(ptep, clr, set);
if (!huge)
assert_pte_locked(mm, addr);
return old_pte;
}
static inline pte_t radix__ptep_get_and_clear_full(struct mm_struct *mm,
unsigned long addr,
pte_t *ptep, int full)
{
unsigned long old_pte;
if (full) {
old_pte = pte_val(*ptep);
*ptep = __pte(0);
} else
old_pte = radix__pte_update(mm, addr, ptep, ~0ul, 0, 0);
return __pte(old_pte);
}
static inline int radix__pte_same(pte_t pte_a, pte_t pte_b)
{
return ((pte_raw(pte_a) ^ pte_raw(pte_b)) == 0);
}
static inline int radix__pte_none(pte_t pte)
{
return (pte_val(pte) & ~RADIX_PTE_NONE_MASK) == 0;
}
static inline void radix__set_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t pte, int percpu)
{
*ptep = pte;
/*
* The architecture suggests a ptesync after setting the pte, which
* orders the store that updates the pte with subsequent page table
* walk accesses which may load the pte. Without this it may be
* possible for a subsequent access to result in spurious fault.
*
* This is not necessary for correctness, because a spurious fault
* is tolerated by the page fault handler, and this store will
* eventually be seen. In testing, there was no noticable increase
* in user faults on POWER9. Avoiding ptesync here is a significant
* win for things like fork. If a future microarchitecture benefits
* from ptesync, it should probably go into update_mmu_cache, rather
* than set_pte_at (which is used to set ptes unrelated to faults).
*
* Spurious faults from the kernel memory are not tolerated, so there
* is a ptesync in flush_cache_vmap, and __map_kernel_page() follows
* the pte update sequence from ISA Book III 6.10 Translation Table
* Update Synchronization Requirements.
*/
}
static inline int radix__pmd_bad(pmd_t pmd)
{
return !!(pmd_val(pmd) & RADIX_PMD_BAD_BITS);
}
static inline int radix__pmd_same(pmd_t pmd_a, pmd_t pmd_b)
{
return ((pmd_raw(pmd_a) ^ pmd_raw(pmd_b)) == 0);
}
static inline int radix__pud_bad(pud_t pud)
{
return !!(pud_val(pud) & RADIX_PUD_BAD_BITS);
}
static inline int radix__pud_same(pud_t pud_a, pud_t pud_b)
{
return ((pud_raw(pud_a) ^ pud_raw(pud_b)) == 0);
}
static inline int radix__p4d_bad(p4d_t p4d)
{
return !!(p4d_val(p4d) & RADIX_P4D_BAD_BITS);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline int radix__pmd_trans_huge(pmd_t pmd)
{
return (pmd_val(pmd) & _PAGE_PTE) == _PAGE_PTE;
}
static inline pmd_t radix__pmd_mkhuge(pmd_t pmd)
{
return __pmd(pmd_val(pmd) | _PAGE_PTE);
}
static inline int radix__pud_trans_huge(pud_t pud)
{
return (pud_val(pud) & _PAGE_PTE) == _PAGE_PTE;
}
static inline pud_t radix__pud_mkhuge(pud_t pud)
{
return __pud(pud_val(pud) | _PAGE_PTE);
}
extern unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp, unsigned long clr,
unsigned long set);
extern unsigned long radix__pud_hugepage_update(struct mm_struct *mm, unsigned long addr,
pud_t *pudp, unsigned long clr,
unsigned long set);
extern pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp);
extern void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
pgtable_t pgtable);
extern pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
extern pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp);
pud_t radix__pudp_huge_get_and_clear(struct mm_struct *mm,
unsigned long addr, pud_t *pudp);
static inline int radix__has_transparent_hugepage(void)
{
/* For radix 2M at PMD level means thp */
if (mmu_psize_defs[MMU_PAGE_2M].shift == PMD_SHIFT)
return 1;
return 0;
}
static inline int radix__has_transparent_pud_hugepage(void)
{
/* For radix 1G at PUD level means pud hugepage support */
if (mmu_psize_defs[MMU_PAGE_1G].shift == PUD_SHIFT)
return 1;
return 0;
}
#endif
struct vmem_altmap;
struct dev_pagemap;
extern int __meminit radix__vmemmap_create_mapping(unsigned long start,
unsigned long page_size,
unsigned long phys);
int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end,
int node, struct vmem_altmap *altmap);
void __ref radix__vmemmap_free(unsigned long start, unsigned long end,
struct vmem_altmap *altmap);
extern void radix__vmemmap_remove_mapping(unsigned long start,
unsigned long page_size);
extern int radix__map_kernel_page(unsigned long ea, unsigned long pa,
pgprot_t flags, unsigned int psz);
static inline unsigned long radix__get_tree_size(void)
{
unsigned long rts_field;
/*
* We support 52 bits, hence:
* bits 52 - 31 = 21, 0b10101
* RTS encoding details
* bits 0 - 3 of rts -> bits 6 - 8 unsigned long
* bits 4 - 5 of rts -> bits 62 - 63 of unsigned long
*/
rts_field = (0x5UL << 5); /* 6 - 8 bits */
rts_field |= (0x2UL << 61);
return rts_field;
}
#ifdef CONFIG_MEMORY_HOTPLUG
int radix__create_section_mapping(unsigned long start, unsigned long end,
int nid, pgprot_t prot);
int radix__remove_section_mapping(unsigned long start, unsigned long end);
#endif /* CONFIG_MEMORY_HOTPLUG */
#ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP
#define vmemmap_can_optimize vmemmap_can_optimize
bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap);
#endif
#define vmemmap_populate_compound_pages vmemmap_populate_compound_pages
int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
unsigned long start,
unsigned long end, int node,
struct dev_pagemap *pgmap);
#endif /* __ASSEMBLER__ */
#endif
linux-7.2.3/arch/powerpc/include/asm/book3s/64/slice.h
+42
View File
@@ -0,0 +1,42 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_BOOK3S_64_SLICE_H
#define _ASM_POWERPC_BOOK3S_64_SLICE_H
#ifndef __ASSEMBLER__
#ifdef CONFIG_PPC_64S_HASH_MMU
#ifdef CONFIG_HUGETLB_PAGE
#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
#endif
#define HAVE_ARCH_UNMAPPED_AREA
#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
#endif
#define SLICE_LOW_SHIFT 28
#define SLICE_LOW_TOP (0x100000000ul)
#define SLICE_NUM_LOW (SLICE_LOW_TOP >> SLICE_LOW_SHIFT)
#define GET_LOW_SLICE_INDEX(addr) ((addr) >> SLICE_LOW_SHIFT)
#define SLICE_HIGH_SHIFT 40
#define SLICE_NUM_HIGH (H_PGTABLE_RANGE >> SLICE_HIGH_SHIFT)
#define GET_HIGH_SLICE_INDEX(addr) ((addr) >> SLICE_HIGH_SHIFT)
#define SLB_ADDR_LIMIT_DEFAULT DEFAULT_MAP_WINDOW_USER64
struct mm_struct;
unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
unsigned long flags, unsigned int psize,
int topdown);
unsigned int get_slice_psize(struct mm_struct *mm, unsigned long addr);
void slice_set_range_psize(struct mm_struct *mm, unsigned long start,
unsigned long len, unsigned int psize);
void slice_init_new_context_exec(struct mm_struct *mm);
void slice_setup_new_exec(void);
#endif /* __ASSEMBLER__ */
#endif /* _ASM_POWERPC_BOOK3S_64_SLICE_H */
linux-7.2.3/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h
+77
View File
@@ -0,0 +1,77 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H
#define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H
/*
* TLB flushing for 64-bit hash-MMU CPUs
*/
#include <linux/percpu.h>
#include <asm/page.h>
#define PPC64_TLB_BATCH_NR 192
struct ppc64_tlb_batch {
unsigned long index;
struct mm_struct *mm;
real_pte_t pte[PPC64_TLB_BATCH_NR];
unsigned long vpn[PPC64_TLB_BATCH_NR];
unsigned int psize;
int ssize;
};
DECLARE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch);
extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch);
static inline void arch_enter_lazy_mmu_mode(void)
{
if (radix_enabled())
return;
/*
* apply_to_page_range can call us this preempt enabled when
* operating on kernel page tables.
*/
preempt_disable();
}
static inline void arch_flush_lazy_mmu_mode(void)
{
struct ppc64_tlb_batch *batch;
if (radix_enabled())
return;
batch = this_cpu_ptr(&ppc64_tlb_batch);
if (batch->index)
__flush_tlb_pending(batch);
}
static inline void arch_leave_lazy_mmu_mode(void)
{
if (radix_enabled())
return;
arch_flush_lazy_mmu_mode();
preempt_enable();
}
extern void hash__tlbiel_all(unsigned int action);
extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize,
int ssize, unsigned long flags);
extern void flush_hash_range(unsigned long number, int local);
extern void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
pmd_t *pmdp, unsigned int psize, int ssize,
unsigned long flags);
struct mmu_gather;
extern void hash__tlb_flush(struct mmu_gather *tlb);
#ifdef CONFIG_PPC_64S_HASH_MMU
/* Private function for use by PCI IO mapping code */
extern void __flush_hash_table_range(unsigned long start, unsigned long end);
void flush_hash_table_pmd_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr);
#else
static inline void __flush_hash_table_range(unsigned long start, unsigned long end) { }
#endif
#endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H */
linux-7.2.3/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+98
View File
@@ -0,0 +1,98 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_TLBFLUSH_RADIX_H
#define _ASM_POWERPC_TLBFLUSH_RADIX_H
#include <asm/hvcall.h>
#define RIC_FLUSH_TLB 0
#define RIC_FLUSH_PWC 1
#define RIC_FLUSH_ALL 2
struct vm_area_struct;
struct mm_struct;
struct mmu_gather;
static inline u64 psize_to_rpti_pgsize(unsigned long psize)
{
if (psize == MMU_PAGE_4K)
return H_RPTI_PAGE_4K;
if (psize == MMU_PAGE_64K)
return H_RPTI_PAGE_64K;
if (psize == MMU_PAGE_2M)
return H_RPTI_PAGE_2M;
if (psize == MMU_PAGE_1G)
return H_RPTI_PAGE_1G;
return H_RPTI_PAGE_ALL;
}
static inline int mmu_get_ap(int psize)
{
return mmu_psize_defs[psize].ap;
}
#ifdef CONFIG_PPC_RADIX_MMU
extern void radix__tlbiel_all(unsigned int action);
extern void radix__flush_tlb_lpid_page(unsigned int lpid,
unsigned long addr,
unsigned long page_size);
extern void radix__flush_pwc_lpid(unsigned int lpid);
extern void radix__flush_all_lpid(unsigned int lpid);
extern void radix__flush_all_lpid_guest(unsigned int lpid);
#else
static inline void radix__tlbiel_all(unsigned int action) { WARN_ON(1); }
static inline void radix__flush_tlb_lpid_page(unsigned int lpid,
unsigned long addr,
unsigned long page_size)
{
WARN_ON(1);
}
static inline void radix__flush_pwc_lpid(unsigned int lpid)
{
WARN_ON(1);
}
static inline void radix__flush_all_lpid(unsigned int lpid)
{
WARN_ON(1);
}
static inline void radix__flush_all_lpid_guest(unsigned int lpid)
{
WARN_ON(1);
}
#endif
extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end);
extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
unsigned long end, int psize);
void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
unsigned long end, int psize);
extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end);
extern void radix__flush_pud_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end);
extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end);
extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
extern void radix__local_flush_all_mm(struct mm_struct *mm);
extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
int psize);
extern void radix__tlb_flush(struct mmu_gather *tlb);
#ifdef CONFIG_SMP
extern void radix__flush_tlb_mm(struct mm_struct *mm);
extern void radix__flush_all_mm(struct mm_struct *mm);
extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
int psize);
#else
#define radix__flush_tlb_mm(mm) radix__local_flush_tlb_mm(mm)
#define radix__flush_all_mm(mm) radix__local_flush_all_mm(mm)
#define radix__flush_tlb_page(vma,addr) radix__local_flush_tlb_page(vma,addr)
#define radix__flush_tlb_page_psize(mm,addr,p) radix__local_flush_tlb_page_psize(mm,addr,p)
#endif
extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr);
extern void radix__flush_tlb_all(void);
#endif
linux-7.2.3/arch/powerpc/include/asm/book3s/64/tlbflush.h
+225
View File
@@ -0,0 +1,225 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H
#define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H
#define MMU_NO_CONTEXT ~0UL
#include <linux/mm_types.h>
#include <linux/mmu_notifier.h>
#include <asm/book3s/64/tlbflush-hash.h>
#include <asm/book3s/64/tlbflush-radix.h>
/* TLB flush actions. Used as argument to tlbiel_all() */
enum {
TLB_INVAL_SCOPE_GLOBAL = 0, /* invalidate all TLBs */
TLB_INVAL_SCOPE_LPID = 1, /* invalidate TLBs for current LPID */
};
static inline void tlbiel_all(void)
{
/*
* This is used for host machine check and bootup.
*
* This uses early_radix_enabled and implementations use
* early_cpu_has_feature etc because that works early in boot
* and this is the machine check path which is not performance
* critical.
*/
if (early_radix_enabled())
radix__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL);
else
hash__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL);
}
static inline void tlbiel_all_lpid(bool radix)
{
/*
* This is used for guest machine check.
*/
if (radix)
radix__tlbiel_all(TLB_INVAL_SCOPE_LPID);
else
hash__tlbiel_all(TLB_INVAL_SCOPE_LPID);
}
#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
if (radix_enabled())
radix__flush_pmd_tlb_range(vma, start, end);
}
#define __HAVE_ARCH_FLUSH_PUD_TLB_RANGE
static inline void flush_pud_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
if (radix_enabled())
radix__flush_pud_tlb_range(vma, start, end);
}
#define __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
unsigned long start,
unsigned long end)
{
if (radix_enabled())
radix__flush_hugetlb_tlb_range(vma, start, end);
}
static inline void flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
if (radix_enabled())
radix__flush_tlb_range(vma, start, end);
}
static inline void flush_tlb_kernel_range(unsigned long start,
unsigned long end)
{
if (radix_enabled())
radix__flush_tlb_kernel_range(start, end);
}
static inline void local_flush_tlb_mm(struct mm_struct *mm)
{
if (radix_enabled())
radix__local_flush_tlb_mm(mm);
}
static inline void local_flush_tlb_page(struct vm_area_struct *vma,
unsigned long vmaddr)
{
if (radix_enabled())
radix__local_flush_tlb_page(vma, vmaddr);
}
static inline void local_flush_tlb_page_psize(struct mm_struct *mm,
unsigned long vmaddr, int psize)
{
if (radix_enabled())
radix__local_flush_tlb_page_psize(mm, vmaddr, psize);
}
static inline void tlb_flush(struct mmu_gather *tlb)
{
if (radix_enabled())
radix__tlb_flush(tlb);
else
hash__tlb_flush(tlb);
}
#ifdef CONFIG_SMP
static inline void flush_tlb_mm(struct mm_struct *mm)
{
if (radix_enabled())
radix__flush_tlb_mm(mm);
}
static inline void flush_tlb_page(struct vm_area_struct *vma,
unsigned long vmaddr)
{
if (radix_enabled())
radix__flush_tlb_page(vma, vmaddr);
}
#else
#define flush_tlb_mm(mm) local_flush_tlb_mm(mm)
#define flush_tlb_page(vma, addr) local_flush_tlb_page(vma, addr)
#endif /* CONFIG_SMP */
#define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault
static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma,
unsigned long address,
pte_t *ptep)
{
/*
* Book3S 64 does not require spurious fault flushes because the PTE
* must be re-fetched in case of an access permission problem. So the
* only reason for a spurious fault should be concurrent modification
* to the PTE, in which case the PTE will eventually be re-fetched by
* the MMU when it attempts the access again.
*
* See: Power ISA Version 3.1B, 6.10.1.2 Modifying a Translation Table
* Entry, Setting a Reference or Change Bit or Upgrading Access
* Authority (PTE Subject to Atomic Hardware Updates):
*
* "If the only change being made to a valid PTE that is subject to
* atomic hardware updates is to set the Reference or Change bit to
* 1 or to upgrade access authority, a simpler sequence suffices
* because the translation hardware will refetch the PTE if an
* access is attempted for which the only problems were reference
* and/or change bits needing to be set or insufficient access
* authority."
*
* The nest MMU in POWER9 does not perform this PTE re-fetch, but
* it avoids the spurious fault problem by flushing the TLB before
* upgrading PTE permissions, see radix__ptep_set_access_flags.
*/
}
static inline bool __pte_flags_need_flush(unsigned long oldval,
unsigned long newval)
{
unsigned long delta = oldval ^ newval;
/*
* The return value of this function doesn't matter for hash,
* ptep_modify_prot_start() does a pte_update() which does or schedules
* any necessary hash table update and flush.
*/
if (!radix_enabled())
return true;
/*
* We do not expect kernel mappings or non-PTEs or not-present PTEs.
*/
VM_WARN_ON_ONCE(oldval & _PAGE_PRIVILEGED);
VM_WARN_ON_ONCE(newval & _PAGE_PRIVILEGED);
VM_WARN_ON_ONCE(!(oldval & _PAGE_PTE));
VM_WARN_ON_ONCE(!(newval & _PAGE_PTE));
VM_WARN_ON_ONCE(!(oldval & _PAGE_PRESENT));
VM_WARN_ON_ONCE(!(newval & _PAGE_PRESENT));
/*
* Must flush on any change except READ, WRITE, EXEC, DIRTY, ACCESSED.
*
* In theory, some changed software bits could be tolerated, in
* practice those should rarely if ever matter.
*/
if (delta & ~(_PAGE_RWX | _PAGE_DIRTY | _PAGE_ACCESSED))
return true;
/*
* If any of the above was present in old but cleared in new, flush.
* With the exception of _PAGE_ACCESSED, don't worry about flushing
* if that was cleared (see the comment in ptep_clear_flush_young()).
*/
if ((delta & ~_PAGE_ACCESSED) & oldval)
return true;
return false;
}
static inline bool pte_needs_flush(pte_t oldpte, pte_t newpte)
{
return __pte_flags_need_flush(pte_val(oldpte), pte_val(newpte));
}
#define pte_needs_flush pte_needs_flush
static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd)
{
return __pte_flags_need_flush(pmd_val(oldpmd), pmd_val(newpmd));
}
#define huge_pmd_needs_flush huge_pmd_needs_flush
extern bool tlbie_capable;
extern bool tlbie_enabled;
static inline bool cputlb_use_tlbie(void)
{
return tlbie_enabled;
}
#endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H */