159 lines
4.6 KiB
C
159 lines
4.6 KiB
C
// SPDX-License-Identifier: MIT
|
|
/*
|
|
* Copyright © 2025 Intel Corporation
|
|
*/
|
|
|
|
#include <linux/bitfield.h>
|
|
#include <linux/kref.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/slab.h>
|
|
|
|
#include "xe_page_reclaim.h"
|
|
|
|
#include "xe_gt_stats.h"
|
|
#include "xe_guc_tlb_inval.h"
|
|
#include "xe_macros.h"
|
|
#include "xe_pat.h"
|
|
#include "xe_sa.h"
|
|
#include "xe_tlb_inval_types.h"
|
|
|
|
/**
|
|
* xe_page_reclaim_skip() - Decide whether PRL should be skipped for a VMA
|
|
* @tile: Tile owning the VMA
|
|
* @vma: VMA under consideration
|
|
*
|
|
* PPC flushing may be handled by HW for specific PAT encodings.
|
|
* Skip PPC flushing/Page Reclaim for scenarios below due to redundant
|
|
* flushes.
|
|
* - pat_index is transient display (1)
|
|
*
|
|
* For cases of NULL VMA, there should be no corresponding PRL entry
|
|
* so skip over.
|
|
*
|
|
* Return: true when page reclamation is unnecessary, false otherwise.
|
|
*/
|
|
bool xe_page_reclaim_skip(struct xe_tile *tile, struct xe_vma *vma)
|
|
{
|
|
u8 l3_policy;
|
|
|
|
if (xe_vma_is_null(vma))
|
|
return true;
|
|
|
|
l3_policy = xe_pat_index_get_l3_policy(tile->xe, vma->attr.pat_index);
|
|
|
|
/*
|
|
* - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 3=UC
|
|
* Transient display flushes is taken care by HW, l3_policy = 1.
|
|
*
|
|
* HW will sequence these transient flushes at various sync points so
|
|
* any event of page reclamation will hit these sync points before
|
|
* page reclamation could execute.
|
|
*/
|
|
return (l3_policy == XE_L3_POLICY_XD);
|
|
}
|
|
|
|
/**
|
|
* xe_page_reclaim_create_prl_bo() - Back a PRL with a suballocated GGTT BO
|
|
* @tlb_inval: TLB invalidation frontend associated with the request
|
|
* @prl: page reclaim list data that bo will copy from
|
|
* @fence: tlb invalidation fence that page reclaim action is paired to
|
|
*
|
|
* Suballocates a 4K BO out of the tile reclaim pool, copies the PRL CPU
|
|
* copy into the BO and queues the buffer for release when @fence signals.
|
|
*
|
|
* Return: struct drm_suballoc pointer on success or ERR_PTR on failure.
|
|
*/
|
|
struct drm_suballoc *xe_page_reclaim_create_prl_bo(struct xe_tlb_inval *tlb_inval,
|
|
struct xe_page_reclaim_list *prl,
|
|
struct xe_tlb_inval_fence *fence)
|
|
{
|
|
struct xe_gt *gt = container_of(tlb_inval, struct xe_gt, tlb_inval);
|
|
struct xe_tile *tile = gt_to_tile(gt);
|
|
/* (+1) for NULL page_reclaim_entry to indicate end of list */
|
|
int prl_size = min(prl->num_entries + 1, XE_PAGE_RECLAIM_MAX_ENTRIES) *
|
|
sizeof(struct xe_guc_page_reclaim_entry);
|
|
struct drm_suballoc *prl_sa;
|
|
|
|
/* Maximum size of PRL is 1 4K-page */
|
|
prl_sa = __xe_sa_bo_new(tile->mem.reclaim_pool,
|
|
prl_size, GFP_ATOMIC);
|
|
if (IS_ERR(prl_sa))
|
|
return prl_sa;
|
|
|
|
memcpy(xe_sa_bo_cpu_addr(prl_sa), prl->entries,
|
|
prl_size);
|
|
xe_sa_bo_flush_write(prl_sa);
|
|
/* Queue up sa_bo_free on tlb invalidation fence signal */
|
|
xe_sa_bo_free(prl_sa, &fence->base);
|
|
|
|
return prl_sa;
|
|
}
|
|
|
|
/**
|
|
* xe_page_reclaim_list_invalidate() - Mark a PRL as invalid
|
|
* @prl: Page reclaim list to reset
|
|
*
|
|
* Clears the entries pointer and marks the list as invalid so
|
|
* future use knows PRL is unusable. It is expected that the entries
|
|
* have already been released.
|
|
*/
|
|
void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl)
|
|
{
|
|
xe_page_reclaim_entries_put(prl->entries);
|
|
prl->entries = NULL;
|
|
prl->num_entries = XE_PAGE_RECLAIM_INVALID_LIST;
|
|
}
|
|
|
|
/**
|
|
* xe_page_reclaim_list_init() - Initialize a page reclaim list
|
|
* @prl: Page reclaim list to initialize
|
|
*
|
|
* NULLs both values in list to prepare on initalization.
|
|
*/
|
|
void xe_page_reclaim_list_init(struct xe_page_reclaim_list *prl)
|
|
{
|
|
prl->entries = NULL;
|
|
prl->num_entries = 0;
|
|
}
|
|
|
|
/**
|
|
* xe_page_reclaim_list_alloc_entries() - Allocate page reclaim list entries
|
|
* @prl: Page reclaim list to allocate entries for
|
|
*
|
|
* Allocate one 4K page for the PRL entries, otherwise assign prl->entries to NULL.
|
|
*/
|
|
int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl)
|
|
{
|
|
struct page *page;
|
|
|
|
if (XE_WARN_ON(prl->entries))
|
|
return 0;
|
|
|
|
page = alloc_page(GFP_KERNEL | __GFP_ZERO);
|
|
if (page) {
|
|
prl->entries = page_address(page);
|
|
prl->num_entries = 0;
|
|
}
|
|
|
|
return page ? 0 : -ENOMEM;
|
|
}
|
|
|
|
/**
|
|
* xe_guc_page_reclaim_done_handler() - Page reclaim done handler
|
|
* @guc: guc
|
|
* @msg: message indicating page reclamation done
|
|
* @len: length of message
|
|
*
|
|
* Page reclamation is an extension of TLB invalidation. Both
|
|
* operations share the same seqno and fence. When either
|
|
* action completes, we need to signal the corresponding
|
|
* fence. Since the handling logic is currently identical, this
|
|
* function delegates to the TLB invalidation handler.
|
|
*
|
|
* Return: 0 on success, -EPROTO for malformed messages.
|
|
*/
|
|
int xe_guc_page_reclaim_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
|
|
{
|
|
return xe_guc_tlb_inval_done_handler(guc, msg, len);
|
|
}
|