367 lines
9.0 KiB
C
367 lines
9.0 KiB
C
// SPDX-License-Identifier: MIT
|
|
/*
|
|
* Copyright © 2025 Intel Corporation
|
|
*/
|
|
|
|
#include <drm/drm_managed.h>
|
|
|
|
#include "xe_device.h"
|
|
#include "xe_gt_sriov_pf_control.h"
|
|
#include "xe_gt_sriov_pf_migration.h"
|
|
#include "xe_pm.h"
|
|
#include "xe_sriov.h"
|
|
#include "xe_sriov_packet.h"
|
|
#include "xe_sriov_packet_types.h"
|
|
#include "xe_sriov_pf_helpers.h"
|
|
#include "xe_sriov_pf_migration.h"
|
|
#include "xe_sriov_printk.h"
|
|
|
|
static struct xe_sriov_migration_state *pf_pick_migration(struct xe_device *xe, unsigned int vfid)
|
|
{
|
|
xe_assert(xe, IS_SRIOV_PF(xe));
|
|
xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe));
|
|
|
|
return &xe->sriov.pf.vfs[vfid].migration;
|
|
}
|
|
|
|
/**
|
|
* xe_sriov_pf_migration_waitqueue() - Get waitqueue for migration.
|
|
* @xe: the &xe_device
|
|
* @vfid: the VF identifier
|
|
*
|
|
* Return: pointer to the migration waitqueue.
|
|
*/
|
|
wait_queue_head_t *xe_sriov_pf_migration_waitqueue(struct xe_device *xe, unsigned int vfid)
|
|
{
|
|
return &pf_pick_migration(xe, vfid)->wq;
|
|
}
|
|
|
|
/**
|
|
* xe_sriov_pf_migration_supported() - Check if SR-IOV VF migration is supported by the device
|
|
* @xe: the &xe_device
|
|
*
|
|
* Return: true if migration is supported, false otherwise
|
|
*/
|
|
bool xe_sriov_pf_migration_supported(struct xe_device *xe)
|
|
{
|
|
xe_assert(xe, IS_SRIOV_PF(xe));
|
|
|
|
return IS_ENABLED(CONFIG_DRM_XE_DEBUG) || !xe->sriov.pf.migration.disabled;
|
|
}
|
|
|
|
/**
|
|
* xe_sriov_pf_migration_disable() - Turn off SR-IOV VF migration support on PF.
|
|
* @xe: the &xe_device instance.
|
|
* @fmt: format string for the log message, to be combined with following VAs.
|
|
*/
|
|
void xe_sriov_pf_migration_disable(struct xe_device *xe, const char *fmt, ...)
|
|
{
|
|
struct va_format vaf;
|
|
va_list va_args;
|
|
|
|
xe_assert(xe, IS_SRIOV_PF(xe));
|
|
|
|
va_start(va_args, fmt);
|
|
vaf.fmt = fmt;
|
|
vaf.va = &va_args;
|
|
xe_sriov_notice(xe, "migration %s: %pV\n",
|
|
IS_ENABLED(CONFIG_DRM_XE_DEBUG) ?
|
|
"missing prerequisite" : "disabled",
|
|
&vaf);
|
|
va_end(va_args);
|
|
|
|
xe->sriov.pf.migration.disabled = true;
|
|
}
|
|
|
|
static void pf_migration_check_support(struct xe_device *xe)
|
|
{
|
|
if (!xe_device_has_memirq(xe))
|
|
xe_sriov_pf_migration_disable(xe, "requires memory-based IRQ support");
|
|
}
|
|
|
|
static void pf_migration_cleanup(void *arg)
|
|
{
|
|
struct xe_sriov_migration_state *migration = arg;
|
|
|
|
xe_sriov_packet_free(migration->pending);
|
|
xe_sriov_packet_free(migration->trailer);
|
|
xe_sriov_packet_free(migration->descriptor);
|
|
}
|
|
|
|
/**
|
|
* xe_sriov_pf_migration_init() - Initialize support for SR-IOV VF migration.
|
|
* @xe: the &xe_device
|
|
*
|
|
* Return: 0 on success or a negative error code on failure.
|
|
*/
|
|
int xe_sriov_pf_migration_init(struct xe_device *xe)
|
|
{
|
|
unsigned int n, totalvfs;
|
|
int err;
|
|
|
|
xe_assert(xe, IS_SRIOV_PF(xe));
|
|
|
|
pf_migration_check_support(xe);
|
|
|
|
if (!xe_sriov_pf_migration_supported(xe))
|
|
return 0;
|
|
|
|
totalvfs = xe_sriov_pf_get_totalvfs(xe);
|
|
for (n = 1; n <= totalvfs; n++) {
|
|
struct xe_sriov_migration_state *migration = pf_pick_migration(xe, n);
|
|
|
|
err = drmm_mutex_init(&xe->drm, &migration->lock);
|
|
if (err)
|
|
return err;
|
|
|
|
init_waitqueue_head(&migration->wq);
|
|
|
|
err = devm_add_action_or_reset(xe->drm.dev, pf_migration_cleanup, migration);
|
|
if (err)
|
|
return err;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static bool pf_migration_data_ready(struct xe_device *xe, unsigned int vfid)
|
|
{
|
|
struct xe_gt *gt;
|
|
u8 gt_id;
|
|
|
|
for_each_gt(gt, xe, gt_id) {
|
|
if (xe_gt_sriov_pf_control_check_save_failed(gt, vfid) ||
|
|
xe_gt_sriov_pf_control_check_save_data_done(gt, vfid) ||
|
|
!xe_gt_sriov_pf_migration_ring_empty(gt, vfid))
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
static struct xe_sriov_packet *
|
|
pf_migration_consume(struct xe_device *xe, unsigned int vfid)
|
|
{
|
|
struct xe_sriov_packet *data;
|
|
bool more_data = false;
|
|
struct xe_gt *gt;
|
|
u8 gt_id;
|
|
|
|
for_each_gt(gt, xe, gt_id) {
|
|
data = xe_gt_sriov_pf_migration_save_consume(gt, vfid);
|
|
if (!data)
|
|
continue;
|
|
if (!IS_ERR(data) || PTR_ERR(data) != -EAGAIN)
|
|
return data;
|
|
more_data = true;
|
|
}
|
|
|
|
if (!more_data)
|
|
return NULL;
|
|
|
|
return ERR_PTR(-EAGAIN);
|
|
}
|
|
|
|
/**
|
|
* xe_sriov_pf_migration_save_consume() - Consume a VF migration data packet from the device.
|
|
* @xe: the &xe_device
|
|
* @vfid: the VF identifier
|
|
*
|
|
* Called by the save migration data consumer (userspace) when
|
|
* processing migration data.
|
|
* If there is no migration data to process, wait until more data is available.
|
|
*
|
|
* Return: Pointer to &xe_sriov_packet on success,
|
|
* NULL if ring is empty and no more migration data is expected,
|
|
* ERR_PTR value in case of error.
|
|
*/
|
|
struct xe_sriov_packet *
|
|
xe_sriov_pf_migration_save_consume(struct xe_device *xe, unsigned int vfid)
|
|
{
|
|
struct xe_sriov_migration_state *migration = pf_pick_migration(xe, vfid);
|
|
struct xe_sriov_packet *data;
|
|
int ret;
|
|
|
|
xe_assert(xe, IS_SRIOV_PF(xe));
|
|
|
|
for (;;) {
|
|
data = pf_migration_consume(xe, vfid);
|
|
if (PTR_ERR(data) != -EAGAIN)
|
|
break;
|
|
|
|
ret = wait_event_interruptible(migration->wq,
|
|
pf_migration_data_ready(xe, vfid));
|
|
if (ret)
|
|
return ERR_PTR(ret);
|
|
}
|
|
|
|
return data;
|
|
}
|
|
|
|
static int pf_handle_descriptor(struct xe_device *xe, unsigned int vfid,
|
|
struct xe_sriov_packet *data)
|
|
{
|
|
int ret;
|
|
|
|
if (data->hdr.tile_id != 0 || data->hdr.gt_id != 0)
|
|
return -EINVAL;
|
|
|
|
ret = xe_sriov_packet_process_descriptor(xe, vfid, data);
|
|
if (ret)
|
|
return ret;
|
|
|
|
xe_sriov_packet_free(data);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int pf_handle_trailer(struct xe_device *xe, unsigned int vfid,
|
|
struct xe_sriov_packet *data)
|
|
{
|
|
struct xe_gt *gt;
|
|
u8 gt_id;
|
|
|
|
if (data->hdr.tile_id != 0 || data->hdr.gt_id != 0)
|
|
return -EINVAL;
|
|
if (data->hdr.offset != 0 || data->hdr.size != 0 || data->buff || data->bo)
|
|
return -EINVAL;
|
|
|
|
xe_sriov_packet_free(data);
|
|
|
|
for_each_gt(gt, xe, gt_id)
|
|
xe_gt_sriov_pf_control_restore_data_done(gt, vfid);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* xe_sriov_pf_migration_restore_produce() - Produce a VF migration data packet to the device.
|
|
* @xe: the &xe_device
|
|
* @vfid: the VF identifier
|
|
* @data: Pointer to &xe_sriov_packet
|
|
*
|
|
* Called by the restore migration data producer (userspace) when processing
|
|
* migration data.
|
|
* If the underlying data structure is full, wait until there is space.
|
|
*
|
|
* Return: 0 on success or a negative error code on failure.
|
|
*/
|
|
int xe_sriov_pf_migration_restore_produce(struct xe_device *xe, unsigned int vfid,
|
|
struct xe_sriov_packet *data)
|
|
{
|
|
struct xe_gt *gt;
|
|
|
|
xe_assert(xe, IS_SRIOV_PF(xe));
|
|
|
|
if (data->hdr.type == XE_SRIOV_PACKET_TYPE_DESCRIPTOR)
|
|
return pf_handle_descriptor(xe, vfid, data);
|
|
if (data->hdr.type == XE_SRIOV_PACKET_TYPE_TRAILER)
|
|
return pf_handle_trailer(xe, vfid, data);
|
|
|
|
gt = xe_device_get_gt(xe, data->hdr.gt_id);
|
|
if (!gt || data->hdr.tile_id != gt->tile->id || data->hdr.type == 0) {
|
|
xe_sriov_err_ratelimited(xe, "Received invalid restore packet for VF%u (type:%u, tile:%u, GT:%u)\n",
|
|
vfid, data->hdr.type, data->hdr.tile_id, data->hdr.gt_id);
|
|
return -EINVAL;
|
|
}
|
|
|
|
return xe_gt_sriov_pf_migration_restore_produce(gt, vfid, data);
|
|
}
|
|
|
|
/**
|
|
* xe_sriov_pf_migration_read() - Read migration data from the device.
|
|
* @xe: the &xe_device
|
|
* @vfid: the VF identifier
|
|
* @buf: start address of userspace buffer
|
|
* @len: requested read size from userspace
|
|
*
|
|
* Return: number of bytes that has been successfully read,
|
|
* 0 if no more migration data is available,
|
|
* -errno on failure.
|
|
*/
|
|
ssize_t xe_sriov_pf_migration_read(struct xe_device *xe, unsigned int vfid,
|
|
char __user *buf, size_t len)
|
|
{
|
|
struct xe_sriov_migration_state *migration = pf_pick_migration(xe, vfid);
|
|
ssize_t ret, consumed = 0;
|
|
|
|
xe_assert(xe, IS_SRIOV_PF(xe));
|
|
|
|
scoped_cond_guard(mutex_intr, return -EINTR, &migration->lock) {
|
|
while (consumed < len) {
|
|
ret = xe_sriov_packet_read_single(xe, vfid, buf, len - consumed);
|
|
if (ret == -ENODATA)
|
|
break;
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
consumed += ret;
|
|
buf += ret;
|
|
}
|
|
}
|
|
|
|
return consumed;
|
|
}
|
|
|
|
/**
|
|
* xe_sriov_pf_migration_write() - Write migration data to the device.
|
|
* @xe: the &xe_device
|
|
* @vfid: the VF identifier
|
|
* @buf: start address of userspace buffer
|
|
* @len: requested write size from userspace
|
|
*
|
|
* Return: number of bytes that has been successfully written,
|
|
* -errno on failure.
|
|
*/
|
|
ssize_t xe_sriov_pf_migration_write(struct xe_device *xe, unsigned int vfid,
|
|
const char __user *buf, size_t len)
|
|
{
|
|
struct xe_sriov_migration_state *migration = pf_pick_migration(xe, vfid);
|
|
ssize_t ret, produced = 0;
|
|
|
|
xe_assert(xe, IS_SRIOV_PF(xe));
|
|
|
|
scoped_cond_guard(mutex_intr, return -EINTR, &migration->lock) {
|
|
while (produced < len) {
|
|
ret = xe_sriov_packet_write_single(xe, vfid, buf, len - produced);
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
produced += ret;
|
|
buf += ret;
|
|
}
|
|
}
|
|
|
|
return produced;
|
|
}
|
|
|
|
/**
|
|
* xe_sriov_pf_migration_size() - Total size of migration data from all components within a device
|
|
* @xe: the &xe_device
|
|
* @vfid: the VF identifier (can't be 0)
|
|
*
|
|
* This function is for PF only.
|
|
*
|
|
* Return: total migration data size in bytes or a negative error code on failure.
|
|
*/
|
|
ssize_t xe_sriov_pf_migration_size(struct xe_device *xe, unsigned int vfid)
|
|
{
|
|
size_t size = 0;
|
|
struct xe_gt *gt;
|
|
ssize_t ret;
|
|
u8 gt_id;
|
|
|
|
xe_assert(xe, IS_SRIOV_PF(xe));
|
|
xe_assert(xe, vfid);
|
|
|
|
for_each_gt(gt, xe, gt_id) {
|
|
ret = xe_gt_sriov_pf_migration_size(gt, vfid);
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
size += ret;
|
|
}
|
|
|
|
return size;
|
|
}
|