409 lines
11 KiB
C
409 lines
11 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* vfio-ISM driver for s390
|
|
*
|
|
* Copyright IBM Corp.
|
|
*/
|
|
|
|
#include <linux/slab.h>
|
|
#include "../vfio_pci_priv.h"
|
|
|
|
#define ISM_VFIO_PCI_OFFSET_SHIFT 48
|
|
#define ISM_VFIO_PCI_OFFSET_TO_INDEX(off) ((off) >> ISM_VFIO_PCI_OFFSET_SHIFT)
|
|
#define ISM_VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << ISM_VFIO_PCI_OFFSET_SHIFT)
|
|
#define ISM_VFIO_PCI_OFFSET_MASK (((u64)(1) << ISM_VFIO_PCI_OFFSET_SHIFT) - 1)
|
|
|
|
/*
|
|
* Use __zpci_load() to bypass automatic use of
|
|
* PCI MIO instructions which are not supported on ISM devices
|
|
*/
|
|
#define ISM_READ(size) \
|
|
static int ism_read##size(struct zpci_dev *zdev, int bar, \
|
|
size_t *filled, char __user *buf, \
|
|
loff_t off) \
|
|
{ \
|
|
u64 req, tmp; \
|
|
u##size val; \
|
|
int ret; \
|
|
\
|
|
req = ZPCI_CREATE_REQ(READ_ONCE(zdev->fh), bar, sizeof(val)); \
|
|
ret = __zpci_load(&tmp, req, off); \
|
|
if (ret) \
|
|
return ret; \
|
|
val = (u##size)tmp; \
|
|
if (copy_to_user(buf, &val, sizeof(val))) \
|
|
return -EFAULT; \
|
|
*filled = sizeof(val); \
|
|
return 0; \
|
|
}
|
|
|
|
ISM_READ(64);
|
|
ISM_READ(32);
|
|
ISM_READ(16);
|
|
ISM_READ(8);
|
|
|
|
struct ism_vfio_pci_core_device {
|
|
struct vfio_pci_core_device core_device;
|
|
struct kmem_cache *store_block_cache;
|
|
};
|
|
|
|
static int ism_vfio_pci_open_device(struct vfio_device *core_vdev)
|
|
{
|
|
struct ism_vfio_pci_core_device *ivpcd;
|
|
struct vfio_pci_core_device *vdev;
|
|
int ret;
|
|
|
|
ivpcd = container_of(core_vdev, struct ism_vfio_pci_core_device,
|
|
core_device.vdev);
|
|
vdev = &ivpcd->core_device;
|
|
|
|
ret = vfio_pci_core_enable(vdev);
|
|
if (ret)
|
|
return ret;
|
|
|
|
vfio_pci_core_finish_enable(vdev);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* ism_vfio_pci_do_io_r()
|
|
*
|
|
* On s390, kernel primitives such as ioread() and iowrite() are switched over
|
|
* from function-handle-based PCI load/stores instructions to PCI memory-I/O (MIO)
|
|
* loads/stores when these are available and not explicitly disabled. Since these
|
|
* instructions cannot be used with ISM devices, ensure that classic
|
|
* function-handle-based PCI instructions are used instead.
|
|
*/
|
|
static ssize_t ism_vfio_pci_do_io_r(struct vfio_pci_core_device *vdev,
|
|
char __user *buf, loff_t off, size_t count,
|
|
int bar)
|
|
{
|
|
struct zpci_dev *zdev = to_zpci(vdev->pdev);
|
|
ssize_t done = 0;
|
|
int ret;
|
|
|
|
while (count) {
|
|
size_t filled;
|
|
|
|
if (count >= 8 && IS_ALIGNED(off, 8)) {
|
|
ret = ism_read64(zdev, bar, &filled, buf, off);
|
|
if (ret)
|
|
return ret;
|
|
} else if (count >= 4 && IS_ALIGNED(off, 4)) {
|
|
ret = ism_read32(zdev, bar, &filled, buf, off);
|
|
if (ret)
|
|
return ret;
|
|
} else if (count >= 2 && IS_ALIGNED(off, 2)) {
|
|
ret = ism_read16(zdev, bar, &filled, buf, off);
|
|
if (ret)
|
|
return ret;
|
|
} else {
|
|
ret = ism_read8(zdev, bar, &filled, buf, off);
|
|
if (ret)
|
|
return ret;
|
|
}
|
|
|
|
count -= filled;
|
|
done += filled;
|
|
off += filled;
|
|
buf += filled;
|
|
}
|
|
|
|
return done;
|
|
}
|
|
|
|
/*
|
|
* ism_vfio_pci_do_io_w()
|
|
*
|
|
* Ensure that the PCI store block (PCISTB) instruction is used as required by the
|
|
* ISM device. The ISM device also uses a 256 TiB BAR 0 for write operations,
|
|
* which requires a 48bit region address space (ISM_VFIO_PCI_OFFSET_SHIFT).
|
|
*/
|
|
static ssize_t ism_vfio_pci_do_io_w(struct vfio_pci_core_device *vdev,
|
|
char __user *buf, loff_t off, size_t count,
|
|
int bar)
|
|
{
|
|
struct zpci_dev *zdev = to_zpci(vdev->pdev);
|
|
struct ism_vfio_pci_core_device *ivpcd;
|
|
ssize_t ret;
|
|
void *data;
|
|
u64 req;
|
|
|
|
if (count > zdev->maxstbl)
|
|
return -EINVAL;
|
|
if (((off % PAGE_SIZE) + count) > PAGE_SIZE)
|
|
return -EINVAL;
|
|
|
|
ivpcd = container_of(vdev, struct ism_vfio_pci_core_device,
|
|
core_device);
|
|
data = kmem_cache_alloc(ivpcd->store_block_cache, GFP_KERNEL);
|
|
if (!data)
|
|
return -ENOMEM;
|
|
|
|
if (copy_from_user(data, buf, count)) {
|
|
ret = -EFAULT;
|
|
goto out_free;
|
|
}
|
|
|
|
req = ZPCI_CREATE_REQ(READ_ONCE(zdev->fh), bar, count);
|
|
ret = __zpci_store_block(data, req, off);
|
|
if (ret)
|
|
goto out_free;
|
|
|
|
ret = count;
|
|
|
|
out_free:
|
|
kmem_cache_free(ivpcd->store_block_cache, data);
|
|
return ret;
|
|
}
|
|
|
|
static ssize_t ism_vfio_pci_bar_rw(struct vfio_pci_core_device *vdev,
|
|
char __user *buf, size_t count, loff_t *ppos,
|
|
bool iswrite)
|
|
{
|
|
int bar = ISM_VFIO_PCI_OFFSET_TO_INDEX(*ppos);
|
|
loff_t pos = *ppos & ISM_VFIO_PCI_OFFSET_MASK;
|
|
resource_size_t end;
|
|
ssize_t done = 0;
|
|
|
|
if (pci_resource_start(vdev->pdev, bar))
|
|
end = pci_resource_len(vdev->pdev, bar);
|
|
else
|
|
return -EINVAL;
|
|
|
|
if (pos >= end)
|
|
return -EINVAL;
|
|
|
|
count = min(count, (size_t)(end - pos));
|
|
|
|
if (iswrite)
|
|
done = ism_vfio_pci_do_io_w(vdev, buf, pos, count, bar);
|
|
else
|
|
done = ism_vfio_pci_do_io_r(vdev, buf, pos, count, bar);
|
|
|
|
if (done >= 0)
|
|
*ppos += done;
|
|
|
|
return done;
|
|
}
|
|
|
|
static ssize_t ism_vfio_pci_config_rw(struct vfio_pci_core_device *vdev,
|
|
char __user *buf, size_t count,
|
|
loff_t *ppos, bool iswrite)
|
|
{
|
|
loff_t pos = *ppos;
|
|
size_t done = 0;
|
|
int ret = 0;
|
|
|
|
pos &= ISM_VFIO_PCI_OFFSET_MASK;
|
|
|
|
while (count) {
|
|
/*
|
|
* zPCI must not use MIO instructions for config space access,
|
|
* so we can use common code path here.
|
|
*/
|
|
ret = vfio_pci_config_rw_single(vdev, buf, count, &pos, iswrite);
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
count -= ret;
|
|
done += ret;
|
|
buf += ret;
|
|
pos += ret;
|
|
}
|
|
|
|
*ppos += done;
|
|
|
|
return done;
|
|
}
|
|
|
|
static ssize_t ism_vfio_pci_rw(struct vfio_device *core_vdev, char __user *buf,
|
|
size_t count, loff_t *ppos, bool iswrite)
|
|
{
|
|
unsigned int index = ISM_VFIO_PCI_OFFSET_TO_INDEX(*ppos);
|
|
struct vfio_pci_core_device *vdev;
|
|
int ret;
|
|
|
|
vdev = container_of(core_vdev, struct vfio_pci_core_device, vdev);
|
|
|
|
if (!count)
|
|
return 0;
|
|
|
|
switch (index) {
|
|
case VFIO_PCI_CONFIG_REGION_INDEX:
|
|
ret = ism_vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
|
|
break;
|
|
|
|
case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
|
|
ret = ism_vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
|
|
break;
|
|
|
|
default:
|
|
return -EINVAL;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static ssize_t ism_vfio_pci_read(struct vfio_device *core_vdev,
|
|
char __user *buf, size_t count, loff_t *ppos)
|
|
{
|
|
return ism_vfio_pci_rw(core_vdev, buf, count, ppos, false);
|
|
}
|
|
|
|
static ssize_t ism_vfio_pci_write(struct vfio_device *core_vdev,
|
|
const char __user *buf, size_t count,
|
|
loff_t *ppos)
|
|
{
|
|
return ism_vfio_pci_rw(core_vdev, (char __user *)buf, count, ppos,
|
|
true);
|
|
}
|
|
|
|
static int ism_vfio_pci_ioctl_get_region_info(struct vfio_device *core_vdev,
|
|
struct vfio_region_info *info,
|
|
struct vfio_info_cap *caps)
|
|
{
|
|
struct vfio_pci_core_device *vdev =
|
|
container_of(core_vdev, struct vfio_pci_core_device, vdev);
|
|
struct pci_dev *pdev = vdev->pdev;
|
|
|
|
switch (info->index) {
|
|
case VFIO_PCI_CONFIG_REGION_INDEX:
|
|
info->offset = ISM_VFIO_PCI_INDEX_TO_OFFSET(info->index);
|
|
info->size = pdev->cfg_size;
|
|
info->flags = VFIO_REGION_INFO_FLAG_READ |
|
|
VFIO_REGION_INFO_FLAG_WRITE;
|
|
break;
|
|
case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
|
|
info->offset = ISM_VFIO_PCI_INDEX_TO_OFFSET(info->index);
|
|
info->size = pci_resource_len(pdev, info->index);
|
|
if (!info->size) {
|
|
info->flags = 0;
|
|
break;
|
|
}
|
|
info->flags = VFIO_REGION_INFO_FLAG_READ |
|
|
VFIO_REGION_INFO_FLAG_WRITE;
|
|
break;
|
|
default:
|
|
info->offset = 0;
|
|
info->size = 0;
|
|
info->flags = 0;
|
|
return -EINVAL;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int ism_vfio_pci_init_dev(struct vfio_device *core_vdev)
|
|
{
|
|
struct zpci_dev *zdev = to_zpci(to_pci_dev(core_vdev->dev));
|
|
struct ism_vfio_pci_core_device *ivpcd;
|
|
char cache_name[20];
|
|
int ret;
|
|
|
|
ivpcd = container_of(core_vdev, struct ism_vfio_pci_core_device,
|
|
core_device.vdev);
|
|
|
|
snprintf(cache_name, sizeof(cache_name), "ism_sb_fid_%08x", zdev->fid);
|
|
|
|
ivpcd->store_block_cache =
|
|
kmem_cache_create(cache_name, zdev->maxstbl,
|
|
(&(struct kmem_cache_args){
|
|
.align = PAGE_SIZE,
|
|
.useroffset = 0,
|
|
.usersize = zdev->maxstbl,
|
|
}),
|
|
(SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT));
|
|
if (!ivpcd->store_block_cache)
|
|
return -ENOMEM;
|
|
|
|
ret = vfio_pci_core_init_dev(core_vdev);
|
|
if (ret)
|
|
kmem_cache_destroy(ivpcd->store_block_cache);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void ism_vfio_pci_release_dev(struct vfio_device *core_vdev)
|
|
{
|
|
struct ism_vfio_pci_core_device *ivpcd = container_of(
|
|
core_vdev, struct ism_vfio_pci_core_device, core_device.vdev);
|
|
|
|
kmem_cache_destroy(ivpcd->store_block_cache);
|
|
vfio_pci_core_release_dev(core_vdev);
|
|
}
|
|
|
|
static const struct vfio_device_ops ism_pci_ops = {
|
|
.name = "ism-vfio-pci",
|
|
.init = ism_vfio_pci_init_dev,
|
|
.release = ism_vfio_pci_release_dev,
|
|
.open_device = ism_vfio_pci_open_device,
|
|
.close_device = vfio_pci_core_close_device,
|
|
.ioctl = vfio_pci_core_ioctl,
|
|
.get_region_info_caps = ism_vfio_pci_ioctl_get_region_info,
|
|
.device_feature = vfio_pci_core_ioctl_feature,
|
|
.read = ism_vfio_pci_read,
|
|
.write = ism_vfio_pci_write,
|
|
.request = vfio_pci_core_request,
|
|
.match = vfio_pci_core_match,
|
|
.match_token_uuid = vfio_pci_core_match_token_uuid,
|
|
.bind_iommufd = vfio_iommufd_physical_bind,
|
|
.unbind_iommufd = vfio_iommufd_physical_unbind,
|
|
.attach_ioas = vfio_iommufd_physical_attach_ioas,
|
|
.detach_ioas = vfio_iommufd_physical_detach_ioas,
|
|
};
|
|
|
|
static int ism_vfio_pci_probe(struct pci_dev *pdev,
|
|
const struct pci_device_id *id)
|
|
{
|
|
struct ism_vfio_pci_core_device *ivpcd;
|
|
int ret;
|
|
|
|
ivpcd = vfio_alloc_device(ism_vfio_pci_core_device, core_device.vdev,
|
|
&pdev->dev, &ism_pci_ops);
|
|
if (IS_ERR(ivpcd))
|
|
return PTR_ERR(ivpcd);
|
|
|
|
dev_set_drvdata(&pdev->dev, &ivpcd->core_device);
|
|
|
|
ret = vfio_pci_core_register_device(&ivpcd->core_device);
|
|
if (ret)
|
|
vfio_put_device(&ivpcd->core_device.vdev);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void ism_vfio_pci_remove(struct pci_dev *pdev)
|
|
{
|
|
struct vfio_pci_core_device *core_device;
|
|
struct ism_vfio_pci_core_device *ivpcd;
|
|
|
|
core_device = dev_get_drvdata(&pdev->dev);
|
|
ivpcd = container_of(core_device, struct ism_vfio_pci_core_device,
|
|
core_device);
|
|
|
|
vfio_pci_core_unregister_device(&ivpcd->core_device);
|
|
vfio_put_device(&ivpcd->core_device.vdev);
|
|
}
|
|
|
|
static const struct pci_device_id ism_device_table[] = {
|
|
{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_IBM,
|
|
PCI_DEVICE_ID_IBM_ISM) },
|
|
{}
|
|
};
|
|
MODULE_DEVICE_TABLE(pci, ism_device_table);
|
|
|
|
static struct pci_driver ism_vfio_pci_driver = {
|
|
.name = KBUILD_MODNAME,
|
|
.id_table = ism_device_table,
|
|
.probe = ism_vfio_pci_probe,
|
|
.remove = ism_vfio_pci_remove,
|
|
.err_handler = &vfio_pci_core_err_handlers,
|
|
.driver_managed_dma = true,
|
|
};
|
|
|
|
module_pci_driver(ism_vfio_pci_driver);
|
|
|
|
MODULE_LICENSE("GPL");
|
|
MODULE_DESCRIPTION("vfio-pci variant driver for the IBM Internal Shared Memory (ISM) device");
|
|
MODULE_AUTHOR("IBM Corporation");
|