linux-kernel/linux-7.2.3/drivers/media/pci/hws/hws_pci.c

// SPDX-License-Identifier: GPL-2.0-only
#include <linux/pci.h>
#include <linux/types.h>
#include <linux/iopoll.h>
#include <linux/bitfield.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/kthread.h>
#include <linux/interrupt.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/ktime.h>
#include <linux/math64.h>
#include <linux/pm.h>
#include <linux/freezer.h>
#include <linux/pci_regs.h>

#include <media/v4l2-ctrls.h>

#include "hws.h"
#include "hws_reg.h"
#include "hws_video.h"
#include "hws_irq.h"
#include "hws_v4l2_ioctl.h"

#define DRV_NAME "hws"
#define HWS_BUSY_POLL_DELAY_US 10
#define HWS_BUSY_POLL_TIMEOUT_US 1000000

static unsigned long long hws_elapsed_us(u64 start_ns)
{
	return div_u64(ktime_get_mono_fast_ns() - start_ns, 1000);
}

/* register layout inside HWS_REG_DEVICE_INFO */
#define DEVINFO_VER GENMASK(7, 0)
#define DEVINFO_SUBVER GENMASK(15, 8)
#define DEVINFO_YV12 GENMASK(31, 28)
#define DEVINFO_HWKEY GENMASK(27, 24)
#define DEVINFO_PORTID GENMASK(25, 24) /* low 2 bits of HW-key */

#define MAKE_ENTRY(__vend, __chip, __subven, __subdev, __configptr) \
	{ .vendor = (__vend),                                       \
	  .device = (__chip),                                       \
	  .subvendor = (__subven),                                  \
	  .subdevice = (__subdev),                                  \
	  .driver_data = (unsigned long)(__configptr) }

/*
 * PCI IDs for HWS family cards.
 *
 * The subsystem IDs are fixed at 0x8888:0x0007 for this family. Some boards
 * enumerate with vendor ID 0x8888 or 0x1f33. Exact SKU names are not fully
 * pinned down yet; update these comments when vendor documentation or INF
 * strings are available.
 */
static const struct pci_device_id hws_pci_table[] = {
	/* HWS family, SKU unknown. */
	MAKE_ENTRY(0x8888, 0x9534, 0x8888, 0x0007, NULL),
	MAKE_ENTRY(0x1F33, 0x8534, 0x8888, 0x0007, NULL),
	MAKE_ENTRY(0x1F33, 0x8554, 0x8888, 0x0007, NULL),

	/* HWS 2x2 HDMI family. */
	MAKE_ENTRY(0x8888, 0x8524, 0x8888, 0x0007, NULL),
	/* HWS 2x2 SDI family. */
	MAKE_ENTRY(0x1F33, 0x6524, 0x8888, 0x0007, NULL),

	/* HWS X4 HDMI family. */
	MAKE_ENTRY(0x8888, 0x8504, 0x8888, 0x0007, NULL),
	/* HWS X4 SDI family. */
	MAKE_ENTRY(0x8888, 0x6504, 0x8888, 0x0007, NULL),

	/* HWS family, SKU unknown. */
	MAKE_ENTRY(0x8888, 0x8532, 0x8888, 0x0007, NULL),
	MAKE_ENTRY(0x8888, 0x8512, 0x8888, 0x0007, NULL),
	MAKE_ENTRY(0x8888, 0x8501, 0x8888, 0x0007, NULL),
	MAKE_ENTRY(0x1F33, 0x6502, 0x8888, 0x0007, NULL),

	/* HWS X4 HDMI family (alternate vendor ID). */
	MAKE_ENTRY(0x1F33, 0x8504, 0x8888, 0x0007, NULL),
	/* HWS 2x2 HDMI family (alternate vendor ID). */
	MAKE_ENTRY(0x1F33, 0x8524, 0x8888, 0x0007, NULL),

	{}
};

static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
{
	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
}

static void hws_configure_hardware_capabilities(struct hws_pcie_dev *hdev)
{
	u16 id = hdev->device_id;

	/* select per-chip channel counts */
	switch (id) {
	case 0x9534:
	case 0x6524:
	case 0x8524:
	case 0x8504:
	case 0x6504:
		hdev->cur_max_video_ch = 4;
		break;
	case 0x8532:
		hdev->cur_max_video_ch = 2;
		break;
	case 0x8512:
	case 0x6502:
		hdev->cur_max_video_ch = 2;
		break;
	case 0x8501:
		hdev->cur_max_video_ch = 1;
		break;
	default:
		hdev->cur_max_video_ch = 4;
		break;
	}

	/* universal buffer capacity */
	hdev->max_hw_video_buf_sz = MAX_MM_VIDEO_SIZE;

	/* decide hardware-version and program DMA max size if needed */
	if (hdev->device_ver > 121) {
		if (id == 0x8501 && hdev->device_ver == 122) {
			hdev->hw_ver = 0;
		} else {
			hdev->hw_ver = 1;
			u32 dma_max = (u32)(MAX_VIDEO_SCALER_SIZE / 16);

			writel(dma_max, hdev->bar0_base + HWS_REG_DMA_MAX_SIZE);
			/* readback to flush posted MMIO write */
			(void)readl(hdev->bar0_base + HWS_REG_DMA_MAX_SIZE);
		}
	} else {
		hdev->hw_ver = 0;
	}
}

static void hws_stop_device(struct hws_pcie_dev *hws);

static void hws_log_lifecycle_snapshot(struct hws_pcie_dev *hws,
				       const char *action,
				       const char *phase)
{
	struct device *dev;
	u32 int_en, int_status, vcap, sys_status, dec_mode;

	if (!hws || !hws->pdev)
		return;

	dev = &hws->pdev->dev;
	if (!hws->bar0_base) {
		dev_dbg(dev,
			"lifecycle:%s:%s bar0-unmapped suspended=%d start_run=%d pci_lost=%d irq=%d\n",
			action, phase, READ_ONCE(hws->suspended), hws->start_run,
			hws->pci_lost, hws->irq);
		return;
	}

	int_en = readl(hws->bar0_base + INT_EN_REG_BASE);
	int_status = readl(hws->bar0_base + HWS_REG_INT_STATUS);
	vcap = readl(hws->bar0_base + HWS_REG_VCAP_ENABLE);
	sys_status = readl(hws->bar0_base + HWS_REG_SYS_STATUS);
	dec_mode = readl(hws->bar0_base + HWS_REG_DEC_MODE);

	dev_dbg(dev,
		"lifecycle:%s:%s suspended=%d start_run=%d pci_lost=%d irq=%d INT_EN=0x%08x INT_STATUS=0x%08x VCAP=0x%08x SYS=0x%08x DEC=0x%08x\n",
		action, phase, READ_ONCE(hws->suspended), hws->start_run,
		hws->pci_lost, hws->irq, int_en, int_status, vcap,
		sys_status, dec_mode);
}

static int read_chip_id(struct hws_pcie_dev *hdev)
{
	u32 reg;
	/* mirror PCI IDs for later switches */
	hdev->device_id = hdev->pdev->device;
	hdev->vendor_id = hdev->pdev->vendor;

	reg = readl(hdev->bar0_base + HWS_REG_DEVICE_INFO);

	hdev->device_ver = FIELD_GET(DEVINFO_VER, reg);
	hdev->sub_ver = FIELD_GET(DEVINFO_SUBVER, reg);
	hdev->support_yv12 = FIELD_GET(DEVINFO_YV12, reg);
	hdev->port_id = FIELD_GET(DEVINFO_PORTID, reg);

	hdev->max_hw_video_buf_sz = MAX_MM_VIDEO_SIZE;
	hdev->max_channels = 4;
	hdev->buf_allocated = false;
	hdev->main_task = NULL;
	hdev->start_run = false;
	hdev->pci_lost = 0;

	writel(0x00, hdev->bar0_base + HWS_REG_DEC_MODE);
	writel(0x10, hdev->bar0_base + HWS_REG_DEC_MODE);

	hws_configure_hardware_capabilities(hdev);

	dev_info(&hdev->pdev->dev,
		 "chip detected: ver=%u subver=%u port=%u yv12=%u\n",
		 hdev->device_ver, hdev->sub_ver, hdev->port_id,
		 hdev->support_yv12);

	return 0;
}

static int main_ks_thread_handle(void *data)
{
	struct hws_pcie_dev *pdx = data;

	set_freezable();

	while (!kthread_should_stop()) {
		/* If we're suspending, don't touch hardware; just sleep/freeze. */
		if (READ_ONCE(pdx->suspended)) {
			try_to_freeze();
			schedule_timeout_interruptible(msecs_to_jiffies(1000));
			continue;
		}

		/* avoid MMIO when suspended (guarded above) */
		check_video_format(pdx);

		try_to_freeze(); /* cooperate with freezer each loop */

		/* Sleep 1s or until signaled to wake/stop */
		schedule_timeout_interruptible(msecs_to_jiffies(1000));
	}

	dev_dbg(&pdx->pdev->dev, "%s: exiting\n", __func__);
	return 0;
}

static void hws_stop_kthread_action(void *data)
{
	struct hws_pcie_dev *hws = data;
	struct task_struct *t;
	u64 start_ns;

	if (!hws)
		return;

	t = READ_ONCE(hws->main_task);
	if (!IS_ERR_OR_NULL(t)) {
		start_ns = ktime_get_mono_fast_ns();
		dev_dbg(&hws->pdev->dev,
			"lifecycle:kthread-stop:begin task=%s[%d]\n",
			t->comm, t->pid);
		WRITE_ONCE(hws->main_task, NULL);
		kthread_stop(t);
		dev_dbg(&hws->pdev->dev,
			"lifecycle:kthread-stop:done (%lluus)\n",
			hws_elapsed_us(start_ns));
	}
}

static int hws_alloc_seed_buffers(struct hws_pcie_dev *hws)
{
	int ch;
	/* 64 KiB is plenty for a safe dummy; hardware needs 64-byte alignment. */
	const size_t need = ALIGN(64 * 1024, 64);

	for (ch = 0; ch < hws->cur_max_video_ch; ch++) {
#if defined(CONFIG_HAS_DMA) /* normal on PCIe platforms */
		void *cpu = dma_alloc_coherent(&hws->pdev->dev, need,
					       &hws->scratch_vid[ch].dma,
					       GFP_KERNEL);
#else
		void *cpu = NULL;
#endif
		if (!cpu) {
			dev_warn(&hws->pdev->dev,
				 "scratch: dma_alloc_coherent failed ch=%d\n", ch);
			/* not fatal: free earlier ones and continue without seeding */
			while (--ch >= 0) {
				if (hws->scratch_vid[ch].cpu)
					dma_free_coherent(&hws->pdev->dev,
							  hws->scratch_vid[ch].size,
							  hws->scratch_vid[ch].cpu,
							  hws->scratch_vid[ch].dma);
				hws->scratch_vid[ch].cpu = NULL;
				hws->scratch_vid[ch].size = 0;
			}
			return -ENOMEM;
		}
		hws->scratch_vid[ch].cpu  = cpu;
		hws->scratch_vid[ch].size = need;
	}
	return 0;
}

static void hws_free_seed_buffers(struct hws_pcie_dev *hws)
{
	int ch;

	for (ch = 0; ch < hws->cur_max_video_ch; ch++) {
		if (hws->scratch_vid[ch].cpu) {
			dma_free_coherent(&hws->pdev->dev,
					  hws->scratch_vid[ch].size,
					  hws->scratch_vid[ch].cpu,
					  hws->scratch_vid[ch].dma);
			hws->scratch_vid[ch].cpu = NULL;
			hws->scratch_vid[ch].size = 0;
		}
	}
}

static void hws_seed_channel(struct hws_pcie_dev *hws, int ch)
{
	dma_addr_t paddr = hws->scratch_vid[ch].dma;
	u32 lo = lower_32_bits(paddr);
	u32 hi = upper_32_bits(paddr);
	u32 pci_addr = lo & PCI_E_BAR_ADD_LOWMASK;

	lo &= PCI_E_BAR_ADD_MASK;

	/* Program 64-bit BAR remap entry for this channel (table @ 0x208 + ch * 8) */
	writel_relaxed(hi, hws->bar0_base +
			    PCI_ADDR_TABLE_BASE + 0x208 + ch * 8);
	writel_relaxed(lo, hws->bar0_base +
			    PCI_ADDR_TABLE_BASE + 0x208 + ch * 8 +
			    PCIE_BARADDROFSIZE);

	/* Program capture engine per-channel base/half */
	writel_relaxed((ch + 1) * PCIEBAR_AXI_BASE + pci_addr,
		       hws->bar0_base + CVBS_IN_BUF_BASE +
		       ch * PCIE_BARADDROFSIZE);

	/* Half size: use either the current format's half or half of scratch. */
	{
		u32 half = hws->video[ch].pix.half_size ?
			hws->video[ch].pix.half_size :
			(u32)(hws->scratch_vid[ch].size / 2);

		writel_relaxed(half / 16,
			       hws->bar0_base + CVBS_IN_BUF_BASE2 +
			       ch * PCIE_BARADDROFSIZE);
	}

	(void)readl(hws->bar0_base + HWS_REG_INT_STATUS); /* flush posted writes */
}

static void hws_seed_all_channels(struct hws_pcie_dev *hws)
{
	int ch;

	for (ch = 0; ch < hws->cur_max_video_ch; ch++) {
		if (hws->scratch_vid[ch].cpu)
			hws_seed_channel(hws, ch);
	}
}

static void hws_irq_mask_gate(struct hws_pcie_dev *hws)
{
	writel(0x00000000, hws->bar0_base + INT_EN_REG_BASE);
	(void)readl(hws->bar0_base + INT_EN_REG_BASE);
}

static void hws_irq_unmask_gate(struct hws_pcie_dev *hws)
{
	writel(HWS_INT_EN_MASK, hws->bar0_base + INT_EN_REG_BASE);
	(void)readl(hws->bar0_base + INT_EN_REG_BASE);
}

static void hws_irq_clear_pending(struct hws_pcie_dev *hws)
{
	u32 st = readl(hws->bar0_base + HWS_REG_INT_STATUS);

	if (st) {
		writel(st, hws->bar0_base + HWS_REG_INT_STATUS); /* W1C */
		(void)readl(hws->bar0_base + HWS_REG_INT_STATUS);
	}
}

static void hws_block_hotpaths(struct hws_pcie_dev *hws)
{
	WRITE_ONCE(hws->suspended, true);
	if (hws->irq >= 0)
		disable_irq(hws->irq);

	if (!hws->bar0_base)
		return;

	hws_irq_mask_gate(hws);
	hws_irq_clear_pending(hws);
}

static int hws_probe(struct pci_dev *pdev, const struct pci_device_id *pci_id)
{
	struct hws_pcie_dev *hws;
	int i, ret, irq;
	unsigned long irqf = 0;
	bool v4l2_registered = false;

	/* devres-backed device object */
	hws = devm_kzalloc(&pdev->dev, sizeof(*hws), GFP_KERNEL);
	if (!hws)
		return -ENOMEM;

	hws->pdev = pdev;
	hws->irq = -1;
	hws->suspended = false;
	pci_set_drvdata(pdev, hws);

	/* 1) Enable device + bus mastering (managed) */
	ret = pcim_enable_device(pdev);
	if (ret)
		return dev_err_probe(&pdev->dev, ret, "pcim_enable_device\n");
	pci_set_master(pdev);

	/* 2) Map BAR0 (managed) */
	ret = pcim_iomap_regions(pdev, BIT(0), KBUILD_MODNAME);
	if (ret)
		return dev_err_probe(&pdev->dev, ret, "pcim_iomap_regions BAR0\n");
	hws->bar0_base = pcim_iomap_table(pdev)[0];

	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
	if (ret) {
		dev_warn(&pdev->dev,
			 "64-bit DMA mask unavailable, falling back to 32-bit (%d)\n",
			 ret);
		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
		if (ret)
			return dev_err_probe(&pdev->dev, ret,
					     "No suitable DMA configuration\n");
	} else {
		dev_dbg(&pdev->dev, "Using 64-bit DMA mask\n");
	}

	/* 3) Apply optional PCIe tuning. */
	enable_pcie_relaxed_ordering(pdev);
#ifdef CONFIG_ARCH_TI816X
	pcie_set_readrq(pdev, 128);
#endif

	/* 4) Identify chip & capabilities */
	read_chip_id(hws);
	dev_info(&pdev->dev, "Device VID=0x%04x DID=0x%04x\n",
		 pdev->vendor, pdev->device);
	hws_init_video_sys(hws, false);

	/* 5) Init channels (video state, locks, vb2, ctrls) */
	for (i = 0; i < hws->max_channels; i++) {
		ret = hws_video_init_channel(hws, i);
		if (ret) {
			dev_err(&pdev->dev, "video channel init failed (ch=%d)\n", i);
			goto err_unwind_channels;
		}
	}

	/* 6) Allocate scratch DMA and seed BAR table + channel base/half (legacy SetDMAAddress) */
	ret = hws_alloc_seed_buffers(hws);
	if (!ret)
		hws_seed_all_channels(hws);

	/* 7) Start-run sequence. */
	hws_init_video_sys(hws, false);

	/* A) Force legacy INTx; legacy used request_irq(pdev->irq, ..., IRQF_SHARED) */
	pci_intx(pdev, 1);
	irqf = IRQF_SHARED;
	irq = pdev->irq;
	hws->irq = irq;
	dev_info(&pdev->dev, "IRQ mode: legacy INTx (shared), irq=%d\n", irq);

	/* B) Mask the device's global/bridge gate (INT_EN_REG_BASE) */
	hws_irq_mask_gate(hws);

	/* C) Clear any sticky pending interrupt status (W1C) before we arm the line */
	hws_irq_clear_pending(hws);

	/* D) Request the legacy shared interrupt line (no vectors/MSI/MSI-X) */
	ret = devm_request_irq(&pdev->dev, irq, hws_irq_handler, irqf,
			       dev_name(&pdev->dev), hws);
	if (ret) {
		dev_err(&pdev->dev, "request_irq(%d) failed: %d\n", irq, ret);
		goto err_unwind_channels;
	}

	/* E) Set the global interrupt enable bit in main control register */
	{
		u32 ctl_reg = readl(hws->bar0_base + HWS_REG_CTL);

		ctl_reg |= HWS_CTL_IRQ_ENABLE_BIT;
		writel(ctl_reg, hws->bar0_base + HWS_REG_CTL);
		(void)readl(hws->bar0_base + HWS_REG_CTL); /* flush write */
		dev_info(&pdev->dev, "Global IRQ enable bit set in control register\n");
	}

	/* F) Open the global gate just like legacy did */
	hws_irq_unmask_gate(hws);
	dev_info(&pdev->dev, "INT_EN_GATE readback=0x%08x\n",
		 readl(hws->bar0_base + INT_EN_REG_BASE));

	/* 11) Register V4L2 */
	ret = hws_video_register(hws);
	if (ret) {
		dev_err(&pdev->dev, "video_register: %d\n", ret);
		goto err_unwind_channels;
	}
	v4l2_registered = true;

	/* 12) Background monitor thread (managed) */
	hws->main_task = kthread_run(main_ks_thread_handle, hws, "hws-mon");
	if (IS_ERR(hws->main_task)) {
		ret = PTR_ERR(hws->main_task);
		hws->main_task = NULL;
		dev_err(&pdev->dev, "kthread_run: %d\n", ret);
		goto err_unregister_va;
	}
	ret = devm_add_action_or_reset(&pdev->dev, hws_stop_kthread_action, hws);
	if (ret) {
		dev_err(&pdev->dev, "devm_add_action kthread_stop: %d\n", ret);
		goto err_unregister_va; /* reset already stopped the thread */
	}

	/* 13) Final: show the line is armed */
	dev_info(&pdev->dev, "irq handler installed on irq=%d\n", irq);
	return 0;

err_unregister_va:
	hws_stop_device(hws);
	hws_video_unregister(hws);
	hws_free_seed_buffers(hws);
	return ret;
err_unwind_channels:
	hws_free_seed_buffers(hws);
	if (!v4l2_registered) {
		while (--i >= 0)
			hws_video_cleanup_channel(hws, i);
	}
	return ret;
}

static int hws_check_busy(struct hws_pcie_dev *pdx)
{
	void __iomem *reg = pdx->bar0_base + HWS_REG_SYS_STATUS;
	u32 val;
	int ret;

	/* poll until !(val & BUSY_BIT), sleeping HWS_BUSY_POLL_DELAY_US between reads */
	ret = readl_poll_timeout(reg, val, !(val & HWS_SYS_DMA_BUSY_BIT),
				 HWS_BUSY_POLL_DELAY_US,
				 HWS_BUSY_POLL_TIMEOUT_US);
	if (ret) {
		dev_err(&pdx->pdev->dev,
			"SYS_STATUS busy bit never cleared (0x%08x)\n", val);
		return -ETIMEDOUT;
	}

	return 0;
}

static void hws_stop_dsp(struct hws_pcie_dev *hws)
{
	u32 status;

	/* Read the decoder mode/status register */
	status = readl(hws->bar0_base + HWS_REG_DEC_MODE);
	dev_dbg(&hws->pdev->dev, "%s: status=0x%08x\n", __func__, status);

	/* If the device looks unplugged/stuck, bail out */
	if (status == 0xFFFFFFFF)
		return;

	/* Tell the DSP to stop */
	writel(0x10, hws->bar0_base + HWS_REG_DEC_MODE);

	if (hws_check_busy(hws))
		dev_warn(&hws->pdev->dev, "DSP busy timeout on stop\n");
	/* Disable video capture engine in the DSP */
	writel(0x0, hws->bar0_base + HWS_REG_VCAP_ENABLE);
}

/* Publish stop so ISR/BH will not touch video buffers anymore. */
static void hws_publish_stop_flags(struct hws_pcie_dev *hws)
{
	unsigned int i;

	for (i = 0; i < hws->cur_max_video_ch; ++i) {
		struct hws_video *v = &hws->video[i];

		WRITE_ONCE(v->cap_active,     false);
		WRITE_ONCE(v->stop_requested, true);
	}

	smp_wmb(); /* make flags visible before we touch MMIO/queues */
}

/* Drain engines + ISR/BH after flags are published. */
static void hws_drain_after_stop(struct hws_pcie_dev *hws)
{
	u32 ackmask = 0;
	unsigned int i;
	u64 start_ns = ktime_get_mono_fast_ns();

	/* Mask device enables: no new DMA starts. */
	writel(0x0, hws->bar0_base + HWS_REG_VCAP_ENABLE);
	(void)readl(hws->bar0_base + HWS_REG_INT_STATUS); /* flush */

	/* Let any in-flight DMAs finish (best-effort). */
	(void)hws_check_busy(hws);

	/* Ack any latched VDONE. */
	for (i = 0; i < hws->cur_max_video_ch; ++i)
		ackmask |= HWS_INT_VDONE_BIT(i);
	if (ackmask) {
		writel(ackmask, hws->bar0_base + HWS_REG_INT_STATUS);
		(void)readl(hws->bar0_base + HWS_REG_INT_STATUS);
	}

	/* Ensure no hard IRQ is still running. */
	if (hws->irq >= 0)
		synchronize_irq(hws->irq);

	dev_dbg(&hws->pdev->dev, "lifecycle:drain-after-stop:done (%lluus)\n",
		hws_elapsed_us(start_ns));
}

static void hws_stop_device(struct hws_pcie_dev *hws)
{
	u32 status = readl(hws->bar0_base + HWS_REG_SYS_STATUS);
	u64 start_ns = ktime_get_mono_fast_ns();
	bool live = status != 0xFFFFFFFF;

	dev_dbg(&hws->pdev->dev, "%s: status=0x%08x\n", __func__, status);
	if (!live) {
		hws->pci_lost = true;
		goto out;
	}
	hws_log_lifecycle_snapshot(hws, "stop-device", "begin");

	/* Make ISR/BH a no-op, then drain engines/IRQ. */
	hws_publish_stop_flags(hws);
	hws_drain_after_stop(hws);

	/* 1) Stop the on-board DSP */
	hws_stop_dsp(hws);

out:
	hws->start_run = false;
	if (live)
		hws_log_lifecycle_snapshot(hws, "stop-device", "end");
	else
		dev_dbg(&hws->pdev->dev, "lifecycle:stop-device:device-lost\n");
	dev_dbg(&hws->pdev->dev, "lifecycle:stop-device:done (%lluus)\n",
		hws_elapsed_us(start_ns));
	dev_dbg(&hws->pdev->dev, "%s: complete\n", __func__);
}

static int hws_quiesce_for_transition(struct hws_pcie_dev *hws,
				      const char *action,
				      bool stop_thread)
{
	struct device *dev = &hws->pdev->dev;
	u64 start_ns = ktime_get_mono_fast_ns();
	u64 step_ns;
	int vret;

	hws_log_lifecycle_snapshot(hws, action, "begin");

	step_ns = ktime_get_mono_fast_ns();
	hws_block_hotpaths(hws);
	dev_dbg(dev, "lifecycle:%s:block-hotpaths (%lluus)\n", action,
		hws_elapsed_us(step_ns));
	hws_log_lifecycle_snapshot(hws, action, "blocked");

	if (stop_thread) {
		step_ns = ktime_get_mono_fast_ns();
		hws_stop_kthread_action(hws);
		dev_dbg(dev, "lifecycle:%s:stop-kthread (%lluus)\n", action,
			hws_elapsed_us(step_ns));
	}

	step_ns = ktime_get_mono_fast_ns();
	vret = hws_video_quiesce(hws, action);
	dev_dbg(dev, "lifecycle:%s:video-quiesce ret=%d (%lluus)\n", action,
		vret, hws_elapsed_us(step_ns));
	if (vret)
		dev_warn(dev, "lifecycle:%s video quiesce returned %d\n",
			 action, vret);

	step_ns = ktime_get_mono_fast_ns();
	hws_stop_device(hws);
	dev_dbg(dev, "lifecycle:%s:stop-device (%lluus)\n", action,
		hws_elapsed_us(step_ns));
	hws_log_lifecycle_snapshot(hws, action, "end");
	dev_dbg(dev, "lifecycle:%s:quiesce-done ret=%d (%lluus)\n", action,
		vret, hws_elapsed_us(start_ns));

	return vret;
}

static void hws_remove(struct pci_dev *pdev)
{
	struct hws_pcie_dev *hws = pci_get_drvdata(pdev);
	u64 start_ns;

	if (!hws)
		return;

	start_ns = ktime_get_mono_fast_ns();
	dev_info(&pdev->dev, "lifecycle:remove begin\n");
	hws_log_lifecycle_snapshot(hws, "remove", "begin");

	/* Stop the monitor thread before tearing down V4L2/vb2 objects. */
	hws_block_hotpaths(hws);
	hws_stop_kthread_action(hws);

	/* Stop hardware and capture cleanly. */
	hws_stop_device(hws);

	/* Unregister V4L2 resources. */
	hws_video_unregister(hws);

	/* Release seeded DMA buffers */
	hws_free_seed_buffers(hws);
	/* kthread is stopped by the devm action registered in probe. */
	hws_log_lifecycle_snapshot(hws, "remove", "end");
	dev_info(&pdev->dev, "lifecycle:remove done (%lluus)\n",
		 hws_elapsed_us(start_ns));
}

#ifdef CONFIG_PM_SLEEP
static int hws_pm_suspend(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct hws_pcie_dev *hws = pci_get_drvdata(pdev);
	int vret;
	u64 start_ns = ktime_get_mono_fast_ns();
	u64 step_ns;

	dev_info(dev, "lifecycle:pm_suspend begin\n");
	vret = hws_quiesce_for_transition(hws, "pm_suspend", false);

	step_ns = ktime_get_mono_fast_ns();
	pci_save_state(pdev);
	pci_clear_master(pdev);
	pci_disable_device(pdev);
	pci_set_power_state(pdev, PCI_D3hot);
	dev_dbg(dev, "lifecycle:pm_suspend:pci-d3hot (%lluus)\n",
		hws_elapsed_us(step_ns));
	dev_info(dev, "lifecycle:pm_suspend done ret=%d (%lluus)\n", vret,
		 hws_elapsed_us(start_ns));

	return 0;
}

static int hws_pm_resume(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct hws_pcie_dev *hws = pci_get_drvdata(pdev);
	int ret;
	u64 start_ns = ktime_get_mono_fast_ns();
	u64 step_ns;

	dev_info(dev, "lifecycle:pm_resume begin\n");

	/* Back to D0 and re-enable the function */
	step_ns = ktime_get_mono_fast_ns();
	pci_set_power_state(pdev, PCI_D0);

	ret = pci_enable_device(pdev);
	if (ret) {
		dev_err(dev, "pci_enable_device: %d\n", ret);
		return ret;
	}
	pci_restore_state(pdev);
	pci_set_master(pdev);
	dev_dbg(dev, "lifecycle:pm_resume:pci-enable (%lluus)\n",
		hws_elapsed_us(step_ns));

	/* Reapply any PCIe tuning lost across D3 */
	enable_pcie_relaxed_ordering(pdev);

	/* Reinitialize chip-side capabilities / registers */
	step_ns = ktime_get_mono_fast_ns();
	read_chip_id(hws);
	/* Re-seed BAR remaps/DMA windows and restart the capture core */
	hws_seed_all_channels(hws);
	hws_init_video_sys(hws, true);
	hws_irq_clear_pending(hws);
	dev_dbg(dev, "lifecycle:pm_resume:chip-reinit (%lluus)\n",
		hws_elapsed_us(step_ns));

	/* IRQs can be re-enabled now that MMIO is sane */
	step_ns = ktime_get_mono_fast_ns();
	if (hws->irq >= 0)
		enable_irq(hws->irq);

	WRITE_ONCE(hws->suspended, false);
	dev_dbg(dev, "lifecycle:pm_resume:irq-unsuspend (%lluus)\n",
		hws_elapsed_us(step_ns));

	/* vb2: nothing mandatory; userspace will STREAMON again when ready */
	step_ns = ktime_get_mono_fast_ns();
	hws_video_pm_resume(hws);
	dev_dbg(dev, "lifecycle:pm_resume:video-resume (%lluus)\n",
		hws_elapsed_us(step_ns));
	hws_log_lifecycle_snapshot(hws, "pm_resume", "end");
	dev_info(dev, "lifecycle:pm_resume done (%lluus)\n",
		 hws_elapsed_us(start_ns));

	return 0;
}

static SIMPLE_DEV_PM_OPS(hws_pm_ops, hws_pm_suspend, hws_pm_resume);
# define HWS_PM_OPS (&hws_pm_ops)
#else
# define HWS_PM_OPS NULL
#endif

static void hws_shutdown(struct pci_dev *pdev)
{
	struct hws_pcie_dev *hws = pci_get_drvdata(pdev);
	int vret = 0;
	u64 start_ns = ktime_get_mono_fast_ns();
	u64 step_ns;

	if (!hws)
		return;

	dev_info(&pdev->dev, "lifecycle:pci_shutdown begin\n");
	vret = hws_quiesce_for_transition(hws, "pci_shutdown", true);

	step_ns = ktime_get_mono_fast_ns();
	pci_clear_master(pdev);
	dev_dbg(&pdev->dev, "lifecycle:pci_shutdown:clear-master (%lluus)\n",
		hws_elapsed_us(step_ns));
	dev_info(&pdev->dev, "lifecycle:pci_shutdown done ret=%d (%lluus)\n",
		 vret, hws_elapsed_us(start_ns));
}

static struct pci_driver hws_pci_driver = {
	.name = KBUILD_MODNAME,
	.id_table = hws_pci_table,
	.probe = hws_probe,
	.remove = hws_remove,
	.shutdown = hws_shutdown,
	.driver = {
		.pm = HWS_PM_OPS,
	},
};

MODULE_DEVICE_TABLE(pci, hws_pci_table);

static int __init pcie_hws_init(void)
{
	return pci_register_driver(&hws_pci_driver);
}

static void __exit pcie_hws_exit(void)
{
	pci_unregister_driver(&hws_pci_driver);
}

module_init(pcie_hws_init);
module_exit(pcie_hws_exit);

MODULE_DESCRIPTION(DRV_NAME);
MODULE_AUTHOR("Ben Hoff <hoff.benjamin.k@gmail.com>");
MODULE_AUTHOR("Sales <sales@avmatrix.com>");
MODULE_LICENSE("GPL");
MODULE_IMPORT_NS("DMA_BUF");