[PATCH 2/2] iommu/amd: add vIOMMU HW queue initialization

[Suravee Suthikulpanit]

Plumb HW queue flags through IOMMU_HW_QUEUE_ALLOC and add AMD HW
queue types and flags to the UAPI. Program VFCTRL guest command,
event, and PPR queue base addresses and lengths during hw_queue_init.

Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@xxxxxxx>
---
 drivers/iommu/amd/amd_iommu_types.h |   8 ++
 drivers/iommu/amd/iommufd.c         | 129 ++++++++++++++++++++++++++++
 drivers/iommu/iommufd/viommu.c      |   3 +-
 include/linux/iommufd.h             |   1 +
 include/uapi/linux/iommufd.h        |  31 ++++++-
 5 files changed, 170 insertions(+), 2 deletions(-)

diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index 1090d7796ce5..ffd28f2a72b2 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -507,6 +507,10 @@ extern bool amdr_ivrs_remap_support;
 #define VIOMMU_VFCTRL_MMIO_BASE(iommu, guestId) \
 	(iommu->vfctrl_base + (guestId * VIOMMU_VFCTRL_MMIO_ENTRY_SIZE))
 
+#define VIOMMU_VFCTRL_MMIO_GUEST_COMMAND_CONTROL_OFFSET	0x20
+#define VIOMMU_VFCTRL_MMIO_GUEST_EVENT_CONTROL_OFFSET	0x28
+#define VIOMMU_VFCTRL_MMIO_GUEST_PPR_CONTROL_OFFSET	0x30
+
 struct amd_iommu;
 struct iommu_domain;
 struct irq_domain;
@@ -1153,6 +1157,10 @@ struct amd_iommu_vdevice {
 	struct iommufd_vdevice core;
 };
 
+struct amd_iommu_hw_queue {
+	struct iommufd_hw_queue core;
+};
+
 #ifdef CONFIG_IRQ_REMAP
 extern struct amd_irte_ops irte_32_ops;
 extern struct amd_irte_ops irte_128_ops;
diff --git a/drivers/iommu/amd/iommufd.c b/drivers/iommu/amd/iommufd.c
index 0d80eaeee662..38b2fc4d79da 100644
--- a/drivers/iommu/amd/iommufd.c
+++ b/drivers/iommu/amd/iommufd.c
@@ -4,6 +4,7 @@
  */
 
 #include <linux/iommu.h>
+#include <linux/amd-iommu.h>
 
 #include "iommufd.h"
 #include "amd_iommu.h"
@@ -185,6 +186,132 @@ static int _amd_viommu_vdevice_init(struct iommufd_vdevice *vdev)
 	return 0;
 }
 
+static size_t _amd_viommu_get_hw_queue_size(struct iommufd_viommu *viommu,
+					    enum iommu_hw_queue_type queue_type)
+{
+	/* Currently do not support Eventlog B and PPRlog B */
+	if ((queue_type != IOMMU_HW_QUEUE_TYPE_AMD_CMD) &&
+	    (queue_type != IOMMU_HW_QUEUE_TYPE_AMD_EVT) &&
+	    (queue_type != IOMMU_HW_QUEUE_TYPE_AMD_PPR))
+		return 0;
+
+	return HW_QUEUE_STRUCT_SIZE(struct amd_iommu_hw_queue, core);
+}
+
+static void _amd_viommu_set_cmdbuf_flags(struct iommufd_hw_queue *hw_queue)
+{
+	u8 __iomem *vfctrl, *vf;
+	u32 flags = hw_queue->flags;
+	u64 val;
+	struct iommufd_viommu *viommu = hw_queue->viommu;
+	struct amd_iommu_viommu *aviommu = container_of(viommu, struct amd_iommu_viommu, core);
+	struct amd_iommu *iommu = container_of(viommu->iommu_dev, struct amd_iommu, iommu);
+	int gid = aviommu->gid;
+
+	vf = VIOMMU_VF_MMIO_BASE(iommu, gid);
+	vfctrl = VIOMMU_VFCTRL_MMIO_BASE(iommu, gid);
+
+	/* Clear fields in VFCTRL MMIO */
+	val = readq(vfctrl + VIOMMU_VFCTRL_MMIO_GUEST_COMMAND_CONTROL_OFFSET);
+	val &= ~(GENMASK_ULL(51, 12) | GENMASK_ULL(9, 8) | GENMASK_ULL(3, 0));
+
+	/* Set Command buffer base, length, enable, command wait enable */
+	val |= FIELD_PREP(GENMASK_ULL(3, 0), hw_queue->length);
+	val |= FIELD_PREP(GENMASK_ULL(51, 12), (hw_queue->base_addr >> 12));
+	val |= FIELD_PREP(BIT_ULL_MASK(8), !!(flags & IOMMU_HW_QUEUE_FLAG_AMD_CMDBUF_EN));
+	val |= FIELD_PREP(BIT_ULL_MASK(9), !!(flags & IOMMU_HW_QUEUE_FLAG_AMD_COMWAIT_EN));
+
+	writeq(val, vfctrl + VIOMMU_VFCTRL_MMIO_GUEST_COMMAND_CONTROL_OFFSET);
+
+	pr_debug("%s: iommu_devid=%#x, gid=%#x, type=%#x, addr=%#llx, len=%#lx, flags=%#x, val=%#llx\n",
+		 __func__, iommu->devid, gid, hw_queue->type,
+		 hw_queue->base_addr, hw_queue->length, flags, val);
+}
+
+static void _amd_viommu_set_evtbuf_flags(struct iommufd_hw_queue *hw_queue)
+{
+	u8 __iomem *vfctrl, *vf;
+	u32 flags = hw_queue->flags;
+	u64 val;
+	struct iommufd_viommu *viommu = hw_queue->viommu;
+	struct amd_iommu_viommu *aviommu = container_of(viommu, struct amd_iommu_viommu, core);
+	struct amd_iommu *iommu = container_of(viommu->iommu_dev, struct amd_iommu, iommu);
+	int gid = aviommu->gid;
+
+	vf = VIOMMU_VF_MMIO_BASE(iommu, gid);
+	vfctrl = VIOMMU_VFCTRL_MMIO_BASE(iommu, gid);
+
+	/* Clear fields in VFCTRL MMIO */
+	val = readq(vfctrl + VIOMMU_VFCTRL_MMIO_GUEST_EVENT_CONTROL_OFFSET);
+	val &= ~GENMASK_ULL(51, 0);
+
+	/* Set Event buffer base and length */
+	val |= FIELD_PREP(GENMASK_ULL(3, 0), hw_queue->length);
+	val |= FIELD_PREP(GENMASK_ULL(51, 12), (hw_queue->base_addr >> 12));
+	val |= FIELD_PREP(BIT_ULL_MASK(8), !!(flags & IOMMU_HW_QUEUE_FLAG_AMD_EVT_LOG_EN));
+	val |= FIELD_PREP(BIT_ULL_MASK(9), !!(flags & IOMMU_HW_QUEUE_FLAG_AMD_EVT_INT_EN));
+	writeq(val, vfctrl + VIOMMU_VFCTRL_MMIO_GUEST_EVENT_CONTROL_OFFSET);
+
+	pr_debug("%s: iommu_devid=%#x, gid=%#x, type=%#x, addr=%#llx, len=%#lx, flags=%#x, val=%#llx\n",
+		 __func__, iommu->devid, gid, hw_queue->type,
+		 hw_queue->base_addr, hw_queue->length, flags, val);
+}
+
+static void _amd_viommu_set_pprbuf_flags(struct iommufd_hw_queue *hw_queue)
+{
+	u8 __iomem *vfctrl, *vf;
+	u32 flags = hw_queue->flags;
+	u64 val;
+	struct iommufd_viommu *viommu = hw_queue->viommu;
+	struct amd_iommu_viommu *aviommu = container_of(viommu, struct amd_iommu_viommu, core);
+	struct amd_iommu *iommu = container_of(viommu->iommu_dev, struct amd_iommu, iommu);
+	int gid = aviommu->gid;
+
+	vf = VIOMMU_VF_MMIO_BASE(iommu, gid);
+	vfctrl = VIOMMU_VFCTRL_MMIO_BASE(iommu, gid);
+
+	/* Clear fields in VFCTRL MMIO */
+	val = readq(vfctrl + VIOMMU_VFCTRL_MMIO_GUEST_PPR_CONTROL_OFFSET);
+	val &= ~GENMASK_ULL(55, 0);
+
+	/* Set PPR buffer base and length */
+	val |= FIELD_PREP(GENMASK_ULL(3, 0), hw_queue->length);
+	val |= FIELD_PREP(GENMASK_ULL(55, 16), (hw_queue->base_addr >> 12));
+	val |= FIELD_PREP(BIT_ULL_MASK(8), !!(flags & IOMMU_HW_QUEUE_FLAG_AMD_PPRLOG_EN));
+	val |= FIELD_PREP(BIT_ULL_MASK(9), !!(flags & IOMMU_HW_QUEUE_FLAG_AMD_PPRINT_EN));
+	val |= FIELD_PREP(BIT_ULL_MASK(10), !!(flags & IOMMU_HW_QUEUE_FLAG_AMD_PPR_EN));
+	val |= FIELD_PREP(BIT_ULL_MASK(13), !!(flags & IOMMU_HW_QUEUE_FLAG_AMD_PPR_AUTO_RSP_EN));
+	val |= FIELD_PREP(BIT_ULL_MASK(14), !!(flags & IOMMU_HW_QUEUE_FLAG_AMD_BLKSTOPMRK_EN));
+	val |= FIELD_PREP(BIT_ULL_MASK(15), !!(flags & IOMMU_HW_QUEUE_FLAG_AMD_PPR_AUTO_RSP_AON));
+	writeq(val, vfctrl + VIOMMU_VFCTRL_MMIO_GUEST_PPR_CONTROL_OFFSET);
+
+	pr_debug("%s: iommu_devid=%#x, gid=%#x, type=%#x, addr=%#llx, len=%#lx, flags=%#x, val=%#llx\n",
+		 __func__, iommu->devid, gid, hw_queue->type,
+		 hw_queue->base_addr, hw_queue->length, flags, val);
+}
+
+static int _amd_viommu_hw_queue_init(struct iommufd_hw_queue *hw_queue, u32 index)
+{
+	int ret = 0;
+
+	switch (hw_queue->type) {
+	case IOMMU_HW_QUEUE_TYPE_AMD_CMD:
+		_amd_viommu_set_cmdbuf_flags(hw_queue);
+		break;
+	case IOMMU_HW_QUEUE_TYPE_AMD_EVT:
+		_amd_viommu_set_evtbuf_flags(hw_queue);
+		break;
+	case IOMMU_HW_QUEUE_TYPE_AMD_PPR:
+		_amd_viommu_set_pprbuf_flags(hw_queue);
+		break;
+	default:
+		pr_err("%s: Invalid type (%#x)\n", __func__, hw_queue->type);
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
 /*
  * See include/linux/iommufd.h
  * struct iommufd_viommu_ops - vIOMMU specific operations
@@ -194,4 +321,6 @@ static const struct iommufd_viommu_ops amd_viommu_ops = {
 	.destroy = amd_iommufd_viommu_destroy,
 	.vdevice_size = VDEVICE_STRUCT_SIZE(struct amd_iommu_vdevice, core),
 	.vdevice_init = _amd_viommu_vdevice_init,
+	.get_hw_queue_size = _amd_viommu_get_hw_queue_size,
+	.hw_queue_init = _amd_viommu_hw_queue_init,
 };
diff --git a/drivers/iommu/iommufd/viommu.c b/drivers/iommu/iommufd/viommu.c
index b278cbde360e..e81948f3402a 100644
--- a/drivers/iommu/iommufd/viommu.c
+++ b/drivers/iommu/iommufd/viommu.c
@@ -392,6 +392,7 @@ static int _iommufd_hw_queue_init(struct iommufd_ucmd *ucmd,
 	refcount_inc(&viommu->obj.users);
 	hw_queue->access = access;
 	hw_queue->type = cmd->type;
+	hw_queue->flags = cmd->flags;
 	hw_queue->length = cmd->length;
 	hw_queue->base_addr = cmd->nesting_parent_iova;
 
@@ -417,7 +418,7 @@ int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd)
 	u64 last;
 	int rc;
 
-	if (cmd->flags || cmd->type == IOMMU_HW_QUEUE_TYPE_DEFAULT)
+	if (cmd->type == IOMMU_HW_QUEUE_TYPE_DEFAULT)
 		return -EOPNOTSUPP;
 	if (!cmd->length)
 		return -EINVAL;
diff --git a/include/linux/iommufd.h b/include/linux/iommufd.h
index 5285d96f1d8c..7a3d11f03dec 100644
--- a/include/linux/iommufd.h
+++ b/include/linux/iommufd.h
@@ -135,6 +135,7 @@ struct iommufd_hw_queue {
 	size_t length;
 
 	enum iommu_hw_queue_type type;
+	u32 flags;
 
 	/* Clean up all driver-specific parts of an iommufd_hw_queue */
 	void (*destroy)(struct iommufd_hw_queue *hw_queue);
diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h
index 52bcd92975da..3f9c972e703a 100644
--- a/include/uapi/linux/iommufd.h
+++ b/include/uapi/linux/iommufd.h
@@ -1322,12 +1322,41 @@ enum iommu_hw_queue_type {
 	 *   emulated vSMMU's IDR1.CMDQS to log2(huge page size / 16 bytes)
 	 */
 	IOMMU_HW_QUEUE_TYPE_TEGRA241_CMDQV = 1,
+	IOMMU_HW_QUEUE_TYPE_AMD_CMD,
+	IOMMU_HW_QUEUE_TYPE_AMD_EVT,
+	IOMMU_HW_QUEUE_TYPE_AMD_PPR,
+};
+
+/**
+ * enum iommu_hw_queue_flags_amd - AMD HW Queue Flags
+ * @IOMMU_HW_QUEUE_FLAG_AMD_CMDBUF_EN        : Command buffer enable
+ * @IOMMU_HW_QUEUE_FLAG_AMD_COMWAIT_EN       : Command wait enable
+ * @IOMMU_HW_QUEUE_FLAG_AMD_EVT_LOG_EN       : Event log enable
+ * @IOMMU_HW_QUEUE_FLAG_AMD_EVT_INT_EN       : Event interrupt enable
+ * @IOMMU_HW_QUEUE_FLAG_AMD_PPRLOG_EN        : PPR log enable
+ * @IOMMU_HW_QUEUE_FLAG_AMD_PPRINT_EN        : PPR print enable
+ * @IOMMU_HW_QUEUE_FLAG_AMD_PPR_EN           : PPR enable
+ * @IOMMU_HW_QUEUE_FLAG_AMD_PPR_AUTO_RSP_EN  : PPR auto response enable
+ * @IOMMU_HW_QUEUE_FLAG_AMD_BLKSTOPMRK_EN    : Block stop mark enable
+ * @IOMMU_HW_QUEUE_FLAG_AMD_PPR_AUTO_RSP_AON : PPR auto response on next enable
+ */
+enum iommu_hw_queue_flags_amd {
+	IOMMU_HW_QUEUE_FLAG_AMD_CMDBUF_EN = 1 << 0,
+	IOMMU_HW_QUEUE_FLAG_AMD_COMWAIT_EN = 1 << 1,
+	IOMMU_HW_QUEUE_FLAG_AMD_EVT_LOG_EN = 1 << 2,
+	IOMMU_HW_QUEUE_FLAG_AMD_EVT_INT_EN = 1 << 3,
+	IOMMU_HW_QUEUE_FLAG_AMD_PPRLOG_EN = 1 << 4,
+	IOMMU_HW_QUEUE_FLAG_AMD_PPRINT_EN = 1 << 5,
+	IOMMU_HW_QUEUE_FLAG_AMD_PPR_EN = 1 << 6,
+	IOMMU_HW_QUEUE_FLAG_AMD_PPR_AUTO_RSP_EN = 1 << 7,
+	IOMMU_HW_QUEUE_FLAG_AMD_BLKSTOPMRK_EN = 1 << 8,
+	IOMMU_HW_QUEUE_FLAG_AMD_PPR_AUTO_RSP_AON = 1 << 9,
 };
 
 /**
  * struct iommu_hw_queue_alloc - ioctl(IOMMU_HW_QUEUE_ALLOC)
  * @size: sizeof(struct iommu_hw_queue_alloc)
- * @flags: Must be 0
+ * @flags: HW queue flags based on enum iommu_hw_queue_type
  * @viommu_id: Virtual IOMMU ID to associate the HW queue with
  * @type: One of enum iommu_hw_queue_type
  * @index: The logical index to the HW queue per virtual IOMMU for a multi-queue
-- 
2.34.1