Re: [PATCH v2 3/8] s390: vfio-ap: allow assignment of unavailable AP resources to mdev device

From: Tony Krowiak
Date: Tue Apr 23 2019 - 09:19:14 EST

On 4/23/19 8:46 AM, Pierre Morel wrote:
On 20/04/2019 23:49, Tony Krowiak wrote:
The AP architecture does not preclude assignment of AP resources that are
not yet in the AP configuration (i.e., not available or not online).
Let's go ahead and implement this facet of the AP architecture for linux
guests.

Access to AP resources is controlled by bit masks in a guest's SIE
state description (i.e., the control block containing the state of a
guest). When an AP instruction is executed on the guest, the firmware
combines the masks from the guest's SIE state description with the masks
from the host by doing a logical bitwise AND of the masks to create what
are defined as effective masks. The effective masks determine which AP
resources can be accessed by the guest.

Effective masking allows the assignment of AP resources to a guest even
if the underlying device is not in the AP configuration. If the device
subsequently becomes configured, the guest will automatically have access
to the associated AP resources (e.g., AP queues).

The current implementation does not allow assignment of AP resources to
an mdev device if the AP queues identified by the assignment are not
bound to the vfio_ap device driver. This patch allows assignment of AP
resources to the mdev device even if the AP queues are not bound to the
vfio_ap device driver, as long as the AP queues are not reserved by the
AP BUS for use by the zcrypt device drivers. If the queues are subsequently
bound to the vfio_ap device driver while a running guest is using the mdev
device, the guest will automatically gain access to the queues. Effective
masking prevents access to queues until such time as they become available.

Signed-off-by: Tony Krowiak <akrowiak@xxxxxxxxxxxxx>
---
 drivers/s390/crypto/vfio_ap_ops.c | 160 +++++---------------------------------
 1 file changed, 18 insertions(+), 142 deletions(-)

diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index eae4ee24460a..221491a9ba95 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -113,122 +113,6 @@ static struct attribute_group *vfio_ap_mdev_type_groups[] = {
ÂÂÂÂÂ NULL,
 };
-struct vfio_ap_queue_reserved {
-ÂÂÂ unsigned long *apid;
-ÂÂÂ unsigned long *apqi;
-ÂÂÂ bool reserved;
-};
-
-/**
- * vfio_ap_has_queue
- *
- * @dev: an AP queue device
- * @data: a struct vfio_ap_queue_reserved reference
- *
- * Flags whether the AP queue device (@dev) has a queue ID containing the APQN,
- * apid or apqi specified in @data:
- *
- * - If @data contains both an apid and apqi value, then @data will be flagged
- *ÂÂ as reserved if the APID and APQI fields for the AP queue device matches
- *
- * - If @data contains only an apid value, @data will be flagged as
- *ÂÂ reserved if the APID field in the AP queue device matches
- *
- * - If @data contains only an apqi value, @data will be flagged as
- *ÂÂ reserved if the APQI field in the AP queue device matches
- *
- * Returns 0 to indicate the input to function succeeded. Returns -EINVAL if
- * @data does not contain either an apid or apqi.
- */
-static int vfio_ap_has_queue(struct device *dev, void *data)
-{
-ÂÂÂ struct vfio_ap_queue_reserved *qres = data;
-ÂÂÂ struct ap_queue *ap_queue = to_ap_queue(dev);
-ÂÂÂ ap_qid_t qid;
-ÂÂÂ unsigned long id;
-
-ÂÂÂ if (qres->apid && qres->apqi) {
-ÂÂÂÂÂÂÂ qid = AP_MKQID(*qres->apid, *qres->apqi);
-ÂÂÂÂÂÂÂ if (qid == ap_queue->qid)
-ÂÂÂÂÂÂÂÂÂÂÂ qres->reserved = true;
-ÂÂÂ } else if (qres->apid && !qres->apqi) {
-ÂÂÂÂÂÂÂ id = AP_QID_CARD(ap_queue->qid);
-ÂÂÂÂÂÂÂ if (id == *qres->apid)
-ÂÂÂÂÂÂÂÂÂÂÂ qres->reserved = true;
-ÂÂÂ } else if (!qres->apid && qres->apqi) {
-ÂÂÂÂÂÂÂ id = AP_QID_QUEUE(ap_queue->qid);
-ÂÂÂÂÂÂÂ if (id == *qres->apqi)
-ÂÂÂÂÂÂÂÂÂÂÂ qres->reserved = true;
-ÂÂÂ } else {
-ÂÂÂÂÂÂÂ return -EINVAL;
-ÂÂÂ }
-
-ÂÂÂ return 0;
-}
-
-/**
- * vfio_ap_verify_queue_reserved
- *
- * @matrix_dev: a mediated matrix device
- * @apid: an AP adapter ID
- * @apqi: an AP queue index
- *
- * Verifies that the AP queue with @apid/@apqi is reserved by the VFIO AP device
- * driver according to the following rules:
- *
- * - If both @apid and @apqi are not NULL, then there must be an AP queue
- *ÂÂ device bound to the vfio_ap driver with the APQN identified by @apid and
- *ÂÂ @apqi
- *
- * - If only @apid is not NULL, then there must be an AP queue device bound
- *ÂÂ to the vfio_ap driver with an APQN containing @apid
- *
- * - If only @apqi is not NULL, then there must be an AP queue device bound
- *ÂÂ to the vfio_ap driver with an APQN containing @apqi
- *
- * Returns 0 if the AP queue is reserved; otherwise, returns -EADDRNOTAVAIL.
- */
-static int vfio_ap_verify_queue_reserved(unsigned long *apid,
-ÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂ unsigned long *apqi)
-{
-ÂÂÂ int ret;
-ÂÂÂ struct vfio_ap_queue_reserved qres;
-
-ÂÂÂ qres.apid = apid;
-ÂÂÂ qres.apqi = apqi;
-ÂÂÂ qres.reserved = false;
-
-ÂÂÂ ret = driver_for_each_device(&matrix_dev->vfio_ap_drv->driver, NULL,
-ÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂ &qres, vfio_ap_has_queue);
-ÂÂÂ if (ret)
-ÂÂÂÂÂÂÂ return ret;
-
-ÂÂÂ if (qres.reserved)
-ÂÂÂÂÂÂÂ return 0;
-
-ÂÂÂ return -EADDRNOTAVAIL;
-}
-
-static int
-vfio_ap_mdev_verify_queues_reserved_for_apid(struct ap_matrix_mdev *matrix_mdev,
-ÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂ unsigned long apid)

You suppress this function but it is still in use in assign_adapter_store...

Right you are. I must have been removed in a subsequent patch, because
this code was compiled and tested. It should have been removed here
though.



-{
-ÂÂÂ int ret;
-ÂÂÂ unsigned long apqi;
-ÂÂÂ unsigned long nbits = matrix_mdev->matrix.aqm_max + 1;
-
-ÂÂÂ if (find_first_bit_inv(matrix_mdev->matrix.aqm, nbits) >= nbits)
-ÂÂÂÂÂÂÂ return vfio_ap_verify_queue_reserved(&apid, NULL);
-
-ÂÂÂ for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, nbits) {
-ÂÂÂÂÂÂÂ ret = vfio_ap_verify_queue_reserved(&apid, &apqi);
-ÂÂÂÂÂÂÂ if (ret)
-ÂÂÂÂÂÂÂÂÂÂÂ return ret;
-ÂÂÂ }
-
-ÂÂÂ return 0;
-}
-
 /**
ÂÂ * vfio_ap_mdev_verify_no_sharing
ÂÂ *
@@ -359,9 +243,17 @@ static ssize_t assign_adapter_store(struct device *dev,
ÂÂÂÂÂ set_bit_inv(apid, matrix_mdev->matrix.apm);
+ÂÂÂ ret = ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,
+ÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂ matrix_mdev->matrix.aqm);
+
+ÂÂÂ /* If any APQN is owned by a default driver */
+ÂÂÂ ret = (ret == 1) ? -EADDRNOTAVAIL : ret;
+ÂÂÂ if (ret)
+ÂÂÂÂÂÂÂ goto error;
+
ÂÂÂÂÂ ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev);
ÂÂÂÂÂ if (ret)
-ÂÂÂÂÂÂÂ goto share_err;
+ÂÂÂÂÂÂÂ goto error;
ÂÂÂÂÂ if (matrix_mdev->shadow_crycb)
ÂÂÂÂÂÂÂÂÂ set_bit_inv(apid, matrix_mdev->shadow_crycb->apm);
@@ -369,7 +261,7 @@ static ssize_t assign_adapter_store(struct device *dev,
ÂÂÂÂÂ ret = count;
ÂÂÂÂÂ goto done;
-share_err:
+error:
ÂÂÂÂÂ clear_bit_inv(apid, matrix_mdev->matrix.apm);
 done:
ÂÂÂÂÂ mutex_unlock(&matrix_dev->lock);
@@ -426,26 +318,6 @@ static ssize_t unassign_adapter_store(struct device *dev,
 }
 static DEVICE_ATTR_WO(unassign_adapter);
-static int
-vfio_ap_mdev_verify_queues_reserved_for_apqi(struct ap_matrix_mdev *matrix_mdev,
-ÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂ unsigned long apqi)
-{
-ÂÂÂ int ret;
-ÂÂÂ unsigned long apid;
-ÂÂÂ unsigned long nbits = matrix_mdev->matrix.apm_max + 1;
-
-ÂÂÂ if (find_first_bit_inv(matrix_mdev->matrix.apm, nbits) >= nbits)
-ÂÂÂÂÂÂÂ return vfio_ap_verify_queue_reserved(NULL, &apqi);
-
-ÂÂÂ for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, nbits) {
-ÂÂÂÂÂÂÂ ret = vfio_ap_verify_queue_reserved(&apid, &apqi);
-ÂÂÂÂÂÂÂ if (ret)
-ÂÂÂÂÂÂÂÂÂÂÂ return ret;
-ÂÂÂ }
-
-ÂÂÂ return 0;
-}
-
 /**
ÂÂ * assign_domain_store
ÂÂ *
@@ -500,12 +372,16 @@ static ssize_t assign_domain_store(struct device *dev,
ÂÂÂÂÂ mutex_lock(&matrix_dev->lock);
-ÂÂÂ ret = vfio_ap_mdev_verify_queues_reserved_for_apqi(matrix_mdev, apqi);
-ÂÂÂ if (ret)
-ÂÂÂÂÂÂÂ goto done;
-
ÂÂÂÂÂ set_bit_inv(apqi, matrix_mdev->matrix.aqm);
+ÂÂÂ ret = ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,
+ÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂÂ matrix_mdev->matrix.aqm);
+
+ÂÂÂ /* If any APQN is owned by a default driver */
+ÂÂÂ ret = (ret == 1) ? -EADDRNOTAVAIL : ret;
+ÂÂÂ if (ret)
+ÂÂÂÂÂÂÂ goto error;

forget to add the error label ?

Apparently I did. I'm not sure what happened here since I compiled
and tested the code. I must have messed something up when formatting
the patches.


+
ÂÂÂÂÂ ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev);
ÂÂÂÂÂ if (ret)
ÂÂÂÂÂÂÂÂÂ goto share_err;