qemu: Process RDMA GID state change event

This event is emitted on the monitor when a GID table in pvrdma device
is modified and the change needs to be propagate to the backend RDMA
device's GID table.

The control over the RDMA device's GID table is done by updating the
device's Ethernet function addresses.
Usually the first GID entry is determine by the MAC address, the second
by the first IPv6 address and the third by the IPv4 address. Other
entries can be added by adding more IP addresses. The opposite is the
same, i.e. whenever an address is removed, the corresponding GID entry
is removed.

The process is done by the network and RDMA stacks. Whenever an address
is added the ib_core driver is notified and calls the device driver's
add_gid function which in turn update the device.

To support this in pvrdma device we need to hook into the create_bind
and destroy_bind HW commands triggered by pvrdma driver in guest.
Whenever a changed is made to the pvrdma device's GID table a special
QMP messages is sent to be processed by libvirt to update the address of
the backend Ethernet device.

Signed-off-by: Yuval Shaia <yuval.shaia@oracle.com>
Signed-off-by: Michal Privoznik <mprivozn@redhat.com>

diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index bc6eaff3186d336a63f48a420068ac75dd254c07..ec6b34030863859b6521e385ac1fe63efac3ecce 100644 (file)
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -13605,6 +13605,9 @@ qemuProcessEventFree(struct qemuProcessEvent *event)
     case QEMU_PROCESS_EVENT_GUESTPANIC:
         qemuMonitorEventPanicInfoFree(event->data);
         break;
+    case QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED:
+        qemuMonitorEventRdmaGidStatusFree(event->data);
+        break;
     case QEMU_PROCESS_EVENT_WATCHDOG:
     case QEMU_PROCESS_EVENT_DEVICE_DELETED:
     case QEMU_PROCESS_EVENT_NIC_RX_FILTER_CHANGED:

diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
index 73d16966e59b20133534971155dfbd6d5d2644cd..a837b8a731b163316b99e751ecabce1d7100acf9 100644 (file)
--- a/src/qemu/qemu_domain.h
+++ b/src/qemu/qemu_domain.h
@@ -490,6 +490,7 @@ typedef enum {
     QEMU_PROCESS_EVENT_BLOCK_JOB,
     QEMU_PROCESS_EVENT_MONITOR_EOF,
     QEMU_PROCESS_EVENT_PR_DISCONNECT,
+    QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED,
 
     QEMU_PROCESS_EVENT_LAST
 } qemuProcessEventType;

diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index ea316f61d11b284ac7507cfafab86a3e51593742..1d961707cc393508a2411114b2814a337a807020 100644 (file)
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -4792,6 +4792,50 @@ processPRDisconnectEvent(virDomainObjPtr vm)
 }
 
 
+static void
+processRdmaGidStatusChangedEvent(virDomainObjPtr vm,
+                                 qemuMonitorRdmaGidStatusPtr info)
+{
+    unsigned int prefix_len;
+    virSocketAddr addr;
+    VIR_AUTOFREE(char *) addrStr = NULL;
+    int rc;
+
+    if (!virDomainObjIsActive(vm))
+        return;
+
+    VIR_DEBUG("netdev=%s, gid_status=%d, subnet_prefix=0x%llx, interface_id=0x%llx",
+              info->netdev, info->gid_status, info->subnet_prefix,
+              info->interface_id);
+
+    if (info->subnet_prefix) {
+        uint32_t ipv6[4] = {0};
+
+        prefix_len = 64;
+        memcpy(&ipv6[0], &info->subnet_prefix, sizeof(info->subnet_prefix));
+        memcpy(&ipv6[2], &info->interface_id, sizeof(info->interface_id));
+        virSocketAddrSetIPv6AddrNetOrder(&addr, ipv6);
+    } else {
+        prefix_len = 24;
+        virSocketAddrSetIPv4AddrNetOrder(&addr, info->interface_id >> 32);
+    }
+
+    if (!(addrStr = virSocketAddrFormat(&addr)))
+        return;
+
+    if (info->gid_status) {
+        VIR_DEBUG("Adding %s to %s", addrStr, info->netdev);
+        rc = virNetDevIPAddrAdd(info->netdev, &addr, NULL, prefix_len);
+    } else {
+        VIR_DEBUG("Removing %s from %s", addrStr, info->netdev);
+        rc = virNetDevIPAddrDel(info->netdev, &addr, prefix_len);
+    }
+
+    if (rc < 0)
+        VIR_WARN("Fail to update address %s to %s", addrStr, info->netdev);
+}
+
+
 static void qemuProcessEventHandler(void *data, void *opaque)
 {
     struct qemuProcessEvent *processEvent = data;
@@ -4832,6 +4876,9 @@ static void qemuProcessEventHandler(void *data, void *opaque)
     case QEMU_PROCESS_EVENT_PR_DISCONNECT:
         processPRDisconnectEvent(vm);
         break;
+    case QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED:
+        processRdmaGidStatusChangedEvent(vm, processEvent->data);
+        break;
     case QEMU_PROCESS_EVENT_LAST:
         break;
     }

diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c
index 5232cefd75ef3264da7c5d465f038d6a01363bb5..419ea66a6ace4b11556d568e87b7f9d8b965d0da 100644 (file)
--- a/src/qemu/qemu_monitor.c
+++ b/src/qemu/qemu_monitor.c
@@ -1684,6 +1684,24 @@ qemuMonitorEmitPRManagerStatusChanged(qemuMonitorPtr mon,
 }
 
 
+int
+qemuMonitorEmitRdmaGidStatusChanged(qemuMonitorPtr mon,
+                                    const char *netdev,
+                                    bool gid_status,
+                                    uint64_t subnet_prefix,
+                                    uint64_t interface_id)
+{
+    int ret = -1;
+    VIR_DEBUG("netdev=%s, gid_status=%d, subnet_prefix=0x%lx, interface_id=0x%lx",
+              netdev, gid_status, subnet_prefix, interface_id);
+
+    QEMU_MONITOR_CALLBACK(mon, ret, domainRdmaGidStatusChanged, mon->vm,
+                          netdev, gid_status, subnet_prefix, interface_id);
+
+    return ret;
+}
+
+
 int
 qemuMonitorSetCapabilities(qemuMonitorPtr mon)
 {
@@ -4315,6 +4333,17 @@ qemuMonitorEventPanicInfoFree(qemuMonitorEventPanicInfoPtr info)
 }
 
 
+void
+qemuMonitorEventRdmaGidStatusFree(qemuMonitorRdmaGidStatusPtr info)
+{
+    if (!info)
+        return;
+
+    VIR_FREE(info->netdev);
+    VIR_FREE(info);
+}
+
+
 int
 qemuMonitorSetWatchdogAction(qemuMonitorPtr mon,
                              const char *action)

diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h
index 824c3e6845ca8faa0b1c1d439d2dd4853b6f37d4..93804233db697cc01941f3c0e077540fc8e7678c 100644 (file)
--- a/src/qemu/qemu_monitor.h
+++ b/src/qemu/qemu_monitor.h
@@ -106,8 +106,20 @@ struct _qemuMonitorEventPanicInfo {
     } data;
 };
 
+
+typedef struct _qemuMonitorRdmaGidStatus qemuMonitorRdmaGidStatus;
+typedef qemuMonitorRdmaGidStatus *qemuMonitorRdmaGidStatusPtr;
+struct _qemuMonitorRdmaGidStatus {
+    char *netdev;
+    bool gid_status;
+    unsigned long long subnet_prefix;
+    unsigned long long interface_id;
+};
+
+
 char *qemuMonitorGuestPanicEventInfoFormatMsg(qemuMonitorEventPanicInfoPtr info);
 void qemuMonitorEventPanicInfoFree(qemuMonitorEventPanicInfoPtr info);
+void qemuMonitorEventRdmaGidStatusFree(qemuMonitorRdmaGidStatusPtr info);
 
 typedef void (*qemuMonitorDestroyCallback)(qemuMonitorPtr mon,
                                            virDomainObjPtr vm,
@@ -278,6 +290,14 @@ typedef int (*qemuMonitorDomainPRManagerStatusChangedCallback)(qemuMonitorPtr mo
                                                                bool connected,
                                                                void *opaque);
 
+typedef int (*qemuMonitorDomainRdmaGidStatusChangedCallback)(qemuMonitorPtr mon,
+                                                             virDomainObjPtr vm,
+                                                             const char *netdev,
+                                                             bool gid_status,
+                                                             uint64_t subnet_prefix,
+                                                             uint64_t interface_id,
+                                                             void *opaque);
+
 typedef struct _qemuMonitorCallbacks qemuMonitorCallbacks;
 typedef qemuMonitorCallbacks *qemuMonitorCallbacksPtr;
 struct _qemuMonitorCallbacks {
@@ -311,6 +331,7 @@ struct _qemuMonitorCallbacks {
     qemuMonitorDomainBlockThresholdCallback domainBlockThreshold;
     qemuMonitorDomainDumpCompletedCallback domainDumpCompleted;
     qemuMonitorDomainPRManagerStatusChangedCallback domainPRManagerStatusChanged;
+    qemuMonitorDomainRdmaGidStatusChangedCallback domainRdmaGidStatusChanged;
 };
 
 char *qemuMonitorEscapeArg(const char *in);
@@ -445,6 +466,10 @@ int qemuMonitorEmitPRManagerStatusChanged(qemuMonitorPtr mon,
                                           const char *prManager,
                                           bool connected);
 
+int qemuMonitorEmitRdmaGidStatusChanged(qemuMonitorPtr mon, const char *netdev,
+                                        bool gid_status, uint64_t subnet_prefix,
+                                        uint64_t interface_id);
+
 int qemuMonitorStartCPUs(qemuMonitorPtr mon);
 int qemuMonitorStopCPUs(qemuMonitorPtr mon);
 

diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c
index e148cae8f0e7540ea55ae2ee117529b0930072c9..1f7eb20a25f4281f21fca3a544fdbfd9aa2fcbad 100644 (file)
--- a/src/qemu/qemu_monitor_json.c
+++ b/src/qemu/qemu_monitor_json.c
@@ -89,6 +89,7 @@ static void qemuMonitorJSONHandleAcpiOstInfo(qemuMonitorPtr mon, virJSONValuePtr
 static void qemuMonitorJSONHandleBlockThreshold(qemuMonitorPtr mon, virJSONValuePtr data);
 static void qemuMonitorJSONHandleDumpCompleted(qemuMonitorPtr mon, virJSONValuePtr data);
 static void qemuMonitorJSONHandlePRManagerStatusChanged(qemuMonitorPtr mon, virJSONValuePtr data);
+static void qemuMonitorJSONHandleRdmaGidStatusChanged(qemuMonitorPtr mon, virJSONValuePtr data);
 
 typedef struct {
     const char *type;
@@ -112,6 +113,7 @@ static qemuEventHandler eventHandlers[] = {
     { "NIC_RX_FILTER_CHANGED", qemuMonitorJSONHandleNicRxFilterChanged, },
     { "POWERDOWN", qemuMonitorJSONHandlePowerdown, },
     { "PR_MANAGER_STATUS_CHANGED", qemuMonitorJSONHandlePRManagerStatusChanged, },
+    { "RDMA_GID_STATUS_CHANGED", qemuMonitorJSONHandleRdmaGidStatusChanged, },
     { "RESET", qemuMonitorJSONHandleReset, },
     { "RESUME", qemuMonitorJSONHandleResume, },
     { "RTC_CHANGE", qemuMonitorJSONHandleRTCChange, },
@@ -1349,6 +1351,40 @@ static void qemuMonitorJSONHandlePRManagerStatusChanged(qemuMonitorPtr mon,
 }
 
 
+static void qemuMonitorJSONHandleRdmaGidStatusChanged(qemuMonitorPtr mon,
+                                                      virJSONValuePtr data)
+{
+    const char *netdev;
+    bool gid_status;
+    unsigned long long subnet_prefix, interface_id;
+
+    if (!(netdev = virJSONValueObjectGetString(data, "netdev"))) {
+        VIR_WARN("missing netdev in GID_STATUS_CHANGED event");
+        return;
+    }
+
+    if (virJSONValueObjectGetBoolean(data, "gid-status", &gid_status)) {
+        VIR_WARN("missing gid-status in GID_STATUS_CHANGED event");
+        return;
+    }
+
+    if (virJSONValueObjectGetNumberUlong(data, "subnet-prefix",
+                                         &subnet_prefix)) {
+        VIR_WARN("missing subnet-prefix in GID_STATUS_CHANGED event");
+        return;
+    }
+
+    if (virJSONValueObjectGetNumberUlong(data, "interface-id",
+                                         &interface_id)) {
+        VIR_WARN("missing interface-id in GID_STATUS_CHANGED event");
+        return;
+    }
+
+    qemuMonitorEmitRdmaGidStatusChanged(mon, netdev, gid_status, subnet_prefix,
+                                        interface_id);
+}
+
+
 int
 qemuMonitorJSONHumanCommandWithFd(qemuMonitorPtr mon,
                                   const char *cmd_str,

diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 31ae3f39a6a243e7475cf5677b2477bb62b79369..83a054663f220e9aa1c8d4b3834384887e22be44 100644 (file)
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -1715,6 +1715,54 @@ qemuProcessHandlePRManagerStatusChanged(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
 }
 
 
+static int
+qemuProcessHandleRdmaGidStatusChanged(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
+                                      virDomainObjPtr vm,
+                                      const char *netdev,
+                                      bool gid_status,
+                                      uint64_t subnet_prefix,
+                                      uint64_t interface_id,
+                                      void *opaque)
+{
+    virQEMUDriverPtr driver = opaque;
+    struct qemuProcessEvent *processEvent = NULL;
+    qemuMonitorRdmaGidStatusPtr info = NULL;
+    int ret = -1;
+
+    virObjectLock(vm);
+
+    VIR_DEBUG("netdev=%s,gid_status=%d,subnet_prefix=0x%lx,interface_id=0x%lx",
+              netdev, gid_status, subnet_prefix, interface_id);
+
+    if (VIR_ALLOC(info) < 0 ||
+        VIR_STRDUP(info->netdev, netdev) < 0)
+        goto cleanup;
+
+    info->gid_status = gid_status;
+    info->subnet_prefix = subnet_prefix;
+    info->interface_id = interface_id;
+
+    if (VIR_ALLOC(processEvent) < 0)
+        goto cleanup;
+
+    processEvent->eventType = QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED;
+    processEvent->vm = virObjectRef(vm);
+    VIR_STEAL_PTR(processEvent->data, info);
+
+    if (virThreadPoolSendJob(driver->workerPool, 0, processEvent) < 0) {
+        qemuProcessEventFree(processEvent);
+        virObjectUnref(vm);
+        goto cleanup;
+    }
+
+    ret = 0;
+ cleanup:
+    qemuMonitorEventRdmaGidStatusFree(info);
+    virObjectUnlock(vm);
+    return ret;
+}
+
+
 static qemuMonitorCallbacks monitorCallbacks = {
     .eofNotify = qemuProcessHandleMonitorEOF,
     .errorNotify = qemuProcessHandleMonitorError,
@@ -1744,6 +1792,7 @@ static qemuMonitorCallbacks monitorCallbacks = {
     .domainBlockThreshold = qemuProcessHandleBlockThreshold,
     .domainDumpCompleted = qemuProcessHandleDumpCompleted,
     .domainPRManagerStatusChanged = qemuProcessHandlePRManagerStatusChanged,
+    .domainRdmaGidStatusChanged = qemuProcessHandleRdmaGidStatusChanged,
 };
 
 static void