Run an RPC protocol over the LXC controller monitor

This defines a new RPC protocol to be used between the LXC
controller and the libvirtd LXC driver. There is only a
single RPC message defined thus far, an asynchronous "EXIT"
event that is emitted just before the LXC controller process
exits. This provides the LXC driver with details about how
the container shutdown - normally, or abnormally (crashed),
thus allowing the driver to emit better libvirt events.

Emitting the event in the LXC controller requires a few
little tricks with the RPC service. Simply calling the
virNetServiceClientSendMessage does not work, since this
merely queues the message for asynchronous processing.
In addition the main event loop is no longer running at
the point the event is emitted, so no I/O is processed.

Thus after invoking virNetServiceClientSendMessage it is
necessary to mark the client as being in "delayed close"
mode. Then the event loop is run again, until the client
completes its close - this happens only after the queued
message has been fully transmitted. The final complexity
is that it is not safe to run virNetServerQuit() from the
client close callback, since that is invoked from a
context where the server is locked. Thus a zero-second
timer is used to trigger shutdown of the event loop,
causing the controller to finally exit.

* src/Makefile.am: Add rules for generating RPC protocol
  files and dispatch methods
* src/lxc/lxc_controller.c: Emit an RPC event immediately
  before exiting
* src/lxc/lxc_domain.h: Record the shutdown reason
  given by the controller
* src/lxc/lxc_monitor.c, src/lxc/lxc_monitor.h: Register
  RPC program and event handler. Add callback to let
  driver receive EXIT event.
* src/lxc/lxc_process.c: Use monitor exit event to decide
  what kind of domain event to emit
* src/lxc/lxc_protocol.x: Define wire protocol for LXC
  controller monitor.

Signed-off-by: Daniel P. Berrange <berrange@redhat.com>

diff --git a/.gitignore b/.gitignore
index b4cbb5f83de18b0d3f74b78f38ace72761824bdd..e4b3932d33885c5ef9b1c06196ee2f6e12a0441a 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -105,6 +105,10 @@
 /src/locking/qemu-sanlock.conf
 /src/locking/test_libvirt_sanlock.aug
 /src/lxc/test_libvirtd_lxc.aug
+/src/lxc/lxc_controller_dispatch.h
+/src/lxc/lxc_monitor_dispatch.h
+/src/lxc/lxc_protocol.c
+/src/lxc/lxc_protocol.h
 /src/qemu/test_libvirtd_qemu.aug
 /src/remote/*_client_bodies.h
 /src/remote/*_protocol.[ch]

diff --git a/src/Makefile.am b/src/Makefile.am
index 0b984205216f610e96e29e7cc1c0699f6bca688a..da3d0cdc06dd2b8841c33dd6fc301e893b23668b 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -387,7 +387,47 @@ if WITH_XEN_INOTIFY
 XEN_DRIVER_SOURCES += xen/xen_inotify.c xen/xen_inotify.h
 endif
 
+LXC_PROTOCOL_GENERATED = \
+       $(srcdir)/lxc/lxc_protocol.h \
+       $(srcdir)/lxc/lxc_protocol.c \
+       $(NULL)
+
+LXC_MONITOR_GENERATED = \
+       $(srcdir)/lxc/lxc_monitor_dispatch.h \
+       $(NULL)
+
+LXC_CONTROLLER_GENERATED = \
+       $(srcdir)/lxc/lxc_controller_dispatch.h \
+       $(NULL)
+
+LXC_GENERATED = \
+       $(LXC_PROTOCOL_GENERATED) \
+       $(LXC_MONITOR_GENERATED) \
+       $(LXC_CONTROLLER_GENERATED) \
+       $(NULL)
+
+LXC_PROTOCOL = $(srcdir)/lxc/lxc_protocol.x
+
+$(srcdir)/lxc/lxc_monitor_dispatch.h: $(srcdir)/rpc/gendispatch.pl \
+               $(LXC_PROTOCOL)
+       $(AM_V_GEN)$(PERL) -w $(srcdir)/rpc/gendispatch.pl \
+         -k virLXCProtocol VIR_LXC_PROTOCOL $(LXC_PROTOCOL) > $@
+
+$(srcdir)/lxc/lxc_controller_dispatch.h: $(srcdir)/rpc/gendispatch.pl \
+               $(REMOTE_PROTOCOL)
+       $(AM_V_GEN)$(PERL) -w $(srcdir)/rpc/gendispatch.pl \
+         -b virLXCProtocol VIR_LXC_PROTOCOL $(LXC_PROTOCOL) > $@
+
+EXTRA_DIST += \
+       $(LXC_PROTOCOL) \
+       $(LXC_GENERATED) \
+       $(NULL)
+
+BUILT_SOURCES += $(LXC_GENERATED)
+
 LXC_DRIVER_SOURCES =                                           \
+               $(LXC_PROTOCOL_GENERATED)                       \
+               $(LXC_MONITOR_GENERATED)                        \
                lxc/lxc_conf.c lxc/lxc_conf.h                   \
                lxc/lxc_container.c lxc/lxc_container.h         \
                lxc/lxc_cgroup.c lxc/lxc_cgroup.h               \
@@ -397,6 +437,8 @@ LXC_DRIVER_SOURCES =                                                \
                lxc/lxc_driver.c lxc/lxc_driver.h
 
 LXC_CONTROLLER_SOURCES =                                       \
+               $(LXC_PROTOCOL_GENERATED)                       \
+               $(LXC_CONTROLLER_GENERATED)                     \
                lxc/lxc_conf.c lxc/lxc_conf.h                   \
                lxc/lxc_container.c lxc/lxc_container.h         \
                lxc/lxc_cgroup.c lxc/lxc_cgroup.h               \

diff --git a/src/lxc/lxc_controller.c b/src/lxc/lxc_controller.c
index e39c2365ca91cc89d3c20e2af57fd77e1233b9b0..e388302a8b0d85027e2fd2b1ff4ac62b809dae61 100644 (file)
--- a/src/lxc/lxc_controller.c
+++ b/src/lxc/lxc_controller.c
@@ -59,6 +59,7 @@
 #include "lxc_conf.h"
 #include "lxc_container.h"
 #include "lxc_cgroup.h"
+#include "lxc_protocol.h"
 #include "virnetdev.h"
 #include "virnetdevveth.h"
 #include "memory.h"
@@ -121,10 +122,25 @@ struct _virLXCController {
 
     /* Server socket */
     virNetServerPtr server;
+    virNetServerClientPtr client;
+    virNetServerProgramPtr prog;
+    bool inShutdown;
+    int timerShutdown;
 };
 
+#include "lxc_controller_dispatch.h"
+
 static void virLXCControllerFree(virLXCControllerPtr ctrl);
 
+static void virLXCControllerQuitTimer(int timer ATTRIBUTE_UNUSED, void *opaque)
+{
+    virLXCControllerPtr ctrl = opaque;
+
+    VIR_DEBUG("Triggering event loop quit");
+    virNetServerQuit(ctrl->server);
+}
+
+
 static virLXCControllerPtr virLXCControllerNew(const char *name)
 {
     virLXCControllerPtr ctrl = NULL;
@@ -134,6 +150,8 @@ static virLXCControllerPtr virLXCControllerNew(const char *name)
     if (VIR_ALLOC(ctrl) < 0)
         goto no_memory;
 
+    ctrl->timerShutdown = -1;
+
     if (!(ctrl->name = strdup(name)))
         goto no_memory;
 
@@ -150,6 +168,11 @@ static virLXCControllerPtr virLXCControllerNew(const char *name)
                                            0)) == NULL)
         goto error;
 
+    if ((ctrl->timerShutdown = virEventAddTimeout(-1,
+                                                  virLXCControllerQuitTimer, ctrl,
+                                                  NULL)) < 0)
+        goto error;
+
 cleanup:
     VIR_FREE(configFile);
     virCapabilitiesFree(caps);
@@ -238,6 +261,9 @@ static void virLXCControllerFree(virLXCControllerPtr ctrl)
     virDomainDefFree(ctrl->def);
     VIR_FREE(ctrl->name);
 
+    if (ctrl->timerShutdown != -1)
+        virEventRemoveTimeout(ctrl->timerShutdown);
+
     virNetServerFree(ctrl->server);
 
     VIR_FREE(ctrl);
@@ -539,12 +565,28 @@ static int virLXCControllerSetupResourceLimits(virLXCControllerPtr ctrl)
 }
 
 
+static void virLXCControllerClientCloseHook(virNetServerClientPtr client)
+{
+    virLXCControllerPtr ctrl = virNetServerClientGetPrivateData(client);
+
+    VIR_DEBUG("Client %p has closed", client);
+    if (ctrl->client == client)
+        ctrl->client = NULL;
+    if (ctrl->inShutdown) {
+        VIR_DEBUG("Arm timer to quit event loop");
+        virEventUpdateTimeout(ctrl->timerShutdown, 0);
+    }
+}
+
 static int virLXCControllerClientHook(virNetServerPtr server ATTRIBUTE_UNUSED,
                                       virNetServerClientPtr client,
                                       void *opaque)
 {
     virLXCControllerPtr ctrl = opaque;
     virNetServerClientSetPrivateData(client, ctrl, NULL);
+    virNetServerClientSetCloseHook(client, virLXCControllerClientCloseHook);
+    VIR_DEBUG("Got new client %p", client);
+    ctrl->client = client;
     return 0;
 }
 
@@ -581,6 +623,12 @@ static int virLXCControllerSetupServer(virLXCControllerPtr ctrl)
     virNetServerServiceFree(svc);
     svc = NULL;
 
+    if (!(ctrl->prog = virNetServerProgramNew(VIR_LXC_PROTOCOL_PROGRAM,
+                                              VIR_LXC_PROTOCOL_PROGRAM_VERSION,
+                                              virLXCProtocolProcs,
+                                              virLXCProtocolNProcs)))
+        goto error;
+
     virNetServerUpdateServices(ctrl->server, true);
     VIR_FREE(sockpath);
     return 0;
@@ -1219,6 +1267,79 @@ virLXCControllerSetupConsoles(virLXCControllerPtr ctrl,
 }
 
 
+static void
+virLXCControllerEventSend(virLXCControllerPtr ctrl,
+                          int procnr,
+                          xdrproc_t proc,
+                          void *data)
+{
+    virNetMessagePtr msg;
+
+    if (!ctrl->client)
+        return;
+
+    VIR_DEBUG("Send event %d client=%p", procnr, ctrl->client);
+    if (!(msg = virNetMessageNew(false)))
+        goto error;
+
+    msg->header.prog = virNetServerProgramGetID(ctrl->prog);
+    msg->header.vers = virNetServerProgramGetVersion(ctrl->prog);
+    msg->header.proc = procnr;
+    msg->header.type = VIR_NET_MESSAGE;
+    msg->header.serial = 1;
+    msg->header.status = VIR_NET_OK;
+
+    if (virNetMessageEncodeHeader(msg) < 0)
+        goto error;
+
+    if (virNetMessageEncodePayload(msg, proc, data) < 0)
+        goto error;
+
+    VIR_DEBUG("Queue event %d %zu", procnr, msg->bufferLength);
+    virNetServerClientSendMessage(ctrl->client, msg);
+
+    xdr_free(proc, data);
+    return;
+
+error:
+    virNetMessageFree(msg);
+    xdr_free(proc, data);
+}
+
+
+static int
+virLXCControllerEventSendExit(virLXCControllerPtr ctrl,
+                              int exitstatus)
+{
+    virLXCProtocolExitEventMsg msg;
+
+    VIR_DEBUG("Exit status %d", exitstatus);
+    memset(&msg, 0, sizeof(msg));
+    switch (exitstatus) {
+    case 0:
+        msg.status = VIR_LXC_PROTOCOL_EXIT_STATUS_SHUTDOWN;
+        break;
+    default:
+        msg.status = VIR_LXC_PROTOCOL_EXIT_STATUS_ERROR;
+        break;
+    }
+
+    virLXCControllerEventSend(ctrl,
+                              VIR_LXC_PROTOCOL_PROC_EXIT_EVENT,
+                              (xdrproc_t)xdr_virLXCProtocolExitEventMsg,
+                              (void*)&msg);
+
+    if (ctrl->client) {
+        VIR_DEBUG("Waiting for client to complete dispatch");
+        ctrl->inShutdown = true;
+        virNetServerClientDelayedClose(ctrl->client);
+        virNetServerRun(ctrl->server);
+    }
+    VIR_DEBUG("Client has gone away");
+    return 0;
+}
+
+
 static int
 virLXCControllerRun(virLXCControllerPtr ctrl)
 {
@@ -1306,6 +1427,8 @@ virLXCControllerRun(virLXCControllerPtr ctrl)
 
     rc = virLXCControllerMain(ctrl);
 
+    virLXCControllerEventSendExit(ctrl, rc);
+
 cleanup:
     VIR_FORCE_CLOSE(control[0]);
     VIR_FORCE_CLOSE(control[1]);
@@ -1527,5 +1650,5 @@ cleanup:
 
     virLXCControllerFree(ctrl);
 
-    return rc ? EXIT_FAILURE : EXIT_SUCCESS;
+    return rc < 0? EXIT_FAILURE : EXIT_SUCCESS;
 }

diff --git a/src/lxc/lxc_domain.h b/src/lxc/lxc_domain.h
index 318c78eb6c6b3cf73100556a21eafca852b420b1..f4a83841264a538ef1d44fe27821fb4aa0291644 100644 (file)
--- a/src/lxc/lxc_domain.h
+++ b/src/lxc/lxc_domain.h
@@ -32,6 +32,7 @@ typedef virLXCDomainObjPrivate *virLXCDomainObjPrivatePtr;
 struct _virLXCDomainObjPrivate {
     virLXCMonitorPtr monitor;
     bool doneStopEvent;
+    int stopReason;
 };
 
 void virLXCDomainSetPrivateDataHooks(virCapsPtr caps);

diff --git a/src/lxc/lxc_monitor.c b/src/lxc/lxc_monitor.c
index da170ae6a25b9f78a9b0f67a46ce6e1b43b19717..a2a25995f6d131aff08da3bf0f3fa2b71c97d75f 100644 (file)
--- a/src/lxc/lxc_monitor.c
+++ b/src/lxc/lxc_monitor.c
@@ -22,6 +22,8 @@
 
 #include "lxc_monitor.h"
 #include "lxc_conf.h"
+#include "lxc_protocol.h"
+#include "lxc_monitor_dispatch.h"
 
 #include "memory.h"
 
@@ -41,9 +43,35 @@ struct _virLXCMonitor {
     virLXCMonitorCallbacksPtr cb;
 
     virNetClientPtr client;
+    virNetClientProgramPtr program;
 };
 
 static void virLXCMonitorFree(virLXCMonitorPtr mon);
+static void
+virLXCMonitorHandleEventExit(virNetClientProgramPtr prog,
+                             virNetClientPtr client,
+                             void *evdata, void *opaque);
+
+static virNetClientProgramEvent virLXCProtocolEvents[] = {
+    { VIR_LXC_PROTOCOL_PROC_EXIT_EVENT,
+      virLXCMonitorHandleEventExit,
+      sizeof(virLXCProtocolExitEventMsg),
+      (xdrproc_t)xdr_virLXCProtocolExitEventMsg },
+};
+
+
+static void
+virLXCMonitorHandleEventExit(virNetClientProgramPtr prog ATTRIBUTE_UNUSED,
+                             virNetClientPtr client ATTRIBUTE_UNUSED,
+                             void *evdata, void *opaque)
+{
+    virLXCMonitorPtr mon = opaque;
+    virLXCProtocolExitEventMsg *msg = evdata;
+
+    VIR_DEBUG("Event exit %d", msg->status);
+    if (mon->cb->exitNotify)
+        mon->cb->exitNotify(mon, msg->status, mon->vm);
+}
 
 
 static void virLXCMonitorEOFNotify(virNetClientPtr client ATTRIBUTE_UNUSED,
@@ -54,6 +82,7 @@ static void virLXCMonitorEOFNotify(virNetClientPtr client ATTRIBUTE_UNUSED,
     virLXCMonitorCallbackEOFNotify eofNotify;
     virDomainObjPtr vm;
 
+    VIR_DEBUG("EOF notify");
     virLXCMonitorLock(mon);
     eofNotify = mon->cb->eofNotify;
     vm = mon->vm;
@@ -101,6 +130,17 @@ virLXCMonitorPtr virLXCMonitorNew(virDomainObjPtr vm,
         goto error;
 
 
+    if (!(mon->program = virNetClientProgramNew(VIR_LXC_PROTOCOL_PROGRAM,
+                                                VIR_LXC_PROTOCOL_PROGRAM_VERSION,
+                                                virLXCProtocolEvents,
+                                                ARRAY_CARDINALITY(virLXCProtocolEvents),
+                                                mon)))
+        goto error;
+
+    if (virNetClientAddProgram(mon->client,
+                               mon->program) < 0)
+        goto error;
+
     mon->vm = vm;
     mon->cb = cb;
 
@@ -130,6 +170,7 @@ static void virLXCMonitorFree(virLXCMonitorPtr mon)
     if (mon->cb && mon->cb->destroy)
         (mon->cb->destroy)(mon, mon->vm);
     virMutexDestroy(&mon->lock);
+    virNetClientProgramFree(mon->program);
     VIR_FREE(mon);
 }
 

diff --git a/src/lxc/lxc_monitor.h b/src/lxc/lxc_monitor.h
index d3b6387b184e15c7cf7ce38d7faa899bf3538663..53301f1f216a2fd0e790cfb18857cc31ff5543de 100644 (file)
--- a/src/lxc/lxc_monitor.h
+++ b/src/lxc/lxc_monitor.h
@@ -22,6 +22,7 @@
 # define __LXC_MONITOR_H__
 
 # include "domain_conf.h"
+# include "lxc_protocol.h"
 
 typedef struct _virLXCMonitor virLXCMonitor;
 typedef virLXCMonitor *virLXCMonitorPtr;
@@ -34,9 +35,14 @@ typedef void (*virLXCMonitorCallbackDestroy)(virLXCMonitorPtr mon,
 typedef void (*virLXCMonitorCallbackEOFNotify)(virLXCMonitorPtr mon,
                                                virDomainObjPtr vm);
 
+typedef void (*virLXCMonitorCallbackExitNotify)(virLXCMonitorPtr mon,
+                                                virLXCProtocolExitStatus status,
+                                                virDomainObjPtr vm);
+
 struct _virLXCMonitorCallbacks {
     virLXCMonitorCallbackDestroy destroy;
     virLXCMonitorCallbackEOFNotify eofNotify;
+    virLXCMonitorCallbackExitNotify exitNotify;
 };
 
 virLXCMonitorPtr virLXCMonitorNew(virDomainObjPtr vm,

diff --git a/src/lxc/lxc_process.c b/src/lxc/lxc_process.c
index 89c2b214ba8fa481dc8743325a593b33148fc762..5ffdff5c65dc093465c73aba6cdfef62e739c75c 100644 (file)
--- a/src/lxc/lxc_process.c
+++ b/src/lxc/lxc_process.c
@@ -167,6 +167,9 @@ static void virLXCProcessCleanup(virLXCDriverPtr driver,
     virLXCDomainObjPrivatePtr priv = vm->privateData;
     virNetDevVPortProfilePtr vport = NULL;
 
+    VIR_DEBUG("Stopping VM name=%s pid=%d reason=%d",
+              vm->def->name, (int)vm->pid, (int)reason);
+
     /* now that we know it's stopped call the hook if present */
     if (virHookPresent(VIR_HOOK_DRIVER_LXC)) {
         char *xml = virDomainDefFormat(vm->def, 0);
@@ -509,7 +512,7 @@ static void virLXCProcessMonitorEOFNotify(virLXCMonitorPtr mon ATTRIBUTE_UNUSED,
     if (!priv->doneStopEvent) {
         event = virDomainEventNewFromObj(vm,
                                          VIR_DOMAIN_EVENT_STOPPED,
-                                         VIR_DOMAIN_EVENT_STOPPED_SHUTDOWN);
+                                         priv->stopReason);
         virDomainAuditStop(vm, "shutdown");
     } else {
         VIR_DEBUG("Stop event has already been sent");
@@ -528,10 +531,31 @@ static void virLXCProcessMonitorEOFNotify(virLXCMonitorPtr mon ATTRIBUTE_UNUSED,
     }
 }
 
+static void virLXCProcessMonitorExitNotify(virLXCMonitorPtr mon ATTRIBUTE_UNUSED,
+                                           virLXCProtocolExitStatus status,
+                                           virDomainObjPtr vm)
+{
+    virLXCDomainObjPrivatePtr priv = vm->privateData;
+
+    switch (status) {
+    case VIR_LXC_PROTOCOL_EXIT_STATUS_SHUTDOWN:
+        priv->stopReason = VIR_DOMAIN_EVENT_STOPPED_SHUTDOWN;
+        break;
+    case VIR_LXC_PROTOCOL_EXIT_STATUS_ERROR:
+        priv->stopReason = VIR_DOMAIN_EVENT_STOPPED_FAILED;
+        break;
+    default:
+        priv->stopReason = VIR_DOMAIN_EVENT_STOPPED_FAILED;
+        break;
+    }
+    VIR_DEBUG("Domain shutoff reason %d (from status %d)",
+              priv->stopReason, status);
+}
 
 static virLXCMonitorCallbacks monitorCallbacks = {
     .eofNotify = virLXCProcessMonitorEOFNotify,
     .destroy = virLXCProcessMonitorDestroy,
+    .exitNotify = virLXCProcessMonitorExitNotify,
 };
 
 
@@ -573,6 +597,8 @@ int virLXCProcessStop(virLXCDriverPtr driver,
     virCgroupPtr group = NULL;
     int rc;
 
+    VIR_DEBUG("Stopping VM name=%s pid=%d reason=%d",
+              vm->def->name, (int)vm->pid, (int)reason);
     if (vm->pid <= 0) {
         virReportError(VIR_ERR_INTERNAL_ERROR,
                        _("Invalid PID %d for container"), vm->pid);
@@ -1006,6 +1032,7 @@ int virLXCProcessStart(virConnectPtr conn,
         goto cleanup;
     }
 
+    priv->stopReason = VIR_DOMAIN_EVENT_STOPPED_FAILED;
     vm->def->id = vm->pid;
     virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, reason);
     priv->doneStopEvent = false;

diff --git a/src/lxc/lxc_protocol.x b/src/lxc/lxc_protocol.x
new file mode 100644 (file)
index 0000000..4fdbe34
--- /dev/null
+++ b/src/lxc/lxc_protocol.x
@@ -0,0 +1,21 @@
+/* -*- c -*-
+ * Define wire protocol for communication between the
+ * LXC driver in libvirtd, and the LXC controller in
+ * the libvirt_lxc helper program.
+ */
+
+enum virLXCProtocolExitStatus {
+    VIR_LXC_PROTOCOL_EXIT_STATUS_ERROR,
+    VIR_LXC_PROTOCOL_EXIT_STATUS_SHUTDOWN
+};
+
+struct virLXCProtocolExitEventMsg {
+    enum virLXCProtocolExitStatus status;
+};
+
+const VIR_LXC_PROTOCOL_PROGRAM = 0x12341234;
+const VIR_LXC_PROTOCOL_PROGRAM_VERSION = 1;
+
+enum virLXCProtocolProcedure {
+    VIR_LXC_PROTOCOL_PROC_EXIT_EVENT = 1 /* skipgen skipgen */
+};