xxx wip: multiqueue support

Performance is still the same, there must be other bottleneck

diff --git a/sys/dev/xen/netfront/netfront.c b/sys/dev/xen/netfront/netfront.c
index 16aefc810015a9e7bac62abc1af22582e5d42d91..0b4e8af56c088bd63fe4e0963f86979f0f8b0d41 100644 (file)
--- a/sys/dev/xen/netfront/netfront.c
+++ b/sys/dev/xen/netfront/netfront.c
@@ -101,6 +101,12 @@ __FBSDID("$FreeBSD$");
 static int xn_enable_lro = 1;
 TUNABLE_INT("hw.xn.enable_lro", &xn_enable_lro);
 
+/*
+ * Number of pairs of queues.
+ */
+static unsigned long xn_num_queues = 4;
+TUNABLE_ULONG("hw.xn.num_queues", &xn_num_queues);
+
 /**
  * \brief The maximum allowed data fragments in a single transmit
  *        request.
@@ -151,7 +157,8 @@ static int netfront_detach(device_t dev);
 static int talk_to_backend(device_t dev, struct netfront_info *info);
 static int create_netdev(device_t dev);
 static void netif_disconnect_backend(struct netfront_info *info);
-static int setup_device(device_t dev, struct netfront_info *info);
+static int setup_device(device_t dev, struct netfront_info *info,
+                       unsigned long num_queues);
 static void free_ring(int *ref, void *ring_ptr_ref);
 
 static int  xn_ifmedia_upd(struct ifnet *ifp);
@@ -193,6 +200,7 @@ struct netfront_stats
        u_long  tx_errors;              /* packet transmit problems     */
 };
 
+#define XN_LOCK_NAME_LEN  8  /* xn{t,r}x_%u, allow for two digits */
 struct netfront_queue_info {
        struct netfront_info *info;
        u_int num;
@@ -204,6 +212,8 @@ struct netfront_queue_info {
 
        struct mtx   tx_lock;
        struct mtx   rx_lock;
+       char         tx_mtx_name[XN_LOCK_NAME_LEN];
+       char         rx_mtx_name[XN_LOCK_NAME_LEN];
        xen_intr_handle_t xen_intr_handle;
 
        grant_ref_t gref_tx_head;
@@ -419,17 +429,22 @@ netfront_attach(device_t dev)
 {
        int err;
 
+       SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
+           SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+           OID_AUTO, "enable_lro", CTLFLAG_RW,
+           &xn_enable_lro, 0, "Large Receive Offload");
+
+       SYSCTL_ADD_ULONG(device_get_sysctl_ctx(dev),
+           SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+           OID_AUTO, "num_queues", CTLFLAG_RW,
+           &xn_num_queues, "Number of pairs of queues");
+
        err = create_netdev(dev);
        if (err) {
                xenbus_dev_fatal(dev, err, "creating netdev");
                return (err);
        }
 
-       SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
-           SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
-           OID_AUTO, "enable_lro", CTLFLAG_RW,
-           &xn_enable_lro, 0, "Large Receive Offload");
-
        return (0);
 }
 
@@ -470,6 +485,56 @@ netfront_resume(device_t dev)
        return (0);
 }
 
+static int
+write_queue_xenstore_keys(device_t dev,
+                         struct netfront_queue_info *queue,
+                         struct xs_transaction *xst, int hierachy)
+{
+       int err;
+       const char *message;
+       const char *node = xenbus_get_node(dev);
+       char *path;
+       size_t path_size;
+
+       if (hierachy) {
+               path_size = strlen(node) + 10;
+               path = malloc(path_size, M_DEVBUF, M_WAITOK|M_ZERO);
+               snprintf(path, path_size, "%s/queue-%u", node,
+                        queue->num);
+       } else {
+               path_size = strlen(node) + 1;
+               path = malloc(path_size, M_DEVBUF, M_WAITOK|M_ZERO);
+               snprintf(path, path_size, "%s", node);
+       }
+
+       err = xs_printf(*xst, path, "tx-ring-ref","%u", queue->tx_ring_ref);
+       if (err) {
+               message = "writing tx ring-ref";
+               goto error;
+       }
+       err = xs_printf(*xst, path, "rx-ring-ref","%u", queue->rx_ring_ref);
+       if (err) {
+               message = "writing rx ring-ref";
+               goto error;
+       }
+       err = xs_printf(*xst, path, "event-channel", "%u",
+                       xen_intr_port(queue->xen_intr_handle));
+       if (err) {
+               message = "writing event-channel";
+               goto error;
+       }
+
+       free(path, M_DEVBUF);
+
+       return (0);
+
+ error:
+       free(path, M_DEVBUF);
+       xenbus_dev_fatal(dev, err, "%s", message);
+
+       return (err);
+}
+
 /* Common code used when first setting up, and when resuming. */
 static int
 talk_to_backend(device_t dev, struct netfront_info *info)
@@ -478,6 +543,8 @@ talk_to_backend(device_t dev, struct netfront_info *info)
        struct xs_transaction xst;
        const char *node = xenbus_get_node(dev);
        int err;
+       unsigned long num_queues, max_queues = 0;
+       unsigned int i;
 
        err = xen_net_read_mac(dev, info->mac);
        if (err) {
@@ -485,10 +552,18 @@ talk_to_backend(device_t dev, struct netfront_info *info)
                goto out;
        }
 
-       /* XXX negotiate other features as well. */
+       err = xs_scanf(XST_NIL, xenbus_get_otherend_path(info->xbdev),
+                      "multi-queue-max-queues", NULL, "%lu", &max_queues);
+       if (err < 0)
+               max_queues = 1;
+       num_queues = xn_num_queues;
+       if (num_queues > max_queues) num_queues = max_queues;
+
+       /* TODO: relinquish resources when reconnecting. */
+       KASSERT(info->queue == NULL, ("info->queue != NULL"));
 
        /* Create shared ring, alloc event channel. */
-       err = setup_device(dev, info);
+       err = setup_device(dev, info, num_queues);
        if (err)
                goto out;
 
@@ -499,24 +574,25 @@ talk_to_backend(device_t dev, struct netfront_info *info)
                goto destroy_ring;
        }
 
-       err = xs_printf(xst, node, "tx-ring-ref","%u",
-                       info->queue[0].tx_ring_ref);
-       if (err) {
-               message = "writing tx ring-ref";
-               goto abort_transaction;
-       }
-       err = xs_printf(xst, node, "rx-ring-ref","%u",
-                       info->queue[0].rx_ring_ref);
-       if (err) {
-               message = "writing rx ring-ref";
-               goto abort_transaction;
-       }
-       err = xs_printf(xst, node,
-                       "event-channel", "%u",
-                       xen_intr_port(info->queue[0].xen_intr_handle));
-       if (err) {
-               message = "writing event-channel";
-               goto abort_transaction;
+       if (info->num_queues == 1) {
+               err = write_queue_xenstore_keys(dev, &info->queue[0],
+                                               &xst, 0);
+               if (err)
+                       goto abort_transaction_no_def_error;
+       } else {
+               err = xs_printf(xst, node, "multi-queue-num-queues",
+                               "%u", info->num_queues);
+               if (err) {
+                       message = "writing multi-queue-num-queues";
+                       goto abort_transaction;
+               }
+
+               for (i = 0; i < info->num_queues; i++) {
+                       err = write_queue_xenstore_keys(dev, &info->queue[i],
+                                                       &xst, 1);
+                       if (err)
+                               goto abort_transaction_no_def_error;
+               }
        }
 
        err = xs_printf(xst, node, "request-rx-copy", "%u", 1);
@@ -551,8 +627,9 @@ talk_to_backend(device_t dev, struct netfront_info *info)
        return 0;
 
  abort_transaction:
-       xs_transaction_end(xst, 1);
        xenbus_dev_fatal(dev, err, "%s", message);
+ abort_transaction_no_def_error:
+       xs_transaction_end(xst, 1);
  destroy_ring:
        netif_free(info);
  out:
@@ -560,59 +637,112 @@ talk_to_backend(device_t dev, struct netfront_info *info)
 }
 
 static int
-setup_device(device_t dev, struct netfront_info *info)
+setup_device(device_t dev, struct netfront_info *info,
+            unsigned long num_queues)
 {
        netif_tx_sring_t *txs;
        netif_rx_sring_t *rxs;
        int error;
+       u_int q, i;
+
+       info->queue = malloc(sizeof(struct netfront_queue_info) * num_queues,
+                            M_DEVBUF,
+                            M_WAITOK|M_ZERO);
+
+       for (q = 0; q < num_queues; q++) {
+               info->queue[q].num = q;
+               info->queue[q].info = info;
+               info->queue[q].rx_target = RX_MIN_TARGET;
+               info->queue[q].tx_ring_ref = GRANT_REF_INVALID;
+               info->queue[q].rx_ring_ref = GRANT_REF_INVALID;
+               info->queue[q].rx.sring = NULL;
+               info->queue[q].tx.sring = NULL;
+
+               snprintf(info->queue[q].tx_mtx_name, XN_LOCK_NAME_LEN,
+                        "xntx_%u", q);
+               snprintf(info->queue[q].rx_mtx_name, XN_LOCK_NAME_LEN,
+                        "xnrx_%u", q);
+
+               mtx_init(&info->queue[q].tx_lock, info->queue[q].tx_mtx_name,
+                        "netfront transmit lock",
+                        MTX_DEF);
+               mtx_init(&info->queue[q].rx_lock, info->queue[q].rx_mtx_name,
+                        "netfront receive lock",
+                        MTX_DEF);
+
+               /* Initialise {tx,rx}_skbs to be a free chain containing every entry. */
+               for (i = 0; i <= NET_TX_RING_SIZE; i++) {
+                       info->queue[q].tx_mbufs[i] = (void *) ((u_long) i+1);
+                       info->queue[q].grant_tx_ref[i] = GRANT_REF_INVALID;
+               }
+               info->queue[q].tx_mbufs[NET_TX_RING_SIZE] = (void *)0;
 
-       u_int q = 0;  /* XXX temporary arrangement */
-       KASSERT(info->num_queues == 1, ("num_queues != 1"));
-
-       info->queue[q].tx_ring_ref = GRANT_REF_INVALID;
-       info->queue[q].rx_ring_ref = GRANT_REF_INVALID;
-       info->queue[q].rx.sring = NULL;
-       info->queue[q].tx.sring = NULL;
+               for (i = 0; i <= NET_RX_RING_SIZE; i++) {
+                       info->queue[q].rx_mbufs[i] = NULL;
+                       info->queue[q].grant_rx_ref[i] = GRANT_REF_INVALID;
+               }
 
-       txs = (netif_tx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT|M_ZERO);
-       if (!txs) {
-               error = ENOMEM;
-               xenbus_dev_fatal(dev, error, "allocating tx ring page");
-               goto fail;
-       }
-       SHARED_RING_INIT(txs);
-       FRONT_RING_INIT(&info->queue[q].tx, txs, PAGE_SIZE);
-       error = xenbus_grant_ring(dev, virt_to_mfn(txs),
-                                 &info->queue[q].tx_ring_ref);
-       if (error)
-               goto fail;
+               mbufq_init(&info->queue[q].xn_rx_batch, INT_MAX);
 
-       rxs = (netif_rx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT|M_ZERO);
-       if (!rxs) {
-               error = ENOMEM;
-               xenbus_dev_fatal(dev, error, "allocating rx ring page");
-               goto fail;
-       }
-       SHARED_RING_INIT(rxs);
-       FRONT_RING_INIT(&info->queue[q].rx, rxs, PAGE_SIZE);
+               /* A grant for every tx ring slot */
+               if (gnttab_alloc_grant_references(NET_TX_RING_SIZE,
+                                         &info->queue[q].gref_tx_head) != 0) {
+                       IPRINTK("#### netfront can't alloc tx grant refs\n");
+                       error = ENOMEM;
+                       goto fail;
+               }
+               /* A grant for every rx ring slot */
+               if (gnttab_alloc_grant_references(RX_MAX_TARGET,
+                                         &info->queue[q].gref_rx_head) != 0) {
+                       WPRINTK("#### netfront can't alloc rx grant refs\n");
+                       gnttab_free_grant_references(info->queue[q].gref_tx_head);
+                       error = ENOMEM;
+                       goto fail;
+               }
 
-       error = xenbus_grant_ring(dev, virt_to_mfn(rxs),
-                                 &info->queue[q].rx_ring_ref);
-       if (error)
-               goto fail;
-
-       error = xen_intr_alloc_and_bind_local_port(dev,
-           xenbus_get_otherend_id(dev), /*filter*/NULL, xn_intr,
-           &info->queue[q],
-           INTR_TYPE_NET | INTR_MPSAFE | INTR_ENTROPY,
-           &info->queue[q].xen_intr_handle);
-
-       if (error) {
-               xenbus_dev_fatal(dev, error,
-                                "xen_intr_alloc_and_bind_local_port failed");
-               goto fail;
+               txs = (netif_tx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF,
+                                                M_NOWAIT|M_ZERO);
+               if (!txs) {
+                       error = ENOMEM;
+                       xenbus_dev_fatal(dev, error, "allocating tx ring page");
+                       goto fail;
+               }
+               SHARED_RING_INIT(txs);
+               FRONT_RING_INIT(&info->queue[q].tx, txs, PAGE_SIZE);
+               error = xenbus_grant_ring(dev, virt_to_mfn(txs),
+                                         &info->queue[q].tx_ring_ref);
+               if (error)
+                       goto fail;
+
+               rxs = (netif_rx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF,
+                                                M_NOWAIT|M_ZERO);
+               if (!rxs) {
+                       error = ENOMEM;
+                       xenbus_dev_fatal(dev, error, "allocating rx ring page");
+                       goto fail;
+               }
+               SHARED_RING_INIT(rxs);
+               FRONT_RING_INIT(&info->queue[q].rx, rxs, PAGE_SIZE);
+
+               error = xenbus_grant_ring(dev, virt_to_mfn(rxs),
+                                         &info->queue[q].rx_ring_ref);
+               if (error)
+                       goto fail;
+
+               error = xen_intr_alloc_and_bind_local_port(dev,
+                           xenbus_get_otherend_id(dev), /*filter*/NULL, xn_intr,
+                           &info->queue[q],
+                           INTR_TYPE_NET | INTR_MPSAFE | INTR_ENTROPY,
+                           &info->queue[q].xen_intr_handle);
+
+               if (error) {
+                       xenbus_dev_fatal(dev, error,
+                                        "xen_intr_alloc_and_bind_local_port failed");
+                       goto fail;
+               }
        }
 
+       info->num_queues = num_queues;
        return (0);
 
  fail:
@@ -1910,7 +2040,6 @@ xn_configure_features(struct netfront_info *np)
 int
 create_netdev(device_t dev)
 {
-       int i;
        struct netfront_info *np;
        int err;
        struct ifnet *ifp;
@@ -1928,49 +2057,6 @@ create_netdev(device_t dev)
        np->rx_min_target = RX_MIN_TARGET;
        np->rx_max_target = RX_MAX_TARGET;
 
-       np->num_queues = 1;
-       np->queue = malloc(sizeof(struct netfront_queue_info), M_DEVBUF,
-                          M_WAITOK|M_ZERO);
-
-       np->queue[0].num = 0;
-       np->queue[0].info = np;
-       np->queue[0].rx_target = RX_MIN_TARGET;
-
-       mtx_init(&np->queue[0].tx_lock, "xntx", "netfront transmit lock",
-                MTX_DEF);
-       mtx_init(&np->queue[0].rx_lock, "xnrx", "netfront receive lock",
-                MTX_DEF);
-
-       /* Initialise {tx,rx}_skbs to be a free chain containing every entry. */
-       for (i = 0; i <= NET_TX_RING_SIZE; i++) {
-               np->queue[0].tx_mbufs[i] = (void *) ((u_long) i+1);
-               np->queue[0].grant_tx_ref[i] = GRANT_REF_INVALID;
-       }
-       np->queue[0].tx_mbufs[NET_TX_RING_SIZE] = (void *)0;
-
-       for (i = 0; i <= NET_RX_RING_SIZE; i++) {
-               np->queue[0].rx_mbufs[i] = NULL;
-               np->queue[0].grant_rx_ref[i] = GRANT_REF_INVALID;
-       }
-
-       mbufq_init(&np->queue[0].xn_rx_batch, INT_MAX);
-
-       /* A grant for every tx ring slot */
-       if (gnttab_alloc_grant_references(NET_TX_RING_SIZE,
-                                         &np->queue[0].gref_tx_head) != 0) {
-               IPRINTK("#### netfront can't alloc tx grant refs\n");
-               err = ENOMEM;
-               goto error;
-       }
-       /* A grant for every rx ring slot */
-       if (gnttab_alloc_grant_references(RX_MAX_TARGET,
-                                         &np->queue[0].gref_rx_head) != 0) {
-               WPRINTK("#### netfront can't alloc rx grant refs\n");
-               gnttab_free_grant_references(np->queue[0].gref_tx_head);
-               err = ENOMEM;
-               goto error;
-       }
-
        err = xen_net_read_mac(dev, np->mac);
        if (err)
                goto error;