From: Paul Durrant Date: Thu, 18 Aug 2016 11:27:44 +0000 (+0100) Subject: xen-netback: multi page rings X-Git-Url: http://xenbits.xensource.com/gitweb?a=commitdiff_plain;h=refs%2Fheads%2Fmulti-page-netback;p=people%2Fpauldu%2Flinux.git xen-netback: multi page rings Signed-off-by: Paul Durrant --- diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h index 3a562683603c..63d8bccf3527 100644 --- a/drivers/net/xen-netback/common.h +++ b/drivers/net/xen-netback/common.h @@ -66,8 +66,15 @@ struct pending_tx_info { struct ubuf_info callback_struct; }; -#define XEN_NETIF_TX_RING_SIZE __CONST_RING_SIZE(xen_netif_tx, XEN_PAGE_SIZE) -#define XEN_NETIF_RX_RING_SIZE __CONST_RING_SIZE(xen_netif_rx, XEN_PAGE_SIZE) +#define XEN_NETIF_TX_RING_SIZE(order) \ + __CONST_RING_SIZE(xen_netif_tx, XEN_PAGE_SIZE << (order)) +#define XEN_NETIF_RX_RING_SIZE(order) \ + __CONST_RING_SIZE(xen_netif_rx, XEN_PAGE_SIZE << (order)) + +#define XEN_NETIF_MAX_TX_RING_SIZE \ + XEN_NETIF_TX_RING_SIZE(XENBUS_MAX_RING_GRANT_ORDER) +#define XEN_NETIF_MAX_RX_RING_SIZE \ + XEN_NETIF_RX_RING_SIZE(XENBUS_MAX_RING_GRANT_ORDER) struct xenvif_rx_meta { int id; @@ -84,7 +91,7 @@ struct xenvif_rx_meta { #define MAX_BUFFER_OFFSET XEN_PAGE_SIZE -#define MAX_PENDING_REQS XEN_NETIF_TX_RING_SIZE +#define MAX_PENDING_REQS XEN_NETIF_MAX_TX_RING_SIZE /* The maximum number of frags is derived from the size of a grant (same * as a Xen page size for now). @@ -96,7 +103,7 @@ struct xenvif_rx_meta { * worst-case number of copy operations is MAX_XEN_SKB_FRAGS per * ring slot. */ -#define MAX_GRANT_COPY_OPS (MAX_XEN_SKB_FRAGS * XEN_NETIF_RX_RING_SIZE) +#define MAX_GRANT_COPY_OPS (MAX_XEN_SKB_FRAGS * XEN_NETIF_MAX_RX_RING_SIZE) #define NETBACK_INVALID_HANDLE -1 @@ -158,6 +165,7 @@ struct xenvif_queue { /* Per-queue data for xenvif */ struct gnttab_unmap_grant_ref tx_unmap_ops[MAX_PENDING_REQS]; /* passed to gnttab_[un]map_refs with pages under (un)mapping */ struct page *pages_to_map[MAX_PENDING_REQS]; + u16 pending_idx_release[MAX_PENDING_REQS]; struct page *pages_to_unmap[MAX_PENDING_REQS]; /* This prevents zerocopy callbacks to race over dealloc_ring */ @@ -194,7 +202,7 @@ struct xenvif_queue { /* Per-queue data for xenvif */ /* We create one meta structure per ring request we consume, so * the maximum number is the same as the ring size. */ - struct xenvif_rx_meta meta[XEN_NETIF_RX_RING_SIZE]; + struct xenvif_rx_meta meta[XEN_NETIF_MAX_RX_RING_SIZE]; /* Transmit shaping: allow 'credit_bytes' every 'credit_usec'. */ unsigned long credit_bytes; @@ -266,6 +274,7 @@ struct xenvif { u8 ip_csum:1; u8 ipv6_csum:1; u8 multicast_control:1; + u8 multi_page_ring:1; /* Is this interface disabled? True when backend discovers * frontend is rogue. @@ -276,6 +285,7 @@ struct xenvif { unsigned long stall_timeout; /* Queues */ + unsigned int ring_page_order; struct xenvif_queue *queues; unsigned int num_queues; /* active queues, resource allocated */ unsigned int stalled_queues; @@ -322,8 +332,9 @@ int xenvif_init_queue(struct xenvif_queue *queue); void xenvif_deinit_queue(struct xenvif_queue *queue); int xenvif_connect_data(struct xenvif_queue *queue, - unsigned long tx_ring_ref, - unsigned long rx_ring_ref, + grant_ref_t *tx_ring_ref, + grant_ref_t *rx_ring_ref, + unsigned int nr_ring_refs, unsigned int tx_evtchn, unsigned int rx_evtchn); void xenvif_disconnect_data(struct xenvif *vif); @@ -343,8 +354,9 @@ void xenvif_wake_queue(struct xenvif_queue *queue); /* (Un)Map communication rings. */ void xenvif_unmap_frontend_data_rings(struct xenvif_queue *queue); int xenvif_map_frontend_data_rings(struct xenvif_queue *queue, - grant_ref_t tx_ring_ref, - grant_ref_t rx_ring_ref); + grant_ref_t *tx_ring_ref, + grant_ref_t *rx_ring_ref, + unsigned int nr_ring_refs); /* Check for SKBs from frontend and schedule backend processing */ void xenvif_napi_schedule_or_enable_events(struct xenvif_queue *queue); @@ -385,6 +397,7 @@ extern unsigned int rx_drain_timeout_msecs; extern unsigned int rx_stall_timeout_msecs; extern unsigned int xenvif_max_queues; extern unsigned int xenvif_hash_cache_size; +extern unsigned int xenvif_max_ring_page_order; #ifdef CONFIG_DEBUG_FS extern struct dentry *xen_netback_dbg_root; diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c index 83deeebfc2d1..6e038b2b038f 100644 --- a/drivers/net/xen-netback/interface.c +++ b/drivers/net/xen-netback/interface.c @@ -44,7 +44,8 @@ #define XENVIF_NAPI_WEIGHT 64 /* Number of bytes allowed on the internal guest Rx queue. */ -#define XENVIF_RX_QUEUE_BYTES (XEN_NETIF_RX_RING_SIZE/2 * PAGE_SIZE) +#define XENVIF_RX_QUEUE_BYTES(order) \ + (XEN_NETIF_RX_RING_SIZE(order) / 2 * PAGE_SIZE) /* This function is used to set SKBTX_DEV_ZEROCOPY as well as * increasing the inflight counter. We need to increase the inflight @@ -517,7 +518,7 @@ int xenvif_init_queue(struct xenvif_queue *queue) queue->credit_timeout.function = xenvif_tx_credit_callback; queue->credit_window_start = get_jiffies_64(); - queue->rx_queue_max = XENVIF_RX_QUEUE_BYTES; + queue->rx_queue_max = XENVIF_RX_QUEUE_BYTES(queue->vif->ring_page_order); skb_queue_head_init(&queue->rx_queue); skb_queue_head_init(&queue->tx_queue); @@ -623,8 +624,9 @@ err: } int xenvif_connect_data(struct xenvif_queue *queue, - unsigned long tx_ring_ref, - unsigned long rx_ring_ref, + grant_ref_t *tx_ring_ref, + grant_ref_t *rx_ring_ref, + unsigned int nr_ring_refs, unsigned int tx_evtchn, unsigned int rx_evtchn) { @@ -636,7 +638,7 @@ int xenvif_connect_data(struct xenvif_queue *queue, BUG_ON(queue->dealloc_task); err = xenvif_map_frontend_data_rings(queue, tx_ring_ref, - rx_ring_ref); + rx_ring_ref, nr_ring_refs); if (err < 0) goto err; diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c index edbae0b1e8f0..9675bacab5f4 100644 --- a/drivers/net/xen-netback/netback.c +++ b/drivers/net/xen-netback/netback.c @@ -72,6 +72,12 @@ module_param_named(max_queues, xenvif_max_queues, uint, 0644); MODULE_PARM_DESC(max_queues, "Maximum number of queues per virtual interface"); +unsigned int xenvif_max_ring_page_order = XENBUS_MAX_RING_GRANT_ORDER; +module_param_named(max_ring_page_order, xenvif_max_ring_page_order, uint, + 0644); +MODULE_PARM_DESC(max_ring_page_order, + "Maximum order of pages to be used for the TX and RX shared rings"); + /* * This is the maximum slots a skb can have. If a guest sends a skb * which exceeds this limit it is considered malicious. @@ -1371,12 +1377,12 @@ static void xenvif_tx_build_gops(struct xenvif_queue *queue, pending_ring_idx_t index; if (queue->tx.sring->req_prod - queue->tx.req_cons > - XEN_NETIF_TX_RING_SIZE) { + XEN_NETIF_TX_RING_SIZE(queue->vif->ring_page_order)) { netdev_err(queue->vif->dev, "Impossible number of requests. " "req_prod %d, req_cons %d, size %ld\n", queue->tx.sring->req_prod, queue->tx.req_cons, - XEN_NETIF_TX_RING_SIZE); + XEN_NETIF_TX_RING_SIZE(queue->vif->ring_page_order)); xenvif_fatal_tx_err(queue->vif); break; } @@ -1806,7 +1812,7 @@ static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue) { struct gnttab_unmap_grant_ref *gop; pending_ring_idx_t dc, dp; - u16 pending_idx, pending_idx_release[MAX_PENDING_REQS]; + u16 pending_idx; unsigned int i = 0; dc = queue->dealloc_cons; @@ -1826,7 +1832,7 @@ static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue) pending_idx = queue->dealloc_ring[pending_index(dc++)]; - pending_idx_release[gop - queue->tx_unmap_ops] = + queue->pending_idx_release[gop - queue->tx_unmap_ops] = pending_idx; queue->pages_to_unmap[gop - queue->tx_unmap_ops] = queue->mmap_pages[pending_idx]; @@ -1864,7 +1870,7 @@ static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue) } for (i = 0; i < gop - queue->tx_unmap_ops; ++i) - xenvif_idx_release(queue, pending_idx_release[i], + xenvif_idx_release(queue, queue->pending_idx_release[i], XEN_NETIF_RSP_OKAY); } @@ -2023,8 +2029,9 @@ void xenvif_unmap_frontend_data_rings(struct xenvif_queue *queue) } int xenvif_map_frontend_data_rings(struct xenvif_queue *queue, - grant_ref_t tx_ring_ref, - grant_ref_t rx_ring_ref) + grant_ref_t *tx_ring_ref, + grant_ref_t *rx_ring_ref, + unsigned int nr_ring_refs) { void *addr; struct xen_netif_tx_sring *txs; @@ -2033,20 +2040,20 @@ int xenvif_map_frontend_data_rings(struct xenvif_queue *queue, int err = -ENOMEM; err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif), - &tx_ring_ref, 1, &addr); + tx_ring_ref, nr_ring_refs, &addr); if (err) goto err; txs = (struct xen_netif_tx_sring *)addr; - BACK_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE); + BACK_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE * nr_ring_refs); err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif), - &rx_ring_ref, 1, &addr); + rx_ring_ref, nr_ring_refs, &addr); if (err) goto err; rxs = (struct xen_netif_rx_sring *)addr; - BACK_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE); + BACK_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE * nr_ring_refs); return 0; @@ -2398,6 +2405,13 @@ static int __init netback_init(void) fatal_skb_slots = XEN_NETBK_LEGACY_SLOTS_MAX; } + if (xenvif_max_ring_page_order > XENBUS_MAX_RING_GRANT_ORDER) { + pr_info("Invalid max_ring_page_order (%d), will use default max: %d.\n", + xenvif_max_ring_page_order, + XENBUS_MAX_RING_GRANT_ORDER); + xenvif_max_ring_page_order = XENBUS_MAX_RING_GRANT_ORDER; + } + rc = xenvif_xenbus_init(); if (rc) goto failed_init; diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c index bacf6e0c12b9..1b5fb5d50cfe 100644 --- a/drivers/net/xen-netback/xenbus.c +++ b/drivers/net/xen-netback/xenbus.c @@ -416,6 +416,11 @@ static int netback_probe(struct xenbus_device *dev, be->hotplug_script = script; + err = xenbus_printf(XBT_NIL, dev->nodename, "max-ring-page-order", "%u", + xenvif_max_ring_page_order); + if (err) + pr_warn("%s write out 'max-ring-page-order' failed\n", __func__); + err = xenbus_switch_state(dev, XenbusStateInitWait); if (err) goto fail; @@ -914,9 +919,27 @@ static void connect(struct backend_info *be) struct xenbus_device *dev = be->dev; unsigned long credit_bytes, credit_usec; unsigned int queue_index; + unsigned int ring_page_order; unsigned int requested_num_queues; struct xenvif_queue *queue; + /* Check whether the frontend supports multiple ring pages */ + err = xenbus_scanf(XBT_NIL, dev->otherend, + "ring-page-order", + "%u", &ring_page_order); + if (err < 0) { + be->vif->ring_page_order = 0; + } else if (ring_page_order <= xenvif_max_ring_page_order) { + be->vif->ring_page_order = ring_page_order; + be->vif->multi_page_ring = 1; + } else { + /* buggy or malicious guest */ + xenbus_dev_fatal(dev, err, + "guest requested ring-page-order %u, exceeding the maximum of %u.", + ring_page_order, xenvif_max_ring_page_order); + return; + } + /* Check whether the frontend requested multiple queues * and read the number requested. */ @@ -1033,13 +1056,16 @@ err: return; } +#define RINGREF_NAME_LEN sizeof("Xx-ring-ref-XXXX") static int connect_data_rings(struct backend_info *be, struct xenvif_queue *queue) { struct xenbus_device *dev = be->dev; unsigned int num_queues = queue->vif->num_queues; - unsigned long tx_ring_ref, rx_ring_ref; + grant_ref_t tx_ring_ref[XENBUS_MAX_RING_GRANTS]; + grant_ref_t rx_ring_ref[XENBUS_MAX_RING_GRANTS]; + unsigned int nr_ring_refs; unsigned int tx_evtchn, rx_evtchn; int err; char *xspath; @@ -1072,14 +1098,44 @@ static int connect_data_rings(struct backend_info *be, queue->id); } - err = xenbus_gather(XBT_NIL, xspath, - "tx-ring-ref", "%lu", &tx_ring_ref, - "rx-ring-ref", "%lu", &rx_ring_ref, NULL); - if (err) { - xenbus_dev_fatal(dev, err, - "reading %s/ring-ref", - xspath); - goto err; + if (!queue->vif->multi_page_ring) { + err = xenbus_gather(XBT_NIL, xspath, + "tx-ring-ref", "%lu", &tx_ring_ref[0], + "rx-ring-ref", "%lu", &rx_ring_ref[0], NULL); + if (err) { + xenbus_dev_fatal(dev, err, + "reading %s/ring-ref", + xspath); + goto err; + } + + nr_ring_refs = 1; + } else { + unsigned int i; + + nr_ring_refs = 1 << queue->vif->ring_page_order; + + for (i = 0; i < nr_ring_refs; i++) { + char tx_ring_ref_name[RINGREF_NAME_LEN]; + char rx_ring_ref_name[RINGREF_NAME_LEN]; + + snprintf(tx_ring_ref_name, RINGREF_NAME_LEN, + "tx-ring-ref%u", i); + snprintf(rx_ring_ref_name, RINGREF_NAME_LEN, + "rx-ring-ref%u", i); + + err = xenbus_gather(XBT_NIL, xspath, + tx_ring_ref_name, "%lu", &tx_ring_ref[i], + rx_ring_ref_name, "%lu", &rx_ring_ref[i], + NULL); + if (err) { + xenbus_dev_fatal(dev, err, + "reading %s/ring-ref%u", + xspath, + i); + goto err; + } + } } /* Try split event channels first, then single event channel. */ @@ -1099,13 +1155,12 @@ static int connect_data_rings(struct backend_info *be, } /* Map the shared frame, irq etc. */ - err = xenvif_connect_data(queue, tx_ring_ref, rx_ring_ref, + err = xenvif_connect_data(queue, tx_ring_ref, rx_ring_ref, nr_ring_refs, tx_evtchn, rx_evtchn); if (err) { xenbus_dev_fatal(dev, err, - "mapping shared-frames %lu/%lu port tx %u rx %u", - tx_ring_ref, rx_ring_ref, - tx_evtchn, rx_evtchn); + "connecting rings (queue %u)", + queue->id); goto err; }