Avoid using the bounce buffer when the source or destination buffer is

32-bits aligned. Merge the two bounce buffers into a single one. Some
rough tests showed that the DWC OTG throughput on RPI2 increased by
10% after this patch.

MFC after:	1 week

diff --git a/sys/dev/usb/controller/dwc_otg.c b/sys/dev/usb/controller/dwc_otg.c
index 6243d8e85c8ef926024cf81c18b6102b78c0190f..5e51527d150e1873d7a1927a3d7b1d90d0a02297 100644 (file)
--- a/sys/dev/usb/controller/dwc_otg.c
+++ b/sys/dev/usb/controller/dwc_otg.c
@@ -180,6 +180,110 @@ dwc_otg_get_hw_ep_profile(struct usb_device *udev,
                *ppf = NULL;
 }
 
+static void
+dwc_otg_write_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
+    uint32_t offset, uint32_t fifo, uint32_t count)
+{
+       uint32_t temp;
+
+       /* round down length to nearest 4-bytes */
+       temp = count & ~3;
+
+       /* check if we can write the data directly */
+       if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
+               struct usb_page_search buf_res;
+
+               /* pre-subtract length */
+               count -= temp;
+
+               /* iterate buffer list */
+               do {
+                       /* get current buffer pointer */
+                       usbd_get_page(pc, offset, &buf_res);
+
+                       if (buf_res.length > temp)
+                               buf_res.length = temp;
+
+                       /* transfer data into FIFO */
+                       bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
+                           fifo, buf_res.buffer, buf_res.length / 4);
+
+                       offset += buf_res.length;
+                       fifo += buf_res.length;
+                       temp -= buf_res.length;
+               } while (temp != 0);
+       }
+
+       /* check for remainder */
+       if (count != 0) {
+               /* clear topmost word before copy */
+               sc->sc_bounce_buffer[(count - 1) / 4] = 0;
+
+               /* copy out data */
+               usbd_copy_out(pc, offset,
+                   sc->sc_bounce_buffer, count);
+
+               /* transfer data into FIFO */
+               bus_space_write_region_4(sc->sc_io_tag,
+                   sc->sc_io_hdl, fifo, sc->sc_bounce_buffer,
+                   (count + 3) / 4);
+       }
+}
+
+static void
+dwc_otg_read_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
+    uint32_t offset, uint32_t count)
+{
+       uint32_t temp;
+
+       /* round down length to nearest 4-bytes */
+       temp = count & ~3;
+
+       /* check if we can read the data directly */
+       if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
+               struct usb_page_search buf_res;
+
+               /* pre-subtract length */
+               count -= temp;
+
+               /* iterate buffer list */
+               do {
+                       /* get current buffer pointer */
+                       usbd_get_page(pc, offset, &buf_res);
+
+                       if (buf_res.length > temp)
+                               buf_res.length = temp;
+
+                       /* transfer data from FIFO */
+                       bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
+                           sc->sc_current_rx_fifo, buf_res.buffer, buf_res.length / 4);
+
+                       offset += buf_res.length;
+                       sc->sc_current_rx_fifo += buf_res.length;
+                       sc->sc_current_rx_bytes -= buf_res.length;
+                       temp -= buf_res.length;
+               } while (temp != 0);
+       }
+
+       /* check for remainder */
+       if (count != 0) {
+               /* read data into bounce buffer */
+               bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
+                       sc->sc_current_rx_fifo,
+                       sc->sc_bounce_buffer, (count + 3) / 4);
+
+               /* store data into proper buffer */
+               usbd_copy_in(pc, offset, sc->sc_bounce_buffer, count);
+
+               /* round length up to nearest 4 bytes */
+               count = (count + 3) & ~3;
+
+               /* update counters */
+               sc->sc_current_rx_bytes -= count;
+               sc->sc_current_rx_fifo += count;
+       }
+}
+
 static void
 dwc_otg_tx_fifo_reset(struct dwc_otg_softc *sc, uint32_t value)
 {
@@ -579,6 +683,14 @@ dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
        sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
        DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 
+       if (sc->sc_current_rx_bytes != 0) {
+               /* need to dump remaining data */
+               bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
+                   sc->sc_current_rx_fifo, sc->sc_bounce_buffer,
+                   sc->sc_current_rx_bytes / 4);
+               /* clear number of active bytes to receive */
+               sc->sc_current_rx_bytes = 0;
+       }
        /* clear cached status */
        sc->sc_last_rx_status = 0;
 }
@@ -761,6 +873,7 @@ dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
                    td->channel[x] != GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status))
                        continue;
                dwc_otg_common_rx_ack(sc);
+               break;
        }
 }
 
@@ -898,6 +1011,7 @@ send_pkt:
                td->state = DWC_CHAN_ST_WAIT_ANE;
        }
 
+       /* copy out control request */
        usbd_copy_out(td->pc, 0, &req, sizeof(req));
 
        DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
@@ -1024,11 +1138,11 @@ dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
                goto not_complete;
        }
 
-       /* copy in control request */
-       memcpy(&req, sc->sc_rx_bounce_buffer, sizeof(req));
+       /* read FIFO */
+       dwc_otg_read_fifo(sc, td->pc, 0, sizeof(req));
 
-       /* copy data into real buffer */
-       usbd_copy_in(td->pc, 0, &req, sizeof(req));
+       /* copy out control request */
+       usbd_copy_out(td->pc, 0, &req, sizeof(req));
 
        td->offset = sizeof(req);
        td->remainder = 0;
@@ -1212,8 +1326,8 @@ dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td,
                        goto complete;
                }
 
-               usbd_copy_in(td->pc, td->offset,
-                   sc->sc_rx_bounce_buffer, count);
+               /* read data from FIFO */
+               dwc_otg_read_fifo(sc, td->pc, td->offset, count);
 
                td->remainder -= count;
                td->offset += count;
@@ -1583,7 +1697,9 @@ dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
                return (0);             /* we are complete */
        }
 
-       usbd_copy_in(td->pc, td->offset, sc->sc_rx_bounce_buffer, count);
+       /* read data from FIFO */
+       dwc_otg_read_fifo(sc, td->pc, td->offset, count);
+
        td->remainder -= count;
        td->offset += count;
 
@@ -1904,17 +2020,9 @@ send_pkt:
                DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
 
                if (count != 0) {
-                       /* clear topmost word before copy */
-                       sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
-
-                       /* copy out data */
-                       usbd_copy_out(td->pc, td->offset + td->tx_bytes,
-                           sc->sc_tx_bounce_buffer, count);
-
-                       /* transfer data into FIFO */
-                       bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
-                           DOTG_DFIFO(channel),
-                           sc->sc_tx_bounce_buffer, (count + 3) / 4);
+                       /* write data into FIFO */
+                       dwc_otg_write_fifo(sc, td->pc, td->offset +
+                           td->tx_bytes, DOTG_DFIFO(channel), count);
                }
 
                /* store number of bytes transmitted */
@@ -2054,18 +2162,9 @@ repeat:
                        count = fifo_left;
 
                if (count != 0) {
-
-                       /* clear topmost word before copy */
-                       sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
-
-                       /* copy out data */
-                       usbd_copy_out(td->pc, td->offset,
-                           sc->sc_tx_bounce_buffer, count);
-
-                       /* transfer data into FIFO */
-                       bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
-                           DOTG_DFIFO(td->ep_no),
-                           sc->sc_tx_bounce_buffer, (count + 3) / 4);
+                       /* write data into FIFO */
+                       dwc_otg_write_fifo(sc, td->pc, td->offset,
+                           DOTG_DFIFO(td->ep_no), count);
 
                        td->tx_bytes -= count;
                        td->remainder -= count;
@@ -2616,6 +2715,11 @@ repeat:
                                        sc->sc_chan_state[ep_no].wait_halted = 0;
                                        DPRINTFN(5, "channel halt complete ch=%u\n", ep_no);
                                }
+                               /* store bytes and FIFO offset */
+                               sc->sc_current_rx_bytes = 0;
+                               sc->sc_current_rx_fifo = 0;
+
+                               /* acknowledge status */
                                dwc_otg_common_rx_ack(sc);
                                goto repeat;
                        }
@@ -2625,13 +2729,11 @@ repeat:
                        ep_no = GRXSTSRD_CHNUM_GET(
                            sc->sc_last_rx_status);
 
-                       /* receive data, if any */
-                       if (temp != 0) {
-                               DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
-                               bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
-                                   DOTG_DFIFO(ep_no),
-                                   sc->sc_rx_bounce_buffer, (temp + 3) / 4);
-                       }
+                       /* store bytes and FIFO offset */
+                       sc->sc_current_rx_bytes = (temp + 3) & ~3;
+                       sc->sc_current_rx_fifo = DOTG_DFIFO(ep_no);
+
+                       DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
 
                        /* check if we should dump the data */
                        if (!(sc->sc_active_rx_ep & (1U << ep_no))) {

diff --git a/sys/dev/usb/controller/dwc_otg.h b/sys/dev/usb/controller/dwc_otg.h
index f5e9887ac37bfb2230e7ba26a7bc62397684ca60..038b072aeef70fa28cf162c2ddc8d24392188145 100644 (file)
--- a/sys/dev/usb/controller/dwc_otg.h
+++ b/sys/dev/usb/controller/dwc_otg.h
@@ -173,8 +173,7 @@ struct dwc_otg_softc {
        bus_space_tag_t sc_io_tag;
        bus_space_handle_t sc_io_hdl;
 
-       uint32_t sc_rx_bounce_buffer[1024 / 4];
-       uint32_t sc_tx_bounce_buffer[MAX(512 * DWC_OTG_MAX_TXP, 1024) / 4];
+       uint32_t sc_bounce_buffer[MAX(512 * DWC_OTG_MAX_TXP, 1024) / 4];
 
        uint32_t sc_fifo_size;
        uint32_t sc_irq_mask;
@@ -186,6 +185,9 @@ struct dwc_otg_softc {
        uint32_t sc_hprt_val;
        uint32_t sc_xfer_complete;
 
+       uint16_t sc_current_rx_bytes;
+       uint16_t sc_current_rx_fifo;
+
        uint16_t sc_active_rx_ep;
        uint16_t sc_last_frame_num;