In all the places that we use the polled for completion interface, except crash

dump support code, move the while loop into an inline function. These aren't
done in the fast path, so if the compiler choses to not inline, any performance
hit is tiny.

diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c
index 5bc98160011d7718ff63ac207913bd8b803c6438..2a13f2f223e9417330c1972d934e1e0f561a7d99 100644 (file)
--- a/sys/dev/nvme/nvme_ctrlr.c
+++ b/sys/dev/nvme/nvme_ctrlr.c
@@ -394,8 +394,7 @@ nvme_ctrlr_identify(struct nvme_controller *ctrlr)
        status.done = 0;
        nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,
            nvme_completion_poll_cb, &status);
-       while (!atomic_load_acq_int(&status.done))
-               pause("nvme", 1);
+       nvme_completion_poll(&status);
        if (nvme_completion_is_error(&status.cpl)) {
                nvme_printf(ctrlr, "nvme_identify_controller failed!\n");
                return (ENXIO);
@@ -424,8 +423,7 @@ nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr)
        status.done = 0;
        nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues,
            nvme_completion_poll_cb, &status);
-       while (!atomic_load_acq_int(&status.done))
-               pause("nvme", 1);
+       nvme_completion_poll(&status);
        if (nvme_completion_is_error(&status.cpl)) {
                nvme_printf(ctrlr, "nvme_ctrlr_set_num_qpairs failed!\n");
                return (ENXIO);
@@ -463,8 +461,7 @@ nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr)
                status.done = 0;
                nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair, qpair->vector,
                    nvme_completion_poll_cb, &status);
-               while (!atomic_load_acq_int(&status.done))
-                       pause("nvme", 1);
+               nvme_completion_poll(&status);
                if (nvme_completion_is_error(&status.cpl)) {
                        nvme_printf(ctrlr, "nvme_create_io_cq failed!\n");
                        return (ENXIO);
@@ -473,8 +470,7 @@ nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr)
                status.done = 0;
                nvme_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair,
                    nvme_completion_poll_cb, &status);
-               while (!atomic_load_acq_int(&status.done))
-                       pause("nvme", 1);
+               nvme_completion_poll(&status);
                if (nvme_completion_is_error(&status.cpl)) {
                        nvme_printf(ctrlr, "nvme_create_io_sq failed!\n");
                        return (ENXIO);
@@ -496,8 +492,7 @@ nvme_ctrlr_destroy_qpairs(struct nvme_controller *ctrlr)
                status.done = 0;
                nvme_ctrlr_cmd_delete_io_sq(ctrlr, qpair,
                    nvme_completion_poll_cb, &status);
-               while (!atomic_load_acq_int(&status.done))
-                       pause("nvme", 1);
+               nvme_completion_poll(&status);
                if (nvme_completion_is_error(&status.cpl)) {
                        nvme_printf(ctrlr, "nvme_destroy_io_sq failed!\n");
                        return (ENXIO);
@@ -506,8 +501,7 @@ nvme_ctrlr_destroy_qpairs(struct nvme_controller *ctrlr)
                status.done = 0;
                nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair,
                    nvme_completion_poll_cb, &status);
-               while (!atomic_load_acq_int(&status.done))
-                       pause("nvme", 1);
+               nvme_completion_poll(&status);
                if (nvme_completion_is_error(&status.cpl)) {
                        nvme_printf(ctrlr, "nvme_destroy_io_cq failed!\n");
                        return (ENXIO);
@@ -789,8 +783,7 @@ nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)
        status.done = 0;
        nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD,
            0, NULL, 0, nvme_completion_poll_cb, &status);
-       while (!atomic_load_acq_int(&status.done))
-               pause("nvme", 1);
+       nvme_completion_poll(&status);
        if (nvme_completion_is_error(&status.cpl) ||
            (status.cpl.cdw0 & 0xFFFF) == 0xFFFF ||
            (status.cpl.cdw0 & 0xFFFF) == 0x0000) {

diff --git a/sys/dev/nvme/nvme_ns.c b/sys/dev/nvme/nvme_ns.c
index 8eff0b4802cd44b3d9c4783c7a095919792b8fa9..006d059257c4509591d50c054ba043eb1bfcf233 100644 (file)
--- a/sys/dev/nvme/nvme_ns.c
+++ b/sys/dev/nvme/nvme_ns.c
@@ -530,8 +530,7 @@ nvme_ns_construct(struct nvme_namespace *ns, uint32_t id,
        status.done = 0;
        nvme_ctrlr_cmd_identify_namespace(ctrlr, id, &ns->data,
            nvme_completion_poll_cb, &status);
-       while (!atomic_load_acq_int(&status.done))
-               pause("nvme", 1);
+       nvme_completion_poll(&status);
        if (nvme_completion_is_error(&status.cpl)) {
                nvme_printf(ctrlr, "nvme_identify_namespace failed\n");
                return (ENXIO);

diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h
index 910c579a94e2ffa68b7a1ef96ad8c138310e58c7..57de5926d32f85aec874785f7ba5b2088fbd9a68 100644 (file)
--- a/sys/dev/nvme/nvme_private.h
+++ b/sys/dev/nvme/nvme_private.h
@@ -446,6 +446,14 @@ int        nvme_attach(device_t dev);
 int    nvme_shutdown(device_t dev);
 int    nvme_detach(device_t dev);
 
+static __inline
+void
+nvme_completion_poll(struct nvme_completion_poll_status *status)
+{
+       while (!atomic_load_acq_int(&status->done))
+               pause("nvme", 1);
+}
+
 static __inline void
 nvme_single_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
 {