dm io: Support IO priority

[ Upstream commit 6e5f0f6383b4896c7e9b943d84b136149d0f45e9 ]

Some IO will dispatch from kworker with different io_context settings
than the submitting task, we may need to specify a priority to avoid
losing priority.

Add IO priority parameter to dm_io() and update all callers.

Co-developed-by: Yibin Ding <yibin.ding@unisoc.com>
Signed-off-by: Yibin Ding <yibin.ding@unisoc.com>
Signed-off-by: Hongyu Jin <hongyu.jin@unisoc.com>
Reviewed-by: Eric Biggers <ebiggers@google.com>
Reviewed-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Mike Snitzer <snitzer@kernel.org>
Stable-dep-of: b4d78cfeb304 ("dm-integrity: align the outgoing bio in integrity_recheck")
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index 100a6a236d92ad8baf5838ddc426fd9b263c4fd2..ec662f97ba82881437eac5d6ca4509a633347518 100644 (file)
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -614,7 +614,7 @@ static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector,
                io_req.mem.ptr.vma = (char *)b->data + offset;
        }
 
-       r = dm_io(&io_req, 1, &region, NULL);
+       r = dm_io(&io_req, 1, &region, NULL, IOPRIO_DEFAULT);
        if (unlikely(r))
                b->end_io(b, errno_to_blk_status(r));
 }
@@ -1375,7 +1375,7 @@ int dm_bufio_issue_flush(struct dm_bufio_client *c)
 
        BUG_ON(dm_bufio_in_request());
 
-       return dm_io(&io_req, 1, &io_reg, NULL);
+       return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_issue_flush);
 
@@ -1398,7 +1398,7 @@ int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t c
 
        BUG_ON(dm_bufio_in_request());
 
-       return dm_io(&io_req, 1, &io_reg, NULL);
+       return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_issue_discard);
 

diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c
index 94382e43ea50617cba5b46d181f8fb2454739c16..aff818eb31fbbc9c269dd56abd0417db43eac161 100644 (file)
--- a/drivers/md/dm-integrity.c
+++ b/drivers/md/dm-integrity.c
@@ -579,7 +579,7 @@ static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf)
                }
        }
 
-       r = dm_io(&io_req, 1, &io_loc, NULL);
+       r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
        if (unlikely(r))
                return r;
 
@@ -1089,7 +1089,7 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf,
        io_loc.sector = ic->start + SB_SECTORS + sector;
        io_loc.count = n_sectors;
 
-       r = dm_io(&io_req, 1, &io_loc, NULL);
+       r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
        if (unlikely(r)) {
                dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ?
                                      "reading journal" : "writing journal", r);
@@ -1205,7 +1205,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, u
        io_loc.sector = target;
        io_loc.count = n_sectors;
 
-       r = dm_io(&io_req, 1, &io_loc, NULL);
+       r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
        if (unlikely(r)) {
                WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
                fn(-1UL, data);
@@ -1532,7 +1532,7 @@ static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_dat
                fr.io_reg.count = 0,
                fr.ic = ic;
                init_completion(&fr.comp);
-               r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL);
+               r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT);
                BUG_ON(r);
        }
 
@@ -1737,7 +1737,7 @@ static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checks
                        io_loc.sector = sector;
                        io_loc.count = ic->sectors_per_block;
 
-                       r = dm_io(&io_req, 1, &io_loc, NULL);
+                       r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
                        if (unlikely(r)) {
                                dio->bi_status = errno_to_blk_status(r);
                                goto free_ret;
@@ -2774,7 +2774,7 @@ next_chunk:
        io_loc.sector = get_data_sector(ic, area, offset);
        io_loc.count = n_sectors;
 
-       r = dm_io(&io_req, 1, &io_loc, NULL);
+       r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
        if (unlikely(r)) {
                dm_integrity_io_error(ic, "reading data", r);
                goto err;

diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c
index e488b05e35fa39767ef8c06db79d0ad9c847f992..ec97658387c39d80dd71c4eef562007544d2fdbb 100644 (file)
--- a/drivers/md/dm-io.c
+++ b/drivers/md/dm-io.c
@@ -295,7 +295,7 @@ static void km_dp_init(struct dpages *dp, void *data)
  *---------------------------------------------------------------*/
 static void do_region(const blk_opf_t opf, unsigned int region,
                      struct dm_io_region *where, struct dpages *dp,
-                     struct io *io)
+                     struct io *io, unsigned short ioprio)
 {
        struct bio *bio;
        struct page *page;
@@ -344,6 +344,7 @@ static void do_region(const blk_opf_t opf, unsigned int region,
                                       &io->client->bios);
                bio->bi_iter.bi_sector = where->sector + (where->count - remaining);
                bio->bi_end_io = endio;
+               bio->bi_ioprio = ioprio;
                store_io_and_region_in_bio(bio, io, region);
 
                if (op == REQ_OP_DISCARD || op == REQ_OP_WRITE_ZEROES) {
@@ -371,7 +372,7 @@ static void do_region(const blk_opf_t opf, unsigned int region,
 
 static void dispatch_io(blk_opf_t opf, unsigned int num_regions,
                        struct dm_io_region *where, struct dpages *dp,
-                       struct io *io, int sync)
+                       struct io *io, int sync, unsigned short ioprio)
 {
        int i;
        struct dpages old_pages = *dp;
@@ -388,7 +389,7 @@ static void dispatch_io(blk_opf_t opf, unsigned int num_regions,
        for (i = 0; i < num_regions; i++) {
                *dp = old_pages;
                if (where[i].count || (opf & REQ_PREFLUSH))
-                       do_region(opf, i, where + i, dp, io);
+                       do_region(opf, i, where + i, dp, io, ioprio);
        }
 
        /*
@@ -413,7 +414,7 @@ static void sync_io_complete(unsigned long error, void *context)
 
 static int sync_io(struct dm_io_client *client, unsigned int num_regions,
                   struct dm_io_region *where, blk_opf_t opf, struct dpages *dp,
-                  unsigned long *error_bits)
+                  unsigned long *error_bits, unsigned short ioprio)
 {
        struct io *io;
        struct sync_io sio;
@@ -435,7 +436,7 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions,
        io->vma_invalidate_address = dp->vma_invalidate_address;
        io->vma_invalidate_size = dp->vma_invalidate_size;
 
-       dispatch_io(opf, num_regions, where, dp, io, 1);
+       dispatch_io(opf, num_regions, where, dp, io, 1, ioprio);
 
        wait_for_completion_io(&sio.wait);
 
@@ -447,7 +448,8 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions,
 
 static int async_io(struct dm_io_client *client, unsigned int num_regions,
                    struct dm_io_region *where, blk_opf_t opf,
-                   struct dpages *dp, io_notify_fn fn, void *context)
+                   struct dpages *dp, io_notify_fn fn, void *context,
+                   unsigned short ioprio)
 {
        struct io *io;
 
@@ -467,7 +469,7 @@ static int async_io(struct dm_io_client *client, unsigned int num_regions,
        io->vma_invalidate_address = dp->vma_invalidate_address;
        io->vma_invalidate_size = dp->vma_invalidate_size;
 
-       dispatch_io(opf, num_regions, where, dp, io, 0);
+       dispatch_io(opf, num_regions, where, dp, io, 0, ioprio);
        return 0;
 }
 
@@ -509,7 +511,8 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp,
 }
 
 int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
-         struct dm_io_region *where, unsigned long *sync_error_bits)
+         struct dm_io_region *where, unsigned long *sync_error_bits,
+         unsigned short ioprio)
 {
        int r;
        struct dpages dp;
@@ -520,11 +523,11 @@ int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
 
        if (!io_req->notify.fn)
                return sync_io(io_req->client, num_regions, where,
-                              io_req->bi_opf, &dp, sync_error_bits);
+                              io_req->bi_opf, &dp, sync_error_bits, ioprio);
 
        return async_io(io_req->client, num_regions, where,
                        io_req->bi_opf, &dp, io_req->notify.fn,
-                       io_req->notify.context);
+                       io_req->notify.context, ioprio);
 }
 EXPORT_SYMBOL(dm_io);
 

diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c
index 0ef78e56aa88c6caf8e28db68125690a0557f9a7..fda51bd140ed3d44369911953689eba44e668248 100644 (file)
--- a/drivers/md/dm-kcopyd.c
+++ b/drivers/md/dm-kcopyd.c
@@ -572,9 +572,9 @@ static int run_io_job(struct kcopyd_job *job)
        io_job_start(job->kc->throttle);
 
        if (job->op == REQ_OP_READ)
-               r = dm_io(&io_req, 1, &job->source, NULL);
+               r = dm_io(&io_req, 1, &job->source, NULL, IOPRIO_DEFAULT);
        else
-               r = dm_io(&io_req, job->num_dests, job->dests, NULL);
+               r = dm_io(&io_req, job->num_dests, job->dests, NULL, IOPRIO_DEFAULT);
 
        return r;
 }

diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c
index b7dd5a0cd58ba774cdf895680c045b358ebbaa20..da77878cb2c029570ec37583faeec55455477618 100644 (file)
--- a/drivers/md/dm-log.c
+++ b/drivers/md/dm-log.c
@@ -295,7 +295,7 @@ static int rw_header(struct log_c *lc, enum req_op op)
 {
        lc->io_req.bi_opf = op;
 
-       return dm_io(&lc->io_req, 1, &lc->header_location, NULL);
+       return dm_io(&lc->io_req, 1, &lc->header_location, NULL, IOPRIO_DEFAULT);
 }
 
 static int flush_header(struct log_c *lc)
@@ -308,7 +308,7 @@ static int flush_header(struct log_c *lc)
 
        lc->io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 
-       return dm_io(&lc->io_req, 1, &null_location, NULL);
+       return dm_io(&lc->io_req, 1, &null_location, NULL, IOPRIO_DEFAULT);
 }
 
 static int read_header(struct log_c *log)

diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
index 2327645fc0648205d16e4319a292c0a153f6627e..1004199ae77ac8c1fa7ad7431cd4aeb8b87b2ccc 100644 (file)
--- a/drivers/md/dm-raid1.c
+++ b/drivers/md/dm-raid1.c
@@ -273,7 +273,7 @@ static int mirror_flush(struct dm_target *ti)
        }
 
        error_bits = -1;
-       dm_io(&io_req, ms->nr_mirrors, io, &error_bits);
+       dm_io(&io_req, ms->nr_mirrors, io, &error_bits, IOPRIO_DEFAULT);
        if (unlikely(error_bits != 0)) {
                for (i = 0; i < ms->nr_mirrors; i++)
                        if (test_bit(i, &error_bits))
@@ -543,7 +543,7 @@ static void read_async_bio(struct mirror *m, struct bio *bio)
 
        map_region(&io, m, bio);
        bio_set_m(bio, m);
-       BUG_ON(dm_io(&io_req, 1, &io, NULL));
+       BUG_ON(dm_io(&io_req, 1, &io, NULL, IOPRIO_DEFAULT));
 }
 
 static inline int region_in_sync(struct mirror_set *ms, region_t region,
@@ -670,7 +670,7 @@ static void do_write(struct mirror_set *ms, struct bio *bio)
         */
        bio_set_m(bio, get_default_mirror(ms));
 
-       BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL));
+       BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL, IOPRIO_DEFAULT));
 }
 
 static void do_writes(struct mirror_set *ms, struct bio_list *writes)

diff --git a/drivers/md/dm-snap-persistent.c b/drivers/md/dm-snap-persistent.c
index 80b95746a43e066237de3be8a34b1e25f05b9842..eee1cd3aa3fcf2fe302bf1e2fc9815df345f63f3 100644 (file)
--- a/drivers/md/dm-snap-persistent.c
+++ b/drivers/md/dm-snap-persistent.c
@@ -220,7 +220,7 @@ static void do_metadata(struct work_struct *work)
 {
        struct mdata_req *req = container_of(work, struct mdata_req, work);
 
-       req->result = dm_io(req->io_req, 1, req->where, NULL);
+       req->result = dm_io(req->io_req, 1, req->where, NULL, IOPRIO_DEFAULT);
 }
 
 /*
@@ -244,7 +244,7 @@ static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, blk_opf_t opf,
        struct mdata_req req;
 
        if (!metadata)
-               return dm_io(&io_req, 1, &where, NULL);
+               return dm_io(&io_req, 1, &where, NULL, IOPRIO_DEFAULT);
 
        req.where = &where;
        req.io_req = &io_req;

diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c
index b48e1b59e6da424327024cda97c13de5a35611e1..6a707b41dc865f62cafa56228d915accacecb764 100644 (file)
--- a/drivers/md/dm-verity-target.c
+++ b/drivers/md/dm-verity-target.c
@@ -503,7 +503,7 @@ static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io,
        io_loc.bdev = v->data_dev->bdev;
        io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT);
        io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT);
-       r = dm_io(&io_req, 1, &io_loc, NULL);
+       r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
        if (unlikely(r))
                goto free_ret;
 

diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c
index a705e24d3e2b6ebefa5f781d927cbe36b5009b14..20fc84b24fc75f36782616d248fa7661113b8584 100644 (file)
--- a/drivers/md/dm-writecache.c
+++ b/drivers/md/dm-writecache.c
@@ -531,7 +531,7 @@ static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
                req.notify.context = &endio;
 
                /* writing via async dm-io (implied by notify.fn above) won't return an error */
-               (void) dm_io(&req, 1, &region, NULL);
+               (void) dm_io(&req, 1, &region, NULL, IOPRIO_DEFAULT);
                i = j;
        }
 
@@ -568,7 +568,7 @@ static void ssd_commit_superblock(struct dm_writecache *wc)
        req.notify.fn = NULL;
        req.notify.context = NULL;
 
-       r = dm_io(&req, 1, &region, NULL);
+       r = dm_io(&req, 1, &region, NULL, IOPRIO_DEFAULT);
        if (unlikely(r))
                writecache_error(wc, r, "error writing superblock");
 }
@@ -596,7 +596,7 @@ static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev)
        req.client = wc->dm_io;
        req.notify.fn = NULL;
 
-       r = dm_io(&req, 1, &region, NULL);
+       r = dm_io(&req, 1, &region, NULL, IOPRIO_DEFAULT);
        if (unlikely(r))
                writecache_error(wc, r, "error flushing metadata: %d", r);
 }
@@ -984,7 +984,7 @@ static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors
        req.client = wc->dm_io;
        req.notify.fn = NULL;
 
-       return dm_io(&req, 1, &region, NULL);
+       return dm_io(&req, 1, &region, NULL, IOPRIO_DEFAULT);
 }
 
 static void writecache_resume(struct dm_target *ti)

diff --git a/include/linux/dm-io.h b/include/linux/dm-io.h
index 92e7abfe04f9299fe2459696ddce861498f5baa9..70b3737052dd241f4241c357c5c09666e988fcb9 100644 (file)
--- a/include/linux/dm-io.h
+++ b/include/linux/dm-io.h
@@ -79,7 +79,8 @@ void dm_io_client_destroy(struct dm_io_client *client);
  * error occurred doing io to the corresponding region.
  */
 int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
-         struct dm_io_region *region, unsigned int long *sync_error_bits);
+         struct dm_io_region *region, unsigned int long *sync_error_bits,
+         unsigned short ioprio);
 
 #endif /* __KERNEL__ */
 #endif /* _LINUX_DM_IO_H */