aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/md/dm.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/md/dm.c')
-rw-r--r--drivers/md/dm.c93
1 files changed, 81 insertions, 12 deletions
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 874bd542a744..9a9b2adcf39e 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -146,6 +146,16 @@ EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
#define DM_NUMA_NODE NUMA_NO_NODE
static int dm_numa_node = DM_NUMA_NODE;
+#define DEFAULT_SWAP_BIOS (8 * 1048576 / PAGE_SIZE)
+static int swap_bios = DEFAULT_SWAP_BIOS;
+static int get_swap_bios(void)
+{
+ int latch = READ_ONCE(swap_bios);
+ if (unlikely(latch <= 0))
+ latch = DEFAULT_SWAP_BIOS;
+ return latch;
+}
+
/*
* For mempools pre-allocation at the table loading time.
*/
@@ -269,7 +279,6 @@ out_free_rq_tio_cache:
static void local_exit(void)
{
- flush_scheduled_work();
destroy_workqueue(deferred_remove_workqueue);
kmem_cache_destroy(_rq_cache);
@@ -462,7 +471,6 @@ static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
struct block_device **bdev)
- __acquires(md->io_barrier)
{
struct dm_target *tgt;
struct dm_table *map;
@@ -496,7 +504,6 @@ retry:
}
static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
- __releases(md->io_barrier)
{
dm_put_live_table(md, srcu_idx);
}
@@ -517,7 +524,7 @@ static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
* subset of the parent bdev; require extra privileges.
*/
if (!capable(CAP_SYS_RAWIO)) {
- DMWARN_LIMIT(
+ DMDEBUG_LIMIT(
"%s: sending ioctl %x to DM device without required privilege.",
current->comm, cmd);
r = -ENOIOCTLCMD;
@@ -623,21 +630,20 @@ static void start_io_acct(struct dm_io *io)
false, 0, &io->stats_aux);
}
-static void end_io_acct(struct dm_io *io)
+static void end_io_acct(struct mapped_device *md, struct bio *bio,
+ unsigned long start_time, struct dm_stats_aux *stats_aux)
{
- struct mapped_device *md = io->md;
- struct bio *bio = io->orig_bio;
- unsigned long duration = jiffies - io->start_time;
+ unsigned long duration = jiffies - start_time;
int pending;
int rw = bio_data_dir(bio);
generic_end_io_acct(md->queue, bio_op(bio), &dm_disk(md)->part0,
- io->start_time);
+ start_time);
if (unlikely(dm_stats_used(&md->stats)))
dm_stats_account_io(&md->stats, bio_data_dir(bio),
bio->bi_iter.bi_sector, bio_sectors(bio),
- true, duration, &io->stats_aux);
+ true, duration, stats_aux);
/*
* After this is decremented the bio must not be touched if it is
@@ -864,6 +870,8 @@ static void dec_pending(struct dm_io *io, blk_status_t error)
blk_status_t io_error;
struct bio *bio;
struct mapped_device *md = io->md;
+ unsigned long start_time = 0;
+ struct dm_stats_aux stats_aux;
/* Push-back supersedes any I/O errors */
if (unlikely(error)) {
@@ -890,8 +898,10 @@ static void dec_pending(struct dm_io *io, blk_status_t error)
io_error = io->status;
bio = io->orig_bio;
- end_io_acct(io);
+ start_time = io->start_time;
+ stats_aux = io->stats_aux;
free_io(md, io);
+ end_io_acct(md, bio, start_time, &stats_aux);
if (io_error == BLK_STS_DM_REQUEUE)
return;
@@ -937,6 +947,11 @@ void disable_write_zeroes(struct mapped_device *md)
limits->max_write_zeroes_sectors = 0;
}
+static bool swap_bios_limit(struct dm_target *ti, struct bio *bio)
+{
+ return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios);
+}
+
static void clone_endio(struct bio *bio)
{
blk_status_t error = bio->bi_status;
@@ -974,6 +989,11 @@ static void clone_endio(struct bio *bio)
}
}
+ if (unlikely(swap_bios_limit(tio->ti, bio))) {
+ struct mapped_device *md = io->md;
+ up(&md->swap_bios_semaphore);
+ }
+
free_tio(tio);
dec_pending(io, error);
}
@@ -1252,6 +1272,22 @@ void dm_remap_zone_report(struct dm_target *ti, struct bio *bio, sector_t start)
}
EXPORT_SYMBOL_GPL(dm_remap_zone_report);
+static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch)
+{
+ mutex_lock(&md->swap_bios_lock);
+ while (latch < md->swap_bios) {
+ cond_resched();
+ down(&md->swap_bios_semaphore);
+ md->swap_bios--;
+ }
+ while (latch > md->swap_bios) {
+ cond_resched();
+ up(&md->swap_bios_semaphore);
+ md->swap_bios++;
+ }
+ mutex_unlock(&md->swap_bios_lock);
+}
+
static blk_qc_t __map_bio(struct dm_target_io *tio)
{
int r;
@@ -1272,6 +1308,14 @@ static blk_qc_t __map_bio(struct dm_target_io *tio)
atomic_inc(&io->io_count);
sector = clone->bi_iter.bi_sector;
+ if (unlikely(swap_bios_limit(ti, clone))) {
+ struct mapped_device *md = io->md;
+ int latch = get_swap_bios();
+ if (unlikely(latch != md->swap_bios))
+ __set_swap_bios_limit(md, latch);
+ down(&md->swap_bios_semaphore);
+ }
+
r = ti->type->map(ti, clone);
switch (r) {
case DM_MAPIO_SUBMITTED:
@@ -1286,10 +1330,18 @@ static blk_qc_t __map_bio(struct dm_target_io *tio)
ret = generic_make_request(clone);
break;
case DM_MAPIO_KILL:
+ if (unlikely(swap_bios_limit(ti, clone))) {
+ struct mapped_device *md = io->md;
+ up(&md->swap_bios_semaphore);
+ }
free_tio(tio);
dec_pending(io, BLK_STS_IOERR);
break;
case DM_MAPIO_REQUEUE:
+ if (unlikely(swap_bios_limit(ti, clone))) {
+ struct mapped_device *md = io->md;
+ up(&md->swap_bios_semaphore);
+ }
free_tio(tio);
dec_pending(io, BLK_STS_DM_REQUEUE);
break;
@@ -1861,6 +1913,7 @@ static void cleanup_mapped_device(struct mapped_device *md)
mutex_destroy(&md->suspend_lock);
mutex_destroy(&md->type_lock);
mutex_destroy(&md->table_devices_lock);
+ mutex_destroy(&md->swap_bios_lock);
dm_mq_cleanup_mapped_device(md);
}
@@ -1935,6 +1988,10 @@ static struct mapped_device *alloc_dev(int minor)
init_completion(&md->kobj_holder.completion);
md->kworker_task = NULL;
+ md->swap_bios = get_swap_bios();
+ sema_init(&md->swap_bios_semaphore, md->swap_bios);
+ mutex_init(&md->swap_bios_lock);
+
md->disk->major = _major;
md->disk->first_minor = minor;
md->disk->fops = &dm_blk_dops;
@@ -1964,7 +2021,9 @@ static struct mapped_device *alloc_dev(int minor)
bio_set_dev(&md->flush_bio, md->bdev);
md->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
- dm_stats_init(&md->stats);
+ r = dm_stats_init(&md->stats);
+ if (r < 0)
+ goto bad;
/* Populate the mapping, nobody knows we exist yet */
spin_lock(&_minor_lock);
@@ -2417,6 +2476,8 @@ static int dm_wait_for_completion(struct mapped_device *md, long task_state)
}
finish_wait(&md->wait, &wait);
+ smp_rmb(); /* paired with atomic_dec_return in end_io_acct */
+
return r;
}
@@ -3062,6 +3123,11 @@ static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
goto out;
ti = dm_table_get_target(table, 0);
+ if (dm_suspended_md(md)) {
+ ret = -EAGAIN;
+ goto out;
+ }
+
ret = -EINVAL;
if (!ti->type->iterate_devices)
goto out;
@@ -3230,6 +3296,9 @@ MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
+module_param(swap_bios, int, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs");
+
MODULE_DESCRIPTION(DM_NAME " driver");
MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
MODULE_LICENSE("GPL");
generated by cgit 1.2.3-korg (git 2.39.0) at 2024-05-24 07:29:18 +0000