diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2013-09-10 13:06:15 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2013-09-10 13:06:15 -0700 |
commit | 7426d62871dafbeeed087d609c6469a515c88389 (patch) | |
tree | 7d935f360eeb5e78ba633238a29e9213c291aad7 /drivers/md | |
parent | 4d7696f1b05f4aeb586c74868fe3da2731daca4b (diff) | |
parent | 7fff5e8f727285cf54e6aba10f31b196f207b98a (diff) |
Merge tag 'dm-3.12-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm
Pull device-mapper updates from Mike Snitzer:
"Add the ability to collect I/O statistics on user-defined regions of a
device-mapper device. This dm-stats code required the reintroduction
of a div64_u64_rem() helper, but as a separate method that doesn't
slow down div64_u64() -- especially on 32-bit systems.
Allow the error target to replace request-based DM devices (e.g.
multipath) in addition to bio-based DM devices.
Various other small code fixes and improvements to thin-provisioning,
DM cache and the DM ioctl interface"
* tag 'dm-3.12-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm:
dm stripe: silence a couple sparse warnings
dm: add statistics support
dm thin: always return -ENOSPC if no_free_space is set
dm ioctl: cleanup error handling in table_load
dm ioctl: increase granularity of type_lock when loading table
dm ioctl: prevent rename to empty name or uuid
dm thin: set pool read-only if breaking_sharing fails block allocation
dm thin: prefix pool error messages with pool device name
dm: allow error target to replace bio-based and request-based targets
math64: New separate div64_u64_rem helper
dm space map: optimise sm_ll_dec and sm_ll_inc
dm btree: prefetch child nodes when walking tree for a dm_btree_del
dm btree: use pop_frame in dm_btree_del to cleanup code
dm cache: eliminate holes in cache structure
dm cache: fix stacking of geometry limits
dm thin: fix stacking of geometry limits
dm thin: add data block size limits to Documentation
dm cache: add data block size limits to code and Documentation
dm cache: document metadata device is exclussive to a cache
dm: stop using WQ_NON_REENTRANT
Diffstat (limited to 'drivers/md')
-rw-r--r-- | drivers/md/Makefile | 2 | ||||
-rw-r--r-- | drivers/md/dm-cache-target.c | 59 | ||||
-rw-r--r-- | drivers/md/dm-crypt.c | 10 | ||||
-rw-r--r-- | drivers/md/dm-ioctl.c | 60 | ||||
-rw-r--r-- | drivers/md/dm-kcopyd.c | 3 | ||||
-rw-r--r-- | drivers/md/dm-raid1.c | 3 | ||||
-rw-r--r-- | drivers/md/dm-stats.c | 969 | ||||
-rw-r--r-- | drivers/md/dm-stats.h | 40 | ||||
-rw-r--r-- | drivers/md/dm-stripe.c | 1 | ||||
-rw-r--r-- | drivers/md/dm-table.c | 20 | ||||
-rw-r--r-- | drivers/md/dm-target.c | 9 | ||||
-rw-r--r-- | drivers/md/dm-thin.c | 122 | ||||
-rw-r--r-- | drivers/md/dm.c | 70 | ||||
-rw-r--r-- | drivers/md/dm.h | 27 | ||||
-rw-r--r-- | drivers/md/persistent-data/dm-block-manager.c | 5 | ||||
-rw-r--r-- | drivers/md/persistent-data/dm-block-manager.h | 5 | ||||
-rw-r--r-- | drivers/md/persistent-data/dm-btree.c | 28 | ||||
-rw-r--r-- | drivers/md/persistent-data/dm-space-map-common.c | 77 |
18 files changed, 1359 insertions, 151 deletions
diff --git a/drivers/md/Makefile b/drivers/md/Makefile index 5ef78efc27f..2acc43fe022 100644 --- a/drivers/md/Makefile +++ b/drivers/md/Makefile @@ -3,7 +3,7 @@ # dm-mod-y += dm.o dm-table.o dm-target.o dm-linear.o dm-stripe.o \ - dm-ioctl.o dm-io.o dm-kcopyd.o dm-sysfs.o + dm-ioctl.o dm-io.o dm-kcopyd.o dm-sysfs.o dm-stats.o dm-multipath-y += dm-path-selector.o dm-mpath.o dm-snapshot-y += dm-snap.o dm-exception-store.o dm-snap-transient.o \ dm-snap-persistent.o diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c index 0df3ec085eb..29569768ffb 100644 --- a/drivers/md/dm-cache-target.c +++ b/drivers/md/dm-cache-target.c @@ -67,9 +67,11 @@ static void free_bitset(unsigned long *bits) #define MIGRATION_COUNT_WINDOW 10 /* - * The block size of the device holding cache data must be >= 32KB + * The block size of the device holding cache data must be + * between 32KB and 1GB. */ #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT) +#define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT) /* * FIXME: the cache is read/write for the time being. @@ -101,6 +103,8 @@ struct cache { struct dm_target *ti; struct dm_target_callbacks callbacks; + struct dm_cache_metadata *cmd; + /* * Metadata is written to this device. */ @@ -117,11 +121,6 @@ struct cache { struct dm_dev *cache_dev; /* - * Cache features such as write-through. - */ - struct cache_features features; - - /* * Size of the origin device in _complete_ blocks and native sectors. */ dm_oblock_t origin_blocks; @@ -138,8 +137,6 @@ struct cache { uint32_t sectors_per_block; int sectors_per_block_shift; - struct dm_cache_metadata *cmd; - spinlock_t lock; struct bio_list deferred_bios; struct bio_list deferred_flush_bios; @@ -148,8 +145,8 @@ struct cache { struct list_head completed_migrations; struct list_head need_commit_migrations; sector_t migration_threshold; - atomic_t nr_migrations; wait_queue_head_t migration_wait; + atomic_t nr_migrations; /* * cache_size entries, dirty if set @@ -160,9 +157,16 @@ struct cache { /* * origin_blocks entries, discarded if set. */ - uint32_t discard_block_size; /* a power of 2 times sectors per block */ dm_dblock_t discard_nr_blocks; unsigned long *discard_bitset; + uint32_t discard_block_size; /* a power of 2 times sectors per block */ + + /* + * Rather than reconstructing the table line for the status we just + * save it and regurgitate. + */ + unsigned nr_ctr_args; + const char **ctr_args; struct dm_kcopyd_client *copier; struct workqueue_struct *wq; @@ -187,14 +191,12 @@ struct cache { bool loaded_mappings:1; bool loaded_discards:1; - struct cache_stats stats; - /* - * Rather than reconstructing the table line for the status we just - * save it and regurgitate. + * Cache features such as write-through. */ - unsigned nr_ctr_args; - const char **ctr_args; + struct cache_features features; + + struct cache_stats stats; }; struct per_bio_data { @@ -1687,24 +1689,25 @@ static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as, static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as, char **error) { - unsigned long tmp; + unsigned long block_size; if (!at_least_one_arg(as, error)) return -EINVAL; - if (kstrtoul(dm_shift_arg(as), 10, &tmp) || !tmp || - tmp < DATA_DEV_BLOCK_SIZE_MIN_SECTORS || - tmp & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) { + if (kstrtoul(dm_shift_arg(as), 10, &block_size) || !block_size || + block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS || + block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS || + block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) { *error = "Invalid data block size"; return -EINVAL; } - if (tmp > ca->cache_sectors) { + if (block_size > ca->cache_sectors) { *error = "Data block size is larger than the cache device"; return -EINVAL; } - ca->block_size = tmp; + ca->block_size = block_size; return 0; } @@ -2609,9 +2612,17 @@ static void set_discard_limits(struct cache *cache, struct queue_limits *limits) static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits) { struct cache *cache = ti->private; + uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT; - blk_limits_io_min(limits, 0); - blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT); + /* + * If the system-determined stacked limits are compatible with the + * cache's blocksize (io_opt is a factor) do not override them. + */ + if (io_opt_sectors < cache->sectors_per_block || + do_div(io_opt_sectors, cache->sectors_per_block)) { + blk_limits_io_min(limits, 0); + blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT); + } set_discard_limits(cache, limits); } diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 6d2d41ae9e3..0fce0bc1a95 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -1645,20 +1645,14 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) } ret = -ENOMEM; - cc->io_queue = alloc_workqueue("kcryptd_io", - WQ_NON_REENTRANT| - WQ_MEM_RECLAIM, - 1); + cc->io_queue = alloc_workqueue("kcryptd_io", WQ_MEM_RECLAIM, 1); if (!cc->io_queue) { ti->error = "Couldn't create kcryptd io queue"; goto bad; } cc->crypt_queue = alloc_workqueue("kcryptd", - WQ_NON_REENTRANT| - WQ_CPU_INTENSIVE| - WQ_MEM_RECLAIM, - 1); + WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1); if (!cc->crypt_queue) { ti->error = "Couldn't create kcryptd queue"; goto bad; diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index f1b758675ec..afe08146f73 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c @@ -877,7 +877,7 @@ static int dev_rename(struct dm_ioctl *param, size_t param_size) unsigned change_uuid = (param->flags & DM_UUID_FLAG) ? 1 : 0; if (new_data < param->data || - invalid_str(new_data, (void *) param + param_size) || + invalid_str(new_data, (void *) param + param_size) || !*new_data || strlen(new_data) > (change_uuid ? DM_UUID_LEN - 1 : DM_NAME_LEN - 1)) { DMWARN("Invalid new mapped device name or uuid string supplied."); return -EINVAL; @@ -1262,44 +1262,37 @@ static int table_load(struct dm_ioctl *param, size_t param_size) r = dm_table_create(&t, get_mode(param), param->target_count, md); if (r) - goto out; + goto err; + /* Protect md->type and md->queue against concurrent table loads. */ + dm_lock_md_type(md); r = populate_table(t, param, param_size); - if (r) { - dm_table_destroy(t); - goto out; - } + if (r) + goto err_unlock_md_type; immutable_target_type = dm_get_immutable_target_type(md); if (immutable_target_type && (immutable_target_type != dm_table_get_immutable_target_type(t))) { DMWARN("can't replace immutable target type %s", immutable_target_type->name); - dm_table_destroy(t); r = -EINVAL; - goto out; + goto err_unlock_md_type; } - /* Protect md->type and md->queue against concurrent table loads. */ - dm_lock_md_type(md); if (dm_get_md_type(md) == DM_TYPE_NONE) /* Initial table load: acquire type of table. */ dm_set_md_type(md, dm_table_get_type(t)); else if (dm_get_md_type(md) != dm_table_get_type(t)) { DMWARN("can't change device type after initial table load."); - dm_table_destroy(t); - dm_unlock_md_type(md); r = -EINVAL; - goto out; + goto err_unlock_md_type; } /* setup md->queue to reflect md's type (may block) */ r = dm_setup_md_queue(md); if (r) { DMWARN("unable to set up device queue for new table."); - dm_table_destroy(t); - dm_unlock_md_type(md); - goto out; + goto err_unlock_md_type; } dm_unlock_md_type(md); @@ -1309,9 +1302,8 @@ static int table_load(struct dm_ioctl *param, size_t param_size) if (!hc || hc->md != md) { DMWARN("device has been removed from the dev hash table."); up_write(&_hash_lock); - dm_table_destroy(t); r = -ENXIO; - goto out; + goto err_destroy_table; } if (hc->new_map) @@ -1322,7 +1314,6 @@ static int table_load(struct dm_ioctl *param, size_t param_size) param->flags |= DM_INACTIVE_PRESENT_FLAG; __dev_status(md, param); -out: if (old_map) { dm_sync_table(md); dm_table_destroy(old_map); @@ -1330,6 +1321,15 @@ out: dm_put(md); + return 0; + +err_unlock_md_type: + dm_unlock_md_type(md); +err_destroy_table: + dm_table_destroy(t); +err: + dm_put(md); + return r; } @@ -1455,20 +1455,26 @@ static int table_status(struct dm_ioctl *param, size_t param_size) return 0; } -static bool buffer_test_overflow(char *result, unsigned maxlen) -{ - return !maxlen || strlen(result) + 1 >= maxlen; -} - /* - * Process device-mapper dependent messages. + * Process device-mapper dependent messages. Messages prefixed with '@' + * are processed by the DM core. All others are delivered to the target. * Returns a number <= 1 if message was processed by device mapper. * Returns 2 if message should be delivered to the target. */ static int message_for_md(struct mapped_device *md, unsigned argc, char **argv, char *result, unsigned maxlen) { - return 2; + int r; + + if (**argv != '@') + return 2; /* no '@' prefix, deliver to target */ + + r = dm_stats_message(md, argc, argv, result, maxlen); + if (r < 2) + return r; + + DMERR("Unsupported message sent to DM core: %s", argv[0]); + return -EINVAL; } /* @@ -1542,7 +1548,7 @@ static int target_message(struct dm_ioctl *param, size_t param_size) if (r == 1) { param->flags |= DM_DATA_OUT_FLAG; - if (buffer_test_overflow(result, maxlen)) + if (dm_message_test_buffer_overflow(result, maxlen)) param->flags |= DM_BUFFER_FULL_FLAG; else param->data_size = param->data_start + strlen(result) + 1; diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index d581fe5d2fa..3a7cade5e27 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c @@ -833,8 +833,7 @@ struct dm_kcopyd_client *dm_kcopyd_client_create(struct dm_kcopyd_throttle *thro goto bad_slab; INIT_WORK(&kc->kcopyd_work, do_work); - kc->kcopyd_wq = alloc_workqueue("kcopyd", - WQ_NON_REENTRANT | WQ_MEM_RECLAIM, 0); + kc->kcopyd_wq = alloc_workqueue("kcopyd", WQ_MEM_RECLAIM, 0); if (!kc->kcopyd_wq) goto bad_workqueue; diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index 699b5be68d3..9584443c561 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -1080,8 +1080,7 @@ static int mirror_ctr(struct dm_target *ti, unsigned int argc, char **argv) ti->per_bio_data_size = sizeof(struct dm_raid1_bio_record); ti->discard_zeroes_data_unsupported = true; - ms->kmirrord_wq = alloc_workqueue("kmirrord", - WQ_NON_REENTRANT | WQ_MEM_RECLAIM, 0); + ms->kmirrord_wq = alloc_workqueue("kmirrord", WQ_MEM_RECLAIM, 0); if (!ms->kmirrord_wq) { DMERR("couldn't start kmirrord"); r = -ENOMEM; diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c new file mode 100644 index 00000000000..8ae31e8d3d6 --- /dev/null +++ b/drivers/md/dm-stats.c @@ -0,0 +1,969 @@ +#include <linux/errno.h> +#include <linux/numa.h> +#include <linux/slab.h> +#include <linux/rculist.h> +#include <linux/threads.h> +#include <linux/preempt.h> +#include <linux/irqflags.h> +#include <linux/vmalloc.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/device-mapper.h> + +#include "dm.h" +#include "dm-stats.h" + +#define DM_MSG_PREFIX "stats" + +static int dm_stat_need_rcu_barrier; + +/* + * Using 64-bit values to avoid overflow (which is a + * problem that block/genhd.c's IO accounting has). + */ +struct dm_stat_percpu { + unsigned long long sectors[2]; + unsigned long long ios[2]; + unsigned long long merges[2]; + unsigned long long ticks[2]; + unsigned long long io_ticks[2]; + unsigned long long io_ticks_total; + unsigned long long time_in_queue; +}; + +struct dm_stat_shared { + atomic_t in_flight[2]; + unsigned long stamp; + struct dm_stat_percpu tmp; +}; + +struct dm_stat { + struct list_head list_entry; + int id; + size_t n_entries; + sector_t start; + sector_t end; + sector_t step; + const char *program_id; + const char *aux_data; + struct rcu_head rcu_head; + size_t shared_alloc_size; + size_t percpu_alloc_size; + struct dm_stat_percpu *stat_percpu[NR_CPUS]; + struct dm_stat_shared stat_shared[0]; +}; + +struct dm_stats_last_position { + sector_t last_sector; + unsigned last_rw; +}; + +/* + * A typo on the command line could possibly make the kernel run out of memory + * and crash. To prevent the crash we account all used memory. We fail if we + * exhaust 1/4 of all memory or 1/2 of vmalloc space. + */ +#define DM_STATS_MEMORY_FACTOR 4 +#define DM_STATS_VMALLOC_FACTOR 2 + +static DEFINE_SPINLOCK(shared_memory_lock); + +static unsigned long shared_memory_amount; + +static bool __check_shared_memory(size_t alloc_size) +{ + size_t a; + + a = shared_memory_amount + alloc_size; + if (a < shared_memory_amount) + return false; + if (a >> PAGE_SHIFT > totalram_pages / DM_STATS_MEMORY_FACTOR) + return false; +#ifdef CONFIG_MMU + if (a > (VMALLOC_END - VMALLOC_START) / DM_STATS_VMALLOC_FACTOR) + return false; +#endif + return true; +} + +static bool check_shared_memory(size_t alloc_size) +{ + bool ret; + + spin_lock_irq(&shared_memory_lock); + + ret = __check_shared_memory(alloc_size); + + spin_unlock_irq(&shared_memory_lock); + + return ret; +} + +static bool claim_shared_memory(size_t alloc_size) +{ + spin_lock_irq(&shared_memory_lock); + + if (!__check_shared_memory(alloc_size)) { + spin_unlock_irq(&shared_memory_lock); + return false; + } + + shared_memory_amount += alloc_size; + + spin_unlock_irq(&shared_memory_lock); + + return true; +} + +static void free_shared_memory(size_t alloc_size) +{ + unsigned long flags; + + spin_lock_irqsave(&shared_memory_lock, flags); + + if (WARN_ON_ONCE(shared_memory_amount < alloc_size)) { + spin_unlock_irqrestore(&shared_memory_lock, flags); + DMCRIT("Memory usage accounting bug."); + return; + } + + shared_memory_amount -= alloc_size; + + spin_unlock_irqrestore(&shared_memory_lock, flags); +} + +static void *dm_kvzalloc(size_t alloc_size, int node) +{ + void *p; + + if (!claim_shared_memory(alloc_size)) + return NULL; + + if (alloc_size <= KMALLOC_MAX_SIZE) { + p = kzalloc_node(alloc_size, GFP_KERNEL | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN, node); + if (p) + return p; + } + p = vzalloc_node(alloc_size, node); + if (p) + return p; + + free_shared_memory(alloc_size); + + return NULL; +} + +static void dm_kvfree(void *ptr, size_t alloc_size) +{ + if (!ptr) + return; + + free_shared_memory(alloc_size); + + if (is_vmalloc_addr(ptr)) + vfree(ptr); + else + kfree(ptr); +} + +static void dm_stat_free(struct rcu_head *head) +{ + int cpu; + struct dm_stat *s = container_of(head, struct dm_stat, rcu_head); + + kfree(s->program_id); + kfree(s->aux_data); + for_each_possible_cpu(cpu) + dm_kvfree(s->stat_percpu[cpu], s->percpu_alloc_size); + dm_kvfree(s, s->shared_alloc_size); +} + +static int dm_stat_in_flight(struct dm_stat_shared *shared) +{ + return atomic_read(&shared->in_flight[READ]) + + atomic_read(&shared->in_flight[WRITE]); +} + +void dm_stats_init(struct dm_stats *stats) +{ + int cpu; + struct dm_stats_last_position *last; + + mutex_init(&stats->mutex); + INIT_LIST_HEAD(&stats->list); + stats->last = alloc_percpu(struct dm_stats_last_position); + for_each_possible_cpu(cpu) { + last = per_cpu_ptr(stats->last, cpu); + last->last_sector = (sector_t)ULLONG_MAX; + last->last_rw = UINT_MAX; + } +} + +void dm_stats_cleanup(struct dm_stats *stats) +{ + size_t ni; + struct dm_stat *s; + struct dm_stat_shared *shared; + + while (!list_empty(&stats->list)) { + s = container_of(stats->list.next, struct dm_stat, list_entry); + list_del(&s->list_entry); + for (ni = 0; ni < s->n_entries; ni++) { + shared = &s->stat_shared[ni]; + if (WARN_ON(dm_stat_in_flight(shared))) { + DMCRIT("leaked in-flight counter at index %lu " + "(start %llu, end %llu, step %llu): reads %d, writes %d", + (unsigned long)ni, + (unsigned long long)s->start, + (unsigned long long)s->end, + (unsigned long long)s->step, + atomic_read(&shared->in_flight[READ]), + atomic_read(&shared->in_flight[WRITE])); + } + } + dm_stat_free(&s->rcu_head); + } + free_percpu(stats->last); +} + +static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, + sector_t step, const char *program_id, const char *aux_data, + void (*suspend_callback)(struct mapped_device *), + void (*resume_callback)(struct mapped_device *), + struct mapped_device *md) +{ + struct list_head *l; + struct dm_stat *s, *tmp_s; + sector_t n_entries; + size_t ni; + size_t shared_alloc_size; + size_t percpu_alloc_size; + struct dm_stat_percpu *p; + int cpu; + int ret_id; + int r; + + if (end < start || !step) + return -EINVAL; + + n_entries = end - start; + if (dm_sector_div64(n_entries, step)) + n_entries++; + + if (n_entries != (size_t)n_entries || !(size_t)(n_entries + 1)) + return -EOVERFLOW; + + shared_alloc_size = sizeof(struct dm_stat) + (size_t)n_entries * sizeof(struct dm_stat_shared); + if ((shared_alloc_size - sizeof(struct dm_stat)) / sizeof(struct dm_stat_shared) != n_entries) + return -EOVERFLOW; + + percpu_alloc_size = (size_t)n_entries * sizeof(struct dm_stat_percpu); + if (percpu_alloc_size / sizeof(struct dm_stat_percpu) != n_entries) + return -EOVERFLOW; + + if (!check_shared_memory(shared_alloc_size + num_possible_cpus() * percpu_alloc_size)) + return -ENOMEM; + + s = dm_kvzalloc(shared_alloc_size, NUMA_NO_NODE); + if (!s) + return -ENOMEM; + + s->n_entries = n_entries; + s->start = start; + s->end = end; + s->step = step; + s->shared_alloc_size = shared_alloc_size; + s->percpu_alloc_size = percpu_alloc_size; + + s->program_id = kstrdup(program_id, GFP_KERNEL); + if (!s->program_id) { + r = -ENOMEM; + goto out; + } + s->aux_data = kstrdup(aux_data, GFP_KERNEL); + if (!s->aux_data) { + r = -ENOMEM; + goto out; + } + + for (ni = 0; ni < n_entries; ni++) { + atomic_set(&s->stat_shared[ni].in_flight[READ], 0); + atomic_set(&s->stat_shared[ni].in_flight[WRITE], 0); + } + + for_each_possible_cpu(cpu) { + p = dm_kvzalloc(percpu_alloc_size, cpu_to_node(cpu)); + if (!p) { + r = -ENOMEM; + goto out; + } + s->stat_percpu[cpu] = p; + } + + /* + * Suspend/resume to make sure there is no i/o in flight, + * so that newly created statistics will be exact. + * + * (note: we couldn't suspend earlier because we must not + * allocate memory while suspended) + */ + suspend_callback(md); + + mutex_lock(&stats->mutex); + s->id = 0; + list_for_each(l, &stats->list) { + tmp_s = container_of(l, struct dm_stat, list_entry); + if (WARN_ON(tmp_s->id < s->id)) { + r = -EINVAL; + goto out_unlock_resume; + } + if (tmp_s->id > s->id) + break; + if (unlikely(s->id == INT_MAX)) { + r = -ENFILE; + goto out_unlock_resume; + } + s->id++; + } + ret_id = s->id; + list_add_tail_rcu(&s->list_entry, l); + mutex_unlock(&stats->mutex); + + resume_callback(md); + + return ret_id; + +out_unlock_resume: + mutex_unlock(&stats->mutex); + resume_callback(md); +out: + dm_stat_free(&s->rcu_head); + return r; +} + +static struct dm_stat *__dm_stats_find(struct dm_stats *stats, int id) +{ + struct dm_stat *s; + + list_for_each_entry(s, &stats->list, list_entry) { + if (s->id > id) + break; + if (s->id == id) + return s; + } + + return NULL; +} + +static int dm_stats_delete(struct dm_stats *stats, int id) +{ + struct dm_stat *s; + int cpu; + + mutex_lock(&stats->mutex); + + s = __dm_stats_find(stats, id); + if (!s) { + mutex_unlock(&stats->mutex); + return -ENOENT; + } + + list_del_rcu(&s->list_entry); + mutex_unlock(&stats->mutex); + + /* + * vfree can't be called from RCU callback + */ + for_each_possible_cpu(cpu) + if (is_vmalloc_addr(s->stat_percpu)) + goto do_sync_free; + if (is_vmalloc_addr(s)) { +do_sync_free: + synchronize_rcu_expedited(); + dm_stat_free(&s->rcu_head); + } else { + ACCESS_ONCE(dm_stat_need_rcu_barrier) = 1; + call_rcu(&s->rcu_head, dm_stat_free); + } + return 0; +} + +static int dm_stats_list(struct dm_stats *stats, const char *program, + char *result, unsigned maxlen) +{ + struct dm_stat *s; + sector_t len; + unsigned sz = 0; + + /* + * Output format: + * <region_id>: <start_sector>+<length> <step> <program_id> <aux_data> + */ + + mutex_lock(&stats->mutex); + list_for_each_entry(s, &stats->list, list_entry) { + if (!program || !strcmp(program, s->program_id)) { + len = s->end - s->start; + DMEMIT("%d: %llu+%llu %llu %s %s\n", s->id, + (unsigned long long)s->start, + (unsigned long long)len, + (unsigned long long)s->step, + s->program_id, + s->aux_data); + } + } + mutex_unlock(&stats->mutex); + + return 1; +} + +static void dm_stat_round(struct dm_stat_shared *shared, struct dm_stat_percpu *p) +{ + /* + * This is racy, but so is part_round_stats_single. + */ + unsigned long now = jiffies; + unsigned in_flight_read; + unsigned in_flight_write; + unsigned long difference = now - shared->stamp; + + if (!difference) + return; + in_flight_read = (unsigned)atomic_read(&shared->in_flight[READ]); + in_flight_write = (unsigned)atomic_read(&shared->in_flight[WRITE]); + if (in_flight_read) + p->io_ticks[READ] += difference; + if (in_flight_write) + p->io_ticks[WRITE] += difference; + if (in_flight_read + in_flight_write) { + p->io_ticks_total += difference; + p->time_in_queue += (in_flight_read + in_flight_write) * difference; + } + shared->stamp = now; +} + +static void dm_stat_for_entry(struct dm_stat *s, size_t entry, + unsigned long bi_rw, sector_t len, bool merged, + bool end, unsigned long duration) +{ + unsigned long idx = bi_rw & REQ_WRITE; + struct dm_stat_shared *shared = &s->stat_shared[entry]; + struct dm_stat_percpu *p; + + /* + * For strict correctness we should use local_irq_disable/enable + * instead of preempt_disable/enable. + * + * This is racy if the driver finishes bios from non-interrupt + * context as well as from interrupt context or from more different + * interrupts. + * + * However, the race only results in not counting some events, + * so it is acceptable. + * + * part_stat_lock()/part_stat_unlock() have this race too. + */ + preempt_disable(); + p = &s->stat_percpu[smp_processor_id()][entry]; + + if (!end) { + dm_stat_round(shared, p); + atomic_inc(&shared->in_flight[idx]); + } else { + dm_stat_round(shared, p); + atomic_dec(&shared->in_flight[idx]); + p->sectors[idx] += len; + p->ios[idx] += 1; + p->merges[idx] += merged; + p->ticks[idx] += duration; + } + + preempt_enable(); +} + +static void __dm_stat_bio(struct dm_stat *s, unsigned long bi_rw, + sector_t bi_sector, sector_t end_sector, + bool end, unsigned long duration, + struct dm_stats_aux *stats_aux) +{ + sector_t rel_sector, offset, todo, fragment_len; + size_t entry; + + if (end_sector <= s->start || bi_sector >= s->end) + return; + if (unlikely(bi_sector < s->start)) { + rel_sector = 0; + todo = end_sector - s->start; + } else { + rel_sector = bi_sector - s->start; + todo = end_sector - bi_sector; + } + if (unlikely(end_sector > s->end)) + todo -= (end_sector - s->end); + + offset = dm_sector_div64(rel_sector, s->step); + entry = rel_sector; + do { + if (WARN_ON_ONCE(entry >= s->n_entries)) { + DMCRIT("Invalid area access in region id %d", s->id); + return; + } + fragment_len = todo; + if (fragment_len > s->step - offset) + fragment_len = s->step - offset; + dm_stat_for_entry(s, entry, bi_rw, fragment_len, + stats_aux->merged, end, duration); + todo -= fragment_len; + entry++; + offset = 0; + } while (unlikely(todo != 0)); +} + +void dm_stats_account_io(struct dm_stats *stats, unsigned long bi_rw, + sector_t bi_sector, unsigned bi_sectors, bool end, + unsigned long duration, struct dm_stats_aux *stats_aux) +{ + struct dm_stat *s; + sector_t end_sector; + struct dm_stats_last_position *last; + + if (unlikely(!bi_sectors)) + return; + + end_sector = bi_sector + bi_sectors; + + if (!end) { + /* + * A race condition can at worst result in the merged flag being + * misrepresented, so we don't have to disable preemption here. + */ + last = __this_cpu_ptr(stats->last); + stats_aux->merged = + (bi_sector == (ACCESS_ONCE(last->last_sector) && + ((bi_rw & (REQ_WRITE | REQ_DISCARD)) == + (ACCESS_ONCE(last->last_rw) & (REQ_WRITE | REQ_DISCARD))) + )); + ACCESS_ONCE(last->last_sector) = end_sector; + ACCESS_ONCE(last->last_rw) = bi_rw; + } + + rcu_read_lock(); + + list_for_each_entry_rcu(s, &stats->list, list_entry) + __dm_stat_bio(s, bi_rw, bi_sector, end_sector, end, duration, stats_aux); + + rcu_read_unlock(); +} + +static void __dm_stat_init_temporary_percpu_totals(struct dm_stat_shared *shared, + struct dm_stat *s, size_t x) +{ + int cpu; + struct dm_stat_percpu *p; + + local_irq_disable(); + p = &s->stat_percpu[smp_processor_id()][x]; + dm_stat_round(shared, p); + local_irq_enable(); + + memset(&shared->tmp, 0, sizeof(shared->tmp)); + for_each_possible_cpu(cpu) { + p = &s->stat_percpu[cpu][x]; + shared->tmp.sectors[READ] += ACCESS_ONCE(p->sectors[READ]); + shared->tmp.sectors[WRITE] += ACCESS_ONCE(p->sectors[WRITE]); + shared->tmp.ios[READ] += ACCESS_ONCE(p->ios[READ]); + shared->tmp.ios[WRITE] += ACCESS_ONCE(p->ios[WRITE]); + shared->tmp.merges[READ] += ACCESS_ONCE(p->merges[READ]); + shared->tmp.merges[WRITE] += ACCESS_ONCE(p->merges[WRITE]); + shared->tmp.ticks[READ] += ACCESS_ONCE(p->ticks[READ]); + shared->tmp.ticks[WRITE] += ACCESS_ONCE(p->ticks[WRITE]); + shared->tmp.io_ticks[READ] += ACCESS_ONCE(p->io_ticks[READ]); + shared->tmp.io_ticks[WRITE] += ACCESS_ONCE(p->io_ticks[WRITE]); + shared->tmp.io_ticks_total += ACCESS_ONCE(p->io_ticks_total); + shared->tmp.time_in_queue += ACCESS_ONCE(p->time_in_queue); + } +} + +static void __dm_stat_clear(struct dm_stat *s, size_t idx_start, size_t idx_end, + bool init_tmp_percpu_totals) +{ + size_t x; + struct dm_stat_shared *shared; + struct dm_stat_percpu *p; + + for (x = idx_start; x < idx_end; x++) { + shared = &s->stat_shared[x]; + if (init_tmp_percpu_totals) + __dm_stat_init_temporary_percpu_totals(shared, s, x); + local_irq_disable(); + p = &s->stat_percpu[smp_processor_id()][x]; + |