path: root/drivers/md/dm-stats.c

#include <linux/errno.h>
#include <linux/numa.h>
#include <linux/slab.h>
#include <linux/rculist.h>
#include <linux/threads.h>
#include <linux/preempt.h>
#include <linux/irqflags.h>
#include <linux/vmalloc.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/device-mapper.h>

#include "dm.h"
#include "dm-stats.h"

#define DM_MSG_PREFIX "stats"

static int dm_stat_need_rcu_barrier;

/*
 * Using 64-bit values to avoid overflow (which is a
 * problem that block/genhd.c's IO accounting has).
 */
struct dm_stat_percpu {
	unsigned long long sectors[2];
	unsigned long long ios[2];
	unsigned long long merges[2];
	unsigned long long ticks[2];
	unsigned long long io_ticks[2];
	unsigned long long io_ticks_total;
	unsigned long long time_in_queue;
};

struct dm_stat_shared {
	atomic_t in_flight[2];
	unsigned long stamp;
	struct dm_stat_percpu tmp;
};

struct dm_stat {
	struct list_head list_entry;
	int id;
	size_t n_entries;
	sector_t start;
	sector_t end;
	sector_t step;
	const char *program_id;
	const char *aux_data;
	struct rcu_head rcu_head;
	size_t shared_alloc_size;
	size_t percpu_alloc_size;
	struct dm_stat_percpu *stat_percpu[NR_CPUS];
	struct dm_stat_shared stat_shared[0];
};

struct dm_stats_last_position {
	sector_t last_sector;
	unsigned last_rw;
};

/*
 * A typo on the command line could possibly make the kernel run out of memory
 * and crash. To prevent the crash we account all used memory. We fail if we
 * exhaust 1/4 of all memory or 1/2 of vmalloc space.
 */
#define DM_STATS_MEMORY_FACTOR		4
#define DM_STATS_VMALLOC_FACTOR		2

static DEFINE_SPINLOCK(shared_memory_lock);

static unsigned long shared_memory_amount;

static bool __check_shared_memory(size_t alloc_size)
{
	size_t a;

	a = shared_memory_amount + alloc_size;
	if (a < shared_memory_amount)
		return false;
	if (a >> PAGE_SHIFT > totalram_pages / DM_STATS_MEMORY_FACTOR)
		return false;
#ifdef CONFIG_MMU
	if (a > (VMALLOC_END - VMALLOC_START) / DM_STATS_VMALLOC_FACTOR)
		return false;
#endif
	return true;
}

static bool check_shared_memory(size_t alloc_size)
{
	bool ret;

	spin_lock_irq(&shared_memory_lock);

	ret = __check_shared_memory(alloc_size);

	spin_unlock_irq(&shared_memory_lock);

	return ret;
}

static bool claim_shared_memory(size_t alloc_size)
{
	spin_lock_irq(&shared_memory_lock);

	if (!__check_shared_memory(alloc_size)) {
		spin_unlock_irq(&shared_memory_lock);
		return false;
	}

	shared_memory_amount += alloc_size;

	spin_unlock_irq(&shared_memory_lock);

	return true;
}

static void free_shared_memory(size_t alloc_size)
{
	unsigned long flags;

	spin_lock_irqsave(&shared_memory_lock, flags);

	if (WARN_ON_ONCE(shared_memory_amount < alloc_size)) {
		spin_unlock_irqrestore(&shared_memory_lock, flags);
		DMCRIT("Memory usage accounting bug.");
		return;
	}

	shared_memory_amount -= alloc_size;

	spin_unlock_irqrestore(&shared_memory_lock, flags);
}

static void *dm_kvzalloc(size_t alloc_size, int node)
{
	void *p;

	if (!claim_shared_memory(alloc_size))
		return NULL;

	if (alloc_size <= KMALLOC_MAX_SIZE) {
		p = kzalloc_node(alloc_size, GFP_KERNEL | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN, node);
		if (p)
			return p;
	}
	p = vzalloc_node(alloc_size, node);
	if (p)
		return p;

	free_shared_memory(alloc_size);

	return NULL;
}

static void dm_kvfree(void *ptr, size_t alloc_size)
{
	if (!ptr)
		return;

	free_shared_memory(alloc_size);

	if (is_vmalloc_addr(ptr))
		vfree(ptr);
	else
		kfree(ptr);
}

static void dm_stat_free(struct rcu_head *head)
{
	int cpu;
	struct dm_stat *s = container_of(head, struct dm_stat, rcu_head);

	kfree(s->program_id);
	kfree(s->aux_data);
	for_each_possible_cpu(cpu)
		dm_kvfree(s->stat_percpu[cpu], s->percpu_alloc_size);
	dm_kvfree(s, s->shared_alloc_size);
}

static int dm_stat_in_flight(struct dm_stat_shared *shared)
{
	return atomic_read(&shared->in_flight[READ]) +
	       atomic_read(&shared-&