path: root/tools/testing/selftests/mqueue/mq_perf_tests.c

/*
 * This application is Copyright 2012 Red Hat, Inc.
 *	Doug Ledford <dledford@redhat.com>
 *
 * mq_perf_tests is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, version 3.
 *
 * mq_perf_tests is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * For the full text of the license, see <http://www.gnu.org/licenses/>.
 *
 * mq_perf_tests.c
 *   Tests various types of message queue workloads, concentrating on those
 *   situations that invole large message sizes, large message queue depths,
 *   or both, and reports back useful metrics about kernel message queue
 *   performance.
 *
 */
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
#include <limits.h>
#include <errno.h>
#include <signal.h>
#include <pthread.h>
#include <sched.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/stat.h>
#include <mqueue.h>
#include <popt.h>

static char *usage =
"Usage:\n"
"  %s [-c #[,#..] -f] path\n"
"\n"
"	-c #	Skip most tests and go straight to a high queue depth test\n"
"		and then run that test continuously (useful for running at\n"
"		the same time as some other workload to see how much the\n"
"		cache thrashing caused by adding messages to a very deep\n"
"		queue impacts the performance of other programs).  The number\n"
"		indicates which CPU core we should bind the process to during\n"
"		the run.  If you have more than one physical CPU, then you\n"
"		will need one copy per physical CPU package, and you should\n"
"		specify the CPU cores to pin ourself to via a comma separated\n"
"		list of CPU values.\n"
"	-f	Only usable with continuous mode.  Pin ourself to the CPUs\n"
"		as requested, then instead of looping doing a high mq\n"
"		workload, just busy loop.  This will allow us to lock up a\n"
"		single CPU just like we normally would, but without actually\n"
"		thrashing the CPU cache.  This is to make it easier to get\n"
"		comparable numbers from some other workload running on the\n"
"		other CPUs.  One set of numbers with # CPUs locked up running\n"
"		an mq workload, and another set of numbers with those same\n"
"		CPUs locked away from the test workload, but not doing\n"
"		anything to trash the cache like the mq workload might.\n"
"	path	Path name of the message queue to create\n"
"\n"
"	Note: this program must be run as root in order to enable all tests\n"
"\n";

char *MAX_MSGS = "/proc/sys/fs/mqueue/msg_max";
char *MAX_MSGSIZE = "/proc/sys/fs/mqueue/msgsize_max";

#define min(a, b) ((a) < (b) ? (a) : (b))
#define MAX_CPUS 64
char *cpu_option_string;
int cpus_to_pin[MAX_CPUS];
int num_cpus_to_pin;
pthread_t cpu_threads[MAX_CPUS];
pthread_t main_thread;
cpu_set_t *cpu_set;
int cpu_set_size;
int cpus_online;

#define MSG_SIZE 16
#define TEST1_LOOPS 10000000
#define TEST2_LOOPS 100000
int continuous_mode;
int continuous_mode_fake;

struct rlimit saved_limits, cur_limits;
int saved_max_msgs, saved_max_msgsize;
int cur_max_msgs, cur_max_msgsize;
FILE *max_msgs, *max_msgsize;
int cur_nice;
char *queue_path = "/mq_perf_tests";
mqd_t queue = -1;
struct mq_attr result;
int mq_prio_max;

const struct poptOption options[] = {
	{
		.longName = "continuous",
		.shortName = 'c',
		.argInfo = POPT_ARG_STRING,
		.arg = &cpu_option_string,
		.val = 'c',
		.descrip = "Run continuous tests at a high queue depth in "
			"order to test the effects of cache thrashing on "
			"other tasks on the system.  This test is intended "
			"to be run on one core of each physical CPU while "
			"some other CPU intensive task is run on all the other "
			"cores of that same physical CPU and the other task "
			"is timed.  It is assumed that the process of adding "
			"messages to the message queue in a tight loop will "
			"impact that other task to some degree.  Once the "
			"tests are performed in this way, you should then "
			"re-run the tests using fake mode in order to check "
			"the difference in time required to perform the CPU "
			"intensive task",
		.argDescrip = "cpu[,cpu]",
	},
	{
		.longName = "fake",
		.shortName = 'f',
		.argInfo = POPT_ARG_NONE,
		.arg = &continuous_mode_fake,
		.val = 0,
		.descrip = "Tie up the CPUs that we would normally tie up in"
			"continuous mode, but don't actually do any mq stuff, "
			"just keep the CPU busy so it can't be used to process "
			"system level tasks as this would free up resources on "
			"the other CPU cores and skew the comparison between "
			"the no-mqueue work and mqueue work tests",
		.argDescrip = NULL,
	},
	{
		.longName = "path",
		.shortName = 'p',
		.argInfo = POPT_ARG_STRING | POPT_ARGFLAG_SHOW_DEFAULT,
		.arg = &queue_path,
		.val = 'p',
		.descrip = "The name of the path to use in the mqueue "
			"filesystem for our tests",
		.argDescrip = "pathname",
	},
	POPT_AUTOHELP
	POPT_TABLEEND
};

static inline void __set(FILE *stream, int value, char *err_msg);
void shutdown(int exit_val, char *err_cause, int line_no);
void sig_action_SIGUSR1(int signum, siginfo_t *info, void *context);
void sig_action(int signum, siginfo_t *info, void *context);
static inline int get(FILE *stream);
static inline void set(FILE *stream, int value);
static inline int try_set(FILE *stream, int value);
static inline void getr(int type, struct rlimit *rlim);
static inline void setr(int type, struct rlimit *rlim);
static inline void open_queue(struct mq_attr *attr);
void increase_limits(void);

static inline void __set(FILE *stream, int value, char *err_msg)
{
	rewind(stream);
	if (fprintf(stream, "%d", value) < 0)
		perror(err_msg);
}


void shutdown(int exit_val, char *err_cause, int line_no)
{
	static int in_shutdown = 0;
	int errno_at_shutdown = errno;
	int i;

	/* In case we get called by multiple threads or from an sighandler */
	if (in_shutdown++)
		return;

	for (i = 0; i < num_cpus_to_pin; i++)
		if (cpu_threads[i]) {
			pthread_kill(cpu_threads[i], SIGUSR1);
			pthread_join(cpu_threads[i], NULL);
		}

	if (queue != -1)
		if (mq_close(queue))
			perror("mq_close() during shutdown");
	if (queue_path)
		/*
		 * Be silent if this fails, if we cleaned up already it's
		 * expected to fail
		 */
		mq_unlink(queue_path);
	if (saved_max_msgs)
		__set(max_msgs, saved_max_msgs,
		      "failed to restore saved_max_msgs");
	if (saved_max_msgsize)
		__set(max_msgsize, saved_max_msgsize,
		      "failed to restore saved_max_msgsize");
	if (exit_val)
		error(exit_val, errno_at_shutdown, "%s at %d",
		      err_cause, line_no);
	exit(0);
}

void sig_action_SIGUSR1(int signum, siginfo_t *info, void *context)
{
	if (pthread_self() != main_thread)
		pthread_exit(0);
	else {
		fprintf(stderr, "Caught signal %d in SIGUSR1 handler, "
				"exiting\n", signum);
		shutdown(0,