[PATCH-tip v5 02/21] perf bench: New microbenchmark for userspace rwlock performance

From: Waiman Long
Date: Fri Feb 03 2017 - 13:04:23 EST

This microbenchmark simulates how the use of different futex types
can affect the actual performanace of userspace rwlock locks. The
usage is:

perf bench futex rwlock <options>

Two sets of simple rwlock lock and unlock functions are implemented
using the wait-wake futexes and glibc rwlock respectively. This
microbenchmark then runs the locking rate measurement tests using
either one of those rwlocks or all of them consecutively.

Sample output from this microbenchmark was as follows:

[PID 7434]: 72 threads doing WW futex lockings (load=1) for 10 secs.

Locking statistics:
Test run time = 10.00s
Total exclusive locking ops = 27,147,143
Total shared locking ops = 27,147,143
Exclusive lock futex calls = 8,812,299
Exclusive unlock futex calls = 9,034,693
Shared lock futex calls = 1,685,538
Shared unlock futex calls = 4,605,343
EAGAIN lock errors = 10,472,722
Process wakeups = 25,115

Percentages:
Exclusive lock futex calls = 32.5%
Shared lock futex calls = 6.2%
Exclusive unlock futex calls = 33.3%
Shared unlock futex calls = 17.0%
EAGAIN lock errors = 99.8%
Process wakeups = 0.2%

Per-thread Locking Rates:
Avg = 75,392 ops/sec (+- 0.30%)
Min = 71,259 ops/sec
Max = 78,211 ops/sec

Signed-off-by: Waiman Long <longman@xxxxxxxxxx>
---
tools/perf/Documentation/perf-bench.txt | 3 +
tools/perf/bench/bench.h | 1 +
tools/perf/bench/futex-locks.c | 683 +++++++++++++++++++++++++++++++-
tools/perf/builtin-bench.c | 1 +
4 files changed, 672 insertions(+), 16 deletions(-)

diff --git a/tools/perf/Documentation/perf-bench.txt b/tools/perf/Documentation/perf-bench.txt
index 1fa5a74..e5a7079 100644
--- a/tools/perf/Documentation/perf-bench.txt
+++ b/tools/perf/Documentation/perf-bench.txt
@@ -206,6 +206,9 @@ Suite for evaluating futex lock_pi calls.
*mutex*::
Suite for evaluating futex calls for implementing userspace mutexes.

+*rwlock*::
+Suite for evaluating futex calls for implementing userspace rwlocks.
+
SEE ALSO
--------
linkperf:perf[1]
diff --git a/tools/perf/bench/bench.h b/tools/perf/bench/bench.h
index b0632df..a7e2037 100644
--- a/tools/perf/bench/bench.h
+++ b/tools/perf/bench/bench.h
@@ -37,6 +37,7 @@
/* pi futexes */
int bench_futex_lock_pi(int argc, const char **argv, const char *prefix);
int bench_futex_mutex(int argc, const char **argv, const char *prefix);
+int bench_futex_rwlock(int argc, const char **argv, const char *prefix);

#define BENCH_FORMAT_DEFAULT_STR "default"
#define BENCH_FORMAT_DEFAULT 0
diff --git a/tools/perf/bench/futex-locks.c b/tools/perf/bench/futex-locks.c
index 622a57c..18dc71d 100644
--- a/tools/perf/bench/futex-locks.c
+++ b/tools/perf/bench/futex-locks.c
@@ -10,6 +10,10 @@
* unlock functions are written to implenment a mutex lock using the
* wait-wake (2 versions) and PI futexes respectively. These functions serve
* as the basis for measuring the locking throughput.
+ *
+ * Two sets of simple reader/writer lock and unlock functions are also
+ * implemented using the wait-wake futexes as well as the Glibc rwlock
+ * respectively for performance measurement purpose.
*/

#include <pthread.h>
@@ -21,11 +25,13 @@
#include <subcmd/parse-options.h>
#include <linux/compiler.h>
#include <linux/kernel.h>
+#include <asm/byteorder.h>
#include <errno.h>
#include "bench.h"
#include "futex.h"

#include <err.h>
+#include <limits.h>
#include <stdlib.h>
#include <sys/time.h>

@@ -42,13 +48,19 @@
*/
enum {
STAT_OPS, /* # of exclusive locking operations */
+ STAT_SOPS, /* # of shared locking operations */
STAT_LOCKS, /* # of exclusive lock futex calls */
STAT_UNLOCKS, /* # of exclusive unlock futex calls */
STAT_SLEEPS, /* # of exclusive lock sleeps */
+ STAT_SLOCKS, /* # of shared lock futex calls */
+ STAT_SUNLOCKS, /* # of shared unlock futex calls */
+ STAT_SSLEEPS, /* # of shared lock sleeps */
STAT_EAGAINS, /* # of EAGAIN errors */
STAT_WAKEUPS, /* # of wakeups (unlock return) */
STAT_LOCKERRS, /* # of exclusive lock errors */
STAT_UNLKERRS, /* # of exclusive unlock errors */
+ STAT_SLOCKERRS, /* # of shared lock errors */
+ STAT_SUNLKERRS, /* # of shared unlock errors */
STAT_NUM /* Total # of statistical count */
};

@@ -58,6 +70,8 @@ enum {
enum {
TIME_LOCK, /* Total exclusive lock syscall time */
TIME_UNLK, /* Total exclusive unlock syscall time */
+ TIME_SLOCK, /* Total shared lock syscall time */
+ TIME_SUNLK, /* Total shared unlock syscall time */
TIME_NUM,
};

@@ -100,7 +114,11 @@ struct worker {
static unsigned int threads_stopping;
static struct stats throughput_stats;
static lock_fn_t mutex_lock_fn;
+static lock_fn_t read_lock_fn;
+static lock_fn_t write_lock_fn;
static unlock_fn_t mutex_unlock_fn;
+static unlock_fn_t read_unlock_fn;
+static unlock_fn_t write_unlock_fn;

/*
* Lock/unlock syscall time macro
@@ -148,6 +166,26 @@ static inline void stat_inc(int tid __maybe_unused, int item __maybe_unused)
#endif

/*
+ * For rwlock, the default is to have each thread acts as both reader and
+ * writer. The use of the -p option will force the use of separate threads
+ * for readers and writers. The numbers of reader and writer threads are
+ * determined by reader percentage and the total number of threads used.
+ * So the actual ratio of reader and writer operations may not be close
+ * to the given reader percentage.
+ */
+static bool xthread;
+static bool pwriter; /* Prefer writer flag */
+static int rthread_threshold = -1; /* (tid < threshold) => reader */
+static unsigned int rpercent = 50; /* Reader percentage */
+
+/*
+ * Glibc rwlock
+ */
+static pthread_rwlock_t __cacheline_aligned rwlock;
+static pthread_rwlockattr_t rwlock_attr;
+static bool rwlock_inited, rwlock_attr_inited;
+
+/*
* The latency value within a lock critical section (load) and between locking
* operations is in term of the number of cpu_relax() calls that are being
* issued.
@@ -170,6 +208,27 @@ static inline void stat_inc(int tid __maybe_unused, int item __maybe_unused)
NULL
};

+static const struct option rwlock_options[] = {
+ OPT_INTEGER ('d', "locklat", &locklat, "Specify inter-locking latency (default = 1)"),
+ OPT_STRING ('f', "ftype", &ftype, "type", "Specify futex type: WW, GC, all (default)"),
+ OPT_INTEGER ('L', "loadlat", &loadlat, "Specify load latency (default = 1)"),
+ OPT_UINTEGER('R', "read-%", &rpercent, "Specify reader percentage (default 50%)"),
+ OPT_UINTEGER('r', "runtime", &nsecs, "Specify runtime (in seconds, default = 10s)"),
+ OPT_BOOLEAN ('S', "shared", &fshared, "Use shared futexes instead of private ones"),
+ OPT_BOOLEAN ('T', "timestat", &timestat, "Track lock/unlock syscall times"),
+ OPT_UINTEGER('t', "threads", &nthreads, "Specify number of threads, default = # of CPUs"),
+ OPT_BOOLEAN ('v', "verbose", &verbose, "Verbose mode: display thread-level details"),
+ OPT_BOOLEAN ('W', "prefer-wr", &pwriter, "Prefer writers instead of readers"),
+ OPT_INTEGER ('w', "wait-ratio", &wratio, "Specify <n>/1024 of load is 1us sleep, default = 0"),
+ OPT_BOOLEAN ('x', "xthread", &xthread, "Use separate reader/writer threads"),
+ OPT_END()
+};
+
+static const char * const bench_futex_rwlock_usage[] = {
+ "perf bench futex rwlock <options>",
+ NULL
+};
+
/**
* futex_cmpxchg() - atomic compare and exchange
* @uaddr: The address of the futex to be modified
@@ -219,6 +278,17 @@ static inline int atomic_inc_return(futex_t *uaddr)
return __sync_add_and_fetch(uaddr, 1);
}

+/**
+ * atomic_add_return - atomically add a number & return the new value
+ * @uaddr: The address of the futex to be added
+ * @val : The integer value to be added
+ * Return: The new value
+ */
+static inline int atomic_add_return(futex_t *uaddr, int val)
+{
+ return __sync_add_and_fetch(uaddr, val);
+}
+
/**********************[ MUTEX lock/unlock functions ]*********************/

/*
@@ -434,6 +504,366 @@ static void pi_mutex_unlock(futex_t *futex, int tid)
stat_inc(tid, STAT_UNLOCKS);
}

+/**********************[ RWLOCK lock/unlock functions ]********************/
+
+/*
+ * Wait-wake futex reader/writer lock/unlock functions
+ *
+ * This implementation is based on the reader-preferring futex eventcount
+ * rwlocks posted on http://locklessinc.com/articles/sleeping_rwlocks with
+ * some modification.
+ *
+ * It is assumed the passed-in futex have sufficient trailing space to
+ * be used by the bigger reader/writer lock structure.
+ */
+#ifdef __LITTLE_ENDIAN
+#define LSB(field) unsigned char field
+#else
+#define LSB(field) struct { \
+ unsigned char __pad[3]; \
+ unsigned char field; \
+ }
+#endif
+
+#define RW_WLOCKED (1U << 0)
+#define RW_READER (1U << 8)
+#define RW_EC_CONTEND (1U << 0)
+#define RW_EC_INC (1U << 8)
+
+struct rwlock {
+ /*
+ * Bits 0-7 : writer lock
+ * Bits 8-31: reader count
+ */
+ union {
+ futex_t val;
+ LSB(wlocked);
+ } lock;
+
+ /* Writer event count */
+ union {
+ futex_t val;
+ LSB(contend);
+ } write_ec;
+
+ /* Reader event count */
+ union {
+ futex_t val;
+ LSB(contend);
+ } read_ec;
+};
+
+static struct rwlock __cacheline_aligned rwfutex;
+
+/*
+ * Reader preferring rwlock functions
+ */
+static void ww_write_lock(futex_t *futex __maybe_unused, int tid)
+{
+ struct timespec stime, etime;
+ struct rwlock *rw = &rwfutex;
+
+ for (;;) {
+ futex_t ec = rw->write_ec.val | RW_EC_CONTEND;
+ int ret;
+
+ /* Set the write lock if there is no reader */
+ if (!rw->lock.val &&
+ (futex_cmpxchg(&rw->lock.val, 0, RW_WLOCKED) == 0))
+ return;
+
+ rw->write_ec.contend = 1;
+
+ if (timestat) {
+ clock_gettime(CLOCK_REALTIME, &stime);
+ ret = futex_wait(&rw->write_ec.val, ec, NULL, flags);
+ clock_gettime(CLOCK_REALTIME, &etime);
+ systime_add(tid, TIME_LOCK, &stime, &etime);
+ } else {
+ ret = futex_wait(&rw->write_ec.val, ec, NULL, flags);
+ }
+
+ stat_inc(tid, STAT_LOCKS);
+ if (ret < 0) {
+ if (errno == EAGAIN)
+ stat_inc(tid, STAT_EAGAINS);
+ else
+ stat_inc(tid, STAT_LOCKERRS);
+ }
+
+ /* Other writers may exist */
+ rw->write_ec.contend = 1;
+ }
+}
+
+static void ww_write_unlock(futex_t *futex __maybe_unused, int tid)
+{
+ struct timespec stime, etime;
+ struct rwlock *rw = &rwfutex;
+ int ret;
+
+ rw->lock.wlocked = 0; /* Unlock */
+
+ /* Wake all the readers */
+ atomic_add_return(&rw->read_ec.val, RW_EC_INC);
+ if (rw->read_ec.contend) {
+ rw->read_ec.contend = 0;
+ if (timestat) {
+ clock_gettime(CLOCK_REALTIME, &stime);
+ ret = futex_wake(&rw->read_ec.val, INT_MAX, flags);
+ clock_gettime(CLOCK_REALTIME, &etime);
+ systime_add(tid, TIME_UNLK, &stime, &etime);
+ } else {
+ ret = futex_wake(&rw->read_ec.val, INT_MAX, flags);
+ }
+ stat_inc(tid, STAT_UNLOCKS);
+ if (ret < 0)
+ stat_inc(tid, STAT_UNLKERRS);
+ else
+ stat_add(tid, STAT_WAKEUPS, ret);
+ if (ret > 0)
+ return;
+ }
+
+ /* Wake a writer */
+ atomic_add_return(&rw->write_ec.val, RW_EC_INC);
+ if (rw->write_ec.contend) {
+ rw->write_ec.contend = 0;
+ if (timestat) {
+ clock_gettime(CLOCK_REALTIME, &stime);
+ ret = futex_wake(&rw->write_ec.val, 1, flags);
+ clock_gettime(CLOCK_REALTIME, &etime);
+ systime_add(tid, TIME_UNLK, &stime, &etime);
+ } else {
+ ret = futex_wake(&rw->write_ec.val, 1, flags);
+ }
+ stat_inc(tid, STAT_UNLOCKS);
+ if (ret < 0)
+ stat_inc(tid, STAT_UNLKERRS);
+ else
+ stat_add(tid, STAT_WAKEUPS, ret);
+ }
+}
+
+static void ww_read_lock(futex_t *futex __maybe_unused, int tid)
+{
+ struct timespec stime, etime;
+ struct rwlock *rw = &rwfutex;
+ futex_t ec = rw->read_ec.val, state;
+ int ret;
+
+ state = atomic_add_return(&rw->lock.val, RW_READER);
+
+ while (state & RW_WLOCKED) {
+ ec |= RW_EC_CONTEND;
+ rw->read_ec.contend = 1;
+
+ /* Sleep until no longer held by a writer */
+ if (timestat) {
+ clock_gettime(CLOCK_REALTIME, &stime);
+ ret = futex_wait(&rw->read_ec.val, ec, NULL, flags);
+ clock_gettime(CLOCK_REALTIME, &etime);
+ systime_add(tid, TIME_SLOCK, &stime, &etime);
+ } else {
+ ret = futex_wait(&rw->read_ec.val, ec, NULL, flags);
+ }
+ stat_inc(tid, STAT_SLOCKS);
+ if (ret < 0) {
+ if (errno == EAGAIN)
+ stat_inc(tid, STAT_EAGAINS);
+ else
+ stat_inc(tid, STAT_SLOCKERRS);
+ }
+
+ ec = rw->read_ec.val;
+ barrier();
+ state = rw->lock.val;
+ }
+}
+
+static void ww_read_unlock(futex_t *futex __maybe_unused, int tid)
+{
+ struct timespec stime, etime;
+ struct rwlock *rw = &rwfutex;
+ futex_t state;
+ int ret;
+
+ /* Read unlock */
+ state = atomic_add_return(&rw->lock.val, -RW_READER);
+
+ /* Other readers there, don't do anything */
+ if (state >> 8)
+ return;
+
+ /* We may need to wake up a writer */
+ atomic_add_return(&rw->write_ec.val, RW_EC_INC);
+ if (rw->write_ec.contend) {
+ rw->write_ec.contend = 0;
+ if (timestat) {
+ clock_gettime(CLOCK_REALTIME, &stime);
+ ret = futex_wake(&rw->write_ec.val, 1, flags);
+ clock_gettime(CLOCK_REALTIME, &etime);
+ systime_add(tid, TIME_SUNLK, &stime, &etime);
+ } else {
+ ret = futex_wake(&rw->write_ec.val, 1, flags);
+ }
+ stat_inc(tid, STAT_SUNLOCKS);
+ if (ret < 0)
+ stat_inc(tid, STAT_SUNLKERRS);
+ else
+ stat_add(tid, STAT_WAKEUPS, ret);
+ }
+}
+
+/*
+ * Writer perferring rwlock functions
+ */
+#define ww2_write_lock ww_write_lock
+#define ww2_read_unlock ww_read_unlock
+
+static void ww2_write_unlock(futex_t *futex __maybe_unused, int tid)
+{
+ struct timespec stime, etime;
+ struct rwlock *rw = &rwfutex;
+ int ret;
+
+ rw->lock.wlocked = 0; /* Unlock */
+
+ /* Wake a writer */
+ atomic_add_return(&rw->write_ec.val, RW_EC_INC);
+ if (rw->write_ec.contend) {
+ rw->write_ec.contend = 0;
+ if (timestat) {
+ clock_gettime(CLOCK_REALTIME, &stime);
+ ret = futex_wake(&rw->write_ec.val, 1, flags);
+ clock_gettime(CLOCK_REALTIME, &etime);
+ systime_add(tid, TIME_UNLK, &stime, &etime);
+ } else {
+ ret = futex_wake(&rw->write_ec.val, 1, flags);
+ }
+ stat_inc(tid, STAT_UNLOCKS);
+ if (ret < 0)
+ stat_inc(tid, STAT_UNLKERRS);
+ else
+ stat_add(tid, STAT_WAKEUPS, ret);
+ if (ret > 0)
+ return;
+ }
+
+ /* Wake all the readers */
+ atomic_add_return(&rw->read_ec.val, RW_EC_INC);
+ if (rw->read_ec.contend) {
+ rw->read_ec.contend = 0;
+ if (timestat) {
+ clock_gettime(CLOCK_REALTIME, &stime);
+ ret = futex_wake(&rw->read_ec.val, INT_MAX, flags);
+ clock_gettime(CLOCK_REALTIME, &etime);
+ systime_add(tid, TIME_UNLK, &stime, &etime);
+ } else {
+ ret = futex_wake(&rw->read_ec.val, INT_MAX, flags);
+ }
+ stat_inc(tid, STAT_UNLOCKS);
+ if (ret < 0)
+ stat_inc(tid, STAT_UNLKERRS);
+ else
+ stat_add(tid, STAT_WAKEUPS, ret);
+ }
+}
+
+static void ww2_read_lock(futex_t *futex __maybe_unused, int tid)
+{
+ struct timespec stime, etime;
+ struct rwlock *rw = &rwfutex;
+
+ for (;;) {
+ futex_t ec = rw->read_ec.val | RW_EC_CONTEND;
+ futex_t state;
+ int ret;
+
+ if (!rw->write_ec.contend) {
+ state = atomic_add_return(&rw->lock.val, RW_READER);
+
+ if (!(state & RW_WLOCKED))
+ return;
+
+ /* Unlock */
+ state = atomic_add_return(&rw->lock.val, -RW_READER);
+ } else {
+ atomic_add_return(&rw->write_ec.val, RW_EC_INC);
+ if (rw->write_ec.contend) {
+ rw->write_ec.contend = 0;
+
+ /* Wake a writer, and then try again */
+ if (timestat) {
+ clock_gettime(CLOCK_REALTIME, &stime);
+ ret = futex_wake(&rw->write_ec.val, 1,
+ flags);
+ clock_gettime(CLOCK_REALTIME, &etime);
+ systime_add(tid, TIME_SUNLK, &stime,
+ &etime);
+ } else {
+ ret = futex_wake(&rw->write_ec.val, 1,
+ flags);
+ }
+ stat_inc(tid, STAT_SUNLOCKS);
+ if (ret < 0)
+ stat_inc(tid, STAT_SUNLKERRS);
+ else
+ stat_add(tid, STAT_WAKEUPS, ret);
+ continue;
+ }
+ }
+
+ rw->read_ec.contend = 1;
+ if (rw->read_ec.val != ec)
+ continue;
+
+ /* Sleep until no longer held by a writer */
+ if (timestat) {
+ clock_gettime(CLOCK_REALTIME, &stime);
+ ret = futex_wait(&rw->read_ec.val, ec, NULL, flags);
+ clock_gettime(CLOCK_REALTIME, &etime);
+ systime_add(tid, TIME_SLOCK, &stime, &etime);
+ } else {
+ ret = futex_wait(&rw->read_ec.val, ec, NULL, flags);
+ }
+ stat_inc(tid, STAT_SLOCKS);
+ if (ret < 0) {
+ if (errno == EAGAIN)
+ stat_inc(tid, STAT_EAGAINS);
+ else
+ stat_inc(tid, STAT_SLOCKERRS);
+ }
+ }
+}
+
+/*
+ * Glibc read/write lock
+ */
+static void gc_write_lock(futex_t *futex __maybe_unused,
+ int tid __maybe_unused)
+{
+ pthread_rwlock_wrlock(&rwlock);
+}
+
+static void gc_write_unlock(futex_t *futex __maybe_unused,
+ int tid __maybe_unused)
+{
+ pthread_rwlock_unlock(&rwlock);
+}
+
+static void gc_read_lock(futex_t *futex __maybe_unused,
+ int tid __maybe_unused)
+{
+ pthread_rwlock_rdlock(&rwlock);
+}
+
+static void gc_read_unlock(futex_t *futex __maybe_unused,
+ int tid __maybe_unused)
+{
+ pthread_rwlock_unlock(&rwlock);
+}
+
/**************************************************************************/

/*
@@ -508,6 +938,77 @@ static void *mutex_workerfn(void *arg)
return NULL;
}

+static void *rwlock_workerfn(void *arg)
+{
+ long tid = (long)arg;
+ struct worker *w = &worker[tid];
+ lock_fn_t rlock_fn = read_lock_fn;
+ lock_fn_t wlock_fn = write_lock_fn;
+ unlock_fn_t runlock_fn = read_unlock_fn;
+ unlock_fn_t wunlock_fn = write_unlock_fn;
+
+ thread_id = gettid();
+ counter = 0;
+
+ atomic_dec_return(&threads_starting);
+
+ /*
+ * Busy wait until asked to start
+ */
+ while (!worker_start)
+ cpu_relax();
+
+ if (rthread_threshold >= 0) {
+ if (tid < rthread_threshold) {
+ do {
+ rlock_fn(w->futex, tid);
+ load(tid);
+ runlock_fn(w->futex, tid);
+ w->stats[STAT_SOPS]++;
+ csdelay();
+ } while (!done);
+ } else {
+ do {
+ wlock_fn(w->futex, tid);
+ load(tid);
+ wunlock_fn(w->futex, tid);
+ w->stats[STAT_OPS]++;
+ csdelay();
+ } while (!done);
+ }
+ goto out;
+ }
+
+ while (!done) {
+ int rcnt = rpercent;
+ int wcnt = 100 - rcnt;
+
+ do {
+ if (wcnt) {
+ wlock_fn(w->futex, tid);
+ load(tid);
+ wunlock_fn(w->futex, tid);
+ w->stats[STAT_OPS]++;
+ wcnt--;
+ csdelay();
+ }
+ if (rcnt) {
+ rlock_fn(w->futex, tid);
+ load(tid);
+ runlock_fn(w->futex, tid);
+ w->stats[STAT_SOPS]++;
+ rcnt--;
+ csdelay();
+ }
+ } while (!done && (rcnt + wcnt));
+ }
+out:
+ if (verbose)
+ printf("[thread %3ld (%d)] exited.\n", tid, thread_id);
+ atomic_inc_return(&threads_stopping);
+ return NULL;
+}
+
static void create_threads(struct worker *w, pthread_attr_t *thread_attr,
void *(*workerfn)(void *arg), long tid)
{
@@ -549,6 +1050,50 @@ static int futex_mutex_type(const char **ptype)
return 0;
}

+static int futex_rwlock_type(const char **ptype)
+{
+ const char *type = *ptype;
+
+ if (!strcasecmp(type, "WW")) {
+ *ptype = "WW";
+ pfutex = &rwfutex.lock.val;
+ if (pwriter) {
+ read_lock_fn = ww2_read_lock;
+ read_unlock_fn = ww2_read_unlock;
+ write_lock_fn = ww2_write_lock;
+ write_unlock_fn = ww2_write_unlock;
+ } else {
+ read_lock_fn = ww_read_lock;
+ read_unlock_fn = ww_read_unlock;
+ write_lock_fn = ww_write_lock;
+ write_unlock_fn = ww_write_unlock;
+ }
+ } else if (!strcasecmp(type, "GC")) {
+ pthread_rwlockattr_t *attr = NULL;
+
+ *ptype = "GC";
+ read_lock_fn = gc_read_lock;
+ read_unlock_fn = gc_read_unlock;
+ write_lock_fn = gc_write_lock;
+ write_unlock_fn = gc_write_unlock;
+ if (pwriter || fshared) {
+ pthread_rwlockattr_init(&rwlock_attr);
+ attr = &rwlock_attr;
+ rwlock_attr_inited = true;
+ if (fshared)
+ pthread_rwlockattr_setpshared(attr, true);
+ if (pwriter)
+ pthread_rwlockattr_setkind_np(attr,
+ PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
+ }
+ pthread_rwlock_init(&rwlock, attr);
+ rwlock_inited = true;
+ } else {
+ return -1;
+ }
+ return 0;
+}
+
static void futex_test_driver(const char *futex_type,
int (*proc_type)(const char **ptype),
void *(*workerfn)(void *arg))
@@ -565,13 +1110,19 @@ static void futex_test_driver(const char *futex_type,
*/
const char *desc[STAT_NUM] = {
[STAT_OPS] = "Total exclusive locking ops",
+ [STAT_SOPS] = "Total shared locking ops",
[STAT_LOCKS] = "Exclusive lock futex calls",
[STAT_UNLOCKS] = "Exclusive unlock futex calls",
[STAT_SLEEPS] = "Exclusive lock sleeps",
+ [STAT_SLOCKS] = "Shared lock futex calls",
+ [STAT_SUNLOCKS] = "Shared unlock futex calls",
+ [STAT_SSLEEPS] = "Shared lock sleeps",
[STAT_WAKEUPS] = "Process wakeups",
[STAT_EAGAINS] = "EAGAIN lock errors",
[STAT_LOCKERRS] = "\nExclusive lock errors",
[STAT_UNLKERRS] = "\nExclusive unlock errors",
+ [STAT_SLOCKERRS] = "\nShared lock errors",
+ [STAT_SUNLKERRS] = "\nShared unlock errors",
};

if (exit_now)
@@ -583,9 +1134,18 @@ static void futex_test_driver(const char *futex_type,
}

printf("\n=====================================\n");
- printf("[PID %d]: %d threads doing %s futex lockings (load=%d) for %d secs.\n\n",
+ printf("[PID %d]: %d threads doing %s futex lockings (load=%d) for %d secs.\n",
getpid(), nthreads, futex_type, loadlat, nsecs);

+ if (xthread) {
+ /*
+ * Compute numbers of reader and writer threads.
+ */
+ rthread_threshold = (rpercent * nthreads + 50)/100;
+ printf("\t\t{%d reader threads, %d writer threads}\n",
+ rthread_threshold, nthreads - rthread_threshold);
+ }
+ printf("\n");
init_stats(&throughput_stats);

*pfutex = 0;
@@ -655,11 +1215,15 @@ static void futex_test_driver(const char *futex_type,
/*
* Get a rounded estimate of the # of locking ops/sec.
*/
- u64 tp = (u64)worker[i].stats[STAT_OPS] * 1000000 / us;
+ u64 tp = (u64)(worker[i].stats[STAT_OPS] +
+ worker[i].stats[STAT_SOPS]) * 1000000 / us;

for (j = 0; j < STAT_NUM; j++)
total.stats[j] += worker[i].stats[j];

+ for (j = 0; j < TIME_NUM; j++)
+ total.times[j] += worker[i].times[j];
+
update_stats(&throughput_stats, tp);
if (verbose)
printf("[thread %3d] futex: %p [ %'ld ops/sec ]\n",
@@ -683,21 +1247,41 @@ static void futex_test_driver(const char *futex_type,
if (total.stats[STAT_UNLOCKS])
printf("Avg exclusive unlock syscall = %'ldns\n",
total.times[TIME_UNLK]/total.stats[STAT_UNLOCKS]);
+ if (total.stats[STAT_SLOCKS])
+ printf("Avg shared lock syscall = %'ldns\n",
+ total.times[TIME_SLOCK]/total.stats[STAT_SLOCKS]);
+ if (total.stats[STAT_SUNLOCKS])
+ printf("Avg shared unlock syscall = %'ldns\n",
+ total.times[TIME_SUNLK]/total.stats[STAT_SUNLOCKS]);
}

printf("\nPercentages:\n");
if (total.stats[STAT_LOCKS])
printf("Exclusive lock futex calls = %.1f%%\n",
stat_percent(&total, STAT_LOCKS, STAT_OPS));
+ if (total.stats[STAT_SLOCKS])
+ printf("Shared lock futex calls = %.1f%%\n",
+ stat_percent(&total, STAT_SLOCKS, STAT_SOPS));
if (total.stats[STAT_UNLOCKS])
printf("Exclusive unlock futex calls = %.1f%%\n",
stat_percent(&total, STAT_UNLOCKS, STAT_OPS));
+ if (total.stats[STAT_SUNLOCKS])
+ printf("Shared unlock futex calls = %.1f%%\n",
+ stat_percent(&total, STAT_SUNLOCKS, STAT_SOPS));
if (total.stats[STAT_EAGAINS])
printf("EAGAIN lock errors = %.1f%%\n",
- stat_percent(&total, STAT_EAGAINS, STAT_LOCKS));
+ (double)total.stats[STAT_EAGAINS] * 100 /
+ (total.stats[STAT_LOCKS] + total.stats[STAT_SLOCKS]));
if (total.stats[STAT_WAKEUPS])
printf("Process wakeups = %.1f%%\n",
- stat_percent(&total, STAT_WAKEUPS, STAT_UNLOCKS));
+ (double)total.stats[STAT_WAKEUPS] * 100 /
+ (total.stats[STAT_UNLOCKS] +
+ total.stats[STAT_SUNLOCKS]));
+ if (xthread)
+ printf("Reader operations = %.1f%%\n",
+ (double)total.stats[STAT_SOPS] * 100 /
+ (total.stats[STAT_OPS] + total.stats[STAT_SOPS]));
+

printf("\nPer-thread Locking Rates:\n");
printf("Avg = %'d ops/sec (+- %.2f%%)\n", (int)(avg + 0.5),
@@ -705,28 +1289,55 @@ static void futex_test_driver(const char *futex_type,
printf("Min = %'d ops/sec\n", (int)throughput_stats.min);
printf("Max = %'d ops/sec\n", (int)throughput_stats.max);

+ /*
+ * Compute the averagge reader and writer locking operation rates
+ * with separate reader and writer threads.
+ */
+ if (xthread) {
+ u64 tp;
+
+ /* Reader stats */
+ memset(&throughput_stats, 0, sizeof(throughput_stats));
+ for (i = 0, tp = 0; i < rthread_threshold; i++) {
+ tp = (u64)worker[i].stats[STAT_SOPS] * 1000000 / us;
+ update_stats(&throughput_stats, tp);
+ }
+ avg = avg_stats(&throughput_stats);
+ stddev = stddev_stats(&throughput_stats);
+ printf("\nReader avg = %'d ops/sec (+- %.2f%%)\n",
+ (int)(avg + 0.5), rel_stddev_stats(stddev, avg));
+
+ /* Writer stats */
+ memset(&throughput_stats, 0, sizeof(throughput_stats));
+ for (tp = 0; i < (int)nthreads; i++) {
+ tp = (u64)worker[i].stats[STAT_OPS] * 1000000 / us;
+ update_stats(&throughput_stats, tp);
+ }
+ avg = avg_stats(&throughput_stats);
+ stddev = stddev_stats(&throughput_stats);
+ printf("Writer avg = %'d ops/sec (+- %.2f%%)\n",
+ (int)(avg + 0.5), rel_stddev_stats(stddev, avg));
+ }
+
if (*pfutex != 0)
printf("\nResidual futex value = 0x%x\n", *pfutex);

/* Clear the workers area */
memset(worker, 0, sizeof(*worker) * nthreads);
+
+ if (rwlock_inited)
+ pthread_rwlock_destroy(&rwlock);
+ if (rwlock_attr_inited)
+ pthread_rwlockattr_destroy(&rwlock_attr);
}

-int bench_futex_mutex(int argc, const char **argv,
- const char *prefix __maybe_unused)
+static void bench_futex_common(struct sigaction *act)
{
- struct sigaction act;
-
- argc = parse_options(argc, argv, mutex_options,
- bench_futex_mutex_usage, 0);
- if (argc)
- goto err;
-
ncpus = sysconf(_SC_NPROCESSORS_ONLN);

- sigfillset(&act.sa_mask);
- act.sa_sigaction = toggle_done;
- sigaction(SIGINT, &act, NULL);
+ sigfillset(&act->sa_mask);
+ act->sa_sigaction = toggle_done;
+ sigaction(SIGINT, act, NULL);

if (!nthreads)
nthreads = ncpus;
@@ -744,6 +1355,19 @@ int bench_futex_mutex(int argc, const char **argv,

if (!fshared)
flags = FUTEX_PRIVATE_FLAG;
+}
+
+int bench_futex_mutex(int argc, const char **argv,
+ const char *prefix __maybe_unused)
+{
+ struct sigaction act;
+
+ argc = parse_options(argc, argv, mutex_options,
+ bench_futex_mutex_usage, 0);
+ if (argc)
+ goto err;
+
+ bench_futex_common(&act);

if (!ftype || !strcmp(ftype, "all")) {
futex_test_driver("WW", futex_mutex_type, mutex_workerfn);
@@ -757,3 +1381,30 @@ int bench_futex_mutex(int argc, const char **argv,
usage_with_options(bench_futex_mutex_usage, mutex_options);
exit(EXIT_FAILURE);
}
+
+int bench_futex_rwlock(int argc, const char **argv,
+ const char *prefix __maybe_unused)
+{
+ struct sigaction act;
+
+ argc = parse_options(argc, argv, rwlock_options,
+ bench_futex_rwlock_usage, 0);
+ if (argc)
+ goto err;
+
+ ncpus = sysconf(_SC_NPROCESSORS_ONLN);
+
+ bench_futex_common(&act);
+
+ if (!ftype || !strcmp(ftype, "all")) {
+ futex_test_driver("WW", futex_rwlock_type, rwlock_workerfn);
+ futex_test_driver("GC", futex_rwlock_type, rwlock_workerfn);
+ } else {
+ futex_test_driver(ftype, futex_rwlock_type, rwlock_workerfn);
+ }
+ free(worker_alloc);
+ return 0;
+err:
+ usage_with_options(bench_futex_rwlock_usage, rwlock_options);
+ exit(EXIT_FAILURE);
+}
diff --git a/tools/perf/builtin-bench.c b/tools/perf/builtin-bench.c
index bf4418d..424a014 100644
--- a/tools/perf/builtin-bench.c
+++ b/tools/perf/builtin-bench.c
@@ -64,6 +64,7 @@ struct bench {
/* pi-futexes */
{ "lock-pi", "Benchmark for futex lock_pi calls", bench_futex_lock_pi },
{ "mutex", "Benchmark for mutex locks using futexes", bench_futex_mutex },
+ { "rwlock", "Benchmark for rwlocks using futexes", bench_futex_rwlock },
{ "all", "Run all futex benchmarks", NULL },
{ NULL, NULL, NULL }
};
--
1.8.3.1