Re: [patch] x86: unify/rewrite SMP TSC sync code

[Robert Crocombe]

The difference that Wink reports is tiny compared to that measured on
my Opteron machines:

dual (2.6.17):

rcrocomb@netfires-aptest:cyclecounter_test$ ./rdtsc-pref 1000000
rdtsc:   average ticks=  10
gtod:    average ticks=4296
gtod_us: average ticks=4328

quad (2.6.16-rt29):

rcrocomb@spanky:wink_saville_test$ ./rdtsc-pref 1000000
rdtsc:   average ticks=  10
gtod:    average ticks=5688
gtod_us: average ticks=5711

I have my own little test that I'll attach, but it gives a similar
result.  Here are the results from the 2x box:

rcrocomb@netfires-aptest:cyclecounter_test$ ./timing
Using the cycle counter
Calibrated timer as 2593081969.758825 Hz
4194304 iterations in 0.016 seconds is 0.004 useconds per iteration.

rcrocomb@netfires-aptest:cyclecounter_test$ ./timing_gettimeofday
Using gettimeofday
4194304 iterations in 6.793 seconds is 1.620 useconds per iteration.

I have used the pthread affinity and/or cpuset, etc. mechanisms to try
and inject some reliability into the measurement.

Using gtod() can amount to a substantial disturbance of the thing to
be measured.  Using rdtsc, things seem reliable so far, and we have an
FPGA (accessed through the PCI bus) that has been programmed to give
access to an 8MHz clock and we do some checks against that.

--
Robert Crocombe
rcrocomb@xxxxxxxxx
#include <stdio.h>                                  // printf()
#include <stdint.h>                                 // uint64_t
#include <stdlib.h>                                 // drand48()
#include <sys/select.h>                             // select()
#include <sys/time.h>                               // gettimeofday
#include <time.h>
#include <asm-x86_64/msr.h>                         // rdtscll()


////////////////////////////////////////////////////////////////////////////////
// Globals
////////////////////////////////////////////////////////////////////////////////

enum
{
    ITERATIONS  = 1 << 22
};

static double seconds_per_tick;

////////////////////////////////////////////////////////////////////////////////
// Prototypes
////////////////////////////////////////////////////////////////////////////////

double gimme_timeofday(void);
double get_time(void);

void selectsleep(unsigned us);
void init(void);

////////////////////////////////////////////////////////////////////////////////
// Definitions
////////////////////////////////////////////////////////////////////////////////

double
gimme_timeofday(void)
{
    struct timeval tv;
    gettimeofday(&tv, 0);
    return tv.tv_sec + 1e-6 * tv.tv_usec;
}


double
get_time(void)
{
    uint64_t t;
    rdtscll(t);
    return t * seconds_per_tick;
}


/**
    A good way to simply hang around doing nothing for awhile.
*/

void
selectsleep(unsigned us)
{
	struct timeval tv;
	tv.tv_sec = 0;
	tv.tv_usec = us;
	select(0,0,0,0,&tv);
}

/**
    Figure out how fast rdtscll() ticks.  This should be equal to the
    frequency of the clock on the processor.  Here's the bad news: I don't
    know if rdtscll() always uses the same processor so it may very well be
    necessary to set a processor affinity to get really good results over
    time.

    This piece of code by Mark Hahn from brain.mcmaster.ca/~hahn/.
*/

void
init(void)
{
	double sumx = 0;
	double sumy = 0;
	double sumxx = 0;
	double sumxy = 0;
	double slope;

	// least squares linear regression of ticks onto real time
	// as returned by gettimeofday.

	const unsigned n = 30;
	unsigned i;

	for ( unsigned int i = 0; i < n; ++i)
        {
		double breal,real,ticks;
		uint64_t aticks, bticks;
	
		breal = gimme_timeofday();
		rdtscll(bticks);

		selectsleep((unsigned)(10000 + drand48() * 200000));

                rdtscll(aticks);
		ticks = aticks - bticks;
		real = gimme_timeofday() - breal;

		sumx += real;
		sumxx += real * real;
		sumxy += real * ticks;
		sumy += ticks;
	}

	slope = ((sumxy - (sumx*sumy) / n) / (sumxx - (sumx*sumx) / n));
	seconds_per_tick = 1.0 / slope;

        printf("Calibrated timer as %.6f Hz\n", slope);
}

int
main(int argc, char *argv[])
{
    printf("Doing stuff\n");

#if 0   // Using rdtscll()
    printf("Using the cycle counter\n");
    init();
    double time_start = gimme_timeofday();
    for (unsigned int i = 0; i < ITERATIONS; ++i)
    {
        double the_time;
        the_time = get_time();
    }
    double time_end = gimme_timeofday();
#else   // using gettimeofday()
    printf("Using gettimeofday\n");
    double time_start = gimme_timeofday();
    for (unsigned int i = 0; i < ITERATIONS; ++i)
    {
        double the_time;
        the_time = gimme_timeofday();
    }
    double time_end = gimme_timeofday();
#endif

    double diff = time_end - time_start;

    double useconds = (diff / ITERATIONS) * 1e6;

    printf("%u iterations in %.3f seconds is %.3f useconds per iteration.\n",
          ITERATIONS, diff, useconds);

    printf("Done\n");
    return 0;
}