Re: Improve spinlock performance by moving work to one core

[Ling Ma]

Hi Longman,

Attachments include user space application thread.c and kernel patch
spinlock-test.patch based on kernel 4.3.0-rc4

we run thread.c with kernel patch, test original and new spinlock respectively,
perf top -G indicates thread.c cause cache_alloc_refill and
cache_flusharray functions to spend ~25% time on original spinlock,
after introducing new spinlock in two functions, the cost time become ~22%.

The printed data  also tell us the new spinlock improves performance
by about 15%( 93841765576 / 81036259588) on E5-2699V3

Appreciate your comments.

Thanks
Ling

2015-11-07 1:38 GMT+08:00 Waiman Long <waiman.long@xxxxxxx>:
>
> On 11/05/2015 11:28 PM, Ling Ma wrote:
>>
>> Longman
>>
>> Thanks for your suggestion.
>> We will look for real scenario to test, and could you please introduce
>> some benchmarks on spinlock ?
>>
>> Regards
>> Ling
>>
>>
>
> The kernel has been well optimized for most common workloads that spinlock contention is usually not a performance bottleneck. There are still corner cases where there is heavy spinlock contention.
>
> I used a spinlock loop microbenchmark like what you are doing as well as AIM7 for application level testing.
>
> Cheers,
> Longman
>
>

Attachment: spinlock-test.patch
Description: Binary data

/**
	Test Case:
		OpenDir, Get status and close it.
*/
#include <unistd.h>
#include <dirent.h>
#include <sys/stat.h>
#include <sys/fcntl.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <pthread.h>

#define TEST_DIR "/tmp/thread"
#define MAX_TEST_THREAD (80)
#define MAX_TEST_FILE 5000

static unsigned long *result[MAX_TEST_THREAD];
static int stop = 0;

static void* case_function(void *para)
{
	int id = (int)(long)para;
	DIR *pDir;
	struct stat f_stat;
	struct dirent *entry=NULL;
	char path[256];
	char cmd[512];
	
	int filecnt       = 0;
	int dircnt        = 0;
	int filetotalsize = 0;
	unsigned long myresult = 0;
	int f = 0;
	
	result[id] = &myresult;

	/* Goto my path and construct empty file */
	sprintf(path, "%s/%d", TEST_DIR, id);
	printf("Creating temp file at %s....\n", path);

	sprintf(cmd, "mkdir %s", path);
	system(cmd);
	chdir(path);
	for (f = 0; f < MAX_TEST_FILE; f++)
	{
		char name[256];

		sprintf(name, "%s/%d", path, f);
		int t = open(name,  O_RDWR | O_CREAT | O_TRUNC, S_IRWXU);
		if (t != -1)
			close(t);
		else
		{
			printf("Errno = %d.\n", errno);
			exit(errno);
		}		
	}

again:
	if ((pDir = opendir(path)) == NULL)
	{
		printf("打开 %s 错误：没有那个文件或目录\n", TEST_DIR);
		goto err;
	}
	
	while ((entry = readdir(pDir)) != NULL)
	{
		struct stat buf;
		if (entry->d_name[0] == '.')
			continue;
		
		//f = open(entry->d_name, 0);
		f = stat(entry->d_name, &buf);
		
		if (f)
			close(f);
		myresult++;
		
		
		//printf("Filename %s, size %10d",entry->d_name, f_stat.st_size);
	}

	closedir(pDir);
	

	/* Need to stop */
	if (!stop)
		goto again;
	return 0;

err:
	;
}

void main()
{
	int i;
	pthread_t thread;

	system("mkdir "TEST_DIR);
		
	for (i = 0; i < MAX_TEST_THREAD; i++)
	{
		pthread_create(&thread, NULL, case_function, (void*)(long)i);
	}

	while (1)
	{
		sleep(1);
		printf("Statistics:\n");

		for (i = 0; i < MAX_TEST_THREAD; i++)
		{
			printf("%d\t", *result[i]);
		}
		printf("\n");
		for (i = 0; i < MAX_TEST_THREAD; i++)
			*result[i] = 0;
	}
}