I am having some trouble with atomic operations in "mmap" mapped
address space... These work fine when the same memory is mapped with "shmat"
!!!
Here is the test code I am using in a Linux SMP Machine with
2.0.27... What it does is to launch several processes that try to increment
a shared counter a certain number of times.
It shoud be compiled with
gcc -O -o lock_test lock_test.c
Before using it you should create with dd 2 1k files in the same directory
named ./foo and ./pid_t
The program usage is : lock_test mmap|shm <n_workers> <number>
When used with the "shm" option everything works fine..
When used with the "mmap" option the atomic operations do not work the
way they should !! 8-(...
Try it with big numbers... Otherwise the bug will not show up !!!
ex) ./lock mmap 5 100000
I expect you to see lost count and deadlocks with mmap and no
problems with shm !!
I would like to know if anyone else is getting this same results and if
by any chance there is a patch that solves this mmap problem...
If someone is already working in this problem I would like to
volunteer to receive patches and do some beta testing...
Thanks,
mcc
Here is the code :
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/ipc.h>
#include <sys/shm.h>
/*
* Some hacks to defeat gcc over-optimizations.. Based in Kernel code !!
*/
#define ADDR (*(struct __dummy *) addr)
struct __dummy { unsigned long a[100]; };
typedef volatile int slock_t;
extern inline int set_bit(int nr, volatile void * addr)
{
int oldbit;
asm volatile ("
lock ;
btsl %2,%1\n\tsbbl %0,%0"
:"=r" (oldbit),"=m" (ADDR)
:"ir" (nr));
return oldbit;
}
extern inline int clear_bit(int nr, volatile void * addr)
{
int oldbit;
asm volatile("
lock ;
btrl %2,%1\n\tsbbl %0,%0"
:"=r" (oldbit),"=m" (ADDR)
:"ir" (nr));
return oldbit;
}
#define S_LOCK(p)\
{\
while(set_bit(0,(p)) ) ;\
}
#define S_UNLOCK(p) (clear_bit(0,(p)))
#define S_CLOCK(p) (!set_bit(0,(p)))
#define S_INIT_LOCK(p) S_UNLOCK(p)
#define ATOMIC_INCR(addr) \
{ \
asm volatile("/* inlined atomic_incr */
lock
incl %0" \
: \
: "rm" (*(addr))); \
}
#define LOCK(X) S_LOCK(&(X))
#define UNLOCK(X) S_UNLOCK(&(X))
#define INIT_LOCK(X) S_INIT_LOCK(&(X))
#define COND_LOCK(X) S_CLOCK(&(X))
#define SHM 1
#define MMAP 2
int shared_mem = MMAP;
typedef struct {
int last_worker;
slock_t lock;
volatile long counter;
volatile int n_workers;
} COUNTER;
long n_times;
pid_t *worker_pid;
work(int worker, COUNTER *c)
{
int i;
if( worker > 0 )
{
if( fork() )
{
return;
}
else
if( worker_pid[worker] == -1 )
{
printf("Could not initiate worker[%d]", worker );
perror("");
}
}
worker_pid[worker] = getpid();
printf("Worker[pid=%-4d,n=%-4d] started\n", worker_pid[worker],worker);
fflush(stdout);
for( i = 0 ; i < n_times; i++ )
{
#ifdef ATOMIC
ATOMIC_INCR(&(c->counter));
#else
LOCK(c->lock);
#ifdef MCC_DEBUG
if( worker != c->last_worker && c->last_worker != -1 )
{
fprintf(stderr,"worker[pid=%-4d,n=%-2d] grabbed lock from worker[pid=%-4d,n=%-2d]\t",
worker_pid[worker], worker,
worker_pid[c->last_worker],
c->last_worker );
fprintf(stderr,"counter=%d\n", c->counter );
fflush(stdout);
}
c->last_worker = worker;
#endif
c->counter ++;
UNLOCK(c->lock);
#endif
}
printf("Worker[%d] ended\n", worker);
fflush(stdout);
LOCK(c->lock);
--c->n_workers;
UNLOCK(c->lock);
printf("Worker[%d] exiting with counter=%d\n", worker, c->counter );
if( worker != 0 ) exit(0);
}
main(int argc, char *argv[])
{
COUNTER *counter;
int n_workers,i;
int fd_counter, fd_pid;
char *c;
int shmid;
umask(077);
if( argc < 3 )
{
printf("Usage : %s mmap|shm <n_workers> <number>\n", *argv );
exit(1);
}
else c = argv[1];
if( strcmp(c,"mmap" ) == 0 )
shared_mem = MMAP;
else if( strcmp(c,"shm") == 0 )
shared_mem = SHM;
else
{
printf("Usage : %s -mmap|shm <n_workers> <number>\n", *argv );
exit(1);
}
if( shared_mem == MMAP )
{
printf("Using BSD mmap\n");
fflush(stdout);
if( (fd_counter = open("./foo", O_RDWR, 0700 )) == -1 )
{
perror("Could not open ./foo\n");
exit(3);
}
if( (long) ( counter = (COUNTER *)
mmap( 0, sizeof(COUNTER),
PROT_READ | PROT_WRITE, MAP_SHARED ,
fd_counter , 0 ) ) == -1 )
{
perror("Allocation of shared memory failed");
exit(2);
}
}
else
{
printf("Using SYSV SHM \n");
fflush(stdout);
if( ( shmid = shmget(IPC_PRIVATE, sizeof(COUNTER), 0600)) == -1 )
{
perror("shmget failed :");
exit(1);
}
if( ( counter = (COUNTER *) shmat(shmid,0,0) ) == (void *) -1 )
{
perror("shmat failed");
}
}
printf("\nCounter allocated at %lx\n\n", (long) counter );
fflush(stdout);
n_workers = atoi(argv[2]);
if( (fd_pid = open("./pid_t", O_RDWR, 0700 )) == -1 )
{
perror("Could not open ./pid_t\n");
exit(3);
}
if( ( worker_pid = (pid_t *) mmap( 0,sizeof(pid_t) * n_workers, PROT_READ | PROT_WRITE, MAP_SHARED, fd_pid , 0 ) ) == (pid_t *) -1 )
{
printf("Could not allocate pid_t space, Aborting --> ");
perror(NULL);
}
worker_pid[0] = getpid();
n_times = atoi(argv[3]);
INIT_LOCK(counter->lock);
counter -> counter = 0;
counter -> last_worker = -1;
counter -> n_workers = n_workers;
for( i = 1 ; i < n_workers ; i++ ) work(i, counter );
work(0, counter);
while( counter->n_workers ) ; /* Wait for everyone to finish */
puts("\n\n");
for( i = 1 ; i < n_workers ; i++ )
{
printf("Going to wait for process %d to finish\n", worker_pid[i]);
waitpid(worker_pid[i], NULL, 0 );
printf("Worker[%d] finished\n", i );
}
puts("Worker[0] finished\n");
printf("\n\n -----> %d * %d == %d <----- \n\n", n_workers, n_times , counter -> counter );
if( shared_mem == SHM )
{
shmdt((void *)counter);
shmctl(shmid,IPC_RMID,NULL);
}
}