It seems that if you use mmap(),memcpy(),munmap() to read/write to a
portion of a file, you can get corrupt data. There appears to be no
thread-safe alternative besides this method to do offset-thread-safe
reads and writes on Linux.
We are using RedHat's kernel 2.0.36-3, and glibc 2.0.7-29.
Does this program work on the "2.2" kernels?
This program makes it appear that mmap() is really broken in 2.0.36.
-Rob
----
/*****************************************************************************
File: writerTestLinux.C
Contents: mmap() Bug Illustration
Created: 1-Apr-1999
Compile as:
g++ -D_REENTRANT -Wall -Werror -g -o writerTestLinux writerTestLinux.C -lpthread
Description:
This program spawns 10 threads that write and then read data from
a file using mmap() [mostly because pread/pwrite don't exist on
Linux]. The program shows that although the data being written and
read do not overlap, the data becomes corrupt.
Operation:
We create a large file filled with 'x's
We create 10 threads. Each with a "letter" (a,b,c etc.) to write into the
file.
Each thread, reserves a 500 byte chunk of the file by locking the
globalOffset variable, incrementing it by 500, and unlocking
Now that the "chunk" is reserved, the thread write's and read's
its "letter" to the reserved region of the file.
It compares what it read to what it wrote and dies if they don't match.
Theory/Guess:
Because we don't have pread/pwrite, we have to use mmap(). mmap() requires
page alignment on the "offset" parameter. This causes us to mmap() a region
larger than what we are writing/reading. Although we don't TOUCH those extra
bits, perhaps there is a bug/race-condition involving overlapping mmap()'s
that causes the data to become corrupt.
*****************************************************************************/
#include <assert.h>
#include <fcntl.h>
#include <memory.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/time.h>
#include <unistd.h>
unsigned int threadCount = 10;
const char *fileName = "testFile";
pthread_mutex_t offsetLock;
size_t globalOffset = 0;
int fd = -1;
//
// Linux doesn't have "pread()/pwrite()", so, we emulate it using mmap().
// We do, of course, have to play a little page alignment trick. Not
// too tricky though.
//
static bool preadLinuxEmulation(int fd, void *buf, size_t nbyte, off_t offset)
{
if (!buf || (offset < 0)) { return false; }
int modulo = offset % ::getpagesize();
caddr_t address = static_cast<caddr_t>(::mmap(0, nbyte + modulo,
PROT_READ, MAP_SHARED, fd,
offset - modulo));
if (address == MAP_FAILED) { return false; }
::memcpy(buf, address + modulo, nbyte);
if (::munmap(address, nbyte + modulo)) { return false; }
return true;
}
static bool pwriteLinuxEmulation(int fd, void *buf, size_t nbyte, off_t offset)
{
if (!buf || (offset < 0)) { return false; }
int modulo = offset % ::getpagesize();
caddr_t outBuf = static_cast<caddr_t>(::mmap(0, nbyte + modulo,
PROT_READ | PROT_WRITE,
MAP_SHARED, fd,
offset - modulo));
if (outBuf == MAP_FAILED) { return false; }
::memcpy(outBuf + modulo, buf, nbyte);
if (::munmap(outBuf, nbyte + modulo)) { return false; }
return true;
}
static void *writeThread(void *arg)
{
// Get the Letter to write() from the argument
char letter = *(static_cast<char *>(arg));
delete arg;
// We are going to write 500 bytes of our letter to SOMEPLACE in the file!
char bufIn[500], bufOut[500];
::memset(bufIn, letter, 500);
::printf("Thread %c starting Write/Read Loop.\n", letter);
for (int x = 0; x < 10000; ++x) {
// SAFELY grab an offset so you don't overlap another thread.
::pthread_mutex_lock(&offsetLock);
size_t offset = globalOffset;
globalOffset += 500;
::pthread_mutex_unlock(&offsetLock);
// Write and Read to the file!
assert(pwriteLinuxEmulation(fd, bufIn, 500, offset));
assert(preadLinuxEmulation(fd, bufOut, 500, offset));
// What you read, BETTER match what you wrote!!!
if (::memcmp(bufIn, bufOut, 500)) {
//
// YUCK!!!! MMAP() IS BROKEN!!!!
//
::printf("mmap() is seriously broken!\n");
::printf("Thread %c wrote data at %u that doesn't match the read that followed\n", letter, offset);
char buf[501];
buf[500] = '\0';
::memcpy(buf, bufIn, 500);
::printf("Data written: %s\n", buf);
::memcpy(buf, bufOut, 500);
::printf("Data read: %s\n", buf);
::abort();
}
}
::printf("%c Exiting.\n", letter);
return 0;
}
int main(int, char **)
{
::printf("Initializing File\n");
::unlink(fileName);
// Create a 8192 * 10000 byte file full of "x"'s
char buf[8192];
::memset(buf, 'x', sizeof(buf));
fd = ::open(fileName, O_RDWR | O_CREAT, 0666);
assert(fd != -1);
for (int x = 0; x < 10000; ++x) {
assert(::write(fd, buf, sizeof(buf)) == sizeof(buf));
}
// Initialize the global offset lock (to keep things synchronized)
pthread_mutexattr_t attr;
assert(!::pthread_mutexattr_init(&attr));
assert(!::pthread_mutex_init(&offsetLock, &attr));
assert(!::pthread_mutexattr_destroy(&attr));
// Okay, "fileName" is now a big file full of "x"'s
// Start some threads, each assigned a letter ("a", "b", "c", etc.) to
// begin writing and reading from that file.
::printf("Starting Threads\n");
char argument = 'a'; // Start with the letter 'a'
for (unsigned int k = 0; k < threadCount; ++k) {
// Initialize Thread Attributes
pthread_t tid;
pthread_attr_t attr;
assert(!::pthread_attr_init(&attr));
assert(!::pthread_attr_setdetachstate(&attr,
PTHREAD_CREATE_DETACHED));
assert(!::pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM));
// Spawn the thread
assert(!::pthread_create(&tid, &attr, writeThread,
new char(argument)));
// Cleanup Thread Attributes
assert(!::pthread_attr_destroy(&attr));
++argument; // Go to the next letter
}
// Just sleep for 30 seconds.
::printf("Waiting for Threads\n");
timeval timeout;
timeout.tv_sec = 30;
timeout.tv_usec = 0;
::select(0, 0, 0, 0, &timeout);
::printf("Test Exiting\n");
::unlink(fileName);
return EXIT_SUCCESS;
}
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.rutgers.edu
Please read the FAQ at http://www.tux.org/lkml/