Re: TCP crashes kernel

Wolfgang Walter (wolfgang.walter@stusta.mhn.de)
Fri, 27 Feb 1998 11:25:42 +0100


On Tue, Feb 24, 1998 at 04:40:23PM -0500, William T. Waters wrote:
> I have discovered a problem with what looks like the networking layer of the
> Linux kernel. The following program will consistently crash an intel box.
> It is crippling our efforts to port our software to Linux -- under any sort of
> load whatsoever, machines are rolling over & just dying. Sometimes the
> program will SEGV or get a Bus Error, sometimes we get kernel syslogs, and
> frequently the whole machine will seize up completely.

For the following I used libc 5.4.44 and linux-pthread 0.7, kernel 2.0.33

I run your program over and over again, several hours, but I did not have any
kernel related problems. The only change I made was to replace the poll()
syscall with an equivalent select() as I have libc5. Of course, your program
segfaulted, but for good reasons.

If I compile it with -D_REENTRANT (which one should), the compile gave me an
error. Indeed, at least under libc 5.4.44 the gethostbyname_r() and
gethostbyaddr_r() require different arguments than those you gave:

I had to replace
gethostbyaddr_r((char *)(&IP), sizeof(int),
AF_INET, &ent, buf, MAXHOSTENT, &pRes, &err)
by
gethostbyaddr_r((char *)(&IP), sizeof(int),
AF_INET, &ent, buf, MAXHOSTENT, &err)

and
gethostbyname_r(HOSTNAME, &hp, buf,
MAXHOSTENT * sizeof(char), &pRes, &err)
by
gethostbyname_r(HOSTNAME, &ent, buf, MAXHOSTENT, &err)

Now, I get again segfaults because I left HOSTNAME and IP_ADDRESS untouched
(i.e. gethostbyname_r and gethostbyaddr_r should fail, which means
gethostbyname_r and gethostbyaddr_r should return NULL, at least with libc5).
But you continues exactly in this situation:
assert(!gethostbyaddr_r....)
....

So you dereference in generateTraffic an invalid pointer:

memcpy(&location.sin_addr.s_addr, hp.h_addr_list[0],
sizeof(location.sin_addr.s_addr));

Changing HOSTNAME and IP_ADDRESS to valid ones cause your program to abort.

So, if one corrects these assert() statements, your program works perfectly
for me and this for several hours if I encrease the time I wait in select.

Here my exact changes against your code

>
> Alan Cox has confirmed that this is a problem in 2.0.33, and I just repeated
> it on the latest 2.1.88 kernel as well.
>
> Is anyone else seeing this error in the "real world"?
>
> Any help in fixing this would be GREATLY appreciated.
>
> Bill
>
>
> /*
>
> This C program spawns 3 types of threads and lets them run.
> The first simply loops reverse DNS'ing an IP address
> The second loops forward DNS'ing a hostname
> The third simply generates network traffic by reading from a neighboring
> machine's chargen port
>
> To compile & run:
> 1) Set the following constants:
> */
> #define HOSTNAME "your.chargen.machine" /* For forward DNS & chargen */
> /* Best to use a machine besides */
> /* the one on which we're running */

replaced with an valid one.

>
> #define IP_ADDRESS "111.222.333.444" /* for reverse DNS */

dito.

>
> #define CHARGEN_PORT 19 /* Probably don't need to change */
>
> #define NUMTIMES 5 /* How many of each thread type to */
> /* create -- you may need to */
> /* adjust this higher especially */
> /* if you have a fast box/network*/
>
>
>
> /*

replace

> 2) Type: gcc -o crash -lpthread crash.c

by

2) Type: gcc -o crash -lpthread -D_REENTRANT crash.c

> 3) Type: ./crash
>
> This crashes, usually generating kernel syslogs and/or crashing the box.
> */
>
> #include <arpa/inet.h>
> #include <assert.h>
> #include <netdb.h>
> #include <netinet/in.h>
> #include <errno.h>
> #include <poll.h>
> #include <pthread.h>
> #include <stdlib.h>
> #include <string.h>
> #include <sys/socket.h>
> #include <unistd.h>
>
> #define MAXHOSTENT 16384
>
> static pthread_t spawnThread(void *(*function)(void *))
> {
> pthread_t tid;
> pthread_attr_t attr;
>
> /* sanity */
> if (!function) { return 0; }
>
> /* Initialize thread attributes */
>
> assert(!pthread_attr_init(&attr));
> assert(!pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED));
> assert(!pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM));
> assert(!pthread_create(&tid, &attr, function, 0));
> assert(!pthread_attr_destroy(&attr));
>
> return tid;
> }
>
> void *reverseDNS(void *arg)
> {
> struct hostent ent;
> struct hostent *pRes = 0;
> char *buf = (char *)malloc(sizeof(char) * MAXHOSTENT);
> int err, IP;
>
> while (1) {
> IP = inet_addr(IP_ADDRESS);

replace

> assert (!gethostbyaddr_r((char *)(&IP), sizeof(int),
> AF_INET, &ent, buf, MAXHOSTENT, &pRes, &err));

by

assert (gethostbyaddr_r((char *)(&IP), sizeof(int),
AF_INET, &ent, buf, MAXHOSTENT, &err));

> }
>
> free(buf);
> return 0;
> }
>
> void *forwardDNS(void *arg)
> {
> int err;
> char *buf = (char *)malloc(sizeof(char) * MAXHOSTENT);
> struct hostent ent;
> struct hostent *pRes = 0;
>
> memset(buf, 0, MAXHOSTENT);
>
> while(1) {

replace

> assert(!gethostbyname_r(HOSTNAME, &ent, buf, MAXHOSTENT, &pRes, &err));

by

assert(gethostbyname_r(HOSTNAME, &ent, buf, MAXHOSTENT, &err));

> }
>
> free(buf);
> return 0;
> }
>
> void *generateTraffic(void *arg)
> {
> /* Look up the IP */
> struct hostent hp, *pRes;
> char *buf = (char *)malloc(sizeof(char) * MAXHOSTENT);
> int err, fd;
> struct sockaddr_in location;
> char buffer[1024];
>

replace

> assert(!gethostbyname_r(HOSTNAME, &hp, buf,
> MAXHOSTENT * sizeof(char), &pRes, &err));

by

assert(gethostbyname_r(HOSTNAME, &hp, buf,
MAXHOSTENT * sizeof(char), &err));

> memset(&location, 0, sizeof(location));
> location.sin_family = AF_INET;
> memcpy(&location.sin_addr.s_addr, pRes->h_addr_list[0],
> sizeof(location.sin_addr.s_addr));
> location.sin_port = htons(CHARGEN_PORT);
> free(buf);
>
> /* Create a socket & connect */
> assert((fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) != -1);
> assert(connect(fd, (struct sockaddr *)(&location),
> sizeof(location)) != -1);
>
> while (1) {
> assert(read(fd, buffer, 1024) != -1);
> }
>
> return 0;
> }
>
> int main(int argc, char **argv)
> {
> int i;

add

struct timeval tv;

> for (i = 0; i < NUMTIMES; i++) {
> spawnThread(forwardDNS);
> spawnThread(reverseDNS);
> spawnThread(generateTraffic);
> }
>
> /* Sleep without generating signals */

replace

> poll(0, 0, 60 * 1000);

by

tv.tv_sec = 60;
tv.tv_usec = 0;
select(0, 0, 0, 0, &tv);

>
> return EXIT_SUCCESS;
> }
>
> -
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.rutgers.edu

Here is the complete changed code for those also using libc5:

-------------

/*

This C program spawns 3 types of threads and lets them run.
The first simply loops reverse DNS'ing an IP address
The second loops forward DNS'ing a hostname
The third simply generates network traffic by reading from a neighboring
machine's chargen port

To compile & run:
1) Set the following constants:
*/
#define HOSTNAME "your.chargen.machine" /* For forward DNS & chargen */
/* Best to use a machine besides */
/* the one on which we're running */

#define IP_ADDRESS "111.222.333.444" /* for reverse DNS */

#define CHARGEN_PORT 19 /* Probably don't need to change */

#define NUMTIMES 5 /* How many of each thread type to */
/* create -- you may need to */
/* adjust this higher especially */
/* if you have a fast box/network*/

/*
2) Type: gcc -o crash -D_REENTRANT -lpthread crash.c
3) Type: ./crash

This crashes, usually generating kernel syslogs and/or crashing the box.
*/

#include <arpa/inet.h>
#include <assert.h>
#include <netdb.h>
#include <netinet/in.h>
#include <errno.h>
#include <pthread.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <unistd.h>

#define MAXHOSTENT 16384

static pthread_t spawnThread(void *(*function)(void *))
{
pthread_t tid;
pthread_attr_t attr;

/* sanity */
if (!function) { return 0; }

/* Initialize thread attributes */

assert(!pthread_attr_init(&attr));
assert(!pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED));
assert(!pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM));
assert(!pthread_create(&tid, &attr, function, 0));
assert(!pthread_attr_destroy(&attr));

return tid;
}

void *reverseDNS(void *arg)
{
struct hostent ent;
struct hostent *pRes = 0;
char *buf = (char *)malloc(sizeof(char) * MAXHOSTENT);
int err, IP;

while (1) {
IP = inet_addr(IP_ADDRESS);
assert (gethostbyaddr_r((char *)(&IP), sizeof(int),
AF_INET, &ent, buf, MAXHOSTENT, &err));
}

free(buf);
return 0;
}

void *forwardDNS(void *arg)
{
int err;
char *buf = (char *)malloc(sizeof(char) * MAXHOSTENT);
struct hostent ent;
struct hostent *pRes = 0;

memset(buf, 0, MAXHOSTENT);

while(1) {
assert(gethostbyname_r(HOSTNAME, &ent, buf, MAXHOSTENT, &err));
}

free(buf);
return 0;
}

void *generateTraffic(void *arg)
{
/* Look up the IP */
struct hostent hp, *pRes;
char *buf = (char *)malloc(sizeof(char) * MAXHOSTENT);
int err, fd;
struct sockaddr_in location;
char buffer[1024];

fprintf(stderr, "miau\n");

assert(gethostbyname_r(HOSTNAME, &hp, buf,
MAXHOSTENT * sizeof(char), &err));

memset(&location, 0, sizeof(location));
location.sin_family = AF_INET;
memcpy(&location.sin_addr.s_addr, hp.h_addr_list[0],
sizeof(location.sin_addr.s_addr));
location.sin_port = htons(CHARGEN_PORT);
free(buf);

/* Create a socket & connect */
assert((fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) != -1);
assert(connect(fd, (struct sockaddr *)(&location),
sizeof(location)) != -1);

while (1) {
assert(read(fd, buffer, 1024) != -1);
}

return 0;
}

int main(int argc, char **argv)
{
int i;
struct timeval tv;

for (i = 0; i < NUMTIMES; i++) {
spawnThread(forwardDNS);
spawnThread(reverseDNS);
fprintf(stderr, "hallo: %d\n", i);
spawnThread(generateTraffic);
}

/* Sleep without generating signals */
tv.tv_sec = 60;
tv.tv_usec = 0;
select(0, 0, 0, 0, &tv);

return EXIT_SUCCESS;
}

----

Wolfgang Walter

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.rutgers.edu