Linux TCP/IP problems with "TCP_NODELAY" - hangup in Threads Kernel 2.2.13-rtl2.0

From: Thomas Mohr (TMohr@spb-bremen.de)
Date: Mon Feb 14 2000 - 05:23:50 EST


Some problems with a small socket server on a Linux 2.2.13.

I use a simple Socketserver with TCP/IP based on a pthread concept.
The usage of the sockets is nonblocking and there is a thread for each
connected
client.
In every serve-thread there is small transaction :
  - a) the client sends length of the following data at first (int)
    b) then sends data and waits for the answer of the
       server.
    c) the client reads out data like the server.. and again and again

  - a) the server reads out at first the length (int)
    b) then receives the n-length bytes data
    when the data is fully received it sends back a small positive message
the same way
    length at first and then the data

If the error occurs the server-thread for the connection is in state
reading of the length or reading of data.. the client says he sends the
length and data
but he waits for the answer and waits..
In the situation the server is not killable anymore even with kill -9 and
the reboot cant
terminate too !!

In my following source of the server the location where the server hang is
marked as
as "MARK 11" or "MARK 22" in the function serve.
The server is allways in of these locatition..

How do i test ->
    I start the server with portnumber = 1234 or 3490 !
    I start 4 to 6 clients (that follows 4- 6 server threads)
    the number of succesful transaction is between 2.000.000 up to
14.000.000 (best)
    until all thread are not running anymore !

It seems that the problem only occurs with setting the option
   TCP_NODELAY !
if not its running slow but nice !

If you have any comments to the source or what i did wrong (or not) please
let me know.
It would be great if someone could help me

   Thanks a lot
       Thomas Mohr

source follows !!

#include <stdio.h>
#include <pthread.h>
#include <signal.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <sys/socket.h>
#include <errno.h>

#define MAX_THREADS 10
#define GOODBYE 0

typedef struct s_theadInfo //for thread
{
  pthread_t *threads;
  int idx;
  int sockfd;
} threadInfo;

/* prototypes */
void * serve(void *arg); // Thread-Funktion
void sig_break(int signo); // Signal-Handler
void sig_pipe(int signo); // Signal-Handler
int spbsRecvn(int fd, void *aptr, int nbytes);

/* globals */
int nThreads;
char dbgstatus[MAX_THREADS];
int sockfd;
int DM_ANZ;

/* MAIN */
int main( int argc, char *argv[] )
{
struct sockaddr_in server;
struct sockaddr_in their_addr;
int sin_size;
int length;
int msgsock;
int rval;
pthread_t threadid;
pthread_t threads[MAX_THREADS];
threadInfo *ti;
int idx;
int sockoption;

    if ( argc < 2 ) {
        printf("Aufruf -> dmserver Port\n");
        exit (-1);
    }

    dbgstatus[MAX_THREADS] = 0;

    nThreads = 0;
    bzero(threads, sizeof(threads));
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (sockfd < 0) {
        printf("Unable to open stream socket\n");
        exit(-1);
    }

    /* CTRL-C Handler einhaengen */
    signal(SIGINT, sig_break);
    signal(SIGPIPE, sig_pipe);

    // Name socket using wildcards
    server.sin_family = AF_INET; // TCP/IP
    server.sin_addr.s_addr = INADDR_ANY;
    server.sin_port = htons(atoi(argv[1]));

#ifdef MY_SOCKOPTION
    sockoption = 1;
    if (setsockopt (sockfd, IPPROTO_TCP, TCP_NODELAY, (char *) &sockoption,
sizeof(sockoption)) != 0) {
        printf("Failed to set socket option TCP_NODELAY\n");
        exit (1);
    }
#endif

    // Dem Socket wird eine Port-Nummer zugewiesen
    if (bind(sockfd, (struct sockaddr *)&server, sizeof(server))) {
        printf("Unable to bind stream socket\n");
        exit(-1);
    }
    // name
    length = sizeof(server);
    if (getsockname(sockfd, (struct sockaddr *)&server, &length)) {
        printf("Unable to get socket name\n");
        exit(-1);
    }

    printf("Socket has port #%d\n", ntohs(server.sin_port));

    // Start accepting connections
    listen(sockfd, 1);

    for (;;) {
        sin_size = sizeof(struct sockaddr_in);
        msgsock = accept(sockfd, (struct sockaddr *)&their_addr,
&sin_size );

        /* anyone connected */
        if (msgsock < 0)
            printf("Unable to accept message\n");
        else {
            printf("server: connection from %s\n",
inet_ntoa(their_addr.sin_addr));
            if (nThreads >= MAX_THREADS) {
                printf("MaxThreads reached\n"); continue;
            }

            // Durch ti wirden dem Thread alle Infos uebergeben
            ti = (threadInfo *)malloc(sizeof(threadInfo));
            if (!ti) {
                printf ("Memory\n");
                exit(-1);
            }

            // Freie Stelle in der Tabelle suchen:
            idx = 0;
            while (threads[idx] && idx < MAX_THREADS)
                ++idx;

            // Folgendes braucht unser Thread:
            ti->threads = threads; // Tabelle mit allen Thread-IDs
            ti->idx = idx; // Seine stelle in dieser
Tabelle
            ti->sockfd = msgsock; // Socket-Handle fuer seine
Connection

            if (pthread_create(&threadid, NULL, &serve, (void *)ti)) {
                free(ti);
                printf("Thread could not be started\n");
                continue;
            }

            threads[idx] = threadid; // Thread-ID in die Tabelle
einfuegen
            ++nThreads;
        } /* end else Connected */
    } /* end for ever */
}

/* Thread for serving clients */
void * serve(void *arg) // wird pro Connection aufgerufen
{
int rval = 0; // Return-Wert
char buf[1024]; // recbuf
char sbuf[1024]; // sendbuf
char *sbufzgr = sbuf;
threadInfo *ti = (threadInfo *)arg; // Zeiger auf ThreadInfo von
main-Thread
int len, nbytes, len2read; // Diverse Zaehler
int templen;

    pthread_detach(pthread_self()); // Thread wird mit der Funktion
beendet

    dbgstatus[ti->idx] = '0';
    while (1) {
// MARK 11
        dbgstatus[ti->idx] = '1';
        /* Laenge einlesen */
        nbytes = spbsRecvn(ti->sockfd, (char *)&len, sizeof(int));
        dbgstatus[ti->idx] = '-';
        if ( nbytes < 0 ) {
            printf("Laenge fehlerhaft empfangen\n");
            break;
        }
        if (nbytes != sizeof(int)) {
            printf("Laenge nicht vollstaendig empfangen\n");
            break;
        }
        len2read = ntohl(len); // # bytes that follow
        if (len2read < 0 || len2read > 1024 ) {
            printf("Laenge ist falsch\n");
            break;
        }
// MARK 22
        dbgstatus[ti->idx] = '2';
        // Daten = len2read Bytes aus dem Stream einlesen
        nbytes = spbsRecvn(ti->sockfd, buf, len2read);
        dbgstatus[ti->idx] = '+';
        if ( nbytes < 0 ) {
            printf("Daten fehlerhaft empfangen\n");
            break;
        }
        if ( nbytes != len2read ) {
            printf("Daten nicht vollstaendig empfangen\n");
            break;
        }
        dbgstatus[ti->idx] = '3';
        /* Befehl */
        sbufzgr = sbuf;
        * (int *) sbufzgr = htonl(sizeof(short)); sbufzgr += sizeof(int);
        * (short *)sbufzgr = htons(6);
        templen = htonl(6);
        rval = send(ti->sockfd, (char *)&templen, 4, 0);
        if (rval < 0 || rval != 4) {
            printf("send Antwortlaenge fehlerhaft\n");
            break;
        }
        rval = send(ti->sockfd, sbuf, 6, 0);
        if (rval < 0 || rval != 6) {
            printf("send Antwort fehlerhaft\n");
            break;
        }
        dbgstatus[ti->idx] = '4';
        printf("%d->%s\r", DM_ANZ++,dbgstatus); fflush(stdout);
    } /* end while */

    // Aufraeumen:
    printf("\nEnde Thread-> %d\n", ti->idx);
    close(ti->sockfd);
    ti->threads[ti->idx] = GOODBYE;
    --nThreads;
    free(ti);
    return NULL;
}

void sig_break(int signo)
{
pid_t pid;
int stat;

    close(sockfd);
    printf("*** terminated\n");
    fflush(stdout);
    exit(0);
    return;
}

/* recv defined number of bytes */
int spbsRecvn(int fd, void *aptr, int nbytes)
{
int nleft, nread;
char *ptr = aptr;

    nleft = nbytes;
    while (nleft > 0) {
        if ( (nread = recv(fd, ptr, nleft, 0)) < 0) {
            printf("ERROR spbsRecvn\n"); exit (2);
            if (errno == EINTR)// Signal caught -> recv again
                nread = 0;
            else
                return -1;
        } /* end if return < 0 */
        else if (nread == 0)
            break;
        nleft -= nread;
        ptr += nread;
    } /* end while all to receive */

    return (nbytes - nleft); // return >= 0

}

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.rutgers.edu
Please read the FAQ at http://www.tux.org/lkml/



This archive was generated by hypermail 2b29 : Sat Apr 15 2000 - 21:00:24 EST