Asynchronous I/O:

When handling multiple connections from different hosts there are usually two ways to handle this:

1. Fork a new process or create a thread to handle the new connect exclusively in the same manner we handle a single connection. This is the easiest way of handling multiple clients from a programmer perspective, but is limited to servers that do not require or provide communication between clients (such as a chat server.)

2. Use the system calls select(2), poll() or epoll(7) to wait for some activity on one or more of the descriptors to process. This may require that some state information be kept about each connection (for example if they are logging in or out, communicating explicitly with another active connection, etc.)

   This method typically involves a "main" loop:

   • Setup select / poll / epoll

   • Loop (forever):

     • Call select / poll / epoll

     • Loop through active descriptors:

       • Handle activity

       • Update descriptors watch-list if necessary.

The select(2) system call:

  #include <sys/select.h>

  /* According to earlier standards */
  #include <sys/time.h>
  #include <sys/types.h>
  #include <unistd.h>

  int select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds,
        struct timeval *timeout);

  void FD_CLR(int fd, fd_set *set);
  int  FD_ISSET(int fd, fd_set *set);
  void FD_SET(int fd, fd_set *set);
  void FD_ZERO(fd_set *set);

  struct timeval {
    time_t  tv_sec;         /* seconds */
    suseconds_t tv_usec;    /* microseconds */
  };

Select will watch up to FD_SETSIZE (1024) number of descriptors for activity. An fd_set is a bit-array representing descriptors to watch and is maintained by the macro functions:

FD_CLR	Clears all the bits in the fd_set
FD_ISSET	Check if a specific bit is set in the fd_set
FD_SET	Sets a specific bit in the fd_set
FD_ZERO	Clears a specific bit in the fd_set

A bit in a fd_set represents a file-descriptor by its number (fd 0 is bit 0 in the fd_set, fd 1 is bit 1 and so on.)

readfds are those descriptors with data to read, writefds are those descriptors that can be written to without blocking and exceptfds are those descriptors with exceptions (usually OOB/ urgent data on TCP sockets.) In general we only care about readfds unless there is a lot of outgoing data that might fill up the buffer and cause blocking on write().

nfds should be set to the value of the highest file descriptor in any of the above three sets plus one.

The fd_sets are set with the descriptors to wait for events on, and are modified by the select function to reflect the descriptors which have changed their status, thus will need to be reset every time the select() system call is made. NULL may be used if no descriptors of a specific class are to be watched for.

The timeout is the amount of time to wait for events, it will wait forever if NULL is given.

Select returns the number of file descriptors which changed state, or -1 on error.

Example code, wait for read events on descriptors 0, 3 and 4:

  // in is the reference fd_set, copied to s_in which select will use/modify:
  fd_set in, s_in;
  // initialize in:
  FD_CLR(&in);
  FD_SET(0, &in); FD_SET(3, &in); FD_SET(4, &in);

  while (1) {
    // Copy in to s_in:
    memcpy(&s_in, &in, sizeof(fd_set));
    int r = select(5, &s_in, NULL, NULL, NULL);
    if (r < 0) perror("select");

    if (FD_ISSET(0, &s_in)) // ... can read from descriptor 0;
    if (FD_ISSET(3, &s_in)) // ... can read from descriptor 3;
    if (FD_ISSET(4, &s_in)) // ... can read from descriptor 4;
  }

Note: select() should be considered deprecated in favor of using the epoll mechanism.

Example code: chat.c

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/select.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>

#define K   1024
#define PORT    47350

int chat_connect(char *hostname, short port)
{
  struct hostent *he;
  struct sockaddr_in server;

  int sock = socket(AF_INET, SOCK_STREAM, 0);
  if (sock < 0) {
    perror("socket");
    exit(1);
  }

  server.sin_family = AF_INET;
  if ((he = gethostbyname(hostname)) == NULL) {
    fprintf(stderr,"Unknown host: %s\n", hostname);
    exit(1);
  }
  memcpy(&server.sin_addr, he->h_addr, he->h_length);
  server.sin_port = htons(port);

  if (connect(sock, (struct sockaddr *)&server, sizeof(server)) < 0) {
    perror("connect");
    exit(1);
  }
  printf("connected.\n");
  return sock;
}

int main(int argc, char *argv[])
{
  char *username = getenv("USER");
  fd_set rfds;
  char cbuf[K], buf[K];
  int sock, r, n;

  if (argc < 2) {
    fprintf(stderr,"Usage: chat <hostname> [<port>]\n");
    return 0;
  }
  if (username == NULL) username = "anonymous";

  sock = chat_connect(argv[1], argc > 2? atoi(argv[2]) : PORT);

  while(1) {
    FD_ZERO(&rfds);
    FD_SET(STDIN_FILENO, &rfds);
    FD_SET(sock, &rfds);

    r = select(sock+1, &rfds, NULL, NULL, NULL);

    if (FD_ISSET(STDIN_FILENO, &rfds)) {
      if (fgets(cbuf, K, stdin) == NULL) break;
      snprintf(buf, K, "%s: %s", username, cbuf);
      if (write(sock, buf, strlen(buf)) < 0) break;
    }
    if (FD_ISSET(sock, &rfds)) {
      if ((r = read(sock, buf, K)) < 0) break;
      printf("%.*s", r, buf);
    }
  }
  return 0;
}

The epoll system (man 7 epoll):

epoll_create(2)

  #include <sys/epoll.h>

  int epoll_create(int size);

Creates an epoll instance. size is ignored, but must be > 0. Returns a file descriptor representing an epoll instance or -1 on error.

Example:

int epfd = epoll_create(1);

epoll_ctl(2)

  #include <sys/epoll.h>

  int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event);

Requests that the epoll system (epfd) monitor a file descriptor (fd). Returns 0 on success or -1 on error.

epfd = The epoll descriptor created by epoll_create().
fd = The file descriptor to monitor.
op = one of:

EPOLL_CTL_ADD	Register the target file descriptor fd on the epoll instance and associate the event "event" with the internal file linked to fd.
EPOLL_CTL_MOD	Change the event event associated with the target file descriptor fd.
EPOLL_CTL_DEL	Remove (de-register) the target file descriptor fd from the epoll instance. "event" is ignored and can be NULL (but must be present for kernels < 2.6.9)

Epoll events

    struct epoll_event {
      uint32_t     events;      /* Epoll events */
      epoll_data_t data;        /* User data variable */
    };
    typedef union epoll_data {
      void      *ptr;
      int       fd;
      uint32_t  u32;
      uint64_t  u64;
    } epoll_data_t;

The epoll_event structure is used to both setup events and get them. The events member of the epoll_event structure is one or more of the following binary OR'ed together (ala. poll):

epoll_wait(2)

  int epoll_wait(int epfd, struct epoll_event *events, int maxevents,
     int timeout);

This function waits for events on the epoll instance epfd

• maxevents is the size of the events array.

• timeout is number of milli-seconds to wait before timing out, use -1 to wait forever.

• The data field will contain the user data that was registered with epoll_ctl(), while the events field will be the bit-mask of returned events.

• Returns # of descriptors to process or -1 on error.

Example code: e-chat.c

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/epoll.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>

#define K   1024
#define PORT    47350

/* Use same chat_connect() defined in chat.c above */

/**
 * Adds a descriptor to watch to the epoll instance:
 */
int ewatch(int epfd, int fd)
{
  struct epoll_event event;

  event.events = EPOLLIN | EPOLLRDHUP | EPOLLERR | EPOLLHUP;
  event.data.fd = fd;

  if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &event) < 0) {
    perror("epoll_ctl");
    exit(1);
  }
  return 0;
}

int main(int argc, char *argv[])
{
  char *username = getenv("USER");
  struct epoll_event events[2];
  char buf[K];
  int sock, i, n, r, epfd, done = 0;

  if (argc < 2) {
    fprintf(stderr,"Usage: chat <hostname> [<port>]\n");
    return 0;
  }
  if (username == NULL) username = "anonymous";

  sock = chat_connect(argv[1], argc > 2? atoi(argv[2]) : PORT);

  if ((epfd = epoll_create(1)) < 0) {
    perror("epoll_create");
    exit(1);
  }
  ewatch(epfd, STDIN_FILENO);
  ewatch(epfd, sock);

  while(!done) {
    // We could just wait for one event, and avoid looping through them, this way
    // is technically more efficient, since we may spend less time in the kernel
    n = epoll_wait(epfd, events, 2, -1);

    // Loop through the returned events:
    for(i=0; i < n; i++) {
      // check for hangup or errors on any of the descriptors:
      if (events[i].events & (EPOLLRDHUP | EPOLLERR | EPOLLHUP)) { done = 1; break; }

      // We can read from one or the other:
      if (events[i].data.fd == STDIN_FILENO) {
        if ((r = read(STDIN_FILENO, buf, K)) <= 0) { done=1; break; } 
        if (write(sock, buf, r) < 0) { done=1; break; }
      } else {
        if ((r = read(sock, buf, K)) <= 0) { done=1; break; }
        if (write(STDOUT_FILENO, buf, r) < 0) { done=1; break; }
      }
      // The if-else above could be written as:
      // if ((r = read(events[i].data.fd, buf, K)) <=0) { done=1; break; }
      // if (write(events[i].data.fd==sock? sock : STDOUT_FILENO, buf, r) < 0) { done=1; break; }
    }
  }
  return 0;
}