Fix initialization and pointer bugs in new code
[krb5.git] / src / lib / apputils / net-server.c
index 9c0d5849d74c2d745f28337a10d6a519246ffc73..11a7c122fffa3bf03168c76c036be5c154b91841 100644 (file)
@@ -58,6 +58,9 @@
 
 #include "fake-addrinfo.h"
 #include "net-server.h"
+#ifdef INTERNAL_VERTO
+#include "verto-k5ev.h"
+#endif
 
 #include <signal.h>
 
@@ -191,34 +194,33 @@ struct connection {
     void *handle;
     const char *prog;
     enum conn_type type;
-    union {
-        /* Type-specific information.  */
-        struct {
-            /* connection */
-            struct sockaddr_storage addr_s;
-            socklen_t addrlen;
-            char addrbuf[56];
-            krb5_fulladdr faddr;
-            krb5_address kaddr;
-            /* incoming */
-            size_t bufsiz;
-            size_t offset;
-            char *buffer;
-            size_t msglen;
-            /* outgoing */
-            krb5_data *response;
-            unsigned char lenbuf[4];
-            sg_buf sgbuf[2];
-            sg_buf *sgp;
-            int sgnum;
-            /* crude denial-of-service avoidance support */
-            time_t start_time;
-        } tcp;
-        struct {
-            SVCXPRT *transp;
-            int closed;
-        } rpc;
-    } u;
+
+    /* Connection fields (TCP or RPC) */
+    struct sockaddr_storage addr_s;
+    socklen_t addrlen;
+    char addrbuf[56];
+    krb5_fulladdr faddr;
+    krb5_address kaddr;
+
+    /* Incoming data (TCP) */
+    size_t bufsiz;
+    size_t offset;
+    char *buffer;
+    size_t msglen;
+
+    /* Outgoing data (TCP) */
+    krb5_data *response;
+    unsigned char lenbuf[4];
+    sg_buf sgbuf[2];
+    sg_buf *sgp;
+    int sgnum;
+
+    /* Crude denial-of-service avoidance support (TCP or RPC) */
+    time_t start_time;
+
+    /* RPC-specific fields */
+    SVCXPRT *transp;
+    int rpc_force_close;
 };
 
 \f
@@ -267,6 +269,20 @@ static SET(unsigned short) udp_port_data, tcp_port_data;
 static SET(struct rpc_svc_data) rpc_svc_data;
 static SET(verto_ev *) events;
 
+verto_ctx *
+loop_init(verto_ev_type types)
+{
+    types |= VERTO_EV_TYPE_IO;
+    types |= VERTO_EV_TYPE_SIGNAL;
+    types |= VERTO_EV_TYPE_TIMEOUT;
+
+#ifdef INTERNAL_VERTO
+    return verto_default_k5ev();
+#else
+    return verto_default(NULL, types);
+#endif
+}
+
 static void
 do_break(verto_ctx *ctx, verto_ev *ev)
 {
@@ -296,39 +312,30 @@ free_sighup_context(verto_ctx *ctx, verto_ev *ev)
     free(verto_get_private(ev));
 }
 
-verto_ctx *
-loop_init(verto_ev_type types, void *handle, void (*reset)())
+krb5_error_code
+loop_setup_signals(verto_ctx *ctx, void *handle, void (*reset)())
 {
     struct sighup_context *sc;
-    verto_ctx *ctx;
-    verto_ev  *ev;
+    verto_ev *ev;
 
-    types |= VERTO_EV_TYPE_IO;
-    types |= VERTO_EV_TYPE_SIGNAL;
-    types |= VERTO_EV_TYPE_TIMEOUT;
-    ctx = verto_default(NULL, types);
     if (!verto_add_signal(ctx, VERTO_EV_FLAG_PERSIST, do_break, SIGINT)  ||
         !verto_add_signal(ctx, VERTO_EV_FLAG_PERSIST, do_break, SIGTERM) ||
         !verto_add_signal(ctx, VERTO_EV_FLAG_PERSIST, do_break, SIGQUIT) ||
         !verto_add_signal(ctx, VERTO_EV_FLAG_PERSIST, VERTO_SIG_IGN, SIGPIPE))
-        goto error;
+        return ENOMEM;
 
     ev = verto_add_signal(ctx, VERTO_EV_FLAG_PERSIST, do_reset, SIGHUP);
     if (!ev)
-        goto error;
+        return ENOMEM;
 
     sc = malloc(sizeof(*sc));
     if (!sc)
-        goto error;
+        return ENOMEM;
+
     sc->handle = handle;
     sc->reset = reset;
-
     verto_set_private(ev, sc, free_sighup_context);
-    return ctx;
-
-error:
-    verto_free(ctx);
-    return NULL;
+    return 0;
 }
 
 krb5_error_code
@@ -415,12 +422,12 @@ free_connection(struct connection *conn)
 {
     if (!conn)
         return;
-    if (conn->u.tcp.response)
-        krb5_free_data(get_context(conn->handle), conn->u.tcp.response);
-    if (conn->u.tcp.buffer)
-        free(conn->u.tcp.buffer);
-    if (conn->type == CONN_RPC_LISTENER && conn->u.rpc.transp != NULL)
-        svc_destroy(conn->u.rpc.transp);
+    if (conn->response)
+        krb5_free_data(get_context(conn->handle), conn->response);
+    if (conn->buffer)
+        free(conn->buffer);
+    if (conn->type == CONN_RPC_LISTENER && conn->transp != NULL)
+        svc_destroy(conn->transp);
     free(conn);
 }
 
@@ -452,14 +459,14 @@ free_socket(verto_ctx *ctx, verto_ev *ev)
 
     /* Close the file descriptor. */
     krb5_klog_syslog(LOG_INFO, _("closing down fd %d"), fd);
-    if (fd >= 0 && (!conn || conn->type != CONN_RPC || conn->u.rpc.closed))
+    if (fd >= 0 && (!conn || conn->type != CONN_RPC || conn->rpc_force_close))
         close(fd);
 
     /* Free the connection struct. */
     if (conn) {
         switch (conn->type) {
             case CONN_RPC:
-                if (conn->u.rpc.closed) {
+                if (conn->rpc_force_close) {
                     FD_ZERO(&fds);
                     FD_SET(fd, &fds);
                     svc_getreqset(&fds);
@@ -507,32 +514,6 @@ make_event(verto_ctx *ctx, verto_ev_flag flags, verto_callback callback,
     return ev;
 }
 
-static verto_ev *
-convert_event(verto_ctx *ctx, verto_ev *ev, verto_ev_flag flags,
-              verto_callback callback)
-{
-    struct connection *conn;
-    verto_ev *newev;
-    int sock;
-
-    conn = verto_get_private(ev);
-    sock = verto_get_fd(ev);
-    if (sock < 0)
-        return NULL;
-
-    newev = make_event(ctx, flags, callback, sock, conn, 1);
-
-    /* Delete the read event without closing the socket
-     * or freeing the connection struct. */
-    if (newev) {
-        verto_set_private(ev, NULL, NULL); /* Reset the destructor. */
-        remove_event_from_set(ev); /* Remove it from the set. */
-        verto_del(ev);
-    }
-
-    return newev;
-}
-
 static verto_ev *
 add_fd(struct socksetup *data, int sock, enum conn_type conntype,
        verto_ev_flag flags, verto_callback callback, int addevent)
@@ -573,7 +554,9 @@ static verto_ev *
 add_udp_fd(struct socksetup *data, int sock, int pktinfo)
 {
     return add_fd(data, sock, pktinfo ? CONN_UDP_PKTINFO : CONN_UDP,
-                  VERTO_EV_FLAG_IO_READ | VERTO_EV_FLAG_PERSIST,
+                  VERTO_EV_FLAG_IO_READ |
+                  VERTO_EV_FLAG_PERSIST |
+                  VERTO_EV_FLAG_REINITIABLE,
                   process_packet, 1);
 }
 
@@ -581,7 +564,9 @@ static verto_ev *
 add_tcp_listener_fd(struct socksetup *data, int sock)
 {
     return add_fd(data, sock, CONN_TCP_LISTENER,
-                  VERTO_EV_FLAG_IO_READ | VERTO_EV_FLAG_PERSIST,
+                  VERTO_EV_FLAG_IO_READ |
+                  VERTO_EV_FLAG_PERSIST |
+                  VERTO_EV_FLAG_REINITIABLE,
                   accept_tcp_connection, 1);
 }
 
@@ -659,14 +644,16 @@ add_rpc_listener_fd(struct socksetup *data, struct rpc_svc_data *svc, int sock)
     verto_ev *ev;
 
     ev = add_fd(data, sock, CONN_RPC_LISTENER,
-                VERTO_EV_FLAG_IO_READ | VERTO_EV_FLAG_PERSIST,
+                VERTO_EV_FLAG_IO_READ |
+                VERTO_EV_FLAG_PERSIST |
+                VERTO_EV_FLAG_REINITIABLE,
                 accept_rpc_connection, 1);
     if (ev == NULL)
         return NULL;
 
     conn = verto_get_private(ev);
-    conn->u.rpc.transp = svctcp_create(sock, 0, 0);
-    if (conn->u.rpc.transp == NULL) {
+    conn->transp = svctcp_create(sock, 0, 0);
+    if (conn->transp == NULL) {
         krb5_klog_syslog(LOG_ERR,
                          _("Cannot create RPC service: %s; continuing"),
                          strerror(errno));
@@ -674,7 +661,7 @@ add_rpc_listener_fd(struct socksetup *data, struct rpc_svc_data *svc, int sock)
         return NULL;
     }
 
-    if (!svc_register(conn->u.rpc.transp, svc->prognum, svc->versnum,
+    if (!svc_register(conn->transp, svc->prognum, svc->versnum,
                       svc->dispatch, 0)) {
         krb5_klog_syslog(LOG_ERR,
                          _("Cannot register RPC service: %s; continuing"),
@@ -1537,34 +1524,99 @@ send_to_from(int s, void *buf, size_t len, int flags,
 #endif
 }
 
+struct udp_dispatch_state {
+    void *handle;
+    const char *prog;
+    int port_fd;
+    krb5_address addr;
+    krb5_fulladdr faddr;
+    socklen_t saddr_len;
+    socklen_t daddr_len;
+    struct sockaddr_storage saddr;
+    struct sockaddr_storage daddr;
+    union aux_addressing_info auxaddr;
+    krb5_data request;
+    char pktbuf[MAX_DGRAM_SIZE];
+};
+
+static void
+process_packet_response(void *arg, krb5_error_code code, krb5_data *response)
+{
+    struct udp_dispatch_state *state = arg;
+    int cc;
+
+    if (code)
+        com_err(state->prog ? state->prog : NULL, code,
+                _("while dispatching (udp)"));
+    if (code || response == NULL)
+        goto out;
+
+    cc = send_to_from(state->port_fd, response->data,
+                      (socklen_t) response->length, 0,
+                      (struct sockaddr *)&state->saddr, state->saddr_len,
+                      (struct sockaddr *)&state->daddr, state->daddr_len,
+                      &state->auxaddr);
+    if (cc == -1) {
+        /* Note that the local address (daddr*) has no port number
+         * info associated with it. */
+        char saddrbuf[NI_MAXHOST], sportbuf[NI_MAXSERV];
+        char daddrbuf[NI_MAXHOST];
+        int e = errno;
+
+        if (getnameinfo((struct sockaddr *)&state->daddr, state->daddr_len,
+                        daddrbuf, sizeof(daddrbuf), 0, 0,
+                        NI_NUMERICHOST) != 0) {
+            strlcpy(daddrbuf, "?", sizeof(daddrbuf));
+        }
+
+        if (getnameinfo((struct sockaddr *)&state->saddr, state->saddr_len,
+                        saddrbuf, sizeof(saddrbuf), sportbuf, sizeof(sportbuf),
+                        NI_NUMERICHOST|NI_NUMERICSERV) != 0) {
+            strlcpy(saddrbuf, "?", sizeof(saddrbuf));
+            strlcpy(sportbuf, "?", sizeof(sportbuf));
+        }
+
+        com_err(state->prog, e, _("while sending reply to %s/%s from %s"),
+                saddrbuf, sportbuf, daddrbuf);
+        goto out;
+    }
+    if ((size_t)cc != response->length) {
+        com_err(state->prog, 0, _("short reply write %d vs %d\n"),
+                response->length, cc);
+    }
+
+out:
+    krb5_free_data(get_context(state->handle), response);
+    free(state);
+}
+
 static void
 process_packet(verto_ctx *ctx, verto_ev *ev)
 {
     int cc;
-    socklen_t saddr_len, daddr_len;
-    krb5_fulladdr faddr;
-    krb5_error_code retval;
-    struct sockaddr_storage saddr, daddr;
-    krb5_address addr;
-    krb5_data request;
-    krb5_data *response;
-    char pktbuf[MAX_DGRAM_SIZE];
-    int port_fd;
-    union aux_addressing_info auxaddr;
     struct connection *conn;
+    struct udp_dispatch_state *state;
 
-    port_fd = verto_get_fd(ev);
     conn = verto_get_private(ev);
-    assert(port_fd >= 0);
-
-    response = NULL;
-    saddr_len = sizeof(saddr);
-    daddr_len = sizeof(daddr);
-    memset(&auxaddr, 0, sizeof(auxaddr));
-    cc = recv_from_to(port_fd, pktbuf, sizeof(pktbuf), 0,
-                      (struct sockaddr *)&saddr, &saddr_len,
-                      (struct sockaddr *)&daddr, &daddr_len,
-                      &auxaddr);
+
+    state = malloc(sizeof(*state));
+    if (!state) {
+        com_err(conn->prog, ENOMEM, _("while dispatching (udp)"));
+        return;
+    }
+
+    state->handle = conn->handle;
+    state->prog = conn->prog;
+    state->port_fd = verto_get_fd(ev);
+    assert(state->port_fd >= 0);
+
+    state->saddr_len = sizeof(state->saddr);
+    state->daddr_len = sizeof(state->daddr);
+    memset(&state->auxaddr, 0, sizeof(state->auxaddr));
+    cc = recv_from_to(state->port_fd, state->pktbuf, sizeof(state->pktbuf), 0,
+                      (struct sockaddr *)&state->saddr, &state->saddr_len,
+                      (struct sockaddr *)&state->daddr, &state->daddr_len,
+                      &state->auxaddr);
     if (cc == -1) {
         if (errno != EINTR && errno != EAGAIN
             /*
@@ -1575,78 +1627,45 @@ process_packet(verto_ctx *ctx, verto_ev *ev)
             && errno != ECONNREFUSED
         )
             com_err(conn->prog, errno, _("while receiving from network"));
+        free(state);
+        return;
+    }
+    if (!cc) { /* zero-length packet? */
+        free(state);
         return;
     }
-    if (!cc)
-        return;         /* zero-length packet? */
 
 #if 0
-    if (daddr_len > 0) {
+    if (state->daddr_len > 0) {
         char addrbuf[100];
-        if (getnameinfo(ss2sa(&daddr), daddr_len, addrbuf, sizeof(addrbuf),
+        if (getnameinfo(ss2sa(&state->daddr), state->daddr_len,
+                        addrbuf, sizeof(addrbuf),
                         0, 0, NI_NUMERICHOST))
             strlcpy(addrbuf, "?", sizeof(addrbuf));
         com_err(conn->prog, 0, _("pktinfo says local addr is %s"), addrbuf);
     }
 #endif
 
-    if (daddr_len == 0 && conn->type == CONN_UDP) {
+    if (state->daddr_len == 0 && conn->type == CONN_UDP) {
         /*
          * If the PKTINFO option isn't set, this socket should be bound to a
          * specific local address.  This info probably should've been saved in
          * our socket data structure at setup time.
          */
-        daddr_len = sizeof(daddr);
-        if (getsockname(port_fd, (struct sockaddr *)&daddr, &daddr_len) != 0)
-            daddr_len = 0;
+            state->daddr_len = sizeof(state->daddr);
+        if (getsockname(state->port_fd, (struct sockaddr *)&state->daddr,
+                        &state->daddr_len) != 0)
+            state->daddr_len = 0;
         /* On failure, keep going anyways. */
     }
 
-    request.length = cc;
-    request.data = pktbuf;
-    faddr.address = &addr;
-    init_addr(&faddr, ss2sa(&saddr));
+    state->request.length = cc;
+    state->request.data = state->pktbuf;
+    state->faddr.address = &state->addr;
+    init_addr(&state->faddr, ss2sa(&state->saddr));
     /* This address is in net order. */
-    retval = dispatch(conn->handle, ss2sa(&daddr),
-                      &faddr, &request, &response, 0);
-    if (retval) {
-        com_err(conn->prog, retval, _("while dispatching (udp)"));
-        return;
-    }
-    if (response == NULL)
-        return;
-    cc = send_to_from(port_fd, response->data, (socklen_t) response->length, 0,
-                      (struct sockaddr *)&saddr, saddr_len,
-                      (struct sockaddr *)&daddr, daddr_len,
-                      &auxaddr);
-    if (cc == -1) {
-        /* Note that the local address (daddr*) has no port number
-         * info associated with it. */
-        char saddrbuf[NI_MAXHOST], sportbuf[NI_MAXSERV];
-        char daddrbuf[NI_MAXHOST];
-        int e = errno;
-        krb5_free_data(get_context(conn->handle), response);
-        if (getnameinfo((struct sockaddr *)&daddr, daddr_len,
-                        daddrbuf, sizeof(daddrbuf), 0, 0,
-                        NI_NUMERICHOST) != 0) {
-            strlcpy(daddrbuf, "?", sizeof(daddrbuf));
-        }
-        if (getnameinfo((struct sockaddr *)&saddr, saddr_len,
-                        saddrbuf, sizeof(saddrbuf), sportbuf, sizeof(sportbuf),
-                        NI_NUMERICHOST|NI_NUMERICSERV) != 0) {
-            strlcpy(saddrbuf, "?", sizeof(saddrbuf));
-            strlcpy(sportbuf, "?", sizeof(sportbuf));
-        }
-        com_err(conn->prog, e, _("while sending reply to %s/%s from %s"),
-                saddrbuf, sportbuf, daddrbuf);
-        return;
-    }
-    if ((size_t)cc != response->length) {
-        com_err(conn->prog, 0, _("short reply write %d vs %d\n"),
-                response->length, cc);
-    }
-    krb5_free_data(get_context(conn->handle), response);
-    return;
+    dispatch(state->handle, ss2sa(&state->daddr), &state->faddr,
+             &state->request, 0, process_packet_response, state);
 }
 
 static int
@@ -1670,10 +1689,10 @@ kill_lru_tcp_or_rpc_connection(void *handle, verto_ev *newev)
 #if 0
         krb5_klog_syslog(LOG_INFO, "fd %d started at %ld",
                          verto_get_fd(oldest_ev),
-                         c->u.tcp.start_time);
+                         c->start_time);
 #endif
         if (oldest_c == NULL
-            || oldest_c->u.tcp.start_time > c->u.tcp.start_time) {
+            || oldest_c->start_time > c->start_time) {
             oldest_ev = ev;
             oldest_c = c;
         }
@@ -1681,9 +1700,9 @@ kill_lru_tcp_or_rpc_connection(void *handle, verto_ev *newev)
     if (oldest_c != NULL) {
         krb5_klog_syslog(LOG_INFO, _("dropping %s fd %d from %s"),
                          c->type == CONN_RPC ? "rpc" : "tcp",
-                         verto_get_fd(oldest_ev), oldest_c->u.tcp.addrbuf);
+                         verto_get_fd(oldest_ev), oldest_c->addrbuf);
         if (oldest_c->type == CONN_RPC)
-            oldest_c->u.rpc.closed = 1;
+            oldest_c->rpc_force_close = 1;
         verto_del(oldest_ev);
     }
     return fd;
@@ -1727,14 +1746,14 @@ accept_tcp_connection(verto_ctx *ctx, verto_ev *ev)
     newconn = verto_get_private(newev);
 
     if (getnameinfo((struct sockaddr *)&addr_s, addrlen,
-                    newconn->u.tcp.addrbuf, sizeof(newconn->u.tcp.addrbuf),
+                    newconn->addrbuf, sizeof(newconn->addrbuf),
                     tmpbuf, sizeof(tmpbuf),
                     NI_NUMERICHOST | NI_NUMERICSERV))
-        strlcpy(newconn->u.tcp.addrbuf, "???", sizeof(newconn->u.tcp.addrbuf));
+        strlcpy(newconn->addrbuf, "???", sizeof(newconn->addrbuf));
     else {
         char *p, *end;
-        p = newconn->u.tcp.addrbuf;
-        end = p + sizeof(newconn->u.tcp.addrbuf);
+        p = newconn->addrbuf;
+        end = p + sizeof(newconn->addrbuf);
         p += strlen(p);
         if ((size_t)(end - p) > 2 + strlen(tmpbuf)) {
             *p++ = '.';
@@ -1743,42 +1762,103 @@ accept_tcp_connection(verto_ctx *ctx, verto_ev *ev)
     }
 #if 0
     krb5_klog_syslog(LOG_INFO, "accepted TCP connection on socket %d from %s",
-                     s, newconn->u.tcp.addrbuf);
+                     s, newconn->addrbuf);
 #endif
 
-    newconn->u.tcp.addr_s = addr_s;
-    newconn->u.tcp.addrlen = addrlen;
-    newconn->u.tcp.bufsiz = 1024 * 1024;
-    newconn->u.tcp.buffer = malloc(newconn->u.tcp.bufsiz);
-    newconn->u.tcp.start_time = time(0);
+    newconn->addr_s = addr_s;
+    newconn->addrlen = addrlen;
+    newconn->bufsiz = 1024 * 1024;
+    newconn->buffer = malloc(newconn->bufsiz);
+    newconn->start_time = time(0);
 
     if (++tcp_or_rpc_data_counter > max_tcp_or_rpc_data_connections)
         kill_lru_tcp_or_rpc_connection(conn->handle, newev);
 
-    if (newconn->u.tcp.buffer == 0) {
+    if (newconn->buffer == 0) {
         com_err(conn->prog, errno,
                 _("allocating buffer for new TCP session from %s"),
-                newconn->u.tcp.addrbuf);
+                newconn->addrbuf);
         verto_del(newev);
         return;
     }
-    newconn->u.tcp.offset = 0;
-    newconn->u.tcp.faddr.address = &newconn->u.tcp.kaddr;
-    init_addr(&newconn->u.tcp.faddr, ss2sa(&newconn->u.tcp.addr_s));
-    SG_SET(&newconn->u.tcp.sgbuf[0], newconn->u.tcp.lenbuf, 4);
-    SG_SET(&newconn->u.tcp.sgbuf[1], 0, 0);
+    newconn->offset = 0;
+    newconn->faddr.address = &newconn->kaddr;
+    init_addr(&newconn->faddr, ss2sa(&newconn->addr_s));
+    SG_SET(&newconn->sgbuf[0], newconn->lenbuf, 4);
+    SG_SET(&newconn->sgbuf[1], 0, 0);
+}
+
+struct tcp_dispatch_state {
+    struct sockaddr_storage local_saddr;
+    struct connection *conn;
+    krb5_data request;
+    verto_ctx *ctx;
+    int sock;
+};
+
+static void
+process_tcp_response(void *arg, krb5_error_code code, krb5_data *response)
+{
+    struct tcp_dispatch_state *state = arg;
+    verto_ev *ev;
+
+    assert(state);
+    state->conn->response = response;
+
+    if (code)
+        com_err(state->conn->prog, code, _("while dispatching (tcp)"));
+    if (code || !response)
+        goto kill_tcp_connection;
+
+    /* Queue outgoing response. */
+    store_32_be(response->length, state->conn->lenbuf);
+    SG_SET(&state->conn->sgbuf[1], response->data, response->length);
+    state->conn->sgp = state->conn->sgbuf;
+    state->conn->sgnum = 2;
+
+    ev = make_event(state->ctx, VERTO_EV_FLAG_IO_WRITE | VERTO_EV_FLAG_PERSIST,
+                    process_tcp_connection_write, state->sock, state->conn, 1);
+    if (ev) {
+        free(state);
+        return;
+    }
+
+kill_tcp_connection:
+    tcp_or_rpc_data_counter--;
+    free_connection(state->conn);
+    close(state->sock);
+    free(state);
+}
+
+/* Creates the tcp_dispatch_state and deletes the verto event. */
+static struct tcp_dispatch_state *
+prepare_for_dispatch(verto_ctx *ctx, verto_ev *ev)
+{
+    struct tcp_dispatch_state *state;
+
+    state = malloc(sizeof(*state));
+    if (!state) {
+        krb5_klog_syslog(LOG_ERR, _("error allocating tcp dispatch private!"));
+        return NULL;
+    }
+    state->conn = verto_get_private(ev);
+    state->sock = verto_get_fd(ev);
+    state->ctx = ctx;
+    verto_set_private(ev, NULL, NULL); /* Don't close the fd or free conn! */
+    remove_event_from_set(ev); /* Remove it from the set. */
+    verto_del(ev);
+    return state;
 }
 
 static void
 process_tcp_connection_read(verto_ctx *ctx, verto_ev *ev)
 {
-    struct connection *conn;
+    struct tcp_dispatch_state *state = NULL;
+    struct connection *conn = NULL;
     ssize_t nread;
     size_t len;
-    int sock;
 
     conn = verto_get_private(ev);
-    sock = verto_get_fd(ev);
 
     /*
      * Read message length and data into one big buffer, already allocated
@@ -1786,85 +1866,77 @@ process_tcp_connection_read(verto_ctx *ctx, verto_ev *ev)
      * we should only be here if there is no data in the buffer, or only an
      * incomplete message.
      */
-    if (conn->u.tcp.offset < 4) {
+    if (conn->offset < 4) {
+        krb5_data *response = NULL;
+
         /* msglen has not been computed.  XXX Doing at least two reads
          * here, letting the kernel worry about buffering. */
-        len = 4 - conn->u.tcp.offset;
-        nread = SOCKET_READ(sock,
-                            conn->u.tcp.buffer + conn->u.tcp.offset, len);
+        len = 4 - conn->offset;
+        nread = SOCKET_READ(verto_get_fd(ev),
+                            conn->buffer + conn->offset, len);
         if (nread < 0) /* error */
             goto kill_tcp_connection;
         if (nread == 0) /* eof */
             goto kill_tcp_connection;
-        conn->u.tcp.offset += nread;
-        if (conn->u.tcp.offset == 4) {
-            unsigned char *p = (unsigned char *)conn->u.tcp.buffer;
-            conn->u.tcp.msglen = load_32_be(p);
-            if (conn->u.tcp.msglen > conn->u.tcp.bufsiz - 4) {
+        conn->offset += nread;
+        if (conn->offset == 4) {
+            unsigned char *p = (unsigned char *)conn->buffer;
+            conn->msglen = load_32_be(p);
+            if (conn->msglen > conn->bufsiz - 4) {
                 krb5_error_code err;
                 /* Message too big. */
                 krb5_klog_syslog(LOG_ERR, _("TCP client %s wants %lu bytes, "
-                                 "cap is %lu"), conn->u.tcp.addrbuf,
-                                 (unsigned long) conn->u.tcp.msglen,
-                                 (unsigned long) conn->u.tcp.bufsiz - 4);
+                                 "cap is %lu"), conn->addrbuf,
+                                 (unsigned long) conn->msglen,
+                                 (unsigned long) conn->bufsiz - 4);
                 /* XXX Should return an error.  */
                 err = make_toolong_error (conn->handle,
-                                          &conn->u.tcp.response);
+                                          &response);
                 if (err) {
                     krb5_klog_syslog(LOG_ERR, _("error constructing "
                                      "KRB_ERR_FIELD_TOOLONG error! %s"),
                                      error_message(err));
                     goto kill_tcp_connection;
                 }
-                goto have_response;
+
+                state = prepare_for_dispatch(ctx, ev);
+                if (!state) {
+                    krb5_free_data(get_context(conn->handle), response);
+                    goto kill_tcp_connection;
+                }
+                process_tcp_response(state, 0, response);
             }
         }
     } else {
         /* msglen known. */
-        krb5_data request;
-        krb5_error_code err;
-        struct sockaddr_storage local_saddr;
-        socklen_t local_saddrlen = sizeof(local_saddr);
+        socklen_t local_saddrlen = sizeof(struct sockaddr_storage);
         struct sockaddr *local_saddrp = NULL;
 
-        len = conn->u.tcp.msglen - (conn->u.tcp.offset - 4);
-        nread = SOCKET_READ(sock,
-                            conn->u.tcp.buffer + conn->u.tcp.offset, len);
+        len = conn->msglen - (conn->offset - 4);
+        nread = SOCKET_READ(verto_get_fd(ev),
+                            conn->buffer + conn->offset, len);
         if (nread < 0) /* error */
             goto kill_tcp_connection;
         if (nread == 0) /* eof */
             goto kill_tcp_connection;
-        conn->u.tcp.offset += nread;
-        if (conn->u.tcp.offset < conn->u.tcp.msglen + 4)
+        conn->offset += nread;
+        if (conn->offset < conn->msglen + 4)
             return;
+
         /* Have a complete message, and exactly one message. */
-        request.length = conn->u.tcp.msglen;
-        request.data = conn->u.tcp.buffer + 4;
+        state = prepare_for_dispatch(ctx, ev);
+        if (!state)
+            goto kill_tcp_connection;
 
-        if (getsockname(sock, ss2sa(&local_saddr),
+        state->request.length = conn->msglen;
+        state->request.data = conn->buffer + 4;
+
+        if (getsockname(verto_get_fd(ev), ss2sa(&state->local_saddr),
                         &local_saddrlen) == 0)
-            local_saddrp = ss2sa(&local_saddr);
+            local_saddrp = ss2sa(&state->local_saddr);
 
-        err = dispatch(conn->handle, local_saddrp, &conn->u.tcp.faddr,
-                       &request, &conn->u.tcp.response, 1);
-        if (err) {
-            com_err(conn->prog, err, _("while dispatching (tcp)"));
-            goto kill_tcp_connection;
-        }
-        if (conn->u.tcp.response == NULL)
-            goto kill_tcp_connection;
-    have_response:
-        /* Queue outgoing response. */
-        store_32_be(conn->u.tcp.response->length, conn->u.tcp.lenbuf);
-        SG_SET(&conn->u.tcp.sgbuf[1], conn->u.tcp.response->data,
-               conn->u.tcp.response->length);
-        conn->u.tcp.sgp = conn->u.tcp.sgbuf;
-        conn->u.tcp.sgnum = 2;
-
-        if (convert_event(ctx, ev,
-                          VERTO_EV_FLAG_IO_WRITE | VERTO_EV_FLAG_PERSIST,
-                          process_tcp_connection_write))
-            return;
+        dispatch(state->conn->handle, local_saddrp, &conn->faddr,
+                 &state->request, 1, process_tcp_response, state);
     }
 
     return;
@@ -1884,19 +1956,19 @@ process_tcp_connection_write(verto_ctx *ctx, verto_ev *ev)
     conn = verto_get_private(ev);
     sock = verto_get_fd(ev);
 
-    nwrote = SOCKET_WRITEV(sock, conn->u.tcp.sgp,
-                           conn->u.tcp.sgnum, tmp);
+    nwrote = SOCKET_WRITEV(sock, conn->sgp,
+                           conn->sgnum, tmp);
     if (nwrote > 0) { /* non-error and non-eof */
         while (nwrote) {
-            sg_buf *sgp = conn->u.tcp.sgp;
+            sg_buf *sgp = conn->sgp;
             if ((size_t)nwrote < SG_LEN(sgp)) {
                 SG_ADVANCE(sgp, (size_t)nwrote);
                 nwrote = 0;
             } else {
                 nwrote -= SG_LEN(sgp);
-                conn->u.tcp.sgp++;
-                conn->u.tcp.sgnum--;
-                if (conn->u.tcp.sgnum == 0 && nwrote != 0)
+                conn->sgp++;
+                conn->sgnum--;
+                if (conn->sgnum == 0 && nwrote != 0)
                     abort();
             }
         }
@@ -1904,7 +1976,7 @@ process_tcp_connection_write(verto_ctx *ctx, verto_ev *ev)
         /* If we still have more data to send, just return so that
          * the main loop can call this function again when the socket
          * is ready for more writing. */
-        if (conn->u.tcp.sgnum > 0)
+        if (conn->sgnum > 0)
             return;
     }
 
@@ -1983,16 +2055,16 @@ accept_rpc_connection(verto_ctx *ctx, verto_ev *ev)
 
         if (getpeername(s, addr, &addrlen) ||
             getnameinfo(addr, addrlen,
-                        newconn->u.tcp.addrbuf,
-                        sizeof(newconn->u.tcp.addrbuf),
+                        newconn->addrbuf,
+                        sizeof(newconn->addrbuf),
                         tmpbuf, sizeof(tmpbuf),
                         NI_NUMERICHOST | NI_NUMERICSERV)) {
-            strlcpy(newconn->u.tcp.addrbuf, "???",
-                    sizeof(newconn->u.tcp.addrbuf));
+            strlcpy(newconn->addrbuf, "???",
+                    sizeof(newconn->addrbuf));
         } else {
             char *p, *end;
-            p = newconn->u.tcp.addrbuf;
-            end = p + sizeof(newconn->u.tcp.addrbuf);
+            p = newconn->addrbuf;
+            end = p + sizeof(newconn->addrbuf);
             p += strlen(p);
             if ((size_t)(end - p) > 2 + strlen(tmpbuf)) {
                 *p++ = '.';
@@ -2001,18 +2073,18 @@ accept_rpc_connection(verto_ctx *ctx, verto_ev *ev)
         }
 #if 0
         krb5_klog_syslog(LOG_INFO, _("accepted RPC connection on socket %d "
-                         "from %s"), s, newconn->u.tcp.addrbuf);
+                         "from %s"), s, newconn->addrbuf);
 #endif
 
-        newconn->u.tcp.addr_s = addr_s;
-        newconn->u.tcp.addrlen = addrlen;
-        newconn->u.tcp.start_time = time(0);
+        newconn->addr_s = addr_s;
+        newconn->addrlen = addrlen;
+        newconn->start_time = time(0);
 
         if (++tcp_or_rpc_data_counter > max_tcp_or_rpc_data_connections)
             kill_lru_tcp_or_rpc_connection(newconn->handle, newev);
 
-        newconn->u.tcp.faddr.address = &newconn->u.tcp.kaddr;
-        init_addr(&newconn->u.tcp.faddr, ss2sa(&newconn->u.tcp.addr_s));
+        newconn->faddr.address = &newconn->kaddr;
+        init_addr(&newconn->faddr, ss2sa(&newconn->addr_s));
     }
 }