/*
 * nasd_srpc_client.c
 *
 * SRPC client stuff
 *
 * Authors: Jim Zelenka, Nat Lanza
 */
/*
 * Copyright (c) of Carnegie Mellon University, 1999, 2000.
 *
 * Permission to reproduce, use, and prepare derivative works of
 * this software for internal use is granted provided the copyright
 * and "No Warranty" statements are included with all reproductions
 * and derivative works. This software may also be redistributed
 * without charge provided that the copyright and "No Warranty"
 * statements are included in all redistributions.
 *
 * NO WARRANTY. THIS SOFTWARE IS FURNISHED ON AN "AS IS" BASIS.
 * CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, EITHER
 * EXPRESSED OR IMPLIED AS TO THE MATTER INCLUDING, BUT NOT LIMITED
 * TO: WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY
 * OF RESULTS OR RESULTS OBTAINED FROM USE OF THIS SOFTWARE. CARNEGIE
 * MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT
 * TO FREEDOM FROM PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT.
 */


#include <nasd/nasd_options.h>
#include <nasd/nasd_general.h>
#include <nasd/nasd_threadstuff.h>
#include <nasd/nasd_shutdown.h>
#include <nasd/nasd_types.h>
#include <nasd/nasd_common.h>
#include <nasd/nasd_srpc.h>
#include <nasd/nasd_rpcgen_glob_param.h>
#include <nasd/nasd_marshall.h>
#include <nasd/nasd_timer.h>

#define DEBUG_SRPC 0

#define DEBUG_SRPC_PRINT_HEADER_INFO(_h_) { \
  if (DEBUG_SRPC > 0) { \
    switch(_h_.op) { \
      case NASD_SRPC_OP_NOOP: \
        nasd_printf("OP NOOP %u\n", _h_.len); break; \
      case NASD_SRPC_OP_PIPEDATA: \
        nasd_printf("OP PIPEDATA %u %u\n", _h_.callid, _h_.len); break; \
      case NASD_SRPC_OP_PIPETERM: \
        nasd_printf("OP PIPETERM %u %u\n", _h_.callid, _h_.len); break; \
      case NASD_SRPC_OP_REQ: \
        nasd_printf("OP REQ %u %u\n", _h_.callid, _h_.len); break; \
      case NASD_SRPC_OP_SETSEQ: \
        nasd_printf("OP SETSEQ %u\n", _h_.len); break; \
      case NASD_SRPC_OP_REP: \
        nasd_printf("OP REP %u\n", _h_.len); break; \
      case NASD_SRPC_OP_PIPEENAB: \
        nasd_printf("OP PIPEENAB %u\n", _h_.len); break; \
      default: \
        nasd_printf("OP unknown %d %u\n", (int)_h_.op, _h_.len); \
    } \
  } \
}

extern int nasd_srpc_use_counter;
NASD_DECLARE_EXTERN_MUTEX(nasd_srpc_use_counter_lock)

/* totally arbitrary */
#define NASD_SRPC_CLI_MAX_PIPE_COALESCE 8
#define NASD_SRPC_CLI_MAX_PUSH_ALLOC    0x3fffffff
#define NASD_SRPC_CLI_MAX_PULL_ALLOC    0x3fffffff


/*
 * Caller does not hold conn lock
 */
nasd_status_t nasd_srpc_call_complete(nasd_srpc_conn_t    *conn,
                                      nasd_srpc_call_t    *call,
                                      int                  final_state,
                                      int                  reslen,
                                      nasd_srpc_status_t  *status) {
  nasd_srpc_header_t header;
  nasd_srpc_status_t src;
  nasd_status_t rc = NASD_SUCCESS, rc2;
  int bcs = 0;

  NASD_SRPC_LOCK_CONN(conn);

  if (NASD_SRPC_CONN_CONNECTED(conn) &&
      (!(conn->state & (NASD_SRPC_STATE_IDAM
                        | NASD_SRPC_STATE_DISC | NASD_SRPC_STATE_OP_R)))
      && (call->state < final_state)) {
    /* At the moment, no one is doing receive duty.
       We "volunteer" to do so until we get the reply we want. */

    conn->state |= NASD_SRPC_STATE_OP_R;
    rc = nasd_srpc_client_handle(conn, call);
    conn->state &= ~NASD_SRPC_STATE_OP_R;

    if (rc != NASD_SUCCESS) {
      NASD_BROADCAST_COND(conn->state_cond);
      NASD_SRPC_CONN_DEC_USE(conn);
      
      if (rc == NASD_SRPC_CALL_ABORTED_DISC) {
        rc2 = nasd_srpc_conn_disc(conn);
        if (rc2) { NASD_PANIC(); } /* XXX */
      }
      
      NASD_SRPC_UNLOCK_CONN(conn);
      if (call->ishashed) {
        call->prev->next = call->next;
        call->next->prev = call->prev;
        call->ishashed = 0;
      }
      nasd_srpc_call_free(call);
      *status = NASD_SRPC_S_ABORTED;
      return (rc);
    } else {
      bcs = 1;
    }
  }

  while (NASD_SRPC_CONN_CONNECTED(conn)
        && (!(conn->state & NASD_SRPC_STATE_IDAM))
        && (!(conn->state & NASD_SRPC_STATE_DISC))
        && (call->state < final_state)) {
    NASD_WAIT_COND(call->data_ready, conn->lock);
  }

  if (call->ishashed) {
    call->prev->next = call->next;
    call->next->prev = call->prev;
    call->ishashed = 0;
  }

  if (conn->state&(NASD_SRPC_STATE_IDAM|NASD_SRPC_STATE_DISC)) {
    NASD_BROADCAST_COND(conn->state_cond);
    NASD_SRPC_CONN_DEC_USE(conn);
    NASD_SRPC_UNLOCK_CONN(conn);
    nasd_srpc_call_free(call);
    *status = NASD_SRPC_S_ABORTED;
    return (NASD_SUCCESS);
  }

  if (bcs) { NASD_BROADCAST_COND(conn->state_cond); }

  NASD_SRPC_CONN_DEC_USE(conn);
  NASD_SRPC_UNLOCK_CONN(conn);

  header = call->header;

  if ((header.len == 0) && (header.op == NASD_SRPC_OP_REP)) {
    src = call->local_stat ? call->local_stat : header.opstat;
    nasd_srpc_call_free(call);
    *status = src;
    return (NASD_SUCCESS);
  }

  if (header.len != reslen) {
    nasd_srpc_call_free(call);
    *status = NASD_SRPC_S_BAD_OPLEN;
    return (NASD_SUCCESS);
  }

  src = call->local_stat ? call->local_stat : header.opstat;

  *status = src;
  return (rc);
}


/*
 * Caller holds conn lock
 */
nasd_status_t nasd_srpc_conn_disc(nasd_srpc_conn_t  *conn) {
  nasd_srpc_call_t *call, *next;
  nasd_status_t rc = NASD_SUCCESS;
  int h;

  NASD_ASSERT(NASD_SRPC_CONN_LOCK_OWNED(conn));

  conn->state |= NASD_SRPC_STATE_DISC;
  conn->state &= ~NASD_SRPC_STATE_CONN;

  while (conn->use_count) {
    for (h = 0; h < NASD_SRPC_CALLHASH_BUCKETS; h++) {
      for (call = conn->c_call_buckets[h].next;
           call != &conn->c_call_buckets[h];
           call = next) {
        next = call->next;
        call->local_stat = NASD_SRPC_S_FAIL;
        NASD_BROADCAST_COND(call->data_ready);
      }
    }
    NASD_SRPC_CONN_WAIT_USE(conn);
  }

  conn->state &= ~(NASD_SRPC_STATE_IDAM|NASD_SRPC_STATE_ODAM);
  NASD_BROADCAST_COND(conn->state_cond);

  rc = nasd_srpc_sys_sock_destroy(conn->sock);

  return (rc);
}


nasd_status_t nasd_srpc_initiate_call(nasd_srpc_conn_t   *conn,
                                      nasd_srpc_call_t  **callp) {
  nasd_srpc_call_t *call;
  nasd_status_t rc;

  rc = nasd_srpc_call_get(&call);
  if (rc) { return (rc); }

  call->callid = 0;
  call->state = 0;
  call->next = call->prev = NULL;
  call->ishashed = 0;

  *callp = call;

  return (NASD_SUCCESS);
}


/*
 * Caller does not hold conn lock
 * Takes a use_count ref on conn iff return is NASD_SUCCESS
 */
nasd_status_t nasd_srpc_place_call(nasd_srpc_conn_t    *conn,
                                   nasd_srpc_call_t    *call,
                                   nasd_srpc_memvec_t  *memvec,
                                   int                  nmemvecs,
                                   int                  total_len,
                                   nasd_uint32          pipeword) {
  int bytes_sent, h, real_total_len;
  nasd_srpc_header_t header;
  nasd_status_t rc;

  NASD_SRPC_LOCK_CONN(conn);

  if (!NASD_SRPC_CONN_CONNECTED(conn)) {
    rc = nasd_srpc_conn_conn(conn);
    if (rc) { goto out; }
  }

  if (conn->state & NASD_SRPC_STATE_ODAM) {
    rc = NASD_SRPC_SENDSTATE_BAD;
    goto out;
  }

  NASD_SRPC_CONN_INC_USE(conn);

  /* Wait until no one else is sending an op block */
  while (conn->state & NASD_SRPC_STATE_OP_S) {
    NASD_SRPC_CONN_WAIT_STATE(conn);

    if (conn->state & NASD_SRPC_STATE_ODAM) {
      NASD_SRPC_CONN_DEC_USE(conn);
      rc = NASD_SRPC_SENDSTATE_BAD;
      goto out;
    }
  }

  conn->state |= NASD_SRPC_STATE_OP_S;

  call->callid = conn->next_callid;
  conn->next_callid++;
  
  header.callid = call->callid;
  header.len    = total_len;
  header.op     = NASD_SRPC_OP_REQ;
  header.opstat = pipeword;

  nasd_srpc_header_t_marshall(&header, memvec->buf);

  real_total_len = total_len + sizeof(nasd_srpc_header_otw_t);

  /* make sure we can find ourselves again */
  h = NASD_SRPC_CALL_HASH(call->callid);
  call->next       = &conn->c_call_buckets[h];
  call->prev       = conn->c_call_buckets[h].prev;
  call->prev->next = call;
  call->next->prev = call;
  call->state      = 1;
  call->ishashed   = 1;

  NASD_SRPC_UNLOCK_CONN(conn);
  rc = nasd_srpc_sys_sock_send(conn->sock, memvec,
                               real_total_len, &bytes_sent);
  NASD_SRPC_LOCK_CONN(conn);
  
  conn->state &= ~NASD_SRPC_STATE_OP_S;

  if (rc || (bytes_sent != real_total_len)) {
    conn->state |= NASD_SRPC_STATE_ODAM;
    rc = NASD_SRPC_SENDSTATE_BAD;

    if (call->ishashed) {
      call->prev->next = call->next;
      call->next->prev = call->prev;
      call->state      = (-1);

      NASD_SRPC_CONN_DEC_USE(conn);
      call->ishashed   = 0;
    }
  }

  NASD_BROADCAST_COND(conn->state_cond);

 out:
  NASD_SRPC_UNLOCK_CONN(conn);
  return (rc);
}


/*
 * Caller does not hold conn lock.
 */
nasd_status_t nasd_srpc_resume_call(nasd_srpc_conn_t     *conn,
                                    nasd_srpc_call_t     *call,
                                    nasd_srpc_memvec_t   *memvec,
                                    int                  nmemvecs,
                                    int                  total_len,
                                    nasd_uint32          pipeword) {
  NASD_SRPC_LOCK_CONN(conn);
  return NASD_SUCCESS;
}


/*
 * Caller must hold conn lock
 */
nasd_status_t nasd_srpc_conn_conn(nasd_srpc_conn_t  *conn) {
  nasd_srpc_header_otw_t header_otw;
  nasd_srpc_header_t header;
  nasd_srpc_memvec_t vec;
  nasd_status_t rc;
  int bytes_sent;

  rc = NASD_SUCCESS;

  NASD_ASSERT(NASD_SRPC_CONN_LOCK_OWNED(conn));

  NASD_ASSERT(!(conn->state & NASD_SRPC_STATE_OP_R));
  NASD_ASSERT(!(conn->state & NASD_SRPC_STATE_OP_S));
  NASD_ASSERT(!(conn->state & NASD_SRPC_STATE_CONN));
  NASD_ASSERT(conn->state & NASD_SRPC_STATE_IDLE);
  NASD_ASSERT(conn->use_count == 0);

  rc = nasd_srpc_sys_sock_conn(conn->sock, conn->ipaddr, conn->ipport);
  if (rc) { rc = NASD_SRPC_CANNOT_CONNECT; goto out; }

  rc = nasd_srpc_sys_sock_set_options(conn->sock, &nasd_srpc_srv_sockbuf_opts);
#if 0
  /* I'm turning this message off, because it's always printed for localhost
     connections. No idea why, but it's really bugging me.  --jimz */
  if (rc) {
    nasd_printf("nasd_srpc_conn_conn: could not set some or all options on conn->sock "
                "0x%lx rc=0x%x (%s)\n", conn->sock, rc, nasd_error_string(rc));
  }
#endif
  rc = NASD_SUCCESS; /* it's not a huge problem if this fails */

  conn->state |= NASD_SRPC_STATE_CONN;

  if (conn->next_callid != 1) {
    header.callid = conn->next_callid;
    header.len    = 0;
    header.op     = NASD_SRPC_OP_SETSEQ;
    header.opstat = 0;

    nasd_srpc_header_t_marshall(&header, header_otw);

    vec.buf  = header_otw;
    vec.len  = sizeof(nasd_srpc_header_otw_t);
    vec.next = NULL;
    
    rc = nasd_srpc_sys_sock_send(conn->sock, &vec,
                                 sizeof(nasd_srpc_header_otw_t), &bytes_sent);

    if (rc || (bytes_sent != sizeof(nasd_srpc_header_otw_t))) {
      rc = nasd_srpc_conn_disc(conn);
      if (rc) { NASD_PANIC(); }
      rc = NASD_SRPC_CANNOT_CONNECT;
      goto out;
    }
  }

 out:
  return rc;
}


void nasd_srpc_conn_shutdown_conn(void  *conn_arg) {
  nasd_srpc_conn_t *conn;
  
  conn = (nasd_srpc_conn_t *)conn_arg;
  nasd_srpc_conn_free(conn);
}


void nasd_srpc_conn_shutdown_sock(void  *conn_arg) {
  nasd_srpc_conn_t *conn;
  nasd_status_t rc;

  conn = (nasd_srpc_conn_t *)conn_arg;

  if (NASD_SRPC_CONN_CONNECTED(conn)) {
    rc = nasd_srpc_conn_disc(conn);
    if (rc) { NASD_PANIC(); }

    rc = nasd_srpc_sys_sock_destroy(conn->sock);
    if (rc) { nasd_printf("SRPC WARNING: got 0x%x (%s) destroying socket\n",
                          rc, nasd_error_string(rc));                        }
  }

  nasd_srpc_sock_free(conn->sock);
  conn->sock = NULL;
}


nasd_status_t nasd_srpc_bind_to_server(char                *servername,
                                       char                *portnum,
                                       nasd_srpc_handle_t  *handlep) {
  nasd_shutdown_list_t *shutdown_list;
  nasd_srpc_conn_t *conn;
  nasd_srpc_sock_t *sock;
  nasd_status_t rc, rc2;
  nasd_uint32 ipaddr;
  nasd_uint16 ipport;

  if (nasd_srpc_use_counter == 0) { return (NASD_SYS_NOT_INITIALIZED); }

  rc = nasd_shutdown_list_init(&shutdown_list);
  if (rc) { return (rc); }

  rc = nasd_srpc_conn_get(&conn);
  if (conn == NULL) { goto bad; }

  rc = nasd_shutdown_proc(shutdown_list, nasd_srpc_conn_shutdown_conn, conn);
  if (rc) { nasd_srpc_conn_shutdown_conn(conn); goto bad; }

  conn->shutdown_list = shutdown_list;

  rc = nasd_inet_aton(servername, &ipaddr);
  if (rc) { rc = NASD_SRPC_BAD_ADDR; goto bad; }

  rc = nasd_inet_svcton(portnum, &ipport);
  if (rc) { rc = NASD_SRPC_BAD_PORT; goto bad; }

  conn->ipaddr = ipaddr;
  conn->ipport = ipport;

  rc = nasd_srpc_sock_get(&sock);
  if (rc) { goto bad; }

  rc = nasd_shutdown_proc(shutdown_list, nasd_srpc_conn_shutdown_sock, conn);
  if (rc) { nasd_srpc_conn_shutdown_sock(conn); goto bad; }

  conn->sock        = sock;
  conn->kind        = NASD_SRPC_CONN_CLIH;
  conn->state       = NASD_SRPC_STATE_IDLE;
  conn->use_count   = 0;
  conn->next_callid = 1;


  NASD_SRPC_LOCK_CONN(conn);

  rc = nasd_srpc_conn_conn(conn);
  if (rc) { goto bad; }

  rc = nasd_srpc_client_sys_conn_init(conn);
  if (rc) { goto bad; }

  NASD_SRPC_UNLOCK_CONN(conn);

  NASD_LOCK_MUTEX(nasd_srpc_use_counter_lock);

  if (nasd_srpc_use_counter == 0) {
    NASD_UNLOCK_MUTEX(nasd_srpc_use_counter_lock);
    rc = NASD_SYS_NOT_INITIALIZED;
    goto bad;
  }

  nasd_srpc_use_counter++;
  NASD_UNLOCK_MUTEX(nasd_srpc_use_counter_lock);

  *handlep = (nasd_srpc_handle_t *)conn;
  return (NASD_SUCCESS);

bad:
  rc2 = nasd_shutdown_list_shutdown(shutdown_list,
                                    NASD_SHUTDOWN_ANNOUNCE_NONE);
  if (rc2) { NASD_PANIC(); }

  return (rc);
}


nasd_status_t nasd_srpc_unbind_server(nasd_srpc_handle_t  *handlep) {
  nasd_shutdown_list_t *shutdown_list;
  nasd_srpc_conn_t *conn;
  nasd_status_t rc;

  conn = *handlep;
  *handlep = NULL;

  NASD_SRPC_LOCK_CONN(conn);

  if (conn->state & NASD_SRPC_STATE_DISC) {
    NASD_SRPC_UNLOCK_CONN(conn);
    return (NASD_SRPC_ALREADY_DISCONN);
  }

  rc = nasd_srpc_conn_disc(conn);
  if (rc) { NASD_PANIC(); } /* XXX */

  NASD_SRPC_UNLOCK_CONN(conn);
  
  shutdown_list = conn->shutdown_list;
  conn->shutdown_list = NULL;

  rc = nasd_shutdown_list_shutdown(shutdown_list, NASD_SHUTDOWN_ANNOUNCE_NONE);
  if (rc) { NASD_PANIC(); }

  nasd_srpc_shutdown();

  return (NASD_SUCCESS);
}


/*
 * Called by listener thread when data is available.
 * Caller holds no locks.
 */
nasd_status_t nasd_srpc_client_handle(nasd_srpc_conn_t  *conn,
                                      nasd_srpc_call_t  *until_call) {
  nasd_srpc_header_otw_t header_otw;
  nasd_srpc_header_t header;
  nasd_srpc_call_t *call;
  nasd_status_t rc;
  int recvd, h;

  if (until_call == NULL) { NASD_SRPC_LOCK_CONN(conn); }

  if (!NASD_SRPC_CONN_CONNECTED(conn)) {
    if (until_call == NULL) { NASD_SRPC_UNLOCK_CONN(conn); }
    return (NASD_SRPC_CALL_ABORTED);
  }

  if (until_call == NULL) {
    /* When this bit is set, someone else (some other thread)
       has taken over receive duty. */
    while(conn->state&NASD_SRPC_STATE_OP_R) {
      if (!(NASD_SRPC_CONN_CONNECTED(conn))) {
        NASD_SRPC_UNLOCK_CONN(conn);
        return (NASD_SRPC_CALL_ABORTED);
      }
      NASD_SRPC_CONN_WAIT_STATE(conn);
    }
  } else { NASD_ASSERT(conn->state & NASD_SRPC_STATE_OP_R); }

  if (!(NASD_SRPC_CONN_CONNECTED(conn))) {
    if (until_call == NULL) { NASD_SRPC_UNLOCK_CONN(conn); }
    return (NASD_SRPC_CALL_ABORTED);
  }

start_receive:
  if (until_call) {
    rc = nasd_srpc_sys_sock_recv(conn->sock, header_otw, 
                                 sizeof(nasd_srpc_header_otw_t), &recvd,
                                 NASD_SRPC_RECV_FILL);
  } else {
    conn->state |= NASD_SRPC_STATE_OP_R;
    rc = nasd_srpc_sys_sock_recv(conn->sock, header_otw,
                                 sizeof(nasd_srpc_header_otw_t), &recvd,
                                 NASD_SRPC_RECV_FILL|NASD_SRPC_RECV_NOWAIT);
  }

  if (rc != NASD_SUCCESS) { goto recv_idam; }

  if (recvd == 0) { /* Nothing to receive, bail */
    if (until_call == NULL) {
      conn->state &= ~NASD_SRPC_STATE_OP_R;
      NASD_BROADCAST_COND(conn->state_cond);
      NASD_SRPC_UNLOCK_CONN(conn);
      return (NASD_SUCCESS);
    } else { /* On one hand, we could return success and let the client proc
                handle it. On the other hand, we should never get here. */
      NASD_PANIC();
      return (NASD_SUCCESS);
    }
  }

  NASD_ASSERT(NASD_SRPC_CONN_CONNECTED(conn));

  NASD_ASSERT(recvd == sizeof(nasd_srpc_header_otw_t));
  nasd_srpc_header_t_unmarshall(header_otw, &header);

  DEBUG_SRPC_PRINT_HEADER_INFO(header);

  switch(header.op) {
    case NASD_SRPC_OP_NOOP: { break; }
    case NASD_SRPC_OP_PIPEDATA:
    case NASD_SRPC_OP_PIPETERM:
    case NASD_SRPC_OP_PIPEENAB:
    {
      h = NASD_SRPC_CALL_HASH(header.callid);
      if (until_call && (until_call->callid == header.callid)) {
        call = until_call;

        if (header.op == NASD_SRPC_OP_PIPEENAB) { call->pipe_enabled = 1; }
        else { call->pipe_header = header; }

        call->local_stat = NASD_SRPC_S_SUCCESS;
        return (NASD_SUCCESS);
      }

      for(call = conn->c_call_buckets[h].next;
          call != &conn->c_call_buckets[h];
          call = call->next) {
        if (call->callid == header.callid) { break; }
      }
      if ((header.callid == 0) || (call->callid != header.callid)) {
        /* bad callid */
        goto recv_idam;
      }

      /* This is someone else's pipe op. */
      if ((call->pipe_expected == 0) ||
          ((call->last_pipe_finished != NASD_SRPC_PIPENUM_NOPIPES)
           && (call->last_pipe_finished >= header.opstat))) {
        /* Bad mojo
           Either we weren't expecting any pipe ops, or this
           is a pipe op for a finished pipe */
        if (call->pipe_expected == 0) {
          call->local_stat = NASD_SRPC_S_BAD_PIPENO;
        }
        rc = nasd_srpc_slurp_extra(conn, header.len);
        if (rc) { goto recv_idam; }
        break;
      }

      if (header.op == NASD_SRPC_OP_PIPEENAB) { call->pipe_enabled = 1; }

      call->local_stat = NASD_SRPC_S_SUCCESS;
      NASD_BROADCAST_COND(call->data_ready);
      break;
    }
    case NASD_SRPC_OP_REP:
    {
      h = NASD_SRPC_CALL_HASH(header.callid);
      if (until_call && (until_call->callid == header.callid)) {
        call = until_call;
        goto got_call;
      }
      for(call = conn->c_call_buckets[h].next;
          call != &conn->c_call_buckets[h];
          call = call->next) {
        if (call->callid == header.callid) { break; }
      }

      if ((header.callid == 0) || (call->callid != header.callid)) {
        /* bad callid */
        goto recv_idam;
      }
got_call:
      /* This should be the last we hear about this call,
         so delete it from the hash table. */
      call->prev->next = call->next;
      call->next->prev = call->prev;
      call->ishashed = 0;
      if (header.len > NASD_SRPC_MAX_RES_BYTES) {
        /* Bad length on the reply. */
        rc = nasd_srpc_slurp_extra(conn, header.len);
        if (rc) { goto recv_idam; }

        call->local_stat = NASD_SRPC_S_BAD_OPLEN;
        call->state++;
        if (until_call == call) { return (NASD_SUCCESS); }
        NASD_BROADCAST_COND(call->data_ready);
        break;
      }
      if (header.len) {
        rc = nasd_srpc_sys_sock_recv(conn->sock, call->resmem,
          header.len, &recvd, NASD_SRPC_RECV_FILL);
        if (rc != NASD_SUCCESS) { goto recv_idam; }
        NASD_ASSERT(recvd == header.len);
      }
      call->state++;
      call->local_stat = NASD_SRPC_S_SUCCESS;
      call->header = header;
      /* We'll set this here, too, in case this is a misordering by the
         server so we don't get stuck spinning around in a pull pipe. */
      call->pipe_header = header;
      if (until_call == call) { return (NASD_SUCCESS); }
      NASD_BROADCAST_COND(call->data_ready);
      break;
    }
    case NASD_SRPC_OP_SETSEQ:
    case NASD_SRPC_OP_REQ:
    default:
      goto recv_idam;
  }

  /* Either we're waiting for a particular call, and this wasn't
     it, or we're doing anything (ie, client proc), so we don't
     particularly feel the need to go back. */
  goto start_receive;

  /* NOTREACHED */

 recv_idam:

  conn->state |= NASD_SRPC_STATE_IDAM;

  if (until_call == NULL) {
    conn->state &= ~NASD_SRPC_STATE_OP_R;
    NASD_SRPC_UNLOCK_CONN(conn);
  }

  NASD_BROADCAST_COND(conn->state_cond);
  return (NASD_SRPC_CALL_ABORTED_DISC);
}


/*
 * Caller does not hold conn lock
 *
 * Bailing out on running a pipe- if there's
 * data pending for the pipe to run out, drain it
 */
nasd_status_t
nasd_srpc_client_drain_serverpush(nasd_srpc_conn_t     *conn,
                                  nasd_srpc_call_t     *call,
                                  nasd_srpc_pipenum_t   pipenum) {
  nasd_status_t rc;

  NASD_ASSERT(NASD_SRPC_CONN_LOCK_NOT_OWNED(conn));

  if (call->pipe_header.opstat != pipenum) { return (NASD_SUCCESS); }

  if (call->pipe_header.op == NASD_SRPC_OP_PIPETERM) {
    call->pipe_header.opstat = NASD_SRPC_PIPENUM_NOPIPES;
    return (NASD_SUCCESS);
  }

  NASD_ASSERT(call->pipe_header.op == NASD_SRPC_OP_PIPEDATA);
  NASD_ASSERT(conn->state & NASD_SRPC_STATE_OP_R);
  
  rc = nasd_srpc_slurp_extra(conn, call->pipe_header.len);
  if (rc) {
    NASD_SRPC_LOCK_CONN(conn);
    conn->state |= NASD_SRPC_STATE_IDAM;
    NASD_SRPC_UNLOCK_CONN(conn);

    NASD_BROADCAST_COND(conn->state_cond);
    return (rc);
  }

  NASD_SRPC_LOCK_CONN(conn);
  conn->state &= ~NASD_SRPC_STATE_OP_R;
  NASD_SRPC_UNLOCK_CONN(conn);

  return (NASD_SUCCESS);
}


/*
 * Caller does not hold conn lock
 *
 * SERVER is pushing. We're getting data FROM the server.
 */
nasd_status_t
nasd_srpc_client_run_serverpush(nasd_srpc_conn_t     *conn,
                                nasd_srpc_call_t     *call,
                                void                 *rock,
                                nasd_srpc_pipenum_t   pipenum,
                                int                  *bytes_pushed,
                                int                   final_state,
                                nasd_srpc_status_t   *status) {
  int terminated, n, buf_remain, wire_remain, recvd;
  nasd_mem_list_t *cur_elem, *memlist;
  char *this_buf, *cur_buf;
  nasd_status_t rc, rc2;

  memlist       = (nasd_mem_list_t *) rock;
  terminated    = 0;
  *bytes_pushed = 0;
  n             = 0;

  NASD_ASSERT(NASD_SRPC_CONN_LOCK_NOT_OWNED(conn));

  if (memlist == NULL) {
    call->last_pipe_finished = pipenum;
    rc = nasd_srpc_client_drain_serverpush(conn, call, pipenum);

    if (rc == NASD_SUCCESS) { rc = NASD_BAD_MEM_LIST; }
    return (rc);
  }

  *status = NASD_SRPC_S_SUCCESS;

  cur_elem = memlist;
  while (cur_elem && ((cur_elem->len == 0) || (cur_elem->nelem == 0))) {
    cur_elem = cur_elem->next;
  }

  if (cur_elem == NULL) {
    call->last_pipe_finished = pipenum;
    rc = nasd_srpc_client_drain_serverpush(conn, call, pipenum);

    if (rc == NASD_SUCCESS) { rc = NASD_BAD_MEM_LIST; }
    return (rc);
  }

  buf_remain = cur_elem->len;
  cur_buf = this_buf = (char *) cur_elem->addr;

  NASD_SRPC_LOCK_CONN(conn);

  do {
    if (NASD_SRPC_CONN_CONNECTED(conn)
        && (!(conn->state &(NASD_SRPC_STATE_IDAM |
                            NASD_SRPC_STATE_DISC | NASD_SRPC_STATE_OP_R)))
        && (call->pipe_header.callid != call->callid)
        && (call->state < final_state)) {
      /* receive until we get a header we want */

      conn->state |= NASD_SRPC_STATE_OP_R;
      rc = nasd_srpc_client_handle(conn, call);

      if (rc) {
        conn->state &= ~NASD_SRPC_STATE_OP_R;
        conn->state |= NASD_SRPC_STATE_IDAM | NASD_SRPC_STATE_ODAM;

        NASD_SRPC_CONN_DEC_USE(conn);

        if (rc == NASD_SRPC_CALL_ABORTED_DISC) {
          rc2 = nasd_srpc_conn_disc(conn);
          if (rc2) { NASD_PANIC(); } /* XXX */
        }

        NASD_SRPC_UNLOCK_CONN(conn);
        NASD_BROADCAST_COND(conn->state_cond);

        *status = NASD_SRPC_S_ABORTED;
        return (NASD_SUCCESS);
      }
    } else {
      while (NASD_SRPC_CONN_CONNECTED(conn)
             && (!(conn->state & NASD_SRPC_STATE_IDAM))
             && (!(conn->state & NASD_SRPC_STATE_DISC))
             && (call->pipe_header.callid != call->callid)) {
        NASD_WAIT_COND(call->data_ready, conn->lock);
      }
    }

    if (conn->state & (NASD_SRPC_STATE_IDAM | NASD_SRPC_STATE_DISC)) {
      if (call->pipe_header.callid == call->callid) {
        /* we're responsible for being the receiver */
        NASD_ASSERT(conn->state & NASD_SRPC_STATE_OP_R);
        conn->state &= ~NASD_SRPC_STATE_OP_R;
      }

      NASD_SRPC_UNLOCK_CONN(conn);
      NASD_BROADCAST_COND(conn->state_cond);
      *status = NASD_SRPC_S_ABORTED;
      return (NASD_SUCCESS);
    }

    /* As if by magic, we have a header for our call */
    switch(call->pipe_header.op) {
      case NASD_SRPC_OP_PIPEDATA: {
        NASD_ASSERT(conn->state & NASD_SRPC_STATE_OP_R);

        if (call->pipe_header.opstat != pipenum) {
          /* Wrong pipe. */

          rc = nasd_srpc_slurp_extra(conn, call->pipe_header.len);
          if (rc) {
            conn->state &= ~NASD_SRPC_STATE_OP_R;
            conn->state |= NASD_SRPC_STATE_IDAM;

            NASD_SRPC_UNLOCK_CONN(conn);
            NASD_BROADCAST_COND(conn->state_cond);

            *status = NASD_SRPC_S_ABORTED;
            return (NASD_SUCCESS);
          }

          conn->state &= ~NASD_SRPC_STATE_OP_R;

          NASD_SRPC_UNLOCK_CONN(conn);
          NASD_BROADCAST_COND(conn->state_cond);

          *status = NASD_SRPC_S_PIPEORD;
          return (NASD_SUCCESS);
        }

        /* Some data is pending at head-of-wire. If we have buffer space,
           pull it in. Otherwise, discard. */
        wire_remain = call->pipe_header.len;
        while(wire_remain) {
          if (cur_elem) {
            rc = nasd_srpc_sys_sock_recv(conn->sock, cur_buf, 
                                         NASD_MIN(buf_remain, wire_remain),
                                         &recvd, NASD_SRPC_RECV_FILL);
            
            if (rc != NASD_SUCCESS) {
              conn->state &= ~NASD_SRPC_STATE_OP_R;
              conn->state |= NASD_SRPC_STATE_IDAM;
              
              NASD_SRPC_UNLOCK_CONN(conn);
              NASD_BROADCAST_COND(conn->state_cond);

              *status = NASD_SRPC_S_ABORTED;
              return (NASD_SUCCESS);
            }

            wire_remain   -= recvd;
            buf_remain    -= recvd;
            *bytes_pushed += recvd;

            if (buf_remain == 0) {
              n++;
              if (n >= cur_elem->nelem) {
                n = 0;

                do {
                  cur_elem = cur_elem->next;
                } while (cur_elem && ((cur_elem->len == 0) ||
                                      (cur_elem->nelem == 0)));

                if (cur_elem) {
                  this_buf   = (char *)cur_elem->addr;
                  cur_buf    = this_buf;
                  buf_remain = cur_elem->len;
                } else {
                  this_buf   = NULL;
                  cur_buf    = NULL;
                  buf_remain = 0;
                }
              } else {
                cur_buf    = &this_buf[cur_elem->stride];
                this_buf   = cur_buf;
                buf_remain = cur_elem->len;
              }
            } else {
              cur_buf = &cur_buf[recvd];
            }
          } else {
            rc = nasd_srpc_slurp_extra(conn, wire_remain);

            if (rc) {
              conn->state &= ~NASD_SRPC_STATE_OP_R;
              conn->state |= NASD_SRPC_STATE_IDAM;

              NASD_SRPC_UNLOCK_CONN(conn);
              NASD_BROADCAST_COND(conn->state_cond);

              *status = NASD_SRPC_S_ABORTED;
              return (NASD_SUCCESS);
            }
          }
        }

        NASD_ASSERT(conn->state & NASD_SRPC_STATE_OP_R);

        call->pipe_header.callid = 0;
        call->pipe_header.opstat = NASD_SRPC_PIPENUM_NOPIPES;

        conn->state &= ~NASD_SRPC_STATE_OP_R;
        break;
      }
      case NASD_SRPC_OP_PIPETERM: {
        /* Server isn't sending any more. We can escape. */
        if (call->pipe_header.opstat != pipenum) {
          /* Wrong pipe. */
          rc = nasd_srpc_slurp_extra(conn, call->pipe_header.len);

          if (rc) {
            conn->state &= ~NASD_SRPC_STATE_OP_R;
            conn->state |= NASD_SRPC_STATE_IDAM;

            NASD_SRPC_UNLOCK_CONN(conn);
            NASD_BROADCAST_COND(conn->state_cond);

            *status = NASD_SRPC_S_ABORTED;
            return (NASD_SUCCESS);
          }

          conn->state &= ~NASD_SRPC_STATE_OP_R;

          NASD_SRPC_UNLOCK_CONN(conn);
          NASD_BROADCAST_COND(conn->state_cond);

          *status = NASD_SRPC_S_PIPEORD;
          return (NASD_SUCCESS);
        }

        conn->state &= ~NASD_SRPC_STATE_OP_R;
        terminated = 1;
        break;
      }
      case NASD_SRPC_OP_REP: {
        /* Server is cutting us off. Bail out and let the caller know. */
        NASD_SRPC_UNLOCK_CONN(conn);
        NASD_BROADCAST_COND(conn->state_cond);

        *status = NASD_SRPC_S_ABORTED;
        return (NASD_SRPC_PIPE_REPLIED);

        /* NOTREACHED */
        break;
      }
      default: {
        *status = NASD_SRPC_S_BAD_PIPE;
        NASD_SRPC_UNLOCK_CONN(conn);
        NASD_BROADCAST_COND(conn->state_cond);
        return (NASD_SUCCESS);
        /* NOTREACHED */
        break;
      }
    }

  } while (!terminated);

  NASD_SRPC_UNLOCK_CONN(conn);
  NASD_BROADCAST_COND(conn->state_cond);

  return (NASD_SUCCESS);
}


/*
 * Caller does not hold conn lock
 *
 * SERVER is pulling. We're sending data TO the server.
 */
nasd_status_t
nasd_srpc_client_run_serverpull(nasd_srpc_conn_t     *conn,
                                nasd_srpc_call_t     *call,
                                void                 *rock,
                                nasd_srpc_pipenum_t   pipenum,
                                nasd_srpc_status_t   *status) {
  nasd_srpc_memvec_t vec[NASD_SRPC_CLI_MAX_PIPE_COALESCE];
  nasd_mem_list_t *memlist, *cur_elem;
  nasd_srpc_header_otw_t header_otw;
  int n, sent, nvecs, total_len;
  nasd_srpc_header_t header;
  nasd_status_t rc;
  char *this_buf;

  NASD_ASSERT(NASD_SRPC_CONN_LOCK_NOT_OWNED(conn));

  memlist = (nasd_mem_list_t *)rock;
  if (memlist == NULL) { return (NASD_BAD_MEM_LIST); }

  cur_elem = memlist;
  while (cur_elem && ((cur_elem->len == 0) || (cur_elem->nelem == 0))) {
    cur_elem = cur_elem->next;
  }
  if (cur_elem == NULL) { return (NASD_BAD_MEM_LIST); }

  this_buf = (char *) cur_elem->addr;
  n = 0;
  
  NASD_SRPC_LOCK_CONN(conn);
  
  while((conn->state & NASD_SRPC_STATE_OP_S) &&
        (!(conn->state & (NASD_SRPC_STATE_ODAM | NASD_SRPC_STATE_DISC)))) {
    NASD_SRPC_CONN_WAIT_STATE(conn);
  }
  
  if (conn->state & (NASD_SRPC_STATE_ODAM | NASD_SRPC_STATE_DISC)) {
    NASD_SRPC_UNLOCK_CONN(conn);
    *status = NASD_SRPC_S_ABORTED;
    return (NASD_SUCCESS);
  }

  conn->state |= NASD_SRPC_STATE_OP_S;

  do {
    total_len   = sizeof(nasd_srpc_header_otw_t);
    vec[0].buf  = header_otw;
    vec[0].len  = sizeof(nasd_srpc_header_otw_t);
    vec[0].next = NULL;
    nvecs = 1;

    while(cur_elem) {
      vec[nvecs-1].next  = &vec[nvecs];
      vec[nvecs].buf     = this_buf;
      vec[nvecs].len     = cur_elem->len;
      vec[nvecs].next    = NULL;
      total_len         += cur_elem->len;
      nvecs++;
      n++;

      if (n >= cur_elem->nelem) {
        n = 0;
        do {
          cur_elem = cur_elem->next;
        } while(cur_elem && ((cur_elem->len == 0) || (cur_elem->nelem == 0)));

        if (cur_elem == NULL) { break; }
        this_buf = (char *)cur_elem->addr;
      } else {
        this_buf = &this_buf[cur_elem->stride];
      }
      
      if (nvecs >= NASD_SRPC_CLI_MAX_PIPE_COALESCE) { break; }
    }

    /* Send off what we have */
    header.callid = call->callid;
    header.len    = total_len - sizeof(nasd_srpc_header_otw_t);
    header.op     = NASD_SRPC_OP_PIPEDATA;
    header.opstat = pipenum;

    nasd_srpc_header_t_marshall(&header, header_otw);

    rc = nasd_srpc_sys_sock_send(conn->sock, vec, total_len, &sent);
    if (rc) {
      conn->state &= ~NASD_SRPC_STATE_OP_S;
      conn->state |= NASD_SRPC_STATE_IDAM;

      NASD_SRPC_UNLOCK_CONN(conn);
      NASD_BROADCAST_COND(conn->state_cond);

      *status = NASD_SRPC_S_ABORTED;
      return (NASD_SUCCESS);
    }

    NASD_ASSERT(sent == total_len);

    nvecs = 0;

  } while(cur_elem);

  NASD_ASSERT(nvecs == 0);
  
  /* send terminate op */
  vec[0].buf  = header_otw;
  vec[0].len  = sizeof(nasd_srpc_header_otw_t);
  vec[0].next = NULL;

  header.callid = call->callid;
  header.len    = 0;
  header.op     = NASD_SRPC_OP_PIPETERM;
  header.opstat = pipenum;

  nasd_srpc_header_t_marshall(&header, header_otw);

  rc = nasd_srpc_sys_sock_send(conn->sock, vec,
                               sizeof(nasd_srpc_header_otw_t), &sent);
  if (rc) {
    conn->state &= ~NASD_SRPC_STATE_OP_S;
    conn->state |= NASD_SRPC_STATE_IDAM;

    NASD_SRPC_UNLOCK_CONN(conn);
    NASD_BROADCAST_COND(conn->state_cond);

    *status = NASD_SRPC_S_ABORTED;
    return (NASD_SUCCESS);
  }

  NASD_ASSERT(sent == sizeof(nasd_srpc_header_otw_t));

  conn->state &= ~NASD_SRPC_STATE_OP_S;
  
  NASD_SRPC_UNLOCK_CONN(conn);
  NASD_BROADCAST_COND(conn->state_cond);

  *status = NASD_SRPC_S_SUCCESS;
  return (NASD_SUCCESS);
}


/*
 * Caller does not hold conn lock
 *
 * Wait for the "okay" to send pipe data
 */
nasd_status_t nasd_srpc_client_wait_pipeenab(nasd_srpc_conn_t    *conn,
                                             nasd_srpc_call_t    *call,
                                             int                  final_state,
                                             nasd_srpc_status_t  *status) {
  nasd_status_t rc, rc2;
  int dobc = 0;

  NASD_SRPC_LOCK_CONN(conn);

  if (NASD_SRPC_CONN_CONNECTED(conn)
      && (!(conn->state &(NASD_SRPC_STATE_IDAM |
                          NASD_SRPC_STATE_DISC | NASD_SRPC_STATE_OP_R)))
      && (call->state < final_state)
      && (call->pipe_enabled == 0)) {
    /* At the moment, no one is doing receive duty.
       We "volunteer" to do so until we get the reply we want. */

    conn->state |= NASD_SRPC_STATE_OP_R;

    rc = nasd_srpc_client_handle(conn, call);

    conn->state &= ~NASD_SRPC_STATE_OP_R;

    dobc = 1;

    if (rc) {
      NASD_BROADCAST_COND(conn->state_cond);
      NASD_SRPC_CONN_DEC_USE(conn);

      if (rc == NASD_SRPC_CALL_ABORTED_DISC) {
        rc2 = nasd_srpc_conn_disc(conn);
        if (rc2) { NASD_PANIC(); } /* XXX */
      }

      NASD_SRPC_UNLOCK_CONN(conn);
      *status = NASD_SRPC_S_ABORTED;
      return (rc);
    }

    /* If for whatever reason what we got wasn't an enable, we
       fall through and catch whatever's going on below anyway */
  }

  while(NASD_SRPC_CONN_CONNECTED(conn)
        && (!(conn->state & NASD_SRPC_STATE_IDAM))
        && (!(conn->state & NASD_SRPC_STATE_DISC))
        && (call->state < final_state)
        && (call->pipe_enabled == 0)) {
    NASD_WAIT_COND(call->data_ready, conn->lock);
  }

  if (conn->state & (NASD_SRPC_STATE_IDAM | NASD_SRPC_STATE_DISC)) {
    NASD_BROADCAST_COND(conn->state_cond);
    NASD_SRPC_UNLOCK_CONN(conn);

    *status = NASD_SRPC_S_ABORTED;
    return (NASD_SUCCESS);
  }

  NASD_ASSERT(call->pipe_enabled);

  NASD_SRPC_UNLOCK_CONN(conn);
  if (dobc) { NASD_BROADCAST_COND(conn->state_cond); }

  *status = NASD_SRPC_S_SUCCESS;
  return (NASD_SUCCESS);
}


/*
 * Caller does not hold conn lock
 *
 * SERVER is pushing. We're getting data FROM the server.
 */
nasd_status_t
nasd_srpc_client_run_serverpush_cb(nasd_srpc_conn_t     *conn,
                                   nasd_srpc_call_t     *call,
                                   void                 *rock,
                                   nasd_srpc_pipenum_t   pipenum,
                                   int                  *bytes_pushed,
                                   int                   final_state,
                                   nasd_srpc_status_t   *status) {
  int terminated, wire_remain, recvd, buf_remain;
  nasd_srpc_client_pushcb_t *pushcb;
  nasd_byte_t *this_buf, *cur_buf;
  nasd_uint32 cur_buf_len;
  nasd_status_t rc, rc2;

  pushcb = (nasd_srpc_client_pushcb_t *) rock;
  terminated    = 0;
  *bytes_pushed = 0;

  *status = NASD_SRPC_S_SUCCESS;

  NASD_ASSERT(NASD_SRPC_CONN_LOCK_NOT_OWNED(conn));

  pushcb->alloc_cb(pushcb->state, NASD_SRPC_CLI_MAX_PUSH_ALLOC,
                   &this_buf, &cur_buf_len);

  if (cur_buf_len == 0) {
    call->last_pipe_finished = pipenum;
    rc = nasd_srpc_client_drain_serverpush(conn, call, pipenum);

    if (rc == NASD_SUCCESS) { rc = NASD_PIPE_CB_FAIL; }
    return (rc);
  }

  cur_buf    = this_buf;
  buf_remain = cur_buf_len;

  NASD_SRPC_LOCK_CONN(conn);

  do {

    if (NASD_SRPC_CONN_CONNECTED(conn)
        && (!(conn->state &(NASD_SRPC_STATE_IDAM |
                            NASD_SRPC_STATE_DISC|NASD_SRPC_STATE_OP_R)))
        && (call->pipe_header.callid != call->callid)
        && (call->state < final_state)) {

      /* At the moment, no one is doing receive duty.
         We "volunteer" to do so until we get a header we want. */
      conn->state |= NASD_SRPC_STATE_OP_R;
      rc = nasd_srpc_client_handle(conn, call);
      
      if (rc) {
        conn->state &= ~NASD_SRPC_STATE_OP_R;
        conn->state |= NASD_SRPC_STATE_IDAM | NASD_SRPC_STATE_ODAM;
        NASD_SRPC_CONN_DEC_USE(conn);

        if (rc == NASD_SRPC_CALL_ABORTED_DISC) {
          rc2 = nasd_srpc_conn_disc(conn);
          if (rc2) { NASD_PANIC(); } /* XXX */
        }

        NASD_SRPC_UNLOCK_CONN(conn);
        NASD_BROADCAST_COND(conn->state_cond);

        *status = NASD_SRPC_S_ABORTED;
        return (NASD_SUCCESS);
      }
    } else {
      while(NASD_SRPC_CONN_CONNECTED(conn)
            && (!(conn->state & NASD_SRPC_STATE_IDAM))
            && (!(conn->state & NASD_SRPC_STATE_DISC))
            && (call->pipe_header.callid != call->callid)) {
        NASD_WAIT_COND(call->data_ready, conn->lock);
      }
    }

    if (conn->state & (NASD_SRPC_STATE_IDAM | NASD_SRPC_STATE_DISC)) {
      if (call->pipe_header.callid == call->callid) {
        /* we're responsible for being the receiver */
        NASD_ASSERT(conn->state & NASD_SRPC_STATE_OP_R);
        conn->state &= ~NASD_SRPC_STATE_OP_R;
      }

      NASD_SRPC_UNLOCK_CONN(conn);
      NASD_BROADCAST_COND(conn->state_cond);

      *status = NASD_SRPC_S_ABORTED;
      return (NASD_SUCCESS);
    }

    /* As if by magic, we have a header for our call */
    switch(call->pipe_header.op) {
      case NASD_SRPC_OP_PIPEDATA: {
        NASD_ASSERT(conn->state & NASD_SRPC_STATE_OP_R);

        if (call->pipe_header.opstat != pipenum) { /* Wrong pipe. */
          rc = nasd_srpc_slurp_extra(conn, call->pipe_header.len);
          
          if (rc) {
            conn->state |= NASD_SRPC_STATE_IDAM;
            *status = NASD_SRPC_S_ABORTED;
          } else {
            *status = NASD_SRPC_S_PIPEORD;
          }

          conn->state &= ~NASD_SRPC_STATE_OP_R;
          NASD_SRPC_UNLOCK_CONN(conn);
          NASD_BROADCAST_COND(conn->state_cond);
          return (NASD_SUCCESS);
        }

        /* Some data is pending at head-of-wire. If we have buffer space,
           pull it in. Otherwise, discard. */

        wire_remain = call->pipe_header.len;
        while(wire_remain) {
          if (this_buf) {
            rc = nasd_srpc_sys_sock_recv(conn->sock, cur_buf,
                                         NASD_MIN(buf_remain, wire_remain),
                                         &recvd, NASD_SRPC_RECV_FILL);
            if (rc != NASD_SUCCESS) {
              conn->state &= ~NASD_SRPC_STATE_OP_R;
              conn->state |= NASD_SRPC_STATE_IDAM;

              NASD_SRPC_UNLOCK_CONN(conn);
              NASD_BROADCAST_COND(conn->state_cond);

              *status = NASD_SRPC_S_ABORTED;
              return (NASD_SUCCESS);
            }

            wire_remain   -= recvd;
            buf_remain    -= recvd;
            *bytes_pushed += recvd;

            pushcb->push_cb(pushcb->state, cur_buf, recvd);

            if (buf_remain == 0) {
              pushcb->alloc_cb(pushcb->state, NASD_SRPC_CLI_MAX_PUSH_ALLOC,
                               &this_buf, &cur_buf_len);

              if (this_buf) { buf_remain = cur_buf_len; }
              else          { buf_remain = 0;           }

              cur_buf  = this_buf;
              this_buf = NULL;

            } else {
              cur_buf = &cur_buf[recvd];
            }
          } else {
            rc = nasd_srpc_slurp_extra(conn, wire_remain);
            if (rc) {
              conn->state &= ~NASD_SRPC_STATE_OP_R;
              conn->state |= NASD_SRPC_STATE_IDAM;

              NASD_SRPC_UNLOCK_CONN(conn);
              NASD_BROADCAST_COND(conn->state_cond);

              *status = NASD_SRPC_S_ABORTED;
              return (NASD_SUCCESS);
            }
          }
        }

        NASD_ASSERT(conn->state & NASD_SRPC_STATE_OP_R);
        call->pipe_header.callid = 0;
        call->pipe_header.opstat = NASD_SRPC_PIPENUM_NOPIPES;
        conn->state &= ~NASD_SRPC_STATE_OP_R;
        break;
      }
      case NASD_SRPC_OP_PIPETERM: {
        /* Server isn't sending any more. We can escape. */

        if (call->pipe_header.opstat != pipenum) { /* Wrong pipe. */

          rc = nasd_srpc_slurp_extra(conn, call->pipe_header.len);

          if (rc) {
            conn->state |= NASD_SRPC_STATE_IDAM;
            *status = NASD_SRPC_S_ABORTED;
          } else {
            *status = NASD_SRPC_S_PIPEORD;
          }

          conn->state &= ~NASD_SRPC_STATE_OP_R;
          NASD_SRPC_UNLOCK_CONN(conn);
          NASD_BROADCAST_COND(conn->state_cond);
          return (NASD_SUCCESS);
        }

        conn->state &= ~NASD_SRPC_STATE_OP_R;
        terminated = 1;
        break;
      }
      case NASD_SRPC_OP_REP: {
        /* Server is cutting us off. Bail out and let the caller know. */
        NASD_SRPC_UNLOCK_CONN(conn);
        NASD_BROADCAST_COND(conn->state_cond);

        *status = NASD_SRPC_S_ABORTED;
        return (NASD_SRPC_PIPE_REPLIED);

        /* NOTREACHED */
        break;
      }
      default: {
        NASD_SRPC_UNLOCK_CONN(conn);
        NASD_BROADCAST_COND(conn->state_cond);

        *status = NASD_SRPC_S_BAD_PIPE;
        return (NASD_SUCCESS);

        /* NOTREACHED */
        break;
      }
    }

  } while(!terminated);

  NASD_SRPC_UNLOCK_CONN(conn);
  NASD_BROADCAST_COND(conn->state_cond);

  pushcb->push_cb(pushcb->state, NULL, 0);

  return (NASD_SUCCESS);
}


/*
 * Caller does not hold conn lock
 *
 * SERVER is pulling. We're sending data TO the server.
 */
nasd_status_t
nasd_srpc_client_run_serverpull_cb(nasd_srpc_conn_t     *conn,
                                   nasd_srpc_call_t     *call,
                                   void                 *rock,
                                   nasd_srpc_pipenum_t   pipenum,
                                   nasd_srpc_status_t   *status) {
  nasd_srpc_memvec_t vec[NASD_SRPC_CLI_MAX_PIPE_COALESCE];
  nasd_uint32 cur_buf_len, pullcb_out_count;
  nasd_srpc_header_otw_t header_otw;
  nasd_srpc_client_pullcb_t *pullcb;
  int sent, nvecs, total_len, i;
  nasd_srpc_header_t header;
  nasd_byte_t *cur_buf;
  nasd_status_t rc;

  NASD_ASSERT(NASD_SRPC_CONN_LOCK_NOT_OWNED(conn));

  pullcb = (nasd_srpc_client_pullcb_t *)rock;

  pullcb->alloc_cb(pullcb->state, NASD_SRPC_CLI_MAX_PULL_ALLOC,
                   &cur_buf, &cur_buf_len);
  
  NASD_SRPC_LOCK_CONN(conn);
  
  while((conn->state&NASD_SRPC_STATE_OP_S) &&
        (!(conn->state & (NASD_SRPC_STATE_ODAM | NASD_SRPC_STATE_DISC)))) {
    NASD_SRPC_CONN_WAIT_STATE(conn);
  }

  if (conn->state & (NASD_SRPC_STATE_ODAM | NASD_SRPC_STATE_DISC)) {
    NASD_SRPC_UNLOCK_CONN(conn);
    *status = NASD_SRPC_S_ABORTED;
    return (NASD_SUCCESS);
  }

  conn->state |= NASD_SRPC_STATE_OP_S;

  do {
    total_len   = sizeof(nasd_srpc_header_otw_t);
    vec[0].buf  = header_otw;
    vec[0].len  = sizeof(nasd_srpc_header_otw_t);
    vec[0].next = &vec[1];
    nvecs       = 1;

    while(cur_buf) {
      vec[nvecs].buf   = cur_buf;
      vec[nvecs].len   = cur_buf_len;
      vec[nvecs].next  = &vec[nvecs+1];
      total_len       += vec[nvecs].len;
      nvecs++;

      pullcb->alloc_cb(pullcb->state, NASD_SRPC_CLI_MAX_PULL_ALLOC,
                       &cur_buf, &cur_buf_len);

      if (nvecs >= NASD_SRPC_CLI_MAX_PIPE_COALESCE) { break; }
    }

    /* Send off what we have */
    header.callid = call->callid;
    header.len    = total_len - sizeof(nasd_srpc_header_otw_t);
    header.op     = NASD_SRPC_OP_PIPEDATA;
    header.opstat = pipenum;

    nasd_srpc_header_t_marshall(&header, header_otw);

    NASD_ASSERT(nvecs > 0);

    vec[nvecs-1].next = NULL;

    rc = nasd_srpc_sys_sock_send(conn->sock, vec, total_len, &sent);

    if (rc) {
      conn->state &= ~NASD_SRPC_STATE_OP_S;
      conn->state |= NASD_SRPC_STATE_IDAM;

      NASD_SRPC_UNLOCK_CONN(conn);
      NASD_BROADCAST_COND(conn->state_cond);

      *status = NASD_SRPC_S_ABORTED;
      return (NASD_SUCCESS);
    }

    NASD_ASSERT(sent == total_len);

    for(i = 1; i < nvecs; i++) {
      pullcb->pull_cb(pullcb->state, vec[i].buf, (nasd_uint32)vec[i].len,
                      &pullcb_out_count);
    }

    nvecs = 0;

  } while(cur_buf);

  NASD_ASSERT(nvecs == 0);

  /* terminate pipe */
  pullcb->pull_cb(pullcb->state, NULL, (nasd_uint32)0, &pullcb_out_count);

  /* send terminate op */
  vec[0].buf  = header_otw;
  vec[0].len  = sizeof(nasd_srpc_header_otw_t);
  vec[0].next = NULL;

  header.callid = call->callid;
  header.len    = 0;
  header.op     = NASD_SRPC_OP_PIPETERM;
  header.opstat = pipenum;

  nasd_srpc_header_t_marshall(&header, header_otw);

  rc = nasd_srpc_sys_sock_send(conn->sock, vec, sizeof(nasd_srpc_header_otw_t),
                               &sent);
  if (rc) {
    conn->state &= ~NASD_SRPC_STATE_OP_S;
    conn->state |= NASD_SRPC_STATE_IDAM;

    NASD_SRPC_UNLOCK_CONN(conn);
    NASD_BROADCAST_COND(conn->state_cond);

    *status = NASD_SRPC_S_ABORTED;
    return (NASD_SUCCESS);
  }

  NASD_ASSERT(sent == sizeof(nasd_srpc_header_otw_t));

  conn->state &= ~NASD_SRPC_STATE_OP_S;

  NASD_SRPC_UNLOCK_CONN(conn);
  NASD_BROADCAST_COND(conn->state_cond);

  *status = NASD_SRPC_S_SUCCESS;
  return (NASD_SUCCESS);
}

/* Local Variables:  */
/* indent-tabs-mode: nil */
/* tab-width: 2 */
/* End: */
