/*
 * mpdrspeed.c
 *
 * Instrument drive read/write speed
 *
 * Authors: David Rochberg, Jim Zelenka
 */
/*
 * Copyright (c) of Carnegie Mellon University, 1996,1997,1998,1999.
 *
 * Permission to reproduce, use, and prepare derivative works of
 * this software for internal use is granted provided the copyright
 * and "No Warranty" statements are included with all reproductions
 * and derivative works. This software may also be redistributed
 * without charge provided that the copyright and "No Warranty"
 * statements are included in all redistributions.
 *
 * NO WARRANTY. THIS SOFTWARE IS FURNISHED ON AN "AS IS" BASIS.
 * CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, EITHER
 * EXPRESSED OR IMPLIED AS TO THE MATTER INCLUDING, BUT NOT LIMITED
 * TO: WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY
 * OF RESULTS OR RESULTS OBTAINED FROM USE OF THIS SOFTWARE. CARNEGIE
 * MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT
 * TO FREEDOM FROM PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT.
 */


#include <nasd/nasd_options.h>
#include <nasd/nasd_pdrive_client.h>
#include <nasd/nasd_pdrive_client_kpdev.h>
#include <nasd/nasd_threadstuff.h>
#include <nasd/mpdrspeed_common.h>
#include <nasd/nasd_timer.h>
#include <nasd/nasd_common.h>
#include <nasd/nasd_security.h>
#include <nasd/nasd_mem.h>
#include <nasd/nasd_getopt.h>
#include <time.h>
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <netdb.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/param.h>
#include <errno.h>
#include <string.h>
#include <netinet/in.h>
#include <netinet/tcp.h>

#if NASD_CMU_PDL > 0
#include <pdllib.h>
#endif /* NASD_CMU_PDL > 0 */

char *progname;

nasd_error_string_t error_text;
nasd_sec_keyring_t keys;

#define DEF_BLOCKSIZE 131072
int blocksize = DEF_BLOCKSIZE;
#define DEF_NITERS 10
int niters = DEF_NITERS;
#define DEF_OFFSET  0
nasd_offset_t default_offset =  DEF_OFFSET;

int do_multi_offsets = 0;
int do_ufs   = 0;
int do_write = 0;
int use_kp   = 0;
int use_tcp  = 0;
int increment_offset = 0;
int lockstep_mode = 0;

nasd_timer_t lockstep_timer;

int extern_sync = 0;
int extern_sync_go = 0;         /* shared 
                                   0: waiting for go signal 
                                   1: go signalled
                                   10: threads are done, waiting for sync thread to write data
                                   100: threads are done, sync thread data written.
                                */

FILE    *syncfi = NULL;
int main_done = 0;
int sync_done = 0;
int ready = 0;

nasd_uint64 bms_targ = 0;

double total_kbytes_s;

nasd_cookie_t cookie;
nasd_uint16 protection;

char *master_password;


unsigned char bit_pattern = 0x00;
int do_compare = 0;
char *gbuf = NULL;
int force =0;
int drives=1;


int multi_files=0;


nasd_threadgroup_t readers;
nasd_threadgroup_t stats_ready; /* We're using this like a barrier sync */



NASD_DECLARE_COND(lockstep_started_cond)
NASD_DECLARE_COND(lockstep_stopped_cond)
NASD_DECLARE_COND(threads_go)
NASD_DECLARE_COND(extern_sync_cond)
NASD_DECLARE_COND(extern_ready_cond)
NASD_DECLARE_MUTEX(printname)
NASD_DECLARE_MUTEX(lockstep_started_mutex)
NASD_DECLARE_MUTEX(lockstep_stopped_mutex)
int lockstep_started = 0;                   /* counter shared among threads */
int lockstep_stopped = 0;
NASD_DECLARE_MUTEX(extern_sync_mutex)
NASD_DECLARE_MUTEX(extern_ready_mutex)
NASD_DECLARE_MUTEX(danger_mutex)
     
     


typedef struct drspeed_thread_args_s {
  nasd_threadgroup_t   *group;
  nasd_thread_t         handle;
  int                   number;
  nasd_identifier_t     nasdid;
  nasd_offset_t         offset;
  nasd_drive_handle_t   drive;
  int                   partnum; /* Due to some bad decisions earlier
                                    on, the partnum also carries the
                                    file descriptor when we're in UFS
                                    mode. */
  char                 *name;
  double                avg;
  double                avg_rate;
  double                accum_time;
  int                   num_ios;
} drspeed_thread_args_t;


drspeed_thread_args_t *drspeed_threads;


nasd_timespec_t diff;
nasd_timer_t    total_timer;
#if NASD_IDLE_SUPPORT > 0
nasd_timespec_t idle_ts1, idle_ts2;
#endif /* NASD_IDLE_SUPPORT > 0 */

void
usage()
{
  int i;
  fprintf(stderr, "USAGE: %s [options] master_password drive partition# [drive partition# ...] \n", progname);
  fprintf(stderr, "  -u use UFS.  Implies -i (offset increment)\n");
  fprintf(stderr, "  -b blocksize (default=%d)\n", DEF_BLOCKSIZE);
  fprintf(stderr, "  -i increment offsets on each iter\n");
  fprintf(stderr, "  -f do not pause if partition has multiple objects\n");
  fprintf(stderr, "  -k use kpdev\n");
  fprintf(stderr, "  -m use the nth object in the mpdrspeed \"filesystem\" for the nth thread\n");
  fprintf(stderr, "  -n iterations (default=%d)\n", DEF_NITERS);
#if NASD_RPC_PACKAGE == NASD_RPC_PACKAGE_DCE
  fprintf(stderr, "  -T use DCE-TCP\n");
#endif /* NASD_RPC_PACKAGE == NASD_RPC_PACKAGE_DCE */
  fprintf(stderr, "  -t throttle value\n");
  fprintf(stderr, "  -v bit pattern\n");
  fprintf(stderr, "  -w write\n");
  fprintf(stderr, "  -S Extern_sync = 1\n");
  fprintf(stderr, "  -s security level\n");
  for(i = 0; i <= NASD_MAX_SECURITY_LEVEL; i++) {
    fprintf(stderr, "     %d %s\n", i, nasd_sec_level_string(i));
  }
  fflush(stderr);
  exit(1);
}

void
dofastread(
  nasd_drive_handle_t   h,
  int                   partnum,
  nasd_identifier_t     nasdid,
  nasd_offset_t         offset,
  char                 *buf,
  int                   len,
  nasd_status_t        *nasd_status,
  nasd_rpc_status_t    *status)
{
  nasd_p_thrtl_op_dr_args_t trd_args;
  nasd_p_smpl_op_dr_args_t rd_args;
  nasd_p_fastread_dr_res_t rd_res;
  nasd_security_param_t sp;
  nasd_len_t out_len;
  int rc;

  if (do_ufs) {
    
    rc = read(partnum /* really fd  */,buf,len); 
    if (rc!=len) {
      perror("ERROR: UFS read failed: ");
      exit(1);
    }
    return;
  }

  sp.type = cookie.capability.type;
  sp.partnum = partnum;
  sp.actual_protection = cookie.capability.min_protection;

  if (bms_targ) { 
    trd_args.in_partnum = partnum;
    trd_args.in_identifier = nasdid;
    trd_args.in_offset = offset;
    trd_args.in_bms_targ = bms_targ;
    trd_args.in_len = len;
    nasd_cl_p_tread_simple_dr(h, cookie.key, &sp, &cookie.capability,
                              &trd_args, buf, &rd_res, status);
    out_len = rd_res.out_datalen;
    *nasd_status = rd_res.nasd_status;
  }
  else {
    rd_args.in_partnum = partnum;
    rd_args.in_identifier = nasdid;
    rd_args.in_offset = offset;
    rd_args.in_len = len;
    nasd_cl_p_read_simple_dr(h, cookie.key, &sp, &cookie.capability,
                             &rd_args, buf, &rd_res, status);
    out_len = rd_res.out_datalen;
    *nasd_status = rd_res.nasd_status;
  }
  if ((*nasd_status == NASD_SUCCESS) && (*status == 0)) {
    if (out_len != len)
      fprintf(stderr, "WARNING: len=%ld out_len=%ld\n",
        (long)len, (long)out_len);
  }
  else {
    fprintf(stderr, "ERROR: read of nasdid 0x%" NASD_ID_FMT
      "got nasd_status 0x%x (%s) status 0x%x (%s)\n",
      nasdid, *nasd_status, nasd_error_string(*nasd_status),
      *status, nasd_cl_error_string(h, *status, error_text));
    exit(1);
  }
}

void
dofastwrite(
  nasd_drive_handle_t   h,
  int                   partnum,
  nasd_identifier_t     nasdid,
  nasd_offset_t         offset,
  char                 *buf,
  int                   len,
  nasd_status_t        *nasd_status,
  nasd_rpc_status_t    *status)
{
  nasd_p_smpl_op_dr_args_t wr_args;
  nasd_p_fastwrite_dr_res_t wr_res;
  nasd_security_param_t sp;
  nasd_len_t out_len;
  int           rc;

  if (do_ufs) {
    rc = write(partnum/* really fd  */,buf,len);
    if (rc!=len) {
      perror("ERROR: UFS write failed: ");
      exit(1);
    }
    return;
  }

  if (bms_targ) {
    fprintf(stderr, "%s: throttling not supported on writes\n", progname);
    fflush(stderr);
    exit(1);
  }
  else {
    wr_args.in_partnum = partnum;
    wr_args.in_identifier = nasdid;
    wr_args.in_offset = offset;
    wr_args.in_len = len;
    sp.type = cookie.capability.type;
    sp.partnum = partnum;
    sp.actual_protection = cookie.capability.min_protection;
    nasd_cl_p_write_simple_dr(h, cookie.key, &sp, &cookie.capability,
                              &wr_args, buf, &wr_res, status);
    out_len = wr_res.out_datalen;
    *nasd_status = wr_res.nasd_status;
  }
  if ((*nasd_status == NASD_SUCCESS) && (*status == 0)) {
    if (out_len != len)
      fprintf(stderr, "WARNING: len=%ld out_len=%ld\n",
        (long)len, (long)out_len);
  }
  else {
    fprintf(stderr, "ERROR: write got nasd_status 0x%x (%s) status 0x%x (%s)\n",
      *nasd_status, nasd_error_string(*nasd_status), *status,
      nasd_cl_error_string(h, *status, error_text));
    exit(1);
  }
}


void
perform_io(
  nasd_drive_handle_t   H,
  int                   partnum,
  char                 *name,
  nasd_identifier_t     nasdid,
  nasd_offset_t         offset,
  char                 *buf,
  char                 *compare_buf,
  int                   len,
  nasd_status_t        *nasd_statusp,
  nasd_rpc_status_t    *statusp)
{
  if (do_write) {
    dofastwrite(H, partnum, nasdid, offset, buf, blocksize,
      nasd_statusp, statusp);
  }
  else {
    dofastread(H, partnum, nasdid, offset, buf, blocksize,
      nasd_statusp, statusp);
  }

  if (compare_buf) {
    if (bcmp(buf, compare_buf, blocksize)) {
      fprintf(stderr, "Failed buf compare!\n");
      fflush(stderr);
      exit(1);
    }
  }
}


nasd_identifier_t
get_first_object(
  nasd_drive_handle_t   drive,
  int                   partnum,
  char                 *name)
{
  nasd_ctrl_part_info_t ptinfo;
  nasd_status_t         rc;
  nasd_rpc_status_t     status;
  nasd_delaycounter_t   delayer;
  nasd_security_param_t sp;

  NASD_BEGIN_DELAYCNT(&delayer);
  NASD_DELAY_FROM(&delayer, 100);

  /* If the drive is freshly started with no partitions, it seems to
     not want to accept I/Os */
  nasd_cl_p_null_dr(drive, &rc, &status);

  /* XXX don't feel like building a capability for this */
  sp.type = NASD_BLACK_KEY;
  sp.partnum = partnum;
  sp.actual_protection = protection;

  rc = nasd_cl_p_ctrl_get_part_info(drive, keys.black_key, &sp, NULL,
                                    partnum, &ptinfo);

  if ((rc == NASD_CTRL_ID_CHECK_FAILED) && (ptinfo.ctrl_id == 0)) {
    /* got back a page of zeroes - no such partition */
    fprintf(stderr, "ERROR: drive %s partition %d does not exist\n", name, partnum);
    fflush(stderr);
    exit(1);
  }

  if (rc) {
    fprintf(stderr, "ERROR: got 0x%x (%s) getting partition info\n",
      rc, nasd_error_string(rc));
    fflush(stderr);
    exit(1);
  }

  if ((ptinfo.num_obj != 1) &&
      (do_write) &&
      (!multi_files) &&
      (!force)) {
    fprintf(stderr, "WARNING:  Partition contains multiple objects\n");
    fprintf(stderr, "Data will be lost.  Sleeping to allow time for abort.\n");
    fflush(stderr);
    sleep(10);
  }
  
  return ptinfo.first_obj;
  
}


void bind_to_drive(drspeed_thread_args_t *thread)
{
  int rc;
  void *binding_args;
  int binding_args_len;
  nasd_drive_param_kpdev_t *kpbind;
  int binding_type;
  
  if (use_kp) {
    binding_type = NASD_BIND_KPDEV_DEFAULT;
    kpbind = malloc(sizeof(nasd_drive_param_kpdev_t));
    binding_args = kpbind;
    binding_args_len = sizeof(nasd_drive_param_kpdev_t);
    sprintf(kpbind->devname, "/dev/nasdkp%d", thread->number);
  }
#if NASD_RPC_PACKAGE == NASD_RPC_PACKAGE_DCE
  else if (use_tcp) {
    binding_type = NASD_BIND_DCE_DIRECT_TCP;
    binding_args = NULL;
    binding_args_len = 0;
  }
#endif /* NASD_RPC_PACKAGE == NASD_RPC_PACKAGE_DCE */
  else {
    binding_type = NASD_BIND_DEFAULT;
    binding_args = NULL;
    binding_args_len = 0;
  }

  rc = nasd_bind_to_drive(thread->name, NASD_PDRIVE_PORT, 
                          binding_type, binding_args, binding_args_len, &(thread->drive));

  if (rc) {
    fprintf(stderr, "ERROR: cannot bind to server %s\n", thread->name);
    fflush(stderr);
    exit(1);
  }
}



void 
drspeed_thread(
               nasd_threadarg_t arg)
{
  drspeed_thread_args_t        *me=(drspeed_thread_args_t *)arg;
  nasd_identifier_t             nasdid = NASDID_NULL;
  char                         *buf,*compare_buf=NULL; /*we need it init buf*/
  nasd_rpc_status_t             status;
  nasd_timespec_t               diff;
  nasd_timer_t                  timer;
  int                           i;
  double                        d;
  nasd_status_t                 nasd_status;
  nasd_error_string_t           err_str;
  mpdrspeed_header_t            header;
  nasd_len_t                    out_len;
  nasd_p_smpl_op_dr_args_t      rd_args;
  nasd_p_fastread_dr_res_t      rd_res;
  nasd_security_param_t         sp;
  nasd_timespec_t               tm;


  NASD_DECLARE_MUTEX(my_boring_lock);

  me->offset = default_offset;
  if (do_ufs) {
    me->partnum /* really fd  */ = open(me->name, 
                                        do_write?O_WRONLY|O_CREAT:O_RDONLY,
                                        0666);
    if (me->partnum /* really fd */ < 0 ) {
      fprintf(stderr,"ERROR UFS: Could not open file %s",me->name);
      perror("");
      exit(1);
    }
    if (do_multi_offsets) {
      me->offset = ((niters+1) * blocksize * (me->number));
      printf ("%s:  Offset %g MBytes\n",me->name,(double)me->offset/1048576.0);/*DB*/
      if( lseek(me->partnum/* really fd */,me->offset,SEEK_SET) != me->offset) {
        fprintf(stderr,"ERROR UFS: Could not seek to location");
        perror("");
        exit(1);
      }
    }
  } else {
    bind_to_drive(me); /* Currently, bind_to_drive takes care of
                          exiting if there's an error during the
                          binding process */

    nasd_sec_password_to_keys(master_password, me->partnum, &keys);
  
    nasd_cl_p_null_dr(me->drive, &nasd_status, &status);
    if (status || nasd_status) {
      fprintf(stderr,
              "ERROR from drive noop, status=0x%x (%s) nasd_status=0x%x (%s)\n",
              status, nasd_cl_error_string(me->drive, status, err_str),
              nasd_status, nasd_error_string(nasd_status));
      exit(1);
    }
    
    nasdid = get_first_object(me->drive, me->partnum, me->name);  

    rd_args.in_partnum = me->partnum;
    rd_args.in_identifier = nasdid;
    rd_args.in_offset = 0;
    rd_args.in_len = sizeof(header);
    sp.type = cookie.capability.type;
    sp.partnum = me->partnum;
    sp.actual_protection = cookie.capability.min_protection;
    
    nasd_cl_p_read_simple_dr(me->drive, cookie.key, &sp, &cookie.capability,
                             &rd_args, &header, &rd_res, &status);
    out_len = rd_res.out_datalen;
    nasd_status = rd_res.nasd_status;

    if (status || nasd_status) {
      fprintf(stderr,
              "ERROR from read to first file for multi locations, status=0x%x (%s) nasd_status=0x%x (%s)\n",
              status, nasd_cl_error_string(me->drive, status, err_str),
              nasd_status, nasd_error_string(nasd_status));
      exit(1);
    }

    if (out_len != sizeof(header)) {
      fprintf (stderr,"Could not read all of header.  Got %d bytes \n",out_len);/*DB*/
      fflush(stderr);exit(1);
    }

    if (strncmp(header.magic,MPDRSPEED_MULTI_MAGIC,strlen(MPDRSPEED_MULTI_MAGIC)) != 0) {
      fprintf(stderr,"Magic number for mpdrspeed header is incorrect\n");
      exit(1);
    }
    if (multi_files) {
      if (me->number< header.nobjects) {
        nasdid = header.objects[me->number];
      } else {
        fprintf(stderr, "There are only %" NASD_64u_FMT
                " distinct objects on the drive\n",
                header.nobjects);
        exit(1);
      }
    } else {
      nasdid = header.objects[0];
    }
    fprintf (stderr,"Selected nasdid 0x%" NASD_ID_FMT " on drive %s\n",nasdid,me->name);/*DB*/fflush(stderr);
  } /* ! branch of do_ufs */


  NASD_Malloc(buf, blocksize, (char *));
  if (buf == NULL) {
    fprintf(stderr, "ERROR: unable to allocate buffer\n");
    fflush(stderr);
    exit(1);
  }
  
  if (do_compare) {
    NASD_Malloc(compare_buf, blocksize, (char *));
    if (compare_buf == NULL) {
      fprintf(stderr, "ERROR: unable to allocate buffer for comparison data\n");
      fflush(stderr);
      exit(1);
    }
  }
  
  for(i=0;i<blocksize;i++) {
    buf[i] = (char)bit_pattern;
  }


  /*This is where security should go*/
  /* Fill in capability and security param */
  nasd_drive_handle_get_time(me->drive, &tm);
  tm.ts_sec+=(60*60);
  nasd_sec_build_capability(me->partnum,
                            nasdid, 
                            (NASD_ACCESS_RIGHTS_READ |
                             NASD_ACCESS_RIGHTS_WRITE),
                            0,
                            tm.ts_sec,
                            protection,
                            NASD_RED_CAPABILITY,
                            me->offset,
                            blocksize,
                            7,
                            keys.red_key,
                            &cookie);



#if 0
  fprintf (stderr,"Performing initial I/O \n");/*DB*/  
#endif

  fflush(stderr);
  perform_io(me->drive, me->partnum, me->name, nasdid, me->offset,
    buf, compare_buf, blocksize, &nasd_status, &status);
  if (status || nasd_status) {
    fprintf(stderr, "ERROR from drive pre-op, status=0x%x (%s) nasd_status=0x%x (%s)\n",
            status, nasd_cl_error_string(me->drive, status, error_text),
            nasd_status, nasd_error_string(nasd_status));
    exit(1);
  }
  me->num_ios = 0;
  me->accum_time = (double)0.0;
  
  nasd_mutex_init(&my_boring_lock);
  NASD_LOCK_MUTEX(my_boring_lock);

  NASD_LOCK_MUTEX(printname);
  fprintf (stderr, "%s ",me->name); fflush(stderr);/*DB*/
  NASD_UNLOCK_MUTEX(printname);

  NASD_THREADGROUP_RUNNING(me->group);

  NASD_THREADGROUP_STARTED(&stats_ready); /* Join the threadgroup/barrier_sync */
  NASD_THREADGROUP_RUNNING(&stats_ready);

  NASD_WAIT_COND(threads_go,my_boring_lock);
  NASD_UNLOCK_MUTEX(my_boring_lock);

  for(i=0;i<niters;i++) {
    if (increment_offset) {
      me->offset+=blocksize;
    }

    if (lockstep_mode) {
      NASD_LOCK_MUTEX(lockstep_started_mutex);
      lockstep_started++;
      if (lockstep_started == drives) {
        lockstep_started = 0;
        NASD_UNLOCK_MUTEX(lockstep_started_mutex);
        NASD_TM_START(&lockstep_timer);
        NASD_BROADCAST_COND(lockstep_started_cond);
      } else {
        while (lockstep_started) 
          NASD_WAIT_COND(lockstep_started_cond,lockstep_started_mutex);
        NASD_UNLOCK_MUTEX(lockstep_started_mutex);
      }
    } else {
      NASD_TM_START(&timer);
    }
          
    
    NASD_TM_START(&timer);
    perform_io(me->drive, me->partnum, me->name, nasdid, me->offset,
      buf, compare_buf, blocksize, &nasd_status, &status);
    if (status || nasd_status) {
      fprintf(stderr, "ERROR from drive op, offset=%"
        NASD_64u_FMT " status=0x%x (%s) nasd_status=0x%x (%s)\n",
        me->offset, status, nasd_cl_error_string(me->drive, status, error_text),
        nasd_status, nasd_error_string(nasd_status));
      exit(1);
    }

    if (lockstep_mode) {
      NASD_LOCK_MUTEX(lockstep_stopped_mutex);
      lockstep_stopped++;
      if (lockstep_stopped == drives) {
        NASD_TM_STOP(&lockstep_timer);
        lockstep_stopped = 0;
        NASD_UNLOCK_MUTEX(lockstep_stopped_mutex);
        NASD_BROADCAST_COND(lockstep_stopped_cond);
      } else {
        while (lockstep_stopped) 
          NASD_WAIT_COND(lockstep_stopped_cond,lockstep_stopped_mutex);
        NASD_UNLOCK_MUTEX(lockstep_stopped_mutex);
      }
    } else {
      NASD_TM_STOP(&timer);
    }

    if (lockstep_mode) {
      NASD_TM_ELAPSED_TS(&lockstep_timer, &diff);
    } else {
      NASD_TM_ELAPSED_TS(&timer, &diff);
    }
    me->num_ios++;
    d = (double)diff.ts_nsec;
    d /= (double)1000000000.0;
    d += (double)diff.ts_sec;
    me->accum_time += d;
  }
  NASD_THREADGROUP_DONE(me->group);
/*CO  printf("%lf seconds to do %d ops\n", me->accum_time, me->num_ios);*/
  d = (double)me->num_ios;
  me->avg = me->accum_time / d;
/*CO  printf("%lf seconds average\n", me->avg);*/
/*CO  printf("%lf ops / sec\n", (d/me->accum_time));*/
/*CO  printf("blocksize = %d\n", blocksize);*/
  me->avg_rate = (double)blocksize;
  me->avg_rate *= (double)me->num_ios;
  me->avg_rate /= (double)1024.0;
  me->avg_rate /= me->accum_time;
/*CO  printf("avg rate = %.2lf kbytes/sec (%.2lf 8k-chunks/sec)\n",*/
/*CO    me->avg_rate, me->avg_rate/((double)8.0));*/
  NASD_THREADGROUP_DONE(&stats_ready);
}


int sync_wait_ready ()
{
  NASD_LOCK_MUTEX(extern_ready_mutex);
  while (ready == 0)
    NASD_WAIT_COND(extern_ready_cond,extern_ready_mutex);
  NASD_UNLOCK_MUTEX(extern_ready_mutex);
  return 1;
}


void report_stats (FILE *dest)
{
  int i;
  double total_ops_s=0;
  double total_kbytes=0;
  double max_time_s=0;
  double d, elapsed;
#if NASD_IDLE_SUPPORT > 0
  double idle,id1,id2;
#endif /* NASD_IDLE_SUPPORT > 0 */

  char buf[MAXHOSTNAMELEN];
  
  gethostname(buf,MAXHOSTNAMELEN);
  fprintf(dest,"name: %s\n",buf);
  fprintf(dest,"type: client\n");
  fprintf(dest,"drives: %d\n",drives);
  fprintf(dest,"size: %d\n",blocksize);
  fprintf(dest,"kpdev: %s\n",use_kp?"yes":"no");
  fprintf(dest,"multi: %s\n",multi_files?"yes":"no");
  fprintf(dest,"lockstep: %s\n",lockstep_mode?"yes":"no");
  fprintf(dest,"iterations: %d\n",niters);
    
  /*         1   2   3   4   5   6   7  8          1        2        3     4        5       6          7          8 */
  fprintf (dest,"%16s %9s %6s %7s %9s %8s %5s %9s\n","server","total s","s/op","op/s","blocksz","KBytes/s","Mbit/s","8Kblock/s"); 
  for (i=0 ; i<drives ; i++ ) {
    d = (double)drspeed_threads[i].num_ios;
    /*         1     2     3      4      5      6     7     8 */
    fprintf (dest,"%16s %9.04f %6.3f %7.04f %9.04d %8.02f %6.02f %9.02f\n", 
            drspeed_threads[i].name,
            drspeed_threads[i].accum_time,
            drspeed_threads[i].avg,
            d / drspeed_threads[i].accum_time,
            blocksize,
            drspeed_threads[i].avg_rate,
            (drspeed_threads[i].avg_rate * (double) 8192.0) / (double)1000000.0,
            drspeed_threads[i].avg_rate/(double)8.0);
    
    if (drspeed_threads[i].accum_time > max_time_s) {
      max_time_s = drspeed_threads[i].accum_time;
    }
    
    total_ops_s += d / drspeed_threads[i].accum_time;
    total_kbytes += (drspeed_threads[i].num_ios * blocksize) / (double) 1024.0;
  }
  
  total_kbytes_s = total_kbytes / max_time_s;
  fprintf (dest,"%16s %9d %6s %7.04f %9.04d %8.02f %6.02f %9.02f\n", 
           "Total",
           drives,
           "drives",
           total_ops_s,
           blocksize,
           total_kbytes_s,
           (total_kbytes_s * (double)8192.0) / (double)1000000.0,
           total_kbytes_s/(double)8.0);
  
  fprintf(dest,"threadtime: %g\n",max_time_s);


  NASD_TM_ELAPSED_TS(&total_timer, &diff);
  elapsed = (double)diff.ts_nsec;
  elapsed /= (double)1000000000.0;
  elapsed += (double)diff.ts_sec;
  
  /*calculate idle time*/

  
#if NASD_IDLE_SUPPORT > 0
    id1 = (double)idle_ts1.ts_nsec;
    id1 /= (double)1000000000.0;
    id1 += (double)idle_ts1.ts_sec;
    id2 = (double)idle_ts2.ts_nsec;
    id2 /= (double)1000000000.0;
    id2 += (double)idle_ts2.ts_sec;
    idle = id2 - id1;

    fprintf (dest,"elapsed: %g\nidle: %g Idle fraction %0.2g\n",elapsed,idle,idle/elapsed);/*DB*/
#else /* NASD_IDLE_SUPPORT > 0 */
    fprintf (dest,"elapsed: %g\n",elapsed);/*DB*/
#endif /* NASD_IDLE_SUPPORT > 0 */

    fflush(dest);
}

void danger(char *a) {
  FILE          *dangerlog;
  time_t	time_t;
  char   timebuf[1024];

  NASD_LOCK_MUTEX(danger_mutex);
  dangerlog = fopen ("/tmp/mpdrspeed.danger","a");
  time(&time_t);
  _nasd_timestr_r(time_t, timebuf);
  fprintf(dangerlog,"%s, errno = %d, time = %s\n",a,errno,timebuf);
  NASD_UNLOCK_MUTEX(danger_mutex);
}
  

#define CHECKRC(_rc_,a) {if (_rc_ < 0) {perror(a);danger(a);exit(1);}}
void sync_thread(
  nasd_threadarg_t  arg)
{
  int			s,syncfd;
  struct sockaddr_in	addr;

  int			addrlen;
  int			opt;
  int			rc;
  int     step;
  

  printf ("Sync thread started\n");  fflush (stdout);/*DB*/

  s = socket (AF_INET,SOCK_STREAM,0);
  CHECKRC(s,"Could not create socket");
  bzero((char *)&addr,sizeof (struct sockaddr_in));
  opt = 1;
  rc = setsockopt(s, SOL_SOCKET, SO_REUSEADDR,(char *)&opt,sizeof(int));
  CHECKRC(rc,"Could not setsockopt to SO_REUSEADDR");
  opt = 1;
  rc = setsockopt(s, IPPROTO_TCP, TCP_NODELAY,(char *)&opt,sizeof(int));
  CHECKRC(rc,"Turning off delay");
  addr.sin_family = AF_INET;
  addr.sin_addr.s_addr = INADDR_ANY;
  addr.sin_port = htons(3840);
  rc = bind(s,(struct sockaddr *)&addr,sizeof(addr));
  CHECKRC(rc,"syncthread listener: Could not bind socket");
  rc = listen (s,1);
  CHECKRC(rc,"Listen");
  syncfd = accept (s,(struct sockaddr *) &addr,&addrlen);
  CHECKRC(syncfd,"Accept");
  sync_wait_ready();
  step = 1;  
  /* Tell the coordinator that we are ready */
  write (syncfd,&step,4);
  /* Wait for the coordinator to tell us that we should go */
  read(syncfd,&step,4);

  extern_sync_go = 1;
  NASD_BROADCAST_COND(extern_sync_cond); /* Start readers */


  /* Wait 'til we're told the readers are done */
  NASD_LOCK_MUTEX(extern_sync_mutex);
  while (extern_sync_go==1) 
    NASD_WAIT_COND(extern_sync_cond,extern_sync_mutex);
  NASD_UNLOCK_MUTEX(extern_sync_mutex);

  /* Tell the coordinator that we are done: Do this before we report
     stats so that the coordinator knows as soon as the first client
     is done*/
  write(syncfd,&step,4);

 /* Now that we are done with the time-critical part, wait until the
    stats are done and report those too */
  NASD_THREADGROUP_WAIT_STOP(&stats_ready);  
  syncfi = fdopen (syncfd,"w+");
  report_stats(syncfi);

  extern_sync_go = 100;
  NASD_BROADCAST_COND(extern_sync_cond);

}

int
main(int argc, char **argv)
{
  char c;
  nasd_thread_t         sync_thread_handle;
  nasd_delaycounter_t   delayer;
  int sec_level=0;
  int i, rc;
#if NASD_IDLE_SUPPORT > 0
  nasd_status_t nasd_status;
#endif /* NASD_IDLE_SUPPORT > 0 */

  nasd_cl_p_init();

  progname = argv[0];
  bzero((char *)&cookie, sizeof(cookie));

  nasd_mutex_init(&printname);
  nasd_mutex_init(&lockstep_started_mutex);

  nasd_mutex_init(&lockstep_stopped_mutex);
  nasd_mutex_init(&extern_sync_mutex);
  nasd_mutex_init(&extern_ready_mutex);
  nasd_mutex_init(&danger_mutex);
  nasd_cond_init(&extern_sync_cond);
  nasd_cond_init(&extern_ready_cond);
  nasd_cond_init(&lockstep_started_cond);
  nasd_cond_init(&lockstep_stopped_cond);

  while (nasd_getopt(argc, argv, "b:cflmMikn:o:Ss:Tt:uv:w", &c)) {
    switch(c) {
      case 'b':
        if (sscanf(nasd_optarg, "%d", &blocksize) != 1) {
          usage();
        }
        break;
      case 'c':
        do_compare = 1;
        break;
      case 'f':
        force = 1;
        break;
      case 'i':
        increment_offset=1;
        break;
      case 'k':
        use_kp = 1;
        break;
      case 'l':
        lockstep_mode = 1;
        break;
      case 'M':
        do_multi_offsets = 1;
        break;
      case 'm':
        multi_files = 1;
        break;
      case 'n':
        if (sscanf(nasd_optarg, "%d", &niters) != 1) {
          usage();
        }
        break;
      case 'o':
        if (sscanf(nasd_optarg, "%"NASD_64s_FMT, &default_offset) != 1) {
          usage();
        }
        break;
      case 'S':
        extern_sync = 1;
        break;
    case 's':
      if (sscanf(nasd_optarg, "%d", &sec_level) != 1) {
        usage();
      }
      break;
#if NASD_RPC_PACKAGE == NASD_RPC_PACKAGE_DCE
    case 'T':
        use_tcp = 1;
        break;
#endif /* NASD_RPC_PACKAGE == NASD_RPC_PACKAGE_DCE */
      case 't':
        if (sscanf(nasd_optarg, "%"NASD_64u_FMT, &bms_targ) != 1) {
          usage();
        }
        if (bms_targ == 0)
          usage();
        break;
      case 'u':
        do_ufs = 1;
        break;
      case 'v':
        bit_pattern = (unsigned char)nasd_optarg[0];
        break;
      case 'w':
        do_write = 1;
        break;
      default:
        usage();
    }
  }

  if (nasd_optind >= argc)
    usage();
  master_password=argv[nasd_optind];
  nasd_optind++;


  drives = argc-nasd_optind;
  if (drives <= 0 ) usage();
  if (drives &0x01) usage();        /* must have [drive partition] pairs */

  if (blocksize < 1) {
    fprintf(stderr, "Bad blocksize %d\n", blocksize);
    usage();
  }
  if (niters < 1) {
    fprintf(stderr, "Bad niters %d\n", niters);
    usage();
  }
  if (default_offset) {
    printf("Using offset %" NASD_64s_FMT "\n", default_offset);
  }
  if (bit_pattern) {
    printf("Using bit pattern 0x%x\n", bit_pattern&0xff);
  }

  rc = nasd_sec_seclevel_to_protection(sec_level, &protection);
  if(rc) {
    fprintf(stderr, "Unknown security level %d\n", sec_level);
    usage();
  }



  rc = nasd_cl_p_init();
  if (rc) {
    printf("ERROR (%s:%d): cannot init client library, rc=0x%x (%s)\n",
      __FILE__, __LINE__, rc, nasd_error_string(rc));
    return(rc);
  }


  drives /= 2;

  NASD_Malloc(drspeed_threads, drives * sizeof (drspeed_thread_args_t),
    (drspeed_thread_args_t *));
  
  if (extern_sync) {
    rc = nasd_thread_create(&sync_thread_handle, sync_thread, NULL);
    if (rc) {
      fprintf(stderr, "ERROR: cannot create sync_thread rc=0x%x (%s)\n",
        rc, nasd_error_string(rc));
      exit(1);
    }
  }

  nasd_init_threadgroup(&readers);
  nasd_init_threadgroup(&stats_ready);
  nasd_cond_init(&threads_go);
  
  for (i = 0 ; i < drives ; i++) {
    drspeed_threads[i].name = argv[nasd_optind++];
    if (sscanf(argv[nasd_optind++],"%d",&(drspeed_threads[i].partnum)) != 1) {
      fprintf(stderr, "Could not parse %s as a partition number\n",
              argv[nasd_optind-1]);
      usage();
    }
    drspeed_threads[i].group=&readers;      
    drspeed_threads[i].number=i;
    rc = nasd_thread_create(&drspeed_threads[i].handle,
                       drspeed_thread,
                       (nasd_threadarg_t)&(drspeed_threads[i]));
    if (rc) {
      fprintf(stderr, "ERROR: could not create thread i=%d rc=0x%x (%s)\n",
        i, rc, nasd_error_string(rc));
      exit(1);
    }
    NASD_THREADGROUP_STARTED(&readers);
  }
  
  
  /* Wait for threads to start up */
  NASD_THREADGROUP_WAIT_START(&readers);
  fprintf(stderr,"!\n");fflush(stderr);

  /* Why is this here?  */
  NASD_BEGIN_DELAYCNT(&delayer);
  NASD_DELAY_FROM(&delayer, 100);

  ready = 1;

  /* Wait for external sync signal if we're into that */
  NASD_BROADCAST_COND(extern_ready_cond);

  if (extern_sync) {
    NASD_LOCK_MUTEX(extern_sync_mutex);
    while (extern_sync_go==0) {
      NASD_WAIT_COND(extern_sync_cond,extern_sync_mutex);
    }
    NASD_UNLOCK_MUTEX(extern_sync_mutex);
  }

  /* Go */
 
#if NASD_IDLE_SUPPORT > 0
  nasd_status = nasd_get_total_idle_time(&idle_ts1);
  if (nasd_status) {
    fprintf(stderr, "ERROR: got 0x%x (%s) from nasd_get_total_idle_time\n",
            nasd_status, nasd_error_string(nasd_status));
    exit(1);
  }
#endif /* NASD_IDLE_SUPPORT > 0 */
  NASD_TM_START(&total_timer); 
  NASD_BROADCAST_COND(threads_go);

  NASD_THREADGROUP_WAIT_STOP(&readers);
  NASD_TM_STOP(&total_timer);
#if NASD_IDLE_SUPPORT > 0
    nasd_status = nasd_get_total_idle_time(&idle_ts2);
    if (nasd_status) {
      fprintf(stderr, "ERROR: got 0x%x (%s) from nasd_get_total_idle_time\n",
        nasd_status, nasd_error_string(nasd_status));
      exit(1);
    }
#endif /* NASD_IDLE_SUPPORT > 0 */

  /* Tell sync thread that the readers are done */
  extern_sync_go=10;
  NASD_BROADCAST_COND(extern_sync_cond); /* This is picked up in sync_thread */

  NASD_THREADGROUP_WAIT_STOP(&stats_ready); /* Wait til stats are done */


  if (extern_sync) {
    /* Wait for sync thread to finish before we think about exiting */
    NASD_LOCK_MUTEX(extern_sync_mutex);
    while (extern_sync_go!=100) 
      NASD_WAIT_COND(extern_sync_cond,extern_sync_mutex);
    NASD_UNLOCK_MUTEX(extern_sync_mutex);
  }

  report_stats(stdout);
  
  /* must not close syncfi until we are really done*/
  if (syncfi)
    fclose(syncfi);
  exit(0);
}

/* Local Variables:  */
/* indent-tabs-mode: nil */
/* tab-width: 2 */
/* End: */
