/*
 * threadtest.c
 *
 * Test multiple drive clients sharing a single RPC connection
 *
 * Author: Nat Lanza
 */
/*
 * Copyright (c) of Carnegie Mellon University, 2000.
 *
 * Permission to reproduce, use, and prepare derivative works of
 * this software for internal use is granted provided the copyright
 * and "No Warranty" statements are included with all reproductions
 * and derivative works. This software may also be redistributed
 * without charge provided that the copyright and "No Warranty"
 * statements are included in all redistributions.
 *
 * NO WARRANTY. THIS SOFTWARE IS FURNISHED ON AN "AS IS" BASIS.
 * CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, EITHER
 * EXPRESSED OR IMPLIED AS TO THE MATTER INCLUDING, BUT NOT LIMITED
 * TO: WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY
 * OF RESULTS OR RESULTS OBTAINED FROM USE OF THIS SOFTWARE. CARNEGIE
 * MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT
 * TO FREEDOM FROM PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT.
 */

#include <nasd/nasd_options.h>
#include <nasd/nasd_site.h>
#include <nasd/nasd_pdrive.h>
#include <nasd/nasd_pdrive_client.h>
#include <nasd/nasd_pdrive_client_kpdev.h>
#include <nasd/nasd_pdev.h>
#include <nasd/nasd_timer.h>
#include <nasd/nasd_mem.h>
#include <nasd/nasd_getopt.h>
#include <nasd/nasd_threadstuff.h>

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define NUM_THREADS 6

#define OBJECT_SIZE 67108864L     /* 64 MB, needs ~8192 disk blocks */
#define READ_BUFFER_SIZE 4194304L /* 4MB read sizes, so we get long pipes. */
#define NUM_SIMPLE 75             /* number of operations for the simple
				     threads to perform */

typedef struct nasd_threadtest_threadarg_s {
  nasd_threadgroup_t *group;
  int type;
  int threadnum;
} nasd_threadtest_threadarg_t;

NASD_DECLARE_MUTEX(thread_started_mutex);
NASD_DECLARE_COND(thread_started_cv);
int thread_started = 0;

NASD_DECLARE_MUTEX(threads_go_mutex);
NASD_DECLARE_COND(threads_go_cv);
int threads_go = 0;

char *progname;

nasd_error_string_t err_str;

int partnum = 1;
char *server_name;
int nondefault_binding = 0;
nasd_drive_handle_t h;
int binding_type;
int binding_args_len;
void *binding_args;
nasd_drive_param_kpdev_t kpdev_args;

nasd_identifier_t nasdid = NASD_ID_NULL;

nasd_uint64 blocksize = 0;

int verbose = 0;
int skip_write = 0;

nasd_cookie_t cookie;
nasd_sec_keyring_t keys;
nasd_security_param_t sec_param;
nasd_uint16 protection;


void usage(void) {
  int i;
  fprintf(stderr, "USAGE: %s [options] servername password\n", progname);
  fprintf(stderr, "  -a identifier (default = root of partition)\n");
  fprintf(stderr, "  -k use kpdev\n");
  fprintf(stderr, "  -l use colocated drive\n");
  fprintf(stderr, "  -M use message queues\n");
  fprintf(stderr, "  -p partnum (default=%d)\n", partnum);
  fprintf(stderr, "  -s security level\n");
  for(i = 0; i <= NASD_MAX_SECURITY_LEVEL; i++) {
    fprintf(stderr, "     %d %s\n", i, nasd_sec_level_string(i));
  }
  fprintf(stderr, "  -v verbose output\n");
  fprintf(stderr, "  -w skip initial write\n");
  fflush(stderr);
  exit(1);
}


void thread_worker_func_simple(nasd_threadtest_threadarg_t *arg) {
  int i;
  nasd_status_t nasd_status;
  nasd_rpc_status_t rc;

  sleep(1); /* let the reader threads catch up */

  for (i = 0; i < NUM_SIMPLE; i++) {

    if (verbose) {
      printf("Thread %d performing null %d...", arg->threadnum, i);
      fflush(stdout);
    }

    nasd_cl_p_null_dr(h, &nasd_status, &rc);
    if (rc || nasd_status) {
      fprintf(stderr,
        "WARNING: error from drive noop by thread %d, rc=0x%x (%s) nasd_status=0x%x (%s)\n",
	      arg->threadnum, rc, nasd_cl_error_string(h, rc, err_str),
	      nasd_status, nasd_error_string(nasd_status));
      exit(1);
    }

    if (verbose) {
      printf(" done(%d,%d)\n", arg->threadnum, i);
      fflush(stdout);
    }
  }
}


void thread_worker_func_read(nasd_threadtest_threadarg_t *arg) {
  nasd_offset_t offset;
  nasd_p_smpl_op_dr_args_t read_args;
  nasd_p_fastread_dr_res_t read_res;
  nasd_status_t nasd_status;
  nasd_rpc_status_t op_status;
  int i;
  char *buf = NULL;
  
  NASD_Malloc(buf, READ_BUFFER_SIZE, (char *));
  NASD_ASSERT(buf != NULL);

  read_args.in_partnum = partnum;
  read_args.in_identifier = nasdid;

  offset = 0;
  i = 0;

  while (offset < OBJECT_SIZE) {
    read_args.in_offset = offset;
    i++;

    if ((OBJECT_SIZE - offset) < READ_BUFFER_SIZE) {
      read_args.in_len = (OBJECT_SIZE - offset);
    } else {
      read_args.in_len = READ_BUFFER_SIZE;
    }

    if (verbose) {
      printf("Thread %d performing read %d...", arg->threadnum, i);
      fflush(stdout);
    }

    nasd_cl_p_read_simple_dr(h, cookie.key, &sec_param,
			     &cookie.capability,
			     &read_args, buf, &read_res,
			     &op_status);
    nasd_status = read_res.nasd_status;

    if (nasd_status || op_status) {
      fprintf(stderr,
	      "ERROR: read (%lu@%lu) failed, rc=0x%x (%s), op_status=0x%x (%s)\n",
	      (unsigned long) read_args.in_len,
	      (unsigned long) read_args.in_offset,
	      nasd_status, nasd_error_string(nasd_status), op_status,
	      nasd_cl_error_string(h, op_status, err_str));
      exit(1);
    }
    
    if (read_res.out_datalen < READ_BUFFER_SIZE) {
      fprintf(stderr, "ERROR: short read %lu@%lu by thread %d!\n",
	      (unsigned long) read_args.in_len,
	      (unsigned long) read_args.in_offset, arg->threadnum);
      exit(1);
    }

    offset += read_res.out_datalen;

    if (verbose) {
      printf(" done(%d,%d)\n", arg->threadnum, i);
      fflush(stdout);
    }
  }
}


void threadfunc(nasd_threadarg_t arg) {
  nasd_threadtest_threadarg_t *ta;

  ta = (nasd_threadtest_threadarg_t *) arg;
  
  NASD_THREADGROUP_RUNNING(ta->group);

  nasd_printf("Thread %d started [id %" NASD_THREAD_ID_FMT "].\n",
	      ta->threadnum, nasd_thread_self());

  NASD_LOCK_MUTEX(thread_started_mutex);
  thread_started = 1;
  NASD_SIGNAL_COND(thread_started_cv);
  NASD_UNLOCK_MUTEX(thread_started_mutex);

  NASD_LOCK_MUTEX(threads_go_mutex);
  while (!threads_go) { NASD_WAIT_COND(threads_go_cv, threads_go_mutex); }
  NASD_UNLOCK_MUTEX(threads_go_mutex);

  switch (ta->type) {
    case 1: thread_worker_func_simple(arg); break;
    case 2: thread_worker_func_read(arg);   break;
    default:
      fprintf(stderr, "Unknown thread type %d in thread %d\n",
	      ta->type, ta->threadnum);
      exit(1);
  }

  if (verbose) {
    printf("Thread %d exiting\n", ta->threadnum);
    fflush(stdout);
  }

  NASD_THREADGROUP_DONE(ta->group);
  NASD_Free(ta, sizeof(nasd_threadtest_threadarg_t));
  NASD_THREAD_KILL_SELF();
}  


void thread_test(void) {
  nasd_status_t rc;
  nasd_thread_t threads[NUM_THREADS];
  nasd_threadgroup_t group;
  nasd_threadtest_threadarg_t *arg;
  int i;

  rc = nasd_threads_init();
  if (rc) {
    fprintf(stderr, "ERROR: failed to init thread subsystem, rc=0x%x (%s)\n",
	    rc, nasd_error_string(rc));
    exit(rc);
  }
  
  rc = nasd_mutex_init(&thread_started_mutex);
  if (rc) {
    fprintf(stderr, "ERROR: failed to init thread mutex, rc=0x%x (%s)\n",
	    rc, nasd_error_string(rc));
    exit(rc);
  }
    
  rc = nasd_cond_init(&thread_started_cv);
  if (rc) {
    fprintf(stderr, "ERROR: failed to init thread cond var, rc=0x%x (%s)\n",
	    rc, nasd_error_string(rc));
    exit(rc);
  }

  rc = nasd_mutex_init(&threads_go_mutex);
  if (rc) {
    fprintf(stderr, "ERROR: failed to init thread mutex, rc=0x%x (%s)\n",
	    rc, nasd_error_string(rc));
    exit(rc);
  }
    
  rc = nasd_cond_init(&threads_go_cv);
  if (rc) {
    fprintf(stderr, "ERROR: failed to init thread cond var, rc=0x%x (%s)\n",
	    rc, nasd_error_string(rc));
    exit(rc);
  }

  rc = nasd_init_threadgroup(&group);
  if (rc) {
    fprintf(stderr, "ERROR: can't init threadgroup, rc=0x%x (%s)\n",
	    rc, nasd_error_string(rc));
    exit(rc);
  }

  printf("Starting threads.\n"); fflush(stdout);

  nasd_printf("Master thread tid %" NASD_THREAD_ID_FMT ", pid %d.\n",
	      nasd_thread_self(), getpid());
  
  for (i = 1; i <= NUM_THREADS; i++) {
    arg = NULL;
    NASD_Malloc(arg, sizeof(nasd_threadtest_threadarg_t),
		(nasd_threadtest_threadarg_t *));
    NASD_ASSERT(arg != NULL);
    arg->group     = &group;
    arg->threadnum = i;
    arg->type      = (i%2) + 1;

    NASD_LOCK_MUTEX(thread_started_mutex);
    thread_started = 0;
    NASD_UNLOCK_MUTEX(thread_started_mutex);

    rc = nasd_thread_create(&threads[i], threadfunc, (nasd_threadarg_t) arg);
    if (rc) { 
      fprintf(stderr, "ERROR: failed to create thread #%d, rc=0x%x (%s)\n",
	      i, rc, nasd_error_string(rc));
      exit(rc);
    }
 
    NASD_THREADGROUP_STARTED(&group);

    NASD_LOCK_MUTEX(thread_started_mutex);
    while (thread_started != 1) {
      NASD_WAIT_COND(thread_started_cv, thread_started_mutex);
    }
    NASD_UNLOCK_MUTEX(thread_started_mutex);
  }

  NASD_THREADGROUP_WAIT_START(&group);
  printf("All threads started.\n"); fflush(stdout);

  NASD_LOCK_MUTEX(threads_go_mutex);
  threads_go = 1;
  NASD_UNLOCK_MUTEX(threads_go_mutex);

  NASD_BROADCAST_COND(threads_go_cv);
  printf("All threads woken up.\n"); fflush(stdout);
  NASD_THREADGROUP_WAIT_STOP(&group);
  printf("All threads finished.\n"); fflush(stdout);
  
  rc = nasd_cond_destroy(&thread_started_cv);
  if (rc) {
    fprintf(stderr, "ERROR: failed to destroy cond var, rc=0x%x (%s)\n",
	    rc, nasd_error_string(rc));
    exit(rc);
  }

  rc = nasd_mutex_destroy(&thread_started_mutex);
  if (rc) {
    fprintf(stderr, "ERROR: failed to destroy mutex, rc=0x%x (%s)\n",
	    rc, nasd_error_string(rc));
    exit(rc);
  }

  rc = nasd_cond_destroy(&threads_go_cv);
  if (rc) {
    fprintf(stderr, "ERROR: failed to destroy cond var, rc=0x%x (%s)\n",
	    rc, nasd_error_string(rc));
    exit(rc);
  }

  rc = nasd_mutex_destroy(&threads_go_mutex);
  if (rc) {
    fprintf(stderr, "ERROR: failed to destroy mutex, rc=0x%x (%s)\n",
	    rc, nasd_error_string(rc));
    exit(rc);
  }
    
  nasd_threads_shutdown();
}


void setup(void) {
  nasd_p_smpl_op_dr_args_t    write_args;
  nasd_p_fastwrite_dr_res_t   write_res;
  nasd_p_flush_obj_dr_args_t  flush_args;
  nasd_p_flush_obj_dr_res_t   flush_res;
  nasd_p_eject_obj_dr_args_t  eject_args;
  nasd_p_eject_obj_dr_res_t   eject_res;
  nasd_status_t               rc;
  nasd_rpc_status_t           op_status;
  nasd_ctrl_part_info_t       ptinfo;
  char                       *buf = NULL;
  nasd_offset_t               offset;
  nasd_uint64                 numblocks;
  int                         i, numiter, chunksize;

  rc = nasd_cl_p_ctrl_get_part_info(h, keys.black_key, &sec_param, NULL,
				    partnum, &ptinfo);
  if (rc) {
    if ((rc == NASD_CTRL_ID_CHECK_FAILED) && (ptinfo.ctrl_id == 0)) {
      /* got back a zero page - no such partition */
      fprintf(stderr, "ERROR: partition %d does not exist\n", partnum);
    } else {
      fprintf(stderr, "ERROR: got 0x%x (%s) getting partition info\n",
	      rc, nasd_error_string(rc));
    }
    exit(rc);
  }

  if (ptinfo.num_obj == 0) {
    fprintf(stderr, "ERROR: partition has no objects!\n");
    exit(1);
  }
  
  if (nasdid == NASD_ID_NULL) { nasdid = ptinfo.first_obj; }

  blocksize = ptinfo.blocksize;

  printf("Using object 0x%" NASD_ID_FMT " in partition %d.\n",
	 nasdid, partnum);

  numblocks = OBJECT_SIZE / blocksize;

  if (ptinfo.part_size < numblocks) {
    fprintf(stderr, "ERROR: partition %d is too small! (%" NASD_64u_FMT
	    " > %" NASD_64u_FMT ").\n", partnum, numblocks, ptinfo.part_size);
    exit(1);
  }

  if (!skip_write) {
    chunksize = 1048576;
    
    NASD_Malloc(buf, chunksize, (char *));
    NASD_ASSERT(buf != NULL);
    
    write_args.in_partnum    = partnum;
    write_args.in_identifier = nasdid;
    offset = 0;
    
    if (verbose) {
      printf("Writing %ld bytes to object 0x%" NASD_ID_FMT "...",
	     OBJECT_SIZE, nasdid);
      fflush(stdout);
    } else {
      printf("Doing initial write...");
      fflush(stdout);
    }
    
    while (offset < OBJECT_SIZE) {
      write_args.in_offset = offset;
      
      if ((OBJECT_SIZE - offset) < chunksize) {
	write_args.in_len = (OBJECT_SIZE - offset);
      } else {
	write_args.in_len = chunksize;
      }
      
      nasd_cl_p_write_simple_dr(h, cookie.key, &sec_param,
				&cookie.capability,
				&write_args, buf, &write_res,
				&op_status);
      rc = write_res.nasd_status;
      
      if (rc || op_status) {
	fprintf(stderr,
		"ERROR: write (%lu@%lu) failed, rc=0x%x (%s), op_status=0x%x (%s)\n",
		(unsigned long) write_args.in_len,
		(unsigned long) write_args.in_offset,
		rc, nasd_error_string(rc), op_status,
		nasd_cl_error_string(h, op_status, err_str));
	exit(1);
      }
      
      offset += write_res.out_datalen;
      
      if (verbose) { printf("."); fflush(stdout); }
    }
    
    printf(" done\n");
    fflush(stdout);

    NASD_Free(buf, chunksize);
  } else {
    printf("Skipping initial write.\n");
  }

  if (verbose) { printf("Flushing object to disk.\n"); }
  flush_args.in_identifier = nasdid;
  flush_args.in_partnum    = partnum;

  nasd_cl_p_flush_obj_dr(h, cookie.key, &sec_param, &cookie.capability,
			 &flush_args, &flush_res, &op_status);
  rc = flush_res.nasd_status;
  if (rc || op_status) {
    fprintf(stderr,
	    "ERROR: flush failed, rc=0x%x (%s), op_status=0x%x (%s)\n",
	    rc, nasd_error_string(rc), op_status,
	    nasd_cl_error_string(h, op_status, err_str));
    exit(1);
  }
  
  if (verbose) { printf("Ejecting object from cache.\n"); }
  eject_args.in_identifier = nasdid;
  eject_args.in_partnum    = partnum;

  nasd_cl_p_eject_obj_dr(h, cookie.key, &sec_param, &cookie.capability,
			 &eject_args, &eject_res, &op_status);
  rc = eject_res.nasd_status;
  if (rc || op_status) {
    fprintf(stderr,
	    "ERROR: eject failed, rc=0x%x (%s), op_status=0x%x (%s)\n",
	    rc, nasd_error_string(rc), op_status,
	    nasd_cl_error_string(h, op_status, err_str));
    exit(1);
  }

}


int main(int argc, char **argv) {
  int i, rc;
  int sec_level = 0;
  char c;
  char *master_password;
  
  /* get the filename component of argv[0] */
  progname = strrchr(argv[0], '/');
  if (!progname) { progname = argv[0]; } else { progname++; }

  bzero((char *)&cookie, sizeof(nasd_cookie_t));
  bzero(&sec_param, sizeof(nasd_security_param_t));

  binding_type = NASD_BIND_DEFAULT;
  binding_args = NULL;
  binding_args_len = 0;
  
  while (nasd_getopt(argc, argv, "a:klMp:s:vw", &c)) {
    switch (c){ 
      case 'a': {
        rc = nasd_str_to_nasd_id(nasd_optarg, &nasdid);
        if (rc) {
          fprintf(stderr, "ERROR: \"%s\" is not a valid NASD identifier\n",
		  nasd_optarg);
          usage();
        }
        break;
      }
      case 'k': {
        if (nondefault_binding) { usage(); }
        nondefault_binding = 1;
        binding_type = NASD_BIND_KPDEV_DEFAULT;
        binding_args = &kpdev_args;
        binding_args_len = sizeof(kpdev_args);
        strcpy(kpdev_args.devname, "/dev/nasdkp0");
        break;
      }
      case 'l': {
        if (nondefault_binding) { usage(); }
        nondefault_binding = 1;
        binding_type = NASD_BIND_COLOCATE;
        binding_args = &kpdev_args;
        binding_args_len = sizeof(kpdev_args);
        strcpy(kpdev_args.devname, "/dev/nasdkp0");
        break;
      }
      case 'M': {
        if (nondefault_binding) { usage(); }
        nondefault_binding = 1;
        binding_type = NASD_BIND_MSGQ;
        break;
      }
      case 'p': {
        if (sscanf(nasd_optarg, "%d", &partnum) != 1) { usage(); }
        break;
      }
      case 's': {
        if (sscanf(nasd_optarg, "%d", &sec_level) != 1) { usage(); }
        break;
      }
      case 'v': {
	verbose++;
	break;
      }
      case 'w': {
	skip_write++;
	break;
      }
      default:
	usage();
    }
  }

  if (nasd_optind >= argc) { usage(); }
  server_name = argv[nasd_optind];
  nasd_optind++;

  if (nasd_optind >= argc) { usage(); }
  master_password=argv[nasd_optind];
  nasd_optind++;

  nasd_sec_password_to_keys(master_password, partnum, &keys);

  if (nasd_optind < argc) { usage(); }

  rc = nasd_sec_seclevel_to_protection(sec_level, &protection);
  if (rc) {
    fprintf(stderr, "Unknown security level %d\n", sec_level);
    usage();
  }

  rc = nasd_cl_p_init();
  if (rc) {
    printf("ERROR: can't init client lib, rc=0x%x (%s)\n",
	   rc, nasd_error_string(rc));
    exit (rc);
  }

  printf("Binding to server %s... ", server_name);
  fflush(stdout);

  rc = nasd_bind_to_drive(server_name, NASD_PDRIVE_PORT,
			  binding_type, binding_args, binding_args_len, &h);
  if (rc) {
    fprintf(stderr, "ERROR: cannot bind to server %s, rc=0x%x (%s)\n",
	    server_name, rc, nasd_error_string(rc));
    fflush(stderr);
    exit(1);
  }

  printf("done\n");
  fflush(stdout);

  /* lalalala */
  setup();
  thread_test();

  rc = nasd_unbind_drive(&h);
  if (rc) {
    fprintf(stderr, "ERROR: got 0x%x (%s) unbinding drive\n",
	    rc, nasd_error_string(rc));
    exit(1);
  }

  nasd_cl_p_shutdown();

  exit(0);
}
