/*
 * nasd_cheops_cl_doio.c
 *
 * Module responsible for dispatching ios to the physical NASDs
 *
 * Authors: Khalil Amiri, CMU SCS/ECE, July 18 1997
 *          Sean Levy, CMU SCS, July 1999
 */
/*
 * Copyright (c) of Carnegie Mellon University, 1996,1997,1998,1999.
 *
 * Permission to reproduce, use, and prepare derivative works of
 * this software for internal use is granted provided the copyright
 * and "No Warranty" statements are included with all reproductions
 * and derivative works. This software may also be redistributed
 * without charge provided that the copyright and "No Warranty"
 * statements are included in all redistributions.
 *
 * NO WARRANTY. THIS SOFTWARE IS FURNISHED ON AN "AS IS" BASIS.
 * CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, EITHER
 * EXPRESSED OR IMPLIED AS TO THE MATTER INCLUDING, BUT NOT LIMITED
 * TO: WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY
 * OF RESULTS OR RESULTS OBTAINED FROM USE OF THIS SOFTWARE. CARNEGIE
 * MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT
 * TO FREEDOM FROM PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT.
 */


#include <nasd/nasd_options.h>
#include <nasd/nasd_types.h>
#include <nasd/nasd_mem.h>
#include <nasd/nasd_threadstuff.h>

#include <nasd/nasd_cheops_types.h>
#include <nasd/nasd_cheops_common.h>
#include <nasd/nasd_cheops_ios.h>
#include <nasd/nasd_cheops_client_internal.h>

/*
 * XXX this should be fixed. It will not be correctly included
 * as a dependency on many platforms. Also, the relative path
 * thing makes it hard for alternatively-organized trees, and
 * just give up on including something like this in the kernel.
 */
#include "../cheops/shared/nasd_cheops_cl_drive.h"

extern _nasd_cheops_cache_t *_nasd_cheops_dr_handle_cachep;
int _nasd_cheops_doio_n_threads = _NASD_CHEOPS_CL_NUM_DOIO_THREADS;
nasd_thread_t *_nasd_cheops_doio_thread_desc = NULL;
nasd_threadgroup_t _nasd_cheops_doio_thread_group;

static void doio_thread(nasd_threadarg_t myid_arg);

/* debugging */
#define _NASD_CHEOPS_DOIO_DEBUG 0

void
_nasd_cheops_doio_init(void)
{
  int i;
  int rc=0;
  int ss, rss;
  nasd_threadattr_t thread_attr;

#if _NASD_CHEOPS_DOIO_DEBUG > 0
  fprintf(stderr, "CHEOPS: doio_init called\n");
#endif /* _NASD_CHEOPS_DOIO_DEBUG */

  if (_nasd_cheops_doio_n_threads < 1) {
    fprintf(stderr, "CHEOPS: cannot start up with %d I/O threads\n",
            _nasd_cheops_doio_n_threads);
    exit(1);
  }
  rc = nasd_init_threadgroup(&_nasd_cheops_doio_thread_group);
  if (rc) {
    fprintf(stderr, "CHEOPS: could not init doio thread group\n");
    exit(1);
  }
  NASD_Malloc(_nasd_cheops_doio_thread_desc,
              _nasd_cheops_doio_n_threads * sizeof(nasd_thread_t),
              (nasd_thread_t *));

  /* init io queues */
  rc = _nasd_cheops_ioq_init();
  if (rc) {
    fprintf(stderr, "CHEOPS: error initializing io queues..exiting\n");
    exit(1);
  }

#if 0
  /* start up doio threads */
  rc = nasd_threadattr_create(&thread_attr);
  if (rc) {
    fprintf(stderr, "CHEOPS: error creating thread attribute\n");
    exit(1);
  }
#endif
  /* This should not be done here, but rather above us */
#ifndef LINUX
  ss = 32*1024;
  rc = nasd_threadattr_setstacksize(&thread_attr, ss);
  if (rc && (rc != NASD_OP_NOT_SUPPORTED)) {
    fprintf(stderr, "CHEOPS: error setting stacksize\n");
    exit(1);
  }
  rc = nasd_threadattr_getstacksize(&thread_attr, &rss);
  if (rc && (rc != NASD_OP_NOT_SUPPORTED)) {
    fprintf(stderr, "CHEOPS: error getting default stacksize\n");
    exit(1);
  }
  else if ((rc == NASD_SUCCESS) && (rss != ss)) {
    fprintf(stderr, "CHEOPS: error setting/getting thread stack size\n");
    exit(1);
  }
#endif /* LINUX */

  /* XXX consider making #threads proportional to #drives */
  for (i = 0; i < _nasd_cheops_doio_n_threads; i++) {
    rc = nasd_thread_create(&_nasd_cheops_doio_thread_desc[i],
                            doio_thread,
                            (nasd_threadarg_t)((unsigned long)i));

    if (rc) {
      fprintf(stderr, "CHEOPS: error spawning thread...exiting\n");
      exit(1);
    }
    NASD_THREADGROUP_STARTED(&_nasd_cheops_doio_thread_group);
  }
  NASD_THREADGROUP_WAIT_START(&_nasd_cheops_doio_thread_group);

#if _NASD_CHEOPS_DOIO_DEBUG > 0
  fprintf(stderr, "CHEOPS: doio_init finished, %d threads running\n",
          _nasd_cheops_doio_n_threads);
#endif /* _NASD_CHEOPS_DOIO_DEBUG */
     
}

static void
doio_thread(
  nasd_threadarg_t      myid_arg)
{
  int myid = (int)myid_arg;
  int rc =0, num_done=0, failures=0;
  int i, di, size_data_read;
  _nasd_cheops_io_t       *new_io;
  nasd_offset_t		  in_offset;
  nasd_len_t		  in_len;
  nasd_len_t		  out_datalen;
  nasd_uint64             bms;
  nasd_identifier_t       oid;
  nasd_partnum_t          in_partnum;
  nasd_status_t           nasd_status;
  nasd_rpc_status_t       status;

  NASD_THREADGROUP_RUNNING(&_nasd_cheops_doio_thread_group);

#if _NASD_CHEOPS_DOIO_DEBUG > 0
  fprintf(stderr, "CHEOPS: thread %d up for service ...\n", myid);
#endif /* _NASD_CHEOPS_DOIO_DEBUG > 0 */

  while (!NASD_THREADGROUP_SHUTDOWNP(&_nasd_cheops_doio_thread_group)) {
#if _NASD_CHEOPS_DOIO_DEBUG > 1
    fprintf(stderr, "CHEOPS: doio %d top of loop debug level %d\n",
            myid, _NASD_CHEOPS_DOIO_DEBUG);
#endif /* _NASD_CHEOPS_DOIO_DEBUG > 0 */
    /* get an io from the queue */ 
    _nasd_cheops_deq_io(&new_io);
    if (NASD_THREADGROUP_SHUTDOWNP(&_nasd_cheops_doio_thread_group))
      break;
#if _NASD_CHEOPS_DOIO_DEBUG > 2
    fprintf(stderr, "CHEOPS: doio %d have an i/o 0x%08lx\n", myid, new_io);
#endif /* _NASD_CHEOPS_DOIO_DEBUG > 0 */

    /* get handle to target drive */ 
    di = new_io->di;
    in_partnum = 0;

    switch (new_io->type) {
    case _NASD_CHEOPS_RAID0_WRITE_IO:	    
      /* write to target stripe unit */
      in_len = new_io->in_datalen; 
      in_offset = new_io->in_offset;
      oid = new_io->ni;
#if _NASD_CHEOPS_DOIO_DEBUG >0
      fprintf(stderr, "doio: issuing write to obj=0x%" NASD_ID_FMT
       ",di=%d, off=%" NASD_64u_FMT ",len=%d\n",
       oid, di, in_offset, in_len);
#endif

      rc = _nasd_cheops_write_dr(di, in_partnum, 
				 new_io->req_key, oid, 
				 new_io->databuf, in_offset, 
				 in_len, &out_datalen,
				 &nasd_status, &status);
      new_io->nasd_status = nasd_status;
      new_io->status = status;
      new_io->out_datalen = out_datalen;
      break;

    case _NASD_CHEOPS_RAID1_WRITE_IO:	    
      /* write conditional to first stripe unit, if sucessful,
         write conditional to second. This guarantees that at least
         one copy of the data is valid in case of power failure */
      in_len = new_io->in_datalen; 
      in_offset = new_io->in_offset;
      oid = new_io->ni;
      rc = _nasd_cheops_write_dr(di, in_partnum, 
				 new_io->req_key, oid, 
				 new_io->databuf, in_offset, 
				 in_len, &out_datalen,
				 &nasd_status, &status);
      new_io->nasd_status = nasd_status;
      new_io->status = status;
      new_io->out_datalen = out_datalen;
      break;
#if 0
    case _NASD_CHEOPS_TSWRITE_IO:	
      /* write to target stripe unit */
      in_len = new_io->in_datalen; 
      in_offset = new_io->in_offset;
      oid = new_io->ni;

#if _NASD_CHEOPS_DOIO_DEBUG >0
      fprintf(stderr, "doio: issuing write to obj=%d,di=%d, off=%d,len=%d\n",
       oid, di, in_offset, in_len);
#endif

      rc = _nasd_cheops_tswrite_dr(di, in_partnum, 
				   new_io->req_key, oid, 
				   new_io->databuf, in_offset, 
				   in_len, new_io->host_tag, &out_datalen,
				   &nasd_status, &status);
      new_io->nasd_status = nasd_status;
      new_io->out_datalen = out_datalen;
      break;
#endif
    case _NASD_CHEOPS_READ_IO:
      /* read data stripe unit */
      in_len = new_io->in_datalen; 
      in_offset = new_io->in_offset;
      oid = new_io->ni;
      bms = new_io->bms_targ;

      rc = _nasd_cheops_read_dr(di, in_partnum, 
				new_io->req_key, oid, 
				new_io->databuf, in_offset, in_len, bms,
				&out_datalen,
				&nasd_status, &status);
      bcopy(new_io->databuf, &(new_io->host_tag),
	    sizeof(nasd_data_block_tag_t));
      new_io->nasd_status = nasd_status;
      new_io->status = status;
      new_io->out_datalen = out_datalen;

#if _NASD_CHEOPS_DOIO_DEBUG > 0
      fprintf(stderr, "[thread:%d] read obj %" NASD_64u_FMT
              ", len=%d, off=%" NASD_64u_FMT ", buf=%p\n",
       myid, new_io->ni, new_io->in_datalen, new_io->in_offset,
       new_io->databuf);
#endif
      break;	    

    case _NASD_CHEOPS_GETATTR_IO:
      oid = new_io->ni;
      _nasd_cheops_getattr_dr(di, in_partnum, new_io->req_key, oid, 
			      (nasd_attribute_t *)new_io->databuf,
			      &nasd_status, &status);
      new_io->nasd_status = nasd_status;
      new_io->status = status;
#if _NASD_CHEOPS_DOIO_DEBUG > 0
      fprintf(stderr, "[thread:%d] sent getattr to (%" NASD_64u_FMT ", %d)\n", 
       myid, new_io->ni, di); 
#endif
      break;
	     
    case _NASD_CHEOPS_SETATTR_IO:
      oid = new_io->ni;
      _nasd_cheops_setattr_dr(di, in_partnum, new_io->req_key, oid, 
			      (nasd_attribute_t *)new_io->databuf,
			      &nasd_status, &status);
      new_io->nasd_status = nasd_status;
      new_io->status = status;
	    
      break;

    default:
      break;
    } 

#if _NASD_CHEOPS_DOIO_DEBUG > 0
    fprintf(stderr, "[thread:%d] before locked section\n", myid);
#endif /* _NASD_CHEOPS_DOIO_DEBUG > 0 */

    /* set status */
    NASD_LOCK_MUTEX(new_io->parent_io->flag_mutex);

    num_done = new_io->parent_io->num_done;
    num_done++;
    new_io->parent_io->num_done = num_done;
    if (new_io->nasd_status != NASD_SUCCESS)
      new_io->parent_io->nasd_status = new_io->nasd_status; 
    if (new_io->status)
      new_io->parent_io->status = new_io->status; 
    size_data_read = new_io->parent_io->out_datalen;
    new_io->parent_io->out_datalen = size_data_read + new_io->out_datalen; 
#if _NASD_CHEOPS_DOIO_DEBUG > 0
    (cheops_Msg
     "[thread:%d]: child io->out_datalen=%d, status=%d/%d, "
     "parent_io=%x\n", myid, new_io->out_datalen, new_io->nasd_status,
     new_io->status, new_io->parent_io);
    (cheops_Msg
     "[thread:%d]: parent io->out_datalen=%d, "
     "parent_io->status=%d/%d, first_io=%x num_done=%d/%d\n", myid,
     new_io->parent_io->out_datalen, new_io->parent_io->nasd_status,
     new_io->parent_io->status, new_io->parent_io->first_io,
     num_done, new_io->parent_io->num_ios);
#endif /*  _NASD_CHEOPS_DOIO_DEBUG > 0 */
    /* if last io, wake up parent thread */
    if (num_done == new_io->parent_io->num_ios) 
      NASD_SIGNAL_COND(new_io->parent_io->ios_all_done); 
    NASD_UNLOCK_MUTEX(new_io->parent_io->flag_mutex);

#if _NASD_CHEOPS_DOIO_DEBUG > 0
    fprintf(stderr, "[thread:%d] out of locked section\n", myid);
#endif /* _NASD_CHEOPS_DOIO_DEBUG > 0 */
  }

#if _NASD_CHEOPS_DOIO_DEBUG > 0
  fprintf(stderr, "[thread:%d] group finished\n", myid);
#endif /* _NASD_CHEOPS_DOIO_DEBUG > 0 */

  NASD_THREADGROUP_DONE(&_nasd_cheops_doio_thread_group);
  NASD_THREAD_KILL_SELF();

}

/* Local Variables:  */
/* indent-tabs-mode: nil */
/* tab-width: 2 */
/* End: */
