/*
 * nasd_od_drive.c
 *
 * User-level RPC front-end for on-disk filesystem.
 *
 * Authors: Jim Zelenka, Marc Unangst
 */
/*
 * Copyright (c) of Carnegie Mellon University, 1997,1998,1999.
 *
 * Permission to reproduce, use, and prepare derivative works of
 * this software for internal use is granted provided the copyright
 * and "No Warranty" statements are included with all reproductions
 * and derivative works. This software may also be redistributed
 * without charge provided that the copyright and "No Warranty"
 * statements are included in all redistributions.
 *
 * NO WARRANTY. THIS SOFTWARE IS FURNISHED ON AN "AS IS" BASIS.
 * CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, EITHER
 * EXPRESSED OR IMPLIED AS TO THE MATTER INCLUDING, BUT NOT LIMITED
 * TO: WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY
 * OF RESULTS OR RESULTS OBTAINED FROM USE OF THIS SOFTWARE. CARNEGIE
 * MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT
 * TO FREEDOM FROM PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT.
 */


#include <nasd/nasd_options.h>
#include <nasd/nasd_drive_options.h>
#include <nasd/nasd_types.h>
#include <nasd/nasd_freelist.h>
#include <nasd/nasd_itypes.h>
#include <nasd/nasd_mem.h>
#include <nasd/nasd_cache.h>
#include <nasd/nasd_common.h>
#include <nasd/nasd_getopt.h>
#include <nasd/nasd_mq.h>
#include <nasd/nasd_drive_build_stamp.h>

#include <errno.h>
#include <fcntl.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/signal.h>
#include <sys/stat.h>
#include <sys/socket.h>

#include <nasd/nasd_pdrive.h>

#include <nasd/nasd_drive_defaults.h>

char *progname;
char *nasd_devname;

int nasd_drive_stopped = 0;
int nasd_create_partition_on_format = 0;

extern int nasd_od_mq_ignore_eexist;
extern int nasd_od_rusage_at_shutdown;

int service_threads = NASD_DEF_SERVICE_THREADS;

int io_outstanding = NASD_DEF_IO_OUTSTANDING;

int def_cachesize; /* initialized from nasd_odc_size at startup */

nasd_layout_type_t layout = NASD_DEF_LAYOUT;

nasd_ioqueue_type_t ioqueue = NASD_DEF_IOQUEUE;

nasd_od_config_t config;

pid_t nasd_drive_signal_pid;

int signal_pipe[2];
#define signal_pipe_rd  signal_pipe[0]
#define signal_pipe_wr  signal_pipe[1]

#define CLOSEUP_PIPES() { \
  close(signal_pipe[0]); \
  close(signal_pipe[1]); \
}

/* 
 * Under Linux, signals may be delivered to *all* of the threads, not
 * just one (as under DUX).  This is a violation of the POSIX pthreads
 * spec, and happens because Linux "threads" are really "processes"
 * that happen to share the same address space and file descriptor
 * set.  Signals sent with kill() or raise() will go to the thread
 * whose PID is specified (which also violates POSIX, but let's not go
 * there), but signals sent to the process group -- say, when you press
 * ^C on a tty -- go to all threads in the process group.
 * 
 * Because it is impossible to determine which thread will hit this
 * code first, it is possible for the thread that runs the shutdown
 * code to be a thread that also has cleanup work to do before it
 * exits.  If that happens, the drive will deadlock.  Therefore, we
 * create a "supervisor" thread to do the shutdown stuff, and allow
 * the thread that ran the signal handler to return.  When signals come
 * in, we simply ensure that this thread awakens.
 *
 * Under other systems, we're going to go ahead and create this extra
 * thread as well so we can avoid tricky synchronization at cleanup
 * time.
 */
nasd_thread_id_t nasd_od_master_shutdown_thread_id;
nasd_threadgroup_t nasd_od_master_shutdown_thread_group;
nasd_thread_t nasd_od_shutdown_worker_thread;
sig_atomic_t nasd_od_shutdown_signalled = 0;
int alldone = 0;

void handle_sigint(int);
static void handle_interrupt(void);

unsigned long max_len = 0;

static void
nasd_od_shutdown_worker_proc(
  nasd_threadarg_t  ignored)
{
  nasd_thread_id_t self;
  fd_set mrdb, rdb;
  int ret;

  nasd_drive_signal_pid = getpid();
  self = nasd_thread_self();

  nasd_od_master_shutdown_thread_id = self;
  NASD_THREADGROUP_RUNNING(&nasd_od_master_shutdown_thread_group);

  FD_ZERO(&mrdb);
  FD_SET(signal_pipe_rd, &mrdb);

  while(!nasd_od_shutdown_signalled) {
    rdb = mrdb;
    ret = select(signal_pipe_rd+1, &rdb, NULL, NULL, NULL);
    if (nasd_od_shutdown_signalled || FD_ISSET(signal_pipe_rd, &mrdb))
      break;
    if (ret) {
      fprintf(stderr, "DRIVE WARNING: shutdown worker thread got "
        "ret %d errno %d\n", ret, errno);
      fflush(stderr);
    }
  }

  fprintf(stderr, "INTERRUPTED (thread id %" NASD_THREAD_ID_FMT ")\n", self);
  fflush(stderr);

  handle_interrupt();

  /* I don't think we should ever get back here. */
}

void
handle_sigint(
  int  sig)
{
  int ret;
  char c;

  c = 'a';
  ret = write(signal_pipe_wr, &c, 1);
}

static void
handle_interrupt()
{
  nasd_thread_id_t self;
  nasd_status_t rc;

  self = nasd_thread_self();
  fprintf(stderr, "DRIVE: thread id %" NASD_THREAD_ID_FMT " shutting down\n",
    self);
  fflush(stderr);

  if (nasd_drive_stopped)
    return;
  nasd_drive_stopped = 1;

  nasd_drive_stop_rpc();

#if MJU_DEBUG
  fprintf(stderr, "shutdown[%lu]: stop_rpc() complete\n", self);
  fflush(stderr);
#endif /* MJU_DEBUG */

  rc = nasd_basic_shutdown();
  if (rc) {
    fprintf(stderr, "ERROR: got 0x%x (%s) from nasd_basic_shutdown()\n",
      rc, nasd_error_string(rc));
    fflush(stderr);
    CLOSEUP_PIPES();
    alldone = 1;
    exit(1);
  }

  fprintf(stderr, "Completed shutdown.\n");
  fflush(stderr);
#if NASD_MEM_COUNT_ALLOC > 0
  if (nasd_mem_allocated>0) {
    fprintf(stderr,
            "WARNING: %" NASD_MEMALLOC_FMT
            " bytes of memory still outstanding, something leaked core. (Note: there is a known 512-byte leak currently)\n",
            nasd_mem_allocated);
    fflush(stderr);
  }
  else {
    printf("0 bytes of memory still outstanding.\n");
  }
#endif /* NASD_MEM_COUNT_ALLOC > 0 */
  fflush(stdout);

  CLOSEUP_PIPES();
  alldone = 1;
  exit(0);
}

void
usage()
{
  fprintf(stderr, "USAGE: %s [options] devname\n", progname);
  fprintf(stderr, "options:\n");
  fprintf(stderr, "  -c  cache_size [default %d]\n", def_cachesize);
  fprintf(stderr, "  -f  format\n");
  fprintf(stderr, "  -i  I/Os outstanding [default %d]\n",
    NASD_DEF_IO_OUTSTANDING);
  fprintf(stderr, "  -l  layout (default '%c')\n", NASD_DEF_LAYOUT);
  fprintf(stderr, "  -m  maximum disk length to use\n");
  fprintf(stderr, "  -p  if formatting, create partition #1 with all free space on format\n");
  fprintf(stderr, "  -q  disk queue policy (default '%c')\n",
    NASD_DEF_IOQUEUE);
  fprintf(stderr, "  -r  show rusage stats at shutdown\n");
#if NASD_RPC_PACKAGE == NASD_RPC_PACKAGE_DCE
  fprintf(stderr, "  -T  use DCE-TCP\n");
#endif /* NASD_RPC_PACKAGE == NASD_RPC_PACKAGE_DCE */
  fprintf(stderr, "  -t  service threads [default %d]\n",
    NASD_DEF_SERVICE_THREADS);
  fprintf(stderr, "  -x  ignore extant message queue\n");
  fflush(stderr);
  exit(1);
}

int
main(argc, argv)
  int     argc;
  char  **argv;
{
  unsigned long num_sect;
  nasd_uint64 dlen, len;
  nasd_uint16 ipport;
  struct timeval tv;
  nasd_status_t rc;
  int fd, ret, val;
  struct stat s;
  dev_t dev;
  char c;

  progname = argv[0];
  def_cachesize = nasd_odc_size;

#if NASD_RPC_PACKAGE == NASD_RPC_PACKAGE_DCE
  nasd_drive_dce_setup_udp();
#endif /* NASD_RPC_PACKAGE == NASD_RPC_PACKAGE_DCE */

  while (nasd_getopt(argc, argv, "c:fi:l:m:pq:rTt:x", &c)) {
    switch(c) {
      case 'c':
        if (sscanf(nasd_optarg, "%d", &val) != 1)
          usage();
        if (val < 1)
          usage();
        nasd_odc_size = val;
        break;
      case 'f':
        nasd_odc_force_format = 1;
        break;
      case 'i':
        if (sscanf(nasd_optarg, "%d", &io_outstanding) != 1)
          usage();
        break;
      case 'l':
        if (strlen(nasd_optarg) != 1) {
          usage();
        }
        layout = nasd_optarg[0];
        break;
      case 'm':
        if (sscanf(nasd_optarg, "%lu", &max_len) != 1)
          usage();
        if (max_len == 0)
          usage();
        break;
      case 'p':
        nasd_create_partition_on_format = 1;
        break;
      case 'q':
        if (strlen(nasd_optarg) != 1) {
          usage();
        }
        ioqueue = nasd_optarg[0];
        break;
      case 'r':
        nasd_od_rusage_at_shutdown = 1;
        break;
#if NASD_RPC_PACKAGE == NASD_RPC_PACKAGE_DCE
      case 'T':
        nasd_drive_dce_setup_tcp();
        break;
#endif /* NASD_RPC_PACKAGE == NASD_RPC_PACKAGE_DCE */
      case 't':
        if (sscanf(nasd_optarg, "%d", &service_threads) != 1)
          usage();
        break;
      case 'x':
        if(nasd_od_mq_ignore_eexist)
          usage();
        nasd_od_mq_ignore_eexist = 1;
        break;
      default:
        fprintf(stderr, "Unknown option '%c'\n", nasd_optopt);
        usage();
    }
  }
  if (nasd_optind >= argc)
    usage();
  nasd_devname = argv[nasd_optind];
  nasd_optind++;
  if (nasd_optind < argc)
    usage();

  if (service_threads < 1)
    usage();

  if (io_outstanding < 1)
    usage();

  fd = open(nasd_devname, O_RDWR);
  if (fd < 0) {
    perror(nasd_devname);
    fprintf(stderr, "ERROR: cannot open %s\n", nasd_devname);
    fflush(stderr);
    exit(1);
  }

  rc = nasd_raw_disk_len(fd, &dlen);
  if (rc) {
    fprintf(stderr, "ERROR: cannot get raw disk length, error 0x%x (%s)\n",
      rc, nasd_error_string(rc));
    exit(1);
  }

  ret = fstat(fd, &s);
  if (ret) {
    perror("fstat");
    fprintf(stderr, "ERROR: could not stat %s\n", nasd_devname);
    fflush(stderr);
    exit(1);
  }
  dev = s.st_rdev;

  close(fd);

  config.layout_type = layout;
  config.ioqueue_type = ioqueue;
  config.ios_outstanding = io_outstanding;

  if (max_len) {
    len = NASD_MIN(max_len, dlen);
    num_sect = (unsigned long) (len >>= 9);
  }
  else {
    num_sect = (unsigned long) (dlen >>= 9);
  }

  ret = pipe(signal_pipe);
  if (ret) {
    fprintf(stderr, "ERROR: cannot create signal pipe\n");
    fflush(stderr);
    exit(1);
  }

  printf("Initializing %s, %lu sectors\n", nasd_devname, num_sect);
#if NASD_SECURE_RPCS_ENABLE > 0
   printf("\nDrive compiled *WITH* security\n\n");
#else /* NASD_SECURE_RPCS_ENABLE > 0 */
   printf("\nDrive compiled WITHOUT security\n\n");
#endif /* NASD_SECURE_RPCS_ENABLE > 0 */

  fflush(stdout);

  rc = nasd_basic_init();
  if (rc) {
    fprintf(stderr, "ERROR: got %s from nasd_basic_init()\n",
      nasd_error_string(rc));
    fflush(stderr);
    CLOSEUP_PIPES();
    exit(1);
  }
  rc = nasd_od_io_init(nasd_devname, &config);
  if (rc) {
    fprintf(stderr, "ERROR: got %s from nasd_od_io_init()\n",
      nasd_error_string(rc));
    fflush(stderr);
    CLOSEUP_PIPES();
    exit(1);
  }

  printf("Setting up...\n");
  fflush(stdout);

  rc = nasd_setup_disk(num_sect, dev, &config);
  if (rc) {
    fprintf(stderr, "ERROR: got %s from nasd_setup_disk()\n",
      nasd_error_string(rc));
    fflush(stderr);
    goto abort_drive;
  }

  nasd_drive_signal_pid = getpid();

  rc = nasd_init_threadgroup(&nasd_od_master_shutdown_thread_group);
  if (rc) {
    fprintf(stderr,
      "ERROR: got 0x%x (%s) initializing nasd_od_master_shutdown_thread_group\n",
      rc, nasd_error_string(rc));
    fflush(stderr);
    goto abort_drive;
  }

  rc = nasd_thread_create_w_name(&nasd_od_shutdown_worker_thread,
    nasd_od_shutdown_worker_proc, NULL, "nasd_od_shutdown_worker");
  if (rc) {
    fprintf(stderr,
      "ERROR: got 0x%x (%s) creating shutdown worker thread\n",
      rc, nasd_error_string(rc));
    fflush(stderr);
    goto abort_drive;
  }
  NASD_THREADGROUP_WAIT_START(&nasd_od_master_shutdown_thread_group);

  rc = nasd_od_io_go();
  if (rc) {
    fprintf(stderr, "ERROR: got %s from nasd_od_io_go()\n",
      nasd_error_string(rc));
    fflush(stderr);
    goto abort_drive;
  }

  printf("Setup RPC subsystem\n");

  rc = nasd_drive_startup_rpc();
  if (rc) {
    fprintf(stderr, "ERROR: got 0x%x (%s) from nasd_startup_rpc()\n",
      rc, nasd_error_string(rc));
    fflush(stderr);
    CLOSEUP_PIPES();
    exit(1);
  }

#ifndef KERNEL
  printf("Setup message queue subsystem\n");

  rc = nasd_drive_startup_msgq();
  if (rc) {
    fprintf(stderr, "ERROR: got 0x%x (%s) from nasd_startup_msgq()\n",
            rc, nasd_error_string(rc));
    fflush(stderr);
    CLOSEUP_PIPES();
    exit(1);
  }
#endif /* !KERNEL */

#ifndef LINUX
  signal(SIGUSR1, handle_sigint);
#endif /* LINUX */

#ifdef LINUX
  /* Linux's pthreads library uses SIGUSR1 and SIGUSR2, so we can't. */
  signal(SIGPWR, handle_sigint);
#endif /* LINUX */

  signal(SIGINT, handle_sigint);

  nasd_printf("DRIVE: %s\n",nasd_drive_build_stamp);

  printf("Running with pid %d.\n", nasd_drive_signal_pid);
  fflush(stdout);

  if (sscanf(NASD_PDRIVE_PORT, "%hu", &ipport) != 1) {
    NASD_PANIC();
  }

  rc = nasd_drive_rpc_listen(service_threads, ipport);
  if (rc) {
    fprintf(stderr, "ERROR: got 0x%x (%s) from nasd_drive_rpc_listen()\n",
      rc, nasd_error_string(rc));
    fflush(stderr);
  }

abort_drive:
  if (!nasd_drive_stopped) {
    handle_interrupt();
  }
  while (alldone == 0) {
    /*
     * Someone else is stopping. They'll exit for us.
     */
    tv.tv_sec = 1;
    tv.tv_usec = 0;

    ret = select(1, NULL, NULL, NULL, &tv);
    if (ret) {
      break;
    }
  }

  CLOSEUP_PIPES();
  exit(2);
}

/* Local Variables:  */
/* indent-tabs-mode: nil */
/* tab-width: 2 */
/* End: */
