#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          Test squeue filtering (--jobs, --nodelist, --states, --steps and
#          --user options).
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2002 The Regents of the University of California.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Morris Jette <jette1@llnl.gov>
# CODE-OCEC-09-009. All rights reserved.
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id      "5.6"
set exit_code    0
set file_in      "test$test_id.input"
set job_id1      0
set job_id2      0
set step_id      0
set job_name1    "test$test_id.job1"
set job_name2    "test$test_id.job2"
set node_cnt 1-1

print_header $test_id

if {[job_submit_all_partitions] != 0} {
	send_user "\nWARNING: This test is incompatible with JobSubmitPlugins=all_partitions\n"
	exit 0
}

#
# Build input script file
#
make_bash_script $file_in "$srun $bin_sleep $max_job_delay"

#
# Submit a couple of job so we have something to look at
#
set sbatch_pid [spawn $sbatch -N$node_cnt -J$job_name1 --output=/dev/null --error=/dev/null -t5 $file_in]
expect {
	-re "Submitted batch job ($number)" {
		set job_id1 $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: srun not responding\n"
		slow_kill $sbatch_pid
		exit 1
	}
	eof {
		wait
	}
}
if {$job_id1 == 0} {
	send_user "\nFAILURE: srun job submit failure\n"
	exit 1
}

set sbatch_pid [spawn $sbatch -N$node_cnt -J$job_name2 --output=/dev/null --error=/dev/null --hold -t5 $file_in]
expect {
	-re "Submitted batch job ($number)" {
		set job_id2 $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: srun not responding\n"
		slow_kill $sbatch_pid
		cancel_job $job_id1
		exit 1
	}
	eof {
		wait
	}
}
if {$job_id2 == 0} {
	send_user "\nFAILURE: srun job submit failure\n"
	cancel_job $job_id1
	exit 1
}

#
# Check the squeue job filter option
#
set job_found 0
spawn $squeue --format=%i --jobs $job_id2
expect {
	-re "($number)" {
		set tmp_id $expect_out(1,string)
		if {$tmp_id == $job_id2} {
			set job_found 1
		} else {
			send_user "\nFAILURE: squeue reported invalid job_id\n"
			set exit_code 1
		}
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: squeue not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$job_found == 0} {
	send_user "\nFAILURE: squeue failed to locate desired job\n"
	set exit_code 1
}

#
# Check the squeue state filter option
#
spawn $squeue --format=%t --noheader --states=PD
expect {
	-re "($alpha_cap)" {
		if {[string compare $expect_out(1,string) "PD"]} {
			send_user "\nFAILURE: squeue state filter failure\n"
			set exit_code 1
		}
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: squeue not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

#
# Check the squeue user filter option with name
#

set this_user [get_my_user_name]

spawn $squeue --format=%u --noheader --user=$this_user
expect {
	-re "($alpha_numeric_under)" {
		if {[string compare $expect_out(1,string) $this_user]} {
			send_user "\nFAILURE: squeue user filter failure\n"
			set exit_code 1
		}
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: squeue not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

#
# Check the squeue user filter option with id
#
set this_uid [get_my_uid]

spawn $squeue --format=%u --noheader --user=$this_uid
expect {
	-re "($alpha_numeric_under)" {
		if {[string compare $expect_out(1,string) $this_user]} {
			send_user "\nFAILURE: squeue user filter failure\n"
			set exit_code 1
		}
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: squeue not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

#
# Check the squeue node filter option
#
set match 0
spawn $squeue --format=%u --noheader --user=$this_uid --nodelist=dummy_name
expect {

	-re "Invalid node name dummy_name" {
		send_user "\nNo worries. This error is expected\n"
		incr match
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: squeue not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$match != 1} {
	send_user "\nFAILURE: squeue node filter failure\n"
	set exit_code 1
}

# The node filtering really only works if the job has begun execution
set node_name_set 0
spawn $squeue --format=%N --noheader --jobs=$job_id1 --states=RUNNING
expect {
	-re "($alpha_numeric_under)" {
		set node_name $expect_out(1,string)
		set node_name_set 1
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: squeue not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$node_name_set == 1} {
	spawn $squeue --format=%u --noheader --user=$this_uid --nodelist=$node_name
	expect {
		-re "($alpha_numeric_under)" {
			set node_name_set 0
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: squeue not responding\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}
}
if {$node_name_set == 1} {
	send_user "\nFAILURE: squeue node name filtering failed\n"
	set exit_code 1
}

#
# Check the squeue partition filter option
#
# First get partition name(s)
set partition1 ""
set partition2 ""
spawn $squeue --format=%P --noheader
expect {
	-re "($alpha_numeric_under)" {
		if {![string compare $partition1 ""]} {
			set partition1 $expect_out(1,string)
			exp_continue
		}
		if {![string compare $expect_out(1,string) $partition1]} {
			set partition2 $expect_out(1,string)
		}
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: squeue not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}
if {[string compare partition2 ""] == 0} {
	set partition2 $partition1
}
spawn $squeue --format=%P --noheader --partitions=$partition1
expect {
	-re "($alpha_numeric_under)" {
		if {[string compare $expect_out(1,string) $partition1]} {
			send_user "\nFAILURE: squeue partition filter error\n"
			set exit_code 1
		}
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: squeue not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

#
# Wait long enough to make sure the job step is running and has
# started its job steps. Check the squeue job step filter option
#
if {[wait_for_job $job_id1 "RUNNING"] != 0} {
	send_user "\nFAILURE: error waiting for job $job_id1 to start\n"
	cancel_job $job_id1
	set exit_code 1
}
exec $bin_sleep 10
set job_found 0
spawn $squeue --format=%i --steps $job_id1.$step_id
expect {
	-re "($number).($number)" {
		set tmp_id $expect_out(1,string)
		if {$tmp_id == $job_id1} {
			set job_found 1
		} else {
			send_user "\nFAILURE: squeue reported invalid job_step_id\n"
			set exit_code 1
		}
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: squeue not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$job_found == 0} {
	send_user "\nFAILURE: squeue failed to locate desired job step\n"
	set exit_code 1
}

#
# Filters the name associated with the job
#
set job_found 0
spawn $squeue --format=%i --name $job_name1
expect {
	-re "($number)" {
		set tmp_id $expect_out(1,string)
		if {$tmp_id == $job_id1} {
			set job_found 1
		} else {
			send_user "\nFAILURE: squeue reported unexpected job_id ($tmp_id != $job_id1)\n"
			set exit_code 1
		}
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: squeue not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$job_found == 0} {
	send_user "\nFAILURE: squeue failed to locate desired job\n"
	set exit_code 1
}

set job_found 0
spawn $squeue --format=%i --name $job_name2
expect {
	-re "($number)" {
		set tmp_id $expect_out(1,string)
		if {$tmp_id == $job_id2} {
			set job_found 1
		} else {
			send_user "\nFAILURE: squeue reported unexpected job_id ($tmp_id != $job_id2)\n"
			set exit_code 1
		}
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: squeue not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$job_found == 0} {
	send_user "\nFAILURE: squeue failed to locate desired job\n"
	set exit_code 1
}

#
# Cancel jobs and remove files
#
if {[cancel_job $job_id1] != 0} {
	set exit_code 1
}
if {[cancel_job $job_id2] != 0} {
	set exit_code 1
}
if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
	exec $bin_rm -f $file_in
}
exit $exit_code

