#!/bin/sh
###############################################################################
##                                                                           ##
##           l  p  j  s  u  b  m  i  t  _  i  n  t  e  l                     ##
##                                                                           ##
##    sh script to generate and submit parallel LoadLeveler jobs using       ##
##    Intel MPI.                                                             ##
##                                                                           ##
##    Usage: lpjsubmit [-class c] [-group g] [-wtime time] [-blocking n]     ##
##                     [-o output] [-e error] [-q] [-nocheck] ntask          ##
##                     [LPJargs...]                                          ##
##                                                                           ##
##    written by Werner von Bloh, PIK Potsdam                                ##
##                                                                           ##
##    Last change: $Date:: 2013-08-16 11:43:44 +0200 (Fri, 16 Aug 2013)    $ ##
##    By         : $Author:: bloh                            $               ##
##                                                                           ##
###############################################################################

if [ $# -lt 1 ]
then
  echo >&2 Error: Number of tasks missing
  echo >&2 Usage: $0 [-class c] [-group g] [-wtime time] [-blocking n] [-o output] [-e error] [-q] [-nocheck] ntasks [args ...]
  exit 1
fi

if [ "$LPJROOT" = "" ]
then
  echo >&2 Error: environment variable LPJROOT is not set
  echo >&2 "Set by export LPJROOT=<path to lpjml directory>"
  exit 1
fi

if [ $1 = "-class" ]
then
  if [ $# -lt 2 ]
  then
    echo >&2 Error: class missing
    echo >&2 Usage: $0 [-class c] [-group g] [-wtime time] [-blocking n] [-o output] [-e error] [-q] [-nocheck] ntasks [args ...]
    exit 1
  fi
  shift 1
  class=$1
  shift 1
else
  class=short
fi
if [ $1 = "-group" ]
then
  if [ $# -lt 2 ]
  then
    echo >&2 Error: group missing
    echo >&2 Usage: $0 [-class c] [-group g] [-wtime time] [-blocking n] [-o output] [-e error] [-q] [-nocheck] ntasks [args ...]
    exit 1
  fi
  shift 1
  group=$1
  shift 1
else
  group=bios-x
fi

wtime=""
if [ $1 = "-wtime" ]
then
  if [ $# -lt 2 ]
  then
    echo >&2 Error: wall clock time missing
    echo >&2 Usage: $0 [-class c] [-group g] [-wtime time] [-blocking n] [-o output] [-e error] [-q] [-nocheck] ntasks [args ...]
    exit 1
  fi
  shift 1
  wtime=$1
  shift 1
fi
blocking="unlimited"
if [ $1 = "-blocking" ]
then
  if [ $# -lt 2 ]
  then
    echo >&2 Error: blocking factor missing 
    echo >&2 Usage: $0 [-class c] [-group g] [-wtime time] [-blocking n] [-o output] [-e error] [-q] [-nocheck] ntasks [args ...]
    exit 1
  fi
  shift 1
  blocking=$1
  shift 1
fi
output="lpjml.\$(cluster).out"
if [ $1 = "-o" ]
then
  if [ $# -lt 2 ]
  then
    echo >&2 Error: output filename missing 
    echo >&2 Usage: $0 [-class c] [-group g] [-wtime time] [-blocking n] [-o output] [-e error] [-q] [-nocheck] ntasks [args ...]
    exit 1
  fi
  shift 1
  output=$1
  shift 1
fi
error="lpjml.\$(cluster).err"
if [ $1 = "-e" ]
then
  if [ $# -lt 2 ]
  then
    echo >&2 Error: error filename  missing
    echo >&2 Usage: $0 [-class c] [-group g] [-wtime time] [-blocking n] [-o output] [-e error] [-q] [-nocheck] ntasks [args ...]
    exit 1
  fi
  shift 1
  error=$1
  shift 1
fi

if [ $1 = "-q" ]
then
 shift 1
 quiet="-q"
fi
if [ $1 = "-nocheck" ]
then
 shift 1
 nocheck="-nocheck"
fi
if [ $# -lt 1 ]
then
  echo >&2 Error: Number of tasks missing
  echo >&2 Usage: $0 [-class c] [-group g] [-wtime time] [-blocking n] [-o output] [-e error] [-q] [-nocheck] ntasks [args ...]
  exit 1
fi

ntask=$1 # number of tasks
shift 1
args=$*  # runtime arguments for lpjml

# check, whether LPJ configuration is valid

if $LPJROOT/bin/lpjcheck $quiet $nocheck $args ;
then
# yes, create LoadL job control file
  cat <<EOF >job.jcf
#!/bin/ksh 
###############################################################################
##                                                                           ##
##                     j  o  b  .  j  c  f                                   ##
##                                                                           ##
##  LoadLeveler JCF file for running an Intel MPI job on the IPLEX cluster   ##
##  at PIK                                                                   ##
##                                                                           ##
##  Automatically generated by lpjsubmit shell script                        ##
##                                                                           ##
##  Created: $(date +"%d.%m.%Y")                                                      ##
##                                                                           ##
###############################################################################
 
# @ job_type = parallel
# @ total_tasks = $ntask
# @ class = $class
# @ group = $group
EOF
if [ "$wtime" != "" ]
then
  echo "# @ wall_clock_limit = " $wtime >>job.jcf
fi
cat <<EOF >>job.jcf
# @ comment = LPJmL Version $(cat $LPJROOT/VERSION) args: $args
# @ environment = COPY_ALL
# @ blocking = $blocking
# @ output = $output
# @ error = $error
# @ queue

llgetmachinelist > hostlist.\$LOADL_STEP_ID

cat hostlist.\$LOADL_STEP_ID|uniq >mpdhosts.\$LOADL_STEP_ID
machine_count=\`cat mpdhosts.\$LOADL_STEP_ID  | wc -l\`
node_count=\`cat hostlist.\$LOADL_STEP_ID  | wc -l\`
export MPD_CON_EXT=\$LOADL_STEP_ID 
mpdboot -n \$machine_count -r ssh -f mpdhosts.\$LOADL_STEP_ID 

mpiexec -machinefile hostlist.\$LOADL_STEP_ID -n \$node_count \$LPJROOT/bin/lpjml $args
rc=\$?  # save return code of mpiexec
#mpdallexit
rm hostlist.\$LOADL_STEP_ID
rm mpdhosts.\$LOADL_STEP_ID
exit \$rc # exit with return code
EOF
# submit job
  if llsubmit $quiet job.jcf ;
  then
    if [ "$quiet" != "-q" ]
    then
      llq -u $(whoami)
    fi
  else
    exit 1
  fi
else
  echo >&2 "Error in LPJ configuration, job not submitted"
  exit 1
fi
