#!/bin/tcsh -f

#######################################################################
#                            Define Colors
#         Note:  For No Colors, set C1 and C2 to NONE 
#######################################################################

set BLACK   = `tput setaf 0`
set RED     = `tput setaf 1`
set GREEN   = `tput setaf 2`
set YELLOW  = `tput setaf 3`
set BLUE    = `tput setaf 4`
set MAGENTA = `tput setaf 5`
set CYAN    = `tput setaf 6`
set WHITE   = `tput setaf 7`
set RESET   = `tput sgr0`
set BOLD    = `tput bold`
set COLORS  = `echo $BLACK $RED $GREEN $YELLOW $BLUE $MAGENTA $CYAN $WHITE $RESET`

if( -e $HOME/.FV3SETUP ) then
     set FV3SETUPinfo = `cat $HOME/.FV3SETUP`
     set C1 = $FV3SETUPinfo[1]
     set C2 = $FV3SETUPinfo[2]
else
     set C1 = $RED
     set C2 = $BLUE
endif
     set CN = $RESET

#######################################################################
#                     Build Directory Locations
#######################################################################

# Set Current Working Path to fv3_setup
# -------------------------------------
setenv ARCH `uname -s`
if ($ARCH == Darwin) then
   set FINDPATH = realpath
else
   set FINDPATH = 'readlink -f'
endif
set FV3SETUP = `$FINDPATH $0`
set BINDIR   = `dirname $FV3SETUP`
set GEOSDEF  = `dirname $BINDIR`
set ETCDIR   = ${GEOSDEF}/etc

# Test if StandAlone_FV3_Dycore.x is here which means you are in install directory
if (! -x StandAlone_FV3_Dycore.x) then
   echo "You are trying to run $0 in the Applications/StandAlone_FV3_Dycore directory"
   echo "This is no longer supported. Please run from the bin/ directory"
   echo "in your installation"
   exit 1
endif

#######################################################################
#                   Test for Command Line Flags
#######################################################################

# Set default behavior of switches
set LINKX = FALSE

while ( $#argv > 0 )
   set arg = $argv[1]
   shift argv
   switch ( $arg )
      # Set our colors
      case -[Cc]:
      case --[Cc][Oo][Ll][Oo][Rr]:
         goto SETCOLOR

      # Symlink StandAlone_FV3_Dycore.x
      case --link:
         set LINKX = TRUE
         breaksw

      # Here any string not above will trigger USAGE
      case -[Hh]:
      case --[Hh][Ee][Ll][Pp]:
      default:
         goto USAGE
   endsw
end

#######################################################################
#           CVS: Use CVS functionality only at NCCS or NAS
#######################################################################

setenv NODE `uname -n`
setenv ARCH `uname -s`
setenv SITE `awk '{print $2}' $ETCDIR/SITE.rc`

#######################################################################
#                 Test for Compiler and MPI Setup
#######################################################################

setenv BASEDIR `awk '{print $2}' $ETCDIR/BASEDIR.rc`

     if ( `echo $BASEDIR | grep -i mvapich2` != '') then
   set MPI = mvapich2
else if ( `echo $BASEDIR | grep -i openmpi`  != '') then
   set MPI = openmpi
else if ( `echo $BASEDIR | grep -i hpcx`     != '') then
   set MPI = openmpi
else if ( `echo $BASEDIR | grep -i impi`     != '') then
   set MPI = intelmpi
else if ( `echo $BASEDIR | grep -i intelmpi` != '') then
   set MPI = intelmpi
else if ( `echo $BASEDIR | grep -i mpt`      != '') then
   set MPI = mpt
else
   # Assume default is Intel MPI in case of older baselibs
   set MPI = intelmpi 
endif

#######################################################################
#               Enter Experiment Specific Run Parameters
#######################################################################

echo
echo "Enter the ${C1}Experiment ID${CN}:"
set EXPID = $<

DESC:
echo "Enter a 1-line ${C1}Experiment Description${CN}:"
set EXPTMP =  `echo $<`
if( $#EXPTMP == 0 ) goto DESC
set EXPDSC =  $EXPTMP[1]
foreach WORD ($EXPTMP)
if( $WORD !=  $EXPDSC ) set EXPDSC = `echo ${EXPDSC}_${WORD}`
end

#######################################################################
#          Continue to enter in experiment parameters 
#######################################################################

HRCODE:
echo "Enter the ${C1}Horizontal Resolution${CN}: ${C2}IM${CN} (Default: 48)"
set   AGCM_IM = $<
if( .$AGCM_IM == . ) then
  set AGCM_IM = 48
endif

set CUBE_AGCM = ""
set AGCM_NF = 6
set GRID_TYPE = "Cubed-Sphere"

echo "Enter the ${C1}Vertical Resolution${CN}: ${C2}LM${CN} (Default: 72)"
set   AGCM_LM = $<
if( .$AGCM_LM == . ) then
  set AGCM_LM = 72
endif


#######################################################################
#         Test to see if you want to use non-hydrostatic
#######################################################################

ASKHYDRO:

set DEFAULT_NON_HYDROSTATIC = @CFG_NONHYDROSTATIC@
echo "Do you wish to ${C1}NON-HYDROSTATIC${CN} dynamics? (Default: ${C2}${DEFAULT_NON_HYDROSTATIC}${CN})"

set   DO_NONHYDRO  = $<
if( .$DO_NONHYDRO == . ) then
   set   DO_NONHYDRO  = $DEFAULT_NON_HYDROSTATIC
else
   set   DO_NONHYDRO  = `echo   $DO_NONHYDRO | tr "[:lower:]" "[:upper:]"`
   if(  $DO_NONHYDRO == "Y"     | \
        $DO_NONHYDRO == "YES"   | \
        $DO_NONHYDRO == "T"     | \
        $DO_NONHYDRO == "TRUE"  ) set DO_NONHYDRO = TRUE
   if(  $DO_NONHYDRO == "N"     | \
        $DO_NONHYDRO == "NO"    | \
        $DO_NONHYDRO == "F"     | \
        $DO_NONHYDRO == "FALSE" ) set DO_NONHYDRO = FALSE

   if( $DO_NONHYDRO != "TRUE" & $DO_NONHYDRO != "FALSE" ) then
      echo
      echo "${C1}NON-HYDROSTATIC${CN} must be set equal to ${C2}TRUE/YES${CN} or ${C2}FALSE/NO${CN}!"
      goto ASKHYDRO
   endif
endif

if ( $DO_NONHYDRO == "TRUE" ) then
   set USE_NONHYDRO = 1
else
   set USE_NONHYDRO = 0
endif

#######################################################################
#            Test to see if you want to use ioserver
#######################################################################

ASKIOS:

if ( $AGCM_IM < 180 ) then
   set DEFAULT_DO_IOS = FALSE
   echo "Do you wish to ${C1}IOSERVER${CN}? (Default: ${C2}NO${CN} or ${C2}FALSE${CN})"
else
   set DEFAULT_DO_IOS = TRUE
   echo "Do you wish to ${C1}IOSERVER${CN}? (Default: ${C2}YES${CN} or ${C2}TRUE${CN})"
endif

set   DO_IOS  = $<
if( .$DO_IOS == . ) then
   set   DO_IOS  = $DEFAULT_DO_IOS
else
   set   DO_IOS  = `echo   $DO_IOS | tr "[:lower:]" "[:upper:]"`
   if(  $DO_IOS == "Y"     | \
        $DO_IOS == "YES"   | \
        $DO_IOS == "T"     | \
        $DO_IOS == "TRUE"  ) set DO_IOS = TRUE
   if(  $DO_IOS == "N"     | \
        $DO_IOS == "NO"    | \
        $DO_IOS == "F"     | \
        $DO_IOS == "FALSE" ) set DO_IOS = FALSE

   if( $DO_IOS != "TRUE" & $DO_IOS != "FALSE" ) then
      echo
      echo "${C1}IOSERVER${CN} must be set equal to ${C2}TRUE/YES${CN} or ${C2}FALSE/NO${CN}!"
      goto ASKIOS
   endif
endif

if ( $DO_IOS == "TRUE" ) then
   set USE_IOSERVER = 1
else
   set USE_IOSERVER = 0
endif

#######################################################################
#                 What Processor Should We Run On?
#######################################################################

ASKPROC:

if ( $SITE == 'NCCS' ) then
   echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
   echo "   ${C2}hasw (Haswell)${CN} (default)"
   if (-e /etc/os-release) then
      echo "   ${C2}sky  (Skylake)${CN}"
   endif
   set MODEL = `echo $<`
   set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
   if ( .$MODEL == .) then
      set MODEL = 'hasw'
   endif

   if( $MODEL != 'hasw' & \
       $MODEL != 'sky' ) goto ASKPROC

   if (     $MODEL == 'hasw') then
      set NCPUS_PER_NODE = 28
   else if ($MODEL == 'sky') then
      set NCPUS_PER_NODE = 40
   endif

else if ( $SITE == 'NAS' ) then
   echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
   echo "   ${C2}has (Haswell)${CN}"
   echo "   ${C2}bro (Broadwell)${CN}"
   echo "   ${C2}sky (Skylake)${CN} (default)"
   echo "   ${C2}cas (Cascade Lake)${CN}"
   echo "   ${C2}rom (AMD Rome)${CN}"
   echo " "
   echo " NOTE: Due to how FV3 is compiled by default, Sandy Bridge"
   echo "       and Ivy Bridge are not supported by current GEOS"
   echo " "
   set MODEL = `echo $<`
   set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
   if ( .$MODEL == .) then
      set MODEL = 'sky'
   endif

   if( $MODEL != 'has' & \
       $MODEL != 'bro' & \
       $MODEL != 'sky' & \
       $MODEL != 'cas' & \
       $MODEL != 'rom' ) goto ASKPROC

   # Some processors have weird names at NAS
   # ---------------------------------------

   if ($MODEL == bro ) then
      set MODEL = 'bro_ele'
   else if ($MODEL == sky) then
      set MODEL = 'sky_ele'
   else if ($MODEL == cas) then
      set MODEL = 'cas_ait'
   else if ($MODEL == rom) then
      set MODEL = 'rom_ait'
   endif

   if (     $MODEL == 'san') then
      set NCPUS_PER_NODE = 16
   else if ($MODEL == 'ivy') then
      set NCPUS_PER_NODE = 20
   else if ($MODEL == 'has') then
      set NCPUS_PER_NODE = 24
   else if ($MODEL == 'bro_ele') then
      set NCPUS_PER_NODE = 28
   else if ($MODEL == 'sky_ele') then
      set NCPUS_PER_NODE = 40
   else if ($MODEL == 'cas_ait') then
      set NCPUS_PER_NODE = 40
   else if ($MODEL == 'rom_ait') then
      set NCPUS_PER_NODE = 128
      # Romes are on a different aoe
      set MODEL='rom_ait:aoe=sles15'
   endif

else
   set MODEL = 'UNKNOWN'
endif

# Set DEFAULT SHMEM Parameter
# ---------------------------
     set USE_SHMEM = 0

# Set IAU-Forcing and Bias Correction OFF
# ---------------------------------------
     set FORCEDAS = "#"
     set FORCEGCM = "#"

# Set Default Readers and Writers
# -------------------------------
     set NUM_READERS = 1
     set NUM_WRITERS = 1

# Set DEFAULT IOSERVER Parameters
# -------------------------------
     set DEF_IOS_NDS      = 1

# Default Run Parameters
# ----------------------
set AGCM_JM  = `expr $AGCM_IM \* 6`
set HIST_IM  = `expr $AGCM_IM \* 4`
set HIST_JM  = `expr $AGCM_IM \* 2 + 1`

if( $AGCM_IM <=  12 ) then
     set       DT = 900
     set       FV_NX = 2
     set       FV_NY = `expr $FV_NX \* 6`
else if( $AGCM_IM <=  24 ) then
     set       DT = 900
     set       FV_NX = 4
     set       FV_NY = `expr $FV_NX \* 6`
else if( $AGCM_IM <=  48 ) then
     set       DT = 450
     set       FV_NX = 4
     set       FV_NY = `expr $FV_NX \* 6`
else if( $AGCM_IM <=  90 ) then
     set       DT = 450
     set  FV_NX = 4
     set  FV_NY = `expr $FV_NX \* 6`
else if( $AGCM_IM <=  180 ) then
     set       DT = 450
     set  FV_NX = 6
     set  FV_NY = `expr $FV_NX \* 6`
     set NUM_READERS = 2
     set DEF_IOS_NDS = 2
else if( $AGCM_IM <= 360 ) then
     set       DT = 450
     set       FV_NX = 12 
     set       FV_NY = `expr $FV_NX \* 6`
     set NUM_READERS = 4
     set DEF_IOS_NDS = 2
else if( $AGCM_IM <= 500 ) then
     set       DT = 450
     set       FV_NX = 12
     set       FV_NY = `expr $FV_NX \* 6`
     set NUM_READERS = 4
     set USE_SHMEM = 1
     set DEF_IOS_NDS = 2
else if( $AGCM_IM <= 720 ) then
     set       DT = 450
     set       FV_NX = 16
     set       FV_NY = `expr $FV_NX \* 6`
     set NUM_READERS = 6
     set USE_SHMEM = 1
     set DEF_IOS_NDS = 3
else if( $AGCM_IM <= 1440 ) then
     set       DT = 450
     set       FV_NX = 30 
     set       FV_NY = `expr $FV_NX \* 6`
     set NUM_READERS = 6
     set USE_SHMEM = 1
     set DEF_IOS_NDS = 4
else if( $AGCM_IM <= 1536 ) then
     set       DT = 90
     set       FV_NX = 64
     set       FV_NY = `expr $FV_NX \* 6`
     set NUM_READERS = 6
     set USE_SHMEM = 1
     set DEF_IOS_NDS = 4
else 
     set       DT = 45
     set       FV_NX = 64
     set       FV_NY = `expr $FV_NX \* 6`
     set NUM_READERS = 6
     set USE_SHMEM = 1
     set DEF_IOS_NDS = 5
endif

set AGCM_GRIDNAME   = "PE${AGCM_IM}x${AGCM_JM}-CF"

#######################################################################
#                  Architecture Specific Variables
#######################################################################

@ MODEL_NPES = $FV_NX * $FV_NY

if ( $DO_IOS == TRUE ) then
   set IOS_NDS = $DEF_IOS_NDS
else
   set IOS_NDS = 0
endif

setenv     RUN_N  $EXPID # RUN Job Name

if( $SITE == 'NAS' ) then
              setenv     RUN_N  `echo $EXPID | cut -b1-200`_RUN                         # RUN      Job Name
              setenv BATCH_CMD "qsub"                 # PBS Batch command
              setenv BATCH_GROUP "PBS -W group_list=" # PBS Syntax for GROUP
              setenv BATCH_TIME "PBS -l walltime="    # PBS Syntax for walltime
              setenv BATCH_JOBNAME "PBS -N "          # PBS Syntax for job name
              setenv BATCH_OUTPUTNAME "PBS -o "       # PBS Syntax for job output name
              setenv BATCH_JOINOUTERR "PBS -j oe -k oed "    # PBS Syntax for joining output and error
              setenv     RUN_T  "1:00:00"             # Wallclock Time   for fv3.j
              set QTYPE = "normal"                 # Queue to use

              #@ NODES  = `echo "($MODEL_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc`
              @ NODES  = `echo "( ($MODEL_NPES + $NCPUS_PER_NODE) + ($IOS_NDS * $NCPUS_PER_NODE) - 1)/$NCPUS_PER_NODE" | bc`

              setenv     RUN_Q  "PBS -q ${QTYPE}"                                                                              # batch queue name for fv3.j
              setenv     RUN_P  "PBS -l select=${NODES}:ncpus=${NCPUS_PER_NODE}:mpiprocs=${NCPUS_PER_NODE}:model=${MODEL}"     # PE Configuration for fv3.j

              setenv WRKDIR     /nobackup/$LOGNAME                                     # user work directory
              setenv CPEXEC    'mcp -a'                                                # Copy utility for large copies
              setenv TAREXEC    mtar                                                   # Tar utility for large archives
else if( $SITE == 'NCCS' ) then
              setenv BATCH_CMD "sbatch"                                                # SLURM Batch command
              setenv BATCH_GROUP "SBATCH --account="                                   # SLURM Syntax for account name
              setenv BATCH_TIME "SBATCH --time="                                       # SLURM Syntax for walltime
              setenv BATCH_JOBNAME "SBATCH --job-name="                                # SLURM Syntax for job name
              setenv BATCH_OUTPUTNAME "SBATCH --output="                               # SLURM Syntax for job output name
              setenv BATCH_JOINOUTERR "DELETE"                                         # SLURM joins out and err by default
              setenv     RUN_T  "01:00:00"                                             # Wallclock Time   for fv3.j

              #@ NODES  = `echo "($MODEL_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc`
              @ NODES  = `echo "( ($MODEL_NPES + $NCPUS_PER_NODE) + ($IOS_NDS * $NCPUS_PER_NODE) - 1)/$NCPUS_PER_NODE" | bc`

              setenv    RUN_Q   "SBATCH --constraint=${MODEL}"                                # batch queue name for fv3.j
              setenv    RUN_P   "SBATCH --nodes=${NODES} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for fv3.j

              setenv WRKDIR     /discover/nobackup/$LOGNAME                            # user work directory
              setenv CPEXEC     /bin/cp                                                # Copy utility for large copies
              setenv TAREXEC    tar
else
# These are defaults for others (assume slurm)
              setenv BATCH_CMD "sbatch"                                                # SLURM Batch command
              setenv BATCH_GROUP "SBATCH --account="                                   # SLURM Syntax for account name
              setenv BATCH_TIME "SBATCH --time="                                       # SLURM Syntax for walltime
              setenv BATCH_JOBNAME "SBATCH --job-name="                                # SLURM Syntax for job name
              setenv BATCH_OUTPUTNAME "SBATCH --output="                               # SLURM Syntax for job output name
              setenv BATCH_JOINOUTERR "DELETE"                                         # SLURM joins out and err by default
              setenv     RUN_T  "01:00:00"                                             # Wallclock Time   for fv3.j

              setenv    RUN_Q   "DELETE"
              setenv    RUN_P   "SBATCH --ntasks=${MODEL_NPES}" # PE Configuration for fv3.j
              setenv NCPUS_PER_NODE NULL # sed needs something

              setenv WRKDIR     /home/$LOGNAME                                         # user work directory
              setenv CPEXEC     /bin/cp                                                # Copy utility for large copies
              setenv TAREXEC    tar                                                    # Tar utility for large archives
endif

#######################################################################
#                 Create Paths for EXP Directories
#######################################################################

# EXP Directory
# -------------
    set check =  FALSE
while( $check == FALSE )
  if( -e $HOME/.EXPDIRroot ) then
         set EXPDIRroot = `cat $HOME/.EXPDIRroot`
  else
         set EXPDIRroot = $WRKDIR
  endif
  setenv EXPDIR_def $EXPDIRroot/$EXPID
  echo ""
  echo "Enter Desired Location for the ${C1}EXPERIMENT${CN} Directory"
  echo "Hit ENTER to use Default Location:"
  echo "----------------------------------"
  echo "Default:  ${C2}${EXPDIR_def}${CN}"
  set   NUEXPDIR  = $<
  if( .$NUEXPDIR != . ) then
       setenv EXPDIR     $NUEXPDIR
       setenv EXPDIR_def $NUEXPDIR
  else
       setenv EXPDIR  $EXPDIR_def
  endif
  
  if( "$EXPID" != `basename $EXPDIR` ) then
       echo "\!\! The ${C1}EXPERIMENT${CN} Directory MUST point to the ${C1}EXPID${CN}: ${C2}${EXPID}${CN}"
       echo " "
  else
      set check = TRUE
  endif
end
mkdir -p $EXPDIR

# Build EXP Root Directory
# ------------------------
@ n = 1
set root = `echo $EXPDIR | cut -d / -f$n`
while( .$root == . )
@ n = $n + 1
set root = `echo $EXPDIR | cut -d / -f$n`
end

set EXPDIRroot = ''
while( $root != $EXPID )
set EXPDIRroot = `echo ${EXPDIRroot}/${root}`
@ n = $n + 1
set root = `echo $EXPDIR | cut -d / -f$n`
end
if( -e $HOME/.EXPDIRroot ) /bin/rm $HOME/.EXPDIRroot
echo $EXPDIRroot > $HOME/.EXPDIRroot

#######################################################################
#                      Set Build Directory  
#######################################################################

set     GEOSDIR  = $GEOSDEF

#
# GEOSBIN does point to the bin/ directory in each
#
setenv GEOSSRC  ${GEOSDIR}
setenv GEOSBIN  ${GEOSDIR}/bin
setenv GEOSETC  ${GEOSDIR}/etc

setenv GEOSUTIL ${GEOSSRC}

#######################################################################
#                    Check for Group ID Sponsor Code
#         Note: Default GROUP will be kept in $HOME/.GROUProot
#######################################################################

       set GROUPS    = `groups`
if( -e $HOME/.GROUProot ) then
       set GROUProot = `cat $HOME/.GROUProot`
else
       set GROUProot = $GROUPS[1]
endif

echo " "
echo "Current GROUPS: ${GROUPS}"
echo "Enter your ${C1}GROUP ID${CN} for Current EXP: (Default: ${C2}${GROUProot}${CN})"
echo "-----------------------------------"
set   NUGROUP  = $<
if( .$NUGROUP != . ) then
     setenv GROUP $NUGROUP
else
     setenv GROUP $GROUProot
endif

if( -e $HOME/.GROUProot ) /bin/rm $HOME/.GROUProot
echo $GROUP > $HOME/.GROUProot

#######################################################################
#      Copy Model Executable and RC Files to Experiment Directory
#######################################################################

if ( $LINKX == "TRUE" ) then
  if ( -e $GEOSBIN/StandAlone_FV3_Dycore.x ) ln -s $GEOSBIN/StandAlone_FV3_Dycore.x $EXPDIR
else
  if ( -e $GEOSBIN/StandAlone_FV3_Dycore.x ) cp -v $GEOSBIN/StandAlone_FV3_Dycore.x $EXPDIR
endif

#######################################################################
#                      Create SETENV Commands
#######################################################################

/bin/rm -f $EXPDIR/SETENV.commands

if( $MPI == openmpi ) then

# This turns off an annoying warning when running
# Open MPI on a system where TMPDIRs are on a networked
# file system

cat > $EXPDIR/SETENV.commands << EOF
   setenv OMPI_MCA_shmem_mmap_enable_nfs_warning 0
EOF

# The below settings seem to be recommended for hybrid
# systems using MVAPICH2 but could change

else if( $MPI == mvapich2 ) then

cat > $EXPDIR/SETENV.commands << EOF
   #setenv MV2_ON_DEMAND_THRESHOLD 8192
   #setenv MV2_USE_SHMEM_ALLREDUCE 0
   #setenv MV2_USE_SHMEM_COLL      0
   #setenv MV2_USE_UD_HYBRID       0
   setenv MV2_ENABLE_AFFINITY     0
   setenv SLURM_DISTRIBUTION block
EOF

else if( $MPI == mpt ) then

cat > $EXPDIR/SETENV.commands << EOF

   setenv MPI_COLL_REPRODUCIBLE
   setenv SLURM_DISTRIBUTION block

   #setenv MPI_DISPLAY_SETTINGS 1
   #setenv MPI_VERBOSE 1
   
   unsetenv MPI_MEMMAP_OFF
   unsetenv MPI_NUM_MEMORY_REGIONS
   setenv MPI_XPMEM_ENABLED yes
   unsetenv SUPPRESS_XPMEM_TRIM_THRESH

   # For some reason, PMI_RANK is randomly set and interferes
   # with binarytile.x and other executables.
   unsetenv PMI_RANK

   # Often when debugging on MPT, the traceback from Intel Fortran
   # is "absorbed" and only MPT's errors are displayed. To allow the
   # compiler's traceback to be displayed, uncomment this environment
   # variable
   #setenv FOR_IGNORE_EXCEPTIONS false

EOF

else if( $MPI == intelmpi ) then

cat > $EXPDIR/SETENV.commands << EOF
#setenv MPS_STAT_LEVEL 4
setenv PSM2_MEMORY large
setenv PSM2_PATH_SELECTION adaptive
setenv I_MPI_ADJUST_GATHERV 3
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_EXTRA_FILESYSTEM 1
setenv I_MPI_EXTRA_FILESYSTEM_FORCE gpfs
setenv ROMIO_FSTYPE_FORCE "gpfs:"
#setenv I_MPI_DEBUG 6
EOF

endif # if mpi

#######################################################################
#               Create Local Scripts and Resource Files
#######################################################################

cat >      $EXPDIR/sedfile << EOF
/@SETENVS/ {
t success
: success
r $EXPDIR/SETENV.commands
d
}

/@GPUSTART/ {
t success1
: success1
r $EXPDIR/GPUSTART.commands
d
}

/@GPUEND/ {
t success2
: success2
r $EXPDIR/GPUEND.commands
d
}

s?@EXPID?$EXPID?g
s?@RUN_N?$RUN_N?g
s?@RUN_T?$RUN_T?g
s?@RUN_P?$RUN_P?g
s?@RUN_Q?$RUN_Q?g
s?@EXPDIR?$EXPDIR?g
s?@EXPDSC?$EXPDSC?g
s?@BATCH_GROUP?${BATCH_GROUP}${GROUP}?g
s?@BATCH_TIME?$BATCH_TIME?g
s?@BATCH_CMD?$BATCH_CMD?g
s?@BATCH_JOBNAME?$BATCH_JOBNAME?g
s?@BATCH_OUTPUTNAME?$BATCH_OUTPUTNAME?g
s?@BATCH_JOINOUTERR?$BATCH_JOINOUTERR?g
s?@SITE?$SITE?g
s?@GEOSDIR?$GEOSDIR?g
s?@GEOSSRC?$GEOSSRC?g
s?@GEOSBIN?$GEOSBIN?g
s?@GEOSETC?$GEOSETC?g
s?@GEOSUTIL?$GEOSUTIL?g

s^@AGCM_GRIDNAME^$AGCM_GRIDNAME^g

s^@BOOT^YES^g
s/@DT/$DT/g
s/@FV_NX/$FV_NX/g
s/@FV_NY/$FV_NY/g
s/@USE_SHMEM/$USE_SHMEM/g
s/@USE_IOSERVER/$USE_IOSERVER/g
s/@USE_NONHYDRO/$USE_NONHYDRO/g
s/@IOS_NDS/$IOS_NDS/g
s/@NCPUS_PER_NODE/$NCPUS_PER_NODE/g
s/@NUM_READERS/$NUM_READERS/g
s/@NUM_WRITERS/$NUM_WRITERS/g
s?>>>CUBE_AGCM<<<?$CUBE_AGCM?g
s/@GRID_TYPE/$GRID_TYPE/g
s/@AGCM_NF/$AGCM_NF/g
s/@AGCM_IM/$AGCM_IM/g
s/@AGCM_JM/$AGCM_JM/g
s/@AGCM_LM/$AGCM_LM/g
s/@HIST_IM/$HIST_IM/g
s/@HIST_JM/$HIST_JM/g

s?@CPEXEC?$CPEXEC?g
s?@TAREXEC?$TAREXEC?g

EOF

set FILES = "fv3.j"
set FILES = `echo $FILES`

echo " "

# Operate on files in ETCDIR

foreach FILE ($FILES)

   /bin/rm -f $EXPDIR/tmpfile
   /bin/rm -f $EXPDIR/$FILE

   if ( -e $BINDIR/$FILE ) then
      cat       $BINDIR/$FILE > $EXPDIR/tmpfile
   else if ( -e $ETCDIR/$FILE ) then
      cat       $ETCDIR/$FILE > $EXPDIR/tmpfile
   else
      echo "ERROR! Cannot find $FILE in $BINDIR or $ETCDIR!"
      exit 2
   endif


   sed -f $EXPDIR/sedfile $EXPDIR/tmpfile > $EXPDIR/$FILE

   echo "Creating ${C1}${FILE}${CN} for Experiment: $EXPID "
   chmod 755 $EXPDIR/$FILE

end

/bin/rm -f $EXPDIR/SETENV.commands

echo $EXPDIR > $EXPDIR/.EXPDIR
echo " "

#######################################################################
#                 Produce Final script and .rc files
#######################################################################

# Delete or Enable EXP Configuration Variables
# --------------------------------------------
    set FILES = "fv3.j"
foreach FILE ($FILES)

if( -e $EXPDIR/$FILE ) set LOCDIR = $EXPDIR

/bin/rm -f $LOCDIR/$FILE.tmp
/bin/mv -f $LOCDIR/$FILE $LOCDIR/$FILE.tmp
    cat    $LOCDIR/$FILE.tmp | awk '{ if ( $1  !~ "#DELETE") { print } }' > $LOCDIR/$FILE
/bin/rm -f $LOCDIR/$FILE.tmp

end

chmod   +x $EXPDIR/fv3.j

#######################################################################
#                       Echo Settings and Messages
#######################################################################

echo "Done!"
echo "-----"

#######################################################################
#                              Clean-Up
#######################################################################

if( -e $EXPDIR/tmpfile ) /bin/rm $EXPDIR/tmpfile
if( -e $EXPDIR/sedfile ) /bin/rm $EXPDIR/sedfile

exit

# ------------------------------------------
# Set a trap to remove the tempfiles on EXIT
# ------------------------------------------
TRAP:
   echo "Interrupt received, cleaning up temporary files"
   /bin/rm $FILES_TO_PROCESS $OLDEXPFILES $NEWEXPFILES $COPYSCRIPT $SEDFILE
   exit 1

CONTACTMATT:
cat <<EOF

It appears your environment is not set correctly to
run with the GPUs. If you wish to build and run the 
model using GPUs, please contact Matt Thompson at:
    
            matthew.thompson@nasa.gov

EOF
exit 1

SETCOLOR:
echo
echo "\033[1;4mGCM Setup Utility${RESET}"
echo
echo "Enter Desired Color Codes for ${BOLD}Highlighted${RESET} and ${BOLD}Default${RESET} text:"
echo
echo "${BOLD}Highlighted${RESET} Text Color: ${BLACK}0 BLACK${RESET}"
echo "                 Color: ${RED}1 RED${RESET}"
echo "                 Color: ${GREEN}2 GREEN${RESET}"
echo "                 Color: ${YELLOW}3 YELLOW${RESET}"
echo "                 Color: ${BLUE}4 BLUE${RESET}"
echo "                 Color: ${MAGENTA}5 MAGENTA${RESET}"
echo "                 Color: ${CYAN}6 CYAN${RESET}"
echo "                 Color: ${WHITE}7 WHITE${RESET}"
echo "                 Color: ${RESET}8 No Color"
set C1 = $<
  @ C1 = $C1 + 1
echo
echo "    ${BOLD}Default${RESET} Text Color: ${BLACK}0 BLACK${RESET}"
echo "                 Color: ${RED}1 RED${RESET}"
echo "                 Color: ${GREEN}2 GREEN${RESET}"
echo "                 Color: ${YELLOW}3 YELLOW${RESET}"
echo "                 Color: ${BLUE}4 BLUE${RESET}"
echo "                 Color: ${MAGENTA}5 MAGENTA${RESET}"
echo "                 Color: ${CYAN}6 CYAN${RESET}"
echo "                 Color: ${WHITE}7 WHITE${RESET}"
echo "                 Color: ${RESET}8 No Color"
set C2 = $<
  @ C2 = $C2 + 1
if( -e $HOME/.FV3SETUP ) /bin/rm -f $HOME/.FV3SETUP
touch $HOME/.FV3SETUP
echo $COLORS[$C1] >> $HOME/.FV3SETUP
echo $COLORS[$C2] >> $HOME/.FV3SETUP
exit 1

USAGE:
cat <<EOF
fv3_setup, a setup script for the GEOS-5 FV3 Standalone

   Usage: $0:t [optional flag]

   -c --color      Set the colors for $0:t
   -g --gpu        Run the model using the GPUs
   -h --help       Show usage

   If invoked alone, the script runs as normal.

   For more information, please contact Larry Takacs.

   Note: fv3_setup must be run on a machine with CVS access.

EOF
exit 1
