#!/bin/csh -fx

################################################################
#
# !DESCRIPTION: Propagate analysis increment with Oseledec 
#               operator.
#
# !REVISION HISTORY:
#
#  13Feb2008  Todling  Initial code
#  03Apr2008  Todling  Add correction term due to IAU
#  05Apr2008  Todling  Verify with assimilation instead of ana
#  03Oct2008  Todling  Added acquire job request for discover runs (what a mess!)
#  04Mar2009  Todling  Change hdf suffixes to nc4; use esma_mpirun
#  02Aug2011  Stassi   Use getsponsor.pl wrapper script
################################################################

set fcstdir = $1
set expid   = $2
set nymde0  = $3
set nhmse0  = $4
set tauf_sc = $5

#
# Echo environment to make sure all is well
# -----------------------------------------
#env
set myname = "fvoseledec"
set nymde  = $nymde0
set nhmse  = $nhmse0

# WARNING: Must specify appropriate FVINPUT for your resolution
# -------------------------------------------------------------
if ( ! $?FVINPUT ) then
     echo " Invalid path for SVEC bcs "
     exit 1
endif
if ( !($?BATCH_SUBCMD) ) setenv BATCH_SUBCMD "sbatch"
if ( !($?CORRECT4IAU) ) setenv CORRECT4IAU 0
if ( !($?VEXPID) ) setenv VEXPID $EXPID


if( -d $fcstdir/oseledec.$nymde.$nhmse ) /bin/rm -r $fcstdir/oseledec.$nymde.$nhmse
mkdir $fcstdir/oseledec.$nymde.$nhmse
cd    $fcstdir/oseledec.$nymde.$nhmse


# Establish link to trajectory/resource files
# -------------------------------------------
  ln -sf $fcstdir/*.traj.* .
  ln -sf $fcstdir/*.ptrj.* .
  ln -sf $fcstdir/oseledec.rc .
  ln -sf $fcstdir/fvpsas.rc  .
  ln -sf $fcstdir/ana.acq .

# Get namelists, resource files, etc ...
# --------------------------------------
  cp $fcstdir/fvoseledec.ccmrun.namelist.tmpl ccmrun.namelist.tmpl
  cp $fcstdir/fvgcm.ccmflags.namelist         ccmflags.namelist

#
# Compute begin time for adjoint integration
# ------------------------------------------
  set taub  = `echorc.x -rc ccmrun.namelist.tmpl integration_length_hrs`
  @ nday    =  $taub / 24
  if($nday < 1) set nday = 0
  @ taub_sc =  $taub * 3600

  if ( $taub_sc > $tauf_sc ) then
     echo FATAL ERROR: Adjoint forecast length exceeds trajectory
     echo forward $tauf_sc secs,   backward $taub_sc secs
     exit 1
  endif

# Determine actual start date/time of integration
# -----------------------------------------------
  set tlag  = `echorc.x -rc ccmrun.namelist.tmpl integration_time_lag_start_hrs`
    @ ntp1  = $#tlag + 1

@ ic = 1
while ( $ic < $ntp1 ) 

  @ tlag_sc =  $tlag[$ic] * 3600

  set buf  = `tick $nymde0 $nhmse0 -$tlag_sc`
  set nymde  =  $buf[1]
  set nhmse  =  $buf[2]
  set nhe    = `echo $nhmse | cut -c 1-2`

  set buf  = `tick $nymde $nhmse -$taub_sc`
  set nymdb  =  $buf[1]
  set nhmsb  =  $buf[2]
  set nhb    = `echo $nhmsb | cut -c 1-2`

#
# Fill in namelist template for this experiment
# ---------------------------------------------
  rm -f sed_file
  echo "s/>>>EXPID<<</$EXPID/1"    > sed_file
  echo "s/>>>NYMDB<<</$nymdb/1"   >> sed_file
  echo "s/>>>NHMSB<<</$nhmsb/1"   >> sed_file
  echo "s/>>>NYMDE<<</$nymde/1"   >> sed_file
  echo "s/>>>NHMSE<<</$nhmse/1"   >> sed_file
  echo "s/>>>NDAY<<</$nday/1"     >> sed_file

  /bin/rm -f ./ccmrun.namelist
  sed -f sed_file  ./ccmrun.namelist.tmpl > ./ccmrun.namelist


# Strip out f95 style comments from namelists
# -------------------------------------------
  cut -f1 -d!  ccmrun.namelist     | tee          fort.811
  cut -f1 -d!  ccmflags.namelist   | tee          fort.813


#
#        ----------------------------------------------------------------------------
#         PART I - Create initial condition for Oselect propagation of ana increments
#        ----------------------------------------------------------------------------
                                                                                               
# Make sure trajectory file is present at inital/final time of integration
# ------------------------------------------------------------------------
  set trajtmpl   = `grep -i trajtmpl ccmrun.namelist|awk '{printf "%s", $3}' | cut -d"'" -f2 | cut -d"'" -f3`
  set begin_traj = `echorc.x -template $EXPID $nymdb $nhmsb -fill $trajtmpl`
  if ( ! -e $begin_traj ) then
       echo "fvoseledec: initial trajectory not found, cannot proceed "
       exit 1
  endif
 
# Generate d_rst from appropriate trajectory file
# -----------------------------------------------
  echo $begin_traj
  ln -sf $begin_traj rst.nc4

# Determine dimensions of required BCs and static ICs; do error checking
# ----------------------------------------------------------------------
  set tres  = `getgfiodim.x $begin_traj`

  set resol = "none"
  if ( $tres[1] ==  72 && $tres[2] ==  46 ) set resol = "a"
  if ( $tres[1] == 144 && $tres[2] ==  91 ) set resol = "b"
  if ( $tres[1] == 288 && $tres[2] == 181 ) set resol = "c"
  if ( $tres[1] == 576 && $tres[2] == 361 ) set resol = "d"
  if ( $resol == "none" ) then
       echo "fvoseledec: unknown resolution "
       exit 1
  endif

# Determine dimension of forecast fields (from prognostic file or background)
# ---------------------------------------------------------------------------
  set hires_full = `grep bkg.eta ana.acq`
  set hires_temp = $hires_full:t
  set hires_bkg  = `echorc.x -template $EXPID $nymdb $nhmsb -fill $hires_temp`
  if ( ! -e $hires_bkg ) then
      
      if (`uname -n` !~ borg*) then

         acquire -v -rc ana.acq -d . -strict $nymdb $nhmsb 060000 1 # Acquire background
                                                                    # ------------------

      else   # acquire using batch job (discover)
             # ----------------------------------

       set qsub_acquire = 0
       set fname = "acquire.pbs"
       alias fname1 "echo \!* >! $fname"
       alias fname2 "echo \!* >> $fname"

        set qsub_acquire = 1
        if ($?gid) then
           set gsflg = "-grpID $gid"
        else
           set gsflg = "-dflt"
        endif
        set GID = `getsponsor.pl $gsflg`

        fname1 "#\!/bin/csh -xvf"
        fname2 "#SBATCH --account=$GID"
        fname2 "#SBATCH --job-name=acquire"
        fname2 "#SBATCH --output=acquire.log.o%j"
        fname2 "#SBATCH --ntasks=1"
        fname2 "#SBATCH --time=1:00:00"
        fname2 "#SBATCH --partition=datamove"
        fname2 "#PBS -W group_list=$GID"
        fname2 "#PBS -N acquire"
        fname2 "#PBS -o acquire.log.o%j"
        fname2 "#PBS -l nodes=1:ppn=1"
        fname2 "#PBS -l walltime=1:00:00"
        fname2 "#PBS -q datamove"
        fname2 "#PBS -S /bin/csh"
        fname2 "#PBS -V"
        fname2 "#PBS -j eo"
        fname2 ""
        fname2 "cd $SENSWDIR"
        fname2 " acquire -v -rc ana.acq -d . -strict $nymdb $nhmsb 060000 1 "
        fname2 "exit"

        if ( $BATCH_SUBCMD == "sbatch" ) then
           sbatch -W $fname
        else
        qsub -W block=true $fname
        endif

      endif # end of aquire (discover)

      if ( ! -e $hires_bkg[1] ) then
        echo "fvoseledec: background file not found, cannot proceed "
        exit 1
      endif

  endif
  set verify_bkg = $hires_bkg[1]
  set  bres = `getgfiodim.x $verify_bkg`   # resolution of background

                              set halt   = 0
                              set interp = 0
  if ( $bres[1] != $tres[1] ) set interp = 1
  if ( $bres[2] != $tres[2] ) set interp = 1
  if ( $bres[3] != $tres[3] ) set halt   = 1
  if ( $halt ) then
       echo "fvoseledec: unacceptable resolution"
       exit 1
  endif

  /bin/rm -f verify_bkg.nc4
  if ( $interp ) then
    dyn2dyn.x -g5 -o verify_bkg.nc4 -res $resol -nlevs $bres[3] -geos4 -pick $nymdb $nhmsb $verify_bkg
  else
    dyn2dyn.x -g5 -o verify_bkg.nc4 -pick $nymdb $nhmsb $verify_bkg
  endif


# Acquire corresponding analysis
# ------------------------------
  set hires_full = `grep ana.eta ana.acq`
  set hires_temp = $hires_full:t
  set hires_anal = `echorc.x -template $EXPID $nymdb $nhmsb -fill $hires_temp`
  if ( ! -e $hires_anal ) then

      if (`uname -n` !~ borg*) then

         acquire -v -rc ana.acq -d . -strict $nymdb $nhmsb 060000 1

      else   # acquire using batch job (discover)
             # ----------------------------------

       set qsub_acquire = 0
       set fname = "acquire.pbs"
       alias fname1 "echo \!* >! $fname"
       alias fname2 "echo \!* >> $fname"

        set qsub_acquire = 1
        if ($?gid) then
           set gsflg = "-grpID $gid"
        else
           set gsflg = "-dflt"
        endif
        set GID = `getsponsor.pl $gsflg`

        fname1 "#\!/bin/csh -xvf"
        fname2 "#SBATCH --account=$GID"
        fname2 "#SBATCH --job-name=acquire"
        fname2 "#SBATCH --output=acquire.log.o%j"
        fname2 "#SBATCH --ntasks=1"
        fname2 "#SBATCH --time=1:00:00"
        fname2 "#SBATCH --partition=datamove"
        fname2 "#PBS -W group_list=$GID"
        fname2 "#PBS -N acquire"
        fname2 "#PBS -o acquire.log.o%j"
        fname2 "#PBS -l nodes=1:ppn=1"
        fname2 "#PBS -l walltime=1:00:00"
        fname2 "#PBS -q datamove"
        fname2 "#PBS -S /bin/csh"
        fname2 "#PBS -V"
        fname2 "#PBS -j eo"
        fname2 ""
        fname2 "cd $SENSWDIR"
        fname2 " acquire -v -rc ana.acq -d . -strict $nymdb $nhmsb 060000 1 "
        fname2 "exit"

        if ( $BATCH_SUBCMD == "sbatch" ) then
           sbatch -W $fname
        else
        qsub -W block=true $fname
        endif

      endif # end of aquire (discover)
      if ( ! -e $hires_anal[1] ) then
        echo "fvoseledec: verifying analysis not found, cannot proceed "
        exit 1
      endif
  endif

# Check resolution of verifying analysis and map if needed
# --------------------------------------------------------
  set  ares = `getgfiodim.x $hires_anal[1]`    # resolution of veryfying analysis

                              set halt   = 0
                              set interp = 0
  if ( $ares[1] != $tres[1] ) set interp = 1
  if ( $ares[2] != $tres[2] ) set interp = 1
  if ( $ares[3] != $tres[3] ) set   halt = 1
  if ( $halt ) then
       echo "fvoseledec: unacceptable resolution"
       exit 1
  endif

  if ( $interp ) then
    dyn2dyn.x -g5 -o verify_anal.nc4 -res $resol -nlevs $ares[3] -geos4 -pick $nymdb $nhmsb $hires_anal[1]
  else
    /bin/mv $hires_anal[1] verify_anal.nc4
  endif


# Correct due to presence of IAU
# ------------------------------
  if ( $CORRECT4IAU ) then

      set prog_file   = `echorc.x -template $EXPID $nymdb $nhmsb upper-air_prog_filename`
      set verify_fcst = $fcstdir/$prog_file
      set fres = `getgfiodim.x $verify_fcst`   # resolution of verifying forecast

                                  set halt   = 0
                                  set interp = 0
      if ( $fres[1] != $tres[1] ) set interp = 1
      if ( $fres[2] != $tres[2] ) set interp = 1
      if ( $fres[3] != $tres[3] ) set halt   = 1
      if ( $halt ) then
           echo "fvoseledec: unacceptable resolution"
           exit 1
      endif

      /bin/rm -f verify_bkg.nc4
      if ( $interp ) then
        dyn2dyn.x -g5 -o verify_fcst.nc4 -res $resol -nlevs $fres[3] -geos4 -pick $nymdb $nhmsb $verify_fcst
      else
        dyn2dyn.x -g5 -o verify_fcst.nc4 -pick $nymdb $nhmsb $verify_fcst
      endif

#     Construct difference vector between background and forecast at bkg time
#                     del = delx - delz = (a-b) + (b-f) = a-f
#     -----------------------------------------------------------------------
      dyndiff.x -g5 -o fvpert.eta.nc4 verify_anal.nc4 verify_fcst.nc4

  else

#     Generate analysis increment (or diff between analysis and forecast)
#     -------------------------------------------------------------------
      dyndiff.x -g5 -o ainc.nc4 verify_anal.nc4 verify_bkg.nc4
      /bin/ln -sf ainc.nc4 fvpert.eta.nc4                  # filename w/ IC for fvpert.x
  endif # < CORRECT4IAU >


#                ---------------------------------------------------------------------------
#                 PART II - Run Tangent Linear Model and Prepear IC for Adjoint Integration
#                ---------------------------------------------------------------------------


# NOTE: p_rst is fixed for ADM/TLM
# --------------------------------
  set im = $tres[1]
  set jm = $tres[2]
  cp $FVINPUT/rs/g4fixedIC/${im}x${jm}/p_rst   .

  ln -s $FVINPUT/fvgcm/${im}x${jm}/o3.amip2_uars_fub_${jm}x55.nc  g4ozdata.nc
  ln -s $FVINPUT/fvgcm/${im}x${jm}/RandelH2O_${jm}x25.bin         g4h2odata.bin
  ln -s $FVINPUT/fvgcm/${im}x${jm}/SSTM5079_${im}x${jm}.nc        g4sst.nc
 
# Execute program for adjoint sensitivity calculation
# ---------------------------------------------------
  unsetenv KMP_LIBRARY

# Propagate perturbation forward
# ------------------------------
  set rcName = oseledec.rc
  set norms  = `echorc.x -rc $rcName pert_norm`
  set nnorms = $#norms

# Generate d_rst from appropriate trajectory file
# -----------------------------------------------
  echo "Initial trajectory point: $begin_traj"
  ln -sf $begin_traj rst.nc4

  ls -lrt
  /bin/rm -f status.log
  set pertout = $expid.pertinc.eta.${nymde}_${nhe}z.nc4
  set fvpert_exe = `which fvpert.x`
  $ADMRUN_OPT_BEGIN $fvpert_exe -g5 -o $expid.pertinc.eta

  if ( ! -e $pertout ) then
       echo "fvoseledec: initial pert not propagated by TLM"
       exit 1
  endif

# Acquire verifying analysis
# --------------------------
  set averif   = "asm.eta" # if wanted for research, can be set to ana.eta
  set hires_full = `grep $averif ana.acq`
  if ( $EXPID == $VEXPID ) then
       set hires_temp = $hires_full:t
       set hires_anal = `echorc.x -template $EXPID $nymde $nhmse -fill $hires_temp`
  else
       set hires_temp = $hires_full:t
       set hires_anal = `echorc.x -template $VEXPID $nymde $nhmse -fill $hires_temp`
  endif
  if ( ! -e $hires_anal ) then

      if (`uname -n` !~ borg*) then

          acquire -v -rc ana.acq -d . -strict $nymde $nhmse 060000 1

      else   # acquire using batch job (discover)
             # ----------------------------------

          set qsub_acquire = 0
          set fname = "acquire.pbs"
          alias fname1 "echo \!* >! $fname"
          alias fname2 "echo \!* >> $fname"

          set qsub_acquire = 1
          if ($?gid) then
             set gsflg = "-grpID $gid"
          else
             set gsflg = "-dflt"
          endif
          set GID = `getsponsor.pl $gsflg`

          fname1 "#\!/bin/csh -xvf"
          fname2 "#SBATCH --account=$GID"
          fname2 "#SBATCH --job-name=acquire"
          fname2 "#SBATCH --output=acquire.log.o%j"
          fname2 "#SBATCH --ntasks=1"
          fname2 "#SBATCH --time=1:00:00"
          fname2 "#SBATCH --partition=datamove"
          fname2 "#PBS -W group_list=$GID"
          fname2 "#PBS -N acquire"
          fname2 "#PBS -o acquire.log.o%j"
          fname2 "#PBS -l nodes=1:ppn=1"
          fname2 "#PBS -l walltime=1:00:00"
          fname2 "#PBS -q datamove"
          fname2 "#PBS -S /bin/csh"
          fname2 "#PBS -V"
          fname2 "#PBS -j eo"
          fname2 ""
          fname2 "cd $SENSWDIR"
          fname2 " acquire -v -rc ana.acq -d . -strict $nymde $nhmse 060000 1 "
          fname2 "exit"

          if ( $BATCH_SUBCMD == "sbatch" ) then
             sbatch -W $fname
          else
          qsub -W block=true $fname
          endif

      endif # end of aquire (discover)

      if ( ! -e $hires_anal[1] ) then
        echo "fvoseledec: verifying analysis not found, cannot proceed "
        exit 1
      endif

  endif

# Check resolution of verifying analysis and map if needed
# --------------------------------------------------------
  set  ares = `getgfiodim.x $hires_anal[1]`    # resolution of veryfying analysis

                              set halt   = 0
                              set interp = 0
  if ( $ares[1] != $tres[1] ) set interp = 1
  if ( $ares[2] != $tres[2] ) set interp = 1
  if ( $ares[3] != $tres[3] ) set   halt = 1
  if ( $halt ) then
       echo "fvoseledec: unacceptable resolution"
       exit 1
  endif

  /bin/rm verify_anal.nc4
  if ( $interp ) then
    dyn2dyn.x -g5 -o verify_anal.nc4 -res $resol -nlevs $ares[3] -geos4 -pick $nymde $nhmse $hires_anal[1]
  else
    /bin/mv $hires_anal[1] verify_anal.nc4
  endif

# Normalize evolved analysis increment as desired
# -----------------------------------------------
  set initadjx = `which initadj.x`
  esma_mpirun -np 1 $initadjx -g5 -pick $nymde $nhmse -rc $rcName $pertout verify_anal.nc4

#                ----------------------------------------------------------------
#                 PART III - Run Adjoint Model (for each available perturbation)
#                ----------------------------------------------------------------

# Loop over number of available gradient vectors
# ----------------------------------------------
  @ ng = 1
  while ( $ng <= $nnorms )

#   Normalize perturbation
#   ----------------------
    if ( ! -e Jgradf_${norms[$ng]}.eta.nc4 ) then
        echo "fvoseledec: error, expecting to find fcst gradient for $norms[$ng], but did not"
        exit 1
    else
        /bin/rm  -f Jgradf.eta.nc4
        /bin/ln -sf Jgradf_${norms[$ng]}.eta.nc4  Jgradf.eta.nc4
    endif

#   Propagate perturbation backward
#   -------------------------------
    ls -lrt
    /bin/rm -f status.log
    set fvsens_exe = `which fvsens.x`
    $ADMRUN_OPT_BEGIN $fvsens_exe -g5

#   Check whether we ran without errors
#   -----------------------------------
    grep 'Normal Execution.' status.log >& /dev/null
    if ( ${status} ) then
      /bin/rm -f          ${FVHOME}/run/*.abnormal.log
      echo $myname": saving  abnormal logs to ${FVHOME}/run"
      echo $myname": abnormal exit from fvpsas, bye, bye..."
      exit 2
    endif

#   Move output files from sens program to main directory
#   -----------------------------------------------------
    set myfile = `ls -1 *.fsens.*`
    set prefix = `echo $myfile | cut -d. -f1-2`
    set suffix = `echo $myfile | cut -d. -f3-`
    /bin/mv $myfile $fcstdir/${prefix}ainc_${norms[$ng]}.$suffix
 
#   Rename Jgradf.eta.nc4 for archiving
#   -----------------------------------
    /bin/mv Jgradf_${norms[$ng]}.eta.nc4 ${fcstdir}/${EXPID}.Jgradfainc_${norms[$ng]}.eta.${nymde}_${nhe}z.nc4

    @ ng++
  end  # n gradients
  /bin/mv Jnormf.txt  ${fcstdir}/${EXPID}.Jnormfainc.${nymde}_${nhe}z.txt

  @ ic++

end  # < cases >
