#!/bin/bash -x
#===	requested shell"
#PBS -S /bin/bash
#===	select queue
#PBS -q medium
#===	mail to user at abortion of job
#PBS -M stephan.henne@empa.ch
#===
#PBS -m e
#===	name of script
#PBS -N MAIOLICA2
#===	export all environment variables to nodes?
#PBS -V
#===	requested maximum time (estimate of single month run for FLEXPART times number of months)
#PBS -l walltime=72:00:00
#===    requested memory
#PBS -l mem=20Gb
#===	requested number of nodes and cores
#PBS -l nodes=1:ppn=16


#set verbose
###########################################################################
# Run simulations with FLEXPART_CTM, divided into periods.
#	This script writes run jobs that are than submitted to the queueing system.
# Therefore, the script should be run directly on the IPAZIA front end.
###########################################################################

###########################################################################
#	SETTINGS
###########################################################################
# directory where executable is located
EXE_DIR="/home/hes/svn_empa/FLEXPART80CTP/branches/nudging/"
# name of executable to use
EXE_NAME="FLEXPART80CTP_beta"
# directory where output should be stored in sub-folders for each month
MAIN_OUT_DIR="/mnt/project/hes/MAIOLICA2/output/NudgingNoBiasSingle_02"
# currently not tested: use the same as for MAIN_OUT_DIR
MAIN_BACK_DIR="/mnt/project/hes/MAIOLICA2/output/NudgingNoBiasSingle_02"
# directory where input files are stored
OPTIONS_DIR="/mnt/project/hes/MAIOLICA2/input/options/SingleNudging/"
# directory where wind fields reside
FIELDS_DIR="/mnt/project/hes/ecmwf_data/ERA_Interim/"
# path to wind fields AVAILABLE file
AVAILABLE_FN="/mnt/project/hes/ecmwf_data/ERA_Interim/AVAILABLE"

#	WHEN STARTING WITH THE FIRST MONTH RESTART SHOULD BE 0 (so that an 
# initialisation is started)  AND 1 FOR RESTARTED RUNS
RESTART=1

#	start time of simulation (YYYYMMDDhhmmss)
# (possible restart conditions or initial conditions are taken from output 
# sub-folder for this month)
START_DTM=20130101000000
#	end time of simulation (YYYYMMDDhhmmss)
END_DTM=20140101000000

#	Define how the simulation is divided into different FLEXPART runs. If BY_MONTH equal 
# to 1 each month is treated in a new FLEXPART run and the parameter STEP is 
# overwritten with the number of seconds in the current month
BY_MONTH=1

#	If BY_MONTH is set to 1 $STEP will be overwritten and contain the length of the 
# current month in seconds
STEP=86400
#   If OUT_STEP is set to 0 it will be overwritten with the length of the month in seconds,
#       hence, monthly output will be produced. This will only affect the grid output. 
#       Receptor output time step needs to be set in the COMMAND file and is not overwritten 
#       by this script.
OUT_STEP=86400

#	OMP STACKSIZE
STACKSIZE="100M"
#	OMP NUM OF THREADS
THREADS="16"
###########################################################################
#	END OF SETTINGS
###########################################################################

#   increase maximal allowed stack size (limited on hypatia)
ulimit -s unlimited

COMMAND_FN=${OPTIONS_DIR}/COMMAND

#	set JOB_ID
JOB_ID=FCTP{$$}

#	check directories
if [ ! -d $MAIN_OUT_DIR ]
then 
	mkdir $MAIN_OUT_DIR
fi

if [ $MAIN_BACK_DIR != $MAIN_OUT_DIR ] 
then 
	if [ ! -d $MAIN_BACK_DIR ] 
	then 
		 mkdir $MAIN_BACK_DIR
	fi 
fi

#	initial start
CUR_START=$START_DTM


##################################################
#	treatment of first month if not restarted
##################################################
if [ "$RESTART" -eq "0" ] 
then
	PREV_BACK_DIR="$MAIN_BACK_DIR/`echo $CUR_START | cut -c 1-8`"
	
	if [ $BY_MONTH -eq 0 ] 
	then
		CUR_END=`step_time $CUR_START $STEP`
	else 
		CUR_END=`step_time $CUR_START -month 1`
		STEP=`diff_time $CUR_END $CUR_START`
		echo $CUR_END $STEP
	fi

    if [ $CUR_START = "20060201000000" ] 
	then
    	CUR_START="20060203000000"
		STEP=`diff_time $CUR_END $CUR_START`
    fi
	CUR_OUT_DIR="$MAIN_OUT_DIR/`echo $CUR_END | cut -c 1-8`"
	CUR_BACK_DIR="$MAIN_BACK_DIR/`echo $CUR_END | cut -c 1-8`"
	if [ ! -d $CUR_OUT_DIR ] 
	then  
		mkdir $CUR_OUT_DIR
	fi
	if [ ! -d $CUR_BACK_DIR ] 
	then 
		mkdir $CUR_BACK_DIR
	fi

	#	clear directories
	rm -rf $CUR_OUT_DIR/*
	rm -rf $CUR_BACK_DIR/*

  	# copy gridded output fields (processed FLEXPAT output from previous runs; not direct FLEXPART output)
	cp $PREV_BACK_DIR/grid_ppbv*.bin $CUR_OUT_DIR
	
	# copy latitude profiles for initialisation
	cp $PREV_BACK_DIR/latitude_profile_* $CUR_OUT_DIR	

	#	adjust input files
	sed 13c\ "\   `echo $CUR_START | cut -c1-8` `echo $CUR_START | cut -c9-17`" $COMMAND_FN  \
        | sed 17c\ "\   `echo $CUR_END | cut -c1-8` `echo $CUR_END | cut -c9-17` "  \
        | sed 89c\ "\     ${RESTART}" | sed 121c\ "\   `echo $STEP`"  > tmp{$$}
	mv -f tmp{$$} $COMMAND_FN

    if [ "${OUT_STEP}" ==  "0" ]; then 
        sed 21c\ "\ `echo $STEP`" $COMMAND_FN | sed 25c\ "\ `echo $STEP`" > tmp{$$}
    else 
        sed 21c\ "\ `echo $OUT_STEP`" $COMMAND_FN | sed 25c\ "\ `echo $OUT_STEP`" > tmp{$$}
    fi 
	mv -f tmp{$$} $COMMAND_FN

	#	run FLEXPART_CTM 
	EXE="${EXE_DIR}/${EXE_NAME} ${OPTIONS_DIR}/ ${CUR_OUT_DIR}/ ${FIELDS_DIR} ${AVAILABLE_FN}"

	export OMP_NUM_THREADS=${THREADS}
	export OMP_STACKSIZE=${STACKSIZE}

	cd $EXE_DIR
	#	execute FLEXPART
	$EXE
	
	#	zip particle position output
	gzip $CUR_OUT_DIR/partposit_end

	#	remove binary fields of previous output
#	rm -f $CUR_OUT_DIR/grid_ppbv_*.bin
	
	#	zip grid conc output
#    for GRIDOUT in $CUR_OUT_DIR/grid_pptv_* $CUR_OUT_DIR/grid_conc_* 
#	do
#		gzip $GRIDOUT    
#	done

	if [ $MAIN_BACK_DIR != $MAIN_OUT_DIR ] 
	then 
		#	copy output to backup directory
		cp -rf $CUR_OUT_DIR/* $CUR_BACK_DIR

		#	remove current local output if copy was successful
		if [ $? -ne 0 ] 
		then 
			echo "Error when copying results to backup. Check status." | mail -s FLEXPART_CTP $RECIPIENT
		else 
			rm -rf $CUR_OUT_DIR
		fi
	fi


	#	before loop starts
	CUR_START=$CUR_END
	
	PREV_BACK_DIR=$CUR_OUT_DIR
	RESTART=1
else 
	PREV_BACK_DIR="$MAIN_BACK_DIR/`echo $START_DTM | cut -c 1-8`"
fi 
#	end of initial run

##################################################
#	advance time
##################################################
if [ $BY_MONTH -eq 0 ] 
then
	CUR_END=`step_time $CUR_START $STEP`
else 
	CUR_END=`step_time $CUR_START -month 1`
	#       work around missing wind fields on 2006-02-01
    if [ $CUR_START = "20060201000000" ] 
	then
    	CUR_START="20060203000000"
    fi
        #       end  of work around 
	STEP=`diff_time $CUR_END $CUR_START`
fi

##################################################
#	further restarts
##################################################
while [ `echo $CUR_END | cut -c1-8` -le `echo $END_DTM | cut -c1-8` ]
do 
	echo $CUR_START " - " $CUR_END 

	CUR_OUT_DIR="$MAIN_OUT_DIR/`echo $CUR_END | cut -c 1-8`"
	CUR_BACK_DIR="$MAIN_BACK_DIR/`echo $CUR_END | cut -c 1-8`"

	#	create directories
	if [ ! -d $CUR_OUT_DIR ]
	then 
 		mkdir $CUR_OUT_DIR
	fi

	#	clear directory
	rm -rf $CUR_OUT_DIR/*

	#	backup directory 
	if [ $MAIN_BACK_DIR != $MAIN_OUT_DIR ] 
	then 
		if [ ! -d $CUR_BACK_DIR ] 
		then 
			mkdir $CUR_BACK_DIR
		fi
		rm -rf $CUR_BACK_DIR/*
	fi

	echo "Current output: $CUR_OUT_DIR"

	##################################################
	#	copy previous output
	##################################################
	HEADER=$PREV_BACK_DIR/header
	if [ -f $HEADER ] 
	then
		cp -f $HEADER $CUR_OUT_DIR/header_previous
		echo "copy: " $HEADER "to" $CUR_OUT_DIR/header_previous
	fi 
    EMRES=$PREV_BACK_DIR/emission_residuals
	if [ -f $EMRES ] 
	then 
		cp -f $EMRES $CUR_OUT_DIR/emission_residuals
		echo "copy: " $EMRES "to" $CUR_OUT_DIR/emission_residuals
    fi
	PARTPOS=$PREV_BACK_DIR/partposit_end.gz
	if [ -f $PARTPOS ] 
	then 
		cp -f $PARTPOS $CUR_OUT_DIR/partposit_previous.gz
		echo "copy: " $PARTPOS "to" $CUR_OUT_DIR/partposit_previous.gz 
	#	unzip particle position output
		gunzip $CUR_OUT_DIR/partposit_previous.gz
	else 
		#	not zipped	
		PARTPOS=$PREV_BACK_DIR/partposit_end
		if [ -f $PARTPOS ] 
		then
			cp -f $PARTPOS $CUR_OUT_DIR/partposit_previous
			echo "copy: " $PARTPOS "to" $CUR_OUT_DIR/partposit_previous 
		else 
			cp $PREV_BACK_DIR/grid_ppbv*.bin $CUR_OUT_DIR
		fi
	fi 

	#	adjust input files
	sed 13c\ "\   `echo $CUR_START | cut -c1-8` `echo $CUR_START | cut -c9-17`" $COMMAND_FN  \
        | sed 17c\ "\   `echo $CUR_END | cut -c1-8` `echo $CUR_END | cut -c9-17` "  \
        | sed 89c\ "\     ${RESTART}" | sed 121c\ "\   `echo $STEP`"  > tmp{$$}
	mv -f tmp{$$} $COMMAND_FN

    if [ "${OUT_STEP}" ==  "0" ]; then 
        sed 21c\ "\ `echo $STEP`" $COMMAND_FN | sed 25c\ "\ `echo $STEP`" > tmp{$$}
    else 
        sed 21c\ "\ `echo $OUT_STEP`" $COMMAND_FN | sed 25c\ "\ `echo $OUT_STEP`" > tmp{$$}
    fi 
	mv -f tmp{$$} $COMMAND_FN

	#	run LORAT
	EXE="${EXE_DIR}/${EXE_NAME} ${OPTIONS_DIR}/ ${CUR_OUT_DIR}/ ${FIELDS_DIR} ${AVAILABLE_FN}"

	export OMP_NUM_THREADS=${THREADS}
	export OMP_STACKSIZE=${STACKSIZE}

	cd $EXE_DIR

	#	execute FLEXPART
	$EXE

	#	remove previous particle dump
	rm -f $CUR_OUT_DIR/partposit_previous

	rm -f $CUR_OUT_DIR/header_previous

	#	zip particle position output
	gzip $CUR_OUT_DIR/partposit_end

	#	remove binary fields of previous output
	rm -f $CUR_OUT_DIR/grid_ppbv_*.bin
	
#	#	zip grid conc output
#    for GRIDOUT in $CUR_OUT_DIR/grid_pptv_* $CUR_OUT_DIR/grid_conc_* 
#	do 
#		gzip $GRIDOUT    
#	done

	if [ $MAIN_BACK_DIR != $MAIN_OUT_DIR ] 
	then 
		#	copy output to backup directory
		cp -rf $CUR_OUT_DIR/* $CUR_BACK_DIR

		#	remove current local output if copy was successful
		if [ $? -ne 0 ] 
		then 
			echo "Error when copying results to backup. Check status." | mail -s FLEXPART_CTP $RECIPIENT
		else 
			rm -rf $CUR_OUT_DIR
		fi
	fi


	#	advance time
	CUR_START=$CUR_END
	PREV_BACK_DIR=$CUR_BACK_DIR	
	if [ $BY_MONTH = 0 ] 
	then
		CUR_END=`step_time $CUR_START $STEP`
	else 
		CUR_END=`step_time $CUR_START -month 1`
		#       work around missing wind fields on 2006-02-01
	    if [ $CUR_START = "20060201000000" ] 
		then
     		CUR_START="20060203000000"
	    fi
		#	 end of work around
		STEP=`diff_time $CUR_END $CUR_START`
		echo $CUR_END $STEP
	fi
done

echo "FLEXPART_CTP Run finished" | mail -s LORAT $RECIPIENT
