#!/bin/csh
#set verbose
###########################################################################
# Run simulations with FLEXPART_CTM, divided into periods.
#	This script writes run jobs that are than submitted to the queueing system.
# Therefore, the script should be run directly on the IPAZIA front end.
###########################################################################

###########################################################################
#	SETTINGS
###########################################################################
# directory where executable is located
set EXE_DIR =      "/home/hes134/FLEXPART80CTM"
# directory where output should be stored in sub-folders for each month
set MAIN_OUT_DIR =  "/mnt/workspc/hes134/FLEXPART80CTM/output/URMEL"
# obsolete
set MAIN_BACK_DIR = "/mnt/storage/hes134/FLEXPART80CTM/output/URMEL"
# directory where input files are stored
set OPTIONS_DIR =  "/home/hes134/FLEXPART80CTM/input/URMEL"
# directory where wind fields reside
set FIELDS_DIR =   "/mnt/storage/hes134/ecmwf_data/en/"
# path to wind fields AVAILABLE file
set AVAILABLE_FN = "/mnt/storage/hes134/ecmwf_data/en/AVAILABLE"

#	WHEN STARTING WITH THE FIRST MONTH RESTART SHOULD BE 0 (so that an 
# initialisation is started)  AND 1 FOR RESTARTED RUNS
set RESTART = 0

#	TO WHOM EMAIL MESSAGES SHOULD BE SEND
set RECIPIENT = "stephan.henne@empa.ch"

#	start time of simulation (YYYYMMDDhhmmss)
# (possible restart conditions or initial conditions are taken from output 
# sub-folder for this month)
set START_DTM = 20040101000000
#	end time of simulation (YYYYMMDDhhmmss)
set END_DTM =   20040201000000

#	Define how the simulation is divided into different FLEXPART runs. If BY_MONTH equal 
# to 1 each month is treated in a new FLEXPART run and the parameter STEP is 
# overwritten with the number of seconds in the current month
set BY_MONTH = 1

#	If BY_MONTH is set to 1 $STEP will be overwritten and contain the length of the 
# current month in seconds
set STEP = 86400

#	interval (s) for checking queue status
set SLEEP_INT = 30

set WALLTIME = "72:00:00"
set MEM = "3072mb"

set QUEUE = "serial"
###########################################################################
#	END OF SETTINGS
###########################################################################

set COMMAND_FN =  ${OPTIONS_DIR}/COMMAND

#	set JOB_ID
set JOB_ID = FCTM{$$}

#	check directories
if ( ! -e $MAIN_OUT_DIR ) mkdir $MAIN_OUT_DIR
if ( ! -e $MAIN_BACK_DIR ) mkdir $MAIN_BACK_DIR

#	initial start
set CUR_START = $START_DTM

##################################################
#	treatment of first month if not restarted
##################################################
if ( $RESTART == 0 ) then
	set PREV_BACK_DIR = "$MAIN_BACK_DIR/`echo $CUR_START | cut -c 1-8`"
	
	if ( $BY_MONTH == 0 ) then
		set CUR_END = `step_time $CUR_START $STEP`
	else 
		set CUR_END = `step_time $CUR_START -month 1`
		set STEP = `diff_time $CUR_END $CUR_START`
		echo $CUR_END $STEP
	endif

	set CUR_OUT_DIR = "$MAIN_OUT_DIR/`echo $CUR_END | cut -c 1-8`"
	set CUR_BACK_DIR = "$MAIN_BACK_DIR/`echo $CUR_END | cut -c 1-8`"
	if ( ! -e $CUR_OUT_DIR ) mkdir $CUR_OUT_DIR
	if ( ! -e $CUR_BACK_DIR ) mkdir $CUR_BACK_DIR

	#	clear directories
	rm -rf $CUR_OUT_DIR/*
	rm -rf $CUR_BACK_DIR/*

  # copy gridded output fields (processed not direct FLEXPART output)
	cp $PREV_BACK_DIR/grid_ppbv*.bin $CUR_OUT_DIR
	
	# copy latitude profiles for initialisation
	cp $PREV_BACK_DIR/latitude_profile_* $CUR_OUT_DIR	

	#	adjust input files
	sed 13c\ "\   `echo $CUR_START | cut -c1-8` `echo $CUR_START | cut -c9-17`" $COMMAND_FN | sed 17c\ "\   `echo $CUR_END | cut -c1-8` `echo $CUR_END | cut -c9-17` " | sed 21c\ "\ `echo $STEP`" | sed 25c\ "\ `echo $STEP`" | sed 89c\ "\     ${RESTART}" | sed 121c\ "\   `echo $STEP`"  > tmp{$$}
	mv -f tmp{$$} $COMMAND_FN

	#	run FLEXPART_CTM 
	set EXE = "{$EXE_DIR}/FLEXPART80CTM {$OPTIONS_DIR}/ {$CUR_OUT_DIR}/ {$FIELDS_DIR} {$AVAILABLE_FN}"

	qsub -q $QUEUE  > tmp_id  << ENDEND
#!/bin/csh
#===	requested shell
#PBS -S /bin/csh
#===	mail to user
#PBS -M ${RECIPIENT}
#===
#PBS -m e
#===	name of script
#PBS -N $JOB_ID 
#===	export all environment variables to nodes?
#PBS -V
#===	requested maximum time
#PBS -l walltime=${WALLTIME}
#===    requested memory
#PBS -l mem=${MEM}
#===
#PBS -l nodes=1:ppn=1

cd ${EXE_DIR}
echo $EXE
$EXE
ENDEND

	#	copy IPAZIA job id
	set IP_ID = `cat tmp_id`
	rm -f tmp_id
	set IP_ID = `echo $IP_ID | cut -d "." -f 1`

	#	wait for job to be finished on node
	while ( `job_in_q $JOB_ID` != 0 )
		sleep $SLEEP_INT 
	end 

	#	zip particle position output
	gzip $CUR_OUT_DIR/partposit_end
	#	zip grid conc output
	gzip $CUR_OUT_DIR/grid_pptv_* 
	#	copy output to backup directory
	cp -rf $CUR_OUT_DIR/* $CUR_BACK_DIR

	#	remove current local output if copy was successful
	if ($status != 0) then 
		echo "Error when copying results to backup. Check status." | mail -s FLEXPART_CTM $RECIPIENT
	else 
		rm -rf $CUR_OUT_DIR
	endif

	#	email output files 
	set OUT_LOG = $JOB_ID.o$IP_ID
	set OUT_ERR = $JOB_ID.e$IP_ID
	gzip $OUT_LOG
	gzip $OUT_ERR

#	mail does not work with attachments anymore on new IPAZIA
#	set OUT_LOG = ${OUT_LOG}.gz
#	set OUT_ERR = ${OUT_ERR}.gz
#	echo "FLEXPART_CTM Results" | mail -s LORAT -a $OUT_LOG -a $OUT_ERR $RECIPIENT

	#	before loop starts
	set CUR_START = $CUR_END
	
	set PREV_BACK_DIR = $CUR_OUT_DIR
else 
	set PREV_BACK_DIR = "$MAIN_BACK_DIR/`echo $START_DTM | cut -c 1-8`"
endif 
#	end of initial run

##################################################
#	advance time
##################################################
if ( $BY_MONTH == 0 ) then
	set CUR_END = `step_time $CUR_START $STEP`
else 
	set CUR_END = `step_time $CUR_START -month 1`
	set STEP = `diff_time $CUR_END $CUR_START`
	echo $CUR_END $STEP $END_DTM
endif

##################################################
#	further restarts
##################################################
while ( `echo $CUR_END | cut -c1-8` <= `echo $END_DTM | cut -c1-8` ) 
	echo $CUR_START " - " $CUR_END 
	#	create directories
	set CUR_OUT_DIR = "$MAIN_OUT_DIR/`echo $CUR_END | cut -c 1-8`"
	set CUR_BACK_DIR = "$MAIN_BACK_DIR/`echo $CUR_END | cut -c 1-8`"
	if ( ! -e $CUR_OUT_DIR ) mkdir $CUR_OUT_DIR
	if ( ! -e $CUR_BACK_DIR ) mkdir $CUR_BACK_DIR

	echo $CUR_OUT_DIR
	#	clear directory
	rm -rf $CUR_OUT_DIR/*
	rm -rf $CUR_BACK_DIR/*

	##################################################
	#	copy previous output
	##################################################
	set HEADER = $PREV_BACK_DIR/header
	if ( -e $HEADER ) then
		cp -f $HEADER $CUR_OUT_DIR/header_previous
		echo "copy: " $HEADER "to" $CUR_OUT_DIR/header_previous
	endif 
	set PARTPOS = $PREV_BACK_DIR/partposit_end.gz
	if ( -e $PARTPOS ) then 
		cp -f $PARTPOS $CUR_OUT_DIR/partposit_previous.gz
		echo "copy: " $PARTPOS "to" $CUR_OUT_DIR/partposit_previous.gz 
	#	unzip particle position output
		gunzip $CUR_OUT_DIR/partposit_previous.gz
	else 
		set PARTPOS = $PREV_BACK_DIR/partposit_emd
		if ( -e $PARTPOS ) then
			cp -f $PARTPOS $CUR_OUT_DIR/partposit_previous
			echo "copy: " $PARTPOS "to" $CUR_OUT_DIR/partposit_previous 
		else 
			cp $PREV_BACK_DIR/grid_ppbv*.bin $CUR_OUT_DIR
		endif
	endif 

	#	adjust input files
	sed 13c\ "\   `echo $CUR_START | cut -c1-8` `echo $CUR_START | cut -c9-17`" $COMMAND_FN | sed 17c\ "\   `echo $CUR_END | cut -c1-8` `echo $CUR_END | cut -c9-17` " | sed 21c\ "\ `echo $STEP`" | sed 25c\ "\ `echo $STEP`" | sed 89c\ "\     ${RESTART}" | sed 121c\ "\   `echo $STEP`"  > tmp{$$}
	mv -f tmp{$$} $COMMAND_FN

	#	run LORAT
	set EXE = "{$EXE_DIR}/FLEXPART80CTM {$OPTIONS_DIR}/ {$CUR_OUT_DIR}/ {$FIELDS_DIR} {$AVAILABLE_FN}"

	qsub -q $QUEUE > tmp_id  << ENDEND
#!/bin/csh
#===	requested shell"
#PBS -S /bin/csh
#===	mail to user at abortion of job
#PBS -M ${RECIPIENT}
#PBS -m e
#===	name of script
#PBS -N $JOB_ID 
#===	export all environment variables to nodes?
#PBS -V
#===	requested maximum time
#PBS -l walltime=${WALLTIME}
#===    requested memory
#PBS -l mem=${MEM}
#PBS -l nodes=1:ppn=1

cd ${EXE_DIR}
$EXE
ENDEND
	
	#	copy IPAZIA job id
	set IP_ID = `cat tmp_id`
	rm -f tmp_id
	set IP_ID = `echo $IP_ID | cut -d "." -f 1`

#	wait for job to be finished on node
	while ( `job_in_q $JOB_ID` != 0  )
		sleep $SLEEP_INT 
	end 

	#	remove previous particle dump
	rm -f $CUR_OUT_DIR/partposit_previous*

	#	zip particle position output
	gzip $CUR_OUT_DIR/partposit_end

	#	remove binary fields of previous output
	rm -f $CUR_OUT_DIR/grid_ppbv_*.bin
	
	#	zip grid conc output
	gzip $CUR_OUT_DIR/grid_pptv_* 

	#	copy output to backup directory
	cp -rf $CUR_OUT_DIR/* $CUR_BACK_DIR

	#	remove current local output if copy was successful
	if ($status != 0) then 
		echo "Error when copying results to backup. Check status." | mail -s FLEXPART_CTM $RECIPIENT
	else 
		rm -rf $CUR_OUT_DIR
	endif

	#	email output files 
	set OUT_LOG = $JOB_ID.o$IP_ID
	set OUT_ERR = $JOB_ID.e$IP_ID
	gzip $OUT_LOG
	gzip $OUT_ERR

#	mail does not work with attachments anymore on new IPAZIA
#	set OUT_LOG = ${OUT_LOG}.gz
#	set OUT_ERR = ${OUT_ERR}.gz
#	echo "FLEXPART_CTM Results" | mail -s LORAT -a $OUT_LOG -a $OUT_ERR $RECIPIENT
	
	
	#	advance time
	set CUR_START = $CUR_END
	set PREV_BACK_DIR = $CUR_OUT_DIR	
	if ( $BY_MONTH == 0 ) then
		set CUR_END = `step_time $CUR_START $STEP`
	else 
		set CUR_END = `step_time $CUR_START -month 1`
		set STEP = `diff_time $CUR_END $CUR_START`
		echo $CUR_END $STEP
	endif
end
echo "FLEXPART_CTM Run finished" | mail -s LORAT $RECIPIENT
