#!/bin/csh
#set verbose
###########################################################################
# Run simulations with FLEXPART_CTM, divided into periods.
#	This script writes run jobs that are than submitted to the queueing system.
# Therefore, the script should be run directly on the IPAZIA front end.
###########################################################################

###########################################################################
#	SETTINGS
###########################################################################
# directory where executable is located
set EXE_DIR =      "/home/hes134/FLEXPART80CTP"
# name of executable to use
set EXE_NAME = "FLEXPART80CTP"
# directory where output should be stored in sub-folders for each month
set MAIN_OUT_DIR =  "/mnt/storage/hes134/FLEXPART80CTM/output/URMEL_INI_32"
# obsolete
set MAIN_BACK_DIR = "/mnt/storage/hes134/FLEXPART80CTM/output/URMEL_INI_32"
# directory where input files are stored
set OPTIONS_DIR =  "/home/hes134/FLEXPART80CTP/input/URMEL_INI_31"
# directory where wind fields reside
set FIELDS_DIR =   "/mnt/storage/hes134/ecmwf_data/en/"
# path to wind fields AVAILABLE file
set AVAILABLE_FN = "/mnt/storage/hes134/ecmwf_data/en/AVAILABLE"

#	WHEN STARTING WITH THE FIRST MONTH RESTART SHOULD BE 0 (so that an 
# initialisation is started)  AND 1 FOR RESTARTED RUNS
set RESTART = 1

#	TO WHOM EMAIL MESSAGES SHOULD BE SEND
set RECIPIENT = "stephan.henne@empa.ch"

#	start time of simulation (YYYYMMDDhhmmss)
# (possible restart conditions or initial conditions are taken from output 
# sub-folder for this month)
set START_DTM = 20000101000000
#	end time of simulation (YYYYMMDDhhmmss)
set END_DTM =   20010101000000

#	Define how the simulation is divided into different FLEXPART runs. If BY_MONTH equal 
# to 1 each month is treated in a new FLEXPART run and the parameter STEP is 
# overwritten with the number of seconds in the current month
set BY_MONTH = 1

#	If BY_MONTH is set to 1 $STEP will be overwritten and contain the length of the 
# current month in seconds
set STEP = 86400

#	interval (s) for checking queue status
set SLEEP_INT = 30

#	settings for the queuing system
set WALLTIME = "20:00:00"
set MEM = "4096mb"
set QUEUE = "infinijazz"
set STACKSIZE = "270M"
set THREADS = "8" 
###########################################################################
#	END OF SETTINGS
###########################################################################

set COMMAND_FN =  ${OPTIONS_DIR}/COMMAND

#	set JOB_ID
set JOB_ID = FCTP{$$}

#	check directories
if ( ! -e $MAIN_OUT_DIR ) mkdir $MAIN_OUT_DIR
if ( $MAIN_BACK_DIR != $MAIN_OUT_DIR ) then 
	if ( ! -e $MAIN_BACK_DIR ) mkdir $MAIN_BACK_DIR
endif
#	initial start
set CUR_START = $START_DTM

##################################################
#	treatment of first month if not restarted
##################################################
if ( $RESTART == 0 ) then
	set PREV_BACK_DIR = "$MAIN_BACK_DIR/`echo $CUR_START | cut -c 1-8`"
	
	if ( $BY_MONTH == 0 ) then
		set CUR_END = `step_time $CUR_START $STEP`
	else 
		set CUR_END = `step_time $CUR_START -month 1`
		set STEP = `diff_time $CUR_END $CUR_START`
		echo $CUR_END $STEP
	endif

        if ( $CUR_START == "20060201000000" ) then
                set CUR_START = "20060203000000"
		set STEP = `diff_time $CUR_END $CUR_START`
        endif
	set CUR_OUT_DIR = "$MAIN_OUT_DIR/`echo $CUR_END | cut -c 1-8`"
	set CUR_BACK_DIR = "$MAIN_BACK_DIR/`echo $CUR_END | cut -c 1-8`"
	if ( ! -e $CUR_OUT_DIR ) mkdir $CUR_OUT_DIR
	if ( ! -e $CUR_BACK_DIR ) mkdir $CUR_BACK_DIR

	#	clear directories
	rm -rf $CUR_OUT_DIR/*
	rm -rf $CUR_BACK_DIR/*

  # copy gridded output fields (processed not direct FLEXPART output)
	cp $PREV_BACK_DIR/grid_ppbv*.bin $CUR_OUT_DIR
	
	# copy latitude profiles for initialisation
	cp $PREV_BACK_DIR/latitude_profile_* $CUR_OUT_DIR	

	#	adjust input files
	sed 13c\ "\   `echo $CUR_START | cut -c1-8` `echo $CUR_START | cut -c9-17`" $COMMAND_FN | sed 17c\ "\   `echo $CUR_END | cut -c1-8` `echo $CUR_END | cut -c9-17` " | sed 21c\ "\ `echo $STEP`" | sed 25c\ "\ `echo $STEP`" | sed 89c\ "\     ${RESTART}" | sed 121c\ "\   `echo $STEP`"  > tmp{$$}
	mv -f tmp{$$} $COMMAND_FN

	#	run FLEXPART_CTM 
	set EXE = "${EXE_DIR}/${EXE_NAME} ${OPTIONS_DIR}/ ${CUR_OUT_DIR}/ ${FIELDS_DIR} ${AVAILABLE_FN}"

	qsub -q $QUEUE  > tmp_id  << ENDEND
#!/bin/bash
#===	requested shell
#PBS -S /bin/bash
#===	mail to user
#PBS -M ${RECIPIENT}
#===
#PBS -m e
#===	name of script
#PBS -N $JOB_ID 
#===	export all environment variables to nodes?
#PBS -V
#===	requested maximum time
#PBS -l walltime=${WALLTIME}
#===    requested memory
#PBS -l mem=${MEM}
#===
#PBS -l nodes=1:ppn=${THREADS}

export OMP_NUM_THREADS=${THREADS}
export OMP_STACKSIZE=${STACKSIZE}

cd $EXE_DIR
$EXE

ENDEND

	#	copy IPAZIA job id
	set IP_ID = `cat tmp_id`
	rm -f tmp_id
	set IP_ID = `echo $IP_ID | cut -d "." -f 1`

	#	wait for job to be finished on node
	while ( `job_in_q $JOB_ID` != 0 )
		sleep $SLEEP_INT 
	end 

	#	zip particle position output
	gzip $CUR_OUT_DIR/partposit_end

	#	remove binary fields of previous output
#	rm -f $CUR_OUT_DIR/grid_ppbv_*.bin
	
	#	zip grid conc output
	gzip $CUR_OUT_DIR/grid_pptv_* 

	if ( $MAIN_BACK_DIR != $MAIN_OUT_DIR ) then 
		#	copy output to backup directory
		cp -rf $CUR_OUT_DIR/* $CUR_BACK_DIR

		#	remove current local output if copy was successful
		if ($status != 0) then 
			echo "Error when copying results to backup. Check status." | mail -s FLEXPART_CTP $RECIPIENT
		else 
			rm -rf $CUR_OUT_DIR
		endif
	endif

	#	email output files 
	set OUT_LOG = $JOB_ID.o$IP_ID
	set OUT_ERR = $JOB_ID.e$IP_ID
	gzip $OUT_LOG
	gzip $OUT_ERR

#	mail does not work with attachments anymore on new IPAZIA
#	set OUT_LOG = ${OUT_LOG}.gz
#	set OUT_ERR = ${OUT_ERR}.gz
#	echo "FLEXPART_CTM Results" | mail -s LORAT -a $OUT_LOG -a $OUT_ERR $RECIPIENT

	#	before loop starts
	set CUR_START = $CUR_END
	
	set PREV_BACK_DIR = $CUR_OUT_DIR
	set RESTART = 1
else 
	set PREV_BACK_DIR = "$MAIN_BACK_DIR/`echo $START_DTM | cut -c 1-8`"
endif 
#	end of initial run

##################################################
#	advance time
##################################################
if ( $BY_MONTH == 0 ) then
	set CUR_END = `step_time $CUR_START $STEP`
else 
	set CUR_END = `step_time $CUR_START -month 1`
	#       work around missing wind fields on 2006-02-01
        if ( $CUR_START == "20060201000000" ) then
                set CUR_START = "20060203000000"
        endif
        #       end  of work around 
	set STEP = `diff_time $CUR_END $CUR_START`
	echo $CUR_END $STEP $END_DTM
endif

##################################################
#	further restarts
##################################################
while ( `echo $CUR_END | cut -c1-8` <= `echo $END_DTM | cut -c1-8` ) 
	echo $CUR_START " - " $CUR_END 
	#	create directories
	set CUR_OUT_DIR = "$MAIN_OUT_DIR/`echo $CUR_END | cut -c 1-8`"
	set CUR_BACK_DIR = "$MAIN_BACK_DIR/`echo $CUR_END | cut -c 1-8`"
	if ( ! -e $CUR_OUT_DIR ) mkdir $CUR_OUT_DIR
	#	clear directory
	rm -rf $CUR_OUT_DIR/*

	if ( $MAIN_BACK_DIR != $MAIN_OUT_DIR ) then 
		if ( ! -e $CUR_BACK_DIR ) mkdir $CUR_BACK_DIR
		rm -rf $CUR_BACK_DIR/*
	endif

	echo $CUR_OUT_DIR

	##################################################
	#	copy previous output
	##################################################
	set HEADER = $PREV_BACK_DIR/header
	if ( -e $HEADER ) then
		cp -f $HEADER $CUR_OUT_DIR/header_previous
		echo "copy: " $HEADER "to" $CUR_OUT_DIR/header_previous
	endif 
	set PARTPOS = $PREV_BACK_DIR/partposit_end.gz
	if ( -e $PARTPOS ) then 
		cp -f $PARTPOS $CUR_OUT_DIR/partposit_previous.gz
		echo "copy: " $PARTPOS "to" $CUR_OUT_DIR/partposit_previous.gz 
	#	unzip particle position output
		gunzip $CUR_OUT_DIR/partposit_previous.gz
	else 
		set PARTPOS = $PREV_BACK_DIR/partposit_emd
		if ( -e $PARTPOS ) then
			cp -f $PARTPOS $CUR_OUT_DIR/partposit_previous
			echo "copy: " $PARTPOS "to" $CUR_OUT_DIR/partposit_previous 
		else 
			cp $PREV_BACK_DIR/grid_ppbv*.bin $CUR_OUT_DIR
		endif
	endif 

	#	adjust input files
	sed 13c\ "\   `echo $CUR_START | cut -c1-8` `echo $CUR_START | cut -c9-17`" $COMMAND_FN | sed 17c\ "\   `echo $CUR_END | cut -c1-8` `echo $CUR_END | cut -c9-17` " | sed 21c\ "\ `echo $STEP`" | sed 25c\ "\ `echo $STEP`" | sed 89c\ "\     ${RESTART}" | sed 121c\ "\   `echo $STEP`"  > tmp{$$}
	mv -f tmp{$$} $COMMAND_FN

	#	run LORAT
	set EXE = "${EXE_DIR}/${EXE_NAME} ${OPTIONS_DIR}/ ${CUR_OUT_DIR}/ ${FIELDS_DIR} ${AVAILABLE_FN}"

	qsub -q $QUEUE > tmp_id  << ENDEND
#!/bin/bash
#===	requested shell"
#PBS -S /bin/bash
#===	mail to user at abortion of job
#PBS -M ${RECIPIENT}
#PBS -m e
#===	name of script
#PBS -N $JOB_ID 
#===	export all environment variables to nodes?
#PBS -V
#===	requested maximum time
#PBS -l walltime=${WALLTIME}
#===    requested memory
#PBS -l mem=${MEM}
#PBS -l nodes=1:ppn=${THREADS}

export OMP_NUM_THREADS=${THREADS}
export OMP_STACKSIZE=${STACKSIZE}

cd $EXE_DIR
$EXE

ENDEND
	#	copy IPAZIA job id
	set IP_ID = `cat tmp_id`
	rm -f tmp_id
	set IP_ID = `echo $IP_ID | cut -d "." -f 1`

#	wait for job to be finished on node
	while ( `job_in_q $JOB_ID` != 0  )
		sleep $SLEEP_INT 
	end 

	#	remove previous particle dump
	rm -f $CUR_OUT_DIR/partposit_previous

	rm -f $CUR_OUT_DIR/header_previous

	#	zip particle position output
	gzip $CUR_OUT_DIR/partposit_end

	#	remove binary fields of previous output
	rm -f $CUR_OUT_DIR/grid_ppbv_*.bin
	
	#	zip grid conc output
	gzip $CUR_OUT_DIR/grid_pptv_* 

	if ( $MAIN_BACK_DIR != $MAIN_OUT_DIR ) then 
		#	copy output to backup directory
		cp -rf $CUR_OUT_DIR/* $CUR_BACK_DIR

		#	remove current local output if copy was successful
		if ($status != 0) then 
			echo "Error when copying results to backup. Check status." | mail -s FLEXPART_CTP $RECIPIENT
		else 
			rm -rf $CUR_OUT_DIR
		endif
	endif

	#	email output files 
	set OUT_LOG = $JOB_ID.o$IP_ID
	set OUT_ERR = $JOB_ID.e$IP_ID
	gzip $OUT_LOG
	gzip $OUT_ERR

#	mail does not work with attachments anymore on new IPAZIA
#	set OUT_LOG = ${OUT_LOG}.gz
#	set OUT_ERR = ${OUT_ERR}.gz
#	echo "FLEXPART_CTM Results" | mail -s LORAT -a $OUT_LOG -a $OUT_ERR $RECIPIENT
	
	#	advance time
	set CUR_START = $CUR_END
	set PREV_BACK_DIR = $CUR_BACK_DIR	
	if ( $BY_MONTH == 0 ) then
		set CUR_END = `step_time $CUR_START $STEP`
	else 
		set CUR_END = `step_time $CUR_START -month 1`
		#       work around missing wind fields on 2006-02-01
	        if ( $CUR_START == "20060201000000" ) then
        	        set CUR_START = "20060203000000"
	        endif
		#	 end of work around
		set STEP = `diff_time $CUR_END $CUR_START`
		echo $CUR_END $STEP
	endif
end
echo "FLEXPART_CTP Run finished" | mail -s LORAT $RECIPIENT
