#!/bin/bash -l
#PBS -N King_B_large_domain
#PBS -o ${PBS_JOBNAME}${PBS_JOBID}.out
#PBS -e ${PBS_JOBNAME}${PBS_JOBID}.err
#PBS -l walltime=180:20:00
#PBS -l nodes=5:ppn=20:esd1
#PBS -q esd1
##PBS -l pmem=150gb
#=================================================================================================================
module load chains/INTEL-17.0
module load compiler/intel/17.0
module load mpi/impi/2017
module load numlib/mkl/2017.6.256


export ELMER_HOME="/home-link/epivj01/INSTALL/ElmerInstall_all/elmerice/Elmer_devel_05-11-20"
export ELMER_SOLVER_HOME="$ELMER_HOME/bin"

# export PATH=/home-link/epivj01/INSTALL/ElmerInstall_all/elmerice/Elmer_devel_05-11-20/bin:$PATH
# export LD_LIBRARY_PATH=/home-link/epivj01/INSTALL/ElmerInstall_all/elmerice/Elmer_devel_05-11-20/share/elmersolver/lib/:$LD_LIBRARY_PATH
# #export LD_LIBRARY_PATH=/home-link/epivj01/INSTALL/ElmerInstall_all/elmerice/Elmer_devel_05-11-20/lib:$LD_LIBRARY_PATH
# #export LD_LIBRARY_PATH=/home-link/epivj01/INSTALL/ElmerInstall_all/elmerice/Elmer_devel_05-11-20/lib/elmersolver/:$LD_LIBRARY_PATH
#
# export LD_LIBRARY_PATH=/home-link/epivj01/INSTALL/ElmerInstall_all/MUMPS_5.1.2/lib:$LD_LIBRARY_PATH
#
# export PATH=/home-link/epivj01/INSTALL/gmsh-4.5.4-source/build:$PATH

#=================================================================================================================
echo Here comes the partition the job runs in:
echo $PBS_QUEUE
# cd $PBS_O_WORKDIR
cd /home-link/epivj01/Elmer/paper_2d_tests/3D_500in_100_ly_King_B_whole_5_5_km

#ELMEREXEC="$ELMER_HOME/bin/ElmerSolver_mpi"
#cat $PBS_NODEFILE | sort | uniq > nodelist.tmp

#while read line; do
	#echo $line
	#scp $ELMEREXEC $line:/scratch/${PBS_JOBID}/ElmerSolver_mpi
	##scp $ELMEREXEC $line:/scratch/Elmer
#done < nodelist.tmp

#sleep 3s
#
# ncpus=`wc -l < ${PBS_NODEFILE}`
# echo Here comes the Nodelist:${PBS_NODEFILE}}
# echo Number of Nodes: ${ncpus}}
# echo $PBS_NODEFILE


#cp $ELMER_HOME/share/elmersolver/lib/FreeSurfaceSolver.so src/MyFreeSurfaceSolver.so
# make compile
make ini
make grid
make submit
#mpirun -np 60 -machinefile $PBS_NODEFILE ElmerSolver_mpi
# mpirun -n 20 ElmerSolver_mpi  > out 2>&1 &
#make submit
#mpirun -n 60 /scratch/Elmer
#mpirun -n 60 /scratch/${PBS_JOBID}/ElmerSolver_mpi
