#!/bin/bash -l
#SBATCH -N 1
#SBATCH -n 1
#SBATCH -t 23:00:00
##SBATCH --mem=32G 
#SBATCH --gres=gpu:1
#SBATCH --job-name=train

module load anaconda3
conda activate /scratch/gpfs/pinchenx/envs/dp-dev

test $? -ne 0 && exit 1

if [ ! -f tag_0_finished ] ;then
  { if [ ! -f model.ckpt.index ]; then dp train KDP_DP_Train.json; else dp train KDP_DP_Train.json --restart model.ckpt; fi }  1>> train.log 2>> train.log 
  if test $? -ne 0; then exit 1; else touch tag_0_finished; fi 
fi


if [ ! -f tag_1_finished ] ;then
  dp freeze  1>> train.log 2>> train.log 
  if test $? -ne 0; then exit 1; else touch tag_1_finished; fi 
fi


if [ ! -f tag_2_finished ] ;then
  dp compress -i frozen_model.pb -o KDP_DP_Model.pb
  # dp freeze  1>> train.log 2>> train.log 
  if test $? -ne 0; then exit 1; else touch tag_2_finished; fi 
fi
