STEP_DIR="$ROOT_DIR/3-create-train-val-set"

# Reset folder
rm -rf "$STEP_DIR"
mkdir -p "$STEP_DIR"

# activate environment
source "$ROOT_DIR/.venv/bin/activate"

# Export symbols
textgrid-tools-cli grids export-marks \
  "$ROOT_DIR/dataset" \
  "transcription" \
  "$STEP_DIR/marks.txt" \
  --log "$STEP_DIR/logs/1.log"

txt-utils-cli create-unit-occurrence-stats \
  "$STEP_DIR/marks.txt" --sep "|" \
  "$STEP_DIR/marks-stats.csv"

# Export utterances
textgrid-tools-cli grids export-marks \
  "$ROOT_DIR/dataset" \
  "words" --sep " " \
  "$STEP_DIR/utterances.txt" \
  --log "$STEP_DIR/logs/2.log"
  
# Remove double spaces that were inserted because of silence between words
txt-utils-cli replace \
  "$STEP_DIR/utterances.txt" \
  " {2,}" " " \
  --log "$STEP_DIR/logs/3.log"

# Remove silence at start and end of utterances
txt-utils-cli replace-line \
  "$STEP_DIR/utterances.txt" \
  "(^ | $)" "" \
  --log "$STEP_DIR/logs/4.log"

# Export durations
textgrid-tools-cli grids export-durations \
  "$ROOT_DIR/dataset" \
  "$STEP_DIR/durations.txt" \
  --log "$STEP_DIR/logs/5.log"

# Export grid paths
textgrid-tools-cli grids export-paths \
  "$ROOT_DIR/dataset" \
  "$STEP_DIR/grids.txt" \
  --log "$STEP_DIR/logs/6.log"

# Export audio paths
textgrid-tools-cli grids export-audio-paths \
  "$ROOT_DIR/dataset" \
  "$STEP_DIR/audios.txt" \
  --log "$STEP_DIR/logs/7.log"

cp \
  "$STEP_DIR/utterances.txt" \
  "$STEP_DIR/utterances-no-punc.txt"

txt-utils-cli trim-units \
  "$STEP_DIR/utterances-no-punc.txt" \
  "both" \
  "!" '"' "'" '(' ')' ',' '-' '.' ':' ';' '<' '=' '>' '?' '[' '\' ']' '{' '}' \
  --sep " " \
  --log "$STEP_DIR/logs/8.log"

txt-utils-cli replace-line \
  "$STEP_DIR/utterances-no-punc.txt" \
  " {2,}" " " \
  --log "$STEP_DIR/logs/9.log"

cp \
  "$STEP_DIR/utterances-no-punc.txt" \
  "$STEP_DIR/utterances-no-punc-lower.txt"

norm-eng-cli normalize-file \
  "$STEP_DIR/utterances-no-punc-lower.txt" \
  "lower" \
  --log "$STEP_DIR/logs/10.log"

# Create dataset for selection
text-selection-cli dataset create \
  "$STEP_DIR/marks.txt" \
  "base" \
  "$STEP_DIR/dataset.pkl" \
  --log "$STEP_DIR/logs/11.log"

# Create weights from durations
text-selection-cli weights create-from-file \
  "$STEP_DIR/dataset.pkl" \
  "$STEP_DIR/durations.txt" \
  "$STEP_DIR/durations.npy" \
  --log "$STEP_DIR/logs/12.log"

# Create weights from durations
text-selection-cli weights create-from-count \
  "$STEP_DIR/dataset.pkl" \
  "$STEP_DIR/utterances.txt" --sep "" \
  "$STEP_DIR/characters.npy" \
  --log "$STEP_DIR/logs/13.log"

# Create uniform weights
text-selection-cli weights create-uniform \
  "$STEP_DIR/dataset.pkl" \
  1 \
  "$STEP_DIR/uniform.npy" \
  --log "$STEP_DIR/logs/14.log"

# Durations min/avg/med/max: 1.11;6.574;6.766;10.09
text-selection-cli dataset export-statistics \
  "$STEP_DIR/dataset.pkl" \
  "$STEP_DIR/stats.csv" \
  "$STEP_DIR/marks.txt" \
  --weights "$STEP_DIR/durations.npy" \
  --log "$STEP_DIR/logs/15.log"

# Filtered 8841 out of 13100 lines (67.49%). 4259 lines remain.
# Select all sentences
text-selection-cli subsets filter-by-regex \
  "$STEP_DIR/dataset.pkl" \
  "base" "ignore" \
  "$STEP_DIR/utterances.txt" \
  '(^[^A-Z]\S*)|(\S*[\.?\!]+\S+)|(\S*[\.?\!]+) |(\S*[^\.?\!]$)' \
  --log "$STEP_DIR/logs/16.log"

# Remove duplicates (9)
text-selection-cli subsets filter-duplicates \
  "$STEP_DIR/dataset.pkl" \
  "base" "ignore" \
  "$STEP_DIR/utterances.txt" \
  --log "$STEP_DIR/logs/17.log"

text-selection-cli subsets filter-by-weight \
  "$STEP_DIR/dataset.pkl" \
  "base" "ignore"\
  "$STEP_DIR/durations.npy" \
  --min-weight 0 \
  --max-weight 4 \
  --log "$STEP_DIR/logs/18.log"

text-selection-cli subsets filter-by-weight \
  "$STEP_DIR/dataset.pkl" \
  "base" "ignore"\
  "$STEP_DIR/durations.npy" \
  --min-weight 7 \
  --max-weight inf \
  --log "$STEP_DIR/logs/19.log"

cp \
  "$STEP_DIR/utterances-no-punc.txt" \
  "$STEP_DIR/utterances-tmp.txt"

txt-utils-cli replace-line \
  "$STEP_DIR/utterances-tmp.txt" \
  "(Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|January|February|March|April|May|June|July|August|September|October|November|December|Christmas|Thanksgiving|Easter|Halloween|Father's Day|New Year's Day|Independence Day|English|British|American|German|French|Italian|Spanish|Chinese|Russian|America|United States|England|Germany|France|Italy|Spain|China|Russia|North America|South America|Australia|Africa|Asia|Europe|Antarctica|Australia)" " " \
  --log "$STEP_DIR/logs/20.log" 

text-selection-cli subsets filter-by-regex \
  "$STEP_DIR/dataset.pkl" \
  "base" "ignore" \
  "$STEP_DIR/utterances-tmp.txt" \
  ' ([A-Z][a-z]+)' \
  --log "$STEP_DIR/logs/21.log"

# Download dictionary: https://github.com/LibreOffice/dictionaries/tree/master/en
wget https://github.com/LibreOffice/dictionaries/raw/master/en/en_US.dic \
  -O "$STEP_DIR/dict.dic"
wget https://github.com/LibreOffice/dictionaries/raw/master/en/en_US.aff \
  -O "$STEP_DIR/dict.aff"

# Extract dictionary (for OOV filtering)
txt-utils-cli extract-vocabulary \
  "$STEP_DIR/utterances-no-punc.txt" \
  "$STEP_DIR/utterances-vocab.txt" --sep " " \
  --log "$STEP_DIR/logs/22.log"

# Identify unknown words
hunspell \
  -d "$STEP_DIR/dict" \
  -w "$STEP_DIR/utterances-vocab.txt" > \
  "$STEP_DIR/unknown-vocab.txt"

text-selection-cli subsets filter-by-vocabulary \
  "$STEP_DIR/dataset.pkl" \
  "base" "ignore" \
  "$STEP_DIR/utterances-no-punc.txt" \
  "$STEP_DIR/unknown-vocab.txt" \
  --mode "iv" --min-count 1 --sep " " \
  --log "$STEP_DIR/logs/23.log"

# Ignore acronyms
text-selection-cli subsets filter-by-regex \
  "$STEP_DIR/dataset.pkl" \
  "base" "ignore" \
  "$STEP_DIR/utterances.txt" \
  "([A-Z]{2,})" \
  --log "$STEP_DIR/logs/24.log"

# Ignore single letters
text-selection-cli subsets filter-by-regex \
  "$STEP_DIR/dataset.pkl" \
  "base" "ignore" \
  "$STEP_DIR/utterances.txt" \
  "((?:^[B-HJ-Z]) )|((?:^[A-HJ-Z])(?::|;|,)+)| ([B-HJ-Z] )| ([A-HJ-Z](?::|;|,)+)|([A-HJ-Z][\!?\.])|([^:] A(?: |:|;|,)+)" \
  --log "$STEP_DIR/logs/25.log"

# Download male and female names
wget "https://www.cs.cmu.edu/afs/cs/project/ai-repository/ai/areas/nlp/corpora/names/male.txt" \
  -O "$STEP_DIR/male-names.txt"
wget "https://www.cs.cmu.edu/afs/cs/project/ai-repository/ai/areas/nlp/corpora/names/female.txt" \
  -O "$STEP_DIR/female-names.txt"

# Merge male and female names
txt-utils-cli merge \
  "$STEP_DIR/male-names.txt" \
  "$STEP_DIR/female-names.txt" \
  "$STEP_DIR/names.txt" \
  --log "$STEP_DIR/logs/26.log"

text-selection-cli subsets filter-by-vocabulary \
  "$STEP_DIR/dataset.pkl" \
  "base" "ignore" \
  "$STEP_DIR/utterances-no-punc.txt" \
  "$STEP_DIR/names.txt" \
  --mode "iv" --min-count 1 --sep " " \
  --log "$STEP_DIR/logs/27.log"

# Ignore utterances with special characters that are not wanted
text-selection-cli subsets filter-by-regex \
  "$STEP_DIR/dataset.pkl" \
  "base" "ignore" \
  "$STEP_DIR/utterances.txt" \
  "([^A-Za-z\.,;:'\- ]+)" \
  --log "$STEP_DIR/logs/28.log"

# Ignore utterances that contain old English
text-selection-cli subsets filter-by-regex \
  "$STEP_DIR/dataset.pkl" \
  "base" "ignore" \
  "$STEP_DIR/utterances-no-punc-lower.txt" \
  "(?:^| )(ye|yer|thou|thy|thee|thine|hath|heth|hafth|hefth|sire)(?: |'|$)" \
  --log "$STEP_DIR/logs/29.log"

# Ignore utterances that contain 'quote'
text-selection-cli subsets filter-by-regex \
  "$STEP_DIR/dataset.pkl" \
  "base" "ignore" \
  "$STEP_DIR/utterances-no-punc-lower.txt" \
  "(?:^| )(quote)(?: |'|$)" \
  --log "$STEP_DIR/logs/30.log"

# Export utterances 
text-selection-cli subsets export \
  "$STEP_DIR/dataset.pkl" \
  "base" \
  "$STEP_DIR/utterances.txt" \
  "$STEP_DIR/utterances-base.txt" \
  --log "$STEP_DIR/logs/31.log"

rm -rf "$STEP_DIR-backup"
cp -r "$STEP_DIR" "$STEP_DIR-backup"

# Select 120 random utterances
text-selection-cli subsets select-randomly \
  "$STEP_DIR/dataset.pkl" \
  "base" \
  "valset" \
  "$STEP_DIR/uniform.npy" \
  120 \
  --seed 1111 \
  --log "$STEP_DIR/logs/32.log"

# Move all from "base" to new "trainset"
text-selection-cli subsets select-all \
  "$STEP_DIR/dataset.pkl" \
  "base" \
  "trainset" \
  --log "$STEP_DIR/logs/33.log"

# Export training sentence audio paths to generate example audios for MOS test later
text-selection-cli subsets export \
  "$STEP_DIR/dataset.pkl" \
  "trainset" \
  "$STEP_DIR/audios.txt" \
  "$STEP_DIR/audios-trainset-sentences.txt" \
  --log "$STEP_DIR/logs/34.log"

# Export training grid paths
text-selection-cli subsets export \
  "$STEP_DIR/dataset.pkl" \
  "trainset" \
  "$STEP_DIR/grids.txt" \
  "$STEP_DIR/grids-trainset-sentences.txt" \
  --log "$STEP_DIR/logs/35.log"

# Move all from "ignore" to "trainset"
text-selection-cli subsets select-all \
  "$STEP_DIR/dataset.pkl" \
  "ignore" \
  "trainset" \
  --log "$STEP_DIR/logs/36.log"

text-selection-cli subsets remove \
  "$STEP_DIR/dataset.pkl" \
  "base" "ignore" \
  --log "$STEP_DIR/logs/37.log"

text-selection-cli subsets sort-by-weight \
  "$STEP_DIR/dataset.pkl" \
  "valset" \
  "$STEP_DIR/characters.npy" \
  --log "$STEP_DIR/logs/38.log"
  
# Export valset utterances 
text-selection-cli subsets export \
  "$STEP_DIR/dataset.pkl" \
  "valset" \
  "$STEP_DIR/utterances.txt" \
  "$STEP_DIR/utterances-valset-sorted.txt" \
  --log "$STEP_DIR/logs/39.log"

# Sort by line nr
text-selection-cli subsets sort-by-line-nr \
  "$STEP_DIR/dataset.pkl" \
  "valset" "trainset" \
  --log "$STEP_DIR/logs/40.log"
  
# Export trainset utterances 
text-selection-cli subsets export \
  "$STEP_DIR/dataset.pkl" \
  "trainset" \
  "$STEP_DIR/utterances.txt" \
  "$STEP_DIR/utterances-trainset.txt" \
  --log "$STEP_DIR/logs/41.log"

# Export valset utterances 
text-selection-cli subsets export \
  "$STEP_DIR/dataset.pkl" \
  "valset" \
  "$STEP_DIR/utterances.txt" \
  "$STEP_DIR/utterances-valset.txt" \
  --log "$STEP_DIR/logs/42.log"

# Export trainset grid paths
text-selection-cli subsets export \
  "$STEP_DIR/dataset.pkl" \
  "trainset" \
  "$STEP_DIR/grids.txt" \
  "$STEP_DIR/grids-trainset.txt" \
  --log "$STEP_DIR/logs/43.log"

# Export valset grid paths
text-selection-cli subsets export \
  "$STEP_DIR/dataset.pkl" \
  "valset" \
  "$STEP_DIR/grids.txt" \
  "$STEP_DIR/grids-valset.txt" \
  --log "$STEP_DIR/logs/44.log"

# Export trainset audio paths
text-selection-cli subsets export \
  "$STEP_DIR/dataset.pkl" \
  "trainset" \
  "$STEP_DIR/audios.txt" \
  "$STEP_DIR/audios-trainset.txt" \
  --log "$STEP_DIR/logs/45.log"

# Export valset audio paths
text-selection-cli subsets export \
  "$STEP_DIR/dataset.pkl" \
  "valset" \
  "$STEP_DIR/audios.txt" \
  "$STEP_DIR/audios-valset.txt" \
  --log "$STEP_DIR/logs/46.log"

rm -rf "$ROOT_DIR/trainset"
mkdir -p "$ROOT_DIR/trainset"

# Import trainset grids to new directory
textgrid-tools-cli grids import-paths \
  "$STEP_DIR/grids-trainset.txt" \
  "$ROOT_DIR/trainset" --symlink \
  --relative-to "$ROOT_DIR/dataset" \
  --log "$STEP_DIR/logs/47.log"

# Import trainset audios to new directory
textgrid-tools-cli grids import-audio-paths \
  "$STEP_DIR/audios-trainset.txt" \
  "$ROOT_DIR/trainset" --symlink \
  --relative-to "$ROOT_DIR/dataset" \
  --log "$STEP_DIR/logs/48.log"

# Remove duration markers
textgrid-tools-cli intervals replace-text \
  "$ROOT_DIR/trainset" -o \
  "transcription" \
  "[ː˘ˑ]+" ""  

textgrid-tools-cli tier export \
  "$ROOT_DIR/trainset" \
  "words" --sep " " \
  --log "$STEP_DIR/logs/49.log"

# Export trainset marks 
text-selection-cli subsets export \
  "$STEP_DIR/dataset.pkl" \
  "trainset" \
  "$STEP_DIR/marks.txt" \
  "$ROOT_DIR/trainset/marks.txt" \
  --log "$STEP_DIR/logs/50.log"

cp \
  "$STEP_DIR/utterances-trainset.txt" \
  "$ROOT_DIR/trainset/utterances.txt"

rm -rf "$ROOT_DIR/valset"
mkdir -p "$ROOT_DIR/valset"

# Import valset grids to new directory
textgrid-tools-cli grids import-paths \
  "$STEP_DIR/grids-valset.txt" \
  "$ROOT_DIR/valset" --symlink \
  --relative-to "$ROOT_DIR/dataset" \
  --log "$STEP_DIR/logs/51.log"

# Import valset audios to new directory
textgrid-tools-cli grids import-audio-paths \
  "$STEP_DIR/audios-valset.txt" \
  "$ROOT_DIR/valset" --symlink \
  --relative-to "$ROOT_DIR/dataset" \
  --log "$STEP_DIR/logs/52.log"

# Remove duration markers
textgrid-tools-cli intervals replace-text \
  "$ROOT_DIR/valset" -o \
  "transcription" \
  "[ː˘ˑ]+" ""  

textgrid-tools-cli tier export \
  "$ROOT_DIR/valset" \
  "words" --sep " " \
  --log "$STEP_DIR/logs/53.log"
  
# Export valset marks 
text-selection-cli subsets export \
  "$STEP_DIR/dataset.pkl" \
  "valset" \
  "$STEP_DIR/marks.txt" \
  "$ROOT_DIR/valset/marks.txt" \
  --log "$STEP_DIR/logs/54.log"

cp \
  "$STEP_DIR/utterances-valset.txt" \
  "$ROOT_DIR/valset/utterances.txt"

tacotron-cli create-mels \
  "$ROOT_DIR/valset" \
  --log "$STEP_DIR/logs/55.log"

# #Grids  Min (s)  Median (s)  Max (s)  Mean (s)  SD (s)  Kurt (s)    Sum (s)  Sum (min)  Sum (h)
#  12980   1.1101       6.799  10.0962    6.5834  2.1917   -0.7861 85452.9322  1424.2155  23.7369
textgrid-tools-cli grids plot-stats \
  "$ROOT_DIR/trainset" \
  "$ROOT_DIR/trainset-stats.pdf" \
  "$ROOT_DIR/trainset-stats.csv" --debug

# #Grids  Min (s)  Median (s)  Max (s)  Mean (s)  SD (s)  Kurt (s)  Sum (s)  Sum (min)  Sum (h)
#    120   4.0242      5.5335   6.9267    5.5345  0.8326   -1.0964  664.144    11.0691   0.1845
textgrid-tools-cli grids plot-stats \
  "$ROOT_DIR/valset" \
  "$ROOT_DIR/valset-stats.pdf" \
  "$ROOT_DIR/valset-stats.csv"


rm -rf "$ROOT_DIR/trainset-example-sentences"
mkdir -p "$ROOT_DIR/trainset-example-sentences"

# Import trainset grids to new directory
textgrid-tools-cli grids import-paths \
  "$STEP_DIR/grids-trainset-sentences.txt" \
  "$ROOT_DIR/trainset-example-sentences" --symlink \
  --relative-to "$ROOT_DIR/dataset" \
  --log "$STEP_DIR/logs/56.log"

# Import trainset audios to new directory
textgrid-tools-cli grids import-audio-paths \
  "$STEP_DIR/audios-trainset-sentences.txt" \
  "$ROOT_DIR/trainset-example-sentences" --symlink \
  --relative-to "$ROOT_DIR/dataset" \
  --log "$STEP_DIR/logs/57.log"

textgrid-tools-cli tier export \
  "$ROOT_DIR/trainset-example-sentences" \
  "words" --sep " " \
  --log "$STEP_DIR/logs/58.log"
