STEP_DIR="$ROOT_DIR/3-create-train-val-set"

# Reset folder
rm -rf "$STEP_DIR"
mkdir -p "$STEP_DIR"

# activate environment
source "$ROOT_DIR/.venv/bin/activate"

# Export symbols
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  textgrid-tools-cli grids export-marks \
    "$ROOT_DIR/dataset-cut-normed/%" \
    "transcription" \
    "$STEP_DIR/%/marks.txt" \
    --log "$STEP_DIR/%/logs/3.log"

find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  txt-utils-cli create-unit-occurrence-stats \
    "$STEP_DIR/%/marks.txt" --sep "|" \
    "$STEP_DIR/%/marks-stats.csv" \
    --log "$STEP_DIR/%/logs/2.log"

# Export utterances
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  textgrid-tools-cli grids export-marks \
    "$ROOT_DIR/dataset-cut-normed/%" \
    "words" --sep " " \
    "$STEP_DIR/%/utterances.txt" \
    --log "$STEP_DIR/%/logs/4.log"

# Remove double spaces that were inserted because of silence between words
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  txt-utils-cli replace \
    "$STEP_DIR/%/utterances.txt" \
    " {2,}" " " \
    --log "$STEP_DIR/%/logs/5.log"

# Remove silence at start and end of utterances
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  txt-utils-cli replace-line \
    "$STEP_DIR/%/utterances.txt" \
    "(^ | $)" "" \
    --log "$STEP_DIR/%/logs/6.log"

# Export all durations
textgrid-tools-cli grids export-durations \
  "$ROOT_DIR/dataset-cut-normed" \
  "$STEP_DIR/all-durations.txt" \
  --log "$STEP_DIR/7.log"

# Export all durations
textgrid-tools-cli grids plot-stats \
  "$ROOT_DIR/dataset-cut-normed" \
  "$STEP_DIR/plot.pdf" \
  "$STEP_DIR/marks.csv" \
  --log "$STEP_DIR/7.log"

text-selection-cli dataset create \
  "$STEP_DIR/all-durations.txt" \
  "base" \
  "$STEP_DIR/dataset.pkl" \
  --log "$STEP_DIR/10.log"

text-selection-cli weights create-from-file \
  "$STEP_DIR/dataset.pkl" \
  "$STEP_DIR/all-durations.txt" \
  "$STEP_DIR/all-durations.npy" \
  --log "$STEP_DIR/11.log"

# Durations min/avg/med/max: 3.77; 7.73; 7.65; 13.85
text-selection-cli dataset export-statistics \
  "$STEP_DIR/dataset.pkl" \
  "$STEP_DIR/stats.csv" \
  "$STEP_DIR/all-durations.txt" \
  --weights "$STEP_DIR/all-durations.npy" \
  --log "$STEP_DIR/11.log"

# Export durations
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  textgrid-tools-cli grids export-durations \
    "$ROOT_DIR/dataset-cut-normed/%" \
    "$STEP_DIR/%/durations.txt" \
    --log "$STEP_DIR/%/logs/7.log"

# Export grid paths
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  textgrid-tools-cli grids export-paths \
    "$ROOT_DIR/dataset-cut-normed/%" \
    "$STEP_DIR/%/grids.txt" \
    --log "$STEP_DIR/%/logs/8.log"

# Export audio paths
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  textgrid-tools-cli grids export-audio-paths \
    "$ROOT_DIR/dataset-cut-normed/%" \
    "$STEP_DIR/%/audios.txt" \
    --log "$STEP_DIR/%/logs/9.log"

# Create dataset for selection
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli dataset create \
    "$STEP_DIR/%/marks.txt" \
    "base" \
    "$STEP_DIR/%/dataset.pkl" \
    --log "$STEP_DIR/%/logs/10.log"

# Create weights from durations
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli weights create-from-file \
    "$STEP_DIR/%/dataset.pkl" \
    "$STEP_DIR/%/durations.txt" \
    "$STEP_DIR/%/durations.npy" \
    --log "$STEP_DIR/%/logs/11.log"

# Create weights from durations
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli weights create-from-count \
    "$STEP_DIR/%/dataset.pkl" \
    "$STEP_DIR/%/utterances.txt" --sep "" \
    "$STEP_DIR/%/characters.npy" \
    --log "$STEP_DIR/%/logs/15.log"

# Create uniform weights
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli weights create-uniform \
    "$STEP_DIR/%/dataset.pkl" \
    1 \
    "$STEP_DIR/%/uniform.npy" \
    --log "$STEP_DIR/%/logs/12.log"

# Create set "ignore" (those speaker which have nothing to ignore will not create that set automatically in next steps which is why they need to be created in beforehand)
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets add \
    "$STEP_DIR/%/dataset.pkl" \
    "ignore" \
    --log "$STEP_DIR/%/logs/17.log"

find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets filter-by-weight \
    "$STEP_DIR/%/dataset.pkl" \
    "base" "ignore"\
    "$STEP_DIR/%/durations.npy" \
    --min-weight 0 \
    --max-weight 4 \
    --log "$STEP_DIR/%/logs/19.log"

find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets filter-by-weight \
    "$STEP_DIR/%/dataset.pkl" \
    "base" "ignore"\
    "$STEP_DIR/%/durations.npy" \
    --min-weight 8 \
    --max-weight inf \
    --log "$STEP_DIR/%/logs/20.log"

find "$STEP_DIR" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets filter-by-regex \
    "$STEP_DIR/%/dataset.pkl" \
    "base" "ignore" \
    "$STEP_DIR/%/utterances.txt" \
    "？" \
    --log "$ROOT_DIR/logs/evaluation/%/22.log"

# Export utterances
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets export \
    "$STEP_DIR/%/dataset.pkl" \
    "base" \
    "$STEP_DIR/%/utterances.txt" \
    "$STEP_DIR/%/utterances-trainset.txt" \
    --log "$STEP_DIR/%/logs/16.log"

find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  cp \
    "$STEP_DIR/%/marks.txt" \
    "$STEP_DIR/%/marks-raw.txt"

# "terminal.integrated.detectLocale": "on"
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  txt-utils-cli replace \
    "$STEP_DIR/%/marks-raw.txt" \
    "[ːˑ˘˥˧˩]+" "" \
    --log "$STEP_DIR/%/logs/15.log"

find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  txt-utils-cli remove-units \
    "$STEP_DIR/%/marks-raw.txt" \
    "SIL0" "SIL1" "SIL2" "SIL3" "。" "？" --sep "|" \
    --log "$STEP_DIR/%/logs/16.log"

find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  txt-utils-cli create-unit-occurrence-stats \
    "$STEP_DIR/%/marks-raw.txt" --sep "|" \
    "$STEP_DIR/%/marks-raw-stats.csv" \
    --log "$STEP_DIR/%/logs/1.log"

find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  txt-utils-cli extract-vocabulary \
    "$STEP_DIR/%/marks-raw.txt" \
    "$STEP_DIR/%/marks-raw-vocab.txt" --sep "|" \
    --log "$STEP_DIR/%/logs/23.log"

# Create set "trainset"
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets add \
    "$STEP_DIR/%/dataset.pkl" \
    "trainset" "valset" \
    --log "$STEP_DIR/%/logs/17.log"

# Select utterances so that all symbols from marks-selection occur at least three times
# echo $(echo -n "%" | gzip -1 -c | tail -c8 | hexdump -n4 -e '"%u"')
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets select-randomly \
    "$STEP_DIR/%/dataset.pkl" \
    "base" \
    "valset" \
    "$STEP_DIR/%/uniform.npy" \
    2 \
    --seed 1111 \
    --log "$STEP_DIR/%/logs/17.log"

# Move all from "base" and "ignore" to new "trainset"
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets select-all \
    "$STEP_DIR/%/dataset.pkl" \
    "base" "ignore" \
    "trainset" \
    --log "$STEP_DIR/%/logs/17.log"

# Remove other subsets
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets remove \
    "$STEP_DIR/%/dataset.pkl" \
    "base" "ignore" \
    --log "$STEP_DIR/%/logs/17.log"

# # Sort after duration asc
# find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
#   text-selection-cli subsets sort-by-weight \
#     "$STEP_DIR/%/dataset.pkl" \
#     "valset" "trainset" \
#     "$STEP_DIR/%/characters.npy" \
#     --log "$STEP_DIR/%/logs/18.log"

# Sort by line nr
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets sort-by-line-nr \
    "$STEP_DIR/%/dataset.pkl" \
    "valset" "trainset" \
    --log "$STEP_DIR/%/logs/18.log"

# Export trainset utterances
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets export \
    "$STEP_DIR/%/dataset.pkl" \
    "trainset" \
    "$STEP_DIR/%/utterances.txt" \
    "$STEP_DIR/%/utterances-trainset.txt" \
    --log "$STEP_DIR/%/logs/16.log"

# Export valset utterances
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets export \
    "$STEP_DIR/%/dataset.pkl" \
    "valset" \
    "$STEP_DIR/%/utterances.txt" \
    "$STEP_DIR/%/utterances-valset.txt" \
    --log "$STEP_DIR/%/logs/16.log"

# Export trainset grid paths
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets export \
    "$STEP_DIR/%/dataset.pkl" \
    "trainset" \
    "$STEP_DIR/%/grids.txt" \
    "$STEP_DIR/%/grids-trainset.txt" \
    --log "$STEP_DIR/%/logs/20.log"

# Export valset grid paths
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets export \
    "$STEP_DIR/%/dataset.pkl" \
    "valset" \
    "$STEP_DIR/%/grids.txt" \
    "$STEP_DIR/%/grids-valset.txt" \
    --log "$STEP_DIR/%/logs/20.log"

# Export trainset audio paths
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets export \
    "$STEP_DIR/%/dataset.pkl" \
    "trainset" \
    "$STEP_DIR/%/audios.txt" \
    "$STEP_DIR/%/audios-trainset.txt" \
    --log "$STEP_DIR/%/logs/21.log"

# Export valset audio paths
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets export \
    "$STEP_DIR/%/dataset.pkl" \
    "valset" \
    "$STEP_DIR/%/audios.txt" \
    "$STEP_DIR/%/audios-valset.txt" \
    --log "$STEP_DIR/%/logs/21.log"

rm -rf "$ROOT_DIR/trainset"
mkdir -p "$ROOT_DIR/trainset"

# Import trainset grids to new directory
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  textgrid-tools-cli grids import-paths \
    "$STEP_DIR/%/grids-trainset.txt" \
    "$ROOT_DIR/trainset/%" --symlink \
    --relative-to "$ROOT_DIR/dataset-cut-normed/%" \
    --log "$STEP_DIR/%/logs/20.log"

# Import trainset audios to new directory
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  textgrid-tools-cli grids import-audio-paths \
    "$STEP_DIR/%/audios-trainset.txt" \
    "$ROOT_DIR/trainset/%" --symlink \
    --relative-to "$ROOT_DIR/dataset-cut-normed/%" \
    --log "$STEP_DIR/%/logs/20.log"

# Export trainset words
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  textgrid-tools-cli tier export \
    "$ROOT_DIR/trainset/%" \
    "words" --sep " " \
    --log "$STEP_DIR/%/logs/20.log"

# Export trainset marks
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets export \
    "$STEP_DIR/%/dataset.pkl" \
    "trainset" \
    "$STEP_DIR/%/marks.txt" \
    "$ROOT_DIR/trainset/%/marks.txt" \
    --log "$STEP_DIR/%/logs/16.log"

find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  cp \
    "$STEP_DIR/%/utterances-trainset.txt" \
    "$ROOT_DIR/trainset/%/utterances.txt"

rm -rf "$ROOT_DIR/valset"
mkdir -p "$ROOT_DIR/valset"

# Import valset grids to new directory
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  textgrid-tools-cli grids import-paths \
    "$STEP_DIR/%/grids-valset.txt" \
    "$ROOT_DIR/valset/%" --symlink \
    --relative-to "$ROOT_DIR/dataset-cut-normed/%" \
    --log "$STEP_DIR/%/logs/20.log"

# Import valset audios to new directory
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  textgrid-tools-cli grids import-audio-paths \
    "$STEP_DIR/%/audios-valset.txt" \
    "$ROOT_DIR/valset/%" --symlink \
    --relative-to "$ROOT_DIR/dataset-cut-normed/%" \
    --log "$STEP_DIR/%/logs/20.log"

# Export valset words
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  textgrid-tools-cli tier export \
    "$ROOT_DIR/valset/%" \
    "words" --sep " " \
    --log "$STEP_DIR/%/logs/20.log"

# Export valset marks
find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  text-selection-cli subsets export \
    "$STEP_DIR/%/dataset.pkl" \
    "valset" \
    "$STEP_DIR/%/marks.txt" \
    "$ROOT_DIR/valset/%/marks.txt" \
    --log "$STEP_DIR/%/logs/16.log"

find "$ROOT_DIR/dataset-cut-normed" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | xargs --max-procs 12 -I % \
  cp \
    "$STEP_DIR/%/utterances-valset.txt" \
    "$ROOT_DIR/valset/%/utterances.txt"

# Total duration: 836.94s / 13.95min / 0.23h
textgrid-tools-cli grids plot-stats \
  "$ROOT_DIR/valset" -o \
  "$ROOT_DIR/valset/stats.pdf" \
  "$ROOT_DIR/valset/stats.csv"

# Total duration: 102641.66s / 1710.69min / 28.51h
textgrid-tools-cli grids plot-stats \
  "$ROOT_DIR/trainset" -o \
  "$ROOT_DIR/trainset/stats.pdf" \
  "$ROOT_DIR/trainset/stats.csv"

textgrid-tools-cli grids export-vocabulary \
  "$ROOT_DIR/valset" \
  "transcription" \
  "$ROOT_DIR/valset/vocabulary.txt"

textgrid-tools-cli grids export-vocabulary \
  "$ROOT_DIR/trainset" \
  "transcription" \
  "$ROOT_DIR/trainset/vocabulary.txt"

diff \
  "$ROOT_DIR/trainset/vocabulary.txt" \
  "$ROOT_DIR/valset/vocabulary.txt" \
  --side-by-side \
  --suppress-common-lines

cd "$ROOT_DIR/valset"
du -a | cut -d/ -f2 | sort | uniq -c | sort -nr

cd "$ROOT_DIR/trainset"
du -a | cut -d/ -f2 | sort | uniq -c | sort -nr

tacotron-cli create-mels \
  "$ROOT_DIR/valset"
