root_dir="/tmp/ljs-script"

rm -rf "$root_dir"
mkdir -p "$root_dir"

# Install tools
pip install --user --upgrade \
  textgrid-tools==0.0.5 \
  pronunciation-dictionary-utils==0.0.2 \
  speech-dataset-parser==0.0.2 \
  dict-from-dict==0.0.2 \
  dict-from-g2pE==0.0.1 \
  english-text-normalization==0.0.2

# Install rsync
sudo apt-get install rsync -y

# Install MFA in version 2.0.4
conda create -n aligner -c conda-forge montreal-forced-aligner==2.0.4 -y

# Download LJ Speech dataset
wget https://data.keithito.com/data/speech/LJSpeech-1.1.tar.bz2 \
  -O "$root_dir/LJSpeech-1.1.tar.bz2"
rm -rf "$root_dir/LJSpeech-1.1"
mkdir -p "$root_dir/LJSpeech-1.1"
tar -xvf "$root_dir/LJSpeech-1.1.tar.bz2" -C "$root_dir"

# Create TextGrids
rm -rf "$root_dir/ljs"
dataset-converter-cli convert-ljs \
  "$root_dir/LJSpeech-1.1" \
  "$root_dir/ljs" \
  --tier "Symbols" \
  --symlink

step_dir="$root_dir/processing"

rm -rf "$step_dir/grids"
mkdir -p "$step_dir/grids"

# Import all grids and audio files
rsync -am --include='*.TextGrid' -f 'hide,! */' $root_dir/ljs/ "$step_dir/grids"
rsync -am --include='*.wav' -f 'hide,! */' --links $root_dir/ljs/ "$step_dir/grids"

textgrid-tools-cli intervals join-between-pauses \
  "$step_dir/grids" -o \
  "Symbols" --join-with ""

textgrid-tools-cli tier export \
  "$step_dir/grids" \
  "Symbols" \
  -out "$step_dir/grids_txt"
  
norm-eng-cli normalize-folder \
  "$step_dir/grids_txt" \
  "general"

textgrid-tools-cli tier import \
  "$step_dir/grids" -o \
  "Normalized" \
  --text-directory "$step_dir/grids_txt"
  
# In Wörter aufteilen
textgrid-tools-cli intervals split \
  "$step_dir/grids" -o \
  "Normalized" --symbol " "
  
textgrid-tools-cli grids export-vocabulary \
  "$step_dir/grids" \
  "Normalized" \
  "$step_dir/vocabulary.txt"

# Merge everything to one text
textgrid-tools-cli intervals join-between-pauses \
  "$step_dir/grids" -o \
  "Normalized" --join-with " "
  
# Download CMU dictionary
wget https://raw.githubusercontent.com/cmusphinx/cmudict/master/cmudict.dict \
  -O "$step_dir/cmu.dict"

# Change formatting to remove numbers from words and save as UTF-8
dict-cli change-formatting \
  "$step_dir/cmu.dict" \
  --des "ISO-8859-1" \
  --consider-numbers \
  --consider-pronunciation-comments \
  --se "UTF-8"

# Transcribe the vocabulary with the custom pronunciation dictionary
dict-from-dict-cli \
  "$step_dir/vocabulary.txt" \
  "$step_dir/cmu.dict" \
  --split-on-hyphen \
  --ignore-case \
  "$step_dir/dictionary.dict" \
  --oov-out "$step_dir/oov.txt"

dict-from-g2pE-cli \
  "$step_dir/oov.txt" \
  --split-on-hyphen \
  "$step_dir/oov.dict"

# Merge all generated dictionaries together
dict-cli merge \
  "$step_dir/dictionary.dict" \
  "$step_dir/oov.dict" \
  --duplicate-handling add

dict-cli export-phonemes \
  "$step_dir/dictionary.dict" \
  "$step_dir/phoneme-set.txt"

dict-cli select-single-pronunciation \
  "$step_dir/dictionary.dict" \
  --mode "first"

# Copy dictionary to create dictionary for alignment  
cp "$step_dir/dictionary.dict" "$step_dir/align.dict"

dict-cli remove-symbols-from-pronunciations \
  "$step_dir/align.dict" \
  "!" "?" "." "," "-" ":" ";" "'" "\"" "(" ")" "[" "]" \
  --keep-empty --empty-symbol "sil" --mode "all"

# Convert words in dictionary to lower-case because otherwise MFA can not handle words at sentence beginnings.
dict-cli change-word-casing \
  "$step_dir/align.dict" \
  "lower"

dict-cli select-single-pronunciation \
  "$step_dir/align.dict" \
  --mode "first"

dict-cli change-formatting \
  "$step_dir/align.dict" \
  -ps TAB

cat > "$step_dir/align.yaml" << EOF
beam: 240
retry_beam: 480
transition_scale: 1.0
acoustic_scale: 0.1
self_loop_scale: 0.1
boost_silence: 1.0
optional_silence_phone: sil
oov_phone: spn
ignore_case: False
punctuation:
clitic_markers:
compound_markers:
quote_markers:
word_break_markers:
brackets: [[♪, ♪]]
EOF

# Remove temporary tier `tmp`
textgrid-tools-cli tiers remove \
  "$step_dir/grids" -o \
  "Symbols"

# Only for testing
# rm -rf "$step_dir/grids_test"
# mkdir -p "$step_dir/grids_test/Linda Johnson;2;eng;North American/wavs"
# ls "$step_dir/grids/Linda Johnson;2;eng;North American/wavs" | head -n 500 | xargs -P 16 -I% --verbose cp "$step_dir/grids/Linda Johnson;2;eng;North American/wavs/%" "$step_dir/grids_test/Linda Johnson;2;eng;North American/wavs/%"

# Activate MFA environment `aligner` to be able to run `mfa` methods
conda activate aligner

# Download alignment model
mfa model download acoustic english_us_arpa

# Align grids with audio
# Duration ~53 min
# Remove old alignments
rm -rf "$step_dir/grids_aligned"
rm -rf "$step_dir/MFA_tmp"
mfa align \
  "$step_dir/grids" \
  "$step_dir/align.dict" \
  "english_us_arpa" \
  "$step_dir/grids_aligned" \
  --config_path "$step_dir/align.yaml" \
  --audio_directory "$step_dir/grids" \
  --temp_directory "$step_dir/MFA_tmp" \
  --num_jobs 16 \
  --output_format "long_textgrid" \
  --verbose \
  --clean \
  --debug \
  --overwrite
conda deactivate

rm -rf "$step_dir/MFA_tmp"

# Create backup of alignments
rm -rf "$step_dir/grids_aligned_backup"
cp -r "$step_dir/grids_aligned" "$step_dir/grids_aligned_backup"

textgrid-tools-cli tier rename \
  "$step_dir/grids_aligned" -o \
  "phones" \
  "phonemes"

rm -rf "$step_dir/grids_aligned_text"
cp -r "$step_dir/grids_aligned" "$step_dir/grids_aligned_text"
textgrid-tools-cli tier import \
  "$step_dir/grids_aligned_text" -o \
  --text-directory "$step_dir/grids_txt" \
  "tmp"
  
textgrid-tools-cli intervals split \
  "$step_dir/grids_aligned_text" -o \
  "tmp" --symbol " "

textgrid-tools-cli tier map \
  "$step_dir/grids_aligned_text" -o \
  "tmp" \
  "words"

# Map ARPAbet to IPA
cp "$step_dir/dictionary.dict" "$step_dir/transcribe.dict"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AO0" "ɔ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AO1" "ˈɔ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AO2" "ˌɔ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AO" "ɔ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "EY0" "eɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "EY1" "ˈeɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "EY2" "ˌeɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "EY" "eɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "UW0" "u"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "UW1" "ˈu"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "UW2" "ˌu"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "UW" "u"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "ER0" "ɝ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "ER1" "ˈɝ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "ER2" "ˌɝ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "ER" "ɝ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "IH0" "ɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "IH1" "ˈɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "IH2" "ˌɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "IH" "ɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "EH0" "ɛ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "EH1" "ˈɛ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "EH2" "ˌɛ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "EH" "ɛ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "IY0" "i"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "IY1" "ˈi"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "IY2" "ˌi"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "IY" "i"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "UX0" "ʉ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "UX1" "ˈʉ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "UX2" "ˌʉ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "UX" "ʉ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AA0" "ɑ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AA1" "ˈɑ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AA2" "ˌɑ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AA" "ɑ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AE0" "æ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AE1" "ˈæ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AE2" "ˌæ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AE" "æ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "OW0" "oʊ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "OW1" "ˈoʊ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "OW2" "ˌoʊ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "OW" "oʊ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AY0" "aɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AY1" "ˈaɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AY2" "ˌaɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AY" "aɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AXR0" "ɚ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AXR1" "ˈɚ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AXR2" "ˌɚ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AXR" "ɚ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AW0" "aʊ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AW1" "ˈaʊ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AW2" "ˌaʊ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AW" "aʊ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AH0" "ʌ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AH1" "ˈʌ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AH2" "ˌʌ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AH" "ʌ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "OY0" "ɔɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "OY1" "ˈɔɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "OY2" "ˌɔɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "OY" "ɔɪ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AX0" "ə"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AX1" "ˈə"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AX2" "ˌə"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "AX" "ə"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "IX0" "ɨ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "IX1" "ˈɨ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "IX2" "ˌɨ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "IX" "ɨ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "UH0" "ʊ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "UH1" "ˈʊ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "UH2" "ˌʊ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "UH" "ʊ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "SH" "ʃ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "B" "b"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "D" "d"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "EM" "m̩"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "M" "m"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "Q" "ʔ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "L" "l"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "K" "k"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "R" "ɹ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "CH" "t͡ʃ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "NG" "ŋ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "P" "p"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "Z" "z"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "ZH" "ʒ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "JH" "d͡ʒ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "T" "t"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "WH" "ʍ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "G" "ɡ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "EN" "n̩"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "NX" "ɾ̃"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "N" "n"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "F" "f"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "TH" "θ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "V" "v"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "H" "h"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "HH" "h"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "EL" "l̩"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "DH" "ð"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "S" "s"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "DX" "ɾ"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "W" "w"
dict-cli map-symbols-in-pronunciations "$step_dir/transcribe.dict" "Y" "j"

# Manually change "- -" to "—", print file path with:
echo $step_dir/transcribe.dict

dict-cli export-phonemes \
  "$step_dir/transcribe.dict" \
  "$step_dir/transcribe-phoneme-set.txt"

# Join all intervals into one interval containing the complete text
rm -rf "$step_dir/grids_aligned_text_phonemes"
cp -rf "$step_dir/grids_aligned_text" "$step_dir/grids_aligned_text_phonemes"

textgrid-tools-cli intervals join-between-pauses \
  "$step_dir/grids_aligned_text_phonemes" -o \
  "tmp" --join-with " "

# Separate each word by space and keep the space as separate interval between the words intervals (the space is needed for the next steps)
textgrid-tools-cli intervals split \
  "$step_dir/grids_aligned_text_phonemes" -o \
  "tmp" --keep

# Remove space-interval content
textgrid-tools-cli tiers remove-symbols \
  "$step_dir/grids_aligned_text_phonemes" -o \
  "tmp" --marks " "

textgrid-tools-cli tier clone \
  "$step_dir/grids_aligned_text_phonemes" -o \
  "tmp" \
  "tmp-raw"

cp "$step_dir/transcribe.dict" "$step_dir/transcribe-raw.dict"

# Remove everything except em dash; no words will be removed from the dictionary
dict-cli remove-symbols-from-pronunciations \
  "$step_dir/transcribe-raw.dict" \
  "!" "?" "." "," "-" ":" ";" "'" "\"" "(" ")" "[" "]" --mode "all"

dict-cli export-phonemes \
  "$step_dir/transcribe-raw.dict" \
  "$step_dir/transcribe-raw-phonemeset.txt"

textgrid-tools-cli tiers transcribe \
  "$step_dir/grids_aligned_text_phonemes" -o \
  "$step_dir/transcribe.dict" \
  "tmp" -s 500

textgrid-tools-cli tiers transcribe \
  "$step_dir/grids_aligned_text_phonemes" -o \
  "$step_dir/transcribe-raw.dict" \
  "tmp-raw" -s 500

# Merge special characters to next phoneme
textgrid-tools-cli intervals join-symbols \
  "$step_dir/grids_aligned_text_phonemes" -o \
  "tmp" \
  --join-symbols "!" "?" "." "," "-" ":" ";" "'" "\"" "(" ")" "[" "]" \
  --mode "right" --join-with " " \
  --ignore-join-symbols ""

textgrid-tools-cli intervals join-symbols \
  "$step_dir/grids_aligned_text_phonemes" -o \
  "tmp" \
  --join-symbols "!" "?" "." "," "-" ":" ";" "'" "\"" "(" ")" "[" "]" \
  --mode "left" --join-with " " \
  --ignore-join-symbols ""

rm -rf "$step_dir/grids_aligned_text_ipa_punc"
cp -r "$step_dir/grids_aligned_text_phonemes" "$step_dir/grids_aligned_text_ipa_punc"

textgrid-tools-cli tier clone \
  "$step_dir/grids_aligned_text_ipa_punc" -o \
  "phonemes" \
  "phonemes-raw"

textgrid-tools-cli tier map \
  "$step_dir/grids_aligned_text_ipa_punc" -o \
  "tmp-raw" \
  "phonemes-raw"

textgrid-tools-cli tier map \
  "$step_dir/grids_aligned_text_ipa_punc" -o \
  "tmp" \
  "phonemes"

textgrid-tools-cli tiers remove \
  "$step_dir/grids_aligned_text_ipa_punc" -o \
  "tmp" "tmp-raw"
  
rm -rf "$step_dir/add-duration-markers"
cp -r "$step_dir/grids_aligned_text_ipa_punc" "$step_dir/add-duration-markers"

textgrid-tools-cli grids plot-durations \
  "$step_dir/add-duration-markers" \
  "phonemes-raw" \
  "$step_dir/phoneme-durations.pdf"

textgrid-tools-cli tier clone \
  "$step_dir/add-duration-markers" -o \
  "phonemes-raw" \
  "duration-marks" --ignore-marks

# Export phoneme-set to know which phonemes should be marked
textgrid-tools-cli grids export-vocabulary \
  "$step_dir/add-duration-markers" \
  "phonemes-raw" \
  "$step_dir/add-duration-markers/phoneme-set.txt"

# Mark all phonemes (ignore "" and em dash)
# Short IPA -> "\xCC" = ˘
textgrid-tools-cli grids mark-durations \
  "$step_dir/add-duration-markers" -o \
  "phonemes-raw" \
  "duration-marks" \
  "˘" \
  --marks-mode "separate" \
  --range-mode "percentile" \
  --scope "all" \
  --range-min 0 \
  --range-max 20 \
  --min-count 10 \
  --selection "aɪ" "aʊ" "b" "d" "d͡ʒ" "eɪ" "f" "h" "i" "j" "k" "l" "m" "n" "oʊ" "p" "s" "t" "t͡ʃ" "u" "v" "w" "z" "æ" "ð" "ŋ" "ɑ" "ɔ" "ɔɪ" "ɛ" "ɝ" "ɡ" "ɪ" "ɹ" "ʃ" "ʊ" "ʌ" "ʒ" "ˈaɪ" "ˈaʊ" "ˈeɪ" "ˈi" "ˈoʊ" "ˈu" "ˈæ" "ˈɑ" "ˈɔ" "ˈɔɪ" "ˈɛ" "ˈɝ" "ˈɪ" "ˈʊ" "ˈʌ" "ˌaɪ" "ˌaʊ" "ˌeɪ" "ˌi" "ˌoʊ" "ˌu" "ˌæ" "ˌɑ" "ˌɔ" "ˌɔɪ" "ˌɛ" "ˌɝ" "ˌɪ" "ˌʊ" "ˌʌ" "θ" 

# Half long
textgrid-tools-cli grids mark-durations \
  "$step_dir/add-duration-markers" -o \
  "phonemes-raw" \
  "duration-marks" \
  "ˑ" \
  --marks-mode "separate" \
  --range-mode "percentile" \
  --scope "all" \
  --range-min 80 \
  --range-max 90 \
  --min-count 10 \
  --selection "aɪ" "aʊ" "b" "d" "d͡ʒ" "eɪ" "f" "h" "i" "j" "k" "l" "m" "n" "oʊ" "p" "s" "t" "t͡ʃ" "u" "v" "w" "z" "æ" "ð" "ŋ" "ɑ" "ɔ" "ɔɪ" "ɛ" "ɝ" "ɡ" "ɪ" "ɹ" "ʃ" "ʊ" "ʌ" "ʒ" "ˈaɪ" "ˈaʊ" "ˈeɪ" "ˈi" "ˈoʊ" "ˈu" "ˈæ" "ˈɑ" "ˈɔ" "ˈɔɪ" "ˈɛ" "ˈɝ" "ˈɪ" "ˈʊ" "ˈʌ" "ˌaɪ" "ˌaʊ" "ˌeɪ" "ˌi" "ˌoʊ" "ˌu" "ˌæ" "ˌɑ" "ˌɔ" "ˌɔɪ" "ˌɛ" "ˌɝ" "ˌɪ" "ˌʊ" "ˌʌ" "θ" 
  
# Long
textgrid-tools-cli grids mark-durations \
  "$step_dir/add-duration-markers" -o \
  "phonemes-raw" \
  "duration-marks" \
  "ː" \
  --marks-mode "separate" \
  --range-mode "percentile" \
  --scope "all" \
  --range-min 90 \
  --range-max inf \
  --min-count 10 \
  --selection "aɪ" "aʊ" "b" "d" "d͡ʒ" "eɪ" "f" "h" "i" "j" "k" "l" "m" "n" "oʊ" "p" "s" "t" "t͡ʃ" "u" "v" "w" "z" "æ" "ð" "ŋ" "ɑ" "ɔ" "ɔɪ" "ɛ" "ɝ" "ɡ" "ɪ" "ɹ" "ʃ" "ʊ" "ʌ" "ʒ" "ˈaɪ" "ˈaʊ" "ˈeɪ" "ˈi" "ˈoʊ" "ˈu" "ˈæ" "ˈɑ" "ˈɔ" "ˈɔɪ" "ˈɛ" "ˈɝ" "ˈɪ" "ˈʊ" "ˈʌ" "ˌaɪ" "ˌaʊ" "ˌeɪ" "ˌi" "ˌoʊ" "ˌu" "ˌæ" "ˌɑ" "ˌɔ" "ˌɔɪ" "ˌɛ" "ˌɝ" "ˌɪ" "ˌʊ" "ˌʌ" "θ" 

# Short break
textgrid-tools-cli grids mark-durations \
  "$step_dir/add-duration-markers" -o \
  "phonemes-raw" \
  "duration-marks" \
  "SIL1" \
  --marks-mode "separate" \
  --range-mode "percentile" \
  --scope "all" \
  --range-min 0 \
  --range-max 33.33 \
  --min-count 3 \
  --selection "" 

# Normal break
textgrid-tools-cli grids mark-durations \
  "$step_dir/add-duration-markers" -o \
  "phonemes-raw" \
  "duration-marks" \
  "SIL2" \
  --marks-mode "separate" \
  --range-mode "percentile" \
  --scope "all" \
  --range-min 33.33 \
  --range-max 66.66 \
  --min-count 3 \
  --selection "" 

# Long break
textgrid-tools-cli grids mark-durations \
  "$step_dir/add-duration-markers" -o \
  "phonemes-raw" \
  "duration-marks" \
  "SIL3" \
  --marks-mode "separate" \
  --range-mode "percentile" \
  --scope "all" \
  --range-min 66.66 \
  --range-max inf \
  --min-count 3 \
  --selection ""
  
rm -rf "$step_dir/add-SIL0"
cp -r "$step_dir/add-duration-markers" "$step_dir/add-SIL0"

textgrid-tools-cli tier clone \
  "$step_dir/add-SIL0" -o \
  "phonemes" "phonemes-tmp"
  
textgrid-tools-cli intervals join-by-boundary \
  "$step_dir/add-SIL0" -o \
  "words" "phonemes-tmp" --join-with "|"
  
textgrid-tools-cli intervals join-between-marks \
  "$step_dir/add-SIL0" -o \
  "phonemes-tmp" \
  "" --join-with "|SIL0|"

# Split SIL0
textgrid-tools-cli intervals split \
  "$step_dir/add-SIL0" -o \
  "phonemes-tmp" --symbol "|"

# Split punctuation
textgrid-tools-cli intervals split \
  "$step_dir/add-SIL0" -o \
  "phonemes-tmp" --symbol " "

rm -rf "$step_dir/merge-markers-to-phonemes"
cp -r "$step_dir/add-SIL0" "$step_dir/merge-markers-to-phonemes"

textgrid-tools-cli tier clone \
  "$step_dir/merge-markers-to-phonemes" -o \
  "phonemes-raw" "phonemes-dur"

# Merge phonemes with duration markers
textgrid-tools-cli tier map \
  "$step_dir/merge-markers-to-phonemes" -o \
  "duration-marks" \
  "phonemes-dur" \
  --filter-from --filter-to --mode "append"

# Remove markers tier
textgrid-tools-cli tiers remove \
  "$step_dir/merge-markers-to-phonemes" -o \
  "duration-marks"
  
rm -rf "$step_dir/add-complete-transcription"
cp -r "$step_dir/merge-markers-to-phonemes" "$step_dir/add-complete-transcription"
  
# Tier phonemes-tmp contains SIL0 and punctuation in separate intervals; don't ignore/filter em dash because it is also in phonemes-dur
textgrid-tools-cli tier map \
  "$step_dir/add-complete-transcription" -o \
  "phonemes-dur" \
  "phonemes-tmp" --filter-to "!" "?" "." "," "-" ":" ";" "'" "\"" "(" ")" "[" "]" "SIL0"

textgrid-tools-cli tier rename \
  "$step_dir/add-complete-transcription" -o \
  "phonemes-tmp" "transcription"

rm -rf "$step_dir/create-dictionary"
cp -r "$step_dir/add-complete-transcription" "$step_dir/create-dictionary"
  
textgrid-tools-cli tier clone \
  "$step_dir/create-dictionary" -o \
  "phonemes" "dictionary"

# Separate punctuation
textgrid-tools-cli intervals split \
  "$step_dir/create-dictionary" -o \
  "dictionary" --symbol " "
  
textgrid-tools-cli tier map \
  "$step_dir/create-dictionary" -o \
  "phonemes-dur" \
  "dictionary" \
  --filter-from "" "SIL1" "SIL2" "SIL3" \
  --filter-to "" "!" "?" "." "," "-" ":" ";" "'" "\"" "(" ")" "[" "]"
  
textgrid-tools-cli grids create-dictionary \
  "$step_dir/create-dictionary" \
  "words" "dictionary" \
  "pronunciations.dict" --scope "all"

mv "$step_dir/create-dictionary/pronunciations.dict" "$step_dir/pronunciations.dict"

textgrid-tools-cli tiers remove \
  "$step_dir/create-dictionary" -o \
  "dictionary"
  
rm -rf "$step_dir/final"
cp -r "$step_dir/create-dictionary" "$step_dir/final"

textgrid-tools-cli tiers remove \
  "$step_dir/final" -o \
  "phonemes" "phonemes-raw"
  
textgrid-tools-cli tier rename \
  "$step_dir/final" -o \
  "phonemes-dur" "phonemes"

textgrid-tools-cli tier move \
  "$step_dir/final" -o \
  "words" 1
  
textgrid-tools-cli tier move \
  "$step_dir/final" -o \
  "phonemes" 2
  
textgrid-tools-cli tier move \
  "$step_dir/final" -o \
  "transcription" 3
  
textgrid-tools-cli grids export-vocabulary \
  "$step_dir/final" \
  "words" \
  "$step_dir/final-words-vocabulary.txt"

textgrid-tools-cli grids export-vocabulary \
  "$step_dir/final" \
  "transcription" \
  "$step_dir/final-transcription-vocabulary.txt"

textgrid-tools-cli grids export-vocabulary \
  "$step_dir/final" \
  "phonemes" \
  "$step_dir/final-phonemes-vocabulary.txt"

textgrid-tools-cli grids plot-durations \
  "$step_dir/final" \
  "phonemes" \
  "$step_dir/phoneme-durations-extended.pdf"

rsync -am --include='*.wav' -f 'hide,! */' --links $step_dir/grids/ "$step_dir/final"

cp "$root_dir/ljs/filename-mapping.json" "$step_dir/final/filename-mapping.json"

rm -rf "$step_dir/LJSpeech-1.1"
dataset-converter-cli restore-structure \
  "$step_dir/final" \
  "$step_dir/LJSpeech-1.1" --symlink
  
# Remove all symbolic links to wav files
find "$step_dir/LJSpeech-1.1" -name "*.wav" -type l -delete

upload_dir="$root_dir/upload"

rm -rf "$upload_dir"
mkdir -p "$upload_dir"

cd "$step_dir/LJSpeech-1.1"
rm -rf "$upload_dir/grids.zip"
zip -q "$upload_dir/grids.zip" -r .

cp "$step_dir/final-words-vocabulary.txt" "$upload_dir/words-vocabulary.txt"
cp "$step_dir/final-phonemes-vocabulary.txt" "$upload_dir/phonemes-vocabulary.txt"
cp "$step_dir/final-transcription-vocabulary.txt" "$upload_dir/transcription-vocabulary.txt"
cp "$step_dir/phoneme-durations-extended.pdf" "$upload_dir/phonemes-durations.pdf"
cp "$step_dir/phoneme-durations.pdf" "$upload_dir/phonemes-durations-simple.pdf"
cp "$step_dir/pronunciations.dict" "$upload_dir/pronunciations.dict"