# Download THCHS-30 dataset from OpenSLR: https://www.openslr.org/18/
wget https://openslr.elda.org/resources/18/data_thchs30.tgz \
  -O "$ROOT_DIR/data_thchs30.tgz"

# Unpack dataset
rm -rf "$ROOT_DIR/THCHS-30"
mkdir -p "$ROOT_DIR/THCHS-30"
tar zxf "$ROOT_DIR/data_thchs30.tgz" \
  -C "$ROOT_DIR/THCHS-30"

# Convert dataset to a generic dataset to have all speakers separated in single folders
rm -rf "$ROOT_DIR/data"
dataset-converter-cli convert-thchs \
  "$ROOT_DIR/THCHS-30/data_thchs30" \
  "$ROOT_DIR/data" \
  --tier "Symbols" \
  --symlink --add-punctuation-marks

wget https://zenodo.org/record/7528596/files/grids-sdp.zip \
  -O "$ROOT_DIR/grids-sdp.zip"

rm -rf "$ROOT_DIR/dataset"
unzip -q "$ROOT_DIR/grids-sdp.zip" -d "$ROOT_DIR/dataset"

rsync -am --include='*.wav' -f 'hide,! */' --links $ROOT_DIR/data/ "$ROOT_DIR/dataset"

# Grid duration statistics:
#  #Grids  Min (s)  Median (s)  Max (s)  Mean (s)  SD (s)  Kurt (s)     Sum (s)  Sum (min)  Sum (h)
#   13388    4.464       9.114  16.3125    9.1853  1.4836    0.3851 122972.5915  2049.5432  34.1591
textgrid-tools-cli grids plot-stats \
  "$ROOT_DIR/dataset" \
  "$ROOT_DIR/dataset-plot.pdf" \
  "$ROOT_DIR/dataset-marks.csv"

# Remove SILX intervals (= pauses at start/end)
rm -rf "$ROOT_DIR/dataset-cut"
textgrid-tools-cli intervals remove \
  "$ROOT_DIR/dataset" \
  "transcription" "SILX" --mode "both" \
  --output-directory "$ROOT_DIR/dataset-cut" -o \
  --output-audio-directory "$ROOT_DIR/dataset-cut"

# Remove not required tier "phonemes"
textgrid-tools-cli tiers remove \
  "$ROOT_DIR/dataset-cut" -o \
  "phonemes"

# Normalize audio files and convert them to 22,050 Hz
cat > "$ROOT_DIR/prepare_audio.sh" << EOF
#!/bin/bash
base_dir="$ROOT_DIR/dataset-cut"
target_dir="$ROOT_DIR/dataset-cut-normed"
base_dir_len=\${#base_dir}
for i; do
    i_len=\${#i}
    rel_path=\${i:base_dir_len:i_len-base_dir_len}
    target_path=\$target_dir\$rel_path
    # echo \$target_path
    d=\$(dirname "\$target_path")
    mkdir -p \$d
    file_name=\$(basename "\$i")
    ffmpeg -i "\$i" -acodec pcm_s16le -ar 22050 -ac 1 -af loudnorm=I=-16:LRA=11:TP=-1.5 -hide_banner -loglevel error \$target_path -y
done
EOF
sudo chmod +x $ROOT_DIR/prepare_audio.sh
rm -rf "$ROOT_DIR/dataset-cut-normed"
find "$ROOT_DIR/dataset-cut" -name '*.wav' -print0 | xargs -P 16 -0 $ROOT_DIR/prepare_audio.sh

rsync -am --include='*.TextGrid' -f 'hide,! */' $ROOT_DIR/dataset-cut/ "$ROOT_DIR/dataset-cut-normed"

# Sync TextGrid duration with new audio duration (could be changed through sampling rate conversion)
textgrid-tools-cli grid sync \
  "$ROOT_DIR/dataset-cut-normed" -o
  
# Grid duration statistics:
#  #Grids  Min (s)  Median (s)  Max (s)  Mean (s)  SD (s)  Kurt (s)     Sum (s)  Sum (min)  Sum (h)
#   13388     3.77        7.65    13.85    7.7292  1.3055    0.6287 103478.6012  1724.6434  28.7441
textgrid-tools-cli grids plot-stats \
  "$ROOT_DIR/dataset-cut-normed" \
  "$ROOT_DIR/dataset-cut-normed-plot.pdf" \
  "$ROOT_DIR/dataset-cut-normed-marks.csv"
