root_dir="/tmp/pinyin-ipa"

# Update package list
sudo apt update

# Install Python 3.8
sudo apt install python3-pip python3.8 python3.8-dev python3.8-distutils python3.8-venv -y

# Empty "root_dir"
rm -rf "$root_dir"
mkdir -p "$root_dir"

# Create virtual environment
# All tools will be installed there and can only be run if the venv is activated
python3.8 -m venv "$root_dir/.venv"

# Activate environment
source "$root_dir/.venv/bin/activate"

# Install/update tools for installing software
python3.8 -m pip install \
  wheel==0.38.4 \
  setuptools==65.6.3

# Install tools
python3.8 -m pip install \
  pronunciation-dictionary-utils==0.0.2 \
  dict-from-pypinyin==0.0.1 \
  pinyin-to-ipa==0.0.1 \
  txt-utils==0.0.1
  
# Download corpus
wget https://downloads.wortschatz-leipzig.de/corpora/zho_wikipedia_2018_1M.tar.gz \
  -O "$root_dir/corpus.tar.gz"
  
rm -rf "$root_dir/corpus"
mkdir -p "$root_dir/corpus"
tar -xf "$root_dir/corpus.tar.gz" \
  -C "$root_dir/corpus"

cp "$root_dir/corpus/zho_wikipedia_2018_1M/zho_wikipedia_2018_1M-words.txt" "$root_dir/words.txt"

# Keep only words
txt-utils-cli replace \
  "$root_dir/words.txt" \
  ".+\t(.+)\t.+" "\1"

# Extract all hanzi syllables
txt-utils-cli extract-vocabulary \
  "$root_dir/words.txt" \
  "$root_dir/syllables-hanzi.txt" --sep ""

upload_dir="$root_dir/upload"
rm -rf "$upload_dir"
mkdir -p "$upload_dir"

# Different styles
style="NORMAL"
style="TONE"
style="TONE2"
style="TONE3"

# Create dictionary out of syllables
dict-from-pypinyin-cli \
  "$root_dir/syllables-hanzi.txt" \
  "$root_dir/dictionary-$style.dict" \
  --oov "$root_dir/oov-$style.txt" \
  --style "$style" --trim --neutral-tone-with-five

# Assume that OOV of all Styles is equal therefore overwriting does not matter
cp "$root_dir/oov-$style.txt" "$upload_dir/oov-vocabulary.txt"

# Export phonemes
dict-cli export-vocabulary \
  "$root_dir/dictionary-$style.dict" \
  "$root_dir/syllables-pinyin-$style-vocab.txt"

# Assume that vocabulary of all Styles is equal therefore overwriting does not matter
cp "$root_dir/syllables-pinyin-$style-vocab.txt" "$upload_dir/hanzi-vocabulary.txt"

# Export phonemes
dict-cli export-phonemes \
  "$root_dir/dictionary-$style.dict" \
  "$root_dir/syllables-pinyin-$style.txt"

voc_path="$root_dir/syllables-pinyin-$style.txt"
ipa_path="$root_dir/syllables-pinyin-$style-ipa.json"
ipa_oov_path="$root_dir/syllables-pinyin-$style-ipa-oov.txt"

# add new newline at end
echo "" >> "$voc_path"

# Create OOV file
rm -rf "$ipa_oov_path"
touch "$ipa_oov_path"

# Create map
echo "{" > "$ipa_path"
cat "$voc_path" | 
while read -r pinyin
do
ipa=$(pinyin-to-ipa-cli $pinyin --first --sep " ")
if [ ! -z "$ipa" ]
then
echo "  \"$pinyin\": \"$ipa\"," >> "$ipa_path"
else
echo "$pinyin" >> "$ipa_oov_path"
fi
done
echo "}" >> "$ipa_path"

# Remove last ","
txt-utils-cli replace \
  "$ipa_path" \
  ",\n}\n" "\n}"

# Copy to upload directory
cp "$ipa_path" "$upload_dir/pinyin-ipa-map-$style.json"

# Tone3 - all possibilities

tone3_voc_gen_path="$root_dir/syllables-pinyin-tone3-all.txt"

# Create file
rm -rf "$tone3_voc_gen_path"
touch "$tone3_voc_gen_path"

cat "$root_dir/syllables-pinyin-NORMAL.txt" | 
while read -r pinyin
do
cat >> "$tone3_voc_gen_path" << EOF
${pinyin}
${pinyin}1
${pinyin}2
${pinyin}3
${pinyin}4
${pinyin}5
EOF
done

voc_path="$tone3_voc_gen_path"
ipa_path="$root_dir/syllables-pinyin-TONE3-all-ipa.json"
ipa_oov_path="$root_dir/syllables-pinyin-TONE3-all-ipa-oov.txt"

# Create OOV file
rm -rf "$ipa_oov_path"
touch "$ipa_oov_path"

# Create map
echo "{" > "$ipa_path"
cat "$voc_path" | 
while read -r pinyin
do
ipa=$(pinyin-to-ipa-cli $pinyin --first --sep " ")
if [ ! -z "$ipa" ]
then
echo "  \"$pinyin\": \"$ipa\"," >> "$ipa_path"
else
echo "$pinyin" >> "$ipa_oov_path"
fi
done
echo "}" >> "$ipa_path"

# Remove last ","
txt-utils-cli replace \
  "$ipa_path" \
  ",\n}\n" "\n}"

# Copy to upload directory
cp "$ipa_path" "$upload_dir/pinyin-ipa-map-TONE3-all.json"
