Adding preprocessing steps

preprocess.sh

+#!/bin/sh
+
+# Directories
+#############
+
+echo Choose directory in the data folder
+read DIR
+export DIR
+mkdir -p ./data/${DIR}/subword
+mkdir -p ./data/${DIR}/vocab
+mkdir -p ./data/${DIR}/tok
+
+# Tokenization
+##############
+
+sacremoses -l en tokenize < ./data/${DIR}/trn.en > ./data/${DIR}/tok/trn.en
+sacremoses -l fr tokenize < ./data/${DIR}/trn.fr > ./data/${DIR}/tok/trn.fr
+sacremoses -l en tokenize < ./data/${DIR}/val.en > ./data/${DIR}/tok/val.en
+sacremoses -l fr tokenize < ./data/${DIR}/val.fr > ./data/${DIR}/tok/val.fr
+sacremoses -l en tokenize < ./data/${DIR}/tra.en > ./data/${DIR}/tok/tra.en
+sacremoses -l fr tokenize < ./data/${DIR}/tra.fr > ./data/${DIR}/tok/tra.fr
+
+# Subword segmentation
+######################
+
+# For fine-tuning
+
+spm_train \
+	--input=./data/books/tok/trn.en,./data/europarl/tok/trn.en,./data/globalvoices/tok/trn.en,./data/news/tok/trn.en,./data/ted/tok/trn.en,./data/${DIR}/tok/trn.en, \
+	--model_prefix=./data/${DIR}/subword/unigram_multi_en \
+	--vocab_size=32000 \
+	--character_coverage=1.0 \
+	--model_type=unigram
+
+spm_train \
+	--input=./data/books/tok/trn.fr,./data/europarl/tok/trn.fr,./data/globalvoices/tok/trn.fr,./data/news/tok/trn.fr,./data/ted/tok/trn.fr,./data/${DIR}/tok/trn.fr \
+	--model_prefix=./data/${DIR}/subword/unigram_multi_fr \
+	--vocab_size=32000 \
+	--character_coverage=1.0 \
+	--model_type=unigram
+
+spm_encode \
+	--model=./data/${DIR}/subword/unigram_multi_en.model \
+	--output_format=piece \
+	< ./data/${DIR}/tok/tra.en \
+	> ./data/${DIR}/tok/tra_unigram_multi.en
+	
+# For video game only
+
+spm_train \
+	--input=./data/${DIR}/tok/trn.en, \
+	--model_prefix=./data/${DIR}/subword/unigram_only_en \
+	--vocab_size=32000 \
+	--character_coverage=1.0 \
+	--model_type=unigram
+
+spm_train \
+	--input=./data/${DIR}/tok/trn.fr \
+	--model_prefix=./data/${DIR}/subword/unigram_only_fr \
+	--vocab_size=32000 \
+	--character_coverage=1.0 \
+	--model_type=unigram
+
+spm_encode \
+	--model=./data/${DIR}/subword/unigram_only_en.model \
+	--output_format=piece \
+	< ./data/${DIR}/tok/tra.en \
+	> ./data/${DIR}/tok/tra_unigram_only.en
+
+# Building vocab
+################
+
+onmt_build_vocab --config configs/${DIR}_only.yaml --n_sample -1
+
+onmt_build_vocab --config configs/${DIR}_tuned.yaml --n_sample -1
\ No newline at end of file