Prepare WMT'17 Datasets about seq2seq HOT 3 OPEN

#! /usr/bin/env bash
# Dependencies:
#   - https://github.com/google/sentencepiece

CORPUS_DIR=$(pwd)
SOURCE_LANG="en"
TARGET_LANG="de"
VOCBA_SIZE=32000

# Learn BPE across both corpora
spm_train \
  --input=${CORPUS_DIR}/corpus.tc.${SOURCE_LANG},${CORPUS_DIR}/corpus.tc.${TARGET_LANG} \
  --model_prefix=${CORPUS_DIR}/bpe \
  --vocab_size=32000 \
  --model_type=bpe

# Apply BPE to corpus
spm_encode --model=${CORPUS_DIR}/bpe.model --output_format=piece \
    < ${CORPUS_DIR}/corpus.tc.${SOURCE_LANG} \
    > ${CORPUS_DIR}/corpus.tc.bpe.${SOURCE_LANG}
spm_encode --model=${CORPUS_DIR}/bpe.model --output_format=piece \
    < ${CORPUS_DIR}/corpus.tc.${TARGET_LANG} \
    > ${CORPUS_DIR}/corpus.tc.bpe.${TARGET_LANG}

# Apply BPE to all dev data
for lang in ${SOURCE_LANG} ${TARGET_LANG}; do
  for infile in $(find ${CORPUS_DIR}/dev | grep tc.${lang}); do
    echo $infile
    outfile="${infile%.*}.bpe.${lang}"
    spm_encode --model=${CORPUS_DIR}/bpe.model --output_format=piece < $infile > $outfile
    echo $outfile
  done
done