Dependency-Based Self-Attention for Transformer NMT (Deguchi et al., 2019)
git clone https://github.com/de9uch1/dbsa.git
cd dbsa/
poetry install
git clone https://github.com/de9uch1/dbsa.git
cd dbsa/
pip install -e ./
Training a Transformer + DBSA model on ASPEC-JE
1. Extract and preprocess the ASPEC-JE data (including dependency parsing)
export NUM_WORKERS=8 # specify the number of CPUs
bash scripts/prepare-aspec.sh
2. Preprocess the dataset
# binarize the dataset
fairseq-preprocess --source-lang ja --target-lang en \
--trainpref aspec_ja_en/train \
--validpref aspec_ja_en/valid \
--testpref aspec_ja_en/test \
--destdir data-bin/ \
--workers $NUM_WORKERS
# deploy the dependency labels
for split in train valid test; do
for l in ja en; do
cp aspec_ja_en/$split.$l.dep data-bin/$split.dep.$l
done
done
fairseq-train \
data-bin/ \
--user-dir dbsa/ \
--task translation_dep \
--arch transformer_dep \
--optimizer adam --adam-betas '(0.9, 0.98)' --clip-norm 0.0 --activation-fn relu \
--lr 7e-4 --lr-scheduler inverse_sqrt --warmup-updates 4000 --warmup-init-lr 1e-7 \
--dropout 0.1 --attention-dropout 0.1 --activation-dropout 0.1 --weight-decay 0.0 \
--max-tokens 12000 --label-smoothing 0.1 \
--save-dir ./checkpoints --log-interval 100 --max-update 100000 \
--keep-interval-updates -1 --save-interval-updates 0 \
--load-dependency --criterion label_smoothed_cross_entropy_with_dependency \
--source-dependency-lambda 0.5 --target-dependency-lambda 0.5 \
--dependency-layer 2 \
--fp16
4. Evaluate and generate the translations
fairseq-generate \
data-bin --gen-subset test \
--user-dir dbsa/ \
--task translation_dep \
--path checkpoints/checkpoint_last.pt \
--max-len-a 1 --max-len-b 50 \
--remove-bpe \
--beam 4 --nbest 1
- You can also generate dependencies (BPE level)
python ./generate.py \
data-bin --gen-subset test \
--user-dir dbsa/ \
--task translation_dep \
--path checkpoints/checkpoint_last.pt \
--max-len-a 1 --max-len-b 50 \
--print-dependency \
--beam 4 --nbest 1