export LNG=de && python3 lm/preprocessing.py with configs/${LNG}_word_pp.json >configs/${LNG}_word_pp.json.output

export LNG=es && CUDA_VISIBLE_DEVICES=3 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &

WORD LMs

• sl: gpu 0 cutoff 20 started MONDAY 18h export LNG=sl && CUDA_VISIBLE_DEVICES=0 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &

• pt: gpu 1 cutoff 98

  • command export LNG=pt && CUDA_VISIBLE_DEVICES=1 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &
  • start 2019-05-06 18:35:46 end training duration validation ppl

• es: gpu 3 cutoff 200

  • command export LNG=es && CUDA_VISIBLE_DEVICES=3 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &
  • start 2019-05-06 18:27:23 end training duration validation ppl

• de: gpu 4 Flair vocabulary of types (UNK <= 220) has 100398 types. Coverage: 0.780

  • command export LNG=de && CUDA_VISIBLE_DEVICES=4 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &
  • start 2019-05-07 02:11:03 end training duration validation ppl

• sv: gpu 2 Flair vocabulary of types (UNK <= 49) has 90786 types. Coverage: 0.8

  • command export LNG=sv && CUDA_VISIBLE_DEVICES=2 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &
  • start `` end training duration ` ` validation ppl

• hu: gpu 2 Flair vocabulary of types (UNK <= 4) has 532247 types. Coverage: 0.797
  UNK has a train set frequency of 3293080 (max: 6812055) and a relative frequency of 0.04075

  • command export LNG=hu && CUDA_VISIBLE_DEVICES=2 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &
  • start `` end training duration ` ` validation ppl

• is: gpu 2

ON s3it

• is: Flair vocabulary of types (UNK <= 4) has 59766 types. Coverage: 0.824
  UNK has a train set frequency of 379524 (max: 315204) and a relative frequency of 0.07392
  export LNG=is && CUDA_VISIBLE_DEVICES=0 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &

CHAR LMs

export LNG=XXXX && python3 lm/preprocessing.py with configs/${LNG}_char_pp.json >configs/${LNG}_char_pp.json.output

export LNG=XXXX && CUDA_VISIBLE_DEVICES=666 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log &

• is: PID 8938 gpu 2 cutoff 14 export LNG=is && CUDA_VISIBLE_DEVICES=2 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log &

• es: gpu ? cutoff 14 export LNG=es && CUDA_VISIBLE_DEVICES=3 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log &

• sl: PID 16202 gpu 1 cutoff 51 export LNG=sl && CUDA_VISIBLE_DEVICES=1 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log &

• pt: PID 13588 gpu 3 cutoff 346 export LNG=pt && CUDA_VISIBLE_DEVICES=3 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log &
  -de: gpu4 export LNG=de && CUDA_VISIBLE_DEVICES=4 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log &

-en: gpu2 export LNG=en && CUDA_VISIBLE_DEVICES=2 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log &

deleting the corpus output directory of a language

rm -r wplmdata-preprocessed/sl/