.
├── EACL_exp      # experiments in the EACL paper
│   ├── m2converter # script to create m2 format from plain texts
│   ├── mturk     # mechanical turk experiments
│   │   ├── sample.csv
│   │   ├── pairwise.csv
│   │   └── template.html
│   └── manual_eval # manual analysis of 100 sentences
│       ├── README.md
│       └── coded_sentences.csv
├── README.md     # This file
├── EACLshort037.pdf
├── dev           # dev set (754 sentences originally from the GUG **test** set)
│   ├── dev.ref0
│   ├── dev.ref1
│   ├── dev.ref2
│   ├── dev.ref3
│   ├── dev.spellchecked.src (spellchecked by enchant)
│   └── dev.src   # source (This should be the input for your system.)
├── eval
│   └── gleu.py   # evaluation script (sentence-level GLEU score)
└── test          # test set (747 sentenses ogirinally from the GUG **dev** set)
    ├── test.ref0
    ├── test.ref1
    ├── test.ref2
    ├── test.ref3
    ├── test.spellchecked.src (spellchecked by enchant)
    └── test.src   # source (This should be the input for your system.)

e.g. python ./eval/gleu.py -r ./dev/dev.ref[0-3] -s ./dev/dev.src --hyp YOUR_SYSTEM_OUTPUT

This returns the mean, standard deviation, and confidence interval.

@InProceedings{napoles-sakaguchi-tetreault:2017:EACLshort,
  author    = {Napoles, Courtney  and  Sakaguchi, Keisuke  and  Tetreault, Joel},
  title     = {JFLEG: A Fluency Corpus and Benchmark for Grammatical Error Correction},
  booktitle = {Proceedings of the 15th Conference of the European Chapter of the Association for Computational Linguistics: Volume 2, Short Papers},
  month     = {April},
  year      = {2017},
  address   = {Valencia, Spain},
  publisher = {Association for Computational Linguistics},
  pages     = {229--234},
  url       = {http://www.aclweb.org/anthology/E17-2037}
}

@InProceedings{heilman-EtAl:2014:P14-2,
  author    = {Heilman, Michael  and  Cahill, Aoife  and  Madnani, Nitin  and  Lopez, Melissa  and  Mulholland, Matthew  and  Tetreault, Joel},
  title     = {Predicting Grammaticality on an Ordinal Scale},
  booktitle = {Proceedings of the 52nd Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)},
  month     = {June},
  year      = {2014},
  address   = {Baltimore, Maryland},
  publisher = {Association for Computational Linguistics},
  pages     = {174--180},
  url       = {http://www.aclweb.org/anthology/P14-2029}
}