准确率对比时，jieba在不同训练集上重新统计了词频信息，用什么统计? about pkuseg-python HOT 3 CLOSED

def cal(file_name, outname):
    mp = {}
    with open(file_name, encoding='utf-8') as f:
        lines = f.readlines()
    for line in lines:
        for i in line.strip().split(' '):
            if len(i)==0:
                continue
            if not i in mp:
                mp[i] = 0
            mp[i] += 1
    d = sorted(list(mp.items()), key=lambda x:(x[1], x[0]), reverse=True)
    with open(outname, 'w', encoding='utf-8') as f:
        for i, freq in d:
            f.write(i+' '+str(freq)+'\n')
cal('icwb2-data/training/msr_training.utf8', 'msr_dict.txt')

tokenizer = jieba.Tokenizer(dictionary='msr_dict.txt')
tokenizer.cut(data)