saved models about parser HOT 5 CLOSED

    transform = state['transform']
    if args.feat == 'bert':
        tokenizer = SubwordField.tokenizer(args.bert)
        transform.FORM[1].tokenize = tokenizer.tokenize
    return cls(args, model, transform)

@classmethod
def tokenizer(cls, name):
    """                                                                                
    Create an instance of tokenizer from either path or name.                          
    :param name: path or name of tokenizer.                                            
    """

    from transformers import AutoTokenizer
    tokenizer = AutoTokenizer.from_pretrained(name)
    tokenizer.bos_token = tokenizer.bos_token or tokenizer.cls_token
tokenizer.eos_token = tokenizer.eos_token or tokenizer.sep_token
    return tokenizer

    elif args.feat == 'bert':
        tokenizer = SubwordField.tokenizer(args.bert)

    if args.feat == 'bert':
        tokenize = self.transform.FORM[1].tokenize # save it                           
        self.transform.FORM[1].tokenize = None
    state = {'name': self.NAME,
             'args': args,
             'state_dict': state_dict,
             'pretrained': pretrained,
             'transform': self.transform}
    torch.save(state, path)
    if args.feat == 'bert':
        self.transform.FORM[1].tokenize = tokenize # restore