More flexible tokenize() function about aho-corasick HOT 1 OPEN

    public Collection<Token> tokenize(final String text, final Collection<Emit> collectedEmits) {
        final Collection<Token> tokens = new ArrayList<>();
        int lastCollectedPosition = -1;

        for (final Emit emit : collectedEmits) {
            if (emit.getStart() - lastCollectedPosition > 1) {
                tokens.add(createFragment(emit, text, lastCollectedPosition));
            }

            tokens.add(createMatch(emit, text));
            lastCollectedPosition = emit.getEnd();
        }

        if (text.length() - lastCollectedPosition > 1) {
            tokens.add(createFragment(null, text, lastCollectedPosition));
        }

        return tokens;
    }

trie.tokenize(text, trie.parseText(text, new StatefulEmitHandler() {

	@Override
	public boolean emit(Emit emit) {
		// TODO Auto-generated method stub
		return false;
	}

	@Override
	public List<Emit> getEmits() {
		// TODO Auto-generated method stub
		return null;
	}
}));