import java.util.; import java.io.;

/**

• Tokenizer serves as an implementation of the StringTokenizer class

• built into Java. It parses a given file into individual Tokens and stores

• these individual Tokens throughout.

• @author Asma Awad

• @version 1.0 */ public class Tokenizer {

  private ArrayList tokens; private int count = 0; //keeps track of the Token being looked at

  Tokenizer(String filename) throws IOException { tokens = new ArrayList<>(); //open the file Scanner infile = new Scanner(new File(filename)); StringBuilder str = new StringBuilder();

   //put each word from the file into a single string
   while (infile.hasNext()) {
       str.append(infile.next()).append(" ");
   }
   //split the string by spaces, and add each word into an array
   String[] words = str.toString().split(" ");
  
   //add each word of the array into the ArrayList as Tokens as opposed to Strings
   for (int i = 0; i < words.length; i++) {
       tokens.add(new Token(filename, words[i], i));
   }
  

  }

  /**

  • next provides the next Token to be looked at in a list of Tokens created through the Tokenizer
  • @return the next Token that has not yet been looked at */ public Token next() { //save the current Token value at count's value Token t = tokens.get(count); //increment count for when next is called again in order to return the next Token count++; return t; }

  /**

  • hasNext determines whether or not there are more Tokens to be looked at
  • @return true if there are more Tokens to be traversed, and false otherwise */ public boolean hasNext() { //if count represents an index that's out of bounds if (count >= tokens.size()) return false; return true; } }