code: public Token[] tokenize(String input) 
      {
          List tokens = new ArrayList();
          int cursor = 0;
          while (cursor<input.length()) {
  	    char ch = input.charAt(cursor);
  	    if (Character.isWhitespace(ch)) {
                  cursor++;
  	    } else if (Character.isLetter(ch)) {
                  StringBuffer buf = new StringBuffer("");
                  while (cursor<input.length() && Character.isLetter(input.charAt(cursor))) {
                      buf.append(input.charAt(cursor));
                      cursor++;
                  }
                  tokens.add(internSomething(buf.toString()));
  	    } else if (Character.isDigit(ch)) {
                  StringBuffer buf = new StringBuffer("");
                  while (cursor<input.length() && Character.isDigit(input.charAt(cursor))) {
                      buf.append(input.charAt(cursor));
                      cursor++;
                  }
                  tokens.add(internSomething(buf.toString()));
  	    } else {
                  if (!ignorePunctuation) {
                      StringBuffer buf = new StringBuffer("");
                      buf.append(ch);
                      String str = buf.toString();
                      tokens.add(internSomething(str));
                  }
                  cursor++;
  	    }
          }
          return (Token[]) tokens.toArray(new BasicToken[tokens.size()]);
      }
docs: Return tokenized version of a string.  Tokens are sequences
  of alphanumerics, or any single punctuation character.