code: @Override
  	public List<Token> tokenize(String s) {
  		List<Token> result = new ArrayList<Token>();
  		result.add(new UndefinedToken(s));
  
  		for (PredefinedTokenDefinition predefinedTokenDefinition : _predefinedTokenDefitions) {
  			Set<Pattern> patterns = predefinedTokenDefinition.getTokenRegexPatterns();
  			for (Pattern pattern : patterns) {
  				for (ListIterator<Token> it = result.listIterator(); it.hasNext();) {
  					Token token = it.next();
  					if (token instanceof UndefinedToken) {
  						List<Token> replacementTokens = tokenizeInternal(token.getString(), predefinedTokenDefinition,
  								pattern);
  						if (replacementTokens.size() > 1) {
  							it.remove();
  							for (Token newToken : replacementTokens) {
  								it.add(newToken);
  							}
  						}
  					}
  				}
  			}
  		}
  
  		return result;
  	}
docs: Will only return either tokens with type PREDEFINED or UNDEFINED