code: public List<String> tokenize(String text) {
  		List<String> tokens = new ArrayList<String>();
  		text = text.toLowerCase();
  		char[] array = text.toCharArray();
  		
  		//convert all apostrophes to 0x27
  		for(int i = 0; i < array.length; i++) {
  			if(isApostrofe(array[i])) array[i] = 0x27;
  		}
  		
  		int pos = 0;
  		
  		//term cycle
  		while(pos < array.length) {
  			//scan to the start of the term
  			while(pos < array.length  && !Character.isLetterOrDigit(array[pos])) pos++;
  			int start = pos;
  			if(start == array.length) break;
  			//scan to the end of the term
  			while(pos < array.length  && isLetterOrDigitOrApostrofe(array[pos])) pos++;
  			int newpos = pos;
  			while(newpos > start && isApostrofe(array[newpos - 1])) newpos--;
  			if(newpos > start) tokens.add(new String(array, start, newpos - start));
  		}
  		return tokens;
  	}
docs: get list of tokens from the text for indexing