code: public Span[] tokenizePos(String d) {
      Span[] tokens = split(d);
      newTokens.clear();
      tokProbs.clear();
      for (int i = 0, il = tokens.length; i < il; i++) {
        Span s = tokens[i];
        String tok = d.substring(s.getStart(), s.getEnd());
        // Can't tokenize single characters
        if (tok.length() < 2) {
          newTokens.add(s);
          tokProbs.add(ONE);
        }
        else if (useAlphaNumericOptimization() && alphaNumeric.matcher(tok).matches()) {
          newTokens.add(s);
          tokProbs.add(ONE);
        }
        else {
          int start = s.getStart();
          int end = s.getEnd();
          final int origStart = s.getStart();
          double tokenProb = 1.0;
          for (int j = origStart + 1; j < end; j++) {
            double[] probs =
              model.eval(cg.getContext(new ObjectIntPair(tok, j - origStart)));
            String best = model.getBestOutcome(probs);
            //System.err.println("TokenizerME: "+tok.substring(0,j-origStart)+"^"+tok.substring(j-origStart)+" "+best+" "+probs[model.getIndex(best)]);
            tokenProb *= probs[model.getIndex(best)];
            if (best.equals(TokContextGenerator.SPLIT)) {
              newTokens.add(new Span(start, j));
              tokProbs.add(new Double(tokenProb));
              start = j;
              tokenProb = 1.0;
            }
          }
          newTokens.add(new Span(start, end));
          tokProbs.add(new Double(tokenProb));
        }
      }
  
      Span[] spans = new Span[newTokens.size()];
      newTokens.toArray(spans);
      return spans;
    }
docs: Tokenizes the string.
  
  @param d  The string to be tokenized.
  @return   A span array containing individual tokens as elements.