code: public Collection<Token> tokenize(String text)
      {
  
          Collection<Token> tokens = new ArrayList<Token>();
  
          Collection<Emit> collectedEmits = parseText(text);
          // 下面是最长分词的关键
          IntervalTree intervalTree = new IntervalTree((List<Intervalable>) (List<?>) collectedEmits);
          intervalTree.removeOverlaps((List<Intervalable>) (List<?>) collectedEmits);
          // 移除结束
  
          int lastCollectedPosition = -1;
          for (Emit emit : collectedEmits)
          {
              if (emit.getStart() - lastCollectedPosition > 1)
              {
                  tokens.add(createFragment(emit, text, lastCollectedPosition));
              }
              tokens.add(createMatch(emit, text));
              lastCollectedPosition = emit.getEnd();
          }
          if (text.length() - lastCollectedPosition > 1)
          {
              tokens.add(createFragment(null, text, lastCollectedPosition));
          }
  
          return tokens;
      }
docs: 一个最长分词器
  
  @param text 待分词文本
  @return