code: @Override
      @SuppressWarnings("StringSplitter")
      public boolean incrementToken() throws IOException {
          final ArrayDeque<String> tokens = getTokens();
          final CharTermAttribute termAtt = getTermAtt();
          if (tokens.isEmpty() && input.incrementToken()) {
              final String text = new String(termAtt.buffer(), 0, termAtt.length());
              if (UrlStringUtils.containsUrl(text)) {
                  final String[] parts = text.split("\\s");
                  for (String part : parts) {
                      if (UrlStringUtils.isUrl(part)) {
                          try {
                              final List<String> data = UrlStringUtils.extractImportantUrlData(part);
                              tokens.addAll(data);
                          } catch (MalformedURLException ex) {
                              LOGGER.debug("error parsing {}", part, ex);
                              tokens.add(part);
                          }
                      } else {
                          tokens.add(part);
                      }
                  }
              } else {
                  tokens.add(text);
              }
          }
          return addTerm();
      }
docs: Increments the underlying TokenStream and sets CharTermAttributes to
  construct an expanded set of tokens by concatenating tokens with the
  previous token.
  
  @return whether or not we have hit the end of the TokenStream
  @throws IOException is thrown when an IOException occurs