code: @Override
    public List<List<Token>> tokenize(final String[] sentences) {
      final long start = System.nanoTime();
      int noTokens = 0;
      int prevIndex = 0;
      int curIndex = 0;
      final String language = this.lang;
      final List<List<Token>> result = new ArrayList<List<Token>>();
      // TODO improve this
      final String offsetText = this.originalText;
      for (final String sentence : sentences) {
        if (DEBUG) {
          System.err.println("-> Segmented:" + sentence);
        }
        final List<Token> tokens = new ArrayList<Token>();
        final String[] curTokens = getTokens(sentence);
        for (final String arrayToken : curTokens) {
          curIndex = offsetText.indexOf(arrayToken, prevIndex);
          // crap rule for corrected URLs
          if (curIndex == -1) {
            curIndex = prevIndex + 1;
          }
          final Token curToken = this.tokenFactory.createToken(arrayToken,
              curIndex, arrayToken.length());
          // exceptions to WFs
          addTokens(curToken, tokens);
          if (DEBUG) {
            System.err.println("-> Token:" + arrayToken + " curIndex: " + curIndex
                + " prev: " + prevIndex);
          }
          prevIndex = curIndex + curToken.tokenLength();
        }
        result.add(tokens);
        noTokens = noTokens + curTokens.length;
      }
      normalizeTokens(result, language);
      final long duration = System.nanoTime() - start;
      final double toksPerSecond = noTokens / (duration / 1000000000.0);
      if (DEBUG) {
        System.err.printf(
            "ixa-pipe-tok tokenized %d tokens at %.2f tokens per second.%n",
            noTokens, toksPerSecond);
      }
      return result;
    }
docs: /*
  (non-Javadoc)
  
  @see eus.ixa.ixa.pipe.tok.Tokenizer#tokenize(java.lang.String[])