code: private static Iterator<String> getBaseIterator(BufferedReader reader,
                                                      boolean keepOrdering) {
  
          // The final iterator is how the stream will be tokenized after all the
          // tokenizing options have been applied.  This value is iteratively set
          // as the options are applied
          Iterator<String> finalIterator = new WordIterator(reader);
  
          // STEP 1: APPLY TOKEN REPLACEMENT
          if (replacementMap != null)
              finalIterator = 
                  new WordReplacementIterator(finalIterator, replacementMap);
  
          // STEP 2: APPLY COMPOUND TOKENIZING
          if (compoundTokens != null) {
              // Because the initialization step for a CWI has some overhead, use
              // the reset to keep the same tokens.  However, multiple threads may
              // be each using their own CWI, so keep Thread-local storage of what
              // CWI is being used to avoid resetting another thread's iterator.
              CompoundWordIterator cwi = 
                  compoundIterators.get(Thread.currentThread());
              if (cwi == null) {
                  cwi = new CompoundWordIterator(finalIterator, compoundTokens);
                  compoundIterators.put(Thread.currentThread(), cwi);
              } else {
                  // NOTE: if the underlying set of valid compound words is ever
                  // changed, the iterator returned from the compoundIterators map
                  // will have been updated by the setProperties() call, so this
                  // method is guaranteed to pick up the latest set of compound
                  // words
                  cwi.reset(finalIterator);
              }
              finalIterator = cwi;
          } 
  
          // STEP 3: APPLY TOKEN LIMITING
          if (wordLimit > 0)
              finalIterator = new LimitedIterator<String>(
                      finalIterator, wordLimit);
  
          // STEP 4: APPLY TOKEN FILTERING
          if (filter != null) {
              finalIterator = (keepOrdering)
                  ? new OrderPreservingFilteredIterator(finalIterator, filter)
                  : new FilteredIterator(finalIterator, filter);
          }
  
          // STEP 5: APPLY STEMMING
          if (stemmer != null) 
              finalIterator = new StemmingIterator(finalIterator, stemmer);
  
          return finalIterator;
      }
docs: Returns an iterator for the basic tokenization of the stream before
  filtering has been applied to the tokens.
  
  @param reader a reader whose contents are to be tokenized
  
  @return an iterator over the tokens in the stream