public interface Modifier {

    /**
     * Processes the characters in the given character buffer, i.e. deletes or replaces or inserts characters, or keeps
     * the characters as they are.
     * 
     * @param characterBuffer
     *            The next characters provided from the modifiable stream. It contains the modifiable characters,
     *            (optionally) preceded by unmodifiable characters.
     *            <p>
     *            The modifier can modify the content of the buffer as appropriate the modifier must not modify the
     *            unmodifiable characters, i.e. the characters before position
     *            <code>firstModifiableCharacterInBufferIndex</code> must not be modified.
     *            <p>
     *            Your modifier should manage the {@link StringBuilder#capacity() capacity} of the buffer on its own -
     *            as the optimal management of the capacity depends on the specific purpose of the modifier.
     *            <p>
     *            The given buffer must be never null.
     * @param firstModifiableCharacterInBuffer
     *            index of the first modifiable character in the buffer. The index of the first character in the buffer
     *            is zero. If there is not modifiable character in the buffer, then this value is equal the length of
     *            the buffer.
     * @param endOfStreamHit
     *            True if no more characters can be read from the stream, i.e. the character buffer contains the all the
     *            characters up to the end of the stream.
     * @return Returns an object that defines how {@link ModifyingReader} or {@link ModifyingWriter} shall behave before
     *         calling {@link Modifier#modify(StringBuilder, int, boolean)} again.
     */
    public AfterModification modify(StringBuilder characterBuffer, int firstModifiableCharacterInBuffer,
            boolean endOfStreamHit);
}
 public class ModifyingReader extends Reader {

    //
    // injected properties
    //

    /**
     * The reader that the reads from the underlying character stream.
     */
    protected Reader delegate;

    /**
     * The modifier thats replaces, deletes, inserts characters of the underlying stream, and manages the buffer size.
     */
    private Modifier modifier;

    //
    // properties that represent the mutable state
    //

    /**
     * At its begin the buffer contains unmodifiable characters (might be needed for look-behind matches as known from
     * regular expression matching). After the unmodifiable characters the buffer holds the modifiable characters. See
     * {@link #firstModifiableCharacterInBuffer}.
     */
    private StringBuilder characterBuffer;

    /**
     * The position of the first character in the {@link #characterBuffer input buffer} that is modifiable.
     * <p>
     * This character and the following characters in {@link #characterBuffer} can be modified by the {@link #modifier}.
     */
    private int firstModifiableCharacterInBuffer;

    /**
     * The value is taken from {@link AfterModification#getNewMinimumLengthOfLookBehind()} after
     * {@link Modifier#modify(StringBuilder, int, boolean)} is called. The value is <em>requested</em> by the
     * {@link Modifier}.
     * <p>
     * If this value is greater than {@link #firstModifiableCharacterInBuffer}, the modifier is probably faulty.
     */
    private int minimumLengthOfLookBehind;

    /**
     * The value is the sum of {@link AfterModification#getNewMinimumLengthOfLookBehind()} and
     * {@link AfterModification#getNewNumberOfChars()} after {@link Modifier#modify(StringBuilder, int, boolean)} is
     * called. The value is <em>requested</em> by the {@link Modifier}.
     */
    private int requestedNumCharactersInBuffer;

    /**
     * The value is initially taken from {@link AfterModification#getNumberOfCharactersToSkip()} after
     * {@link Modifier#modify(StringBuilder, int, boolean)} is called. Then during reading characters from the reader
     * this value is decremented.
     */
    private int numberOfCharactersToSkip;

    /**
     * True if the end of stream was detected.
     */
    private boolean endOfStreamHit = false;

    /**
     * The holds the last {@link AfterModification} provided by the {@link #modifier}. This property serves debugging
     * purposes only.
     */
    private AfterModification lastAfterModificationForDebuggingOnly = null;

    /**
     * @param reader
     *            The underlying reader that provides the original, not modified characters. For optimal performance
     *            choose a {@link BufferedReader}.
     * @param modifier
     *            the object that modifies the stream.
     */
    public ModifyingReader(Reader reader, Modifier modifier) {
        super();

        this.delegate = reader;

        // take injected properties
        this.modifier = modifier;

        // initialize the mutable state
        this.numberOfCharactersToSkip = 0;
        this.minimumLengthOfLookBehind = 0; // we cannot do anything else
        this.firstModifiableCharacterInBuffer = 0;
        this.requestedNumCharactersInBuffer = 1; // any value
        this.characterBuffer = new StringBuilder(requestedNumCharactersInBuffer);
        adjustCapacityOfBuffer();
    }

    /**
     * Updates the input buffer according to {@link #minimumLengthOfLookBehind} and
     * {@link #requestedNumCharactersInBuffer}, and then fills the buffer up to its capacity.
     */
    private void updateBuffer() throws IOException {

        removeCharactersInBufferNotNeededAnyLonger();

        adjustCapacityOfBuffer();

        fill();
    }

    /**
     * Deletes those characters at the start of the buffer we do not need any longer.
     */
    private void removeCharactersInBufferNotNeededAnyLonger() {

        int charactersToDelete = firstModifiableCharacterInBuffer - minimumLengthOfLookBehind;

        if (charactersToDelete > 0) {
            characterBuffer.delete(0, charactersToDelete);
            firstModifiableCharacterInBuffer -= charactersToDelete;
        } else if (charactersToDelete < 0) {
            onFaultyModifier(-52, charactersToDelete + " characters to delete but this is not possible ("
                    + firstModifiableCharacterInBuffer + "," + minimumLengthOfLookBehind + ")");
        }
    }

    /**
     * Adjusts the capacity of the buffer.
     */
    protected void adjustCapacityOfBuffer() {

        // is the current capacity not big enough?
        if (characterBuffer.capacity() < requestedNumCharactersInBuffer) {

            // increase the capacity (we delegate to the default behavior of
            // StringBuffer)
            characterBuffer.ensureCapacity(requestedNumCharactersInBuffer);

        }

    }

    /**
     * Reads more characters into the input buffer. This method will block until the buffer is filled with
     * {@link #requestedNumCharactersInBuffer} characters , or an I/O error occurs, or the end of the stream is reached.
     */
    private void fill() throws IOException {

        int length = requestedNumCharactersInBuffer - characterBuffer.length();
        if (length <= 0) {
            // nothing to do
            return;
        }

        char[] buffer = new char[length];
        int offset = 0;

        while (length > 0) {

            int readChars = delegate.read(buffer, offset, length);
            if (readChars != -1) {
                characterBuffer.append(buffer, offset, readChars);
                offset += readChars;
                length -= readChars;
            } else {
                endOfStreamHit = true;
                break;
            }

        }

    }

    protected void onFaultyModifier(int errorCode, String errorMessage) {
        // should we silently ignore any errors and fallback to a meaningful
        // behavior? No because a faulty modifier should be fixed by a
        // ValidModifier, i.e. the user must take the responsibility for the
        // errors and deal with them. Therefore, we throw an exception here.

        // (1) collect the current state of the involved objects
        Map<String, Object> description = new HashMap<String, Object>();

        // describe arguments
        description.put("argument errorCode", errorCode);
        description.put("argument errorMessage", errorMessage);

        // describe reader
        description.put("this", this);

        // (2) create an error message
        List<String> keys = new ArrayList<String>(description.keySet());
        Collections.sort(keys);
        StringBuilder sb = new StringBuilder();
        for (String key : keys) {
            sb.append("\n" + key + ": " + description.get(key));
        }

        // (3) throw exception
        throw new FaultyModifierException(sb.toString(), description);
    }

    //
    // interface Reader
    //

    /**
     * TODO more doc
     * 
     * @see java.io.Reader#read(char[], int, int)
     */
    @Override
    public int read(char[] cbuf, int off, int len) throws IOException {
        int index = 0;
        int read = 0;
        while (index < len && (read = readCharacter()) != -1) {
            cbuf[off + index] = (char) read;
            index++;
        }

        if (index == 0 && read == -1) {
            return -1;
        } else {
            return index;
        }
    }

    /**
     * Contract is similar to {@link java.io.Reader#read()}.
     * <p>
     * TODO more doc
     */
    protected int readCharacter() throws IOException {

        if (numberOfCharactersToSkip == 0) {

            boolean modifyAgainImmediately = false;

            AfterModification afterModification = null;
            do {

                updateBuffer();

                afterModification = modifier.modify(characterBuffer, firstModifiableCharacterInBuffer, endOfStreamHit);

                lastAfterModificationForDebuggingOnly = afterModification;

                // update minimumLengthOfLookBehind
                minimumLengthOfLookBehind = afterModification.getNewMinimumLengthOfLookBehind();
                if (minimumLengthOfLookBehind > firstModifiableCharacterInBuffer
                        + afterModification.getNumberOfCharactersToSkip()) {
                    onFaultyModifier(-11, "Requested Look Behind" + " is impossible because there are not enough "
                            + "characters in the stream.");
                }

                // update requestedNumCharactersInBuffer
                requestedNumCharactersInBuffer = minimumLengthOfLookBehind + afterModification.getNewNumberOfChars();
                if (requestedNumCharactersInBuffer < minimumLengthOfLookBehind + 1) {
                    onFaultyModifier(-13, "Requested Capacity" + " is two small because there must at least one"
                            + " unread characters available after the " + "look behind characters characters in the"
                            + " stream.");
                }

                modifyAgainImmediately = false;
                // do we have to read at least a single character as there is no
                // modifiable character left in the character buffer?
                if (firstModifiableCharacterInBuffer >= characterBuffer.length() && !endOfStreamHit) {
                    // yes, we need fresh input ->
                    modifyAgainImmediately = true;
                }
                // has the modifier requested a modification at the current
                // position?
                else if (afterModification.isModifyAgainImmediately()) {
                    modifyAgainImmediately = true;
                }

            } while (modifyAgainImmediately);

            numberOfCharactersToSkip = afterModification.getNumberOfCharactersToSkip();

            if (!afterModification.isModifyAgainImmediately() && numberOfCharactersToSkip == 0 && !endOfStreamHit) {
                onFaultyModifier(-16, "Not a single characters shall be " + "skipped but this is not possible of "
                        + "modifyAgain() returns false and the end of " + "stream is not reached yet.");
            }
            numberOfCharactersToSkip--;
        } else {
            numberOfCharactersToSkip--;
        }

        // is the end of the stream reached?
        if (firstModifiableCharacterInBuffer >= characterBuffer.length()) {
            // yes -> return -1
            return -1;
        } else {
            // no -> return the next unread character
            char result = characterBuffer.charAt(firstModifiableCharacterInBuffer);

            firstModifiableCharacterInBuffer++;

            return result;
        }
    }

    //
    // override Object.*
    //

    /**
     * @see java.lang.Object#toString()
     */
    @Override
    public String toString() {
        StringBuilder builder = new StringBuilder();
        builder.append("ModifyingReader [\ncharacterBuffer=");
        builder.append(characterBuffer);
        builder.append(", \nnextUnreadCharacterInBuffer=");
        builder.append(firstModifiableCharacterInBuffer);
        builder.append(", \nmodifier=");
        builder.append(modifier);
        builder.append(", \nminimumLengthOfLookBehind=");
        builder.append(minimumLengthOfLookBehind);
        builder.append(", \nrequestedCapacityOfCharacterBuffer=");
        builder.append(requestedNumCharactersInBuffer);
        builder.append(", \nlockedCharacters=");
        builder.append(numberOfCharactersToSkip);
        builder.append(", \nendOfStreamHit=");
        builder.append(endOfStreamHit);
        builder.append(", \nlastModificationForDebuggingOnly=");
        builder.append(lastAfterModificationForDebuggingOnly);
        builder.append("]");
        return builder.toString();
    }

    //
    // delegating methods
    //

    /**
     * @see java.io.Reader#close()
     */
    @Override
    public void close() throws IOException {
        delegate.close();
    }

    // What do I need this delegating method for? As long I don't know I don't
    // delegate.
    // /**
    // * @see java.io.Reader#ready()
    // */
    // @Override
    // public boolean ready() throws IOException {
    // return delegate.ready();
    // }

}
 public class RegexModifier implements Modifier {

    // * <p>
    // * <h2>A summary of the internal algorithm (some details are left
    // out)</h2>
    // * <p>
    // * Is there a match in the buffer?
    // * <ul>
    // * <li>we found a match
    // * <ul>
    // * <li>entire buffer matches and end of stream not hit yet -> the match
    // might
    // * change with more input -> FETCH_MORE_INPUT (<i>match_open</i>)
    // * <li>replace the matched content, and then
    // * <ul>
    // * <li>the match processor decides that no skip is needed yet -> continue
    // with
    // * the existing buffer content -> try another match
    // (<i>match_n_continue</i>)
    // * <li>skip needed but no characters left in the buffer after the
    // replacement ->
    // * MODIFY_AGAIN_IMMEDIATELY is only thing we can do
    // (<i>match_n_refill</i>)
    // * <li>skip needed and there are characters left in the buffer -> SKIP
    // * (<i>match_n_skip</i>)
    // * </ul>
    // * </ul>
    // * <li>we haven't found a match.
    // * <ul>
    // * <li>By looking for matches (including the empty string) that start in
    // the
    // * range [from, maxFrom], the end of the buffer is hit.
    // * <ul>
    // * <li>end of stream hit -> no match possible -> SKIP the entire buffer
    // * (<i>nomatch_eos</i>)
    // * <li>end of stream not hit -> match might be possible or end of buffer
    // is hit
    // * -> FETCH_MORE_INPUT cannot be wrong (<i>nomatch_fetch</i>)
    // * </ul>
    // * <li>We did not match a single character or the empty string -> SKIP the
    // * entire buffer (<i>nomatch_skip</i>)
    // * </ul>
    // * </ul>

    //
    // injected
    //

    protected ModificationFactory factory;

    protected MatchProcessor matchProcessor;

    /**
     * The compiled representation of a regular expression. If the regular expression matches, then a modification shall
     * be carried out.
     */
    protected OnStreamMatcher matcher;

    protected int newNumberOfChars = -1;

    //
    // state
    //

    /**
     * The number of characters that shall be skipped automatically if the modifier is called the next time.
     * <p>
     * This property is either zero or one (not greater). If this property is one, then the modifier tries to match
     * behind the first modifiable character. If this property is zero, then the modifier tries to match before the
     * first modifiable character.
     */
    private int unseenCharactersToSkip = 0;

    //
    // constructors
    //

    /**
     * Only for subclasses.
     */
    protected RegexModifier() {
        super();
    }

    /**
     * Like {@link RegexModifier#RegexModifier(String, int, String, int, int)} but uses defaults for
     * <code>minimumLengthOfLookBehind</code> (1) and <code>newNumberOfChars</code> (2048).
     */
    public RegexModifier(String regex, int flags, String replacement) {
        this(regex, flags, replacement, 1, 2048);
    }

    /**
     * Creates a modifier that matches a regular expression on character streams and replaces the matches.
     * <p>
     * This modifier uses {@link OnStreamStandardMatcher} which is not the fastest implementation of
     * {@link OnStreamMatcher}. If you want to use a faster matcher, use
     * {@link #RegexModifier(OnStreamMatcher, MatchProcessor, int, int)} instead.
     * <p>
     * A more convenient use of a {@link RegexModifier} is provided by the {@link ModifyingReaderFactory} respectively
     * {@link ModifyingWriterFactory}.
     * 
     * @param regex
     *            the regular expression that describe the text that shall be replaced. See
     *            {@link Pattern#compile(String, int)}.
     * @param flags
     *            the flags that are to use when the regex is applied on the character stream. See
     *            {@link Pattern#compile(String, int)}.
     * @param replacement
     *            the replacement for the text that is matched via <code>regex</code>. See
     *            {@link Matcher#appendReplacement(StringBuffer, String)}.
     * @param minimumLengthOfLookBehind
     *            See {@link RegexModifier#RegexModifier(OnStreamMatcher, MatchProcessor, int, int)} .
     * @param newNumberOfChars
     *            See {@link RegexModifier#RegexModifier(OnStreamMatcher, MatchProcessor, int, int)} .
     */
    public RegexModifier(String regex, int flags, String replacement, int minimumLengthOfLookBehind,
            int newNumberOfChars) {
        this(regex, flags, new ReplacingProcessor(replacement), minimumLengthOfLookBehind, newNumberOfChars);
    }

    /**
     * See {@link #RegexModifier(String, int, String, int, int)}.
     */
    public RegexModifier(String regex, int flags, MatchProcessor matchProcessor, int minimumLengthOfLookBehind,
            int newNumberOfChars) {

        Matcher jdkMatcher = Pattern.compile(regex, flags).matcher("");
        jdkMatcher.useTransparentBounds(true);
        jdkMatcher.useAnchoringBounds(false);
        init(new OnStreamStandardMatcher(jdkMatcher), matchProcessor, minimumLengthOfLookBehind, newNumberOfChars);
    }

    /**
     * Creates a modifier that matches a regular expression on character streams and does 'something' if matches are
     * found.
     * <p>
     * 
     * @param matcher
     *            Matches a regular expression on a <code>CharSequence</code>.
     * @param matchProcessor
     *            Defines what to do if the regular expression matches some text in the stream.
     * @param minimumLengthOfLookBehind
     *            See {@link AfterModification#getNewMinimumLengthOfLookBehind()}.
     * @param newNumberOfChars
     *            See {@link AfterModification#getNewNumberOfChars()}. This should not be smaller than the length of the
     *            characters sequence the pattern needs to match properly. In case you want to match more than once, the
     *            value should be higher.
     */
    public RegexModifier(OnStreamMatcher matcher, MatchProcessor matchProcessor, int minimumLengthOfLookBehind,
            int newNumberOfChars) {

        init(matcher, matchProcessor, minimumLengthOfLookBehind, newNumberOfChars);
    }

    @SuppressWarnings("hiding")
    protected void init(OnStreamMatcher matcher, MatchProcessor matchProcessor, int minimumLengthOfLookBehind,
            int newNumberOfChars) {

        this.factory = new ModificationFactory(minimumLengthOfLookBehind, newNumberOfChars);
        this.matchProcessor = matchProcessor;
        this.matcher = matcher;
        this.newNumberOfChars = newNumberOfChars;
    }

    //
    // interface Modifier
    //

    /**
     * @see com.github.rwitzel.streamflyer.core.Modifier#modify(java.lang.StringBuilder, int, boolean)
     */
    @Override
    public AfterModification modify(StringBuilder characterBuffer, int firstModifiableCharacterInBuffer,
            boolean endOfStreamHit) {

        // the first position we will match from.
        Integer minFrom = null;

        while (true) {

            // determine the range [minFrom, maxFrom] that will contain the
            // first character of the matching string

            if (minFrom == null) {
                minFrom = firstModifiableCharacterInBuffer;

                if (unseenCharactersToSkip > 0) {

                    // is there at least one modifiable character in the buffer?
                    if (minFrom + unseenCharactersToSkip > characterBuffer.length()) {
                        // no -> we need more input to skip the characters

                        if (endOfStreamHit) {
                            // -> stop
                            return factory.stop(characterBuffer, firstModifiableCharacterInBuffer, endOfStreamHit);
                        } else {
                            // -> fetch more input
                            return factory.fetchMoreInput(0, characterBuffer, firstModifiableCharacterInBuffer,
                                    endOfStreamHit);
                        }

                    } else {
                        // yes -> increase the *minFrom*
                        minFrom += unseenCharactersToSkip;
                        unseenCharactersToSkip = 0;
                    }
                }
            }

            // we have to restrict maxFrom in order to prevent that the
            // requested number of characters increases more and more
            int maxFrom = firstModifiableCharacterInBuffer + newNumberOfChars;

            // adjust maxFrom if it is bigger than the given buffer
            if (maxFrom > characterBuffer.length()) {
                // this is NOT set to characterBuffer.length() -1 by intention
                // as a regular expression might match on the zero length string
                // (but with positive look-behind)!
                maxFrom = characterBuffer.length();
            }

            // find first match
            // (as the match processor might have modified the buffer, we reset
            // the matcher inside the loop instead of outside of the loop)
            matcher.reset(characterBuffer);
            boolean matchFound = matcher.findUnlessHitEnd(minFrom, maxFrom);

            if (matchFound) {
                // we found a match

                // could change this positive match into a negative one
                // (matcher.requireEnd()) or into a longer one (greedy
                // operator)?
                if (matcher.hitEnd() && !endOfStreamHit) {
                    // (match_open) yes, it could -> we need more input

                    int numberOfCharactersToSkip = matcher.lastFrom() - firstModifiableCharacterInBuffer;

                    // read more input (skip some characters if possible)
                    AfterModification mod = factory.fetchMoreInput(numberOfCharactersToSkip, characterBuffer,
                            firstModifiableCharacterInBuffer, endOfStreamHit);

                    assert __checkpoint( //
                            "name", "match_open", //
                            "minLen", firstModifiableCharacterInBuffer, //
                            "characterBuffer", characterBuffer, //
                            "endOfStreamHit", endOfStreamHit, //
                            "afterModification", mod);

                    return mod;
                } else {
                    // no -> thus we can use this match -> process the match

                    // process the match
                    MatchResult matchResult = matcher; // .toMatchResult()?
                    // (I could pass firstModifiableCharacterInBuffer instead of
                    // minFrom as well)
                    MatchProcessorResult matchProcessorResult = matchProcessor.process(characterBuffer, minFrom,
                            matchResult);
                    minFrom = matchProcessorResult.getFirstModifiableCharacterInBuffer();

                    // match again without skip? (even for minFrom == maxFrom we
                    // try a match) (minFrom <= maxFrom is needed so that the
                    // buffer does not increase if the replacement is longer
                    // than the replaced string, i.e. minFrom > maxFrom means
                    // that a SKIP is needed)
                    // (I (rwoo) think an earlier SKIP (minFrom < maxFrom
                    // instead of minFrom <= maxFrom) would also be possible.
                    // This has no impact on the matching and only minimal
                    // impact on the performance)
                    if (minFrom <= maxFrom && matchProcessorResult.isContinueMatching()) {
                        // (match_n_continue) no skip needed yet -> continue
                        // matching on the existing buffer content

                        assert __checkpoint( //
                                "name", "match_n_continue", //
                                "minLen", firstModifiableCharacterInBuffer, //
                                "characterBuffer", characterBuffer, //
                                "endOfStreamHit", endOfStreamHit, //
                                "minFrom", minFrom);

                        // We try the next match on the modified input, i.e.
                        // not match only once -> next loop
                        continue;
                    } else {

                        // we shall not continue matching on the
                        // existing buffer content but skip (keep the buffer
                        // small)

                        int numberOfCharactersToSkip;
                        if (minFrom > characterBuffer.length()) {
                            // this happens when we avoid endless loops after
                            // we matched an empty string
                            unseenCharactersToSkip = minFrom - characterBuffer.length();
                            ZzzAssert.isTrue(unseenCharactersToSkip == 1, "unseenCharactersToSkip must be one but was "
                                    + unseenCharactersToSkip);
                            numberOfCharactersToSkip = characterBuffer.length() - firstModifiableCharacterInBuffer;
                        } else {
                            numberOfCharactersToSkip = minFrom - firstModifiableCharacterInBuffer;
                        }

                        if (numberOfCharactersToSkip == 0) {

                            // (match_n_refill) there are no characters left in
                            // the buffer after the replacement ->
                            // MODIFY_AGAIN_IMMEDIATELY is only thing we can do
                            // (the match processor implementation must not
                            // cause an endless loop)

                            // (passing false for endOfStreamHit is ugly!!!)
                            // we should offer a new method in
                            // ModificationFactory, something like
                            // continueAfterModification(...) that chooses the
                            // appropriate action. the following code is always
                            // a MODIFY_AGAIN_IMMEDIATELY
                            AfterModification mod = factory.fetchMoreInput(numberOfCharactersToSkip, characterBuffer,
                                    firstModifiableCharacterInBuffer, false);

                            assert __checkpoint( //
                                    "name", "match_n_refill", //
                                    "minLen", firstModifiableCharacterInBuffer, //
                                    "characterBuffer", characterBuffer, //
                                    "endOfStreamHit", endOfStreamHit, //
                                    "afterModification", mod);

                            return mod;
                        } else {
                            // (match_n_skip) there are characters left in
                            // the buffer -> SKIP

                            AfterModification mod = factory.skipOrStop(numberOfCharactersToSkip, characterBuffer,
                                    firstModifiableCharacterInBuffer, endOfStreamHit);

                            assert __checkpoint( //
                                    "name", "match_n_skip", //
                                    "minLen", firstModifiableCharacterInBuffer, //
                                    "characterBuffer", characterBuffer, //
                                    "endOfStreamHit", endOfStreamHit, //
                                    "afterModification", mod);

                            return mod;
                        }
                    }

                }
            } else {
                // we haven't found a match

                // By looking for matches (including the empty string) that
                // start in the range [from, maxFrom], is the end of the buffer
                // hit?
                if (matcher.lastFrom() <= maxFrom) {
                    // yes, the end of the buffer was hit

                    // can we get more input?
                    if (endOfStreamHit) {
                        // (nomatch_eos) no, in the entire stream we will not
                        // found more matches that start in range [from,
                        // maxFrom] -> skip the characters from range [from,
                        // maxFrom]

                        int numberOfCharactersToSkip = maxFrom - firstModifiableCharacterInBuffer;
                        AfterModification mod = factory.skipOrStop(numberOfCharactersToSkip, characterBuffer,
                                firstModifiableCharacterInBuffer, endOfStreamHit);

                        assert __checkpoint( //
                                "name", "nomatch_eos", //
                                "minLen", firstModifiableCharacterInBuffer, //
                                "characterBuffer", characterBuffer, //
                                "endOfStreamHit", endOfStreamHit, //
                                "afterModification", mod);

                        return mod;

                    } else {
                        // (nomatch_fetch) yes > we should fetch more input
                        // (because end of stream is not hit yet)

                        // if maxFrom == characterBuffer.length() and lastFrom()
                        // == maxFrom we cannot decide whether this is really an
                        // open match or rather a not a match at all. But by
                        // skipping the characters in front of lastFrom() and
                        // fetching more input we cannot do anything wrong

                        int numberOfCharactersToSkip = matcher.lastFrom() - firstModifiableCharacterInBuffer;
                        AfterModification mod = factory.fetchMoreInput(numberOfCharactersToSkip, characterBuffer,
                                firstModifiableCharacterInBuffer, endOfStreamHit);

                        assert __checkpoint( //
                                "name", "nomatch_fetch", //
                                "minLen", firstModifiableCharacterInBuffer, //
                                "characterBuffer", characterBuffer, //
                                "endOfStreamHit", endOfStreamHit, //
                                "afterModification", mod);

                        return mod;
                    }

                } else { // matcher.lastFrom() == maxFrom + 1

                    // (nomatch_skip) no, we are matching not a single character

                    // -> skip the characters from range [from, maxFrom]
                    int numberOfCharactersToSkip = maxFrom - firstModifiableCharacterInBuffer;
                    AfterModification mod = factory.skipOrStop(numberOfCharactersToSkip, characterBuffer,
                            firstModifiableCharacterInBuffer, endOfStreamHit);

                    assert __checkpoint( //
                            "name", "nomatch_skip", //
                            "minLen", firstModifiableCharacterInBuffer, //
                            "characterBuffer", characterBuffer, //
                            "endOfStreamHit", endOfStreamHit, //
                            "afterModification", mod);

                    return mod;
                }
            }
        }

    }

    /**
     * This method is called if a certain line of code is reached ("checkpoint").
     * <p>
     * This method should be called only if the modifier is tested. Otherwise you might experience performance
     * penalties.
     * 
     * @param checkpointDescription
     *            A list of objects describing the checkpoint. The objects should be given as name-value-pairs.
     * @return Returns true. This allows you to use this method as side-effect in Java assertions.
     */
    protected boolean __checkpoint(Object... checkpointDescription) {
        // nothing to do here
        return true;
    }

    //
    // override Object.*
    //

    /**
     * @see java.lang.Object#toString()
     */
    @Override
    public String toString() {
        StringBuilder builder = new StringBuilder();
        builder.append("RegexModifier [\nfactory=");
        builder.append(factory);
        builder.append(", \nreplacement=");
        builder.append(matchProcessor);
        builder.append(", \nmatcher=");
        builder.append(matcher);
        builder.append(", \nnewNumberOfChars=");
        builder.append(newNumberOfChars);
        builder.append("]");
        return builder.toString();
    }

    //
    // injected
    //

    /**
     * @param matchProcessor
     *            The {@link #matchProcessor} to set.
     */
    public void setMatchProcessor(MatchProcessor matchProcessor) {
        this.matchProcessor = matchProcessor;
    }

}
 public class ReplacingProcessor extends AbstractMatchProcessor implements MatchProcessor {

    //
    // injected properties
    //

    /**
     * This list contains Strings and Numbers. The latter ones refer to groups.
     */
    private List<Object> parts = null;

    /**
     * If not null, then the replacement do not uses groups. Then this property contains the entire replacement.
     */
    private CharSequence replacementWithoutGroupReferences;

    //
    // constructors
    //

    /**
     * For unit tests only.
     */
    public ReplacingProcessor() {
        // nothing to do here
    }

    /**
     * @param replacement
     */
    public ReplacingProcessor(String replacement) {
        super();

        // validate arguments
        ZzzValidate.notNull(replacement, "replacement must not be null");

        // parse replacement
        this.parts = parseReplacement(replacement);

        if (parts.size() == 1 && (parts.get(0) instanceof CharSequence)) {

            this.replacementWithoutGroupReferences = (CharSequence) parts.get(0);

        } else {

            this.replacementWithoutGroupReferences = null;

        }
    }

    /**
     * Package-private for JUnit-tests.
     * 
     * @return Returns the parts of the matchProcessor
     */
    List<Object> parseReplacement(String replacement) {

        List<Object> compiledReplacement = new ArrayList<Object>();

        // we look for escaped literals and references to groups
        int position = 0;
        StringBuilder notGroupReference = new StringBuilder();

        while (position < replacement.length()) {
            char ch = replacement.charAt(position);

            // reference to group?
            if (ch == '$') {

                if (notGroupReference.length() != 0) {
                    compiledReplacement.add(notGroupReference.toString());
                    notGroupReference.setLength(0);
                }

                position++;

                int groupNumberStartPosition = position;
                if (groupNumberStartPosition == replacement.length()) {
                    throw new IllegalArgumentException("group reference $ " + "without number at the end " + "of the "
                            + "replacement string (" + replacement + ")");
                }

                ch = replacement.charAt(position);

                while ('0' <= ch && ch <= '9') {
                    position++;
                    if (position == replacement.length()) {
                        break;
                    }
                    ch = replacement.charAt(position);
                }
                // position is the position of the first character that does not
                // belong to the group reference

                if (position == groupNumberStartPosition) {
                    throw new IllegalArgumentException("invalid reference to group at position " + position
                            + " in replacement string " + replacement.substring(0, position) + "[]"
                            + replacement.substring(position));
                }

                String groupNumberString = replacement.substring(groupNumberStartPosition, position);
                int groupNumber = Integer.parseInt(groupNumberString);

                compiledReplacement.add(groupNumber);
            }
            // escaped literal?
            else if (ch == '\\') {

                // skip the backslash
                position++;
                ch = replacement.charAt(position);

                notGroupReference.append(ch);
                position++;
            }
            // this is a normal character
            else {

                notGroupReference.append(ch);
                position++;
            }
        }

        if (notGroupReference.length() != 0) {
            compiledReplacement.add(notGroupReference.toString());
        }

        return compiledReplacement;
    }

    /**
     * Replaces the match as given by the {@link MatchResult} with the configured replacement.
     * 
     * @see com.github.rwitzel.streamflyer.regex.MatchProcessor#process(java.lang.StringBuilder, int,
     *      java.util.regex.MatchResult)
     */
    @Override
    public MatchProcessorResult process(StringBuilder characterBuffer, int firstModifiableCharacterInBuffer,
            MatchResult matchResult) {

        int start = matchResult.start();
        int end = matchResult.end();

        if (replacementWithoutGroupReferences != null) {

            characterBuffer.delete(start, end);
            characterBuffer.insert(start, replacementWithoutGroupReferences);
            return createResult(matchResult, start + replacementWithoutGroupReferences.length(), true);
        } else {

            // (1) create replacement
            // (one could set a reasonable value for the initial size of the
            // string builder somewhere)
            StringBuilder replacement = new StringBuilder();

            for (Object part : parts) {
                if (part instanceof Integer) {
                    // append the value of the referred group
                    replacement.append(matchResult.group((Integer) part));
                } else {
                    // append the static part
                    replacement.append(part); // part is instance of String
                }
            }

            // (2) replace the match with the replacement
            characterBuffer.delete(start, end);
            characterBuffer.insert(start, replacement);

            return createResult(matchResult, start + replacement.length(), true);
        }

    }
}
 public class NoMatch {

    /**
     * The position of the first character that might belong to a noMatch.
     */
    private int startPosition;

    /**
     * @return the startPosition
     */
    public int getStartPosition() {
        return startPosition;
    }

    /**
     * @param position
     *            the startPosition to set
     */
    public void setStartPosition(int position) {
        this.startPosition = position;
    }

    /**
     * Called by {@link NoMatchAwareTransitionGuard} or {@link NoMatchAwareMatchProcessor} before a match (when the
     * noMatch is not empty). By default, we return the original match result but you are allowed to modify the stream.
     * In the latter case you have to adjust the returned match result.
     * <p>
     * The end of the noMatch is given by {@link MatchResult#start()}.
     */
    public MatchResult processNoMatch(StringBuilder characterBuffer, int firstModifiableCharacterInBuffer,
            MatchResult matchResult) {
        return matchResult;
    }

    /**
     * Called by {@link NoMatchAwareModifier} when new characters are fetched from the stream (when the noMatch is not
     * empty). By default, we return the original 'after modification' but you are allowed to modify the stream. In the
     * latter case you have to adjust the returned match result.
     * <p>
     * The end of the noMatch is given by <code>firstModifiableCharacterInBuffer</code> +
     * {@link AfterModification#getNumberOfCharactersToSkip()}.
     */
    public AfterModification processNoMatch(StringBuilder characterBuffer, int firstModifiableCharacterInBuffer,
            boolean endOfStreamHit, AfterModification mod, Modifier modifier) {
        return mod;
    }

}
 public class NoMatchAwareMatchProcessor implements MatchProcessor {

    protected MatchProcessor delegate;

    protected NoMatch noMatch;

    /**
     * True if this processor shall process noMatches. This should be false if there is already a
     * {@link NoMatchAwareTransitionGuard} configured. This applies only if the {@link RegexModifier} uses a
     * {@link StateMachine}.
     */
    private boolean processNoMatch;

    public NoMatchAwareMatchProcessor(MatchProcessor delegate, NoMatch noMatch, boolean processNoMatch) {
        super();
        this.delegate = delegate;
        this.noMatch = noMatch;
        this.processNoMatch = processNoMatch;
    }

    @Override
    public MatchProcessorResult process(StringBuilder characterBuffer, int firstModifiableCharacterInBuffer,
            MatchResult matchResult) {

        // are there characters that belong to a noMatch?
        if (processNoMatch && noMatch.getStartPosition() < matchResult.start()) {
            // -> process the noMatch
            matchResult = noMatch.processNoMatch(characterBuffer, firstModifiableCharacterInBuffer, matchResult);
        }

        MatchProcessorResult result = delegate.process(characterBuffer, firstModifiableCharacterInBuffer, matchResult);

        // set the start position of the next noMatch
        noMatch.setStartPosition(result.getFirstModifiableCharacterInBuffer());

        return result;
    }

}
 public class NoMatchAwareModifier implements Modifier {

    private Modifier delegate;

    private NoMatch noMatch;

    public NoMatchAwareModifier(Modifier delegate, NoMatch noMatch) {
        super();
        this.delegate = delegate;
        this.noMatch = noMatch;
    }

    @Override
    public AfterModification modify(StringBuilder characterBuffer, int firstModifiableCharacterInBuffer,
            boolean endOfStreamHit) {

        // set the start position of the next noMatch
        noMatch.setStartPosition(firstModifiableCharacterInBuffer);

        AfterModification mod = delegate.modify(characterBuffer, firstModifiableCharacterInBuffer, endOfStreamHit);

        if (noMatch.getStartPosition() < firstModifiableCharacterInBuffer + mod.getNumberOfCharactersToSkip()) {
            // there are characters for the noMatch -> process them
            mod = noMatch.processNoMatch(characterBuffer, firstModifiableCharacterInBuffer, endOfStreamHit, mod,
                    delegate);
        }

        return mod;
    }

}
 public class NoMatchAwareTransitionGuard extends TransitionGuard {

    private TransitionGuard delegate;

    private NoMatch noMatch;

    public NoMatchAwareTransitionGuard(TransitionGuard delegate, NoMatch noMatch) {
        super();
        this.delegate = delegate;
        this.noMatch = noMatch;
    }

    @Override
    public MatchProcessorResult stopTransition(State nextState, StringBuilder characterBuffer,
            int firstModifiableCharacterInBuffer, MatchResult matchResult) {

        // are there characters that belong to a noMatch?
        if (noMatch.getStartPosition() < matchResult.start()) {
            // -> process the noMatch
            matchResult = noMatch.processNoMatch(characterBuffer, firstModifiableCharacterInBuffer, matchResult);
        }

        // delegate to the real guard
        return delegate.stopTransition(nextState, characterBuffer, firstModifiableCharacterInBuffer, matchResult);
    }

}
 public class State {

    /**
     * The token that must be matched to reach this state.
     */
    private Token token;

    /**
     * The transitions that can be applied.
     */
    private Transitions transitions;

    /**
     * A constructor to create an initial state.
     * 
     * @param stateName
     *            A unique name for the state.
     */
    public State(String stateName) {
        this(stateName, "", new DoNothingProcessor());
    }

    /**
     * If this constructed state is reached, the stream is {@link DoNothingProcessor not modified}.
     * 
     * @param stateName
     *            A unique name for the state.
     * @param regex
     *            In order to reach this state, this regular expression must be matched.
     */
    public State(String stateName, String regex) {
        this(stateName, regex, new DoNothingProcessor());
    }

    /**
     * This constructor defines a state so that text of the matched state token is replaced with the given replacement.
     * 
     * @param stateName
     *            A unique name for the state.
     * @param regex
     *            In order to reach this state, this regular expression must be matched.
     * @param replacement
     *            The replacement defines how the text that is matched via <code>regex</code> shall be replaced.
     */
    public State(String stateName, String regex, String replacement) {
        this(stateName, regex, new ReplacingProcessor(replacement));
    }

    /**
     * If the constructed state is reached then the given {@link MatchProcessor} modifies the stream.
     * 
     * @param stateName
     *            A unique name for the state.
     * @param regex
     *            In order to reach this state, this regular expression must be matched.
     * @param matchProcessor
     *            If the constructed state is reached then the given {@link MatchProcessor} modifies the stream.
     */
    public State(String stateName, String regex, MatchProcessor matchProcessor) {
        this(new Token(stateName, regex, matchProcessor));
    }

    /**
     * The constructed state is reached if the given token is matched.
     * 
     * @param token
     *            See {@link #token}
     */
    public State(Token token) {
        super();
        this.token = token;
    }

    /**
     * Convenience method.
     * <p>
     * Sets {@link #transitions} as defined by the reachable states and the transition guard.
     * 
     * @param endStates
     *            the states that can be reached from the this state.
     * @param transitionGuard
     *            this guard can stop the transition or add additional logic to the transition.
     */
    public void setTransitions(List<State> endStates, TransitionGuard transitionGuard) {
        setTransitions(new Transitions(endStates, transitionGuard));
    }

    /**
     * Sets {@link #transitions}.
     * 
     * @param transitions
     *            the transitions to set.
     */
    public void setTransitions(Transitions transitions) {
        this.transitions = transitions;
    }

    /**
     * @return Returns the token that must be matched to reach this state.
     */
    public Token getToken() {
        return token;
    }

    /**
     * @return Returns the token processor that must be used to switch to the next state.
     */
    public Transitions getTransitions() {
        return transitions;
    }

    @Override
    public String toString() {
        return "State [token=" + token + ", transitions=" + transitions + "]";
    }

}
 public class StateMachine implements MatchProcessor {

    /**
     * The most recently reached state.
     */
    private State currentState;

    /**
     * The transitions that starts from {@link #currentState}.
     */
    private Transitions transitions;

    /**
     * The matcher that is used by the {@link RegexModifier}. By changing the delegate we can exchange the regex the
     * modifier looks for. The matcher looks for the tokens that belong to the end states of {@link #transitions}.
     */
    private DelegatingMatcher delegatingMatcher;

    /**
     * 
     * @param initialState
     *            the initial state for the state machine
     * @param delegatingMatcher
     *            See {@link #delegatingMatcher}.
     */
    public StateMachine(State initialState, DelegatingMatcher delegatingMatcher) {
        super();
        this.delegatingMatcher = delegatingMatcher;
        changeStateTo(initialState);
    }

    /**
     * @return the currentState
     */
    public State getCurrentState() {
        return currentState;
    }

    /**
     * Changes the {@link #currentState} to the given state.
     * 
     * @param state
     */
    protected void changeStateTo(State state) {
        currentState = state;
        transitions = state.getTransitions();
        delegatingMatcher.setDelegate(state.getTransitions().getMatcher());
    }

    @Override
    public MatchProcessorResult process(StringBuilder characterBuffer, int firstModifiableCharacterInBuffer,
            MatchResult matchResult) {

        // modify stream
        MatchProcessorResult result = transitions.process(characterBuffer, firstModifiableCharacterInBuffer,
                matchResult);

        // change state if needed
        State newState = transitions.pollNewState();
        if (newState != currentState) {
            changeStateTo(newState);
        }

        return result;
    }

}
 public class TokenCollector extends TransitionGuard {

    private List<String> foundTokens = new ArrayList<String>();

    public TokenCollector(List<String> foundTokens) {
        super();
        this.foundTokens = foundTokens;
    }

    @Override
    public MatchProcessorResult stopTransition(State nextState, StringBuilder characterBuffer,
            int firstModifiableCharacterInBuffer, MatchResult matchResult) {

        // log the found token
        String foundToken = nextState.getToken().getName() + ":" + matchResult.group();
        foundTokens.add(foundToken);

        // don't stop the transition
        return null;
    }
}
 public class DelegatingMatcher implements OnStreamMatcher {

    private OnStreamMatcher delegate;

    public OnStreamMatcher getDelegate() {
        return delegate;
    }

    public void setDelegate(OnStreamMatcher delegate) {
        this.delegate = delegate;
    }

    //
    // override OnStreamMatcher.*
    //

    @Override
    public int start() {
        return delegate.start();
    }

    @Override
    public int start(int group) {
        return delegate.start(group);
    }

    @Override
    public int end() {
        return delegate.end();
    }

    @Override
    public int end(int group) {
        return delegate.end(group);
    }

    @Override
    public String group() {
        return delegate.group();
    }

    @Override
    public String group(int group) {
        return delegate.group(group);
    }

    @Override
    public int groupCount() {
        return delegate.groupCount();
    }

    @Override
    public void reset(CharSequence input) {
        delegate.reset(input);
    }

    @Override
    public boolean findUnlessHitEnd(int minFrom, int maxFrom) {
        return delegate.findUnlessHitEnd(minFrom, maxFrom);
    }

    @Override
    public boolean hitEnd() {
        return delegate.hitEnd();
    }

    @Override
    public boolean requireEnd() {
        return delegate.requireEnd();
    }

    @Override
    public int lastFrom() {
        return delegate.lastFrom();
    }

}
 public class DoNothingProcessor extends AbstractMatchProcessor implements MatchProcessor {

    @Override
    public MatchProcessorResult process(StringBuilder characterBuffer, int firstModifiableCharacterInBuffer,
            MatchResult matchResult) {

        return createResult(matchResult, matchResult.end(), true);
    }

}

import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.junit.Test;
import com.github.rwitzel.streamflyer.core.Modifier;
import com.github.rwitzel.streamflyer.core.ModifyingReader;
import com.github.rwitzel.streamflyer.regex.RegexModifier;
import com.github.rwitzel.streamflyer.regex.ReplacingProcessor;
import com.github.rwitzel.streamflyer.regex.addons.nomatch.NoMatch;
import com.github.rwitzel.streamflyer.regex.addons.nomatch.NoMatchAwareMatchProcessor;
import com.github.rwitzel.streamflyer.regex.addons.nomatch.NoMatchAwareModifier;
import com.github.rwitzel.streamflyer.regex.addons.nomatch.NoMatchAwareTransitionGuard;
import com.github.rwitzel.streamflyer.regex.addons.stateful.State;
import com.github.rwitzel.streamflyer.regex.addons.stateful.StateMachine;
import com.github.rwitzel.streamflyer.regex.addons.stateful.TokenCollector;
import com.github.rwitzel.streamflyer.regex.addons.util.DelegatingMatcher;
import com.github.rwitzel.streamflyer.regex.addons.util.DoNothingProcessor;

/**
 * Copyright (C) 2011 rwitzel75@googlemail.com
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.github.rwitzel.streamflyer.regex.addons.nomatch;





/**
 * Tests {@link NoMatch} and the classes of the same package together with a {@link StateMachine}.
 * 
 * @author rwoo
 * 
 */
public class NoMatchWithStateMachineTest {

    private State createState(String stateName, NoMatch noMatch) {
        // we don't need NoMatchAwareMatchProcessor for the initial state because the initial state is never reached
        return new State(stateName);
    }

    private State createState(String stateName, String regex, NoMatch pos) {

        return new State(stateName, regex, new NoMatchAwareMatchProcessor(new DoNothingProcessor(), pos, false));
