// Path: nicta-ner-common/src/main/java/org/t3as/ner/Token.java
// @Immutable
// public class Token {
// 
//     /** 0-indexed position of the first character of this token from the original text being analysed. */
//     public final int startIndex;
//     /** The text that this token is representing. */
//     public final String text;
// 
//     @JsonCreator
//     public Token(@JsonProperty("startIndex") final int startIndex, @JsonProperty("text") final String text) {
//         this.startIndex = startIndex;
//         this.text = text;
//     }
// 
//     @Override
//     public boolean equals(final Object o) {
//         if (this == o) return true;
//         if (o == null || getClass() != o.getClass()) return false;
//         final Token token = (Token) o;
//         return startIndex == token.startIndex && text.equals(token.text);
//     }
// 
//     @Override
//     public int hashCode() {
//         int result = startIndex;
//         result = 31 * result + text.hashCode();
//         return result;
//     }
// 
//     public String toString() { return startIndex + ":" + text; }
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean endsWith(final String word, final String... suffixes) {
//     if (word == null || suffixes == null) return false;
//     for (final String s : suffixes) {
//         if (word.endsWith(s)) return true;
//     }
//     return false;
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean equalss(final String word, final String... words) {
//     if (word == null || words == null) return false;
//     for (final String s : words) {
//         if (word.equals(s)) return true;
//     }
//     return false;
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static char initChar(final String s) {
//     return s.charAt(0);
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean isSingleUppercaseChar(final String word) {
//     return word.length() == 1 && isUpperCase(word.charAt(0));
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static char lastChar(final String s) {
//     return s.charAt(s.length() -1);
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static String toEngLowerCase(final String s) { return s.toLowerCase(Locale.ENGLISH); }


import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import org.t3as.ner.Token;
import java.io.IOException;
import java.text.BreakIterator;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import static java.lang.Character.isLetterOrDigit;
import static java.lang.Character.isSpaceChar;
import static java.text.BreakIterator.DONE;
import static org.t3as.ner.util.Strings.endsWith;
import static org.t3as.ner.util.Strings.equalss;
import static org.t3as.ner.util.Strings.initChar;
import static org.t3as.ner.util.Strings.isSingleUppercaseChar;
import static org.t3as.ner.util.Strings.lastChar;
import static org.t3as.ner.util.Strings.toEngLowerCase;
import static org.t3as.ner.util.Tokenizer.Mode.WITHOUT_PUNCTUATION;
import static org.t3as.ner.util.Tokenizer.Mode.WITH_PUNCTUATION;
        for (int endIdx = wordIterator.next(); endIdx != DONE; startIdx = endIdx, endIdx = wordIterator.next()) {
            final String word = text.substring(startIdx, endIdx);
            l.add(new Token(startIdx, word));
        }

        return new Tokens(l);
    }

    private void mergeWordsIntoSentence(final Token previousWord, final Token word, final Token nextWord,
                                        final int previousWordIndex) {
        // make sure the previous and next words both start with a letter or digit
        if (isLetterOrDigit(initChar(previousWord.text)) && isLetterOrDigit(initChar(nextWord.text))) {
            // merge the 3 words again and add the result, replace previous word from sentence
            currentSentence.set(previousWordIndex,
                                new Token(previousWord.startIndex, previousWord.text + word.text + nextWord.text));
        }
        // otherwise just add the word and next word
        else {
            currentSentence.add(word);
            currentSentence.add(nextWord);
        }
    }

    private boolean wordIsFullStop(final Token t) {
        boolean canBreakSentence = true;
        if (currentSentence.isEmpty()) currentSentence.add(t);
        else {
            final int previousWordIndex = currentSentence.size() - 1;
            final Token previousWord = currentSentence.get(previousWordIndex);
            if (ABBREVIATION_EXCEPTIONS.contains(toEngLowerCase(previousWord.text))
// Path: nicta-ner-common/src/main/java/org/t3as/ner/Token.java
// @Immutable
// public class Token {
// 
//     /** 0-indexed position of the first character of this token from the original text being analysed. */
//     public final int startIndex;
//     /** The text that this token is representing. */
//     public final String text;
// 
//     @JsonCreator
//     public Token(@JsonProperty("startIndex") final int startIndex, @JsonProperty("text") final String text) {
//         this.startIndex = startIndex;
//         this.text = text;
//     }
// 
//     @Override
//     public boolean equals(final Object o) {
//         if (this == o) return true;
//         if (o == null || getClass() != o.getClass()) return false;
//         final Token token = (Token) o;
//         return startIndex == token.startIndex && text.equals(token.text);
//     }
// 
//     @Override
//     public int hashCode() {
//         int result = startIndex;
//         result = 31 * result + text.hashCode();
//         return result;
//     }
// 
//     public String toString() { return startIndex + ":" + text; }
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean endsWith(final String word, final String... suffixes) {
//     if (word == null || suffixes == null) return false;
//     for (final String s : suffixes) {
//         if (word.endsWith(s)) return true;
//     }
//     return false;
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean equalss(final String word, final String... words) {
//     if (word == null || words == null) return false;
//     for (final String s : words) {
//         if (word.equals(s)) return true;
//     }
//     return false;
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static char initChar(final String s) {
//     return s.charAt(0);
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean isSingleUppercaseChar(final String word) {
//     return word.length() == 1 && isUpperCase(word.charAt(0));
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static char lastChar(final String s) {
//     return s.charAt(s.length() -1);
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static String toEngLowerCase(final String s) { return s.toLowerCase(Locale.ENGLISH); }


// Path: nicta-ner/src/main/java/org/t3as/ner/util/Tokenizer.java
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import org.t3as.ner.Token;
import java.io.IOException;
import java.text.BreakIterator;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import static java.lang.Character.isLetterOrDigit;
import static java.lang.Character.isSpaceChar;
import static java.text.BreakIterator.DONE;
import static org.t3as.ner.util.Strings.endsWith;
import static org.t3as.ner.util.Strings.equalss;
import static org.t3as.ner.util.Strings.initChar;
import static org.t3as.ner.util.Strings.isSingleUppercaseChar;
import static org.t3as.ner.util.Strings.lastChar;
import static org.t3as.ner.util.Strings.toEngLowerCase;
import static org.t3as.ner.util.Tokenizer.Mode.WITHOUT_PUNCTUATION;
import static org.t3as.ner.util.Tokenizer.Mode.WITH_PUNCTUATION;

        for (int endIdx = wordIterator.next(); endIdx != DONE; startIdx = endIdx, endIdx = wordIterator.next()) {
            final String word = text.substring(startIdx, endIdx);
            l.add(new Token(startIdx, word));
        }

        return new Tokens(l);
    }

    private void mergeWordsIntoSentence(final Token previousWord, final Token word, final Token nextWord,
                                        final int previousWordIndex) {
        // make sure the previous and next words both start with a letter or digit
        if (isLetterOrDigit(initChar(previousWord.text)) && isLetterOrDigit(initChar(nextWord.text))) {
            // merge the 3 words again and add the result, replace previous word from sentence
            currentSentence.set(previousWordIndex,
                                new Token(previousWord.startIndex, previousWord.text + word.text + nextWord.text));
        }
        // otherwise just add the word and next word
        else {
            currentSentence.add(word);
            currentSentence.add(nextWord);
        }
    }

    private boolean wordIsFullStop(final Token t) {
        boolean canBreakSentence = true;
        if (currentSentence.isEmpty()) currentSentence.add(t);
        else {
            final int previousWordIndex = currentSentence.size() - 1;
            final Token previousWord = currentSentence.get(previousWordIndex);
            if (ABBREVIATION_EXCEPTIONS.contains(toEngLowerCase(previousWord.text))
                || isSingleUppercaseChar(previousWord.text)