// Path: nicta-ner-common/src/main/java/org/t3as/ner/Token.java
// @Immutable
// public class Token {
// 
//     /** 0-indexed position of the first character of this token from the original text being analysed. */
//     public final int startIndex;
//     /** The text that this token is representing. */
//     public final String text;
// 
//     @JsonCreator
//     public Token(@JsonProperty("startIndex") final int startIndex, @JsonProperty("text") final String text) {
//         this.startIndex = startIndex;
//         this.text = text;
//     }
// 
//     @Override
//     public boolean equals(final Object o) {
//         if (this == o) return true;
//         if (o == null || getClass() != o.getClass()) return false;
//         final Token token = (Token) o;
//         return startIndex == token.startIndex && text.equals(token.text);
//     }
// 
//     @Override
//     public int hashCode() {
//         int result = startIndex;
//         result = 31 * result + text.hashCode();
//         return result;
//     }
// 
//     public String toString() { return startIndex + ":" + text; }
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean endsWith(final String word, final String... suffixes) {
//     if (word == null || suffixes == null) return false;
//     for (final String s : suffixes) {
//         if (word.endsWith(s)) return true;
//     }
//     return false;
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean equalss(final String word, final String... words) {
//     if (word == null || words == null) return false;
//     for (final String s : words) {
//         if (word.equals(s)) return true;
//     }
//     return false;
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static char initChar(final String s) {
//     return s.charAt(0);
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean isSingleUppercaseChar(final String word) {
//     return word.length() == 1 && isUpperCase(word.charAt(0));
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static char lastChar(final String s) {
//     return s.charAt(s.length() -1);
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static String toEngLowerCase(final String s) { return s.toLowerCase(Locale.ENGLISH); }


import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import org.t3as.ner.Token;
import java.io.IOException;
import java.text.BreakIterator;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import static java.lang.Character.isLetterOrDigit;
import static java.lang.Character.isSpaceChar;
import static java.text.BreakIterator.DONE;
import static org.t3as.ner.util.Strings.endsWith;
import static org.t3as.ner.util.Strings.equalss;
import static org.t3as.ner.util.Strings.initChar;
import static org.t3as.ner.util.Strings.isSingleUppercaseChar;
import static org.t3as.ner.util.Strings.lastChar;
import static org.t3as.ner.util.Strings.toEngLowerCase;
import static org.t3as.ner.util.Tokenizer.Mode.WITHOUT_PUNCTUATION;
import static org.t3as.ner.util.Tokenizer.Mode.WITH_PUNCTUATION;
     * Tokenize some text - not thread safe.
     * @param text tokenize this
     * @return tokenized text
     */
    public List<List<Token>> process(final String text) {
        final List<List<Token>> paragraph = new ArrayList<>();
        currentSentence = new ArrayList<>();
        final Tokens tokens = splitText(text);

        while (tokens.hasNext()) {
            final Token t = tokens.next();
            final String trimmedWord = t.text.trim();

            // skip spaces
            if (trimmedWord.isEmpty()) continue;

            if (((mode == WITH_PUNCTUATION) || (mode == WITHOUT_PUNCTUATION && isLetterOrDigit(initChar(t.text))))) {
                boolean canBreakSentence = true;
                if (t.text.contains("'")) {
                    wordContainsApostrophe(t);
                }
                else if (".".equals(trimmedWord)) {
                    canBreakSentence = wordIsFullStop(t);
                }
                else if (":".equals(trimmedWord)) {
                    wordIsColon(tokens, t);
                }
                else currentSentence.add(t);

                // handling the end of a sentence
// Path: nicta-ner-common/src/main/java/org/t3as/ner/Token.java
// @Immutable
// public class Token {
// 
//     /** 0-indexed position of the first character of this token from the original text being analysed. */
//     public final int startIndex;
//     /** The text that this token is representing. */
//     public final String text;
// 
//     @JsonCreator
//     public Token(@JsonProperty("startIndex") final int startIndex, @JsonProperty("text") final String text) {
//         this.startIndex = startIndex;
//         this.text = text;
//     }
// 
//     @Override
//     public boolean equals(final Object o) {
//         if (this == o) return true;
//         if (o == null || getClass() != o.getClass()) return false;
//         final Token token = (Token) o;
//         return startIndex == token.startIndex && text.equals(token.text);
//     }
// 
//     @Override
//     public int hashCode() {
//         int result = startIndex;
//         result = 31 * result + text.hashCode();
//         return result;
//     }
// 
//     public String toString() { return startIndex + ":" + text; }
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean endsWith(final String word, final String... suffixes) {
//     if (word == null || suffixes == null) return false;
//     for (final String s : suffixes) {
//         if (word.endsWith(s)) return true;
//     }
//     return false;
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean equalss(final String word, final String... words) {
//     if (word == null || words == null) return false;
//     for (final String s : words) {
//         if (word.equals(s)) return true;
//     }
//     return false;
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static char initChar(final String s) {
//     return s.charAt(0);
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean isSingleUppercaseChar(final String word) {
//     return word.length() == 1 && isUpperCase(word.charAt(0));
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static char lastChar(final String s) {
//     return s.charAt(s.length() -1);
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static String toEngLowerCase(final String s) { return s.toLowerCase(Locale.ENGLISH); }


// Path: nicta-ner/src/main/java/org/t3as/ner/util/Tokenizer.java
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import org.t3as.ner.Token;
import java.io.IOException;
import java.text.BreakIterator;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import static java.lang.Character.isLetterOrDigit;
import static java.lang.Character.isSpaceChar;
import static java.text.BreakIterator.DONE;
import static org.t3as.ner.util.Strings.endsWith;
import static org.t3as.ner.util.Strings.equalss;
import static org.t3as.ner.util.Strings.initChar;
import static org.t3as.ner.util.Strings.isSingleUppercaseChar;
import static org.t3as.ner.util.Strings.lastChar;
import static org.t3as.ner.util.Strings.toEngLowerCase;
import static org.t3as.ner.util.Tokenizer.Mode.WITHOUT_PUNCTUATION;
import static org.t3as.ner.util.Tokenizer.Mode.WITH_PUNCTUATION;

     * Tokenize some text - not thread safe.
     * @param text tokenize this
     * @return tokenized text
     */
    public List<List<Token>> process(final String text) {
        final List<List<Token>> paragraph = new ArrayList<>();
        currentSentence = new ArrayList<>();
        final Tokens tokens = splitText(text);

        while (tokens.hasNext()) {
            final Token t = tokens.next();
            final String trimmedWord = t.text.trim();

            // skip spaces
            if (trimmedWord.isEmpty()) continue;

            if (((mode == WITH_PUNCTUATION) || (mode == WITHOUT_PUNCTUATION && isLetterOrDigit(initChar(t.text))))) {
                boolean canBreakSentence = true;
                if (t.text.contains("'")) {
                    wordContainsApostrophe(t);
                }
                else if (".".equals(trimmedWord)) {
                    canBreakSentence = wordIsFullStop(t);
                }
                else if (":".equals(trimmedWord)) {
                    wordIsColon(tokens, t);
                }
                else currentSentence.add(t);

                // handling the end of a sentence
                if (canBreakSentence && equalss(trimmedWord, ".", ";", "?", "!")) {