// Path: nicta-ner-common/src/main/java/org/t3as/ner/Token.java
// @Immutable
// public class Token {
// 
//     /** 0-indexed position of the first character of this token from the original text being analysed. */
//     public final int startIndex;
//     /** The text that this token is representing. */
//     public final String text;
// 
//     @JsonCreator
//     public Token(@JsonProperty("startIndex") final int startIndex, @JsonProperty("text") final String text) {
//         this.startIndex = startIndex;
//         this.text = text;
//     }
// 
//     @Override
//     public boolean equals(final Object o) {
//         if (this == o) return true;
//         if (o == null || getClass() != o.getClass()) return false;
//         final Token token = (Token) o;
//         return startIndex == token.startIndex && text.equals(token.text);
//     }
// 
//     @Override
//     public int hashCode() {
//         int result = startIndex;
//         result = 31 * result + text.hashCode();
//         return result;
//     }
// 
//     public String toString() { return startIndex + ":" + text; }
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean endsWith(final String word, final String... suffixes) {
//     if (word == null || suffixes == null) return false;
//     for (final String s : suffixes) {
//         if (word.endsWith(s)) return true;
//     }
//     return false;
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean equalss(final String word, final String... words) {
//     if (word == null || words == null) return false;
//     for (final String s : words) {
//         if (word.equals(s)) return true;
//     }
//     return false;
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static char initChar(final String s) {
//     return s.charAt(0);
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean isSingleUppercaseChar(final String word) {
//     return word.length() == 1 && isUpperCase(word.charAt(0));
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static char lastChar(final String s) {
//     return s.charAt(s.length() -1);
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static String toEngLowerCase(final String s) { return s.toLowerCase(Locale.ENGLISH); }


import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import org.t3as.ner.Token;
import java.io.IOException;
import java.text.BreakIterator;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import static java.lang.Character.isLetterOrDigit;
import static java.lang.Character.isSpaceChar;
import static java.text.BreakIterator.DONE;
import static org.t3as.ner.util.Strings.endsWith;
import static org.t3as.ner.util.Strings.equalss;
import static org.t3as.ner.util.Strings.initChar;
import static org.t3as.ner.util.Strings.isSingleUppercaseChar;
import static org.t3as.ner.util.Strings.lastChar;
import static org.t3as.ner.util.Strings.toEngLowerCase;
import static org.t3as.ner.util.Tokenizer.Mode.WITHOUT_PUNCTUATION;
import static org.t3as.ner.util.Tokenizer.Mode.WITH_PUNCTUATION;
/*
 * #%L
 * NICTA t3as Named-Entity Recognition library
 * %%
 * Copyright (C) 2010 - 2014 NICTA
 * %%
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public
 * License along with this program.  If not, see
 * <http://www.gnu.org/licenses/gpl-3.0.html>.
 * #L%
 */
package org.t3as.ner.util;




/**
 * This class utilizes a Java standard class to token the input sentence.
 * <p/>
 * This class is not thread safe.
 */
public class Tokenizer {

    public enum Mode {
        WITH_PUNCTUATION,
        WITHOUT_PUNCTUATION
    }

    private static final ImmutableCollection<String> ABBREVIATION_EXCEPTIONS;

    static {
        try { ABBREVIATION_EXCEPTIONS = ImmutableList.copyOf(IO.lowercaseLines(Tokenizer.class, "TokenizerAbbreviation")); }
        catch (final IOException e) { throw new RuntimeException("Could not load the TokenizerAbbreviation file.", e); }
    }

    private final Mode mode;
    private List<Token> currentSentence;

    public Tokenizer(final Mode mode) { this.mode = mode; }

    /**
     * Tokenize some text - not thread safe.
     * @param text tokenize this
     * @return tokenized text
     */
    public List<List<Token>> process(final String text) {
        final List<List<Token>> paragraph = new ArrayList<>();
        currentSentence = new ArrayList<>();
        final Tokens tokens = splitText(text);

        while (tokens.hasNext()) {
            final Token t = tokens.next();
            final String trimmedWord = t.text.trim();

            // skip spaces
            if (trimmedWord.isEmpty()) continue;

// Path: nicta-ner-common/src/main/java/org/t3as/ner/Token.java
// @Immutable
// public class Token {
// 
//     /** 0-indexed position of the first character of this token from the original text being analysed. */
//     public final int startIndex;
//     /** The text that this token is representing. */
//     public final String text;
// 
//     @JsonCreator
//     public Token(@JsonProperty("startIndex") final int startIndex, @JsonProperty("text") final String text) {
//         this.startIndex = startIndex;
//         this.text = text;
//     }
// 
//     @Override
//     public boolean equals(final Object o) {
//         if (this == o) return true;
//         if (o == null || getClass() != o.getClass()) return false;
//         final Token token = (Token) o;
//         return startIndex == token.startIndex && text.equals(token.text);
//     }
// 
//     @Override
//     public int hashCode() {
//         int result = startIndex;
//         result = 31 * result + text.hashCode();
//         return result;
//     }
// 
//     public String toString() { return startIndex + ":" + text; }
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean endsWith(final String word, final String... suffixes) {
//     if (word == null || suffixes == null) return false;
//     for (final String s : suffixes) {
//         if (word.endsWith(s)) return true;
//     }
//     return false;
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean equalss(final String word, final String... words) {
//     if (word == null || words == null) return false;
//     for (final String s : words) {
//         if (word.equals(s)) return true;
//     }
//     return false;
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static char initChar(final String s) {
//     return s.charAt(0);
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static boolean isSingleUppercaseChar(final String word) {
//     return word.length() == 1 && isUpperCase(word.charAt(0));
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static char lastChar(final String s) {
//     return s.charAt(s.length() -1);
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static String toEngLowerCase(final String s) { return s.toLowerCase(Locale.ENGLISH); }


// Path: nicta-ner/src/main/java/org/t3as/ner/util/Tokenizer.java
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import org.t3as.ner.Token;
import java.io.IOException;
import java.text.BreakIterator;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import static java.lang.Character.isLetterOrDigit;
import static java.lang.Character.isSpaceChar;
import static java.text.BreakIterator.DONE;
import static org.t3as.ner.util.Strings.endsWith;
import static org.t3as.ner.util.Strings.equalss;
import static org.t3as.ner.util.Strings.initChar;
import static org.t3as.ner.util.Strings.isSingleUppercaseChar;
import static org.t3as.ner.util.Strings.lastChar;
import static org.t3as.ner.util.Strings.toEngLowerCase;
import static org.t3as.ner.util.Tokenizer.Mode.WITHOUT_PUNCTUATION;
import static org.t3as.ner.util.Tokenizer.Mode.WITH_PUNCTUATION;

/*
 * #%L
 * NICTA t3as Named-Entity Recognition library
 * %%
 * Copyright (C) 2010 - 2014 NICTA
 * %%
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public
 * License along with this program.  If not, see
 * <http://www.gnu.org/licenses/gpl-3.0.html>.
 * #L%
 */
package org.t3as.ner.util;




/**
 * This class utilizes a Java standard class to token the input sentence.
 * <p/>
 * This class is not thread safe.
 */
public class Tokenizer {

    public enum Mode {
        WITH_PUNCTUATION,
        WITHOUT_PUNCTUATION
    }

    private static final ImmutableCollection<String> ABBREVIATION_EXCEPTIONS;

    static {
        try { ABBREVIATION_EXCEPTIONS = ImmutableList.copyOf(IO.lowercaseLines(Tokenizer.class, "TokenizerAbbreviation")); }
        catch (final IOException e) { throw new RuntimeException("Could not load the TokenizerAbbreviation file.", e); }
    }

    private final Mode mode;
    private List<Token> currentSentence;

    public Tokenizer(final Mode mode) { this.mode = mode; }

    /**
     * Tokenize some text - not thread safe.
     * @param text tokenize this
     * @return tokenized text
     */
    public List<List<Token>> process(final String text) {
        final List<List<Token>> paragraph = new ArrayList<>();
        currentSentence = new ArrayList<>();
        final Tokens tokens = splitText(text);

        while (tokens.hasNext()) {
            final Token t = tokens.next();
            final String trimmedWord = t.text.trim();

            // skip spaces
            if (trimmedWord.isEmpty()) continue;

            if (((mode == WITH_PUNCTUATION) || (mode == WITHOUT_PUNCTUATION && isLetterOrDigit(initChar(t.text))))) {