// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static String clean(final String s) { return CLEAN.matcher(s).replaceAll(""); }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// @SuppressWarnings("ReturnOfNull")
// public static String simplify(final String s) {
//     if (s == null) return null;
//     final String s1 = Normalizer.normalize(s, Normalizer.Form.NFC);
//     return s1.trim();
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static String toEngLowerCase(final String s) { return s.toLowerCase(Locale.ENGLISH); }


import java.util.List;
import java.util.Set;
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.io.Resources.getResource;
import static com.google.common.io.Resources.readLines;
import static org.t3as.ner.util.Strings.clean;
import static org.t3as.ner.util.Strings.simplify;
import static org.t3as.ner.util.Strings.toEngLowerCase;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.LineProcessor;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
/*
 * #%L
 * NICTA t3as Named-Entity Recognition library
 * %%
 * Copyright (C) 2010 - 2014 NICTA
 * %%
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public
 * License along with this program.  If not, see
 * <http://www.gnu.org/licenses/gpl-3.0.html>.
 * #L%
 */
package org.t3as.ner.util;




public final class IO {

    private static final Splitter SPACES = Splitter.on(' ').trimResults().omitEmptyStrings();
    private static final Splitter TABS = Splitter.on('\t').trimResults().omitEmptyStrings();

    private IO() {}

    /** Return a Set containing trimmed lowercaseLines read from a file, skipping comments. */
    public static Set<String> lowercaseLines(final Class<?> origin, final String resource) throws IOException {
        return ImmutableSet.copyOf(new HashSet<String>() {{
            readResource(origin, resource, new NullReturnLineProcessor() {
                @Override
                public boolean processLine(@Nonnull final String line) {
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static String clean(final String s) { return CLEAN.matcher(s).replaceAll(""); }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// @SuppressWarnings("ReturnOfNull")
// public static String simplify(final String s) {
//     if (s == null) return null;
//     final String s1 = Normalizer.normalize(s, Normalizer.Form.NFC);
//     return s1.trim();
// }
// 
// Path: nicta-ner/src/main/java/org/t3as/ner/util/Strings.java
// public static String toEngLowerCase(final String s) { return s.toLowerCase(Locale.ENGLISH); }


// Path: nicta-ner/src/main/java/org/t3as/ner/util/IO.java
import java.util.List;
import java.util.Set;
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.io.Resources.getResource;
import static com.google.common.io.Resources.readLines;
import static org.t3as.ner.util.Strings.clean;
import static org.t3as.ner.util.Strings.simplify;
import static org.t3as.ner.util.Strings.toEngLowerCase;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.LineProcessor;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;

/*
 * #%L
 * NICTA t3as Named-Entity Recognition library
 * %%
 * Copyright (C) 2010 - 2014 NICTA
 * %%
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public
 * License along with this program.  If not, see
 * <http://www.gnu.org/licenses/gpl-3.0.html>.
 * #L%
 */
package org.t3as.ner.util;




public final class IO {

    private static final Splitter SPACES = Splitter.on(' ').trimResults().omitEmptyStrings();
    private static final Splitter TABS = Splitter.on('\t').trimResults().omitEmptyStrings();

    private IO() {}

    /** Return a Set containing trimmed lowercaseLines read from a file, skipping comments. */
    public static Set<String> lowercaseLines(final Class<?> origin, final String resource) throws IOException {
        return ImmutableSet.copyOf(new HashSet<String>() {{
            readResource(origin, resource, new NullReturnLineProcessor() {
                @Override
                public boolean processLine(@Nonnull final String line) {
                    final String l = simplify(line);