public class Configuration {
  private static final Logger LOG = LoggerFactory
      .getLogger(Configuration.class);

  public static final boolean RUNNING_WITHIN_JAR = Configuration.class
      .getResource("Configuration.class").toString().startsWith("jar:");

  public static final String WORKING_DIR_PATH = (RUNNING_WITHIN_JAR) ? ""
      : System.getProperty("user.dir") + File.separator;

  public static final String TEMP_DIR_PATH = System
      .getProperty("java.io.tmpdir");

  public static final String GLOBAL_RESOURCES_DATASETS_SEMEVAL = "global.resources.datasets.semeval";
  public static final String GLOBAL_RESOURCES_DATASETS_CRAWLER = "global.resources.datasets.crawler";

  public static final String GLOBAL_RESOURCES_DICT = "global.resources.dict";
  public static final String GLOBAL_RESOURCES_DICT_SENTIMENT = "global.resources.dict.sentiment";
  public static final String GLOBAL_RESOURCES_DICT_SENTIMENT_SENTIWORDNET_PATH = "global.resources.dict.sentiment.sentiwordnet.path";
  public static final String GLOBAL_RESOURCES_DICT_SLANG = "global.resources.dict.slang";
  public static final String GLOBAL_RESOURCES_DICT_WORDNET_PATH = "global.resources.dict.wordnet.path";

  public static final Map CONFIG = readConfig();

  @SuppressWarnings("rawtypes")
  public static Map readConfig() {
    Map conf = readConfigFile(WORKING_DIR_PATH + "conf/defaults.yaml", true);
    // read custom config
    LOG.info("Try to load user-specific config...");
    Map customConfig = readConfigFile(WORKING_DIR_PATH
        + "conf/configuration.yaml", false);
    if (customConfig != null) {
      conf.putAll(customConfig);
    } else if (RUNNING_WITHIN_JAR) {
      customConfig = readConfigFile("../conf/configuration.yaml", false);
      if (customConfig != null) {
        conf.putAll(customConfig);
      }
    }
    return conf;
  }

  @SuppressWarnings("rawtypes")
  public static Map readConfigFile(String file, boolean mustExist) {
    Yaml yaml = new Yaml(new SafeConstructor());
    Map ret = null;
    InputStream input = IOUtils.getInputStream(file);
    if (input != null) {
      ret = (Map) yaml.load(new InputStreamReader(input));
      LOG.info("Loaded " + file);
      try {
        input.close();
      } catch (IOException e) {
        LOG.error("IOException: " + e.getMessage());
      }
    } else if (mustExist) {
      LOG.error("Config file " + file + " was not found!");
    }
    if ((ret == null) && (mustExist)) {
      throw new RuntimeException("Config file " + file + " was not found!");
    }
    return ret;
  }

  public static <K, V> V get(K key) {
    return get(key, null);
  }

  public static <K, V> V get(K key, V defaultValue) {
    return get((Map<K, V>) CONFIG, key, defaultValue);
  }

  public static <K, V> V get(Map<K, V> map, K key, V defaultValue) {
    V value = map.get(key);
    if (value == null) {
      value = defaultValue;
    }
    return value;
  }

  public static Dataset getDataSetSemEval2013() {
    return Dataset.readFromYaml((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_SEMEVAL)).get("2013"));
  }

  public static Dataset getDataSetSemEval2013Mixed() {
    return Dataset.readFromYaml((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_SEMEVAL)).get("2013Mixed"));
  }

  public static Dataset getDataSetSemEval2014() {
    return Dataset.readFromYaml((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_SEMEVAL)).get("2014"));
  }

  public static List<Status> getDataSetCrawler() {
    return getDataSetCrawler((String) ((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_CRAWLER)).get("Crawler"))
        .get("language.filter"));
  }

  public static List<Status> getDataSetCrawler(String filterLanguage) {
    return JsonUtils.readTweetsDirectory((String) ((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_CRAWLER)).get("Crawler")).get("path"),
        filterLanguage);
  }

  public static List<Status> getDataSetCrawlerTest() {
    return getDataSetCrawlerTest((String) ((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_CRAWLER)).get("CrawlerTest"))
        .get("language.filter"));
  }

  public static List<Status> getDataSetCrawlerTest(String filterLanguage) {
    return JsonUtils.readTweetsDirectory((String) ((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_CRAWLER)).get("CrawlerTest"))
        .get("path"), filterLanguage);
  }

  public static List<String> getNameEntities() {
    return (List<String>) ((Map) CONFIG.get(GLOBAL_RESOURCES_DICT))
        .get("NameEntities");
  }

  public static List<String> getFirstNames() {
    return (List<String>) ((Map) CONFIG.get(GLOBAL_RESOURCES_DICT))
        .get("FirstNames");
  }

  public static List<Map> getEmoticons() {
    return (List<Map>) ((Map) CONFIG.get(GLOBAL_RESOURCES_DICT))
        .get("Emoticons");
  }

  public static List<Map> getInterjections() {
    return (List<Map>) ((Map) CONFIG.get(GLOBAL_RESOURCES_DICT))
        .get("Interjections");
  }

  public static List<String> getStopWords() {
    return (List<String>) ((Map) CONFIG.get(GLOBAL_RESOURCES_DICT))
        .get("StopWords");
  }

  public static List<Map> getSentimentWordlists() {
    return (List<Map>) CONFIG.get(GLOBAL_RESOURCES_DICT_SENTIMENT);
  }

  public static String getSentiWordNetDict() {
    return (String) CONFIG
        .get(GLOBAL_RESOURCES_DICT_SENTIMENT_SENTIWORDNET_PATH);
  }

  public static List<Map> getSlangWordlists() {
    return (List<Map>) CONFIG.get(GLOBAL_RESOURCES_DICT_SLANG);
  }

  public static String getWordNetDict() {
    return (String) CONFIG.get(GLOBAL_RESOURCES_DICT_WORDNET_PATH);
  }

} public class Dataset implements Serializable {
  private static final long serialVersionUID = -1822543611053727897L;
  public static final String SERIAL_EXTENSION = ".ser";
  private static final Logger LOG = LoggerFactory.getLogger(Dataset.class);

  private String m_datasetPath;
  private String m_trainDataFile;
  private String m_devDataFile;
  private String m_testDataFile;

  private String m_delimiter;
  private int m_idIndex;
  private int m_labelIndex;
  private int m_textIndex;

  private List<String> m_positiveLabels;
  private List<String> m_negativeLabels;
  private List<String> m_neutralLabels;

  private int m_positiveValue;
  private int m_negativeValue;
  private int m_neutralValue;

  private svm_parameter m_svmParam;

  private List<Tweet> m_trainTweets = null;
  private List<Tweet> m_devTweets = null;
  private List<Tweet> m_testTweets = null;

  public Dataset(String datasetPath, String trainDataFile, String devDataFile,
      String testDataFile, String delimiter, int idIndex, int labelIndex,
      int textIndex, List<String> positiveLabels, List<String> negativeLabels,
      List<String> neutralLabels, int positiveValue, int negativeValue,
      int neutralValue, svm_parameter svmParam) {
    this.m_datasetPath = datasetPath;
    this.m_trainDataFile = trainDataFile;
    this.m_devDataFile = devDataFile;
    this.m_testDataFile = testDataFile;

    this.m_delimiter = delimiter;

    this.m_idIndex = idIndex;
    this.m_labelIndex = labelIndex;
    this.m_textIndex = textIndex;

    this.m_positiveLabels = positiveLabels;
    this.m_negativeLabels = negativeLabels;
    this.m_neutralLabels = neutralLabels;

    this.m_positiveValue = positiveValue;
    this.m_negativeValue = negativeValue;
    this.m_neutralValue = neutralValue;

    this.m_svmParam = svmParam;
  }

  public String getDatasetPath() {
    return m_datasetPath;
  }

  public String getTrainDataFile() {
    return (m_trainDataFile != null) ? m_datasetPath + File.separator
        + m_trainDataFile : null;
  }

  public String getTrainDataSerializationFile() {
    return (m_trainDataFile != null) ? m_datasetPath + File.separator
        + m_trainDataFile + SERIAL_EXTENSION : null;
  }

  public String getDevDataFile() {
    return (m_devDataFile != null) ? m_datasetPath + File.separator
        + m_devDataFile : null;
  }

  public String getDevDataSerializationFile() {
    return (m_devDataFile != null) ? m_datasetPath + File.separator
        + m_devDataFile + SERIAL_EXTENSION : null;
  }

  public String getTestDataFile() {
    return (m_testDataFile != null) ? m_datasetPath + File.separator
        + m_testDataFile : null;
  }

  public String getTestDataSerializationFile() {
    return (m_testDataFile != null) ? m_datasetPath + File.separator
        + m_testDataFile + SERIAL_EXTENSION : null;
  }

  public String getDelimiter() {
    return m_delimiter;
  }

  public int getIdIndex() {
    return m_idIndex;
  }

  public int getLabelIndex() {
    return m_labelIndex;
  }

  public int getTextIndex() {
    return m_textIndex;
  }

  public List<String> getNegativeLabels() {
    return m_negativeLabels;
  }

  public List<String> getNeutralLabels() {
    return m_neutralLabels;
  }

  public List<String> getPositiveLabels() {
    return m_positiveLabels;
  }

  public int getNegativeValue() {
    return m_negativeValue;
  }

  public int getNeutralValue() {
    return m_neutralValue;
  }

  public int getPositiveValue() {
    return m_positiveValue;
  }

  public svm_parameter getSVMParam() {
    return m_svmParam;
  }

  public List<Tweet> getTrainTweets(boolean includeDevTweets) {
    if ((m_trainTweets == null) && (this.getTrainDataFile() != null)) {
      m_trainTweets = FileUtils.readTweets(this.getTrainDataFile(), this);
    }

    // optional include dev tweets in training dataset
    if (includeDevTweets) {
      List<Tweet> combinedList = new ArrayList<Tweet>(m_trainTweets);
      combinedList.addAll(getDevTweets());
      return combinedList;
    } else {
      return m_trainTweets;
    }
  }

  public List<Tweet> getDevTweets() {
    if ((m_devTweets == null) && (this.getDevDataFile() != null)) {
      m_devTweets = FileUtils.readTweets(this.getDevDataFile(), this);
    }
    return m_devTweets;
  }

  public List<Tweet> getTestTweets() {
    if ((m_testTweets == null) && (this.getTestDataFile() != null)) {
      m_testTweets = FileUtils.readTweets(this.getTestDataFile(), this);
    }
    return m_testTweets;
  }

  public void printDatasetStats() {
    LOG.info("Dataset: " + getDatasetPath());

    LOG.info("Train Dataset: " + getTrainDataFile());
    printTweetStats(getTrainTweets(false));

    LOG.info("Dev Dataset: " + getDevDataFile());
    printTweetStats(getDevTweets());

    LOG.info("Test Dataset: " + getTestDataFile());
    printTweetStats(getTestTweets());
  }

  public static void printTweetStats(List<Tweet> tweets) {
    if (tweets != null) {
      Map<Integer, Integer> counts = new TreeMap<Integer, Integer>();
      for (Tweet tweet : tweets) {
        int key = tweet.getScore().intValue();
        Integer count = counts.get(key);
        counts.put(key, ((count != null) ? count + 1 : 1));
      }

      int total = 0;
      int max = 0;
      for (Map.Entry<Integer, Integer> entry : counts.entrySet()) {
        LOG.info("Class: \t" + entry.getKey() + "\t" + entry.getValue());
        total += entry.getValue();
        if (entry.getValue() > max) {
          max = entry.getValue();
        }
      }
      LOG.info("Total: " + total);

      LOG.info("Optimal Class Weights: ");
      for (Map.Entry<Integer, Integer> entry : counts.entrySet()) {
        LOG.info("Class: \t" + entry.getKey() + "\t"
            + (max / (double) entry.getValue()));
      }
    }
  }

  @Override
  public String toString() {
    return "Dataset [datasetPath=" + m_datasetPath + ", trainDataFile="
        + m_trainDataFile + ", devDataFile=" + m_devDataFile
        + ", testDataFile=" + m_testDataFile + ", delimiter=" + m_delimiter
        + ", idIndex=" + m_idIndex + ", textIndex=" + m_textIndex
        + ", labelIndex=" + m_labelIndex + ", positiveLabels="
        + m_positiveLabels + ", negativeLabel=" + m_negativeLabels
        + ", neutralLabel=" + m_neutralLabels + ", positiveValue="
        + m_positiveValue + ", negativeValue=" + m_negativeValue
        + ", neutralValue=" + m_neutralValue + "]";
  }

  @SuppressWarnings({ "rawtypes", "unchecked" })
  public static Dataset readFromYaml(Map dataset) {
    svm_parameter svmParam = SVM.getDefaultParameter();

    if (dataset.get("svm.kernel") != null) {
      svmParam.kernel_type = (Integer) dataset.get("svm.kernel");
    }

    if (dataset.get("svm.c") != null) {
      svmParam.C = (Double) dataset.get("svm.c");
    }

    if (dataset.get("svm.gamma") != null) {
      svmParam.gamma = (Double) dataset.get("svm.gamma");
    }

    if (dataset.get("svm.class.weights") != null) {
      Map<Integer, Double> classWeights = (Map<Integer, Double>) dataset
          .get("svm.class.weights");

      svmParam.nr_weight = classWeights.size();
      svmParam.weight_label = new int[svmParam.nr_weight];
      svmParam.weight = new double[svmParam.nr_weight];

      for (Map.Entry<Integer, Double> entry : classWeights.entrySet()) {
        svmParam.weight_label[entry.getKey()] = entry.getKey();
        svmParam.weight[entry.getKey()] = entry.getValue();
      }
    }

    return new Dataset((String) dataset.get("path"),
        (String) dataset.get("train.file"), (String) dataset.get("dev.file"),
        (String) dataset.get("test.file"), (String) dataset.get("delimiter"),
        (Integer) dataset.get("tweetId.index"),
        (Integer) dataset.get("label.index"),
        (Integer) dataset.get("text.index"),
        (List<String>) dataset.get("positive.labels"),
        (List<String>) dataset.get("negative.labels"),
        (List<String>) dataset.get("neutral.labels"),
        (Integer) dataset.get("positive.class.value"),
        (Integer) dataset.get("negative.class.value"),
        (Integer) dataset.get("neutral.class.value"), svmParam);
  }

  public static void main(String[] args) {
    Dataset dataSet2013Mixed = Configuration.getDataSetSemEval2013Mixed();
    LOG.info("Dataset: SemEval2013 mixed");
    LOG.info(dataSet2013Mixed.toString());
    dataSet2013Mixed.printDatasetStats();

    Dataset dataSet2013 = Configuration.getDataSetSemEval2013();
    LOG.info("Dataset: SemEval2013");
    dataSet2013.printDatasetStats();

    // Dataset dataSet2014 = Configuration.getDataSetSemEval2014();
    // LOG.info("Dataset: SemEval2014");
    // dataSet2014.printDatasetStats();
  }

} public class FirstNames {
  private static final Logger LOG = LoggerFactory.getLogger(FirstNames.class);
  private static final FirstNames INSTANCE = new FirstNames();

  private Set<String> m_firstNames = null;

  private FirstNames() {
    for (String file : Configuration.getFirstNames()) {
      LOG.info("Load FirstNames from: " + file);
      if (m_firstNames == null) {
        m_firstNames = FileUtils.readFile(file, true);
      } else {
        m_firstNames.addAll(FileUtils.readFile(file, true));
      }
    }
  }

  public static FirstNames getInstance() {
    return INSTANCE;
  }

  public boolean isFirstName(String value) {
    return m_firstNames.contains(value.toLowerCase());
  }

  public static void main(String[] args) {
    FirstNames firstNames = FirstNames.getInstance();
    // test FirstNames
    String[] testFirstNames = new String[] { "Kevin", "Martin", "martin",
        "justin" };
    for (String s : testFirstNames) {
      System.out
          .println("isFirstName(" + s + "): " + firstNames.isFirstName(s));
    }
  }
} public class Interjections {
  private static final Logger LOG = LoggerFactory
      .getLogger(Interjections.class);
  private static final Interjections INSTANCE = new Interjections();

  private Set<String> m_interjections = null;
  private List<Pattern> m_interjectionPatterns = null;

  private Interjections() {
    List<Map> interjectionFiles = Configuration.getInterjections();
    for (Map interjectionEntry : interjectionFiles) {
      String file = (String) interjectionEntry.get("path");
      Boolean containsRegex = (Boolean) interjectionEntry.get("containsRegex");

      Set<String> interjections = FileUtils.readFile(file);

      if (containsRegex) {
        LOG.info("Loaded Interjections including regex patterns from: " + file);
        if (m_interjectionPatterns == null) {
          m_interjectionPatterns = new ArrayList<Pattern>();
        }
        for (String interjection : interjections) {
          m_interjectionPatterns.add(Pattern.compile("^" + interjection + "$"));
        }
      } else {
        LOG.info("Loaded Interjections from: " + file);
        if (m_interjections == null) {
          m_interjections = new HashSet<String>();
        }
        m_interjections.addAll(interjections);
      }
    }
  }

  public static Interjections getInstance() {
    return INSTANCE;
  }

  public boolean isInterjection(String value) {
    value = value.toLowerCase();
    boolean result = false;
    if (m_interjections != null) {
      result = m_interjections.contains(value);
    }

    if ((result == false) && (m_interjectionPatterns != null)) {
      for (Pattern interjection : m_interjectionPatterns) {
        Matcher m = interjection.matcher(value);
        if (m.find()) {
          return true;
        }
      }
    }

    return result;
  }

  public static void main(String[] args) {
    Interjections interjections = Interjections.getInstance();
    // test Interjections
    String[] testInterjections = new String[] { "wow", "wooow", "wooo" };
    for (String s : testInterjections) {
      System.out.println("isInterjection(" + s + "): "
          + interjections.isInterjection(s));
    }
  }
} public class NameEntities {
  private static final Logger LOG = LoggerFactory.getLogger(NameEntities.class);
  private static final NameEntities INSTANCE = new NameEntities();

  private Set<String> m_nameEntities = null;

  private NameEntities() {
    for (String file : Configuration.getNameEntities()) {
      LOG.info("Load NameEntities from: " + file);
      if (m_nameEntities == null) {
        m_nameEntities = FileUtils.readFile(file);
      } else {
        m_nameEntities.addAll(FileUtils.readFile(file));
      }
    }
  }

  public static NameEntities getInstance() {
    return INSTANCE;
  }

  public boolean isNameEntity(String value) {
    return m_nameEntities.contains(value.toLowerCase());
  }

  public static void main(String[] args) {
    NameEntities nameEntities = NameEntities.getInstance();
    System.out.println("NameEntities of 'apple': '"
        + nameEntities.isNameEntity("apple") + "'");
    System.out.println("NameEntities of 'facebook': '"
        + nameEntities.isNameEntity("facebook") + "'");
  }
} public class SlangCorrection {
  private static final Logger LOG = LoggerFactory
      .getLogger(SlangCorrection.class);
  private static final SlangCorrection INSTANCE = new SlangCorrection();

  private Map<String, String[]> m_slangWordList = new HashMap<String, String[]>();

  private SlangCorrection() {
    List<Map> slangWordLists = Configuration.getSlangWordlists();
    for (Map slangWordListEntry : slangWordLists) {
      String file = (String) slangWordListEntry.get("path");
      String separator = (String) slangWordListEntry.get("delimiter");
      boolean isEnabled = (Boolean) slangWordListEntry.get("enabled");
      if (isEnabled) {
        LOG.info("Load SlangLookupTable from: " + file);
        Map<String, String> slangWordList = FileUtils.readFile(file, separator);
        for (Map.Entry<String, String> entry : slangWordList.entrySet()) {
          if (!m_slangWordList.containsKey(entry.getKey())) {
            m_slangWordList.put(entry.getKey(), entry.getValue().split(" "));
          }
        }
      }
    }
  }

  public static SlangCorrection getInstance() {
    return INSTANCE;
  }

  public String[] getCorrection(String token) {
    // LOG.info("getCorrection('" + token + "'): "
    // + Arrays.toString(m_slangWordList.get(token)));
    return m_slangWordList.get(token);
  }

  public static void main(String[] args) {
    SlangCorrection slangCorrection = SlangCorrection.getInstance();

    // test testSlangCorrection
    String[] testSlangCorrection = new String[] { "afaik", "cum", "w/", "Fri",
        "fri", "Sat", "sat", "Sun", "sun", "U.S.", "U.K." };
    for (String s : testSlangCorrection) {
      System.out.println("slang correction of '" + s + "': '"
          + Arrays.toString(slangCorrection.getCorrection(s)) + "'");
    }
  }

} public class Tokenizer {
  private static final Logger LOG = LoggerFactory.getLogger(Tokenizer.class);
  private static final boolean LOGGING = Configuration.get(
      "commons.tokenizer.logging", false);

  public static enum Type {
    REGEX_TOKENIZER, ARK_TOKENIZER, STANFORD_TOKENIZER
  }

  public static List<List<String>> tokenizeTweets(List<Tweet> tweets) {
    return tokenizeTweets(tweets, Type.REGEX_TOKENIZER);
  }

  public static List<List<String>> tokenizeTweets(List<Tweet> tweets, Type type) {
    List<List<String>> tokenizedTweets = new ArrayList<List<String>>();
    for (Tweet tweet : tweets) {
      tokenizedTweets.add(tokenize(tweet.getText(), type));
    }
    return tokenizedTweets;
  }

  public static List<String> tokenize(String str) {
    return tokenize(str, Type.REGEX_TOKENIZER);
  }

  public static List<String> tokenize(String str, Type type) {
    // Step 1) Trim text
    str = str.trim();

    // Step 2) Replace Unicode symbols \u0000
    if (UnicodeUtils.containsUnicode(str)) {
      String replacedText = UnicodeUtils.replaceUnicodeSymbols(str);
      // LOG.info("Replaced Unicode symbols from '" + str + "' to '"
      // + replacedText + "'");
      if ((LOGGING) && (replacedText.equals(str))) {
        LOG.warn("Unicode symbols could not be replaced: '" + str + "'");
      }
      str = replacedText;
    }

    // Step 3) Replace HTML symbols &#[0-9];
    if (HtmlUtils.containsHtml(str)) {
      String replacedText = HtmlUtils.replaceHtmlSymbols(str);
      // LOG.info("Replaced HTML symbols from '" + text + "' to '"
      // + replacedText + "'");
      if ((LOGGING) && (replacedText.equals(str))) {
        LOG.warn("HTML symbols could not be replaced: '" + str + "'");
      }
      str = replacedText;
    }

    // Step 4) Tokenize
    List<String> tokenizedTokens = null;

    switch (type) {
      case REGEX_TOKENIZER:
        tokenizedTokens = new ArrayList<String>();
        Matcher m = RegexUtils.TOKENIZER_PATTERN.matcher(str);
        while (m.find()) {
          tokenizedTokens.add(m.group());
        }
        break;

      case ARK_TOKENIZER:
        tokenizedTokens = Twokenize.tokenize(str);
        break;

      case STANFORD_TOKENIZER:
        TokenizerFactory<Word> tokenizer = PTBTokenizerFactory
            .newTokenizerFactory();
        tokenizer.setOptions("ptb3Escaping=false");
        List<List<HasWord>> sentences = MaxentTagger.tokenizeText(
            new StringReader(str), tokenizer);
        // Convert sentences to List<String>
        tokenizedTokens = new ArrayList<String>();
        for (List<HasWord> sentence : sentences) {
          for (HasWord word : sentence) {
            tokenizedTokens.add(word.word());
          }
        }
        break;

      default:
        break;
    }

    return tokenizedTokens;
  }

  public static void main(String[] args) {
    boolean extendedTest = true;
    List<Tweet> tweets = null;
    Type type = Type.REGEX_TOKENIZER;

    // load tweets
    if (extendedTest) {
      Dataset dataset = Configuration.getDataSetSemEval2013();
      tweets = dataset.getTrainTweets(true);
    } else { // test tweets
      tweets = Tweet.getTestTweets();
      tweets
          .add(new Tweet(
              0L,
              ":-))) xDD XDD ;) :) :-) :) :D :o) :] :3 :c) :> =] 8) =) :} :^) :-D 8-D 8D x-D xD X-D XD =-D =D =-3 =3"));
      tweets
          .add(new Tweet(0L,
              ">:[ :-( :(  :-c :c :-< :< :-[ :[ :{ ;( :-|| :@ >:( :'-( :'( :'-) :') \\m/"));
      tweets
          .add(new Tweet(
              0L,
              "\"All the money you've spent on my credit card I'm taking it out of your account\".... Hi I'm Sydney and I'm filing bankruptcy\ud83d\ude05\ud83d\ude05\ud83d\ude05\ud83d\ude05 \ud83d\ude05"));
      tweets
          .add(new Tweet(
              0L,
              "word:-) Moon:Oct Jobs! Thursday:... http://t.co/TZZzrrKa +1 (000) 123-4567, (000) 123-4567, and 123-4567 "));
      tweets
          .add(new Tweet(
              0L,
              "t/m k/o w/my b/slisten Rt/follow S/o S/O O/U O/A w/ w/Biden w/deals w/you w/the w/her"));
      tweets
          .add(new Tweet(0L, "5pm 5Am 5% $5 5-6am 7am-10pm U.S. U.K. L.O.V.E"));
    }

    // Tokenize tweets
    long startTime = System.currentTimeMillis();
    for (Tweet tweet : tweets) {
      List<String> tokens = Tokenizer.tokenize(tweet.getText(), type);
      LOG.info("Tweet: '" + tweet.getText() + "'");
      LOG.info("Tokens: " + tokens);
    }
    LOG.info("Tokenize finished after "
        + (System.currentTimeMillis() - startTime) + " ms");
  }

} public final class Tweet implements Serializable {
  private static final long serialVersionUID = 5246263296320608188L;
  private final Long m_id;
  private final String m_text;
  private final Double m_score;

  public Tweet(Long id, String text, Double score) {
    this.m_id = id;
    this.m_text = text;
    this.m_score = score;
  }

  public Tweet(Long id, String text) {
    this(id, text, null);
  }

  public Long getId() {
    return m_id;
  }

  public String getText() {
    return m_text;
  }

  public Double getScore() {
    return m_score;
  }

  @Override
  public boolean equals(Object obj) {
    if (obj == null) {
      return false;
    }
    if (getClass() != obj.getClass()) {
      return false;
    }
    final Tweet other = (Tweet) obj;
    // check if id is matching
    if (this.m_id != other.getId()) {
      return false;
    }
    // check if text is matching
    if (!this.m_text.equals(other.getText())) {
      return false;
    }
    return true;
  }

  @Override
  public String toString() {
    return "Tweet [id=" + m_id + ", text=" + m_text
        + ((m_score != null) ? ", score=" + m_score : "") + "]";
  }

  public static List<Tweet> getTestTweets() {
    List<Tweet> tweets = new ArrayList<Tweet>();
    tweets.add(new Tweet(1L,
        "Gas by my house hit $3.39 !!!! I'm goin to Chapel Hill on Sat . :)",
        1.0));
    tweets
        .add(new Tweet(
            2L,
            "@oluoch @victor_otti @kunjand I just watched it ! Sridevi's comeback .... U remember her from the 90s ?? Sun mornings on NTA ;)",
            1.0));
    tweets
        .add(new Tweet(
            3L,
            "PBR & @mokbpresents bring you Jim White at the @Do317 Lounge on October 23rd at 7 pm ! http://t.co/7x8OfC56.",
            0.5));
    tweets
        .add(new Tweet(
            4L,
            "Why is it so hard to find the @TVGuideMagazine these days ? Went to 3 stores for the Castle cover issue . NONE . Will search again tomorrow ...",
            0.0));
    tweets.add(new Tweet(0L,
        "called in sick for the third straight day.  ugh.", 0.0));
    tweets
        .add(new Tweet(
            5L,
            "Here we go.  BANK FAAAAIL FRIDAY -- The FDIC says the Bradford Bank in Baltimore, Maryland has become the 82nd bank failure of the year.",
            0.0));
    tweets.add(new Tweet(0L,
        "Oh, I'm afraid your Windows-using friends will not survive.", 0.0));

    tweets
        .add(new Tweet(
            6L,
            "Excuse the connectivity of this live stream , from Baba Amr , so many activists using only one Sat Modem . LIVE http://t.co/U283IhZ5 #Homs"));
    tweets
        .add(new Tweet(
            7L,
            "Show your LOVE for your local field & it might win an award ! Gallagher Park #Bedlington current 4th in National Award http://t.co/WeiMDtQt"));
    tweets
        .add(new Tweet(
            8L,
            "@firecore Can you tell me when an update for the Apple TV 3rd gen becomes available ? The missing update holds me back from buying #appletv3"));
    tweets
        .add(new Tweet(
            9L,
            "My #cre blog Oklahoma Per Square Foot returns to the @JournalRecord blog hub tomorrow . I will have some interesting local data to share ."));
    tweets
        .add(new Tweet(
            10L,
            "\" &quot; @bbcburnsy : Loads from SB ; &quot;talks with Chester continue ; no deals 4 out of contract players ' til Jan ; Dev t Roth , Coops to Chest'ld #hcafc \""));

    // thesis examples
    tweets
        .add(new Tweet(
            11L,
            "@user1 @user2 OMG I just watched Michael&#039;s comeback ! U remember him from the 90s ?? yaaaa \uD83D\uDE09 #michaelcomeback"));
    tweets
        .add(new Tweet(12L,
            "@user1 @user2 OMG I just watched it again ! \uD83D\uDE09 #michaelcomeback"));
    tweets.add(new Tweet(13L,
        "@user3 I luv him too ! \uD83D\uDE09 #michaelcomeback"));

    return tweets;
  }

} public class RegexUtils {
  private static final String SPACE_EXCEPTIONS = "\\n\\r";
  public static final String SPACE_CHAR_CLASS = "\\p{C}\\p{Z}&&[^"
      + SPACE_EXCEPTIONS + "\\p{Cs}]";
  public static final String SPACE_REGEX = "[" + SPACE_CHAR_CLASS + "]";

  public static final String PUNCTUATION_CHAR_CLASS = "\\p{P}\\p{M}\\p{S}"
      + SPACE_EXCEPTIONS;
  public static final String PUNCTUATION_REGEX = "[" + PUNCTUATION_CHAR_CLASS
      + "]";

  // URL Regex
  public static final String URL = "(?:" + "(?i)(https?|ftp)://" + "(-\\.)?"
      + "([^\\s/?\\.#-]+\\.?)*(/[^\\s\\.]*)?" + ")";
  public static final Pattern URL_PATTERN = Pattern.compile(URL);

  // Email Regex
  public static final String EMAIL_ADDRESS = "(?:"
      + "[a-zA-Z0-9\\+\\.\\_\\%\\-\\+]{1,256}" + "\\@"
      + "[a-zA-Z0-9][a-zA-Z0-9\\-]{0,64}" + "(" + "\\."
      + "[a-zA-Z0-9][a-zA-Z0-9\\-]{0,25}" + ")+" + ")";
  public static final Pattern EMAIL_ADDRESS_PATTERN = Pattern
      .compile(EMAIL_ADDRESS);

  // Phone Regex
  public static final String PHONE = "(?:" + "(?:\\+?[01][\\-\\s.]*)?"
      + "(?:[\\(]?\\d{3}[\\-\\s.\\)]*)?" + "\\d{3}" + "[\\-\\s.]*" + "\\d{4}"
      + ")";
  public static final Pattern PHONE_PATTERN = Pattern.compile(PHONE);

  // Username Regex
  public static final String USER_NAME = "(?:" + "\\@+([A-Za-z]+[A-Za-z0-9_]+)"
      + ")";
  public static final Pattern USER_NAME_PATTERN = Pattern.compile(USER_NAME);

  // Hashtag Regex
  public static final String HASH_TAG = "(?:"
      + "\\#+([A-Za-z]+[A-Za-z0-9_\\'\\-]*[A-Za-z0-9_]+)" + ")";
  public static final Pattern HASH_TAG_PATTERN = Pattern.compile(HASH_TAG);

  // Retweet Regex
  public static final String RETWEET = "(?:" + "(?i)(RT|retweet|from|via)"
      + "((?:\\b\\W*\\@\\w+)*)" + ")";
  public static final Pattern RETWEET_PATTERN = Pattern.compile(RETWEET);

  // Matches a HTML tag
  public static final String HTML_TAG = "<[^>]+>";
  public static final Pattern HTML_TAG_PATTERN = Pattern.compile(HTML_TAG);

  // Contains HTML symbols
  public static final String CONTAINS_HTML_SYMBOLS = "(?:" + "&#[0-9]{2,4};)"
      + "|" + "(?:&[a-zA-Z0-9]{2,6};" + ")";
  public static final Pattern CONTAINS_HTML_SYMBOLS_PATTERN = Pattern
      .compile(CONTAINS_HTML_SYMBOLS);

  // Contains Unicode symbols
  public static final String CONTAINS_UNICODE_SYMBOLS = "(?:"
      + "[^\\u0000-\\u007F]+" + ")" + "|" + "(?:" + "\\\\u[0-9a-fA-F]{4,5}+"
      + ")";
  public static final Pattern CONTAINS_UNICODE_SYMBOLS_PATTERN = Pattern
      .compile(CONTAINS_UNICODE_SYMBOLS);

  // Two or more repeating chars
  // "(.)\\1{1,}" means any character (added to group 1)
  // followed by itself at least one times, this means two equal chars
  public static final String TWO_OR_MORE_REPEATING_CHARS = "(.)\\1{1,}";
  public static final Pattern TWO_OR_MORE_REPEATING_CHARS_PATTERN = Pattern
      .compile(TWO_OR_MORE_REPEATING_CHARS);

  // Three or more repeating chars
  // "(.)\\1{2,}" means any character (added to group 1)
  // followed by itself at least two times, this means three equal chars
  public static final String THREE_OR_MORE_REPEATING_CHARS = "(.)\\1{2,}";
  public static final Pattern THREE_OR_MORE_REPEATING_CHARS_PATTERN = Pattern
      .compile(THREE_OR_MORE_REPEATING_CHARS);

  // Punctuation Regex
  public static final String PUNCTUATIONS = "^[\\p{Punct}\\s]+$";
  public static final Pattern PUNCTUATIONS_PATTERN = Pattern
      .compile(PUNCTUATIONS);

  // Underscore Regex
  public static final String UNDERSCORES = "^_+$";
  public static final Pattern UNDERSCORES_PATTERN = Pattern
      .compile(UNDERSCORES);

  // Starts with an alphabetic character
  public static final String STARTS_WITH_ALPHABETIC_CHAR = "^[a-zA-Z].*$";
  public static final Pattern STARTS_WITH_ALPHABETIC_CHAR_PATTERN = Pattern
      .compile(STARTS_WITH_ALPHABETIC_CHAR);

  // Number Regex
  public static final String NUMBER = "(?:" + "[+\\-]?" + "\\d+" + "(\\,\\d+)?"
      + "(\\.\\d+)?" + ")";
  public static final Pattern NUMBER_PATTERN = Pattern.compile(NUMBER);

  // Special number Regex
  public static final String SPECIAL_NUMBER = "(?:" + "\\$?" + "[+\\-]?"
      + "\\d+" + "([\\.|\\,|\\:|\\-]\\d+)*" + "(?:" + "(?i)(%|fm|am|pm|p|lb)"
      + "|" + "(c|p|st|nd|rd|th)" + ")?" + ")";
  public static final Pattern SPECIAL_NUMBER_PATTERN = Pattern
      .compile(SPECIAL_NUMBER);

  // Separated number Regex
  public static final String SEPARATED_NUMBER = "(?:" + "\\d+" + "(?:"
      + "(?i)(am|pm)" + "|" + "(c|st|nd|rd|th)" + ")?" + "[\\/|\\,|\\-]+"
      + "\\d+" + "([\\/|\\,|\\-]\\d+)*" + "(?:" + "(?i)(%|fm|am|pm|lb)" + "|"
      + "(c|p|st|nd|rd|th)" + ")?" + ")";
  public static final Pattern SEPARATED_NUMBER_PATTERN = Pattern
      .compile(SEPARATED_NUMBER);

  // Punctuation between words
  public static final String PUNCTUATION_BETWEEN_WORDS = "^(.*[^\\.|\\,|\\!|\\/|\\-])"
      + "[\\.|\\,|\\!|\\/|\\-]+" + "([^\\.|\\,|\\!|\\/|\\-].*)$";
  public static final Pattern PUNCTUATION_BETWEEN_WORDS_PATTERN = Pattern
      .compile(PUNCTUATION_BETWEEN_WORDS);

  // Alternating letter dot pattern e.g., L.O.V.E
  public static final String ALTERNATING_LETTER_DOT = "[a-zA-Z]\\.(?:[a-zA-Z](\\.)?)+";
  public static final Pattern ALTERNATING_LETTER_DOT_PATTERN = Pattern
      .compile(ALTERNATING_LETTER_DOT);

  // Words with apostrophes or dashes
  public static final String WORDS_WITH_APOSTROPHES_DASHES = "(?:"
      + "[a-zA-Z][a-zA-Z\\'\\-\\_]+[a-zA-Z]" + ")";

  // Words without apostrophes or dashes
  public static final String WORDS_WITHOUT_APOSTROPHES_DASHES = "(?:"
      + "[\\w_]+" + ")";

  // Ellipsis dots, sequences of two or more periods
  public static final String ELLIPSIS_DOTS = "(?:" + "\\.(?:\\s*\\.){1,}" + ")";

  // Non-whitespace char
  public static final String NOT_A_WHITESPACE = "(?:" + "\\S" + ")";

  // Emoticon Regex
  // TODO word:-) is not valid
  private static final String EMOTICON_EYES = "[:;=8xX*<>=^|#%]"; // eyes
  private static final String EMOTICON_NOSE = "[']?" + "[-_co^./]?" + "[\\\\]?"; // nose
  private static final String EMOTICON_MOUTH = "[0]?" // mouth
      + "([\\(\\)\\[\\]\\/\\\\}{*.^<>=@|,bdDpPLScoO$X#J3&]" + ")\\1{0,}";
  public static final String EMOTICON = "(?:" + "[<>oO0}3|]?" + EMOTICON_EYES
      + EMOTICON_NOSE + EMOTICON_MOUTH + "|" /* reverse */
      + EMOTICON_MOUTH + EMOTICON_EYES + "[<>]?" + ")";
  private static final String EMOTICON_DELIMITER = SPACE_REGEX + "|"
      + PUNCTUATION_REGEX;
  public static final Pattern EMOTICON_PATTERN = Pattern.compile("(?<=^|"
      + EMOTICON_DELIMITER + ")" + EMOTICON + "(?=$|" + EMOTICON_DELIMITER
      + ")");

  // Slang pattern to match w/
  public static final String SLANG = "[a-zA-Z]\\/[a-zA-z]*" + "|" + "\\\\m\\/";
  public static final Pattern SLANG_PATTERN = Pattern.compile("(?<=^|"
      + EMOTICON_DELIMITER + ")" + SLANG + "(?=$|" + EMOTICON_DELIMITER + ")");

  // Attention the order does matter
  public static final Pattern TOKENIZER_PATTERN = Pattern
      .compile(EMOTICON_PATTERN.pattern() + "|" + URL + "|" + PHONE + "|"
          + EMAIL_ADDRESS + "|" + USER_NAME + "|" + HASH_TAG + "|"
          + SLANG_PATTERN.pattern() + "|" + ALTERNATING_LETTER_DOT + "|"
          + WORDS_WITH_APOSTROPHES_DASHES + "|" + SEPARATED_NUMBER + "|"
          + SPECIAL_NUMBER + "|" + WORDS_WITHOUT_APOSTROPHES_DASHES + "|"
          + ELLIPSIS_DOTS + "|" + NOT_A_WHITESPACE);

} public class StringUtils {
  private static final Logger LOG = LoggerFactory.getLogger(StringUtils.class);

  public static boolean isURL(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.URL_PATTERN.matcher(str).matches();
  }

  public static boolean isEmail(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.EMAIL_ADDRESS_PATTERN.matcher(str).matches();
  }

  public static boolean isPhone(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.PHONE_PATTERN.matcher(str).matches();
  }

  public static boolean isEmoticon(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.EMOTICON_PATTERN.matcher(str).matches();
  }

  public static boolean isHashTag(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.HASH_TAG_PATTERN.matcher(str).matches();
  }

  public static boolean isUser(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.USER_NAME_PATTERN.matcher(str).matches();
  }

  public static boolean isRetweet(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.RETWEET_PATTERN.matcher(str).matches();
  }

  public static boolean consitsOfPunctuations(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.PUNCTUATIONS_PATTERN.matcher(str).matches();
  }

  public static boolean consitsOfUnderscores(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.UNDERSCORES_PATTERN.matcher(str).matches();
  }

  public static boolean isNumeric(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.NUMBER_PATTERN.matcher(str).matches();
  }

  public static boolean isSpecialNumeric(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.SPECIAL_NUMBER_PATTERN.matcher(str).matches();
  }

  public static boolean isSeparatedNumeric(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.SEPARATED_NUMBER_PATTERN.matcher(str).matches();
  }

  public static boolean isSlang(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.SLANG_PATTERN.matcher(str).matches();
  }

  public static boolean startsWithAlphabeticChar(String str) {
    if (str == null) {
      return false;
    }
    return RegexUtils.STARTS_WITH_ALPHABETIC_CHAR_PATTERN.matcher(str)
        .matches();
  }

  public static boolean isPunctuation(char c) {
    return c == ',' || c == '.' || c == '!' || c == '?' || c == ':' || c == ';';
  }

  public static boolean isSpecialChar(char c) {
    return c == '-' || c == '_' || c == '&' || c == '|' || c == '/'
        || c == '\\' || c == '"' || c == '\'' || c == '`' || c == '´'
        || c == '<' || c == '>' || c == '~' || c == '*' || c == '+' || c == '('
        || c == ')' || c == '[' || c == ']';
  }

  public static String trimPunctuation(String str) {
    int valueLen = str.length();
    // count starting punctuations
    int startingPunctuations = 0;
    for (int i = 0; i < valueLen; i++) {
      if (isPunctuation(str.charAt(i)) || isSpecialChar((str.charAt(i)))) {
        startingPunctuations++;
      } else {
        break;
      }
    }

    // count ending punctuations
    int endingPunctuations = 0;
    for (int i = valueLen - 1; i >= 0; i--) {
      if (isPunctuation(str.charAt(i)) || isSpecialChar((str.charAt(i)))) {
        endingPunctuations++;
      } else {
        break;
      }
    }

    // case 1) no characters were punctuation
    if ((startingPunctuations == 0) && (endingPunctuations == 0)) {
      return str;
    }

    // case 2) all characters were punctuation
    if ((startingPunctuations == valueLen) && (endingPunctuations == valueLen)) {
      return "";
    }

    // case 3) substring
    int endIndex = valueLen - endingPunctuations;
    if (startingPunctuations >= endIndex) {
      LOG.error("trimPunctuation is not possible for '" + str + "' valueLen: "
          + valueLen + " startingPunc: " + startingPunctuations
          + " endingPunc: " + endingPunctuations + " endIndex: " + endIndex);
      return str;
    } else {
      LOG.info("trimPunctuation from '" + str + "' to '"
          + str.substring(startingPunctuations, endIndex) + "'");
      return str.substring(startingPunctuations, endIndex);
    }
  }

  public static void main(String[] args) {
    // test URLs
    String[] testURLs = new String[] { "http://", "www.google.com",
        "http://www.google.com", "https://www.google.com",
        "ftp://www.google.com", "google.com" };
    for (String s : testURLs) {
      System.out.println("isURL(" + s + "): " + isURL(s));
    }

    // test userNames
    String[] userNames = new String[] { "@hugo", "@@hugo", " @hugu2", "@hugo_",
        "2@hugo_", "a@a", "@h0_", "@0abc", "@abc0" };
    for (String s : userNames) {
      System.out.println("isUser(" + s + "): " + isUser(s));
    }

    // test hashTags
    String[] hashTags = new String[] { "#hugo", "##hugo", " #hugu2", "#hugo_",
        "a#a", "#h0_", "#0abc", "#abc0" };
    for (String s : hashTags) {
      System.out.println("isHashTag(" + s + "): " + isHashTag(s));
    }

    // test retweets
    String[] retweets = new String[] { "RT", "retweet", "Retweet",
        "RT@username", "RT @ username" };
    for (String s : retweets) {
      System.out.println("isRetweet(" + s + "): " + isRetweet(s));
    }

    // test punctuations
    String[] testPunctuations = new String[] { ".a .", "..--", ".. --" };
    for (String s : testPunctuations) {
      System.out.println("consitsOfPunctuations(" + s + "): "
          + consitsOfPunctuations(s));
    }
    testPunctuations = new String[] { ".asdf.", "asdf.:--", "--asdf-!", ":-)",
        ">:-[", "\"All", "\"abc\"", "\"abc\".." };
    for (String s : testPunctuations) {
      System.out.println("trimPunctuation(" + s + "): " + trimPunctuation(s));
    }

    // test underscores
    String testUnderscores = "_ 1__ ____ _2___ __2";
    for (String s : testUnderscores.split(" ")) {
      boolean consitsOfUnderscores = consitsOfUnderscores(s);
      if (!consitsOfUnderscores) {
        System.out.println("consitsOfUnderscores(" + s + "): "
            + consitsOfUnderscores(s));
      }
    }

    // test startsWithAlphabeticChar
    String[] testStartsWithAlphabeticChar = new String[] { "Hello", "hello",
        "-hello", "0hello", "@hello" };
    for (String s : testStartsWithAlphabeticChar) {
      System.out.println("startsWithAlphabeticChar(" + s + "): "
          + startsWithAlphabeticChar(s));
    }

    // test numerics
    String[] testNumerics = new String[] { "1", "1000", "1,000", "+1", "-1",
        "+1.0", "+1.", "-.5", "000000" };
    for (String s : testNumerics) {
      System.out.println("isNumeric(" + s + "): " + isNumeric(s));
    }

    // test emoticon
    String testEmoticons = ":-) :-)) :) :D :o) :] :3 :c) :> =] 8) =) :} :^)"
        + " :-D 8-D 8D x-D xD X-D XD =-D =D =-3 =3"
        + " >:[ :-( :( :-c :c :-< :< :-[ :[ :{" + " ;( :-|| :@ >:("
        + " :'-( :'( :'-) :')" + " D:< D: D8 D; D= DX"
        + " >:O :-O :O :-o :o 8-0 O_O o-o O_o o_O o_o O-O" + " :* :^*"
        + " ;-) ;) *-) *) ;-] ;] ;D ;^) :-,"
        + " >:P :-P :P X-P x-p xp XP :-p :p =p :-b :b d:"
        + " >:\\ >:/ :-/ :-. :/ :\\ =/ =\\ :L =L :S >.<" + " :| :-|" + " :$"
        + " :-X :X :-# :#" + " O:-) 0:-3 0:3 0:-) 0:) 0;^)" + " >:) >;) >:-)"
        + " }:-) }:) 3:-) 3:)" + " o/\\o" + " |;-) |-O" + " :-J" + " :-& :&"
        + " #-)" + " %-) %)" + " <:-|" + " <3"
        + " </3 :Oc =.= o.0 ^^ Odd 0:00 3:00" + " \\m/";
    for (String s : testEmoticons.split(" ")) {
      boolean isEmoticon = isEmoticon(s);
      if (!isEmoticon) {
        System.out.println("isEmoticon( " + s + " ): " + isEmoticon(s));
      }
    }

    // test slang
    String testSlang = "t/m k/o w/my b/slisten Rt/follow S/o S/O O/U O/A w/ w/Biden w/deals w/you w/the w/her";
    for (String s : testSlang.split(" ")) {
      boolean isSlang = isSlang(s);
      if (!isSlang) {
        System.out.println("isSlang(" + s + "): " + isSlang(s));
      }
    }
  }
} public class WordNet {
  public static final int MAX_DEPTH_OF_HIERARCHY = 16;
  private static final Logger LOG = LoggerFactory.getLogger(WordNet.class);
  private static final WordNet INSTANCE = new WordNet();

  private IRAMDictionary m_dict;
  private File m_wordNetDir;
  private WordnetStemmer m_wordnetStemmer;

  private WordNet() {
    try {
      String wordNetDictPath = Configuration.getWordNetDict();
      LOG.info("WordNet Dictionary: " + wordNetDictPath);
      m_wordNetDir = new File(Configuration.TEMP_DIR_PATH + File.separator
          + "dict");
      LOG.info("WordNet Extract Location: " + m_wordNetDir.getAbsolutePath());

      // check if extract location does exist
      if (m_wordNetDir.exists()) {
        IOUtils.delete(m_wordNetDir);
      }

      // extract tar.gz file
      IOUtils.extractTarGz(wordNetDictPath, m_wordNetDir.getParent());

      m_dict = new RAMDictionary(m_wordNetDir, ILoadPolicy.NO_LOAD);
      m_dict.open();

      // load into memory
      long t = System.currentTimeMillis();
      m_dict.load(true);
      LOG.info("Loaded Wordnet into memory in "
          + (System.currentTimeMillis() - t) + " msec");

      m_wordnetStemmer = new WordnetStemmer(m_dict);

    } catch (IOException e) {
      LOG.error("IOException: " + e.getMessage());
    } catch (InterruptedException e) {
      LOG.error("InterruptedException: " + e.getMessage());
    }
  }

  public static WordNet getInstance() {
    return INSTANCE;
  }

  public void close() {
    if (m_dict != null) {
      m_dict.close();
    }
    try {
      IOUtils.delete(m_wordNetDir);
    } catch (IOException e) {
      LOG.error("IOException: " + e.getMessage());
    }
  }

  public boolean contains(String word) {
    for (POS pos : POS.values()) {
      for (String stem : m_wordnetStemmer.findStems(word, pos)) {
        IIndexWord indexWord = m_dict.getIndexWord(stem, pos);
        if (indexWord != null)
          return true;
      }
    }
    return false;
  }

  public boolean isNoun(String word) {
    return m_dict.getIndexWord(word, POS.NOUN) != null;
  }

  public boolean isAdjective(String word) {
    return m_dict.getIndexWord(word, POS.ADJECTIVE) != null;
  }

  public boolean isAdverb(String word) {
    return m_dict.getIndexWord(word, POS.ADVERB) != null;
  }

  public boolean isVerb(String word) {
    return m_dict.getIndexWord(word, POS.VERB) != null;
  }

  public synchronized POS findPOS(String word) {
    int maxCount = 0;
    POS mostLikelyPOS = null;
    for (POS pos : POS.values()) {
      // From JavaDoc: The surface form may or may not contain whitespace or
      // underscores, and may be in mixed case.
      word = word.replaceAll("\\s", "").replaceAll("_", "");

      List<String> stems = m_wordnetStemmer.findStems(word, pos);
      for (String stem : stems) {
        IIndexWord indexWord = m_dict.getIndexWord(stem, pos);
        if (indexWord != null) {
          int count = 0;
          for (IWordID wordId : indexWord.getWordIDs()) {
            IWord aWord = m_dict.getWord(wordId);
            ISenseEntry senseEntry = m_dict.getSenseEntry(aWord.getSenseKey());
            count += senseEntry.getTagCount();
          }

          if (count > maxCount) {
            maxCount = count;
            mostLikelyPOS = pos;
          }
        }
      }
    }

    return mostLikelyPOS;
  }

  public List<String> findStems(String word, POS pos) {
    return m_wordnetStemmer.findStems(word, pos);
  }

  public IIndexWord getIndexWord(String word, POS pos) {
    List<String> stems = findStems(word, pos);
    if (stems != null) {
      if (stems.size() > 1) {
        LOG.info("Be careful the word '" + word
            + "' has several lemmatized forms.");
      }

      for (String stem : stems) {
        return m_dict.getIndexWord(stem, pos); // return first stem
      }
    }
    return null;
  }

  public String getLemma(ISynset synset) {
    return synset.getWord(0).getLemma();
  }

  public String getLemma(ISynsetID synsetID) {
    return getLemma(m_dict.getSynset(synsetID));
  }

  public Set<IIndexWord> getAllIndexWords(String word) {
    Set<IIndexWord> indexWords = new HashSet<IIndexWord>();
    for (POS pos : POS.values()) {
      IIndexWord indexWord = m_dict.getIndexWord(word, pos);
      if (indexWord != null) {
        indexWords.add(indexWord);
      }
    }
    return indexWords;
  }

  public ISynset getSynset(String word, POS pos) {
    IIndexWord indexWord = m_dict.getIndexWord(word, pos);
    if (indexWord != null) {
      IWordID wordID = indexWord.getWordIDs().get(0); // use first WordID
      return m_dict.getWord(wordID).getSynset();
    }
    return null;
  }

  public Set<ISynset> getSynsets(IIndexWord indexWord) {
    Set<ISynset> synsets = new HashSet<ISynset>();
    for (IWordID wordID : indexWord.getWordIDs()) {
      synsets.add(m_dict.getSynset(wordID.getSynsetID()));
    }
    return synsets;
  }

  public Set<ISynset> getSynsets(Set<IIndexWord> indexWords) {
    Set<ISynset> synsets = new HashSet<ISynset>();
    for (IIndexWord indexWord : indexWords) {
      for (IWordID wordID : indexWord.getWordIDs()) {
        synsets.add(m_dict.getSynset(wordID.getSynsetID()));
      }
    }
    return synsets;
  }

  public List<ISynsetID> getAncestors(ISynset synset) {
    List<ISynsetID> list = new ArrayList<ISynsetID>();
    list.addAll(synset.getRelatedSynsets(Pointer.HYPERNYM));
    list.addAll(synset.getRelatedSynsets(Pointer.HYPERNYM_INSTANCE));
    return list;
  }

  public List<ISynsetID> getChildren(ISynset synset) {
    List<ISynsetID> list = new ArrayList<ISynsetID>();
    list.addAll(synset.getRelatedSynsets(Pointer.HYPONYM));
    list.addAll(synset.getRelatedSynsets(Pointer.HYPONYM_INSTANCE));
    return list;
  }

  public List<List<ISynset>> getPathsToRoot(ISynset synset) {
    List<List<ISynset>> pathsToRoot = null;
    List<ISynsetID> ancestors = getAncestors(synset);

    if (ancestors.isEmpty()) {
      pathsToRoot = new ArrayList<List<ISynset>>();
      List<ISynset> pathToRoot = new ArrayList<ISynset>();
      pathToRoot.add(synset);
      pathsToRoot.add(pathToRoot);

    } else if (ancestors.size() == 1) {
      pathsToRoot = getPathsToRoot(m_dict.getSynset(ancestors.get(0)));

      for (List<ISynset> pathToRoot : pathsToRoot) {
        pathToRoot.add(0, synset);
      }

    } else {
      pathsToRoot = new ArrayList<List<ISynset>>();
      for (ISynsetID ancestor : ancestors) {
        ISynset ancestorSynset = m_dict.getSynset(ancestor);
        List<List<ISynset>> pathsToRootLocal = getPathsToRoot(ancestorSynset);

        for (List<ISynset> pathToRoot : pathsToRootLocal) {
          pathToRoot.add(0, synset);
        }

        pathsToRoot.addAll(pathsToRootLocal);
      }
    }

    return pathsToRoot;
  }

  private ISynset findClosestCommonParent(List<ISynset> pathToRoot1,
      List<ISynset> pathToRoot2) {
    int i = 0;
    int j = 0;

    if (pathToRoot1.size() > pathToRoot2.size()) {
      i = pathToRoot1.size() - pathToRoot2.size();
      j = 0;

    } else if (pathToRoot1.size() < pathToRoot2.size()) {
      i = 0;
      j = pathToRoot2.size() - pathToRoot1.size();
    }

    do {
      ISynset synset1 = pathToRoot1.get(i++);
      ISynset synset2 = pathToRoot2.get(j++);

      if (synset1.equals(synset2)) {
        return synset1;
      }

    } while (i < pathToRoot1.size());

    return null;
  }

  public ISynset findClosestCommonParent(ISynset synset1, ISynset synset2) {
    if ((synset1 == null) || (synset2 == null)) {
      return null;
    }
    if (synset1.equals(synset2)) {
      return synset1;
    }

    List<List<ISynset>> pathsToRoot1 = getPathsToRoot(synset1);
    List<List<ISynset>> pathsToRoot2 = getPathsToRoot(synset2);
    ISynset resultSynset = null;
    int i = 0;

    for (List<ISynset> pathToRoot1 : pathsToRoot1) {
      for (List<ISynset> pathToRoot2 : pathsToRoot2) {

        ISynset synset = findClosestCommonParent(pathToRoot1, pathToRoot2);

        int j = pathToRoot1.size() - (pathToRoot1.indexOf(synset) + 1);
        if (j >= i) {
          i = j;
          resultSynset = synset;
        }
      }
    }

    return resultSynset;
  }

  /**
   * maxDepth
   * 
   * @param synset
   * @return The length of the longest hypernym path from this synset to the
   *         root.
   */
  public int maxDepth(ISynset synset) {
    if (synset == null) {
      return 0;
    }

    List<ISynsetID> ancestors = getAncestors(synset);
    if (ancestors.isEmpty()) {
      return 0;
    }

    int i = 0;
    for (ISynsetID ancestor : ancestors) {
      ISynset ancestorSynset = m_dict.getSynset(ancestor);
      int j = maxDepth(ancestorSynset);
      i = (i > j) ? i : j;
    }

    return i + 1;
  }

  /**
   * Shortest Path Distance
   * 
   * Returns the distance of the shortest path linking the two synsets (if one
   * exists).
   * 
   * For each synset, all the ancestor nodes and their distances are recorded
   * and compared. The ancestor node common to both synsets that can be reached
   * with the minimum number of traversals is used. If no ancestor nodes are
   * common, null is returned. If a node is compared with itself 0 is returned.
   * 
   * @param synset1
   * @param synset2
   * @return The number of edges in the shortest path connecting the two nodes,
   *         or null if no path exists.
   */
  public Integer shortestPathDistance(ISynset synset1, ISynset synset2) {
    Integer distance = null;
    if (synset1.equals(synset2)) {
      return 0;
    }

    ISynset ccp = findClosestCommonParent(synset1, synset2);
    if (ccp != null) {
      distance = maxDepth(synset1) + maxDepth(synset2) - 2 * maxDepth(ccp);

      // Debug
      String w1 = synset1.getWords().get(0).getLemma();
      String w2 = synset2.getWords().get(0).getLemma();
      String w3 = ccp.getWords().get(0).getLemma();
      System.out.println("maxDepth(" + w1 + "): " + maxDepth(synset1));
      System.out.println("maxDepth(" + w2 + "): " + maxDepth(synset2));
      System.out.println("maxDepth(" + w3 + "): " + maxDepth(ccp));
      System.out.println("distance(" + w1 + "," + w2 + "): " + distance);
    }
    return distance;
  }

  /**
   * Path Distance Similarity
   * 
   * Return a score denoting how similar two word senses are, based on the
   * shortest path that connects the senses in the is-a (hypernym/hypnoym)
   * taxonomy.
   * 
   * The score is in the range 0 to 1, except in those cases where a path cannot
   * be found (will only be true for verbs as there are many distinct verb
   * taxonomies), in which case null is returned.
   * 
   * A score of 1 represents identity i.e. comparing a sense with itself will
   * return 1.
   * 
   * @param synset1
   * @param synset2
   * @return A score denoting the similarity of the two ``Synset`` objects,
   *         normally between 0 and 1. null is returned if no connecting path
   *         could be found. 1 is returned if a ``Synset`` is compared with
   *         itself.
   */
  public Double pathSimilarity(ISynset synset1, ISynset synset2) {
    Integer distance = shortestPathDistance(synset1, synset2);
    Double pathSimilarity = null;
    if (distance != null) {
      if (distance < 0) {
        throw new IllegalArgumentException("Distance value is negative!");
      }
      pathSimilarity = 1 / ((double) distance + 1);

      // Debug
      String w1 = synset1.getWords().get(0).getLemma();
      String w2 = synset2.getWords().get(0).getLemma();
      System.out.println("maxDepth(" + w1 + "): " + maxDepth(synset1));
      System.out.println("maxDepth(" + w2 + "): " + maxDepth(synset2));
      System.out.println("distance: " + distance);
      System.out.println("pathSimilarity(" + w1 + "," + w2 + "): "
          + pathSimilarity);
    } else {
      // TODO simulate_root=True
    }
    return pathSimilarity;
  }

  /**
   * Leacock Chodorow Similarity
   * 
   * Return a score denoting how similar two word senses are, based on the
   * shortest path that connects the senses and the maximum depth of the
   * taxonomy in which the senses occur. The relationship is given as -log(p/2d)
   * where p is the shortest path length and d is the taxonomy depth.
   * 
   * lch(c1,c2) = - log(minPathLength(c1,c2) / 2 * depth of the hierarchy)
   * lch(c1,c2) = - log(minPL(c1,c2) / 2 * depth) = log(2*depth / minPL(c1,c2))
   * 
   * minPathLength is measured in nodes, i.e. the distance of a node to itself
   * is 0! This would cause a logarithm error (or a division by zero)). Thus we
   * changed the behaviour in order to return a distance of 1, if the nodes are
   * equal or neighbors.
   * 
   * double relatedness = Math.log( (2*depthOfHierarchy) / (pathLength + 1) );
   * 
   * @param synset1
   * @param synset2
   * @return A score denoting the similarity of the two ``Synset`` objects,
   *         normally greater than 0. null is returned if no connecting path
   *         could be found. If a ``Synset`` is compared with itself, the
   *         maximum score is returned, which varies depending on the taxonomy
   *         depth.
   */
  public Double lchSimilarity(ISynset synset1, ISynset synset2) {
    Integer distance = shortestPathDistance(synset1, synset2);
    Double lchSimilarity = null;
    if (distance != null) {
      if (distance < 0) {
        throw new IllegalArgumentException("Distance value is negative!");
      }
      if (distance == 0) {
        distance = 1;
      }

      lchSimilarity = Math.log((2 * MAX_DEPTH_OF_HIERARCHY)
          / ((double) distance));

      // Debug
      String w1 = synset1.getWords().get(0).getLemma();
      String w2 = synset2.getWords().get(0).getLemma();
      System.out.println("lchSimilarity(" + w1 + "," + w2 + "): "
          + lchSimilarity);
    } else {
      // TODO simulate_root=True
    }
    return lchSimilarity;
  }

  /**
   * Computes a score denoting how similar two word senses are, based on the
   * shortest path that connects the senses in the is-a (hypernym/hypnoym)
   * taxonomy.
   *
   * @param indexWord1
   * @param indexWord2
   * @return Returns a similarity score is in the range 0 to 1. A score of 1
   *         represents identity i.e. comparing a sense with itself will return
   *         1.
   */
  public double similarity(String word1String, POS word1POS,
      String word2String, POS word2POS) {
    IIndexWord indexWord1 = getIndexWord(word1String, word1POS);
    Set<ISynset> synsets1 = getSynsets(indexWord1);

    IIndexWord indexWord2 = getIndexWord(word2String, word2POS);
    Set<ISynset> synsets2 = getSynsets(indexWord2);

    double maxSim = 0;
    for (ISynset synset1 : synsets1) {
      for (ISynset synset2 : synsets2) {
        double sim = pathSimilarity(synset1, synset2);
        if ((sim > 0) && (sim > maxSim)) {
          maxSim = sim;
        }
      }
    }
    return maxSim;
  }

  public ISynset disambiguateWordSenses(List<TaggedWord> sentence, String word,
      POS pos) {
    IIndexWord indexWord = getIndexWord(word, pos);
    Set<ISynset> synsets = getSynsets(indexWord);

    ISynset resultSynset = null;
    double bestScore = 0;
    for (ISynset synset : synsets) {
      for (TaggedWord taggedWord : sentence) {
        double score = 0;
        IIndexWord indexWordLocal = getIndexWord(taggedWord.word(),
            POSTag.convertPTB(taggedWord.tag()));
        Set<ISynset> synsetsLocal = getSynsets(indexWordLocal);
        for (ISynset synsetLocal : synsetsLocal) {
          double sim = shortestPathDistance(synsetLocal, synset);
          if (sim > 0) {
            score += sim;
          }
        }
        if (score > bestScore) {
          bestScore = score;
          resultSynset = synset;
        }
      }
    }
    return resultSynset;
  }

  /**
   * testWordNet implementation
   */
  public void testWordNet() {
    // Performance test - treking across Wordnet
    trek();

    // ************************************************************************
    // Misc Tests
    // ************************************************************************
    System.out.println("\nwn.synsets('dog')");
    Set<IIndexWord> indexWords = getAllIndexWords("dog");
    for (IIndexWord indexWord : indexWords) {
      System.out.println("indexWords: ");
      for (IWordID wordID : indexWord.getWordIDs()) {
        IWord word = m_dict.getWord(wordID);
        System.out.println(word.getSynset());
      }
    }

    System.out.println("\nwn.synsets('dog', pos=wn.NOUN)");
    IIndexWord indexWord = m_dict.getIndexWord("dog", POS.NOUN);
    IWordID wordID = indexWord.getWordIDs().get(0);
    IWord word = m_dict.getWord(wordID);
    System.out.println("Id = " + wordID);
    System.out.println("Lemma = " + word.getLemma());
    System.out.println("Gloss = " + word.getSynset().getGloss());

    System.out
        .println("\ndog = wn.synset('dog.n.01') dog.hypernyms - ancestors");
    List<ISynsetID> ancestors = getAncestors(getSynset("dog", POS.NOUN));
    for (ISynsetID ancestor : ancestors) {
      ISynset synset = m_dict.getSynset(ancestor);
      System.out.println(synset);
    }

    System.out.println("\ndog = wn.synset('dog.n.01') dog.hyponyms - children");
    List<ISynsetID> children = getChildren(getSynset("dog", POS.NOUN));
    for (ISynsetID child : children) {
      ISynset synset = m_dict.getSynset(child);
      System.out.println(synset);
    }

    // ************************************************************************
    // Test similarities
    // ************************************************************************
    Double pathSimilarity = null;
    Double lchSimilarity = null;

    System.out.println("\nwn.path_similarity(dog, dog) = 1.0");
    pathSimilarity = pathSimilarity(getSynset("dog", POS.NOUN),
        getSynset("dog", POS.NOUN));
    System.out.println("pathSimilarity: " + pathSimilarity);

    System.out.println("\nwn.lch_similarity(dog, dog) = 3.63758");
    lchSimilarity = lchSimilarity(getSynset("dog", POS.NOUN),
        getSynset("dog", POS.NOUN));
    System.out.println("lchSimilarity: " + lchSimilarity);

    System.out.println("\nwn.path_similarity(dog, cat) = 0.2");
    pathSimilarity = pathSimilarity(getSynset("dog", POS.NOUN),
        getSynset("cat", POS.NOUN));
    System.out.println("pathSimilarity: " + pathSimilarity);

    System.out.println("\nwn.lch_similarity(dog, cat) = 2.02814");
    lchSimilarity = lchSimilarity(getSynset("dog", POS.NOUN),
        getSynset("cat", POS.NOUN));
    System.out.println("lchSimilarity: " + lchSimilarity);

    System.out.println("\nwn.path_similarity(hit, slap) = 0.14285");
    pathSimilarity = pathSimilarity(getSynset("hit", POS.VERB),
        getSynset("slap", POS.VERB));
    System.out.println("pathSimilarity: " + pathSimilarity);

    System.out.println("\nwn.lch_similarity(hit, slap) = 1.31218");
    lchSimilarity = lchSimilarity(getSynset("hit", POS.VERB),
        getSynset("slap", POS.VERB));
    System.out.println("lchSimilarity: " + lchSimilarity);

    // ************************************************************************
    // Test stemming
    // ************************************************************************
    System.out.println("\nStemming test...");
    String[] stemmingWords = { "cats", "running", "ran", "cactus", "cactuses",
        "community", "communities", "going" };
    POS[] stemmingPOS = { POS.NOUN, POS.VERB, POS.VERB, POS.NOUN, POS.NOUN,
        POS.NOUN, POS.NOUN, POS.VERB };
    String[] stemmingResults = { "cat", "run", "run", "cactus", "cactus",
        "community", "community", "go" };

    for (int i = 0; i < stemmingWords.length; i++) {
      List<String> stemResults = m_wordnetStemmer.findStems(stemmingWords[i],
          stemmingPOS[i]);
      for (String stemResult : stemResults) {
        System.out.println("stems of \"" + stemmingWords[i] + "\": "
            + stemResult);
        // verify result
        if (!stemResult.equals(stemmingResults[i])) {
          System.err.println("Wrong stemming result of \"" + stemmingWords[i]
              + "\" result: \"" + stemResult + "\" expected: \""
              + stemmingResults[i] + "\"");
        }
      }
    }

  }

  /**
   * Treking across Wordnet for performance measurements
   * 
   */
  private void trek() {
    int tickNext = 0;
    int tickSize = 20000;
    int seen = 0;
    System.out.print("Treking across Wordnet");
    long t = System.currentTimeMillis();
    for (POS pos : POS.values()) {
      for (Iterator<IIndexWord> i = m_dict.getIndexWordIterator(pos); i
          .hasNext();) {
        for (IWordID wid : i.next().getWordIDs()) {
          seen += m_dict.getWord(wid).getSynset().getWords().size();
          if (seen > tickNext) {
            System.out.print(".");
            tickNext = seen + tickSize;
          }
        }
      }
    }
    System.out.printf("done (%1d msec)\n", System.currentTimeMillis() - t);
    System.out.println("In my trek I saw " + seen + " words");
  }

  public static void main(String[] args) {
    WordNet wordNet = WordNet.getInstance();
    wordNet.testWordNet();
    wordNet.close();
  }
}
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import at.illecker.storm.commons.Configuration;
import at.illecker.storm.commons.Dataset;
import at.illecker.storm.commons.dict.FirstNames;
import at.illecker.storm.commons.dict.Interjections;
import at.illecker.storm.commons.dict.NameEntities;
import at.illecker.storm.commons.dict.SlangCorrection;
import at.illecker.storm.commons.tokenizer.Tokenizer;
import at.illecker.storm.commons.tweet.Tweet;
import at.illecker.storm.commons.util.RegexUtils;
import at.illecker.storm.commons.util.StringUtils;
import at.illecker.storm.commons.wordnet.WordNet;
import edu.stanford.nlp.ling.TaggedWord;
        LOG.info("NameEntity labelled for " + token);
      }
      preTaggedToken.setTag("NNP");
    } else if ((m_interjections.isInterjection(token))
        || (StringUtils.isEmoticon(token))) {
      if (LOGGING) {
        LOG.info("Interjection or Emoticon labelled for " + token);
      }
      preTaggedToken.setTag("UH");
    }
    return preTaggedToken;
  }

  private String removeRepeatingChars(String value) {
    // if there are three repeating equal chars
    // then remove one char until the word is found in the vocabulary
    // else if the word is not found reduce the repeating chars to one

    // collect matches for sub-token search
    List<int[]> matches = new ArrayList<int[]>();

    Matcher m = RegexUtils.THREE_OR_MORE_REPEATING_CHARS_PATTERN.matcher(value);
    while (m.find()) {
      int start = m.start();
      int end = m.end();
      // String c = m.group(1);
      // LOG.info("token: '" + value + "' match at start: " + start + " end: "
      // + end);

      // check if token is not in the vocabulary
      if (!m_wordnet.contains(value)) {
        // collect matches for subtoken check
        matches.add(new int[] { start, end });

        StringBuilder sb = new StringBuilder(value);
        for (int i = 0; i < end - start - 1; i++) {
          sb.deleteCharAt(start); // delete repeating char

          // LOG.info("check token: '" + sb.toString() + "'");
          // check if token is in the vocabulary
          if (m_wordnet.contains(sb.toString())) {
            if (LOGGING) {
              LOG.info("removeRepeatingChars from token '" + value + "' to '"
                  + sb + "'");
            }
            return sb.toString();
          }

          // if the token is not in the vocabulary check all combinations
          // of prior matches
          // TODO really necessary?
          for (int j = 0; j < matches.size(); j++) {
            int startSub = matches.get(j)[0];
            int endSub = matches.get(j)[1];
            if (startSub != start) {
              StringBuilder subSb = new StringBuilder(sb);
              for (int k = 0; k < endSub - startSub - 1; k++) {
                subSb.deleteCharAt(startSub);

                // LOG.info("check subtoken: '" + subSb.toString() + "'");
                if (m_wordnet.contains(subSb.toString())) {
                  if (LOGGING) {
                    LOG.info("removeRepeatingChars from '" + value + "' to '"
                        + subSb + "'");
                  }
                  return subSb.toString();
                }
              }
            }
          }
        }
      }
    }

    // no match have been found
    // reduce all repeating chars
    if (!matches.isEmpty()) {
      String reducedToken = m.replaceAll("$1");
      if (LOGGING) {
        LOG.info("removeRepeatingChars(not found in dict) from '" + value
            + "' to '" + reducedToken + "'");
      }
      value = reducedToken;
    }
    return value;
  }

  public List<List<String>> preprocessTweets(List<List<String>> tweets) {
    List<List<String>> preprocessedTweets = new ArrayList<List<String>>();
    for (List<String> tweet : tweets) {
      preprocessedTweets.add(preprocess(tweet));
    }
    return preprocessedTweets;
  }

  public List<List<TaggedWord>> preprocessAndTagTweets(List<List<String>> tweets) {
    List<List<TaggedWord>> preprocessedTweets = new ArrayList<List<TaggedWord>>();
    for (List<String> tweet : tweets) {
      preprocessedTweets.add(preprocessAndTag(tweet));
    }
    return preprocessedTweets;
  }

  public static void main(String[] args) {
    Preprocessor preprocessor = Preprocessor.getInstance();
    List<Tweet> tweets = null;
    boolean extendedTest = true;
    boolean debugOutput = false;

    // load tweets
    if (extendedTest) {
      // Twitter crawler
      // List<Status> extendedTweets = Configuration
      // .getDataSetUibkCrawlerTest("en");
      // tweets = new ArrayList<Tweet>();
      // for (Status tweet : extendedTweets) {
      // tweets.add(new Tweet(tweet.getId(), tweet.getText(), 0));
      // }

      // SemEval2013
      Dataset dataset = Configuration.getDataSetSemEval2013();