public class Configuration {
  private static final Logger LOG = LoggerFactory
      .getLogger(Configuration.class);

  public static final boolean RUNNING_WITHIN_JAR = Configuration.class
      .getResource("Configuration.class").toString().startsWith("jar:");

  public static final String WORKING_DIR_PATH = (RUNNING_WITHIN_JAR) ? ""
      : System.getProperty("user.dir") + File.separator;

  public static final String TEMP_DIR_PATH = System
      .getProperty("java.io.tmpdir");

  public static final String GLOBAL_RESOURCES_DATASETS_SEMEVAL = "global.resources.datasets.semeval";
  public static final String GLOBAL_RESOURCES_DATASETS_CRAWLER = "global.resources.datasets.crawler";

  public static final String GLOBAL_RESOURCES_DICT = "global.resources.dict";
  public static final String GLOBAL_RESOURCES_DICT_SENTIMENT = "global.resources.dict.sentiment";
  public static final String GLOBAL_RESOURCES_DICT_SENTIMENT_SENTIWORDNET_PATH = "global.resources.dict.sentiment.sentiwordnet.path";
  public static final String GLOBAL_RESOURCES_DICT_SLANG = "global.resources.dict.slang";
  public static final String GLOBAL_RESOURCES_DICT_WORDNET_PATH = "global.resources.dict.wordnet.path";

  public static final Map CONFIG = readConfig();

  @SuppressWarnings("rawtypes")
  public static Map readConfig() {
    Map conf = readConfigFile(WORKING_DIR_PATH + "conf/defaults.yaml", true);
    // read custom config
    LOG.info("Try to load user-specific config...");
    Map customConfig = readConfigFile(WORKING_DIR_PATH
        + "conf/configuration.yaml", false);
    if (customConfig != null) {
      conf.putAll(customConfig);
    } else if (RUNNING_WITHIN_JAR) {
      customConfig = readConfigFile("../conf/configuration.yaml", false);
      if (customConfig != null) {
        conf.putAll(customConfig);
      }
    }
    return conf;
  }

  @SuppressWarnings("rawtypes")
  public static Map readConfigFile(String file, boolean mustExist) {
    Yaml yaml = new Yaml(new SafeConstructor());
    Map ret = null;
    InputStream input = IOUtils.getInputStream(file);
    if (input != null) {
      ret = (Map) yaml.load(new InputStreamReader(input));
      LOG.info("Loaded " + file);
      try {
        input.close();
      } catch (IOException e) {
        LOG.error("IOException: " + e.getMessage());
      }
    } else if (mustExist) {
      LOG.error("Config file " + file + " was not found!");
    }
    if ((ret == null) && (mustExist)) {
      throw new RuntimeException("Config file " + file + " was not found!");
    }
    return ret;
  }

  public static <K, V> V get(K key) {
    return get(key, null);
  }

  public static <K, V> V get(K key, V defaultValue) {
    return get((Map<K, V>) CONFIG, key, defaultValue);
  }

  public static <K, V> V get(Map<K, V> map, K key, V defaultValue) {
    V value = map.get(key);
    if (value == null) {
      value = defaultValue;
    }
    return value;
  }

  public static Dataset getDataSetSemEval2013() {
    return Dataset.readFromYaml((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_SEMEVAL)).get("2013"));
  }

  public static Dataset getDataSetSemEval2013Mixed() {
    return Dataset.readFromYaml((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_SEMEVAL)).get("2013Mixed"));
  }

  public static Dataset getDataSetSemEval2014() {
    return Dataset.readFromYaml((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_SEMEVAL)).get("2014"));
  }

  public static List<Status> getDataSetCrawler() {
    return getDataSetCrawler((String) ((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_CRAWLER)).get("Crawler"))
        .get("language.filter"));
  }

  public static List<Status> getDataSetCrawler(String filterLanguage) {
    return JsonUtils.readTweetsDirectory((String) ((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_CRAWLER)).get("Crawler")).get("path"),
        filterLanguage);
  }

  public static List<Status> getDataSetCrawlerTest() {
    return getDataSetCrawlerTest((String) ((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_CRAWLER)).get("CrawlerTest"))
        .get("language.filter"));
  }

  public static List<Status> getDataSetCrawlerTest(String filterLanguage) {
    return JsonUtils.readTweetsDirectory((String) ((Map) ((Map) CONFIG
        .get(GLOBAL_RESOURCES_DATASETS_CRAWLER)).get("CrawlerTest"))
        .get("path"), filterLanguage);
  }

  public static List<String> getNameEntities() {
    return (List<String>) ((Map) CONFIG.get(GLOBAL_RESOURCES_DICT))
        .get("NameEntities");
  }

  public static List<String> getFirstNames() {
    return (List<String>) ((Map) CONFIG.get(GLOBAL_RESOURCES_DICT))
        .get("FirstNames");
  }

  public static List<Map> getEmoticons() {
    return (List<Map>) ((Map) CONFIG.get(GLOBAL_RESOURCES_DICT))
        .get("Emoticons");
  }

  public static List<Map> getInterjections() {
    return (List<Map>) ((Map) CONFIG.get(GLOBAL_RESOURCES_DICT))
        .get("Interjections");
  }

  public static List<String> getStopWords() {
    return (List<String>) ((Map) CONFIG.get(GLOBAL_RESOURCES_DICT))
        .get("StopWords");
  }

  public static List<Map> getSentimentWordlists() {
    return (List<Map>) CONFIG.get(GLOBAL_RESOURCES_DICT_SENTIMENT);
  }

  public static String getSentiWordNetDict() {
    return (String) CONFIG
        .get(GLOBAL_RESOURCES_DICT_SENTIMENT_SENTIWORDNET_PATH);
  }

  public static List<Map> getSlangWordlists() {
    return (List<Map>) CONFIG.get(GLOBAL_RESOURCES_DICT_SLANG);
  }

  public static String getWordNetDict() {
    return (String) CONFIG.get(GLOBAL_RESOURCES_DICT_WORDNET_PATH);
  }

} public class Dataset implements Serializable {
  private static final long serialVersionUID = -1822543611053727897L;
  public static final String SERIAL_EXTENSION = ".ser";
  private static final Logger LOG = LoggerFactory.getLogger(Dataset.class);

  private String m_datasetPath;
  private String m_trainDataFile;
  private String m_devDataFile;
  private String m_testDataFile;

  private String m_delimiter;
  private int m_idIndex;
  private int m_labelIndex;
  private int m_textIndex;

  private List<String> m_positiveLabels;
  private List<String> m_negativeLabels;
  private List<String> m_neutralLabels;

  private int m_positiveValue;
  private int m_negativeValue;
  private int m_neutralValue;

  private svm_parameter m_svmParam;

  private List<Tweet> m_trainTweets = null;
  private List<Tweet> m_devTweets = null;
  private List<Tweet> m_testTweets = null;

  public Dataset(String datasetPath, String trainDataFile, String devDataFile,
      String testDataFile, String delimiter, int idIndex, int labelIndex,
      int textIndex, List<String> positiveLabels, List<String> negativeLabels,
      List<String> neutralLabels, int positiveValue, int negativeValue,
      int neutralValue, svm_parameter svmParam) {
    this.m_datasetPath = datasetPath;
    this.m_trainDataFile = trainDataFile;
    this.m_devDataFile = devDataFile;
    this.m_testDataFile = testDataFile;

    this.m_delimiter = delimiter;

    this.m_idIndex = idIndex;
    this.m_labelIndex = labelIndex;
    this.m_textIndex = textIndex;

    this.m_positiveLabels = positiveLabels;
    this.m_negativeLabels = negativeLabels;
    this.m_neutralLabels = neutralLabels;

    this.m_positiveValue = positiveValue;
    this.m_negativeValue = negativeValue;
    this.m_neutralValue = neutralValue;

    this.m_svmParam = svmParam;
  }

  public String getDatasetPath() {
    return m_datasetPath;
  }

  public String getTrainDataFile() {
    return (m_trainDataFile != null) ? m_datasetPath + File.separator
        + m_trainDataFile : null;
  }

  public String getTrainDataSerializationFile() {
    return (m_trainDataFile != null) ? m_datasetPath + File.separator
        + m_trainDataFile + SERIAL_EXTENSION : null;
  }

  public String getDevDataFile() {
    return (m_devDataFile != null) ? m_datasetPath + File.separator
        + m_devDataFile : null;
  }

  public String getDevDataSerializationFile() {
    return (m_devDataFile != null) ? m_datasetPath + File.separator
        + m_devDataFile + SERIAL_EXTENSION : null;
  }

  public String getTestDataFile() {
    return (m_testDataFile != null) ? m_datasetPath + File.separator
        + m_testDataFile : null;
  }

  public String getTestDataSerializationFile() {
    return (m_testDataFile != null) ? m_datasetPath + File.separator
        + m_testDataFile + SERIAL_EXTENSION : null;
  }

  public String getDelimiter() {
    return m_delimiter;
  }

  public int getIdIndex() {
    return m_idIndex;
  }

  public int getLabelIndex() {
    return m_labelIndex;
  }

  public int getTextIndex() {
    return m_textIndex;
  }

  public List<String> getNegativeLabels() {
    return m_negativeLabels;
  }

  public List<String> getNeutralLabels() {
    return m_neutralLabels;
  }

  public List<String> getPositiveLabels() {
    return m_positiveLabels;
  }

  public int getNegativeValue() {
    return m_negativeValue;
  }

  public int getNeutralValue() {
    return m_neutralValue;
  }

  public int getPositiveValue() {
    return m_positiveValue;
  }

  public svm_parameter getSVMParam() {
    return m_svmParam;
  }

  public List<Tweet> getTrainTweets(boolean includeDevTweets) {
    if ((m_trainTweets == null) && (this.getTrainDataFile() != null)) {
      m_trainTweets = FileUtils.readTweets(this.getTrainDataFile(), this);
    }

    // optional include dev tweets in training dataset
    if (includeDevTweets) {
      List<Tweet> combinedList = new ArrayList<Tweet>(m_trainTweets);
      combinedList.addAll(getDevTweets());
      return combinedList;
    } else {
      return m_trainTweets;
    }
  }

  public List<Tweet> getDevTweets() {
    if ((m_devTweets == null) && (this.getDevDataFile() != null)) {
      m_devTweets = FileUtils.readTweets(this.getDevDataFile(), this);
    }
    return m_devTweets;
  }

  public List<Tweet> getTestTweets() {
    if ((m_testTweets == null) && (this.getTestDataFile() != null)) {
      m_testTweets = FileUtils.readTweets(this.getTestDataFile(), this);
    }
    return m_testTweets;
  }

  public void printDatasetStats() {
    LOG.info("Dataset: " + getDatasetPath());

    LOG.info("Train Dataset: " + getTrainDataFile());
    printTweetStats(getTrainTweets(false));

    LOG.info("Dev Dataset: " + getDevDataFile());
    printTweetStats(getDevTweets());

    LOG.info("Test Dataset: " + getTestDataFile());
    printTweetStats(getTestTweets());
  }

  public static void printTweetStats(List<Tweet> tweets) {
    if (tweets != null) {
      Map<Integer, Integer> counts = new TreeMap<Integer, Integer>();
      for (Tweet tweet : tweets) {
        int key = tweet.getScore().intValue();
        Integer count = counts.get(key);
        counts.put(key, ((count != null) ? count + 1 : 1));
      }

      int total = 0;
      int max = 0;
      for (Map.Entry<Integer, Integer> entry : counts.entrySet()) {
        LOG.info("Class: \t" + entry.getKey() + "\t" + entry.getValue());
        total += entry.getValue();
        if (entry.getValue() > max) {
          max = entry.getValue();
        }
      }
      LOG.info("Total: " + total);

      LOG.info("Optimal Class Weights: ");
      for (Map.Entry<Integer, Integer> entry : counts.entrySet()) {
        LOG.info("Class: \t" + entry.getKey() + "\t"
            + (max / (double) entry.getValue()));
      }
    }
  }

  @Override
  public String toString() {
    return "Dataset [datasetPath=" + m_datasetPath + ", trainDataFile="
        + m_trainDataFile + ", devDataFile=" + m_devDataFile
        + ", testDataFile=" + m_testDataFile + ", delimiter=" + m_delimiter
        + ", idIndex=" + m_idIndex + ", textIndex=" + m_textIndex
        + ", labelIndex=" + m_labelIndex + ", positiveLabels="
        + m_positiveLabels + ", negativeLabel=" + m_negativeLabels
        + ", neutralLabel=" + m_neutralLabels + ", positiveValue="
        + m_positiveValue + ", negativeValue=" + m_negativeValue
        + ", neutralValue=" + m_neutralValue + "]";
  }

  @SuppressWarnings({ "rawtypes", "unchecked" })
  public static Dataset readFromYaml(Map dataset) {
    svm_parameter svmParam = SVM.getDefaultParameter();

    if (dataset.get("svm.kernel") != null) {
      svmParam.kernel_type = (Integer) dataset.get("svm.kernel");
    }

    if (dataset.get("svm.c") != null) {
      svmParam.C = (Double) dataset.get("svm.c");
    }

    if (dataset.get("svm.gamma") != null) {
      svmParam.gamma = (Double) dataset.get("svm.gamma");
    }

    if (dataset.get("svm.class.weights") != null) {
      Map<Integer, Double> classWeights = (Map<Integer, Double>) dataset
          .get("svm.class.weights");

      svmParam.nr_weight = classWeights.size();
      svmParam.weight_label = new int[svmParam.nr_weight];
      svmParam.weight = new double[svmParam.nr_weight];

      for (Map.Entry<Integer, Double> entry : classWeights.entrySet()) {
        svmParam.weight_label[entry.getKey()] = entry.getKey();
        svmParam.weight[entry.getKey()] = entry.getValue();
      }
    }

    return new Dataset((String) dataset.get("path"),
        (String) dataset.get("train.file"), (String) dataset.get("dev.file"),
        (String) dataset.get("test.file"), (String) dataset.get("delimiter"),
        (Integer) dataset.get("tweetId.index"),
        (Integer) dataset.get("label.index"),
        (Integer) dataset.get("text.index"),
        (List<String>) dataset.get("positive.labels"),
        (List<String>) dataset.get("negative.labels"),
        (List<String>) dataset.get("neutral.labels"),
        (Integer) dataset.get("positive.class.value"),
        (Integer) dataset.get("negative.class.value"),
        (Integer) dataset.get("neutral.class.value"), svmParam);
  }

  public static void main(String[] args) {
    Dataset dataSet2013Mixed = Configuration.getDataSetSemEval2013Mixed();
    LOG.info("Dataset: SemEval2013 mixed");
    LOG.info(dataSet2013Mixed.toString());
    dataSet2013Mixed.printDatasetStats();

    Dataset dataSet2013 = Configuration.getDataSetSemEval2013();
    LOG.info("Dataset: SemEval2013");
    dataSet2013.printDatasetStats();

    // Dataset dataSet2014 = Configuration.getDataSetSemEval2014();
    // LOG.info("Dataset: SemEval2014");
    // dataSet2014.printDatasetStats();
  }

} public class ArkPOSTagger {
  private static final Logger LOG = LoggerFactory.getLogger(ArkPOSTagger.class);
  private static final boolean LOGGING = Configuration.get(
      "commons.postagger.logging", false);
  private static final ArkPOSTagger INSTANCE = new ArkPOSTagger();
  String m_taggingModel;
  private Model m_model;
  private FeatureExtractor m_featureExtractor;

  private ArkPOSTagger() {
    // Load ARK POS Tagger
    try {
      m_taggingModel = Configuration
          .get("global.resources.postagger.ark.model.path");
      LOG.info("Load ARK POS Tagger with model: " + m_taggingModel);
      // TODO absolute path needed for resource
      if ((Configuration.RUNNING_WITHIN_JAR)
          && (!m_taggingModel.startsWith("/"))) {
        m_taggingModel = "/" + m_taggingModel;
      }
      m_model = Model.loadModelFromText(m_taggingModel);
      m_featureExtractor = new FeatureExtractor(m_model, false);
    } catch (IOException e) {
      LOG.error("IOException: " + e.getMessage());
    }
  }

  public static ArkPOSTagger getInstance() {
    return INSTANCE;
  }

  public List<List<TaggedToken>> tagTweets(List<List<String>> tweets) {
    List<List<TaggedToken>> taggedTweets = new ArrayList<List<TaggedToken>>();
    for (List<String> tweet : tweets) {
      taggedTweets.add(tag(tweet));
    }
    return taggedTweets;
  }

  public List<TaggedToken> tag(List<String> tokens) {
    Sentence sentence = new Sentence();
    sentence.tokens = tokens;
    ModelSentence ms = new ModelSentence(sentence.T());
    m_featureExtractor.computeFeatures(sentence, ms);
    m_model.greedyDecode(ms, false);

    List<TaggedToken> taggedTokens = new ArrayList<TaggedToken>();
    for (int t = 0; t < sentence.T(); t++) {
      TaggedToken tt = new TaggedToken(tokens.get(t),
          m_model.labelVocab.name(ms.labels[t]));
      taggedTokens.add(tt);
    }
    return taggedTokens;
  }

  public void serializeModel() {
    SerializationUtils.serialize(m_model, m_taggingModel + "_model.ser");
  }

  public void serializeFeatureExtractor() {
    SerializationUtils.serialize(m_featureExtractor, m_taggingModel
        + "_featureExtractor.ser");
  }

  public static void main(String[] args) {
    boolean extendedTest = false;
    boolean useSerialization = true;

    // load tweets
    List<Tweet> tweets = null;
    if (extendedTest) {
      // Twitter crawler
      // List<Status> extendedTweets = Configuration
      // .getDataSetUibkCrawlerTest("en");
      // tweets = new ArrayList<Tweet>();
      // for (Status tweet : extendedTweets) {
      // tweets.add(new Tweet(tweet.getId(), tweet.getText(), 0));
      // }

      // SemEval2013
      Dataset dataset = Configuration.getDataSetSemEval2013();
      tweets = dataset.getTrainTweets(true);
    } else {
      tweets = Tweet.getTestTweets();
    }

    Preprocessor preprocessor = Preprocessor.getInstance();
    ArkPOSTagger posTagger = ArkPOSTagger.getInstance();

    if (useSerialization) {
      posTagger.serializeModel();
      posTagger.serializeFeatureExtractor();
    }

    // process tweets
    long startTime = System.currentTimeMillis();
    for (Tweet tweet : tweets) {
      // Tokenize
      List<String> tokens = Tokenizer.tokenize(tweet.getText());

      // Preprocess
      List<String> preprocessedTokens = preprocessor.preprocess(tokens);

      // POS Tagging
      List<TaggedToken> taggedTokens = posTagger.tag(preprocessedTokens);

      LOG.info("Tweet: '" + tweet + "'");
      LOG.info("TaggedTweet: " + taggedTokens);
    }
    long elapsedTime = System.currentTimeMillis() - startTime;
    LOG.info("POSTagger finished after " + elapsedTime + " ms");
    LOG.info("Total tweets: " + tweets.size());
    LOG.info((elapsedTime / (double) tweets.size()) + " ms per Tweet");
  }

} public class Preprocessor {
  private static final Logger LOG = LoggerFactory.getLogger(Preprocessor.class);
  private static final boolean LOGGING = Configuration.get(
      "commons.preprocessor.logging", false);
  private static final Preprocessor INSTANCE = new Preprocessor();

  private WordNet m_wordnet;
  private SlangCorrection m_slangCorrection;
  private FirstNames m_firstNames;
  private NameEntities m_nameEntities;
  private Interjections m_interjections;

  private Preprocessor() {
    // Load WordNet
    m_wordnet = WordNet.getInstance();
    // Load Slang correction
    m_slangCorrection = SlangCorrection.getInstance();
    // Load FirstNames
    m_firstNames = FirstNames.getInstance();
    // Load NameEntities
    m_nameEntities = NameEntities.getInstance();
    // Load Interjections
    m_interjections = Interjections.getInstance();
  }

  public static Preprocessor getInstance() {
    return INSTANCE;
  }

  public List<String> preprocess(List<String> tokens) {
    // tail recursion
    return preprocessAccumulator(new LinkedList<String>(tokens), false,
        new ArrayList<String>());
  }

  public List<TaggedWord> preprocessAndTag(List<String> tokens) {
    // tail recursion
    return preprocessAccumulator(new LinkedList<String>(tokens), true,
        new ArrayList<TaggedWord>());
  }

  @SuppressWarnings("unchecked")
  private <T> List<T> preprocessAccumulator(LinkedList<String> tokens,
      boolean pretag, List<T> processedTokens) {

    if (tokens.isEmpty()) {
      return processedTokens;
    } else {
      // remove token from queue
      String token = tokens.removeFirst();

      // identify token
      boolean tokenContainsPunctuation = StringUtils
          .consitsOfPunctuations(token);
      boolean tokenIsEmoticon = StringUtils.isEmoticon(token);
      boolean tokenIsURL = StringUtils.isURL(token);
      boolean tokenIsNumeric = StringUtils.isNumeric(token);

      // Step 1) Unify Emoticons remove repeating chars
      if ((tokenIsEmoticon) && (!tokenIsURL) && (!tokenIsNumeric)) {
        Matcher m = RegexUtils.TWO_OR_MORE_REPEATING_CHARS_PATTERN
            .matcher(token);
        if (m.find()) {
          boolean isSpecialEmoticon = m.group(1).equals("^");
          String reducedToken = m.replaceAll("$1");
          if (isSpecialEmoticon) { // keep ^^
            reducedToken += "^";
          }
          // else {
          // TODO
          // Preprocess token again if there are recursive patterns in it
          // e.g., :):):) -> :):) -> :) Not possible because of Tokenizer
          // tokens.add(0, reducedToken);
          // }
          if (LOGGING) {
            LOG.info("Unify Emoticon from '" + token + "' to '" + reducedToken
                + "'");
          }

          if (pretag) {
            processedTokens.add((T) new TaggedWord(reducedToken, "UH"));
          } else {
            processedTokens.add((T) reducedToken);
          }
          return preprocessAccumulator(tokens, pretag, processedTokens);
        }
      } else if (tokenContainsPunctuation) {
        // If token is no Emoticon then there is no further
        // preprocessing for punctuations
        if (pretag) {
          processedTokens.add((T) new TaggedWord(token));
        } else {
          processedTokens.add((T) token);
        }
        return preprocessAccumulator(tokens, pretag, processedTokens);
      }

      // identify token
      boolean tokenIsUser = StringUtils.isUser(token);
      boolean tokenIsHashTag = StringUtils.isHashTag(token);
      boolean tokenIsSlang = StringUtils.isSlang(token);
      boolean tokenIsEmail = StringUtils.isEmail(token);
      boolean tokenIsPhone = StringUtils.isPhone(token);
      boolean tokenIsSpecialNumeric = StringUtils.isSpecialNumeric(token);
      boolean tokenIsSeparatedNumeric = StringUtils.isSeparatedNumeric(token);

      // Step 2) Slang Correction
      // TODO prevent slang correction if all UPPERCASE
      // 'FC' to [fruit, cake]
      // 'Ajax' to [Asynchronous, Javascript, and, XML]
      // 'TL' to [dr too, long, didn't, read]
      // S.O.L - SOL - [s**t, outta, luck]
      // 'AC/DC' to 'AC' and 'DC' - 'DC' to [don't, care]
      // TODO update dictionary O/U O/A
      if ((!tokenIsEmoticon) && (!tokenIsUser) && (!tokenIsHashTag)
          && (!tokenIsURL) && (!tokenIsNumeric) && (!tokenIsSpecialNumeric)
          && (!tokenIsSeparatedNumeric) && (!tokenIsEmail) && (!tokenIsPhone)) {
        String[] slangCorrection = m_slangCorrection.getCorrection(token
            .toLowerCase());
        if (slangCorrection != null) {
          for (int i = 0; i < slangCorrection.length; i++) {
            if (pretag) {
              // PreTagging for POS Tagger
              TaggedWord preTaggedToken = pretagToken(slangCorrection[i],
                  tokenIsHashTag, tokenIsUser, tokenIsURL);
              processedTokens.add((T) preTaggedToken);
            } else {
              processedTokens.add((T) slangCorrection[i]);
            }
          }
          if (LOGGING) {
            LOG.info("Slang Correction from '" + token + "' to "
                + Arrays.toString(slangCorrection));
          }
          return preprocessAccumulator(tokens, pretag, processedTokens);
        } else if (tokenIsSlang) {
          if (token.startsWith("w/")) {
            if (pretag) {
              processedTokens.add((T) new TaggedWord("with"));
              // PreTagging for POS Tagger
              TaggedWord preTaggedToken = pretagToken(token.substring(2),
                  tokenIsHashTag, tokenIsUser, tokenIsURL);
              processedTokens.add((T) preTaggedToken);
            } else {
              processedTokens.add((T) "with");
              processedTokens.add((T) token.substring(2));
            }
            if (LOGGING) {
              LOG.info("Slang Correction from '" + token + "' to " + "[with, "
                  + token.substring(2) + "]");
            }
            return preprocessAccumulator(tokens, pretag, processedTokens);
          } else {
            if (LOGGING) {
              LOG.info("Slang Correction might be missing for '" + token + "'");
            }
          }
        }
      }

      // Step 3) Check if there are punctuations between words
      // e.g., L.O.V.E
      if ((!tokenIsEmoticon) && (!tokenIsUser) && (!tokenIsHashTag)
          && (!tokenIsURL) && (!tokenIsNumeric) && (!tokenIsSpecialNumeric)
          && (!tokenIsSeparatedNumeric) && (!tokenIsEmail) && (!tokenIsPhone)) {
        // remove alternating letter dot pattern e.g., L.O.V.E
        Matcher m = RegexUtils.ALTERNATING_LETTER_DOT_PATTERN.matcher(token);
        if (m.matches()) {
          String newToken = token.replaceAll("\\.", "");
          if (m_wordnet.contains(newToken)) {
            if (LOGGING) {
              LOG.info("Remove punctuations in word from '" + token + "' to '"
                  + newToken + "'");
            }
            token = newToken;
            if (pretag) {
              // PreTagging for POS Tagger
              TaggedWord preTaggedToken = pretagToken(token, tokenIsHashTag,
                  tokenIsUser, tokenIsURL);
              processedTokens.add((T) preTaggedToken);
            } else {
              processedTokens.add((T) token);
            }
            return preprocessAccumulator(tokens, pretag, processedTokens);
          }
        }
      }

      // Step 4) Add missing g in gerund forms e.g., goin
      if ((!tokenIsUser) && (!tokenIsHashTag) && (!tokenIsURL)
          && (token.endsWith("in")) && (!m_firstNames.isFirstName(token))
          && (!m_wordnet.contains(token.toLowerCase()))) {
        // append "g" if a word ends with "in" and is not in the vocabulary
        if (LOGGING) {
          LOG.info("Add missing \"g\" from '" + token + "' to '" + token + "g'");
        }
        token = token + "g";
        if (pretag) {
          // PreTagging for POS Tagger, because it could be a interjection
          TaggedWord preTaggedToken = pretagToken(token, tokenIsHashTag,
              tokenIsUser, tokenIsURL);
          processedTokens.add((T) preTaggedToken);
        } else {
          processedTokens.add((T) token);
        }
        return preprocessAccumulator(tokens, pretag, processedTokens);
      }

      // Step 5) Remove elongations of characters (suuuper)
      // 'lollll' to 'loll' because 'loll' is found in dict
      // TODO 'AHHHHH' to 'AH'
      if ((!tokenIsEmoticon) && (!tokenIsUser) && (!tokenIsHashTag)
          && (!tokenIsURL) && (!tokenIsNumeric) && (!tokenIsSpecialNumeric)
          && (!tokenIsSeparatedNumeric) && (!tokenIsEmail) && (!tokenIsPhone)) {

        // remove repeating chars
        token = removeRepeatingChars(token);

        // Step 5b) Try Slang Correction again
        String[] slangCorrection = m_slangCorrection.getCorrection(token
            .toLowerCase());
        if (slangCorrection != null) {
          for (int i = 0; i < slangCorrection.length; i++) {
            if (pretag) {
              // PreTagging for POS Tagger
              TaggedWord preTaggedToken = pretagToken(slangCorrection[i],
                  tokenIsHashTag, tokenIsUser, tokenIsURL);
              processedTokens.add((T) preTaggedToken);
            } else {
              processedTokens.add((T) slangCorrection[i]);
            }
          }
          if (LOGGING) {
            LOG.info("Slang Correction from '" + token + "' to "
                + Arrays.toString(slangCorrection));
          }
          return preprocessAccumulator(tokens, pretag, processedTokens);
        }
      }

      // add token to processed list
      if (pretag) {
        // PreTagging for POS Tagger
        TaggedWord preTaggedToken = pretagToken(token, tokenIsHashTag,
            tokenIsUser, tokenIsURL);
        processedTokens.add((T) preTaggedToken);
      } else {
        processedTokens.add((T) token);
      }
      return preprocessAccumulator(tokens, pretag, processedTokens);
    }
  }

  private TaggedWord pretagToken(String token, boolean tokenIsHashTag,
      boolean tokenIsUser, boolean tokenIsURL) {
    TaggedWord preTaggedToken = new TaggedWord(token);
    if (tokenIsHashTag) {
      preTaggedToken.setTag("HT");
    } else if (tokenIsUser) {
      preTaggedToken.setTag("USR");
    } else if (tokenIsURL) {
      preTaggedToken.setTag("URL");
    } else if (StringUtils.isRetweet(token)) {
      preTaggedToken.setTag("RT");
    } else if (m_nameEntities.isNameEntity(token)) {
      if (LOGGING) {
        LOG.info("NameEntity labelled for " + token);
      }
      preTaggedToken.setTag("NNP");
    } else if ((m_interjections.isInterjection(token))
        || (StringUtils.isEmoticon(token))) {
      if (LOGGING) {
        LOG.info("Interjection or Emoticon labelled for " + token);
      }
      preTaggedToken.setTag("UH");
    }
    return preTaggedToken;
  }

  private String removeRepeatingChars(String value) {
    // if there are three repeating equal chars
    // then remove one char until the word is found in the vocabulary
    // else if the word is not found reduce the repeating chars to one

    // collect matches for sub-token search
    List<int[]> matches = new ArrayList<int[]>();

    Matcher m = RegexUtils.THREE_OR_MORE_REPEATING_CHARS_PATTERN.matcher(value);
    while (m.find()) {
      int start = m.start();
      int end = m.end();
      // String c = m.group(1);
      // LOG.info("token: '" + value + "' match at start: " + start + " end: "
      // + end);

      // check if token is not in the vocabulary
      if (!m_wordnet.contains(value)) {
        // collect matches for subtoken check
        matches.add(new int[] { start, end });

        StringBuilder sb = new StringBuilder(value);
        for (int i = 0; i < end - start - 1; i++) {
          sb.deleteCharAt(start); // delete repeating char

          // LOG.info("check token: '" + sb.toString() + "'");
          // check if token is in the vocabulary
          if (m_wordnet.contains(sb.toString())) {
            if (LOGGING) {
              LOG.info("removeRepeatingChars from token '" + value + "' to '"
                  + sb + "'");
            }
            return sb.toString();
          }

          // if the token is not in the vocabulary check all combinations
          // of prior matches
          // TODO really necessary?
          for (int j = 0; j < matches.size(); j++) {
            int startSub = matches.get(j)[0];
            int endSub = matches.get(j)[1];
            if (startSub != start) {
              StringBuilder subSb = new StringBuilder(sb);
              for (int k = 0; k < endSub - startSub - 1; k++) {
                subSb.deleteCharAt(startSub);

                // LOG.info("check subtoken: '" + subSb.toString() + "'");
                if (m_wordnet.contains(subSb.toString())) {
                  if (LOGGING) {
                    LOG.info("removeRepeatingChars from '" + value + "' to '"
                        + subSb + "'");
                  }
                  return subSb.toString();
                }
              }
            }
          }
        }
      }
    }

    // no match have been found
    // reduce all repeating chars
    if (!matches.isEmpty()) {
      String reducedToken = m.replaceAll("$1");
      if (LOGGING) {
        LOG.info("removeRepeatingChars(not found in dict) from '" + value
            + "' to '" + reducedToken + "'");
      }
      value = reducedToken;
    }
    return value;
  }

  public List<List<String>> preprocessTweets(List<List<String>> tweets) {
    List<List<String>> preprocessedTweets = new ArrayList<List<String>>();
    for (List<String> tweet : tweets) {
      preprocessedTweets.add(preprocess(tweet));
    }
    return preprocessedTweets;
  }

  public List<List<TaggedWord>> preprocessAndTagTweets(List<List<String>> tweets) {
    List<List<TaggedWord>> preprocessedTweets = new ArrayList<List<TaggedWord>>();
    for (List<String> tweet : tweets) {
      preprocessedTweets.add(preprocessAndTag(tweet));
    }
    return preprocessedTweets;
  }

  public static void main(String[] args) {
    Preprocessor preprocessor = Preprocessor.getInstance();
    List<Tweet> tweets = null;
    boolean extendedTest = true;
    boolean debugOutput = false;

    // load tweets
    if (extendedTest) {
      // Twitter crawler
      // List<Status> extendedTweets = Configuration
      // .getDataSetUibkCrawlerTest("en");
      // tweets = new ArrayList<Tweet>();
      // for (Status tweet : extendedTweets) {
      // tweets.add(new Tweet(tweet.getId(), tweet.getText(), 0));
      // }

      // SemEval2013
      Dataset dataset = Configuration.getDataSetSemEval2013();
      tweets = dataset.getTrainTweets(true);

    } else { // test tweets
      tweets = Tweet.getTestTweets();
      tweets.add(new Tweet(0L, "2moro afaik bbq hf lol loool lollll"));
      tweets
          .add(new Tweet(
              0L,
              "suuuper suuper professional tell aahh aaahh aahhh aaahhh aaaahhhhh gaaahh gaaahhhaaag haaahaaa hhhaaaahhhaaa"));
      tweets.add(new Tweet(0L, "Martin martin kevin Kevin Justin justin"));
      tweets.add(new Tweet(0L, "10,000 1000 +111 -111,0000.4444"));
      tweets.add(new Tweet(0L,
          "bankruptcy\ud83d\ude05 happy:-) said:-) ;-)yeah"));
      tweets.add(new Tweet(0L, "I\u2019m shit\u002c fan\\u002c \\u2019t"));
      tweets
          .add(new Tweet(
              0L,
              "like...and vegas.just hosp.now lies\u002c1st lies,1st candy....wasn\u2019t Nevada\u002cFlorida\u002cOhio\u002cTuesday lol.,.lol lol...lol.."));
      tweets.add(new Tweet(0L, "L.O.V.E D.R.U.G.S K.R.I.T"));
      tweets
          .add(new Tweet(
              0L,
              "Lamar.....I free..edom free.edom star.Kisses,Star Yes..a Oh,I it!!!Go Jenks/sagna"));
      tweets
          .add(new Tweet(
              0L,
              "32.50 $3.25 49.3% 97.1FM 97.1fm 8.30pm 12.45am 12.45AM 12.45PM 6-7pm 5-8p 6pm-9pm @9.15 tonight... 10,000 199,400 149,597,900 20,000+ 10.45,9 8/11/12"));
      tweets
          .add(new Tweet(0L,
              "(6ft.10) 2),Chap 85.3%(6513 (att@m80.com) awayDAWN.com www.asdf.org"));
    }

    // Tokenize
    List<List<String>> tokenizedTweets = Tokenizer.tokenizeTweets(tweets);

    // Preprocess only
    long startTime = System.currentTimeMillis();
    List<List<String>> preprocessedTweets = preprocessor
        .preprocessTweets(tokenizedTweets);
    LOG.info("Preprocess finished after "
        + (System.currentTimeMillis() - startTime) + " ms");

    // Preprocess and tag
    startTime = System.currentTimeMillis();
    List<List<TaggedWord>> preprocessedTaggedTweets = preprocessor
        .preprocessAndTagTweets(tokenizedTweets);
    LOG.info("PreprocessAndTag finished after "
        + (System.currentTimeMillis() - startTime) + " ms");

    if (debugOutput) {
      for (int i = 0; i < tweets.size(); i++) {
        LOG.info("Tweet: '" + tweets.get(i).getText() + "'");
        LOG.info("Preprocessed: '" + preprocessedTweets.get(i) + "'");
        LOG.info("PreprocessedTagged: '" + preprocessedTaggedTweets.get(i)
            + "'");
      }
    }
  }
} public class CombinedFeatureVectorGenerator extends FeatureVectorGenerator {
  private static final Logger LOG = LoggerFactory
      .getLogger(CombinedFeatureVectorGenerator.class);

  private final boolean m_useTaggedWords;
  private SentimentFeatureVectorGenerator m_sentimentFeatureVectorGenerator = null;
  private TfIdfFeatureVectorGenerator m_tfidfFeatureVectorGenerator = null;
  private POSFeatureVectorGenerator m_POSFeatureVectorGenerator = null;

  public CombinedFeatureVectorGenerator(boolean useTaggedWords,
      boolean normalizePOSCounts, TweetTfIdf tweetTfIdf) {
    m_useTaggedWords = useTaggedWords;

    m_sentimentFeatureVectorGenerator = new SentimentFeatureVectorGenerator(1);

    m_POSFeatureVectorGenerator = new POSFeatureVectorGenerator(useTaggedWords,
        normalizePOSCounts,
        m_sentimentFeatureVectorGenerator.getFeatureVectorSize() + 1);

    m_tfidfFeatureVectorGenerator = new TfIdfFeatureVectorGenerator(tweetTfIdf,
        m_sentimentFeatureVectorGenerator.getFeatureVectorSize()
            + m_POSFeatureVectorGenerator.getFeatureVectorSize() + 1);

    LOG.info("VectorSize: " + getFeatureVectorSize());
  }

  @Override
  public int getFeatureVectorSize() {
    return m_sentimentFeatureVectorGenerator.getFeatureVectorSize()
        + m_POSFeatureVectorGenerator.getFeatureVectorSize()
        + m_tfidfFeatureVectorGenerator.getFeatureVectorSize();
  }

  @Override
  public Map<Integer, Double> generateFeatureVectorFromTaggedWords(
      List<TaggedWord> tweet) {
    if (!m_useTaggedWords) {
      throw new RuntimeException(
          "Use TaggedWords was set to false! generateFeatureVectorFromTaggedWords is not applicable!");
    }

    Map<Integer, Double> featureVector = m_sentimentFeatureVectorGenerator
        .generateFeatureVectorFromTaggedWords(tweet);

    featureVector.putAll(m_POSFeatureVectorGenerator
        .generateFeatureVectorFromTaggedWords(tweet));

    featureVector.putAll(m_tfidfFeatureVectorGenerator
        .generateFeatureVectorFromTaggedWords(tweet));

    return featureVector;
  }

  @Override
  public Map<Integer, Double> generateFeatureVectorFromTaggedTokens(
      List<TaggedToken> tweet) {
    if (m_useTaggedWords) {
      throw new RuntimeException(
          "Use TaggedWords was set to true! generateFeatureVectorFromTaggedTokens is not applicable!");
    }

    Map<Integer, Double> featureVector = m_sentimentFeatureVectorGenerator
        .generateFeatureVectorFromTaggedTokens(tweet);

    featureVector.putAll(m_POSFeatureVectorGenerator
        .generateFeatureVectorFromTaggedTokens(tweet));

    featureVector.putAll(m_tfidfFeatureVectorGenerator
        .generateFeatureVectorFromTaggedTokens(tweet));

    return featureVector;
  }

  public static void main(String[] args) {
    boolean extendedTest = false;
    boolean useArkPOSTagger = true;
    boolean usePOSTags = true; // use POS tags in terms
    Preprocessor preprocessor = Preprocessor.getInstance();

    // load tweets
    List<Tweet> tweets = null;
    if (extendedTest) {
      // SemEval2013
      Dataset dataset = Configuration.getDataSetSemEval2013();
      tweets = dataset.getTrainTweets(true);
    } else { // test tweets
      tweets = Tweet.getTestTweets();
    }

    // Tokenize
    List<List<String>> tokenizedTweets = Tokenizer.tokenizeTweets(tweets);

    if (useArkPOSTagger) {
      ArkPOSTagger arkPOSTagger = ArkPOSTagger.getInstance();

      // Preprocess only
      long startTime = System.currentTimeMillis();
      List<List<String>> preprocessedTweets = preprocessor
          .preprocessTweets(tokenizedTweets);
      LOG.info("Preprocess finished after "
          + (System.currentTimeMillis() - startTime) + " ms");

      // Ark POS Tagging
      startTime = System.currentTimeMillis();
      List<List<TaggedToken>> taggedTweets = arkPOSTagger
          .tagTweets(preprocessedTweets);
      LOG.info("Ark POS Tagger finished after "
          + (System.currentTimeMillis() - startTime) + " ms");

      // Generate CombinedFeatureVectorGenerator
      TweetTfIdf tweetTfIdf = TweetTfIdf.createFromTaggedTokens(taggedTweets,
          TfType.RAW, TfIdfNormalization.COS, usePOSTags);
      CombinedFeatureVectorGenerator cfvg = new CombinedFeatureVectorGenerator(
          false, true, tweetTfIdf);

      // Combined Feature Vector Generation
      for (List<TaggedToken> taggedTokens : taggedTweets) {
        Map<Integer, Double> combinedFeatureVector = cfvg
            .generateFeatureVectorFromTaggedTokens(taggedTokens);

        // Build feature vector string
        String featureVectorStr = "";
        for (Map.Entry<Integer, Double> feature : combinedFeatureVector
            .entrySet()) {
          featureVectorStr += " " + feature.getKey() + ":" + feature.getValue();
        }
        LOG.info("Tweet: '" + taggedTokens + "'");
        LOG.info("CombinedFeatureVector: " + featureVectorStr);
      }

    } else {
      GatePOSTagger gatePOSTagger = GatePOSTagger.getInstance();

      // Preprocess and tag
      long startTime = System.currentTimeMillis();
      List<List<TaggedWord>> preprocessedTweets = preprocessor
          .preprocessAndTagTweets(tokenizedTweets);
      LOG.info("PreprocessAndTag finished after "
          + (System.currentTimeMillis() - startTime) + " ms");

      // Gate POS Tagging
      startTime = System.currentTimeMillis();
      List<List<TaggedWord>> taggedTweets = gatePOSTagger
          .tagTweets(preprocessedTweets);
      LOG.info("Gate POS Tagger finished after "
          + (System.currentTimeMillis() - startTime) + " ms");

      // Generate CombinedFeatureVectorGenerator
      TweetTfIdf tweetTfIdf = TweetTfIdf.createFromTaggedWords(taggedTweets,
          TfType.RAW, TfIdfNormalization.COS, usePOSTags);
      CombinedFeatureVectorGenerator cfvg = new CombinedFeatureVectorGenerator(
          true, true, tweetTfIdf);

      // Combined Feature Vector Generation
      for (List<TaggedWord> taggedWords : taggedTweets) {
        Map<Integer, Double> combinedFeatureVector = cfvg
            .generateFeatureVectorFromTaggedWords(taggedWords);

        // Build feature vector string
        String featureVectorStr = "";
        for (Map.Entry<Integer, Double> feature : combinedFeatureVector
            .entrySet()) {
          featureVectorStr += " " + feature.getKey() + ":" + feature.getValue();
        }
        LOG.info("Tweet: '" + taggedWords + "'");
        LOG.info("CombinedFeatureVector: " + featureVectorStr);
      }
    }
  }

} public abstract class FeatureVectorGenerator {
  private static final Logger LOG = LoggerFactory
      .getLogger(FeatureVectorGenerator.class);

  public abstract int getFeatureVectorSize();

  public abstract Map<Integer, Double> generateFeatureVectorFromTaggedWords(
      List<TaggedWord> tweet);

  public abstract Map<Integer, Double> generateFeatureVectorFromTaggedTokens(
      List<TaggedToken> tweet);

  public List<Map<Integer, Double>> generateFeatureVectorsFromTaggedWords(
      List<List<TaggedWord>> tweets) {
    return generateFeatureVectorsFromTaggedWords(tweets, false);
  }

  public List<Map<Integer, Double>> generateFeatureVectorsFromTaggedWords(
      List<List<TaggedWord>> taggedTweets, boolean logging) {
    List<Map<Integer, Double>> featuredVectors = new ArrayList<Map<Integer, Double>>();
    for (List<TaggedWord> tweet : taggedTweets) {
      Map<Integer, Double> featureVector = generateFeatureVectorFromTaggedWords(tweet);
      if (logging) {
        LOG.info("Tweet: " + tweet);
        LOG.info("FeatureVector: " + featureVector);
      }
      featuredVectors.add(featureVector);
    }
    return featuredVectors;
  }

  public List<Map<Integer, Double>> generateFeatureVectorsFromTaggedTokens(
      List<List<TaggedToken>> tweets) {
    return generateFeatureVectorsFromTaggedTokens(tweets, false);
  }

  public List<Map<Integer, Double>> generateFeatureVectorsFromTaggedTokens(
      List<List<TaggedToken>> taggedTweets, boolean logging) {
    List<Map<Integer, Double>> featuredVectors = new ArrayList<Map<Integer, Double>>();
    for (List<TaggedToken> tweet : taggedTweets) {
      Map<Integer, Double> featureVector = generateFeatureVectorFromTaggedTokens(tweet);
      if (logging) {
        LOG.info("Tweet: " + tweet);
        LOG.info("FeatureVector: " + featureVector);
      }
      featuredVectors.add(featureVector);
    }
    return featuredVectors;
  }

} public class IdentityScoreClassifier implements ScoreClassifier {

  @Override
  public int classfyScore(double score) {
    return (int) score;
  }
} public interface ScoreClassifier {
  public int classfyScore(double score);
} public enum TfIdfNormalization {
  /**
   * Do not normalize the vector
   */
  NONE,
  /**
   * Normalize by the vector elements added in quadrature
   */
  COS
} public enum TfType {
  /**
   * Raw frequency: tf(t,d) = f(t,d) The number of times that term t occurs in
   * document d.
   */
  RAW,

  /**
   * Log scaled frequency: tf(t,d) = 1 + log f(t,d), or zero if f(t,d) is zero;
   */
  LOG,

  /**
   * Boolean frequency: tf(t,d) = 1 if t occurs in d and 0 otherwise;
   */
  BOOL
} public class TweetTfIdf {
  private static final Logger LOG = LoggerFactory.getLogger(TweetTfIdf.class);

  private TfType m_tfType;
  private TfIdfNormalization m_tfIdfNormalization;
  private List<Map<String, Double>> m_termFreqs;
  private Map<String, Double> m_inverseDocFreq;
  private Map<String, Integer> m_termIds;
  private boolean m_usePOSTags;

  private TweetTfIdf(TfType type, TfIdfNormalization normalization,
      boolean usePOSTags) {
    this.m_tfType = type;
    this.m_tfIdfNormalization = normalization;
    this.m_usePOSTags = usePOSTags;
  }

  public TfType getTfType() {
    return m_tfType;
  }

  public TfIdfNormalization getTfIdfNormalization() {
    return m_tfIdfNormalization;
  }

  public List<Map<String, Double>> getTermFreqs() {
    return m_termFreqs;
  }

  public Map<String, Double> getInverseDocFreq() {
    return m_inverseDocFreq;
  }

  public Map<String, Integer> getTermIds() {
    return m_termIds;
  }

  public Map<String, Double> tfIdfFromTaggedWords(List<TaggedWord> tweet) {
    return TfIdf.tfIdf(tfFromTaggedWords(tweet, m_tfType, m_usePOSTags),
        m_inverseDocFreq, m_tfIdfNormalization);
  }

  public Map<String, Double> tfIdfFromTaggedTokens(List<TaggedToken> tweet) {
    return TfIdf.tfIdf(tfFromTaggedTokens(tweet, m_tfType, m_usePOSTags),
        m_inverseDocFreq, m_tfIdfNormalization);
  }

  public static TweetTfIdf createFromTaggedWords(List<List<TaggedWord>> tweets,
      boolean usePOSTags) {
    return createFromTaggedWords(tweets, TfType.RAW, TfIdfNormalization.NONE,
        usePOSTags);
  }

  public static TweetTfIdf createFromTaggedWords(List<List<TaggedWord>> tweets,
      TfType type, TfIdfNormalization normalization, boolean usePOSTags) {

    TweetTfIdf tweetTfIdf = new TweetTfIdf(type, normalization, usePOSTags);

    tweetTfIdf.m_termFreqs = tfTaggedWordTweets(tweets, type, usePOSTags);
    tweetTfIdf.m_inverseDocFreq = idf(tweetTfIdf.m_termFreqs);

    tweetTfIdf.m_termIds = new HashMap<String, Integer>();
    int i = 0;
    for (String key : tweetTfIdf.m_inverseDocFreq.keySet()) {
      tweetTfIdf.m_termIds.put(key, i);
      i++;
    }

    LOG.info("Found " + tweetTfIdf.m_inverseDocFreq.size() + " terms");
    // Debug
    // print("Term Frequency", m_termFreqs, m_inverseDocFreq);
    // print("Inverse Document Frequency", m_inverseDocFreq);
    return tweetTfIdf;
  }

  public static TweetTfIdf createFromTaggedTokens(
      List<List<TaggedToken>> tweets, boolean usePOSTags) {
    return createFromTaggedTokens(tweets, TfType.RAW, TfIdfNormalization.NONE,
        usePOSTags);
  }

  public static TweetTfIdf createFromTaggedTokens(
      List<List<TaggedToken>> tweets, TfType type,
      TfIdfNormalization normalization, boolean usePOSTags) {

    TweetTfIdf tweetTfIdf = new TweetTfIdf(type, normalization, usePOSTags);

    tweetTfIdf.m_termFreqs = tfTaggedTokenTweets(tweets, type, usePOSTags);
    tweetTfIdf.m_inverseDocFreq = idf(tweetTfIdf.m_termFreqs);

    tweetTfIdf.m_termIds = new HashMap<String, Integer>();
    int i = 0;
    for (String key : tweetTfIdf.m_inverseDocFreq.keySet()) {
      tweetTfIdf.m_termIds.put(key, i);
      i++;
    }

    LOG.info("Found " + tweetTfIdf.m_inverseDocFreq.size() + " terms");
    // Debug
    // print("Term Frequency", m_termFreqs, m_inverseDocFreq);
    // print("Inverse Document Frequency", m_inverseDocFreq);
    return tweetTfIdf;
  }

  public static List<Map<String, Double>> tfTaggedWordTweets(
      List<List<TaggedWord>> tweets, TfType type, boolean usePOSTags) {
    List<Map<String, Double>> termFreqs = new ArrayList<Map<String, Double>>();
    for (List<TaggedWord> tweet : tweets) {
      termFreqs.add(tfFromTaggedWords(tweet, type, usePOSTags));
    }
    return termFreqs;
  }

  public static List<Map<String, Double>> tfTaggedTokenTweets(
      List<List<TaggedToken>> tweets, TfType type, boolean usePOSTags) {
    List<Map<String, Double>> termFreqs = new ArrayList<Map<String, Double>>();
    for (List<TaggedToken> tweet : tweets) {
      termFreqs.add(tfFromTaggedTokens(tweet, type, usePOSTags));
    }
    return termFreqs;
  }

  public static Map<String, Double> tfFromTaggedWords(List<TaggedWord> tweet,
      TfType type, boolean usePOSTags) {
    Map<String, Double> termFreq = new LinkedHashMap<String, Double>();
    WordNet wordNet = WordNet.getInstance();
    StopWords stopWords = StopWords.getInstance();

    List<String> words = new ArrayList<String>();
    for (TaggedWord taggedWord : tweet) {
      String word = taggedWord.word().toLowerCase();
      String pennTag = taggedWord.tag();

      if ((!pennTag.equals(".")) && (!pennTag.equals(","))
          && (!pennTag.equals(":")) && (!pennTag.equals("''"))
          && (!pennTag.equals("(")) && (!pennTag.equals(")"))
          && (!pennTag.equals("URL")) && (!pennTag.equals("USR"))
          && (!pennTag.equals("CC")) && (!pennTag.equals("CD"))
          && (!pennTag.equals("SYM")) && (!pennTag.equals("POS"))
          && (!stopWords.isStopWord(word))) {

        // Remove hashtag
        if (pennTag.equals("HT")) {
          word = word.substring(1);
        }

        // Check if word consists of punctuations
        // if (StringUtils.consitsOfPunctuations(word)
        // && (!pennTag.equals("POS"))) {
        // continue;
        // }

        // Check if word starts with an alphabet
        if (!StringUtils.startsWithAlphabeticChar(word)) {
          continue;
        }

        POS posTag = POSTag.convertPTB(pennTag);
        // LOG.info("word: '" + word + "' pennTag: '" + pennTag + "' tag: '"
        // + posTag + "'");

        // word stemming
        List<String> stems = wordNet.findStems(word, posTag);
        if (!stems.isEmpty()) {
          word = stems.get(0);
        }

        // add word to term frequency
        if (usePOSTags) {
          words.add(word
              + ((posTag != null) ? "#" + POSTag.toString(posTag) : ""));
        } else {
          words.add(word);
        }
      }
    }
    termFreq = TfIdf.tf(termFreq, words);
    termFreq = TfIdf.normalizeTf(termFreq, type);
    return termFreq;
  }

  public static Map<String, Double> tfFromTaggedTokens(List<TaggedToken> tweet,
      TfType type, boolean usePOSTags) {
    Map<String, Double> termFreq = new LinkedHashMap<String, Double>();
    WordNet wordNet = WordNet.getInstance();
    StopWords stopWords = StopWords.getInstance();

    List<String> words = new ArrayList<String>();
    for (TaggedToken taggedToken : tweet) {
      String word = taggedToken.token.toLowerCase();
      String arkTag = taggedToken.tag;

      // http://www.ark.cs.cmu.edu/TweetNLP/annot_guidelines.pdf
      if ((!arkTag.equals(",")) && (!arkTag.equals("$"))
          && (!arkTag.equals("G")) && (!arkTag.equals("@"))
          && (!arkTag.equals("~")) && (!arkTag.equals("U"))
          && (!stopWords.isStopWord(word))) {

        // Remove hashtag
        if (arkTag.equals("#")) {
          word = word.substring(1);
        }

        // Check if word consists of punctuations
        // if (StringUtils.consitsOfPunctuations(word)
        // && (!pennTag.equals("POS"))) {
        // continue;
        // }

        // Check if word starts with an alphabet
        if (!StringUtils.startsWithAlphabeticChar(word)) {
          continue;
        }

        POS posTag = POSTag.convertArk(arkTag);
        // LOG.info("word: '" + word + "' pennTag: '" + pennTag + "' tag: '"
        // + posTag + "'");

        // word stemming
        List<String> stems = wordNet.findStems(word, posTag);
        if (!stems.isEmpty()) {
          word = stems.get(0);
        }

        // add word to term frequency
        if (usePOSTags) {
          words.add(word
              + ((posTag != null) ? "#" + POSTag.toString(posTag) : ""));
        } else {
          words.add(word);
        }
      }
    }
    termFreq = TfIdf.tf(termFreq, words);
    termFreq = TfIdf.normalizeTf(termFreq, type);
    return termFreq;
  }

  public static Map<String, Double> idf(List<Map<String, Double>> termFreq) {
    return TfIdf.idf(termFreq);
  }

  public static List<Map<String, Double>> tfIdf(
      List<Map<String, Double>> termFreqs, Map<String, Double> inverseDocFreq,
      TfIdfNormalization normalization) {

    List<Map<String, Double>> tfIdf = new ArrayList<Map<String, Double>>();
    // compute tfIdf for each document
    for (Map<String, Double> doc : termFreqs) {
      tfIdf.add(TfIdf.tfIdf(doc, inverseDocFreq, normalization));
    }

    return tfIdf;
  }

  public static void print(String title, Map<String, Double> inverseDocFreq) {
    // print title
    LOG.info(String.format("=== %s ===", title));
    // print values
    for (Map.Entry<String, Double> term : inverseDocFreq.entrySet()) {
      String line = String.format("%15s", term.getKey());
      line += String.format("%8.4f", term.getValue());
      LOG.info(line);
    }
  }

  public static void print(String title, List<Map<String, Double>> termFreqs,
      Map<String, Double> inverseDocFreq) {
    // print title
    LOG.info(String.format("=== %s ===", title));

    // print header
    String line = String.format("%15s", " ");
    for (int i = 0; i < termFreqs.size(); i++) {
      line += String.format("%8s", " " + i);
    }
    LOG.info(line);

    // print values
    for (Map.Entry<String, Double> term : inverseDocFreq.entrySet()) {
      line = String.format("%15s", term.getKey());
      for (Map<String, Double> termFreq : termFreqs) {
        line += String.format("%8.4f", termFreq.get(term.getKey()));
      }
      LOG.info(line);
    }
  }

  public static void main(String[] args) {
    boolean extendedTest = false;
    boolean useArkPOSTagger = true;
    boolean usePOSTags = true; // use POS tags in terms

    Preprocessor preprocessor = Preprocessor.getInstance();
    GatePOSTagger gatePOSTagger = null;
    ArkPOSTagger arkPOSTagger = null;

    if (useArkPOSTagger) {
      arkPOSTagger = ArkPOSTagger.getInstance();
    } else {
      gatePOSTagger = GatePOSTagger.getInstance();
    }

    // load tweets
    List<Tweet> tweets = null;
    if (extendedTest) {
      // SemEval2013
      Dataset dataset = Configuration.getDataSetSemEval2013();
      tweets = dataset.getTrainTweets(true);
    } else {
      tweets = Tweet.getTestTweets();
    }

    // Tokenize
    List<List<String>> tokenizedTweets = Tokenizer.tokenizeTweets(tweets);

    if (useArkPOSTagger) {
      // Preprocess only
      long startTime = System.currentTimeMillis();
      List<List<String>> preprocessedTweets = preprocessor
          .preprocessTweets(tokenizedTweets);
      LOG.info("Preprocess finished after "
          + (System.currentTimeMillis() - startTime) + " ms");

      // Ark POS Tagging
      startTime = System.currentTimeMillis();
      List<List<TaggedToken>> taggedTweets = arkPOSTagger
          .tagTweets(preprocessedTweets);
      LOG.info("Ark POS Tagger finished after "
          + (System.currentTimeMillis() - startTime) + " ms");

      // Ark TF-IDF
      List<Map<String, Double>> termFreqs = TweetTfIdf.tfTaggedTokenTweets(
          taggedTweets, TfType.LOG, usePOSTags);
      Map<String, Double> inverseDocFreq = TweetTfIdf.idf(termFreqs);
      List<Map<String, Double>> tfIdf = TweetTfIdf.tfIdf(termFreqs,
          inverseDocFreq, TfIdfNormalization.COS);

      LOG.info("Found " + inverseDocFreq.size() + " terms");
      print("Term Frequency", termFreqs, inverseDocFreq);
      print("Inverse Document Frequency", inverseDocFreq);
      print("Tf-Idf", tfIdf, inverseDocFreq);

    } else {
      // Preprocess and tag
      long startTime = System.currentTimeMillis();
      List<List<TaggedWord>> preprocessedTweets = preprocessor
          .preprocessAndTagTweets(tokenizedTweets);
      LOG.info("PreprocessAndTag finished after "
          + (System.currentTimeMillis() - startTime) + " ms");

      // Gate POS Tagging
      startTime = System.currentTimeMillis();
      List<List<TaggedWord>> taggedTweets = gatePOSTagger
          .tagTweets(preprocessedTweets);
      LOG.info("Gate POS Tagger finished after "
          + (System.currentTimeMillis() - startTime) + " ms");

      // Gate TF-IDF
      List<Map<String, Double>> termFreqs = TweetTfIdf.tfTaggedWordTweets(
          taggedTweets, TfType.LOG, usePOSTags);
      Map<String, Double> inverseDocFreq = TweetTfIdf.idf(termFreqs);
      List<Map<String, Double>> tfIdf = TweetTfIdf.tfIdf(termFreqs,
          inverseDocFreq, TfIdfNormalization.COS);

      LOG.info("Found " + inverseDocFreq.size() + " terms");
      print("Term Frequency", termFreqs, inverseDocFreq);
      print("Inverse Document Frequency", inverseDocFreq);
      print("Tf-Idf", tfIdf, inverseDocFreq);
    }
  }

} public class Tokenizer {
  private static final Logger LOG = LoggerFactory.getLogger(Tokenizer.class);
  private static final boolean LOGGING = Configuration.get(
      "commons.tokenizer.logging", false);

  public static enum Type {
    REGEX_TOKENIZER, ARK_TOKENIZER, STANFORD_TOKENIZER
  }

  public static List<List<String>> tokenizeTweets(List<Tweet> tweets) {
    return tokenizeTweets(tweets, Type.REGEX_TOKENIZER);
  }

  public static List<List<String>> tokenizeTweets(List<Tweet> tweets, Type type) {
    List<List<String>> tokenizedTweets = new ArrayList<List<String>>();
    for (Tweet tweet : tweets) {
      tokenizedTweets.add(tokenize(tweet.getText(), type));
    }
    return tokenizedTweets;
  }

  public static List<String> tokenize(String str) {
    return tokenize(str, Type.REGEX_TOKENIZER);
  }

  public static List<String> tokenize(String str, Type type) {
    // Step 1) Trim text
    str = str.trim();

    // Step 2) Replace Unicode symbols \u0000
    if (UnicodeUtils.containsUnicode(str)) {
      String replacedText = UnicodeUtils.replaceUnicodeSymbols(str);
      // LOG.info("Replaced Unicode symbols from '" + str + "' to '"
      // + replacedText + "'");
      if ((LOGGING) && (replacedText.equals(str))) {
        LOG.warn("Unicode symbols could not be replaced: '" + str + "'");
      }
      str = replacedText;
    }

    // Step 3) Replace HTML symbols &#[0-9];
    if (HtmlUtils.containsHtml(str)) {
      String replacedText = HtmlUtils.replaceHtmlSymbols(str);
      // LOG.info("Replaced HTML symbols from '" + text + "' to '"
      // + replacedText + "'");
      if ((LOGGING) && (replacedText.equals(str))) {
        LOG.warn("HTML symbols could not be replaced: '" + str + "'");
      }
      str = replacedText;
    }

    // Step 4) Tokenize
    List<String> tokenizedTokens = null;

    switch (type) {
      case REGEX_TOKENIZER:
        tokenizedTokens = new ArrayList<String>();
        Matcher m = RegexUtils.TOKENIZER_PATTERN.matcher(str);
        while (m.find()) {
          tokenizedTokens.add(m.group());
        }
        break;

      case ARK_TOKENIZER:
        tokenizedTokens = Twokenize.tokenize(str);
        break;

      case STANFORD_TOKENIZER:
        TokenizerFactory<Word> tokenizer = PTBTokenizerFactory
            .newTokenizerFactory();
        tokenizer.setOptions("ptb3Escaping=false");
        List<List<HasWord>> sentences = MaxentTagger.tokenizeText(
            new StringReader(str), tokenizer);
        // Convert sentences to List<String>
        tokenizedTokens = new ArrayList<String>();
        for (List<HasWord> sentence : sentences) {
          for (HasWord word : sentence) {
            tokenizedTokens.add(word.word());
          }
        }
        break;

      default:
        break;
    }

    return tokenizedTokens;
  }

  public static void main(String[] args) {
    boolean extendedTest = true;
    List<Tweet> tweets = null;
    Type type = Type.REGEX_TOKENIZER;

    // load tweets
    if (extendedTest) {
      Dataset dataset = Configuration.getDataSetSemEval2013();
      tweets = dataset.getTrainTweets(true);
    } else { // test tweets
      tweets = Tweet.getTestTweets();
      tweets
          .add(new Tweet(
              0L,
              ":-))) xDD XDD ;) :) :-) :) :D :o) :] :3 :c) :> =] 8) =) :} :^) :-D 8-D 8D x-D xD X-D XD =-D =D =-3 =3"));
      tweets
          .add(new Tweet(0L,
              ">:[ :-( :(  :-c :c :-< :< :-[ :[ :{ ;( :-|| :@ >:( :'-( :'( :'-) :') \\m/"));
      tweets
          .add(new Tweet(
              0L,
              "\"All the money you've spent on my credit card I'm taking it out of your account\".... Hi I'm Sydney and I'm filing bankruptcy\ud83d\ude05\ud83d\ude05\ud83d\ude05\ud83d\ude05 \ud83d\ude05"));
      tweets
          .add(new Tweet(
              0L,
              "word:-) Moon:Oct Jobs! Thursday:... http://t.co/TZZzrrKa +1 (000) 123-4567, (000) 123-4567, and 123-4567 "));
      tweets
          .add(new Tweet(
              0L,
              "t/m k/o w/my b/slisten Rt/follow S/o S/O O/U O/A w/ w/Biden w/deals w/you w/the w/her"));
      tweets
          .add(new Tweet(0L, "5pm 5Am 5% $5 5-6am 7am-10pm U.S. U.K. L.O.V.E"));
    }

    // Tokenize tweets
    long startTime = System.currentTimeMillis();
    for (Tweet tweet : tweets) {
      List<String> tokens = Tokenizer.tokenize(tweet.getText(), type);
      LOG.info("Tweet: '" + tweet.getText() + "'");
      LOG.info("Tokens: " + tokens);
    }
    LOG.info("Tokenize finished after "
        + (System.currentTimeMillis() - startTime) + " ms");
  }

} public final class FeaturedTweet implements Serializable {
  private static final long serialVersionUID = -2662190284006213666L;
  private final Tweet m_tweet;
  private final List<String> m_tokens;
  private final List<String> m_preprocessedTokens;
  private final List<TaggedWord> m_preprocessedTaggedTokens;
  private final List<TaggedToken> m_arkTaggedTokens;
  private final List<TaggedWord> m_gateTaggedTokens;
  private final Map<Integer, Double> m_arkFeatureVector; // dense vector
  private final Map<Integer, Double> m_gateFeatureVector; // dense vector

  private FeaturedTweet(Tweet tweet, List<String> tokens,
      List<String> preprocessedTokens,
      List<TaggedWord> preprocessedTaggedTokens,
      List<TaggedToken> arkTaggedWords, List<TaggedWord> gateTaggedWords,
      Map<Integer, Double> arkFeatureVector,
      Map<Integer, Double> gateFeatureVector) {
    m_tweet = tweet;
    m_tokens = tokens;
    m_preprocessedTokens = preprocessedTokens;
    m_preprocessedTaggedTokens = preprocessedTaggedTokens;
    m_arkTaggedTokens = arkTaggedWords;
    m_gateTaggedTokens = gateTaggedWords;
    m_arkFeatureVector = arkFeatureVector;
    m_gateFeatureVector = gateFeatureVector;
  }

  public Long getId() {
    return m_tweet.getId();
  }

  public String getText() {
    return m_tweet.getText();
  }

  public Double getScore() {
    return m_tweet.getScore();
  }

  public List<String> getTokens() {
    return m_tokens;
  }

  public List<String> getPreprocessedTokens() {
    return m_preprocessedTokens;
  }

  public List<TaggedWord> getPreprocessedTaggedTokens() {
    return m_preprocessedTaggedTokens;
  }

  public List<TaggedToken> getArkTaggedTokens() {
    return m_arkTaggedTokens;
  }

  public List<TaggedWord> getGateTaggedTokens() {
    return m_gateTaggedTokens;
  }

  public Map<Integer, Double> getArkFeatureVector() {
    return m_arkFeatureVector;
  }

  public Map<Integer, Double> getGateFeatureVector() {
    return m_gateFeatureVector;
  }

  @Override
  public boolean equals(Object obj) {
    return m_tweet.equals(obj);
  }

  @Override
  public String toString() {
    return m_tweet.toString();
  }

  public static FeaturedTweet createFromTaggedWords(Long id, String text,
      Double score, List<String> tokens,
      List<TaggedWord> preprocessedTaggedTokens, List<TaggedWord> taggedWords,
      Map<Integer, Double> featureVector) {
    return new FeaturedTweet(new Tweet(id, text, score), tokens, null,
        preprocessedTaggedTokens, null, taggedWords, null, featureVector);
  }

  public static FeaturedTweet createFromTaggedWords(Tweet tweet,
      List<String> tokens, List<TaggedWord> preprocessedTaggedTokens,
      List<TaggedWord> taggedWords, Map<Integer, Double> featureVector) {
    return new FeaturedTweet(tweet, tokens, null, preprocessedTaggedTokens,
        null, taggedWords, null, featureVector);
  }

  public static FeaturedTweet createFromTaggedTokens(Long id, String text,
      Double score, List<String> tokens, List<String> preprocessedTokens,
      List<TaggedToken> taggedTokens, Map<Integer, Double> featureVector) {
    return new FeaturedTweet(new Tweet(id, text, score), tokens,
        preprocessedTokens, null, taggedTokens, null, featureVector, null);
  }

  public static FeaturedTweet createFromTaggedTokens(Tweet tweet,
      List<String> tokens, List<String> preprocessedTokens,
      List<TaggedToken> taggedTokens, Map<Integer, Double> featureVector) {
    return new FeaturedTweet(tweet, tokens, preprocessedTokens, null,
        taggedTokens, null, featureVector, null);
  }

  public static FeaturedTweet create(Tweet tweet, List<String> tokens,
      List<String> preprocessedTokens,
      List<TaggedWord> preprocessedTaggedTokens,
      List<TaggedToken> taggedTokens, List<TaggedWord> taggedWords,
      Map<Integer, Double> arkFeatureVector,
      Map<Integer, Double> gateFeatureVector) {
    return new FeaturedTweet(tweet, tokens, preprocessedTokens,
        preprocessedTaggedTokens, taggedTokens, taggedWords, arkFeatureVector,
        gateFeatureVector);
  }

  public static final List<List<TaggedWord>> getTaggedWordsFromTweets(
      List<FeaturedTweet> featuredTweets) {
    List<List<TaggedWord>> taggedTweets = new ArrayList<List<TaggedWord>>();
    for (FeaturedTweet tweet : featuredTweets) {
      taggedTweets.add(tweet.getGateTaggedTokens());
    }
    return taggedTweets;
  }

  public static final List<List<TaggedToken>> getTaggedTokensFromTweets(
      List<FeaturedTweet> featuredTweets) {
    List<List<TaggedToken>> taggedTweets = new ArrayList<List<TaggedToken>>();
    for (FeaturedTweet tweet : featuredTweets) {
      taggedTweets.add(tweet.getArkTaggedTokens());
    }
    return taggedTweets;
  }

  public static final void serializeTrainTweets(Dataset dataset,
      boolean includeDevTweets) {

    final Preprocessor preprocessor = Preprocessor.getInstance();
    final ArkPOSTagger arkPOSTagger = ArkPOSTagger.getInstance();
    final GatePOSTagger gatePOSTagger = GatePOSTagger.getInstance();

    List<Tweet> tweets = dataset.getTrainTweets(includeDevTweets);

    // Tokenize
    List<List<String>> tokenizedTweets = Tokenizer.tokenizeTweets(tweets);

    // Preprocess only
    List<List<String>> preprocessedTweets = preprocessor
        .preprocessTweets(tokenizedTweets);

    // Preprocess and tag
    List<List<TaggedWord>> preprocessedTaggedTweets = preprocessor
        .preprocessAndTagTweets(tokenizedTweets);

    // Ark POS Tagging
    List<List<TaggedToken>> arkTaggedTweets = arkPOSTagger
        .tagTweets(preprocessedTweets);

    // Gate POS Tagging
    List<List<TaggedWord>> gateTaggedTweets = gatePOSTagger
        .tagTweets(preprocessedTaggedTweets);

    // Ark Feature Vector Generator
    TweetTfIdf arkTweetTfIdf = TweetTfIdf.createFromTaggedTokens(
        arkTaggedTweets, TfType.RAW, TfIdfNormalization.COS, true);
    FeatureVectorGenerator arkFvg = new CombinedFeatureVectorGenerator(false,
        true, arkTweetTfIdf);

    // Gate Feature Vector Generator
    TweetTfIdf gateTweetTfIdf = TweetTfIdf.createFromTaggedWords(
        gateTaggedTweets, TfType.RAW, TfIdfNormalization.COS, true);
    FeatureVectorGenerator gateFvg = new CombinedFeatureVectorGenerator(true,
        true, gateTweetTfIdf);

    // Feature Vector Generation
    List<FeaturedTweet> featuredTrainTweets = new ArrayList<FeaturedTweet>();
    for (int i = 0; i < tweets.size(); i++) {
      // Ark Feature Vector Generation
      List<TaggedToken> arkTaggedTweet = arkTaggedTweets.get(i);
      Map<Integer, Double> arkFeatureVector = arkFvg
          .generateFeatureVectorFromTaggedTokens(arkTaggedTweet);

      // Gate Feature Vector Generation
      List<TaggedWord> gateTaggedTweet = gateTaggedTweets.get(i);
      Map<Integer, Double> gateFeatureVector = gateFvg
          .generateFeatureVectorFromTaggedWords(gateTaggedTweet);

      featuredTrainTweets.add(FeaturedTweet.create(tweets.get(i),
          tokenizedTweets.get(i), preprocessedTweets.get(i),
          preprocessedTaggedTweets.get(i), arkTaggedTweet, gateTaggedTweet,
          arkFeatureVector, gateFeatureVector));
    }

    SerializationUtils.serializeList(featuredTrainTweets,
        dataset.getTrainDataSerializationFile());
  }

  public static void main(String[] args) {
    serializeTrainTweets(Configuration.getDataSetSemEval2013(), true);
  }

} public final class Tweet implements Serializable {
  private static final long serialVersionUID = 5246263296320608188L;
  private final Long m_id;
  private final String m_text;
  private final Double m_score;

  public Tweet(Long id, String text, Double score) {
    this.m_id = id;
    this.m_text = text;
    this.m_score = score;
  }

  public Tweet(Long id, String text) {
    this(id, text, null);
  }

  public Long getId() {
    return m_id;
  }

  public String getText() {
    return m_text;
  }

  public Double getScore() {
    return m_score;
  }

  @Override
  public boolean equals(Object obj) {
    if (obj == null) {
      return false;
    }
    if (getClass() != obj.getClass()) {
      return false;
    }
    final Tweet other = (Tweet) obj;
    // check if id is matching
    if (this.m_id != other.getId()) {
      return false;
    }
    // check if text is matching
    if (!this.m_text.equals(other.getText())) {
      return false;
    }
    return true;
  }

  @Override
  public String toString() {
    return "Tweet [id=" + m_id + ", text=" + m_text
        + ((m_score != null) ? ", score=" + m_score : "") + "]";
  }

  public static List<Tweet> getTestTweets() {
    List<Tweet> tweets = new ArrayList<Tweet>();
    tweets.add(new Tweet(1L,
        "Gas by my house hit $3.39 !!!! I'm goin to Chapel Hill on Sat . :)",
        1.0));
    tweets
        .add(new Tweet(
            2L,
            "@oluoch @victor_otti @kunjand I just watched it ! Sridevi's comeback .... U remember her from the 90s ?? Sun mornings on NTA ;)",
            1.0));
    tweets
        .add(new Tweet(
            3L,
            "PBR & @mokbpresents bring you Jim White at the @Do317 Lounge on October 23rd at 7 pm ! http://t.co/7x8OfC56.",
            0.5));
    tweets
        .add(new Tweet(
            4L,
            "Why is it so hard to find the @TVGuideMagazine these days ? Went to 3 stores for the Castle cover issue . NONE . Will search again tomorrow ...",
            0.0));
    tweets.add(new Tweet(0L,
        "called in sick for the third straight day.  ugh.", 0.0));
    tweets
        .add(new Tweet(
            5L,
            "Here we go.  BANK FAAAAIL FRIDAY -- The FDIC says the Bradford Bank in Baltimore, Maryland has become the 82nd bank failure of the year.",
            0.0));
    tweets.add(new Tweet(0L,
        "Oh, I'm afraid your Windows-using friends will not survive.", 0.0));

    tweets
        .add(new Tweet(
            6L,
            "Excuse the connectivity of this live stream , from Baba Amr , so many activists using only one Sat Modem . LIVE http://t.co/U283IhZ5 #Homs"));
    tweets
        .add(new Tweet(
            7L,
            "Show your LOVE for your local field & it might win an award ! Gallagher Park #Bedlington current 4th in National Award http://t.co/WeiMDtQt"));
    tweets
        .add(new Tweet(
            8L,
            "@firecore Can you tell me when an update for the Apple TV 3rd gen becomes available ? The missing update holds me back from buying #appletv3"));
    tweets
        .add(new Tweet(
            9L,
            "My #cre blog Oklahoma Per Square Foot returns to the @JournalRecord blog hub tomorrow . I will have some interesting local data to share ."));
    tweets
        .add(new Tweet(
            10L,
            "\" &quot; @bbcburnsy : Loads from SB ; &quot;talks with Chester continue ; no deals 4 out of contract players ' til Jan ; Dev t Roth , Coops to Chest'ld #hcafc \""));

    // thesis examples
    tweets
        .add(new Tweet(
            11L,
            "@user1 @user2 OMG I just watched Michael&#039;s comeback ! U remember him from the 90s ?? yaaaa \uD83D\uDE09 #michaelcomeback"));
    tweets
        .add(new Tweet(12L,
            "@user1 @user2 OMG I just watched it again ! \uD83D\uDE09 #michaelcomeback"));
    tweets.add(new Tweet(13L,
        "@user3 I luv him too ! \uD83D\uDE09 #michaelcomeback"));

    return tweets;
  }

} public class SerializationUtils {
  private static final Logger LOG = LoggerFactory
      .getLogger(SerializationUtils.class);

  public static <T extends Serializable> void serializeList(List<T> objects,
      String fileName) {
    // Assume List is Serializable
    // e.g., LinkedList or ArrayList
    if (objects instanceof java.io.Serializable) {
      SerializationUtils.serialize((Serializable) objects, fileName);
    } else {
      LOG.error("List is not serializable!");
    }
  }

  public static <T extends Serializable> void serialize(T object,
      String fileName) {
    try {
      if (object != null) {
        FileOutputStream fos = new FileOutputStream(fileName);
        ObjectOutputStream oos = new ObjectOutputStream(fos);
        oos.writeObject(object);
        oos.close();
        fos.close();
      }
      LOG.info("Serialized in " + fileName);
    } catch (FileNotFoundException fnfe) {
      LOG.error("FileNotFoundException: " + fnfe.getMessage());
    } catch (IOException ioe) {
      LOG.error("IOException: " + ioe.getMessage());
    }
  }

  public static <T extends Serializable> T deserialize(String fileName) {
    return deserialize(IOUtils.getInputStream(fileName));
  }

  public static <T extends Serializable> T deserialize(InputStream is) {
    T object = null;
    try {
      if (is != null) {
        ObjectInputStream ois = new ObjectInputStream(is);
        object = (T) ois.readObject();
        ois.close();
        is.close();
      }
    } catch (FileNotFoundException fnfe) {
      LOG.error("FileNotFoundException: " + fnfe.getMessage());
    } catch (IOException ioe) {
      LOG.error("IOException: " + ioe.getMessage());
    } catch (ClassNotFoundException c) {
      LOG.error("ClassNotFoundException: " + c.getMessage());
    }
    return object;
  }
}
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import libsvm.svm;
import libsvm.svm_model;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import at.illecker.storm.commons.Configuration;
import at.illecker.storm.commons.Dataset;
import at.illecker.storm.commons.postagger.ArkPOSTagger;
import at.illecker.storm.commons.preprocessor.Preprocessor;
import at.illecker.storm.commons.svm.featurevector.CombinedFeatureVectorGenerator;
import at.illecker.storm.commons.svm.featurevector.FeatureVectorGenerator;
import at.illecker.storm.commons.svm.scoreclassifier.IdentityScoreClassifier;
import at.illecker.storm.commons.svm.scoreclassifier.ScoreClassifier;
import at.illecker.storm.commons.tfidf.TfIdfNormalization;
import at.illecker.storm.commons.tfidf.TfType;
import at.illecker.storm.commons.tfidf.TweetTfIdf;
import at.illecker.storm.commons.tokenizer.Tokenizer;
import at.illecker.storm.commons.tweet.FeaturedTweet;
import at.illecker.storm.commons.tweet.Tweet;
import at.illecker.storm.commons.util.io.SerializationUtils;
import cmu.arktweetnlp.Tagger.TaggedToken;
import com.google.caliper.Benchmark;
import com.google.caliper.Param;
import com.google.caliper.api.Macrobenchmark;
import com.google.caliper.runner.CaliperMain;
/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package at.illecker.storm.commons.svm;






public class SVMCaliperBenchmark extends Benchmark {
  private static final Logger LOG = LoggerFactory
      .getLogger(SVMCaliperBenchmark.class);

  @Param({ "1", "2" })
  private int n; // number of threads

  private int inputCount = 1;

  private final Preprocessor m_preprocessor = Preprocessor.getInstance();
  private final ArkPOSTagger m_arkPOSTagger = ArkPOSTagger.getInstance();