public class LocEntityAnnotation {

  /**
   * Those are filled out after generating the loc entity.
   */
  private int toksStart, toksEnd;

  private Token[] tokens;

  private String NEType;

  /**
   * This is absolutely necessary.
   */
  private double latitude;

  private double longitude;



  public int getToksStart() {
    return toksStart;
  }

  public LocEntityAnnotation setToksStart(int toksStart) {
    this.toksStart = toksStart;
    return this;
  }

  public int getToksEnd() {
    return toksEnd;
  }

  public LocEntityAnnotation setToksEnd(int toksEnd) {
    this.toksEnd = toksEnd;
    return this;
  }

  public String getNEType() {
    return NEType;
  }

  public LocEntityAnnotation setNEType(String type) {
    this.NEType = type;

    for (int i = 0; i < tokens.length; i++) {
      tokens[i].setNE(type + "_" + (i == 0 ? "B" : "I"));
    }
    return this;
  }

  public Token[] getTokens() {
    return tokens;
  }

  public LocEntityAnnotation setTokens(Token[] token) {
    this.tokens = token;
    return this;
  }

  public double getLatitude() {
    return latitude;
  }

  public LocEntityAnnotation setLatitude(double latitude) {
    this.latitude = latitude;
    return this;
  }

  public double getLongitude() {
    return longitude;
  }

  public LocEntityAnnotation setLongitude(double longitude) {
    this.longitude = longitude;
    return this;
  }

  /**
   * tokStart, toke End, NEType, tokens are necessary. Geonames ID, lat and lon are not.
   * 
   * @param tokStart
   * @param tokEnd
   * @param NEType
   * @param tokens
   */
  public LocEntityAnnotation(int tokStart, int tokEnd, String NEType, Token[] tokens) {
    this.NEType = NEType;
    toksStart = tokStart;
    toksEnd = tokEnd;
    this.tokens = tokens;
    latitude = longitude = -999;
  }

  @Override
  public String toString() {
    String s = "[" + NEType + " : ";
    for (Token t : tokens)
      s += t.getToken() + " ";
    s += toksStart + "-" + toksEnd + " " + latitude + "," + longitude + "]";
    return s;
  }

  StringBuffer sb;

  public String getTokenString() {
    sb = new StringBuffer();
    for (Token t : tokens) {
      sb.append(t.getToken()).append(" ");
    }
    return sb.toString().trim();
  }

  @Override
  public boolean equals(Object lc) {
    if (lc instanceof LocEntityAnnotation) {
      LocEntityAnnotation le = (LocEntityAnnotation) lc;
      if (le.getTokenString().equals(this.getTokenString()) && le.toksStart == this.toksStart
              && le.toksEnd == this.toksEnd && le.NEType.equalsIgnoreCase(this.NEType))
        return true;
    }
    return false;
  }

  public static Comparator spanOrderComparator = new Comparator() {

    @Override
    public int compare(Object arg0, Object arg1) {
      LocEntityAnnotation span0 = (LocEntityAnnotation) arg0;
      LocEntityAnnotation span1 = (LocEntityAnnotation) arg1;
      if ((span0.toksStart - span0.toksEnd) > (span1.toksStart - span1.toksEnd))
        return -1;
      if ((span0.toksStart - span0.toksEnd) == (span1.toksStart - span1.toksEnd))
        return 0;
      else
        return 1;

    }

  };

  ArrayList<String> ids;
  
  public LocEntityAnnotation setGeoNamesId(ArrayList<String> ids){
    this.ids=ids;
    return this;
  }
  public LocEntityAnnotation setGeoNamesId(String id){
    if (this.ids==null)
    {
      ids = new ArrayList<String>();
      ids.add(id);
    }
    return this;
    
  }
  
  public ArrayList<String> getGeonamesIds() {
    // TODO Auto-generated method stub
    return ids;
  }
} public class Sentence {

  String sentenceString;

  Token[] tokens;

  String id;

  public Sentence(String s){
    this.setSentenceString(s);
  }
  public void setTokens(Token[] tokens) {
    this.tokens = tokens;
  }

  Dependency[] dependencies;

  public String getSentenceString() {
    return sentenceString;
  }

  public Sentence setSentenceString(String sentenceString) {
    this.sentenceString = sentenceString;
    return this;
  }

  public Token[] getTokens() {
    return tokens;
  }

  public String getId() {
    return id;
  }

  public Sentence setId(String sentenceId) {
    this.id = sentenceId;
    return this;
  }

  public Dependency[] getDependencies() {
    return dependencies;
  }

  public Sentence setDependencies(Dependency[] dependencies) {
    this.dependencies = dependencies;

    return this;
  }

  public int tokenLength() {
    if (tokens == null)
      return 0;
    return tokens.length;
  }

  public int charLength() {
    return sentenceString.length();
  }
  
  public String toString(){
    return "id is " + id + ", sentence string is : " + sentenceString + "\n tokens are : " + Arrays.asList(tokens);
  }
} public class Token implements Comparable<Token> {

  String token;

  /**
   * which token it is.
   */
  int position;

  String sentId;

  String tokenizerType;

  /**
   * use labels to store features and results.
   */
  String norm;

  String lemma;

  String POS;

  boolean inLocationGazetteer;

  boolean inNameGazetteer;

  String NE;

  /** 
   * Token string, sentence id, and position in the sentence.
   * @param token
   * @param sid
   * @param pos
   */
  public Token(String token,String sid, int pos){
    this.token=token;
    sentId = sid;
    position = pos;
  }
  public String getNE() {
    return NE;
  }

  public Token setNE(String nE) {
    NE = nE;
    return this;

  }

  public String getNorm() {
    return norm;
  }

  public Token setNorm(String norm) {
    this.norm = norm;
    return this;
  }

  public String getLemma() {
    return lemma;
  }

  public Token setLemma(String lemma) {
    this.lemma = lemma;
    return this;
  }

  public String getPOS() {
    return POS;
  }

  public Token setPOS(String pOS) {
    POS = pOS;
    return this;
  }

  public boolean isInLocationGazetteer() {
    return inLocationGazetteer;
  }

  public Token setInLocationGazetteer(boolean inLocationGazetteer) {
    this.inLocationGazetteer = inLocationGazetteer;
    return this;
  }

  public boolean isInNameGazetteer() {
    return inNameGazetteer;
  }

  public Token setInNameGazetteer(boolean inNameGazetteer) {
    this.inNameGazetteer = inNameGazetteer;
    return this;
  }

  public String getToken() {
    return token;
  }

  public Token setToken(String token) {
    this.token = token;
    return this;

  }

  public int getPosition() {
    return position;
  }

  public Token setPosition(int position) {
    this.position = position;
    return this;

  }

  public String getSentId() {
    return sentId;
  }

  public Token setSentId(String sentId) {
    this.sentId = sentId;
    return this;

  }

  public String getTokenizerType() {
    return tokenizerType;
  }

  public Token setTokenizerType(String tokenizerType) {
    this.tokenizerType = tokenizerType;
    return this;

  }

  public String toString() {
    return token + " : (TokenizerID:" + tokenizerType + ") (NEtype:" + NE + ") ( position:"
            + position + ")" + "(norm:" + norm + ") (lemma:" + lemma + " ) ( POS:" + POS + " ) ";
  }

  public int hashCode() {
    return token.hashCode();
  }

  @Override
  /**
   * This only compares tokens in a single sentence. Never use it on multiple sentences.
   */
  public int compareTo(Token o) {
    if (this.position == o.position)
      return 0;
    if (this.position > o.position)
      return 1;
    return -1;
  }

  /**
   * Equals only check tokens in a single sentence. Don't use it across sentences.
   */
  @Override
  public boolean equals(Object o) {
    if (o instanceof Token) {
      if (((Token) o).getToken().equals(this.getToken())
              && ((Token) o).getPosition() == this.getPosition())
        return true;
    }
    return false;
  }

  @Override
  /**
   * sentenceID, position should not be cloned, maybe.
   */
  public Token clone() {
    
    Token c = new Token(this.token,this.sentId,this.position);
    // it does not matter the string equal copies the ref of the string.
    
    /**
     * Don't clone the sentence id and position. Those has to be specificed later.
     */
    c.tokenizerType = new String(this.tokenizerType);
    c.norm = new String(this.norm);
    c.lemma = new String(this.lemma);
    c.POS = new String(this.POS);
    c.inLocationGazetteer = this.inLocationGazetteer;
    c.inNameGazetteer = this.inNameGazetteer;
    c.NE = new String(this.NE);
    return c;
  }
} public class Tweet {

  int createFrom; // 0 for null; -1 for non-status. 1 for status.

  String id;

  // backup reference to the status
  Status status;

  Sentence sentence;

  List<LocEntityAnnotation> toponyms;

  String userLocation;

  String timezone;

  String userDescription;

  String place, placeCountry, placeType; // city, admin, poi,neighborhood,country,

  GeoLocation[][] placeBounds;

  double latitude, longitude;

  /**
   * default constructor, all fields are null, except for lat and lon, which are -999.
   */
  public Tweet() {
    latitude = longitude = -999;
  }

  /**
   * constructor with a string. Other fields are defaults.
   * 
   * @param tweetStr
   */
  public Tweet(String tweetStr) {
    this();
    this.sentence = new Sentence(tweetStr);
    this.createFrom = -1;
    // the rest of the fields other than sentence and latitude and longitude has been automatically
    // assigned null;
  }

  /**
   * construct tweet from status. All the fields are filled out automatically.
   * 
   * @param status
   */
  public Tweet(Status status) {
    this();
    this.status = status;
    this.createFrom = 1;
    loadStatus();

  }

  public String getId() {
    return id;
  }

  /**
   * set ID only when tweet is not constructed from status.
   * 
   * @param id
   * @return
   */
  public Tweet setId(String id) {
    if (createFrom == 1){
      System.err.println("Tweet is Created from Status. ID not changable.");
      return this;
    }
    this.id = id;
    return this;
  }

  public Status getStatus() {
    return status;
  }

  /**
   * if the status is re-set, all the fields are changed.
   * 
   * @param status
   * @return
   */
  public Tweet setStatus(Status status) {
    this.status = status;
    loadStatus();
    return this;
  }

  private void loadStatus() {
    // TODO Auto-generated method stub
    this.id = "" + status.getId();
    this.sentence = new Sentence(status.getText());

    if (status.getUser() != null) {
      if (status.getUser().getLocation() != null)
        this.userLocation = status.getUser().getLocation();
      if (status.getUser().getDescription() != null)
        this.userDescription = status.getUser().getDescription();
      if (status.getUser().getTimeZone() != null)
        this.timezone = status.getUser().getTimeZone();
    }
    if (status.getGeoLocation() != null) {
      this.latitude = status.getGeoLocation().getLatitude();
      this.longitude = status.getGeoLocation().getLongitude();
    }
    if (status.getPlace() != null) {
      this.place = status.getPlace().getFullName();
      this.placeCountry = status.getPlace().getCountry();
      placeBounds = status.getPlace().getBoundingBoxCoordinates();
      this.placeType = status.getPlace().getPlaceType();
    }
  }

  public Sentence getSentence() {
    return sentence;
  }

  /**
   * set sentence only when tweet is not constructed from status.
   * 
   * @param sentence
   * @return
   */
  public Tweet setSentence(Sentence sentence) {
    if (createFrom == 1){
      System.err.println("Sentence is Created from Status. Sentence not changable.");
      return this;
    }
    this.sentence = sentence;
    return this;
  }
  
  public Tweet setSentence(String sentence){
    if (createFrom == 1){
      System.err.println("Sentence is Created from Status. Sentence not changable.");
      return this;
    }
    this.sentence = new Sentence(sentence);
    return this;
  }

  public Tweet setText(String text) {
    if (createFrom == 1){
      System.err.println("Text is Created from Status. ID not changable.");
      return this;
    }
    this.sentence = new Sentence(text);
    return this;
  }
  
  public String getText(){
    return this.sentence.getSentenceString();
  }

  public List<LocEntityAnnotation> getToponyms() {
    return toponyms;
  }

  public Tweet setToponyms(List<LocEntityAnnotation> toponyms) {
    this.toponyms = toponyms;
    return this;
  }

  public String getUserLocation() {
    return userLocation;
  }

  public Tweet setUserLocation(String userLocation) {
    if (createFrom == 1){
      System.err.println("UserLocation is Created from Status. ID not changable.");
      return this;
    }
    this.userLocation = userLocation;
    return this;
  }

  public String getTimezone() {
    return timezone;
  }

  public Tweet setTimezone(String timezone) {
    if (createFrom == 1){
      System.err.println("Timezone is Created from Status. ID not changable.");
      return this;
    }

    this.timezone = timezone;
    return this;
  }

  public String getUserDescription() {
    return userDescription;
  }

  public Tweet setUserDescription(String userDescription) {
    if (createFrom == 1){
      System.err.println("UserDescription is Created from Status. ID not changable.");
      return this;
    }

    this.userDescription = userDescription;
    return this;
  }

  /**
   * user information
   * 
   * @return
   */
  public boolean containsUserInfo() {
    if (this.userLocation.length() == 0 && this.userDescription.length() == 0)
      return false;
    else
      return true;
  }

  /**
   * user information
   * 
   * @return
   */
  public String getUserInfo() {
    return userLocation + " : " + userDescription;
  }

  public String getPlace() {
    return place;
  }

  public Tweet setPlace(String place) {
    if (createFrom == 1){
      System.err.println("Place is Created from Status. ID not changable.");
      return this;
    }

    this.place = place;
    return this;

  }

  public String getPlaceCountry() {
    return placeCountry;
  }

  public Tweet setPlaceCountry(String placeCountry) {
    if (createFrom == 1){
      System.err.println("PlaceCountry is Created from Status. ID not changable.");
      return this;
    }

    this.placeCountry = placeCountry;
    return this;
  }

  public GeoLocation[][] getPlaceBounds() {
    return placeBounds;
  }

  public Tweet setPlaceBounds(GeoLocation[][] placeBounds) {
    if (createFrom == 1){
      System.err.println("PlaceBounds is Created from Status. ID not changable.");
      return this;
    }

    this.placeBounds = placeBounds;
    return this;
  }

  public String getPlaceType() {
    return placeType;
  }

  public Tweet setPlaceType(String placeType) {
    if (createFrom == 1){
      System.err.println("PlaceType is Created from Status. ID not changable.");
      return this;
    }

    this.placeType = placeType;
    return this;
  }

  public double getLatitude() {
    return latitude;
  }

  public Tweet setLatitude(double latitude) {
    if (createFrom == 1){
      System.err.println("Latitude is Created from Status. ID not changable.");
      return this;
    }

    this.latitude = latitude;
    return this;
  }

  public double getLongitude() {
    return longitude;
  }

  public Tweet setLongitude(double longitude) {
    if (createFrom == 1){
      System.err.println("longitude is Created from Status. ID not changable.");
      return this;
    }

    this.longitude = longitude;
    return this;
  }

  public boolean containsTweetCoord() {
    if (this.latitude == -999 || this.longitude == -999)
      return false;
    return true;
  }

  public String getToponymsAsText() {
    // TODO Auto-generated method stub
    StringBuilder sb = new StringBuilder();
    for (LocEntityAnnotation e : this.toponyms)
      sb.append(e).append("\n");
    return sb.toString();
  }

  public HashSet<String> getAllIds() {
    // TODO Auto-generated method stub
    if (this.toponyms == null)
      return null;
    HashSet<String> ids = new HashSet<String>();
    for (LocEntityAnnotation topo : this.toponyms) {
      if (topo.getGeonamesIds() == null)
        continue;
      ids.addAll(topo.getGeonamesIds());
    }
    return ids;
  }

  public boolean containsTimezone() {
    // TODO Auto-generated method stub
    return !(timezone == null);
  }

  @Override
  public String toString() {

    return "tweet id: " + id + ", text : " + sentence;
  }

  @Override
  public int hashCode() {
    return id.hashCode();
  }

} public class NLPFactory {
  /**
   * @return the enPosTagger
   */
  public static POSTagger getEnPosTagger() {
    return ENTweetPOSTagger.getInstance();
  }

  public static Lemmatizer getEnUWStemmer() {
    return UWMorphaStemmer.getInstance();
  }

  public static POSTagger getEsPosTagger() {
    return ESAnnaPOSTagger.getInstance();
  }

  public static Lemmatizer getEsAnnaLemmatizer() {
    try {
      return AnnaLemmatizer.getInstance("es");
    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    return null;
  }
  
  public static Lemmatizer getEnAnnaLemmatizer() {
    try {
      return AnnaLemmatizer.getInstance("en");
    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    return null;
  }
} public class FeatureGenerator {

  HashSet<String> preposition, countries;

  Dictionary prepdict, countrydict;

  EuroLangTwokenizer tokenizer;

  Lemmatizer lemmatizer;

  POSTagger postagger;

  Index index;

  public Index getIndex() {
    return index;
  }

  @SuppressWarnings("unchecked")
  public FeatureGenerator(String language, Index index, String resourcepath) {
    // initialize dictionary to lookup.
    // "geoNames.com/allCountries.txt"
    this.index = index;

    if (language.equals("en") || language.equals("es"))
      tokenizer = new EuroLangTwokenizer();
    else
      System.err.println("No proper tokenizer found for this language.");

    if (language.equals("en"))
      // lemmatizer = new AnnaLemmatizer(resourcepath + language+
      // "/CoNLL2009-ST-English-ALL.anna-3.3.lemmatizer.model");
      lemmatizer = NLPFactory.getEnUWStemmer();
    else if (language.equals("es"))
      lemmatizer = new AnnaLemmatizer(resourcepath + language
              + "/CoNLL2009-ST-Spanish-ALL.anna-3.3.lemmatizer.model");
    if (language.equals("en"))
      postagger = NLPFactory.getEnPosTagger();
    else if (language.equals("es"))
      postagger = new ESAnnaPOSTagger(resourcepath + language
              + "/CoNLL2009-ST-Spanish-ALL.anna-3.3.postagger.model");
    try {
      prepdict = Dictionary.getSetFromListFile(resourcepath + language + "/prepositions.txt", true,
              true);
      preposition = (HashSet<String>) prepdict.getDic(DicType.SET);
    } catch (IOException e) {

      e.printStackTrace();
    }
  }

  static int statstreet = 0;

  static int statbuilding = 0;

  static int stattoponym = 0;

  static int statabbr = 0, statadj = 0;

  String tweet;

  public static void main(String argv[]) throws IOException, InterruptedException {
    // argv[0] = "es";
    // argv[1] = "GeoNames/allCountries.txt";
    argv[2] = "res/";

    String featuretype = "-3tok*.3pres.2cap.3caps.1pos.2pos.1gaz.1gazs.1cty.1ctys.-3-1prep";
    // ="ct-only";
    // ="allbutpos";
    FeatureGenerator fgen = new FeatureGenerator(argv[0], ResourceFactory.getClbIndex(),
            argv[2]);
    String traintest[] = new String[] { "train", "test" };
    for (int tt = 0; tt < 2; tt++) {
      CsvReader entries = new CsvReader("trainingdata/" + argv[0] + "NER/" + traintest[tt]
              + "/raw.csv", ',', Charset.forName("utf-8"));

      BufferedWriter fwriter = GetWriter.getFileWriter("trainingdata/" + argv[0] + "NER/"
              + traintest[tt] + "/" + featuretype + "-features.txt");

      entries.readHeaders();

      int i = 1;
      int counttweets = 0;
      while (entries.readRecord()) {
        i++;
        if (i % 200 == 0)
          System.out.println(i + " ");

        String tag = entries.get(entries.getHeaders()[0]);

        if (tag.equals("") == false)
          continue;// filter language
        else
          counttweets++; // count spanish tweets

        String tweet = entries.get(entries.getHeaders()[1]);// System.out.println(tweet);
        tweet = tweet.replace("-", " - ");
        String street = entries.get(entries.getHeaders()[2]);
        street = street.replace("-", " - ");
        String building = entries.get(entries.getHeaders()[3]);
        building = building.replace("-", " - ");
        String toponym = entries.get(entries.getHeaders()[4]);
        toponym = toponym.replace("-", " - ");
        String abbr = entries.get(entries.getHeaders()[5]);
        abbr = abbr.replace("-", " - ");
        // String locadj = entries.get(entries.getHeaders()[6]);
        // locadj = locadj.replace("-", " - ");

        String[] t_tweet = (EuroLangTwokenizer.tokenize(tweet)).toArray(new String[] {});
        String[] t_street = street.trim().split(",");// without
        // trimming
        String[] t_building = building.trim().split(",");// without
        // trimming
        String[] t_toponym = toponym.trim().split(",");// without
        // trimming
        String[] t_abbr = abbr.trim().split(",");// without
        // trimming
        // String[] t_locadj = locadj.trim().split(",");

        if (!EmptyArray(t_street))
          statstreet += t_street.length;
        if (!EmptyArray(t_building))
          statbuilding += t_building.length;
        if (!EmptyArray(t_toponym))
          stattoponym += t_toponym.length;
        if (!EmptyArray(t_abbr))
          statabbr += t_abbr.length;
        // if (!EmptyArray(t_locadj))
        // statadj += t_locadj.length;
        if (t_tweet.length == 0)
          continue;

        // labeling
        HashMap<Integer, String> f_loc = safeTag(t_tweet, t_street, t_building, t_toponym, t_abbr);

        Sentence sent = new Sentence(tweet);
        List<Feature[]> tweetfeatures = fgen.extractFeature(sent);

        for (int j = 0; j < tweetfeatures.size(); j++) {
          initialFeatureWriter();
          for (Feature f : tweetfeatures.get(j)) {
            append(f.toString());
            // System.out.println(f.toString());
          }

          // location class.
          String loctag = "O";
          if (f_loc.containsKey(j))
            loctag = f_loc.get(j);
          // loctag = "LOC";
          append(loctag);
          fwriter.write(emit());
        }

        fwriter.write("\n");
      }
      System.out.println(statstreet);
      System.out.println(statbuilding);
      System.out.println(stattoponym);
      System.out.println(statabbr);
      // System.out.println(statadj);
      System.out.println();
      System.out.println(counttweets);

      fwriter.close();
      entries.close();
    }
  }

  /**
   * MAIN FUNCTION FOR EXTRACTIN FEATURES
   * 
   * @param t_tweet
   * @param trie
   * @param postags
   * @return FEATURE LISTS
   */
  public List<Feature[]> extractFeature(Sentence tweetSentence) {
    // String[] t_tweet; //original input.
    int len = tweetSentence.tokenLength();
    List<List<Feature>> instances = new ArrayList<List<Feature>>(len);
    List<Feature> f = new ArrayList<Feature>();

    // normalize tweet norm_tweet
    for (int i = 0; i < len; i++)
      tweetSentence.getTokens()[i].setNorm(StringUtil
              .getDeAccentLoweredString(tokentype(tweetSentence.getTokens()[i].getToken())));
    // lemmatize norm field. store in lemma field.// originally lemmat_tweet;
    // stored in lemma field
    lemmatizer.lemmatize(tweetSentence);

    // pos tagging, originally postags. input is t_tweet
    // stored in pos field
    postagger.tag(tweetSentence);

    // String[] f_pos = postags.toArray(new String[] {});

    // f_gaz originally. filled in inGaz Field in token. check norm_tweet field.
    // boolean[] f_gaz =
    gazTag(tweetSentence, this.index);

    // use norm_tweet field to tag countries. don't remove f_country, because it's not a type in
    // token.
    boolean[] f_country = countryTag(tweetSentence);

    for (int i = 0; i < len; i++) {
      // clear feature list for this loop
      f = new ArrayList<Feature>();
      // /////////////////////////////// MORPH FEATURES
      // use lemma_tweet to get token features.
      genTokenFeatures(f, tweetSentence, i);
      // ////////////////////////////// SEMANTIC FEATURES
      genPosFeatures(f, tweetSentence, i);
      // ////////////////////////////////// GAZ AND DICT LOOK UP
      genGazFeatures(f, tweetSentence, i);
      // f7: STREET SUFFIX
      // f8 PREPOSITION

      genCountryFeatures(f, f_country, i);// f10: DIRECTION
      // f10 directions

      // FEATURES are not stored in tweetsentence in advance. Those are generated in those features.
      // use t_tweet to get cap.
      genCapFeatures(f, tweetSentence, i);

      // use t_tweet to generate preposition tags.
      genPrepFeatures(f, tweetSentence, i, preposition);

      // f9: COUNTRY
      // f11: DISTANCE
      // f12: STOPWORDS
      // f13: BUILDING
      instances.add(f);
    }

    // convert array to output format.
    ArrayList<Feature[]> newinstances = new ArrayList<Feature[]>();
    for (int i1 = 0; i1 < instances.size(); i1++) {
      newinstances.add(instances.get(i1).toArray(new Feature[] {}));
    }
    return newinstances;
  }

  // //////////////////////////////////////////////////////////////////////////////////
  // FEATURE EXTRACTORS
  // //////////////////////////////////////////////
  /**
   * PREPOSITION OR NOT.
   * 
   * INPUT RAW TOKENS OUTPUT BINARY VALUE YES OR NO.
   * 
   * @param f
   * @param t_tweet
   * @param i
   */
  // prep-2.prep-1
  private static void genPrepFeatures(List<Feature> f, Sentence sent, int i,
          HashSet<String> preposition) {
    // String[] t_tweet;

    if (i - 3 >= 0)
      addFeature(f,
              "-3_cont_prep_" + preposition.contains(TOKLW(sent.getTokens()[i - 3].getToken())));
    if (i - 2 >= 0)
      addFeature(f,
              "-2_cont_prep_" + preposition.contains(TOKLW(sent.getTokens()[i - 2].getToken())));
    if (i - 1 >= 0)
      addFeature(f,
              "-1_cont_prep_" + preposition.contains(TOKLW(sent.getTokens()[i - 1].getToken())));
  }

  /**
   * COUNTRY GAZ EXISTENCE
   * 
   * @param f
   * @param f_country
   * @param i
   */
  // country.-1.+1.seq-1+1
  private static void genCountryFeatures(List<Feature> f, boolean[] f_country, int i) {
    addFeature(f, "0_cont_country_" + f_country[i]);
    String countryseq = "";
    if (i - 1 >= 0) {
      addFeature(f, "-1_cont_country_" + f_country[i - 1]);
      countryseq += f_country[i - 1] + "::";
    }
    if (i + 1 <= f_country.length - 1) {
      addFeature(f, "+1_cont_country_" + f_country[i + 1]);
      countryseq += f_country[i + 1];
    }
    addFeature(f, "-+_cont_country_seq_" + countryseq);

  }

  /**
   * GAZ EXISTENCE
   * 
   * @param f
   * @param f_gaz
   * @param i
   */
  // gaz.-1.+1.seq-1+1
  private static void genGazFeatures(List<Feature> f, Sentence sent, int i) {
    // boolean[] f_gaz;
    // CURRENT WORD
    addFeature(f, "0_cont_gaz_" + sent.getTokens()[i].isInLocationGazetteer());

    String gazseq = "";
    if (i - 1 >= 0) {
      addFeature(f, "-1_cont_gaz_" + sent.getTokens()[i - 1].isInLocationGazetteer());
      gazseq += sent.getTokens()[i - 1].isInLocationGazetteer() + "::";
    }
    if (i + 1 <= sent.tokenLength() - 1) {
      addFeature(f, "+1_cont_gaz_" + sent.getTokens()[i + 1].isInLocationGazetteer());
      gazseq += sent.getTokens()[i + 1].isInLocationGazetteer();
    }
    addFeature(f, "-+_cont_gaz_seq_" + gazseq);
  }

  /**
   * POINT POS FOR EACH SURROUNDING WORD POS SEQUENCE
   * 
   * @param f
   * @param f_pos
   * @param i
   */
  // pos.seq-3-1.seq+1+3
  private static void genPosFeatures(List<Feature> f, Sentence twSent, int i) {
    // String[] f_pos;
    int t_length = twSent.tokenLength();
    // f5 PART OF SPEECH

    // CURRENT WORD
    addFeature(f, "0_pos_" + twSent.getTokens()[i].getPOS());

    String posleft = "", posright = "";
    if (i - 4 >= 0) {
      // addFeature(f, "-4.pos." + f_pos[i - 4]);
      // posleft += f_pos[i - 4];
    }
    if (i - 3 >= 0) {
      // addFeature(f, "-3.pos." + f_pos[i - 3]);
      // posleft += f_pos[i - 3];
    }
    if (i - 2 >= 0) {
      // addFeature(f, "-2_pos_" + f_pos[i - 2]);
      posleft += twSent.getTokens()[i - 2].getPOS();
    }
    if (i - 1 >= 0) {
      addFeature(f, "-1_pos_" + twSent.getTokens()[i - 1].getPOS());
      posleft += twSent.getTokens()[i - 1].getPOS();
    }
    if (i + 1 <= t_length - 1) {
      addFeature(f, "+1_pos_" + twSent.getTokens()[i + 1].getPOS());
      posright += twSent.getTokens()[i + 1].getPOS();
    }
    if (i + 2 <= t_length - 1) {
      // addFeature(f, "+2_pos_" + f_pos[i + 2]);
      posright += twSent.getTokens()[i + 2].getPOS();
    }
    if (i + 3 <= t_length - 1) {
      // addFeature(f, "+3.pos." + f_pos[i + 3]);
      // posright += f_pos[i + 3];
    }
    if (i + 4 <= t_length - 1) {
      // addFeature(f, "+4.pos." + f_pos[i + 4]);
      // posright += f_pos[i + 4];
    }
    addFeature(f, "-pos_seq_" + posleft);
    addFeature(f, "+pos_seq_" + posright);

  }

  /**
   * CAPITALIZATION SEQUENCE POINT CAPs OF SURROUNDING WORDS CAP SEQUENCEs
   * 
   * @param f
   * @param t_tweet
   * @param i
   */
  // cap.seq-3-1.seq+1+3
  private static void genCapFeatures(List<Feature> f, Sentence sent, int i) {
    // String[] t_tweet;
    int t_length = sent.tokenLength();

    // CURRENT WORD
    addFeature(f, "0_mph_cap_" + MPHCAP(sent.getTokens()[i].getToken()));

    String left = "", right = "";
    if (i - 4 >= 0) {
      // addFeature(f, "-4_mph_cap_" + MPHCAP(t_tweet[i - 4]));
      // left += MPHCAP(t_tweet[i - 4]);
    }
    if (i - 3 >= 0) {
      addFeature(f, "-3_mph_cap_" + MPHCAP(sent.getTokens()[i - 3].getToken()));
      // left += MPHCAP(t_tweet[i - 3]);
    }
    if (i - 2 >= 0) {
      addFeature(f, "-2_mph_cap_" + MPHCAP(sent.getTokens()[i - 2].getToken()));
      left += MPHCAP(sent.getTokens()[i - 2].getToken());
    }
    if (i - 1 >= 0) {
      addFeature(f, "-1_mph_cap_" + MPHCAP(sent.getTokens()[i - 1].getToken()));
      left += MPHCAP(sent.getTokens()[i - 1].getToken()) + "::";
    }
    if (i + 1 <= t_length - 1) {
      addFeature(f, "+1_mph_cap_" + MPHCAP(sent.getTokens()[i + 1].getToken()));
      right += MPHCAP(sent.getTokens()[i + 1].getToken());
    }
    if (i + 2 <= t_length - 1) {
      addFeature(f, "+2_mph_cap_" + MPHCAP(sent.getTokens()[i + 2].getToken()));
      right += MPHCAP(sent.getTokens()[i + 2].getToken());
    }
    if (i + 3 <= t_length - 1) {
      addFeature(f, "+3_mph_cap_" + MPHCAP(sent.getTokens()[i + 3].getToken()));
      // right += MPHCAP(t_tweet[i + 3]);
    }
    if (i + 4 <= t_length - 1) {
      // addFeature(f, "+4_mph_cap_" + MPHCAP(t_tweet[i + 4]));
      // right += MPHCAP(t_tweet[i + 4]);
    }
    addFeature(f, "-_mph_cap_seq_" + left);
    addFeature(f, "+_mph_cap_seq_" + right);
    addFeature(f, "-+_mph_cap_seq_" + left + right);

  }

  /**
   * CONTEXT WORD (LEMMA) EXISTENCE The bag of words feature, and position appearance feature
   * together. 1. Each lemma is added in bag of context words 2. Each position has an presence
   * feature for determining the existence of the window position.
   * 
   * @param f
   *          : Feature list
   * @param lemmat_tweet
   *          : lemmas of the tweet,
   * @param i
   *          : position of the current word
   */
  // tok.-1.+1.pres-4+4.
  private static void genTokenFeatures(List<Feature> f, Sentence sent, int i) {
    // String[] lemmat_tweet;
    // CURRENT TOKEN
    addFeature(f, "0_tok_lw_" + TOKLW(sent.getTokens()[i].getLemma()));
    if (i - 4 >= 0) {
      // addFeature(f, "-_tok_lw_" + TOKLW(lemmat_tweet[i - 4]));
      addFeature(f, "-4_tok_present_1");
    } else {
      addFeature(f, "-4_tok_present_0");
    }
    if (i - 3 >= 0) {
      addFeature(f, "-_tok_lw_" + TOKLW(sent.getTokens()[i - 3].getLemma()));
      addFeature(f, "-3_tok_present_1");
    } else {
      addFeature(f, "-3_tok_present_0");
    }
    // this feature has changed into bag of window words feature,
    // which is less specific than just the position.
    if (i - 2 >= 0) {
      addFeature(f, "-2_tok_lw_" + TOKLW(sent.getTokens()[i - 2].getLemma()));
      addFeature(f, "-2_tok_present_1");
    } else {
      addFeature(f, "-2_tok_present_0");
    }
    if (i - 1 >= 0) {
      addFeature(f, "-1_tok_lw_" + TOKLW(sent.getTokens()[i - 1].getLemma()));
      addFeature(f, "-1_tok_present_1");
    } else {
      addFeature(f, "-1_tok_present_0");
    }
    if (i + 1 <= sent.tokenLength() - 1) {
      addFeature(f, "+1_tok_lw_" + TOKLW(sent.getTokens()[i + 1].getLemma()));
      addFeature(f, "+1_tok_present_1");
    } else {
      addFeature(f, "+1_tok_present_0");
    }
    if (i + 2 <= sent.tokenLength() - 1) {
      addFeature(f, "+2_tok_lw_" + TOKLW(sent.getTokens()[i + 2].getLemma()));
      addFeature(f, "+2_tok_present_1");
    } else {
      addFeature(f, "+2_tok_present_0");
    }
    if (i + 3 <= sent.tokenLength() - 1) {
      addFeature(f, "+_tok_lw_" + TOKLW(sent.getTokens()[i + 3].getLemma()));
      addFeature(f, "+3_tok_present_1");
    } else {
      addFeature(f, "+3_tok_present_0");
    }
    if (i + 4 <= sent.tokenLength() - 1) {
      // addFeature(f, "+_tok_lw_" + TOKLW(sent.getTokens()[i+4].getLemma()));
      addFeature(f, "+4_tok_present_1");
    } else {
      addFeature(f, "+4_tok_present_0");
    }
  }

  /**
   * CAPITALIZATION
   * 
   * @param string
   * @return boolean
   */
  private static String MPHCAP(String string) {

    boolean a = Character.isUpperCase(string.charAt(0));
    return Boolean.toString(a);
  }

  /**
   * CONVERT TO LOWER TYPE Input the lemma, 1. Run tokentype() to convert to token 2. lowercase and
   * deaccent the lemma.
   * 
   * @param lemmastring
   * @return
   */
  private static String TOKLW(String lemmastring) {

    lemmastring = StringUtil.getDeAccentLoweredString(tokentype(lemmastring));
    return lemmastring;
  }

  /**
   * CONVERT TO TYPE Naively decide the tweet token type, url, or hashtag, or metion, or number. Or
   * it's not any of them, just return it's original string.
   * 
   * @param token
   * @return
   */
  public static String tokentype(String token) {
    // lower cased word.
    String ltoken = StringUtil.getDeAccentLoweredString(token.trim());

    if (ltoken.startsWith("http:") || ltoken.startsWith("www:")) {
      ltoken = "[http]";
    } else if (ltoken.startsWith("@") || ltoken.startsWith("#")) {
      if (ltoken.length() > 1) {
        ltoken = ltoken.substring(1);
      }
    }
    try {
      Double.parseDouble(ltoken);
      ltoken = "[num]";
    } catch (NumberFormatException e) {
    }

    return ltoken;
  }

  // ////////////////////////////////////////////////////////////////////////////////////
  // GAZ FEATURE HELPER
  // //////////////////////////////////////////////////////////
  /**
   * GAZ TAGGING BASED ON GREEDY SEARCH. FIND THE LONGEST MATCH STARTING FROM THE CURRENT WORD
   * 
   * @param t_tweet
   * @param trie
   * @return
   */
  private static Sentence gazTag(Sentence twSent, Index index) {
    int len = twSent.tokenLength();
    boolean[] gaztag = new boolean[twSent.tokenLength()];
    int i = 0;
    while (i < len) {
      String history = "";
      for (int j = i; j < len; j++) {
        history += twSent.getTokens()[j].getNorm();
        if(history.trim().length()==0)
          continue;
        if (index.inIndex(history)) {
          for (int k = i; k < j + 1; k++)
            gaztag[k] = true;
          // gaztag[j]=true;
        }
      }
      i++;
    }

    for (i = 0; i < gaztag.length; i++) {
      twSent.getTokens()[i].setInLocationGazetteer(gaztag[i]);
    }
    return twSent;
  }

  /**
   * COUNTRY TAGGING BASED ON GREEDY SEARCH FIND THE LONGEST MATCH STARTING FROM THE CURRENT WORD
   * 
   * @param t_tweet
   * @return
   */
  private static boolean[] countryTag(Sentence sent) {
    // String[] t_tweet;
    boolean[] countrytag = new boolean[sent.tokenLength()];
    int i = 0;
    while (i < sent.tokenLength()) {
      String history = "";
      for (int j = i; j < sent.tokenLength(); j++) {
        history += " " + StringUtil.getDeAccentLoweredString(sent.getTokens()[j].getNorm());
        // System.out.println(history);
        // System.out.println(ParserUtils.isCountry(history.trim()));
        if (ParserUtils.isCountry(history.trim())) {
          for (int k = i; k < j + 1; k++)
            countrytag[k] = true;
        }
      }
      i++;
    }
    return countrytag;
  }

  /**
   * HELPER FOR SAFELY TAGGING THE STRINGS
   * 
   * @param t_tweet
   * @param t_street
   * @param t_building
   * @param t_toponym
   * @param t_abbr
   * @param tk
   * @return
   */
  private static HashMap<Integer, String> safeTag(String[] t_tweet, String[] t_street,
          String[] t_building, String[] t_toponym, String[] t_abbr) {
    HashMap<Integer, String> tagresults = new HashMap<Integer, String>();
    if (!EmptyArray(t_toponym)) {
      fillinTag(t_tweet, t_toponym, tagresults, "TP");
    }
    if (!EmptyArray(t_street)) {
      fillinTag(t_tweet, t_street, tagresults, "ST");
    }
    if (!EmptyArray(t_building)) {
      fillinTag(t_tweet, t_building, tagresults, "BD");
    }
    if (!EmptyArray(t_abbr)) {
      fillinTag(t_tweet, t_abbr, tagresults, "AB");
    }
    return tagresults;
  }

  /**
   * AUXILARY FUNCTION FOR SAFETAG. FILL THE TAG IN t_location INTO THE tagresults hashmap.
   * 
   * @param t_tweet
   * @param t_street
   * @param tagresults
   * @param tk
   */
  private static void fillinTag(String[] t_tweet, String[] t_location,
          HashMap<Integer, String> tagresults, String TAG) {

    for (String location : t_location) {

      List<String> loctokens = EuroLangTwokenizer.tokenize(location);
      // if (TAG.equals("AB")) {
      // System.out.println("the original tweet tokenized is : " +
      // Arrays.asList(t_tweet).toString());
      // System.out.println("the location tokenized is :" +
      // loctokens.toString());
      // }
      for (String token : loctokens) {
        boolean have = false;
        String ntoken = StringUtil.getDeAccentLoweredString(token);
        for (int i = 0; i < t_tweet.length; i++) {
          if (StringUtil.getDeAccentLoweredString(
                  (t_tweet[i].startsWith("#") && t_tweet[i].length() > 1) ? t_tweet[i].substring(1)
                          : t_tweet[i]).equals(ntoken)) {
            tagresults.put(i, TAG);
            have = true;
          }
        }
        if (have == false)
          System.out.println("Don't have the tag: " + token);
      }
    }
    // if(TAG.equals("AB"))
    // System.out.println(tagresults);
  }

  // ///////////////////////////////////////////////////////////////////////////////////////////////////////
  // TOOLS
  // //////////////////////////////////
  /**
   * JUDGE EMPTY OF AN ARRAY.
   * 
   * @param array
   * @return
   */
  static boolean EmptyArray(String[] array) {
    if (array.length < 2)
      if (array[0].equals(""))
        return true;
    return false;
  }

  // ////////////////////////////////////////////////////////////////////////////////
  // HELPER FOR FEATURE VECTOR
  // /////////////////////////////////////////
  static StringBuilder sb = new StringBuilder();

  /**
   * helper for building feature vector. sb stores the features on a line, and this func is used to
   * initialize the sb, aka, clear the builder.
   */
  private static void initialFeatureWriter() {
    sb = new StringBuilder();
  }

  private static void append(String featurestring) {

    if (sb.length() > 0)
      sb.append("\t");
    sb.append(featurestring);
  }

  static String emit() {
    return sb.append("\n").toString();
  }

  private static void addFeature(List<Feature> features, String string) {

    features.add(new Feature(string));
  }

  // ////////////////////////////////////////////////////////////////////////////////////
  // GETTER AND SETTERS /////

  public HashSet<String> getPreposition() {
    return preposition;
  }

  public void setPreposition(HashSet<String> preposition) {
    this.preposition = preposition;
  }

  public HashSet<String> getCountries() {
    return countries;
  }

  public void setCountries(HashSet<String> countries) {
    this.countries = countries;
  }

  public EuroLangTwokenizer getTokenizer() {
    return tokenizer;
  }

  public void setTokenizer(EuroLangTwokenizer tokenizer) {
    this.tokenizer = tokenizer;
  }

  public Lemmatizer getLemmatizer() {
    return lemmatizer;
  }

  public void setLemmatizer(Lemmatizer lemmatizer) {
    this.lemmatizer = lemmatizer;
  }

  public POSTagger getPostagger() {
    return postagger;
  }

  public void setPostagger(POSTagger postagger) {
    this.postagger = postagger;
  }

} public class EuroLangTwokenizer {

  static Pattern Contractions = Pattern
          .compile("(?i)(\\w+)(['’′]t|['’′]ve|['’′]ll|['’′]d|['’′]re|['’′]s|['’′]m)$");

  static Pattern Whitespace = Pattern.compile("[\\s\\p{Zs}]+");

  // @ Wei Zhang
  // added question marks and exclamation mark, and long dash.
  // added ( and ) as punctuations
  static String punctChars = "['\"“”‘’\\|.—¿?¡!…,:;]";

  // static String punctSeq = punctChars+"+"; //'anthem'. => ' anthem '.
  static String punctSeq = "['\"“”‘’/_]+|[.¿?¡!,…]+|[:;]+"; // 'anthem'. =>

  // ' anthem '
  // .
  static String entity = "&(?:amp|lt|gt|quot);";

  // URLs

  // BTO 2012-06: everyone thinks the daringfireball regex should be better,
  // but they're wrong.
  // If you actually empirically test it the results are bad.
  // Please see https://github.com/brendano/ark-tweet-nlp/pull/9

  static String urlStart1 = "(?:https?://|\\bwww\\.)";

  static String commonTLDs = "(?:com|org|edu|gov|net|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|pro|tel|travel|xxx)";

  static String ccTLDs = "(?:ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|"
          + "bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|"
          + "er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|"
          + "hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|"
          + "lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|"
          + "nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sk|"
          + "sl|sm|sn|so|sr|ss|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|"
          + "va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw)"; // TODO:

  // remove
  // obscure
  // country
  // domains?
  static String urlStart2 = "\\b(?:[A-Za-z\\d-])+(?:\\.[A-Za-z0-9]+){0,3}\\." + "(?:" + commonTLDs
          + "|" + ccTLDs + ")" + "(?:\\." + ccTLDs + ")?(?=\\W|$)";

  // Wei Zhang added urlStart3 for HTTP://
  static String urlStart3 = "(?:HTTPS?://|\\bwww\\.)";

  // ////////

  static String urlBody = "(?:[^\\.\\s<>][^\\s<>]*?)?";

  static String urlExtraCrapBeforeEnd = "(?:" + punctChars + "|" + entity + ")+?";

  static String urlEnd = "(?:\\.\\.+|[<>]|\\s|$)";

  public static String url = "(?:" + urlStart1 + "|" + urlStart2 + "|" + urlStart3 + ")" + urlBody
          + "(?=(?:" + urlExtraCrapBeforeEnd + ")?" + urlEnd + ")";

  // Numeric
  static String timeLike = "\\d+(?::\\d+){1,2}";

  // static String numNum = "\\d+\\.\\d+";
  static String numberWithCommas = "(?:(?<!\\d)\\d{1,3},)+?\\d{3}" + "(?=(?:[^,\\d]|$))";

  static String numComb = "\\p{Sc}?\\d+(?:\\.\\d+)+%?";

  // Abbreviations
  static String boundaryNotDot = "(?:$|\\s|[“\\u0022?!,:;]|" + entity + ")";

  static String aa1 = "(?:[A-Za-z]\\.){2,}(?=" + boundaryNotDot + ")";

  static String aa2 = "[^A-Za-z](?:[A-Za-z]\\.){1,}[A-Za-z](?=" + boundaryNotDot + ")";

  static String standardAbbreviations = "\\b(?:[Mm]r|[Mm]rs|[Mm]s|[Dd]r|[Ss]r|[Jj]r|[Rr]ep|[Ss]en|[Ss]t)\\.";

  static String arbitraryAbbrev = "(?:" + aa1 + "|" + aa2 + "|" + standardAbbreviations + ")";

  static String separators = "(?:--+|―|—|~|–|=)";

  static String decorations = "(?:[♫♪]+|[★☆]+|[♥❤♡]+|[\\u2639-\\u263b]+|[\\ue001-\\uebbb]+)";

  static String thingsThatSplitWords = "[^\\s\\.,?\"\\-]";

  static String embeddedApostrophe = thingsThatSplitWords + "+['’′]" + thingsThatSplitWords
          + "*";

  public static String OR(String... parts) {
    String prefix = "(?:";
    StringBuilder sb = new StringBuilder();
    for (String s : parts) {
      sb.append(prefix);
      prefix = "|";
      sb.append(s);
    }
    sb.append(")");
    return sb.toString();
  }

  // Emoticons
  static String normalEyes = "(?iu)[:=]"; // 8 and x are eyes but cause

  // problems
  static String wink = "[;]";

  static String noseArea = "(?:|-|[^a-zA-Z0-9 ])"; // doesn't get :'-(

  static String happyMouths = "[D\\)\\]\\}]+";

  static String sadMouths = "[\\(\\[\\{]+";

  static String tongue = "[pPd3]+";

  static String otherMouths = "(?:[oO]+|[/\\\\]+|[vV]+|[Ss]+|[|]+)"; // remove

  // forward
  // slash
  // if
  // http://'s
  // aren't
  // cleaned

  // mouth repetition examples:
  // @aliciakeys Put it in a love song :-))
  // @hellocalyclops =))=))=)) Oh well

  static String bfLeft = "(♥|0|o|°|v|\\$|t|x|;|\\u0CA0|@|ʘ|•|・|◕|\\^|¬|\\*)";

  static String bfCenter = "(?:[\\.]|[_-]+)";

  static String bfRight = "\\2";

  static String s3 = "(?:--['\"])";

  static String s4 = "(?:<|&lt;|>|&gt;)[\\._-]+(?:<|&lt;|>|&gt;)";

  static String s5 = "(?:[.][_]+[.])";

  static String basicface = "(?:(?i)" + bfLeft + bfCenter + bfRight + ")|" + s3 + "|" + s4 + "|"
          + s5;

  static String eeLeft = "[＼\\\\ƪԄ\\(（<>;ヽ\\-=~\\*]+";

  static String eeRight = "[\\-=\\);'\\u0022<>ʃ）/／ノﾉ丿╯σっµ~\\*]+";

  static String eeSymbol = "[^A-Za-z0-9\\s\\(\\)\\*:=-]";

  static String eastEmote = eeLeft + "(?:" + basicface + "|" + eeSymbol + ")+" + eeRight;

  public static String emoticon = OR(
  // Standard version :) :( :] :D :P
          "(?:>|&gt;)?"
                  + OR(normalEyes, wink)
                  + OR(noseArea, "[Oo]")
                  + OR(tongue + "(?=\\W|$|RT|rt|Rt)", otherMouths + "(?=\\W|$|RT|rt|Rt)",
                          sadMouths, happyMouths),

          // reversed version (: D: use positive lookbehind to remove
          // "(word):"
          // because eyes on the right side is more ambiguous with the
          // standard usage of : ;
          "(?<=(?: |^))" + OR(sadMouths, happyMouths, otherMouths) + noseArea
                  + OR(normalEyes, wink) + "(?:<|&lt;)?",

          // inspired by
          // http://en.wikipedia.org/wiki/User:Scapler/emoticons#East_Asian_style
          eastEmote.replaceFirst("2", "1"), basicface
  // iOS 'emoji' characters (some smileys, some symbols) [\ue001-\uebbb]
  // TODO should try a big precompiled lexicon from Wikipedia, Dan Ramage
  // told
  // me (BTO) he does this
  );

  static String Hearts = "(?:<+/?3+)+"; // the other hearts are in

  // decorations

  static String Arrows = "(?:<*[-―—=]*>+|<+[-―—=]*>*)|\\p{InArrows}+";

  // BTO 2011-06: restored Hashtag, AtMention protection (dropped in
  // original
  // scala port) because it fixes
  // "hello (#hashtag)" ==> "hello (#hashtag )" WRONG
  // "hello (#hashtag)" ==> "hello ( #hashtag )" RIGHT
  // "hello (@person)" ==> "hello (@person )" WRONG
  // "hello (@person)" ==> "hello ( @person )" RIGHT
  // ... Some sort of weird interaction with edgepunct I guess, because
  // edgepunct
  // has poor content-symbol detection.

  // This also gets #1 #40 which probably aren't hashtags .. but good as
  // tokens.
  // If you want good hashtag identification, use a different regex.
  static String Hashtag = "#[a-zA-Z0-9_]+"; // optional: lookbehind for \b

  // optional: lookbehind for \b, max length 15
  static String AtMention = "[@＠][a-zA-Z0-9_]+";

  // I was worried this would conflict with at-mentions
  // but seems ok in sample of 5800: 7 changes all email fixes
  // http://www.regular-expressions.info/email.html
  static String Bound = "(?:\\W|^|$)";

  public static String Email = "(?<=" + Bound
          + ")[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,4}(?=" + Bound + ")";

  // We will be tokenizing using these regexps as delimiters
  // Additionally, these things are "protected", meaning they shouldn't be
  // further split themselves.
  static Pattern Protected = Pattern.compile(OR(Hearts, url, Email, timeLike,
          // numNum,
          numberWithCommas, numComb, emoticon, Arrows, entity, punctSeq, arbitraryAbbrev,
          separators, decorations, embeddedApostrophe, Hashtag, AtMention));

  // Edge punctuation
  // Want: 'foo' => ' foo '
  // While also: don't => don't
  // the first is considered "edge punctuation".
  // the second is word-internal punctuation -- don't want to mess with it.
  // BTO (2011-06): the edgepunct system seems to be the #1 source of
  // problems
  // these days.
  // I remember it causing lots of trouble in the past as well. Would be
  // good
  // to revisit or eliminate.

  // Note the 'smart quotes' (http://en.wikipedia.org/wiki/Smart_quotes)
  static String edgePunctChars = "'\"“”‘’«»{}\\(\\)\\[\\]\\*&"; // add

  // \\p{So}?
  // (symbols)
  static String edgePunct = "[" + edgePunctChars + "]";

  static String notEdgePunct = "[a-zA-Z0-9]"; // content characters

  // @Wei Zhang: Fixed bug ??(peru) ->?? ( peru ).
  // added ! and ? and ¿ and ¡.
  static String offEdge = "(^|$|:|;|!|\\?|¡|¿|\\s|\\.|,)"; // colon here

  // gets
  // "(hello):"
  // ==>
  // "( hello ):"
  static Pattern EdgePunctLeft = Pattern.compile(offEdge + "(" + edgePunct + "+)(" + notEdgePunct
          + ")");

  static Pattern EdgePunctRight = Pattern.compile("(" + notEdgePunct + ")(" + edgePunct + "+)"
          + offEdge);

  public static String splitEdgePunct(String input) {
    Matcher m1 = EdgePunctLeft.matcher(input);
    input = m1.replaceAll("$1$2 $3");
    m1 = EdgePunctRight.matcher(input);
    input = m1.replaceAll("$1 $2$3");
    return input;
  }

  private static class Pair<T1, T2> {
    public T1 first;

    public T2 second;

    public Pair(T1 x, T2 y) {
      first = x;
      second = y;
    }
  }

  // The main work of tokenizing a tweet.
  private static List<String> simpleTokenize(String text) {

    // Do the no-brainers first
    String splitPunctText = splitEdgePunct(text);

    int textLength = splitPunctText.length();

    // BTO: the logic here got quite convoluted via the Scala porting
    // detour
    // It would be good to switch back to a nice simple procedural style
    // like in the Python version
    // ... Scala is such a pain. Never again.

    // Find the matches for subsequences that should be protected,
    // e.g. URLs, 1.0, U.N.K.L.E., 12:53
    Matcher matches = Protected.matcher(splitPunctText);
    // Storing as List[List[String]] to make zip easier later on
    List<List<String>> bads = new ArrayList<List<String>>(); // linked
    // list?
    List<Pair<Integer, Integer>> badSpans = new ArrayList<Pair<Integer, Integer>>();
    while (matches.find()) {
      // The spans of the "bads" should not be split.
      if (matches.start() != matches.end()) { // unnecessary?
        List<String> bad = new ArrayList<String>(1);
        bad.add(splitPunctText.substring(matches.start(), matches.end()));
        bads.add(bad);
        badSpans.add(new Pair<Integer, Integer>(matches.start(), matches.end()));
      }
    }

    // Create a list of indices to create the "goods", which can be
    // split. We are taking "bad" spans like
    // List((2,5), (8,10))
    // to create
    // / List(0, 2, 5, 8, 10, 12)
    // where, e.g., "12" here would be the textLength
    // has an even length and no indices are the same
    List<Integer> indices = new ArrayList<Integer>(2 + 2 * badSpans.size());
    indices.add(0);
    for (Pair<Integer, Integer> p : badSpans) {
      indices.add(p.first);
      indices.add(p.second);
    }
    indices.add(textLength);

    // Group the indices and map them to their respective portion of the
    // string
    List<List<String>> splitGoods = new ArrayList<List<String>>(indices.size() / 2);
    for (int i = 0; i < indices.size(); i += 2) {
      String goodstr = splitPunctText.substring(indices.get(i), indices.get(i + 1));
      List<String> splitstr = Arrays.asList(goodstr.trim().split(" "));
      splitGoods.add(splitstr);
    }

    // Reinterpolate the 'good' and 'bad' Lists, ensuring that
    // additonal tokens from last good item get included
    List<String> zippedStr = new ArrayList<String>();
    int i;
    for (i = 0; i < bads.size(); i++) {
      zippedStr = addAllnonempty(zippedStr, splitGoods.get(i));
      zippedStr = addAllnonempty(zippedStr, bads.get(i));
    }
    zippedStr = addAllnonempty(zippedStr, splitGoods.get(i));

    // BTO: our POS tagger wants "ur" and "you're" to both be one token.
    // Uncomment to get "you 're"

    ArrayList<String> splitStr = new ArrayList<String>(zippedStr.size());
    for (String tok : zippedStr)
      splitStr.addAll(splitToken(tok));
    zippedStr = splitStr;

    return zippedStr;
  }

  private static List<String> addAllnonempty(List<String> master, List<String> smaller) {
    for (String s : smaller) {
      String strim = s.trim();
      if (strim.length() > 0)
        master.add(strim);
    }
    return master;
  }

  /** "foo   bar " => "foo bar" */
  public static String squeezeWhitespace(String input) {
    return Whitespace.matcher(input).replaceAll(" ").trim();
  }

  // Final pass tokenization based on special patterns
  private static List<String> splitToken(String token) {

    Matcher m = Contractions.matcher(token);
    if (m.find()) {
      String[] contract = { m.group(1), m.group(2) };
      // System.out.println(Arrays.asList(contract).toString());
      return Arrays.asList(contract);
    }
    String[] contract = { token };
    // System.out.println(Arrays.asList(contract).toString());

    return Arrays.asList(contract);
  }

  // @Wei Zhang
  // modified return type from list to string array
  /** Assume 'text' has no HTML escaping. **/
  public static List<String> tokenize(String text) {
    return simpleTokenize(squeezeWhitespace(text));
  }
/**
 * Take charge of generating tokens in a sentence. A raw sentence won't have tokens until it is tokenized.
 * @param sentence
 * @return
 */
  public static edu.cmu.geolocator.model.Sentence tokenize(edu.cmu.geolocator.model.Sentence sentence) {
    String sSent = sentence.getSentenceString();
    List<String> tokenized = tokenize(sSent);
    List<Token> tokens = new ArrayList<Token>();
    Token t;
    for (int i = 0; i < tokenized.size(); i++) {
      tokens.add(new Token(tokenized.get(i),sentence.getId(),i));
    }
    sentence.setTokens(tokens.toArray(new Token[]{}));
    return sentence;
  }

  public static void main(String[] args) throws IOException {
    BufferedReader br = new BufferedReader(new InputStreamReader(System.in, "utf-8"));
    System.out.print("> ");
    // Read user input
    String inputStr = br.readLine();
    while (!inputStr.equals("")) {

      // inputStr = "terremoto/earthquake";
      List<String> tokens = EuroLangTwokenizer.tokenize(inputStr);
      for (String token : tokens) {
        System.out.print("[" + token + "]");

      }
      System.out.print("\n> ");
      inputStr = br.readLine();
    }
    br.close();
  }

} public class ParserFactory {
  private static EnglishParser enparser;
  private static SpanishParser esparser;
  public static EnglishParser getEnAggrParser(){
    if(enparser==null)
      enparser = new EnglishParser(false);
    return enparser;
  }
  public static SpanishParser getEsAggrParser(){
    if(esparser==null)
      esparser = new SpanishParser(false);
    return esparser;
  }
  public static MTNERParser getEnNERParser() {
      try {
        return MTNERParser.getInstance("en");
      } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
      return null;
  }

  public static EnglishRuleSTBDParser getEnSTBDParser() {
      return EnglishRuleSTBDParser.getInstance();
  }
  
  public static EnglishRuleToponymParser getEnToponymParser() {
      return EnglishRuleToponymParser.getInstance();
  }

  public static MTNERParser getEsNERParser() {
    try {
      return MTNERParser.getInstance("es");
    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    return null;
}

public static SpanishRuleSTBDParser getEsSTBDParser() {
    return SpanishRuleSTBDParser.getInstance();
}

public static SpanishRuleToponymParser getEsToponymParser() {
    return SpanishRuleToponymParser.getInstance();
}

} public interface STBDParser {
	List<LocEntityAnnotation> parse(Tweet tweet);
} public class EnglishRuleSTBDParser implements STBDParser {

  /**
   * Default protected constructor
   */
  static String streetpattern = "\\$+[AN\\^G]+[N\\^G]";

  static String streetpattern2 = "[D]*[AN\\^G][N\\^G]";

  static String streetpattern3 = "[D]*[AN\\^G][AN\\^G][N\\^G]";

  static String streetpattern4 = "[D]*[AN\\^G][AN\\^G][AN\\^G][N\\^G]";

  static String[] stpattern = { streetpattern, streetpattern2, streetpattern3, streetpattern4 };

  static String buildingpattern = "([D]*[AN\\^G]+[N\\^G])";

  static Pattern streetpospattern;

  static Pattern buildingpospattern = Pattern.compile(buildingpattern);

  FeatureGenerator fgen;

  public EnglishRuleSTBDParser(FeatureGenerator fgen) {
    this.fgen = fgen;
  }

  List<LocEntityAnnotation> les;

  /**
   * parse street and buildings by rules, and put them into the tweet.
   * return them as well.
   * 
   * @param tweetMatches
   * @param tweet
   */
  public List<LocEntityAnnotation> parse(Tweet tweet) {
    les = new ArrayList<LocEntityAnnotation>();
    Sentence tweetSent = tweet.getSentence();
    EuroLangTwokenizer.tokenize(tweetSent);
    fgen.getPostagger().tag(tweetSent);

    String posstr = "";
    for (int i = 0; i < tweetSent.tokenLength(); i++)
      posstr += tweetSent.getTokens()[i].getPOS();

    for (int j = 0; j < stpattern.length; j++) {
      streetpospattern = Pattern.compile(stpattern[j]);
      Matcher stmatcher = streetpospattern.matcher(posstr);
      while (stmatcher.find()) {
        if (ParserUtils.isStreetSuffix(tweetSent.getTokens()[stmatcher.end() - 1].getToken())) {
          int startpos = stmatcher.start();
          int endpos = stmatcher.end() - 1;
          Token[] t = new Token[endpos - startpos + 1];
          for (int i = startpos; i <= endpos; i++) {
            t[i - startpos] = tweet.getSentence().getTokens()[i].setNE("st");
          }
          LocEntityAnnotation le = new LocEntityAnnotation(startpos, endpos, "st", t);
          les.add(le);
        }
      }
    }
    Matcher bdmatcher = buildingpospattern.matcher(posstr);
    while (bdmatcher.find()) {
      if (ParserUtils.isBuildingSuffix(tweetSent.getTokens()[bdmatcher.end() - 1].getToken())) {
        int startpos = bdmatcher.start();
        int endpos = bdmatcher.end() - 1;

        Token[] t = new Token[endpos - startpos + 1];
        for (int i = startpos; i <= endpos; i++) {
          t[i - startpos] = tweet.getSentence().getTokens()[i].setNE("bd");
        }
        LocEntityAnnotation le = new LocEntityAnnotation(startpos, endpos, "bd", t);
        les.add(le);
      }
    }
    tweet.setToponyms(les);
    return ParserUtils.ResultReduce(les, true);
  }

  private static EnglishRuleSTBDParser estbdparser;

  public static EnglishRuleSTBDParser getInstance() {
    if (estbdparser == null)
      estbdparser = new EnglishRuleSTBDParser(new FeatureGenerator("en",
              ResourceFactory.getClbIndex(), "res/"));
    return estbdparser;

  }

  public static void main(String argv[]) {
    Tweet t = new Tweet();
    String s = "RT @JenellaHerring: #centraltxfires Big Dog Rescue is evac. THEY NEED TRAILERS, TRUCKS & CRATES!!! shelter is located 589 Cool Water Dri ...";
    s = "Neal St. Park on fire damn man we breathing ashes";
    s = "I'm at Harrah's Louisiana Downs (8000 E Texas St, at I-220, Bossier City) http://t.co/EPvhlnd";
    s = "I'm at Trigon Bus Stop (Texas A&M University, College Station) http://t.co/xLlvSH9";
    s = "RT @ksatweather: Mandatory evacuations ordered for The Abbey Apts & Canyon Ridge Estates in the Stone Oak area. #TxWildfire";
    // s =
    // "MT @roxxsfisher: law officers asking spectators to please leave canyon ridge elem school and stone oak park for their safety. #satxwildfires";
    // s =
    // "Am worried about the #centraltxfires in Cedar Creek bc the Capitol of Texas Zoo is there. Lots of rescues & endangered animals. #pawcircle";
    t.setSentence(new Sentence(s));
    EnglishRuleSTBDParser stparser;
    for (int i = 0; i < 100; i++)
    {  stparser = ParserFactory.getEnSTBDParser();
    List<LocEntityAnnotation> locs = stparser.parse(t);
    System.out.println(locs);
    }
    

  }
} public class ParserUtils {

  public static void main(String argv[]) {
    System.out.println(ParserUtils.isEsFilterword("de"));
    List<String> matches = Arrays.asList(new String[] { "g{chile}g", "g{chile stock}g",
        "g{stock exchange}g" });
    System.out.println(ResultReduce(matches));
    System.out.println(ParserUtils.esdictset.size());
    System.out.println(ParserUtils.isESBuildingPrefix("Oficinas"));
  }

  static HashSet<String> StreetSuffix;

  static HashSet<String> BuildingSuffix;

  static HashSet<String> CountryNames;;

  static HashSet<String> ESStreetPrefix;

  static HashSet<String> ESBuildingPrefix;

  static HashSet<String> states;
  static {
    StreetSuffix = new HashSet<String>(Arrays.asList(new String[] { "street", "st", "st.", "lane",
        "ln", "ln.", "dr", "dri", "dr.", "boulevard", "blvd", "blvd.", "circle", "cir", "cir.",
        "cl", "cl.", "pl", "pl.", "avenue", "ave", "ave.", "square", "sq", "sq.", "road", "rd" }));
    BuildingSuffix = new HashSet<String>(Arrays.asList(new String[] { "downs", "barn", "rescue",
        "coop", "greenhouse", "roadhouse", "silo", "stable", "cellar", "mill", "loft", "house",
        "shed", "grainery", "watermill", "mill", "bar", "pub", "brothel", "casino", "centre",
        "shoppe", "center", "forum", "station", "store", "hotel", "motel", "market", "nightclub",
        "club", "office", "building", "restaurant", "cafe", "skyscraper", "shop", "mall",
        "warehouse", "supermarket", "college", "gym", "gymnasium", "school", "library", "museum",
        "theater", "amphitheater", "cinema", "symphony", "university", "capitol", "consulate",
        "courthouse", "embassy", "station", "palace", "park", "bridge", "tower", "hospital",
        "parliament", "station", "prison", "jail", "diner", "brewery", "factory", "foundry",
        "mine", "plant", "refinery", "rig", "barracks", "bunker", "castle", "citadel", "gate",
        "fort", "church", "bascilica", "cathedral", "duomo", "chapel", "oratory", "martyrium",
        "mosque", "mihrab", "masjid", "surau", "imambargah", "monastery", "temple", "pyramid",
        "shrine", "synagogue", "pagoda", "gurdwara", "airport", "port", "terminal", "marina",
        "stall", "town", "bldg", "centre", "abbey", "apartment", "aquarium", "armory", "asylum",
        "auditorium", "bakery", "barn", "bistro", "boathouse", "cabin", "camp", "cemetery",
        "city hall", "clinic", "concert hall", "convent", "convention center", "cottage", "depot",
        "grove", "headquarters", "homestead", "duomo", "elementary school", "inn", "landmark",
        "library", "lodge", "lounge", "mansion", "mausoleum", "medical center", "grainery",
        "greenhouse", "gurdwara", "gym", "observatory", "opera house", "orchard", "orphanage",
        "parsonage", "pier", "planetarium", "plantation", "pub", "quarter, district", "ranch",
        "resort", "saloon", "studio", "supermarket", "tavern", "motel", "museum", "club", "tomb",
        "town hall", "warehouse", "wharf", "winery", "zoo", "salon", "deli", "delicatessen",
        "shack", "tent", "slum", "ghetto", "hospice", "stable", "orchard", "estate",
        "conservatory", "spa", "discoteque", "cellar", "grocery", "vault", "basement", "den",
        "living room", "kitchen", "dining room", "bedroom", "bath", "attic", "hut", "shop",
        "neighborhood", "garage", "dormitory", "dressing room", "locker", "temple", "depot",
        "terminus", "convent", "college", "consulate", "hotel", "cafe", "cinemas", "hut", "corner",
        "lounge", "stadium", "labs", "village", "gardens", "university", "library", "planetarium",
        "foundation", "bridge", "avenue", "academy", "basilica", "church", "auditorium", "mall",
        "lighthouse", "supermarket", "estate", "sweets", "market", "sarees", "saris", "shop",
        "shoppe", "stationary", "dispensary", "pharmacy", "beach", "traders", "housesdairy",
        "drycleaners", "studios", "district", "enclave", "plaza", "stop", "apts", "estates",
        "university", "univ.", "univ", "u.", "preschool", "dam", "hosp" }));
    CountryNames = new HashSet<String>(Arrays.asList(new String[] { "England", "America", "Canada",
        "Al Urdun", "République Populaire Démocratique de", "Oceanía; Australasia", "Espagne",
        "Sahara Occidental", "Kazakstan", "Ukrayina", "Asia Central",
        "North Eastern South America", "Swaziland", "Islas Malvinas", "Turquie", "Falkland",
        "Cameroon", "Burkina Faso", "Northern Europe", "Serbian: Kosovo", "Panamá",
        "Nederlandse Antillen", "Etiopía", "Népal", "Uzbekistán", "Swazilandia",
        "Russian Federation", "Antártida", "Dominica", "Saint-Vincent-et-les Grenadines",
        "Indonésie", "Bhoutan", "Timor-Leste (East Timor)", "Shqiperia", "Region", "Soudan",
        "Îles Vierges des États-Unis", "Oceanía/Australasia", "Suomen Tasavalta", "Ethiopia",
        "East Timor Timor-Leste", "Cáucaso", "Local Name", "Ile de la Réunion", "Macao", "Turkey",
        "Falkland Islands", "Porto Rico", "Lubnan", "Sureste de América del Sur", "Bahrein",
        "Nueva Caledonia", "Grecia", "Nauru", "República Checa", "Norway", "Korea",
        "Wallis et Futuna", "Francia", "África Occidental", "Montenegro", "Nouvelle-Zélande",
        "Îles (Malvinas)", "Île Christmas", "Caribe", "Slovénie", "Santo Tomé y Príncipe",
        "El sur de América del Sur", "Libye", "Cayman Islands", "Malaisie", "Libya", "Finland",
        "Central African Republic", "Îles Cocos (Keeling)", "Norge", "Liechtenstein", "Sudáfrica",
        "Mauritius", "Dominique", "Portugal", "Cyprus", "Martinica", "Aomen", "Suriyah",
        "Letzebuerg", "Camboya", "Marianas del Norte", "Pitcairn", "Kuwait", "Hagere Ertra",
        "Mongol Uls", "Costa Rica", "y el Mediterráneo", "Guadalupe", "Nigeria", "Algérie",
        "Cocos (Keeling) Islands", "Australia", "Australie", "Pérou", "Tuvalu", "Libéria",
        "Antarctique", "Bielorrusia", "Belgique", "People's Democratic Republic", "Hong Kong",
        "Ityop'iya", "Dominicaine", "Brasil", "Oceanía", "Sak'art'velo", "Denmark", "Philippines",
        "Deutschland", "Inde", "Morocco", "Filastin", "Kypros", "Cap-Vert", "Sierra Leona",
        "Salomon", "Estonie", "Lietuva", "Federación Rusa", "Estonia", "Kosovo", "Sainte-Lucie",
        "Lebanon", "Comores", "Thaïlande", "Colombie", "Arabie Saoudite", "Colombia", "Eslovaquia",
        "Republique Togolaise", "México", "Palau", "Taiwan (Republic of China)", "Mocambique",
        "Aṣ-Ṣaḥrā’ al-Gharbīyah", "Dominicana", "Israël", "Érythrée", "Netherlands",
        "Samoa Américaines", "Suriname", "République Tchèque", "Suisse (French)", "Pologne",
        "Timor-Leste (Timor Oriental)", "Al Kuwayt", "République de", "Greater Antilles",
        "Grande-Bretagne", "Jordan", "République-Unie de", "Alemania", "Moldavia",
        "Isla De Navidad", "Eritrea", "Maritime Southeast Asia", "Pays-Bas", "South-Central Asia",
        "Sudeste Asiático", "Belice", "Jamhuri ya Muungano wa Tanzania", "Croatia", "Syria",
        "Guinéee", "Guine-Bissau", "Antarctic", "Oceania/Australia", "Palaos", "Byelarus",
        "Nigéria", "Mexico", "Serbie", "Serbia", "Surinam", "Bosnia-Herzegovina", "Ouzbékistan",
        "Greece", "Europa Oriental", "Albanian: : Kosova ou Kosovë",
        "Occidental de América del Sur", "Sao Tome and Principe", "Prathet Thai", "Han-guk",
        "Sudán", "Camerún", "Middle East", "East-África Central", "Bolivia", "África Central",
        "Bolivie", "Ghana", "Saudi Arabia", "Eastern Africa", "Magyarorszag", "American Samoa",
        "Antigua-et-Barbuda", "Guatemala", "Émirats Arabes Unis", "Al Yaman", "República", "Spain",
        "Al Jaza'ir", "Brunéi Darussalam", "Îles Marshall", "Amerika Sāmoa", "Oceania; Australia",
        "Allemagne", "Canadá", "Martinique", "Europa del Sur dentro de Roma", "Cabo Verde",
        "Gabon", "Niue", "Western Asia", "Turkménistan", "Singapour", "Antillas Holandesas",
        "Greenland", "Noreste de América del Sur", "Macau", "Corea del Sur",
        "United Arab Emirates", "India", "Lesotho", "República Centroafricana", "Fiyi", "Lettonie",
        "Kenya", "Birmania", "Central América del Sur", "Túnez", "Czech Republic", "Tailandia",
        "Svizzera (Italian)", "Taïwan", "Lesser Antilles", "Mongolie",
        "Democratic People's Rep. (North Korea)", "San Marino", "French Polynesia",
        "Republic of (South Korea)", "Bosna i Hercegovina", "Bermuda", "Groenland", "Peru",
        "Vaticano", "Zhong Guo", "Bharat", "Benin", "Arabian Peninsula",
        "North-Eastern South America", "Tanzania; officially the United Republic of Tanzania",
        "Chine", "Tíbet", "China", "Filipinas", "Azarbaycan", "Ukraine", "Tonga", "Western Sahara",
        "Papouasie-Nouvelle-Guinée", "Indonesia", "Bahreïn", "United States", "Guayana Francesa",
        "Argelia", "Hungría", "Mali", "Île Maurice", "Bulgarie", "Bulgaria", "Angola",
        "French Southern Territories", "Chad", "South Africa", "Tokelau", "Tajikistan",
        "Brunei Darussalam", "Norvège", "Senegal", "Polonia", "Uganda", "Líbano", "Hungary",
        "Niger", "Eastern Europe - Northern Asia", "Brazil", "Virgin Islands (U.S.)", "Косово",
        "Tobago", "Samoa Americana", "Islas Caimán", "La République Démocratique du",
        "Noroccidental de América del Sur", "Guadeloupe", "Iran", "Benín", "Algeria",
        "Status Civitatis Vaticanæ", "Ellas or Ellada", "Autriche", "Leeward Islands",
        "Corea del Norte", "Irán", "Marshall Islands", "Belgium", "Nederland/Holland", "Haiti",
        "Chypre", "Haïti", "Pakistán", "Sao Tomé-et-Principe", "Timor Oriental", "Gambia",
        "Éthiopie", "Gambie", "Letonia", "Guinea-Bissau", "Kiribati", "Isla Pitcairn", "Géorgie",
        "Iraq", "Reunión", "Antillas Mayores", "Equatorial Guinea", "Djibouti", "Japon",
        "Antigua and Barbuda", "Syrienne", "Nicaragua", "Slovenija", "Trinidad", "Koweït",
        "América del Norte", "Reino Unido", "Suiza", "Îles Mariannes du Nord", "Tanzania",
        "Rossiya", "Venezuela", "Polinesia Francesa", "Iceland", "Zambia", "Polynésie Française",
        "Western South America", "Zambie", "Republique Centrafricaine", "Germany",
        "Norte del América del Norte", "Kazakhstan", "Kyrgyzstan", "Corée", "Bermudes",
        "Ciudad del Vaticano", "Southern Africa", "Guyane", "Egypt", "Guyana", "Danemark",
        "English Name", "Egipto", "Burma", "Croatie", "Ucrania", "Qazaqstan", "Honduras",
        "Central America", "Eslovenia", "Antarctica", "Lituanie", "Lituania", "Namibia",
        "Western Europe", "Danmark", "Turkmenistan", "África del Sur", "Azerbaiyán", "Malasia",
        "États Fédérés de Micronésie", "Finlandia", "Reunion Island", "Fidji", "Libia",
        "Dhivehi Raajje", "Jumhurii Tojikiston", "Polinesia", "Mauricio", "Ouganda",
        "Îles Vierges Britanniques", "Japan", "Europa del Norte", "Central de América del Sur",
        "Republic of", "Bosnia and Herzegovina", "Éire", "Timor", "Lyoveldio Island", "Liberia",
        "Kibris", "Maldives", "Maritime South-East Asia", "Yisra'el", "dentro de Italia",
        "Tanzanie", "Schweiz (German)", "Christmas Island", "Viêt Nam", "Nippon", "Monaco",
        "Santa Lucía", "South-East Asia", "Europa del Oeste", "République", "Siria",
        "África Oriental", "Mediterranean", "Tadjikistan", "República Guinea", "Turks et Caïques",
        "Gran Bretaña", "Égypte", "Guinea Ecuatorial", "Afghanistan", "Caribbean", "Rumanía",
        "Tierras Australes y Antárticas Francesas", "Southern Europe", "France", "Vanuatu",
        "Malawi", "Équateur", "Bermudas", "parte de la Bahamas archipiélago.", "Europa del Sur",
        "Sudán del Sur", "Azerbaïdjan", "Virgin Islands", "Dominican Republic",
        "San Cristobal y Nevis", "Chipre", "Ceska Republika", "Oriente Medio", "Islande", "Grèce",
        "Islandia", "Granada", "Laos; oficialmente: República Democrática Popular Lao",
        "África Centraln Republic", "Países Bajos", "Sudeste Asiático y Oceanía",
        "Democratic Republic of the Congo (Kinshasa)", "Palestinian National Authority",
        "Sao Tome and Príncipe", "Malaysia", "parts of the Bahamas island chain.", "Islas Cocos",
        "République Islamique d' Iran", "Guinea", "Panama", "Maroc", "Guinee", "Luxembourg",
        "Cape Verde", "Bahamas", "Arabia Saudita", "Western Africa", "Andorre", "Gibraltar",
        "Ireland", "Groenlandia", "Nueva Zelanda", "Pitcairn Island", "Grandes Lagos de África",
        "Slovenia", "El Salvador", "République Démocratique du Congo", "East-Central Africa",
        "En el sur del Europa Oriental", "Papúa-Nueva Guinea", "Thailand", "Belize",
        "Sierra Leone", "Central South America", "Muritaniyah", "Georgia", "Sverige",
        "Southern Europe within Italy", "Poland", "North North America", "Aotearoa",
        "Islas Salomón", "Polynesia", "Australasia", "Royaume-Uni", "Southern South America",
        "Estados Unidos Mexicanos", "Guyane Française", "Al Arabiyah as Suudiyah",
        "San Vincente y Granadinas", "Negara Brunei Darussalam", "Kampuchea",
        "Europa Oriental - Northern Asia", "Uzbekistan", "Islas Feroe", "Dinamarca", "Taiwan",
        "Latvija", "Misr", "Barbados", "Cote d'Ivoire", "Madagascar", "Italy", "Sudan", "Kenia",
        "Iran (Islamic Republic of)", "Trinité-et-Tobago", "Micronesia", "Sáhara Occidental",
        "Central East South America", "Ruanda", "République du Congo", "Anguilla", "Holy See",
        "Israel", "Kirguistán", "Norte de África", "Papua New Guinea", "Zimbabwe", "Oman",
        "Península arábiga", "Belau", "Suecia", "Mauritania", "Mauritanie", "Kiribas", "Cameroun",
        "Fédération de", "Wallis y Futuna", "Nepal", "Crna Gora", "Trinidad and Tobago", "Latvia",
        "Myanma Naingngandaw", "Îles", "Costa de Marfil", "As-Sudan", "Federal States of",
        "Belgique/Belgie", "España", "Côte D'ivoire", "Libiyah", "United Kingdom",
        "Islas Marshall", "Congo", "Hong-Kong", "Central Eastern South America", "Paraguay",
        "Fiji", "Croacia", "Botswana", "Estados Unidos", "Papua Niu Gini", "Brésil",
        "Soudan du Sud", "Mónaco", "Hrvatska", "Lithuania", "Cambodia", "Asia Occidental",
        "Afghanestan", "The Gambia", "Autorité Nationale Palestinienne", "Aruba",
        "l'ex-République Yougoslave de", "Kalaallit Nunaat", "Province de Chine", "Argentina",
        "Argentine", "Bahrain", "Österreich", "y Asia Occidental", "Monténégro", "Northern Africa",
        "Saint-Siège (État de la Cité du Vatican)", "Al Bahrayn", "Bután", "République Arabe",
        "République Démocratique Populaire", "Finlande", "Holanda", "Islas Cook",
        "Islas Turcas y Caicos", "Trinidad y Tobago", "Jamaica", "República Democrática del Congo",
        "T'ai-wan", "Suisse", "French Guiana", "África austral", "New Zealand", "Yemen",
        "Melanesia", "Pakistan", "Albania", "Samoa", "Macédoine", "Albanie",
        "Terres Australes et Antarctiques Françaises", "Xianggang", "Viet Nam", "Nioué",
        "Emiratos Árabes Unidos", "Islas Virgenes Americanas", "Saint-Kitts-et-Nevis",
        "Saint Vincent and the Grenadines", "En el norte del América del Sur", "West Africa",
        "Kirghizistan", "Guam", "Oceania", "Choson", "Yémen", "Saltanat Uman", "Azerbaijan",
        "Italie", "Somalie", "Syrian Arab Republic", "Rwanda", "Italia", "Somalia", "Tchad",
        "Afganistán", "Cook Islands", "Jamaïque", "Gabón", "Cuba", "Turquía",
        "Antilles Néerlandaises", "Antigua y Barbuda", "Saint Kitts and Nevis", "Togo",
        "African Great Lakes", "Ecuador", "Armenia", "Marruecos", "Nouvelle-Calédonie",
        "Republica Oriental del Uruguay", "Montserrat", "Polska", "Îles Féroé", "Foroyar",
        "North America", "Sweden", "Vietnam", "Centrafricaine", "Russie", "Bod",
        "Slovakia (Slovak Republic)", "Liban", "Romania", "Antillas Menores",
        "Estados Federados de", "Eesti Vabariik", "Barbade", "Vatican City State (Holy See)",
        "Qatar", "Bosnie-Herzégovine", "Austria", "Al Maghrib", "Mozambique", "Hongrie", "Irlanda",
        "Malí", "Irlande", "Slovaquie", "Faroe Islands", "Omán", "Rep. of", "Hayastan",
        "Afrique du Sud", "Kazajstán", "Great Britain", "Ivory Coast", "Central Asia",
        "East Timor (Timor-Leste)", "Arménie", "Bangladesh", "Roumanie", "Belarus", "Mexique",
        "Dawlat Qatar", "Solomon Islands", "Yibuti", "Chile", "Puerto Rico", "Srbija", "Chili",
        "Republica", "Saint Lucia", "Sao Tome e Principe", "Noruega", "Namibie",
        "Terres Australes Françaises", "Guinée Équatoriale", "Tunis", "Mongolia",
        "Al Imarat al Arabiyah al Muttahidah", "Birmanie", "Switzerland", "Grenada", "Bélarus",
        "Grenade", "Bénin", "Seychelles", "Uruguay", "Sénégal", "Pilipinas", "Africa",
        "États-Unis", "Islas Virgenes Británicas", "Northern South America", "Slovensko",
        "Singapur", "French Polinesia", "Burundi", "Suède", "Maldivas", "Turks and Caicos Islands",
        "Republic of (Brazzaville)", "North West South America", "Bhutan", "Southeast Asia",
        "Malte", "Malta", "Asia Oriental", "Northern Mariana Islands", "América Central",
        "Isla Christmas", "Îles Cook", "Netherlands Antilles", "Lao", "Cambodge", "Tibet",
        "Macedonia", "Palestinian territories", "Kyrgyz Respublikasy", "Central Africa",
        "República del Congo", "Turkiye", "Japón", "Asia del Sur", "Bélgica", "Guinea Bissau",
        "Virgin Islands (British)", "Eastern Asia", "Mayotte", "New Caledonia", "Andorra",
        "Jordanie", "Perú", "Jordania", "South Sudan", "Guinée-Bissau", "Moldova", "Luxemburgo",
        "el Mediterráneo", "Wallis and Futuna Islands", "Santa Sede", "Myanmar", "Tunisia",
        "Saint-Marin", "Singapore", "Tunisie", "Palestina", "Comoros", "French Name", "Caïmanes",
        "Eastern Europe", "Makedonija", "Sri Lanka", "Southeastern Europe",
        "Uzbekiston Respublikasi", }));

    ESBuildingPrefix = new HashSet<String>(Arrays.asList(new String[] { "placa", "granero",
        "gallinero", "galpón", "invernadero", "garaje", "cochera", "estacionamiento", "estacion",
        "silo", "estableestablo", "bodega", "molino", "desván", "casa", "cobertizo", "granero",
        "molino", "cantina", "pub", "burdel", "casa de citas", "casino", "centro", "tienda",
        "centro", "foro", "estación", "almacén", "tienda", "hotel", "motel", "mercado",
        "plaza de mercado", "tianguis", "club", "club nocturno", "antro", "oficina", "edificio",
        "restaurante", "café", "cafetería", "rascacielos", "almacén", "tienda", "centro comercial",
        "mall", "almacén", "bodega", "supermercado", "universidad", "gimnasio", "gimnasio",
        "escuela", "universidad", "biblioteca", "museo", "teatro", "anfiteatro", "cine",
        "sinfonía", "universidad", "capitolio", "consulado", "palacio de justicia", "embajada",
        "estación", "palacio", "parque", "puente", "torre", "hospital", "parlamento", "estación",
        "prisión", "cárcel", "comedor", "cervecería", "fábrica", "maquila", "fundición", "mina",
        "planta", "refinería", "plataforma", "petrolera", "cuartel", "búnker", "castillo",
        "ciudadela", "iglesia", "basílica", "catedral", "domo", "capilla", "oratoria", "oratorio",
        "mezquita", "mihrab", "masjid", "surau", "monasterio", "templo", "pirámide", "santuario",
        "sinagoga", "pagoda", "gurdwara", "aeropuerto", "puerto", "terminal", "marina",
        "embarcadero", "malecón", "puesto", "ciudad", "pueblo", "edif", "centro", "granja",
        "rancho", "café", "cafetería", "corral", "escuela primaria", "escuela secundaria",
        "escuela preparatoria", "abadía", "apartamento", "acuario", "armería", "manicomio", "sala",
        "panadería", "granero", "or café", "cobertizo", "cabaña", "campamento", "cementerio",
        "ayuntamiento", "clínica", "sala de conciertos", "convento", "centro de convenciones",
        "casita", "depósito", "arboleda", "sede", "granja", "duomo", "escuela primaria",
        "escuela básica", "posada", "marca", "biblioteca", "logia", "salón", "mansión", "mausoleo",
        "centro médico", "granero", "invernadero", "vivero", "gurdwara", "gimnasio",
        "observatorio", "ópera", "huerta", "orfanato", "casa parroquial", "embarcadero",
        "planetario", "plantación", "pub", "barrio", "rancho", "resort", "bar", "estudio",
        "supermercado", "taberna", "motel", "museo", "club", "sepulcro", "ayuntamiento", "almacén",
        "muelle", "lagar", "zoo", "peluquería", "delicatessen", "delicatessen", "chabola", "carpa",
        "barrio bajo", "gueto", "hospicio", "establo", "huerta", "finca", "hacienda",
        "conservatorio", "spa", "disco", "bodega", "almacén", "cripta", "sótano", "bar",
        "balneario", "health resort", "taberna", "cantina", "edificio consistorial", "embarcadero",
        "bodega", "discoteca", "habitación", "ático", "choza", "tienda", "barriada", "dormitorio",
        "recámara", "armario", "cuarto de baño", "colegio", "bolsa", "viña", "u", "bosque",
        "comuna", "costanera", "metro", "villa", "clinica", "oficinas", "comercios" }));
    ESStreetPrefix = new HashSet<String>(Arrays.asList(new String[] { "calle", "cll", "cl",
        "boulevard", "blvd", "avenida", "plz", "calle", "carretera", "colonia", "col", "municipio",
        "mpio", "carrera", "cra", "cr", "dra", "dr", "paseo", "esquina", "esq", "calzada",
        "prolongacion", "delegacion", "fraccionamiento", "carrera", "ave", "av", "calle", "cll.",
        "cl.", "boulevard", "blvd.", "avenida.", "plaza.", "plz.", "calle.", "carretera.",
        "colonia.", "col.", "municipio.", "mpio.", "carrera.", "cra.", "cr.", "dra.", "dr.",
        "paseo.", "esquina.", "esq.", "calzada.", "prolongacion.", "delegación.",
        "fraccionamiento.", "carrera.", "ave.", "av.", "ruta", "pasaje" }));
    states = new HashSet<String>(Arrays.asList(new String[] { "brc", "bru", "wal", "vlg", "zh",
        "zg", "vd", "vs", "ur", "ti", "tg", "so", "sg", "sz", "sh", "ow", "nw", "ne", "lu", "ju",
        "gr", "gl", "ge", "fr", "be", "bs", "bl", "ar", "ai", "ag", "st", "sa", "no", "os", "wls",
        "sct", "nir", "eng", "gf", "pi", "sr", "sj", "tf", "tm", "um", "yg", "yn", "me", "ma",
        "ah", "cp", "as", "at", "dd", "ba", "an", "in", "mt", "dc", "de", "fl", "ga", "ks", "la",
        "md", "mo", "ms", "nc", "ok", "sc", "tn", "tx", "wv", "al", "ct", "ia", "il", "in", "me",
        "mi", "mn", "ne", "nh", "nj", "ny", "oh", "ri", "vt", "wi", "ca", "co", "nm", "nv", "ut",
        "az", "id", "mt", "nd", "or", "sd", "wa", "wv", "hi", "ak", "ky", "ma", "pa", "va" }));

  }

  static HashSet<String> lowerCountryNames, lowerESBuildingPrefix, dictset, esdictset;
  static {
    lowerCountryNames = new HashSet<String>();
    for (String s : CountryNames) {
      lowerCountryNames.add(StringUtil.getDeAccentLoweredString(s));
    }
    lowerESBuildingPrefix = new HashSet<String>();
    for (String s : ESBuildingPrefix)
      lowerESBuildingPrefix.add(StringUtil.getDeAccentLoweredString(s));
    Dictionary endict = null, esdict = null;
    try {
      endict = Dictionary.getSetFromListFile("res/en/words.filtered_SRC1000PlusCountryPlusStates.txt", true,
              false);
      esdict = Dictionary.getSetFromListFile("res/es/lradic.txt", true, false);
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    dictset = (HashSet<String>) endict.getDic(DicType.SET);
    esdictset = (HashSet<String>) esdict.getDic(DicType.SET);
  }

  // check the word is or is not a street suffix
  public static boolean isStreetSuffix(String word) {
    word = word.toLowerCase().trim();
    return StreetSuffix.contains(word);
  }

  public static boolean isBuildingSuffix(String word) {
    word = word.toLowerCase().trim();
    return BuildingSuffix.contains(word);
  }

  public static boolean isCountry(String word) {
    word = StringUtil.getDeAccentLoweredString(word);
    // System.out.println(word);
    return lowerCountryNames.contains(word);
  }

  public static boolean isESBuildingPrefix(String word) {
    word = StringUtil.getDeAccentLoweredString(word);
    return ESBuildingPrefix.contains(word) || BuildingSuffix.contains(word);

  }

  public static boolean isESStreetPrefix(String word) {
    word = StringUtil.getDeAccentLoweredString(word);
    return ESStreetPrefix.contains(word);

  }

  public static boolean isFilterword(String word) {
    word = word.toLowerCase().trim();
    for (String tok : word.split(" ")) {
      if (dictset.contains(tok) == false)
        return false;
    }
    return true;
  }

  public static boolean isEsFilterword(String word) {
    word = StringUtil.getDeAccentLoweredString(word);
    for (String tok : word.split(" ")) {
      if (esdictset.contains(tok) == false && dictset.contains(tok) == false)
        return false;
    }
    return true;
  }

  // check if all the part of speech includes in the "stop pos list".
  public static boolean containsStopPOS(String postags) {
    String[] taglist = postags.trim().split("[ ]");
    for (String tag : taglist) {
      if (isStopPOS(tag) == false) {
        return false;
      }
    }
    return true;
  }

  private static boolean isStopPOS(String tag) {
    // TODO Auto-generated method stub

    return tag.equals("V") || tag.equals("A") || tag.equals("R") || tag.equals("D")
            || tag.equals("P") || tag.equals("&") || tag.equals("T") || tag.equals("X")
            || tag.equals("Y") || tag.equals("~") || tag.equals("U") || tag.equals("E")
            || tag.equals("$") || tag.equals(",");
  }

  public static boolean hasNum(String str) {
    for (int i = 0; i < str.length(); i++) {
      Character c = str.charAt(i);
      if (Character.isDigit(c)) {
        return true;
      }
    }
    return false;
  }

  /*
   * returns a string from i1 to i2, inclusive, in the form [word[1] word[2] ...] word[n]
   */
  public static String getNewString(List<String> arr, int i1, int i2) {
    String newstr = "";
    for (int i = i1; i < i2; i++) {
      newstr += arr.get(i);
      newstr += " ";
    }
    newstr += " " + arr.get(i2);

    return newstr.replace("  ", " ");
  }

  /*
   * returns a string from i1 to i2, inclusive, in the form [word[1] word[2] ...] word[n]
   */
  public static String getString(List<String> arr, int i1, int i2) {
    String newstr = "[";
    for (int i = i1; i < i2; i++) {
      newstr += arr.get(i);
      newstr += " ";
    }
    newstr += "] " + arr.get(i2);

    return newstr;
  }

  public static List<String> ResultReduce(List<String> matches) {
    // TODO Auto-generated method stub
    ArrayList<String> newmatches = new ArrayList<String>();

    for (String match : matches)
      if (isSuperStrInSet(match, matches))
        continue;
      else
        newmatches.add(match);
    return newmatches;
  }

  public static List<LocEntityAnnotation> ResultReduce(List<LocEntityAnnotation> les, boolean t) {
    // TODO Auto-generated method stub
    Collections.sort(les, LocEntityAnnotation.spanOrderComparator);

    List<LocEntityAnnotation> nles = new ArrayList<LocEntityAnnotation>();
    for (int i = 0; i < les.size(); i++){
      boolean flag = false;
      for (int j = i + 1; j < les.size(); j++) {
        if (les.get(i).getToksStart() >= les.get(j).getToksStart()
                && les.get(i).getToksEnd() <= les.get(j).getToksEnd())
          flag=true;
      }
      if (!flag)
        nles.add(les.get(i));
    }
    return nles;
  }

  private static boolean isSuperStrInSet(String m, List<String> matches) {
    // TODO Auto-generated method stu
    for (String match : matches) {
      String nakedm = m.substring(3, m.length() - 3);
      if (match.contains(nakedm) && (nakedm.length() < match.length() - 6)) {
        // System.out.println(nakedm);
        return true;
      }
    }
    return false;
  }

  public static boolean isStateAbbreviation(String string) {
    // TODO Auto-generated method stub
    if (string.length() != 2 && string.length() != 3)
      return false;
    if (Character.isUpperCase(string.charAt(0)) || Character.isUpperCase(string.charAt(1)))
      return false;
    if (states.contains(string.toLowerCase()) == true)
      return true;
    return false;
  }

  public static boolean isallCap(Tweet tweet) {
    // TODO Auto-generated method stub
    String s = tweet.getSentence().getSentenceString();
    double caps, total;
    caps = 1;
    total = s.length();
    for (char c : s.toCharArray()) {
      if (Character.isUpperCase(c))
        caps += 1;
    }
    if (caps / total > 0.79)
      return true;
    else
      return false;
  }

} public class CollaborativeIndex implements Index {

  private IndexSearcher stringSearcher, infoSearcher;

  private HashSet<String> ids;

  private BooleanQuery q;

  private ArrayList<Document> returnDocs;

  private String stringIndexName, infoIndexName, stringLoad, infoLoad;

  private static CollaborativeIndex ci;

  public static CollaborativeIndex getInstance() {
    if (ci == null)
      ci = new CollaborativeIndex().config(GlobalParam.getGazIndex() + "/StringIndex",
              GlobalParam.getGazIndex() + "/InfoIndex", "mmap", "mmap").open();
    return ci;

  }

  public CollaborativeIndex config(String stringIndexName, String infoIndexName, String stringLoad,
          String infoLoad) {
    this.stringIndexName = stringIndexName;
    this.infoIndexName = infoIndexName;
    this.stringLoad = stringLoad;
    this.infoLoad = infoLoad;
    return this;
  }

  /*
   * public boolean inIndexStrict(String phrase) { if (phrase == null || phrase.length() == 0) throw
   * new NullPointerException(); TermQuery query = new TermQuery(new Term("LOWERED_ORIGIN",
   * phrase.toLowerCase())); //If it's an abbreviation, then return true; ignore case.
   * 
   * if (ResourceFactory.getCountryCode2CountryMap().isCountryAbbreviation(phrase)) return true;
   * TopDocs res = null; try { res = stringSearcher.search(query, 1); } catch (IOException e) { //
   * TODO Auto-generated catch block e.printStackTrace(); } if (res == null) return false; else
   * return res.totalHits > 0 ? true : false; }
   */
  /**
   * Check if the original string is in index. If not, check the non-space version. However, we have
   * to add some heuristics.
   */
  @Override
  public boolean inIndex(String phrase) {
    if (phrase == null || phrase.length() == 0)
      throw new NullPointerException();
    if (phrase.startsWith("#"))
      phrase = phrase.substring(1);
    if (ResourceFactory.getCountryCode2CountryMap().isInMap(phrase.trim().toLowerCase()))
      return true;
    phrase = phrase.toLowerCase().replace(" ", "");
    TermQuery query = new TermQuery(new Term("LOWERED-NO-WS", phrase));
    TopDocs res = null;
    try {
      res = stringSearcher.search(query, 1);
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    if (res == null)
      return false;
    return res.totalHits > 0 ? true : false;
  }

  @Override
  public ArrayList<Document> getDocumentsByPhrase(String phrase) {
    if (phrase == null || phrase.length() == 0)
      throw new NullPointerException();
    if (phrase.startsWith("#"))
      phrase = phrase.substring(1);
    TermQuery query = new TermQuery(
            new Term("LOWERED-NO-WS", phrase.toLowerCase().replace(" ", "")));
    TopDocs res = null;
    try {
      res = stringSearcher.search(query, 2500);
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    String abIds = null;
    if (ResourceFactory.getCountryCode2CountryMap().isInMap(phrase.toLowerCase()))
      abIds = ResourceFactory.getCountryCode2CountryMap().getValue(phrase.toLowerCase()).getId();

    // System.out.println(res.totalHits);
    if (res == null && abIds == null)
      return null;
    if (res.totalHits == 0 && abIds == null)
      return null;
    ids = new HashSet<String>();
    if (res != null)
      try {
        for (ScoreDoc doc : res.scoreDocs) {
          ids.add(stringSearcher.doc(doc.doc).get("ID"));
        }
      } catch (Exception e) {
        e.printStackTrace();
      }
    if (abIds != null)
      ids.add(abIds);
    // System.out.println(ids);
    // System.out.println("total number of String ids are:" + ids.size());
    q = new BooleanQuery();

    for (String id : ids) {
      q.add(new TermQuery(new Term("ID", id)), Occur.SHOULD);
    }
    // use a term filter instead of a query filter.
    try {
      TopDocs docs = infoSearcher.search(q, 2500);
      // System.out.println("total hits in info is:" + docs.totalHits);
      returnDocs = new ArrayList<Document>(docs.totalHits);
      for (ScoreDoc d : docs.scoreDocs) {
        returnDocs.add(infoSearcher.doc(d.doc));
      }
      return returnDocs;
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    return null;
  }

  /*
   * public ArrayList<Document> getDocumentsByPhraseStrict(String phrase) { if (phrase == null ||
   * phrase.length() == 0) throw new NullPointerException(); TermQuery query = new TermQuery(new
   * Term("LOWERED_ORIGIN", phrase.toLowerCase())); TopDocs res = null; try { res =
   * stringSearcher.search(query, 200); } catch (IOException e) { // TODO Auto-generated catch block
   * e.printStackTrace(); } System.out.println(res.totalHits);
   * 
   * if (res == null) return null; if (res.totalHits == 0) return null; ids = new
   * HashSet<String>(res.totalHits); try { for (ScoreDoc doc : res.scoreDocs) {
   * ids.add(stringSearcher.doc(doc.doc).get("ID")); } } catch (Exception e) { e.printStackTrace();
   * } if (ResourceFactory.getCountryCode2CountryMap().isCountryAbbreviation(phrase))
   * ids.add(ResourceFactory.getCountryCode2CountryMap().getValue(phrase).getId()); //
   * System.out.println("total number of String ids are:" + ids.size()); q = new BooleanQuery();
   * 
   * for (String id : ids) { q.add(new TermQuery(new Term("ID", id)), Occur.SHOULD); } // use a term
   * filter instead of a query filter. try { TopDocs docs = infoSearcher.search(q, 200); //
   * System.out.println("total hits in info is:" + docs.totalHits); returnDocs = new
   * ArrayList<Document>(docs.totalHits); for (ScoreDoc d : docs.scoreDocs) {
   * returnDocs.add(infoSearcher.doc(d.doc)); } return returnDocs; } catch (IOException e) { // TODO
   * Auto-generated catch block e.printStackTrace(); } return null; }
   */
  public CollaborativeIndex open() {
    try {
      stringSearcher = GetReader.getIndexSearcher(stringIndexName, stringLoad);
      // for setting the max clause count for search query.
      BooleanQuery.setMaxClauseCount(2500);

    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    try {
      infoSearcher = GetReader.getIndexSearcher(infoIndexName, infoLoad);
    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    return this;
  }

  @Override
  public void close() {
    // TODO Auto-generated method stub

  }

  public String[] getAlternateNames(String id) {
    HashSet<String> names = null;
    Query query = new TermQuery(new Term("ID", id));
    try {
      TopDocs topDocs = infoSearcher.search(query, 2500);
      names = new HashSet<String>(topDocs.totalHits);
      for (ScoreDoc doc : topDocs.scoreDocs) {
        names.add(stringSearcher.doc(doc.doc).get("LOWERED_ORIGIN"));
      }
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    return names.toArray(new String[names.size()]);
  }

  @Override
  public Document getDocumentsById(String id) {
    if (id == null || id.length() == 0)
      throw new NullPointerException();
    TermQuery query = new TermQuery(new Term("ID", id));
    TopDocs res = null;
    try {
      res = infoSearcher.search(query, 1);
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    if (res == null)
      return null;
    if (res.totalHits == 0)
      return null;
    try {
      return infoSearcher.doc(res.scoreDocs[0].doc);
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    return null;

  }

  public static void main(String argv[]) throws IOException {
    GlobalParam.setGazIndex("/users/indri/GazIndex");
    // GlobalParam.setGeoNames("GeoNames");
    CollaborativeIndex ci = ResourceFactory.getClbIndex();
    System.out.println(ci.getDocumentsByPhrase("newport").size());
    for (Document doc : ci.getDocumentsByPhrase("San Francisco"))

      System.out.println(doc.get(InfoFields.name) + " " + doc.get(InfoFields.countryCode) + " "
              + doc.get(InfoFields.adm1Code) + "\t" + doc.get(InfoFields.adm2Code) + "\t"
              + doc.get(InfoFields.alternativeNamesCount) + "\t" +doc.get(InfoFields.population)+"\t" +doc.get(InfoFields.id) + "\t"
              + doc.get(InfoFields.latitude) + "\t" + doc.get(InfoFields.longitude));
    if (true)
      return;
    boolean mode = true; // string
    // mode = false; // id
    /**
     * search string //id
     */

    BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
    String s = null;
    while ((s = br.readLine()) != null) {
      if (mode) {
        if (ResourceFactory.getClbIndex().inIndex(s))
          for (Document d : ResourceFactory.getClbIndex().getDocumentsByPhrase(s))
            System.out.println(d);
        else
          System.out.println("null.");
      } else {
        Document doc = ci.getDocumentsById(s);
        System.out.println(doc);
        CandidateAndFeature gc = new CandidateAndFeature(s, doc, new LocEntityAnnotation(0, 0, s,
                null));
        System.out.println(gc.getId() + " " + gc.getAsciiName() + " " + gc.getCountryCode());
      }
    }
  }
}
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.csvreader.CsvReader;
import edu.cmu.geolocator.model.LocEntityAnnotation;
import edu.cmu.geolocator.model.Sentence;
import edu.cmu.geolocator.model.Token;
import edu.cmu.geolocator.model.Tweet;
import edu.cmu.geolocator.nlp.NLPFactory;
import edu.cmu.geolocator.nlp.ner.FeatureExtractor.FeatureGenerator;
import edu.cmu.geolocator.nlp.tokenizer.EuroLangTwokenizer;
import edu.cmu.geolocator.parser.ParserFactory;
import edu.cmu.geolocator.parser.STBDParser;
import edu.cmu.geolocator.parser.english.EnglishRuleSTBDParser;
import edu.cmu.geolocator.parser.utils.ParserUtils;
import edu.cmu.geolocator.resource.gazindexing.CollaborativeIndex.CollaborativeIndex;
/**
 * 
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
 * 
 * @author Wei Zhang,  Language Technology Institute, School of Computer Science, Carnegie-Mellon University.
 * email: wei.zhang@cs.cmu.edu
 * 
 */
package edu.cmu.geolocator.parser.spanish;




public class SpanishRuleSTBDParser implements STBDParser {

  /**
   * Default protected constructor
   */
  static String streetpattern = "\\$+[an]+n";

  static String streetpattern2 = "[d]*[an]n";

  static String streetpattern3 = "[d]*[an][an]n";

  static String streetpattern4 = "[d]*[an][an][an]n";

  static String highway = "nz";

  static String av_street = "afvz";

  static String st_adj = "na";

  static String st_only = "ndn";

  static String calle = "nn[z]*|na[z]*";

  static String esquina = "ns[an]{0,2}";

  static String[] stpattern = { streetpattern, streetpattern2, streetpattern3, streetpattern4,
      highway, av_street, st_adj, st_only, calle, esquina };

  static String buildingpattern = "[d]*[an]+n";

  static String buildingpattern2 = "ns[an]+[na]";

  static String el_pattern = "[n]+dn";

  static String de_pattern = "nsn";

  static String de_sup_pattern = "nsdn";

  static String recurNS_de_pattern = "nsnsn[a]?";

  static String de_dot_pattern = "nfsn";

  static String buildingpattern3 = "[an]+";

  static String bd_adj = "na";

  static String[] bdpattern = { buildingpattern, buildingpattern2, el_pattern, recurNS_de_pattern,
      de_dot_pattern, buildingpattern3, de_pattern, bd_adj, de_sup_pattern };

  static Pattern streetpospattern;

  static Pattern buildingpospattern;

  
  ArrayList<LocEntityAnnotation> les;

  /**
   * Find streets by street suffixes
   * 
   * @param tweetMatches
   * @param tweet
   */
  public List<LocEntityAnnotation> parse(Tweet tweet) {
    les = new ArrayList<LocEntityAnnotation>();
    Sentence tweetSent = tweet.getSentence();
    EuroLangTwokenizer.tokenize(tweetSent);
    NLPFactory.getEsPosTagger().tag(tweetSent);

    String posstr = "";
    for (int i = 0; i < tweetSent.tokenLength(); i++)
      posstr += tweetSent.getTokens()[i].getPOS();

    for (int j = 0; j < stpattern.length; j++) {
      streetpospattern = Pattern.compile(stpattern[j]);
      Matcher stmatcher = streetpospattern.matcher(posstr);
      if (j == 8) {

      }
      while (stmatcher.find()) {
        if (ParserUtils.isESStreetPrefix(tweetSent.getTokens()[stmatcher.start()].getToken())) {
          int startpos = stmatcher.start();
          int endpos = stmatcher.end() - 1;
          Token[] t = new Token[endpos - startpos + 1];