public class ADRLuceneSearcher {
	static Directory indexDirectory;
	private static  String lucenePath;

	public static int topN=60;
	static{  
		
		setLucenePath(Setting.getValue("LuceneIndexFile")+"/adrProjectLexicon");
//		setLucenePath(Setting.getValue("LuceneIndexFile")+"/adrProjectLexiconSpell");
//		lucenePath = Setting.getValue("LuceneIndexFile")+"/adrProjectLexiconExactMatch";


	}
	
	
	public static void main(String[] args ) throws IOException, ParseException
	{	
		String test_sent =  "I gained an two cardiorespiratory arrests";
		DidYouMeanIndexer si = new DidYouMeanIndexer();
		test_sent = si.getTermByTermCorrectSpell(test_sent);
		
		test_sent= StringUtil.cleanString(test_sent);
		
			test_sent = StringUtil.removeStopWords(test_sent);
		
		
		
		String test = StringUtil.getTermByTermWordnet(test_sent);
//		ADRLexiconEntry ADR_le = getLexiconEntryByContent(test_sent);
		ArrayList<ADRLexiconEntry> ret = ADRLuceneSearcher.getTopRatedADRFromLuceneIndex(test,topN,Setting.getValue("LuceneIndexFile")+"/adrProjectLexicon");
//		int doc = getDocFreq("at",ADRCorpusIndex.getLucenePath());
//		System.out.println(doc);
	}

	public static ADRLexiconEntry getLexiconEntryByContent(String queryContent) throws ParseException {
		String adr_content= null;
		queryContent = queryContent.replaceAll("\\W", "");
		if (!queryContent.matches("\\w+")
				|| queryContent.isEmpty())
		{
			return null;
		}
		try {
			indexDirectory = FSDirectory.open(new File(getLucenePath()));
			
			
			QueryParser qp = new QueryParser(Version.LUCENE_36, "content", new StandardAnalyzer(Version.LUCENE_36));
			Query query = qp.parse("content:"+queryContent);
//			FuzzyQuery query = new FuzzyQuery(new Term("content", queryContent),0,2);
			 
			IndexReader reader = IndexReader.open(indexDirectory);
			
			IndexSearcher searcher = new IndexSearcher(reader);
			ScoreDoc[] hits = searcher.search(query, null, 1).scoreDocs;
//			TopScoreDocCollector collector = TopScoreDocCollector.create(1, true);
//			searcher.search(query,collector);
//			ScoreDoc[] hits = collector.topDocs().scoreDocs;
			if(hits.length>0)
			{
				int i=0;
				for (ScoreDoc h:hits)
				{
					float score = h.score;
					Document hitDoc = searcher.doc(hits[i].doc);
					
					adr_content = hitDoc.get("content");
					
					System.out.println(adr_content);
					System.out.println(score);
					String umls_id = hitDoc.get("umls_id");
//					if (score>=9.0)
//					{
						
						System.out.println(adr_content);
//					}
					i++;
				}
				Document hitDoc = searcher.doc(hits[0].doc);
				
				adr_content = hitDoc.get("content");
				String umls_id = hitDoc.get("umls_id");
				
				ADRLexiconEntry lexicon_entry = new ADRLexiconEntry(adr_content, umls_id);
				
				return lexicon_entry;
			}
				
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} 
//		catch (ParseException e) {
//			// TODO Auto-generated catch block
//			e.printStackTrace();
//		} 
		catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return null;
	}

	public static int getDocFreq(String token,String lecene_path) throws IOException
	{
		indexDirectory = FSDirectory.open(new File(lecene_path));
		
		token =StringUtil.getTermByTermWordnet(token);
		 
		IndexReader reader = IndexReader.open(indexDirectory);
		TermEnum termEnum = reader.terms(new Term("content", token));
		
		int doc_freq = termEnum.docFreq();
		
		return doc_freq;
	}

	public static String getLucenePath() {
		return lucenePath;
	}

	public static void setLucenePath(String lucenePath) {
		ADRLuceneSearcher.lucenePath = lucenePath;
	}
	
	public static ArrayList<ADRLexiconEntry>  getTopRatedADRFromLuceneIndex(String querystr, int hitsPerPage,
			String luceneIndexPath) { 
		ArrayList<ADRLexiconEntry> search_result = new ArrayList<>();
	    try { 
	    	
	      // Instantiate a query parser 
			QueryParser qp = new QueryParser(Version.LUCENE_36, "content", new StandardAnalyzer(Version.LUCENE_36,CharArraySet.EMPTY_SET ));
		  // Parse 
			Query query = qp.parse("content:"+querystr);
//			indexDirectory = FSDirectory.open(new File(Setting.getValue("LuceneIndexFile")+"/adrProjectLexicon"));
			indexDirectory = FSDirectory.open(new File(luceneIndexPath));
	      // Instantiate a searcher 
	  	 IndexReader reader = IndexReader.open(indexDirectory);
		
		 IndexSearcher searcher = new IndexSearcher(reader); 
	      // Ranker 
	      TopScoreDocCollector collector = TopScoreDocCollector.create( 
	          hitsPerPage, true); 
	      // Search! 
	      searcher.search(query, collector); 
	      // Retrieve the top-n documents 
	      ScoreDoc[] hits = collector.topDocs().scoreDocs; 
	 
	      // Display results 
//	      System.out.println("Found " + hits.length + " hits."); 
	      for (int i = 0; i < hits.length; ++i) { 
	        int docId = hits[i].doc; 
	        Document d = searcher.doc(docId); 
	        

			String adr_content = d.get("content");
			String umls_id = d.get("umls_id");
			ADRLexiconEntry lexicon_entry = new ADRLexiconEntry(adr_content, umls_id);
			search_result.add(lexicon_entry);
	        
	        System.out.println((i + 1) + ". " + d.get("content")); 
	      } 
	 
	      // Close the searcher 
	      searcher.close(); 
	    } catch (Exception e) { 
	      System.out.println("Got an Exception: " + e.getMessage()); 
	    } 
	    return search_result;
	  }
	public static boolean isPhraseInIndex(String querystr, int hitsPerPage, String indexPath) { 
		boolean is_in_index=false;
		
		
	    try { 
	    	
	      // Instantiate a query parser 
			QueryParser qp = new QueryParser(Version.LUCENE_36, "content", new StandardAnalyzer(Version.LUCENE_36));
		  // Parse 
			Query query = qp.parse("content:"+querystr);
			indexDirectory = FSDirectory.open(new File(indexPath));
		      // Instantiate a searcher 
		  	 IndexReader reader = IndexReader.open(indexDirectory);
			
			 IndexSearcher searcher = new IndexSearcher(reader); 
			 // Ranker 
			 TopScoreDocCollector collector = TopScoreDocCollector.create( 
	          hitsPerPage, true); 
			 // Search! 
			 searcher.search(query, collector); 
	      // Retrieve the top-n documents 
			 ScoreDoc[] hits = collector.topDocs().scoreDocs; 

		     for (int i = 0; i < hits.length; ++i) { 
		        int docId = hits[i].doc; 
		        Document d = searcher.doc(docId); 
		        String adr_content = d.get("content");
				String umls_id = d.get("umls_id");
				
		        is_in_index =true;

	//	        System.out.println((i + 1) + ". " + d.get("content")); 
		        break;
	      } 
	 
	      // Close the searcher 
	      searcher.close();
	      reader.close();
	      indexDirectory.close();
	      
	    } catch (Exception e) { 
	      System.out.println("Got an Exception: " + e.getMessage()); 
	    } 
	    return is_in_index;
	  }
	public static boolean isPhraseInIndexVerbatim(String querystr, int hitsPerPage, String indexPath) { 
		boolean is_in_index=false;
		
		
	    try { 
	    	
	      // Instantiate a query parser 
			QueryParser qp = new QueryParser(Version.LUCENE_36, "content", new StandardAnalyzer(Version.LUCENE_36));
		  // Parse 
			Query query = qp.parse("content:"+querystr);
			indexDirectory = FSDirectory.open(new File(indexPath));
	      // Instantiate a searcher 
	  	 IndexReader reader = IndexReader.open(indexDirectory);
		
		 IndexSearcher searcher = new IndexSearcher(reader); 
	      // Ranker 
	      TopScoreDocCollector collector = TopScoreDocCollector.create( 
	          hitsPerPage, true); 
	      // Search! 
	      searcher.search(query, collector); 
	      // Retrieve the top-n documents 
	      ScoreDoc[] hits = collector.topDocs().scoreDocs; 
	 
	      // Display results 
	      System.out.println("Found " + hits.length + " hits."); 
	      boolean is_in_index_with_all_tokens = true;
	      for (int i = 0; i < hits.length; ++i) { 
	        int docId = hits[i].doc; 
	        Document d = searcher.doc(docId); 
	        String adr_content = d.get("content");
	        String[] adr_content_tokens =adr_content.split(" ");
	        for (String token:adr_content_tokens)
	        {
	        	if (!querystr.matches(".*"+token+".*"))
	        	{
	        		is_in_index_with_all_tokens=false;
	        		break;
	        	}
	        }
			if (!is_in_index_with_all_tokens)
			{
				continue;
			}
	        is_in_index =true;

	        
	        System.out.println((i + 1) + ". " + d.get("content")); 
	        break;

	      } 
	 
	      // Close the searcher 
	      searcher.close(); 
	    } catch (Exception e) { 
	      System.out.println("Got an Exception: " + e.getMessage()); 
	    } 
	    return is_in_index;
	  }
	
} public class DidYouMeanIndexer {
	String correctText;
	public HashMap<String, String> lemma_correct_wrong_spell_map = new HashMap<>();

    public static void main (String[] args) throws IOException, ParseException
    {
    	DidYouMeanIndexer sc = new DidYouMeanIndexer();
    	String correct = sc.getTermByTermCorrectSpell("diarrhea");
    	System.out.println(correct);
    }
    public static void createSpellIndex(String spellIndexDirectoryPath,String sourceDirectoryPath) throws IOException {

        Directory sourceDirectory = FSDirectory.open(new File(sourceDirectoryPath));
        
        Directory spellIndexDirectory = FSDirectory.open(new File(spellIndexDirectoryPath));
        
        SpellChecker spellChecker = new SpellChecker(spellIndexDirectory);
        IndexReader reader = IndexReader.open(sourceDirectory);
        
       
        LuceneDictionary dictionary = new LuceneDictionary(reader, "content");
        
        IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LUCENE_36, new StandardAnalyzer(Version.LUCENE_36,CharArraySet.EMPTY_SET));
        spellChecker.indexDictionary(dictionary, indexWriterConfig, true);
           
        spellChecker.close();
    }

    public static void search(String queryString) throws IOException, ParseException {
        long startTime = System.currentTimeMillis();
        IndexSearcher is = null;
        try {
        	
        	Directory indexDirectory = FSDirectory.open(new File(Setting.getValue("LuceneIndexFile")+"/adrProjectLexicon"));
			
			
			QueryParser qp = new QueryParser(Version.LUCENE_36, "content", new StandardAnalyzer(Version.LUCENE_36));
			
			Query query = qp.parse("content:"+queryString);
			 
			IndexReader reader = IndexReader.open(indexDirectory);
			
			IndexSearcher searcher = new IndexSearcher(reader);
           
            qp.setDefaultOperator(QueryParser.AND_OPERATOR);
            
            ScoreDoc[] hits = searcher.search(query,1).scoreDocs;
           
//            Hits hits = searcher.search(query);
            long endTime = System.currentTimeMillis();
            
        } finally {
            if (is != null) {
                is.close();
            }
        }
    }
    static String luceneDir = System.getProperty("user.dir")+"/LuceneIndexes/EnglishSpellIndex";
    static Directory spell_directory;
    static IndexReader reader;
    static SpellChecker spellChecker;
    static{
    	 try {
			spell_directory = FSDirectory.open(new File(luceneDir));
			reader = IndexReader.open(spell_directory);
			spellChecker= new SpellChecker(spell_directory);
	        spellChecker.setStringDistance(new JaroWinklerDistance());
		} catch (IOException e) {
			//TODO
			System.out.println(e.getMessage());
		}
    }
	
	
    public static String getCorrectSpell(String wordForSuggestions ) throws IOException
    {
    	String correct_spell = wordForSuggestions;
    	
    	if (StringUtil.isStopWord(wordForSuggestions))
    	{
    		return correct_spell;
    	}
  
 
        
        if (!spellChecker.exist(wordForSuggestions))
        {
//        	String[] suggestions = spellChecker.suggestSimilar(wordForSuggestions, 1,reader,"content" ,SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX);
        	String[] suggestions = spellChecker.suggestSimilar(wordForSuggestions, 5,0.7f);
        	if (suggestions!=null && suggestions.length>0) {
                 for (String word : suggestions) {
                	 if (!(word.startsWith(String.valueOf(wordForSuggestions.charAt(0)))) )
                	 {
                		 continue;
                	 }
                	 correct_spell = word;
                	 break;
//                	 System.out.println("suggestions found for word:"+correct_spell);
                 }
             }
             else {
                 System.out.println("No suggestions found for word:"+wordForSuggestions);
             }
        }
       
    	return correct_spell;
    }
    public String getTermByTermCorrectSpell(String phrase) throws IOException
    {
    	
    	phrase = phrase.toLowerCase();
    	
    	String[] words = phrase.split(" ");
		String correctSpellString = "";
		for(int i=0;i<words.length;i++)
		{
			
			String correct = getCorrectSpell(words[i]);
			lemma_correct_wrong_spell_map.put(StringUtil.getTermByTermWordnet(correct), StringUtil.getTermByTermWordnet(words[i]));
			correctSpellString = correctSpellString.concat(correct+" ");
		}
		
		correctText = correctSpellString.trim();
		//%%% This is an special handling for n't
		if (phrase.matches("n't"))
			return phrase;
		return correctText;
    }
    

    

  
} @Entity
@Table( name = "Artifact" )
public class Artifact {
	String content;
	String generalized_content;
	List<Artifact> childsArtifact;
	Artifact parentArtifact;
	Artifact nextArtifact;
	Artifact previousArtifact;
	Type artifactType;
	private boolean forTrain;
	private boolean forDemo=false;
	
	
	
	@Transient
	boolean isNew;
	@Transient
//	List<ArtifactFeature> features = new ArrayList<ArtifactFeature>();
	
	String POS;
	
	// These two variable are lazy and will be initialized in getter
	Integer startIndex = null;
	Integer endIndex = null;

	Integer lineOffset = null;
	Integer wordOffset = null;
	
	String stanDependency =null;
	private String stanPennTree = null;
	
	int artifactId = -1;
	/*
	 * Physical address of file which artifact loaded from that
	 */
	String associatedFilePath;
	String corpusName;

	public enum Type {
		Document, Sentence, Phrase, Word
	}

	
	public Artifact()
	{
		
	}
	/**
	 * Loads Artifact by id
	 * @param pArtifactID
	 * @return
	 */
	public static Artifact getInstance(int pArtifactID) {
		String hql = "from Artifact where artifactId = "+pArtifactID;
		Artifact artifact_obj = 
			(Artifact)HibernateUtil.executeReader(hql).get(0);
		return artifact_obj;
	}
	/**
	 * Creates completely empty Artifact object
	 * @return
	 */
	public static Artifact getInstance(Type pArtifactType) {
		Artifact artifact_obj = new Artifact();
		artifact_obj.setArtifactType(pArtifactType);
		HibernateUtil.save(artifact_obj);
		return artifact_obj;
	}
	
	/**
	 * Loads or creates the Artifact
	 * @param pArtifactType
	 * @param pFilePath
	 * @param pStartIndex
	 * @return
	 */
	public static Artifact getInstance(Type pArtifactType, String pFilePath, 
			int pStartIndex){
		String hql = "from Artifact where artifactType = "+
			pArtifactType.ordinal()+" and associatedFilePath ='" +
			pFilePath + "' and startIndex="+pStartIndex;
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	artifact_obj.setStartIndex(pStartIndex);
	    	artifact_obj.setAssociatedFilePath(pFilePath);
	    	artifact_obj.setArtifactType(pArtifactType);
	    	artifact_obj.setForTrain(Setting.TrainingMode);
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact getInstance(Type pArtifactType, String pFilePath, 
			int pStartIndex,String corpusName){
		String hql = "from Artifact where artifactType = "+
			pArtifactType.ordinal()+" and associatedFilePath ='" +
			pFilePath + "' and corpusName= '"+corpusName+"' and startIndex="+pStartIndex;
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	artifact_obj.setStartIndex(pStartIndex);
	    	artifact_obj.setAssociatedFilePath(pFilePath);
	    	artifact_obj.setArtifactType(pArtifactType);
	    	artifact_obj.setForTrain(Setting.TrainingMode);
	    	artifact_obj.setCorpusName(corpusName);
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact getInstance(Type pArtifactType, String pFilePath, 
			int pStartIndex,boolean forDemo){
		String hql = "from Artifact where artifactType = "+
			pArtifactType.ordinal()+" and associatedFilePath ='" +
			pFilePath + "' and startIndex="+pStartIndex+" and forDemo="+forDemo;
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	artifact_obj.setStartIndex(pStartIndex);
	    	artifact_obj.setAssociatedFilePath(pFilePath);
	    	artifact_obj.setArtifactType(pArtifactType);
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact getInstance(int pWordIndex,Type pArtifactType, String pFilePath
			){
		String hql = "from Artifact where artifactType = "+
			pArtifactType.ordinal()+" and associatedFilePath ='" +
			pFilePath + "' and wordIndex="+pWordIndex;
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	artifact_obj.setWordIndex(pWordIndex);
	    	artifact_obj.setAssociatedFilePath(pFilePath);
	    	artifact_obj.setArtifactType(pArtifactType);
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact getInstance(Type pArtifactType, String unique_id, 
			String content){
		String hql = "from Artifact where artifactType = "+
			pArtifactType.ordinal()+" and associatedFilePath ='" +
			unique_id+"'";
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	
	    	artifact_obj.setAssociatedFilePath(unique_id);
	    	artifact_obj.setContent(content);
	    	artifact_obj.setArtifactType(pArtifactType);
	    	
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact getInstanceByEndChar(Type pArtifactType, String pFilePath, 
			int pEndIndex){
		String hql = "from Artifact where artifactType = "+
			pArtifactType.ordinal()+" and associatedFilePath ='" +
			pFilePath + "' and endIndex="+pEndIndex;
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	artifact_obj.setEndIndex(pEndIndex);
	    	artifact_obj.setAssociatedFilePath(pFilePath);
	    	artifact_obj.setArtifactType(pArtifactType);
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact getMadeUpInstance(String content){
		String hql = "from Artifact where content = '"+content+"'";
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	artifact_obj.setContent(content);
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE})
//	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE}, fetch=FetchType.LAZY )
//	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} )
    @JoinColumn(name="parentArtifact")
	public Artifact getParentArtifact()  {
		return parentArtifact;
	}
	public void setParentArtifact(Artifact _parentArtifact) {
		parentArtifact = _parentArtifact;
	}
	
//	@OneToMany(mappedBy="parentArtifact")
//    @OrderBy("startIndex")
	@Transient
	public List<Artifact> getChildsArtifact() {
		if(childsArtifact==null && artifactType != Type.Word)
			childsArtifact = 
				(List<Artifact>) HibernateUtil.executeReader( "from Artifact where parentArtifact = "+artifactId+" order by artifactId");

		return childsArtifact;
	}
	

	
	
//	@OneToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE}, fetch=FetchType.LAZY )
	@OneToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} )
	@JoinColumn(name="nextArtifact")
    public Artifact getNextArtifact() {
		return nextArtifact;
	}
	public void setNextArtifact(Artifact pNextArtifact){
		nextArtifact = pNextArtifact;
	}
	

//	@OneToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE}, fetch=FetchType.LAZY )
	@OneToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} )
	@JoinColumn(name="previousArtifact")
    public Artifact getPreviousArtifact() {
		return previousArtifact;
	}
	public void setPreviousArtifact(Artifact pPreviousArtifact){
		previousArtifact = pPreviousArtifact;
	}
	/**
	 * Return oldest parent of this Artifact which is usually a Document
	 */
	@Transient
	public Artifact grandFather() throws SQLException {
		Artifact grandFather = this;

		while (grandFather.getParentArtifact() != null) {
			grandFather = grandFather.getParentArtifact();
		}

		return grandFather;
	}

	@Index(name = "index_start_index")
	public Integer getStartIndex() {
		if (this.getArtifactType() == Type.Document) {
			startIndex = 0;
		} else if (startIndex == null) {
			int _previousArtifactsLength = 0;
			Artifact previous = this.getPreviousArtifact();
			while (previous != null) {
				_previousArtifactsLength += previous.getContent().length() + 1;
				previous = previous.getPreviousArtifact();
			}
			if(getParentArtifact()!=null)
				startIndex = parentArtifact.getStartIndex()
						+ _previousArtifactsLength + 1;
			
		}
		return startIndex;
	}

	public void setEndIndex(Integer _endIndex) {
		if (_endIndex == null && !this.getContent().isEmpty()) {
			_endIndex = startIndex + this.getContent().length();
		}
		endIndex = _endIndex;
	}

	@Index(name = "index_end_index")
	public Integer getEndIndex() {
		if (endIndex == null)
		{
			if(getStartIndex()!=null)
				endIndex = startIndex + getContent().length();
		}
		
		return endIndex;
	}

	

	public String getPOS()
	{
//		if(POS==null){
////			try {
////				POS = 
////					ArtifactAttributeTable.getAttributeValue(artifactId, 
////							AttributeValuePair.AttributeType.POS.name());
////			} catch (SQLException e) {
////				// TODO Auto-generated catch block
////				e.printStackTrace();
////			}
//			if(POS==null && artifactType == Type.Word){
//				String contentRegexCasted = StringUtil.castForRegex(getContent());
//				lineOffset = getLineIndex();
//				if(lineOffset==null) return "";
//				String tagged = 
//					StanfordParser.getTagged(associatedFilePath, lineOffset);
//				String[] taggedTokens = 
//					tagged.split(" ");
//				wordOffset = getWordIndex();
//				boolean index_invalid = false;
//				if(wordOffset!=null && 
//						taggedTokens.length>wordOffset)
//				{
//					String[] parts = taggedTokens[wordOffset].replace("\\/", "/"). // '/' is tagger splitter; decast to match					
//							split(contentRegexCasted+"/");
//					if(parts.length==2)
//						POS = parts[1];
//					else
//						index_invalid = true;
//				}
//				else
//					index_invalid= true;
//
//				if(index_invalid)
//				{
//					
//					nextArtifact = getNextArtifact();
//					previousArtifact = getPreviousArtifact();
//					if(previousArtifact != null && nextArtifact!=null)
//					{
//						String preContent=previousArtifact.getContent();
//						String nextContent=nextArtifact.getContent();
//						if(nextContent.equals("-"))
//						{
//							Artifact nextnext = nextArtifact.getNextArtifact();
//							if(nextnext!=null)
//							{
//								contentRegexCasted = 
//									contentRegexCasted+
//									nextContent+
//									nextnext.getContent();
//							}
//						}
//						// to make sure not casted before
////							contentRegexCasted = 
////								Util.castForRegex(Util.decastRegex(contentRegexCasted));
//						
//						Pattern p = Pattern.compile("(.* )?"+contentRegexCasted+"/(\\S+)( .*)?");
//						Pattern p2 = Pattern.compile("(.* )?\\S*"+contentRegexCasted+"\\S*/(\\S+)( .*)?");
//						Matcher m = p.matcher(tagged);
//						if(m.matches())
//						{
//							POS=m.group(2);
////								Util.log("_POS found: "+_POS+
////										" for "+contentRegexCasted, Level.INFO);
//						}else
//						{
//							m = p2.matcher(tagged);
//							if(m.matches())
//							{
//								POS=m.group(2);
////									Util.log("_POS found: "+_POS+
////											" for "+contentRegexCasted, Level.INFO);
//							}
//						}
//						
//					}
//					
//					if(POS==null &&  previousArtifact != null)
//					{
//						POS = previousArtifact.getPOS();
//					}
//					if(POS==null)
//					{
//						POS = getContent();
////						Util.log("Error in _POS: word index = "+_wordOffset+
////							", tagged:"+tagged+", tagged token:"+
////							" "+contentRegexCasted+" -> " 
////							+getContent(), Level.WARNING);
//					}
//				}
//			}
//		}
//		if(getContent().equals("-"))
//			POS = "-";

		
		return POS;
	}
	public void setPOS(String pPOS) {
		POS = pPOS;
	}
	/**
	 * starts from 0
	 * @return
	 */
	@Index(name = "index_word_index")
	public Integer getWordIndex() {
		
		
		if(wordOffset==null)
		{
			
			if(artifactType==Type.Word)
			{
			
				wordOffset=0;
				Artifact preWord = getPreviousArtifact();
				while(preWord!=null)
				{
					wordOffset++;
					preWord = preWord.getPreviousArtifact();
					previousArtifact = null;
				}
			}
			
		}
		return wordOffset;
	}
	
	public void setWordIndex(Integer _wordOffset) {
		wordOffset = _wordOffset;
	}
	public void setLineIndex(Integer _lineIndex) {
		lineOffset = _lineIndex;
	}
	public void setStartIndex(Integer _startIndex) {
		startIndex = _startIndex;
	}
	
	/**
	 * sentence index, starts from 1
	 * @return
	 */
	@Index(name = "index_line_index")
	public Integer getLineIndex() {
		if(lineOffset==null)
		{
			if(artifactType==Type.Word)
			{
				Artifact sentence = getParentArtifact();
				if(sentence==null) 
				{
					nextArtifact = getNextArtifact();
					if(nextArtifact!=null)
						sentence = nextArtifact.getParentArtifact();
				}
				if(sentence==null) 
				{
					previousArtifact = getPreviousArtifact();
					if(previousArtifact!=null)
						sentence = previousArtifact.getParentArtifact();
				}
				if(sentence==null) 
				{
//						Util.log("Error in getLineIndex: word '"+ _artifactId 
//								+"' doesn't have parent!", Level.SEVERE);
						
					return null;
				}
				lineOffset = sentence.getLineIndex();
			}
			if(artifactType==Type.Sentence)
			{
				lineOffset=1;
				Artifact preSentence = getPreviousArtifact();
				while(preSentence!=null)
				{
					lineOffset++;
					preSentence = preSentence.getPreviousArtifact();
				}
			}
			
		}
		return lineOffset;
	}

	public void setContent(String _content) {
		this.content = _content;
	}
	@Column(length = 2000)
	@Index(name = "index_content")
	public String getContent() {
		if(content==null) return "";
		return content;
	}

	public void setAssociatedFilePath(String _associatedFilePath) {
		this.associatedFilePath = _associatedFilePath;
	}

	public String getAssociatedFilePath() {
		return associatedFilePath;
	}
	
	@Column(nullable = false)
	public void setArtifactType(Type _artifactType) {
		this.artifactType = _artifactType;
	}

	public Type getArtifactType() {
		return artifactType;
	}

	public void setArtifactId(int _artifactId) {
		this.artifactId = _artifactId;
	}
	@Id
	@GeneratedValue(generator="increment")
	@GenericGenerator(name="increment", strategy = "increment")
	public int getArtifactId() {
		return artifactId;
	}
	
	@Override public String toString()
	{
		return associatedFilePath.substring(associatedFilePath.length()-15, 
				associatedFilePath.length()-1) + ":"+
			artifactId+"-"+artifactType.toString()+"-"
			+parentArtifact.artifactId+"-"+
			nextArtifact.artifactId+"-"+
			previousArtifact.artifactId;
	}
	
	@Override public boolean equals(Object pArtifact)
	{
		if(!(pArtifact instanceof Artifact))
			return false;
		Artifact a = (Artifact)pArtifact;
		return (a.getArtifactId() == artifactId) ||
				(a.getStartIndex() == getStartIndex() &&
				a.getContent().equals(getContent()) &&
				a.getAssociatedFilePath().equals(getAssociatedFilePath()) &&
				a.getArtifactType().equals(getArtifactType()));
	}
	@Override public int hashCode()
	{
		return artifactId;
	}
	public static Artifact findInstance(Type pArtifactType, String pFilePath, 
			int pWordIndex, int pLineIndex){
		String hql = "from Artifact where artifactType =:artifactType  and" +
				" associatedFilePath like :associatedFilePath and wordIndex=:wordIndex " +
				" and lineIndex= :lineIndex";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("associatedFilePath", '%'+pFilePath);
		params.put("wordIndex", pWordIndex);
		params.put("artifactType", pArtifactType.ordinal());
		params.put("lineIndex",pLineIndex);
		
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
	    
	    
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstance(Artifact pSentence,int pWordIndex){
		String hql = "from Artifact where parentArtifact = "+pSentence.getArtifactId()+" and wordIndex =" +
		pWordIndex;
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstance(Type pArtifactType, String pFilePath, 
			int pLine, int pWordIndex, String pContent, String pSecondWord){
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		String hql = "from Artifact where artifactType =:artifactType  and" +
				" associatedFilePath like :associatedFilePath and content like:content " +
				" and lineIndex= :lineIndex ";
		if (pSecondWord != null)
		{
			hql += "and nextArtifact.content = :nextContent";
			params.put("nextContent", pSecondWord);
		}
		else
		{
			hql += " and ABS(wordIndex - :pWordIndex) <=5";
			params.put("pWordIndex",pWordIndex);
		}
		
		params.put("associatedFilePath", '%'+pFilePath);
		params.put("content", pContent+'%');
		params.put("artifactType", pArtifactType.ordinal());
		params.put("lineIndex",pLine);
		
		
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
	    
	    
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstanceByContent(Type pArtifactType, String pFilePath, 
			String pContent ){
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		String hql = "from Artifact where artifactType =:artifactType  and" +
				" associatedFilePath like :associatedFilePath and content like:content ";
		
		
		params.put("associatedFilePath", '%'+pFilePath+'%');
		params.put("content", '%'+pContent+'%');
		params.put("artifactType", pArtifactType.ordinal());
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
	    
	    
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstanceByContentEnd(Type pArtifactType, String pFilePath, 
			String pContent ){
		HashMap<String, Object> params = new HashMap<String, Object>();
		String hql = "from Artifact where artifactType =:artifactType  and" +
				" associatedFilePath like :associatedFilePath and content like:content ";
		
		params.put("associatedFilePath", '%'+pFilePath+'%');
		params.put("content", '%'+pContent);
		params.put("artifactType", pArtifactType.ordinal());
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstanceByExactContent(Type pArtifactType, String pFilePath, 
			String pContent ){
		HashMap<String, Object> params = new HashMap<String, Object>();
		String hql = "from Artifact where artifactType =:artifactType  and" +
				" associatedFilePath like :associatedFilePath and content like:content ";
		
		params.put("associatedFilePath", '%'+pFilePath+'%');
		params.put("content", pContent);
		params.put("artifactType", pArtifactType.ordinal());
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstance(Type pArtifactType, String pFilePath, 
			int pStartChar, String pContent, String pSecondWord){
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		String hql = "from Artifact where artifactType =:artifactType  and" +
				" associatedFilePath like :associatedFilePath and content like:content " +
				" and ABS(startIndex - :pStartIndex) <45";
		if (pSecondWord != null)
		{
			hql += "and nextArtifact.content = :nextContent";
			params.put("nextContent", pSecondWord);
		}
	
		
		params.put("associatedFilePath", '%'+pFilePath);
		params.put("content", pContent+'%');
		params.put("artifactType", pArtifactType.ordinal());
		params.put("pStartIndex",pStartChar);
		
		
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
	    
	    
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstanceByStartIndex(String pFilePath,int pStartIndex){
		String hql = "from Artifact where associatedFilePath like :associatedFilePath  and  artifactType=:artifactType and startIndex=:startIndex ";

		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("associatedFilePath", '%'+pFilePath);
		params.put("startIndex", pStartIndex);
		params.put("artifactType", 3);
		

		List<Artifact> artifact_objects = 
					(List<Artifact>) HibernateUtil.executeReader(hql, params);


	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstanceByEndIndex(String pFilePath,int pEndIndex){
		String hql = "from Artifact where associatedFilePath like :associatedFilePath  and  artifactType=:artifactType and endIndex=:endIndex ";

		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("associatedFilePath", '%'+pFilePath);
		params.put("endIndex", pEndIndex);
		params.put("artifactType", 3);
		

		List<Artifact> artifact_objects = 
					(List<Artifact>) HibernateUtil.executeReader(hql, params);


	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static List<Artifact> listByType(Type pSentenceType) {
		String hql = "from Artifact where artifactType = "+pSentenceType.ordinal()
				+" order by associatedFilePath";
			
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
		return artifact_objects;
	}
	//TODO: move this method to the right class
	public static List<Artifact> listTweets(Type pSentenceType) {
		String hql = "from Artifact where artifactType = "+pSentenceType.ordinal()
				+" and associatedFilePath not like '%host%' order by associatedFilePath";
			
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
		return artifact_objects;
	}
	public static List<Artifact> listByTypeByForTrain(Type pSentenceType, boolean forTrain) {
		String hql = "from Artifact where artifactType = "+pSentenceType.ordinal()
				+" and forTrain= "+forTrain+" order by associatedFilePath";
			
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
		return artifact_objects;
	}
	public static List<Artifact> listByTypeByForTrainByCorpus(Type pSentenceType, boolean forTrain,String corpusName) {
		String hql = "from Artifact where artifactType = "+pSentenceType.ordinal()
				+" and forTrain= "+forTrain+" and corpusName= '"+corpusName+"' order by associatedFilePath";
			
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
		return artifact_objects;
	}

	public static List<Artifact> listByTypeLimited(Type pSentenceType,boolean for_train, Integer count) {
		
		String hql = "from Artifact where artifactType = "+pSentenceType.ordinal()
				+"  and forTrain = "+for_train+" order by rand()";
		HashMap<String, Object> params = new HashMap<String, Object>();
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params,count);
		return artifact_objects;
	}
	public static List<Artifact> listByType(Type pSentenceType,boolean for_train) {
		String file_path_pattern = "/train/";
		if (!for_train)
		{
			file_path_pattern = "/test/";
		}
		String hql = "from Artifact where artifactType = "+pSentenceType.ordinal()
				+"  and associatedFilePath like :pPattern order by associatedFilePath";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("pPattern", '%'+file_path_pattern+'%');
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
		return artifact_objects;
	}

	public void setStanDependency(String stanDependency) {
		this.stanDependency = stanDependency;
	}
	@Column(length = 2000)
	public String getStanDependency() {
		return stanDependency;
	}
	
	public void setStanPennTree(String stanPennTree) {
		this.stanPennTree = stanPennTree;
	}
	@Column(length = 2000)
	public String getStanPennTree() {
		return stanPennTree;
	}
	@Transient
	List<Artifact> children = null;
	@Transient
	public Artifact getChildByWordIndex(int word_index) {
		if(children==null)
			loadChildren();
		Artifact foundArtifact = null;
		for(Artifact art : children)
			if(art.getWordIndex() == word_index)
			{
				foundArtifact = art;
				break;
			}
		return foundArtifact;
	}
	private void loadChildren() {
		String hql = "from Artifact where parentArtifact = "+getArtifactId();
		
		children = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	}
	
	public static List<Artifact> listByTypeAndForTrain(Type pSentenceType,boolean for_train) {

		String hql = "from Artifact where forTrain ="+for_train+" and artifactType = "+pSentenceType.ordinal()
				+"  order by associatedFilePath";
		HashMap<String, Object> params = new HashMap<String, Object>();
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
		return artifact_objects;
	}
	public static List<Artifact> listByTypeAndForTrain(Type pSentenceType,boolean for_train,String corpus) {

		String hql = "from Artifact where forTrain ="+for_train+" and artifactType = "+pSentenceType.ordinal()
				+"  and corpusName='"+corpus+"' order by associatedFilePath";
		HashMap<String, Object> params = new HashMap<String, Object>();
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
		return artifact_objects;
	}
	public boolean isForTrain() {
		return forTrain;
	}
	public void setForTrain(boolean forTrain) {
		this.forTrain = forTrain;
	}
	public boolean isForDemo() {
		return forDemo;
	}
	public void setForDemo(boolean forDemo) {
		this.forDemo = forDemo;
	}
	public String getCorpusName() {
		return corpusName;
	}
	public void setCorpusName(String name) {
		this.corpusName = name;
	}
	
} @Entity
@Table( name = "FeatureValuePair" )
public class FeatureValuePair {
	public enum FeatureName {
		// Document Features
		JournalTitle,
		CompletedYear,
		CreatedYear,
		RevisedYear,
		MESHHeading,
		

		// Paragraph Features
		// PositionInDoc,

		// Sentence Features
		ProteinCountInSentence,
		SentenceTFIDF,

		// Words Features
		POS,
		POSNext1,
		POSNext2,
		POSPre1,
		POSPre2,
		
		PorterStem, 
		WordnetStem, 
		OriginalWord, 
		NameEntity, 
		StartWithUppercase, 
		AllUppercase, 
		AllLowercase, 
		HasSpecialChars, 
		HasDigit, 
		
		CommaLeftCount,
		CommaRightCount,
		QuoteLeftCount,
		QuoteRightCount,
		ProteinCountInWindow,
		
		SimilarityToGene_expression, 
		SimilarityToTranscription, 
		SimilarityToProtein_catabolism, 
		SimilarityToLocalization, 
		SimilarityToBinding, 
		SimilarityToPhosphorylation, 
		SimilarityToRegulation, 
		SimilarityToPositive_regulation, 
		SimilarityToNegative_regulation,
		PositionInDoc,
		
		//coreference features
		AnaphoraIsSubject,
		AntecedentInFirstSubject,
		AntecedentInHeader,
		AntecedentIsSubject,
		Appositive,
		NumberAgreement,
		SentenceDistance, TWOGram, TWOGramBackward, ThreeGram, ThreeGramBackward, NellLink, 
		ProblemCountInSentence, TestCountInSentence, TreatmentCountInSentence, TestsBeforeWord, 
		TreatmentsBeforeWord, ProblemsBeforeWord, ProblemPossibleCountInSentence, ProblemHypoCountInSentence, 
		ProblemConditionalCountInSentence, ProblemAWSECountInSentence, ProblemAbsentCountInSentence, ProblemPresentCountInSentence, EdgeType,
		WordWindowNext, WordWindowPre, EdgeParsePath, EdgeParseDistance, DependencyLinkedTokens,
		
		TimexCount, ClinicalEventsCount, LinkWordBetween, LinkArgumentType, LinkFromPhrasePolarity, LinkFromPhraseModality, LinkFromPhraseType, LinkToPhraseModality, LinkToPhraseType, 
		LinkToPhrasePolarity, LinkToPhraseTimexMod, LinkFromPhraseTimexMod, 
		InterMentionLocationType, AreDirectlyConnected, HaveCommonGovernors, AreConjunctedAnd,
		//NGrams
		NonNormalizedNGram2, NonNormalizedNGram3, NorBetweenNGram2, NorBetweenNGram3, Link2GramBetween, 
		Link2GramFrom,Link2GramTo,
		
		//Link Args basic features
		FromPhraseContent, ToPhraseContent, FromPhrasePOS, ToPhrasePOS,
		
		LinkBetweenWordCount, LinkBetweenPhraseCount, 
		
		//ParseDependency features
		FromPhraseRelPrep, ToPhraseRelPrep, FromPhraseGovVerb, ToPhraseGovVerb,  FromPhraseGovVerbTense,
		FromPhraseGovVerbAux, toPhraseGovVerbAux, areGovVerbsConnected,
		normalizedDependencies,
		//pattern statistics
		POverlapGivenPattern, PBeforeGivenPattern, PAfterGivenPattern, PNoLinkGivenPattern, hasFeasibleLink, 
		POverlapGivenPatternTTO, PBeforeGivenPatternTTO, PAfterGivenPatternTTO, PNoLinkGivenPatternTTO,
		maxProbClassByPattern,
		
		ParseTreePath, ParseTreePathSize,
		
		
		//sectime features
		relatedSectionInDoc, AdmissionOrDischarge, 
		//normalized 
		fromPrepArg, toPrepArg, isToPhDirectPrepArgOfFromPh, isEventAfterProblem, norToTypeDep,
		fromToToPathExist, toToFromPathExist, fromToToPathSize, toToFromPathSize, customGraphPath,
		//custom graph

		LabeledGraphNorDepPath,  customGraphIndividualPath, customGraphPathString,
		

		TemporalSignal, FromPhrasePOS1By1, ToPhrasePOS1By1, 
		FromPhrasePOSWindowBefore, ToPhrasePOSWindowBefore,  
		FromPhrasePOSWindowAfter, ToPhrasePOSWindowAfter, 
		ToPhrasePOSBigramAfter, FromPhrasePOSBigramBefore,
		ToPhrasePOSBigramBefore, FromToPhrasePOSBigram, LinkFromToWordDistance, LinkPOSBetween,
		
		
		betweenChunck, 
		
		//for the deext project
		chunkContent, embeddings,
		
		//For Twitter
		content, hasTermInADRDic, 
		
		//for conceptExtraction
		TokenContent, PrevTokenContent, NextTokenContent, SecondPrevTokenContent, SecondNextTokenContent, PositiveTokensInChunk, NegativeTokensInChunk, 
		PositiveTokensInNbrChunk, NegativeTokensInNbrChunk, MaxLexiconSim, isInLexiconEntry, GovVerb, MaxPhraseLexSim, BinaryLexiconSim, DeepClassNumber, PrevDeepClassNumber, SecondPrevDeepClassNumber, NextDeepClassNumber, SecondNextDeepClassNumber, GovVerbDeepClassNumber,
		isTokenInLexicon,isTokenNegated, TokenContentProcessed, isTokenNegatedExpanded,ThirdPrevTokenContent,ThirdNextTokenContent,ThirdPrevDeepClassNumber,
		ThirdNextDeepClassNumber,isPrevTokenInLexicon,isSPrevTokenInLexicon, isNextTokenInLexicon, isSNextTokenInLexicon,
		
		//for SVM classifier
		SVMCandTokenContent, SVMCandPrevTokenContent, SVMCandSecondPrevTokenContent, SVMCandNextTokenContent, SVMCandSecondNextTokenContent, SVMCandGovVerb, Token_IDF, Token_Lexicon_IDF, 
		SVMCandParSentToken, SVMCandParSentTokenIdf, SVMCandBigrams, SVMCandParSenDeepClassNumber, SVMCandThirdNextTokenContent, SVMCandThirdPrevTokenContent, 
		SVMIsPhraseNegatedExpanded, SVMIsPhraseNegated, 
		//hierachical clustering bitstring
		HClusterBitStringCurrent, PrevHClusterBitString, SecondPrevHClusterBitString, ThirdPrevHClusterBitString, NextHClusterBitString, SecondNextHClusterBitString, ThirdNextHClusterBitString,
		HClusterBitStringCurrent8, PrevHClusterBitString8, SecondPrevHClusterBitString8, ThirdPrevHClusterBitString8, NextHClusterBitString8, SecondNextHClusterBitString8, ThirdNextHClusterBitString8,
		HClusterBitStringCurrent4, PrevHClusterBitString4, SecondPrevHClusterBitString4, ThirdPrevHClusterBitString4, NextHClusterBitString4, SecondNextHClusterBitString4, ThirdNextHClusterBitString4, 
		
		Word2vecADRSimilarity, Word2vecIndicationSimilarity,  
		

		
	}

	String featureName;
	String featureValue;
	
	//For string multi features this can be used to handle real value 
	// othewise 1 or 0 would be used for values(for tf_idfs)
	private String featureValueAuxiliary;
	
	private int featureValuePairId = -1;
	
	//reset every time used for training a model;-1 means not used in training
	private int tempFeatureIndex = -1;
	
	
	
	public int getTempFeatureIndex() {
		return tempFeatureIndex;
	}
	public void setTempFeatureIndex(int tempFeatureIndex) {
		this.tempFeatureIndex = tempFeatureIndex;
	}
	public void setFeatureName(String _featureName) {
		featureName = _featureName;
	}
	@NaturalId
	public String getFeatureName() {
		return featureName;
	}

	
	public void setFeatureValue(String _featureValue) {
		featureValue = _featureValue;
	}
	@NaturalId
	public String getFeatureValue() {
		return featureValue;
	}
	
	public FeatureValuePair()
	{
	
	}
	

	
	public void setFeatureValuePairId(int featureValuePairId) {
		this.featureValuePairId = featureValuePairId;
	}
	@Id
	@GeneratedValue(generator="increment")
	@GenericGenerator(name="increment", strategy = "increment")
	public int getFeatureValuePairId() {
		return featureValuePairId;
	}

	
	void setFeatureValueAuxiliary(String featureValueAuxiliary) {
		this.featureValueAuxiliary = featureValueAuxiliary;
	}
	@NaturalId
	public String getFeatureValueAuxiliary() {
		return featureValueAuxiliary;
	}
	
	@Override public String toString()
	{
		return featureName+" = "+ featureValue + 
			" ("+featureValueAuxiliary+")";
	}
	
	@Override public boolean equals(Object pFeatureValuePair)
	{
		if(!(pFeatureValuePair instanceof FeatureValuePair))
			return false;
		FeatureValuePair fvp = (FeatureValuePair)pFeatureValuePair;
		if(fvp.getFeatureValuePairId() == featureValuePairId ||
				(fvp.getFeatureName() == featureName &&
					fvp.getFeatureValue().equals(featureValue)&&
					fvp.getFeatureValueAuxiliary() == featureValueAuxiliary))
			return true;
		else 
			return false;
		
	}
	@Override public int hashCode()
	{
		return featureValuePairId;
	}
	public static FeatureValuePair getInstance(String pFeatureName, 
			String pFeatureValue,
			String pFeatureValueAuxiliary){
		return getInstance(pFeatureName, pFeatureValue, pFeatureValueAuxiliary, false);
	}
	public static FeatureValuePair getInstance(String pFeatureName, 
			String pFeatureValue,
			String pFeatureValueAuxiliary,
			boolean isDense)
	{
		FeatureValuePair feature_value;

	
		
		String hql = "from FeatureValuePair where featureName= :featureName "+
					" AND featureValue= :featureValue ";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("featureValue", pFeatureValue);
		params.put("featureName", pFeatureName);
		
		if(pFeatureValueAuxiliary!=null)
		{
			hql += " AND featureValueAuxiliary= :featureValueAuxiliary ";
			params.put("featureValueAuxiliary", pFeatureValueAuxiliary);
		}
		
		List<FeatureValuePair> featurev_list = 
			(List<FeatureValuePair>) HibernateUtil.executeReader(hql, params);
		
		if(featurev_list.size() == 0)
		{
			feature_value = new FeatureValuePair();
			feature_value.setFeatureName(pFeatureName);
			feature_value.setFeatureValue(pFeatureValue);
			if(pFeatureValueAuxiliary!=null)
				feature_value.setFeatureValueAuxiliary(pFeatureValueAuxiliary);
			HibernateUtil.save(feature_value);
		}else
			feature_value = featurev_list.get(0);
		return feature_value;
	}
	
	public static FeatureValuePair getInstance(String pFeatureName, 
			String pFeatureValue, boolean isDense)
	{
	
		return getInstance(pFeatureName, pFeatureValue, null, isDense);
	}
	public static FeatureValuePair getInstance(String pFeatureName, 
			String pFeatureValue)
	{
	
		return getInstance(pFeatureName, pFeatureValue, null, false);
	}
	
	public static List<String> multiValueFeatures = new ArrayList<String>();
	static
	{
		multiValueFeatures.add(FeatureName.MESHHeading.name());
		multiValueFeatures.add(FeatureName.SentenceTFIDF.name());
		multiValueFeatures.add(FeatureName.LinkWordBetween.name());
	}
	@Transient
	//TODO make it better
	public boolean isMultiValue()
	{
		boolean res = false;
		
		if(multiValueFeatures.contains(featureName))
			res = true;

		return res;
	}
	public static void resetIndexes() {
		String hql = "update FeatureValuePair set tempFeatureIndex = "+Integer.MAX_VALUE;
		HibernateUtil.executeNonReader(hql);
		Session temp_Session = HibernateUtil.sessionFactory.openSession();
		String selecthql = "from FeatureValuePair";


		List<FeatureValuePair> features = 
			(List<FeatureValuePair>) HibernateUtil.executeReader(selecthql,null,null, temp_Session);
		int new_feature_index = 0;
		int count =0;
		for(int i=0;i<features.size();i++) {
			count++;
			
//			FileUtil.logLine(null,"resetIndexes--------feature processed: "+count+"/"+features.size());
			temp_Session = HibernateUtil.clearSession(temp_Session);
			
			//load again fvp to get effect of bulk update
			String selectfvp = "from FeatureValuePair where featureValuePairId = "+features.get(i).getFeatureValuePairId();
			
			FeatureValuePair fvp  = 
				((List<FeatureValuePair>) HibernateUtil.executeReader(selectfvp,null,null, temp_Session)).get(0);
			
			int featureIndex = 
					fvp.getTempFeatureIndex();
			if(featureIndex==Integer.MAX_VALUE)
			{
				new_feature_index ++;
				if(fvp.getFeatureValueAuxiliary()!=null)
				{
					featureIndex = new_feature_index;
					fvp.setTempFeatureIndex(featureIndex);
					HibernateUtil.save(fvp, temp_Session);
				}else
				{
					//feature index for attribute not set before
					//find one for the attribute
					featureIndex = new_feature_index;
					
					String update_attribute_index_hql = 
						"UPDATE FeatureValuePair set tempFeatureIndex = "+ featureIndex
						+ " where featureName='"+fvp.getFeatureName()+"'";
					HibernateUtil.executeNonReader(update_attribute_index_hql);
				}
			}
		}
		temp_Session.clear();
		temp_Session.close();
	}
	public static int getMaxIndex() {
		Session tmp_session = HibernateUtil.sessionFactory.openSession();
		String hql = "select max(tempFeatureIndex) from FeatureValuePair where tempFeatureIndex<"+Integer.MAX_VALUE;
		List res = HibernateUtil.executeReader(hql, null,null,tmp_session);
		tmp_session.clear();
		tmp_session.close();
		return (res.get(0)==null)?0:Integer.valueOf(res.get(0).toString());
	}
	public static int getMinIndexForAttribute(String attributeName) {
		String hql = "select min(tempFeatureIndex) from FeatureValuePair where " +
				"featureName='" + attributeName + "'";
		List res = HibernateUtil.executeReader(hql);
		return (res.get(0)==null)?0:Integer.valueOf(res.get(0).toString());
	}
	public static FeatureValuePair getInstance(FeatureName linkType,
			String pFeatureValue) {
		
		return getInstance(linkType.name(), pFeatureValue);
	}
	
	public static FeatureValuePair getInstance(FeatureName linkType,
			String pFeatureValue, String auxValue) {
		
		return getInstance(linkType.name(), pFeatureValue, auxValue);
	}
	public static Integer getRelatedFromEventTypeFValuePairIds(
			String pFeatureValue)
	{
		Integer feature_value_id =null;
		
//		String featureValueString = "";
//		for(String val: pFeatureValues)
//		{
//			featureValueString = featureValueString.concat(", '"+val+"'");
//		}
//		featureValueString = featureValueString.replaceFirst(",", "");
		
		String hql = "from FeatureValuePair where featureName in ('LinkFromPhraseType') "+
					" AND featureValue = '"+pFeatureValue+"' ";
		
		
		List<FeatureValuePair> featurev_list = 
			(List<FeatureValuePair>) HibernateUtil.executeReader(hql);
		if (featurev_list.size() !=0)
		{
			feature_value_id = featurev_list.get(0).getFeatureValuePairId();
		}
		
		return feature_value_id;
	}
	public static Integer getRelatedToEventTypeFValuePairIds(
		String pFeatureValue)
	{
		Integer feature_value_id =null;
		
//		String featureValueString = "";
//		for(String val: pFeatureValues)
//		{
//			featureValueString = featureValueString.concat(", '"+val+"'");
//		}
//		featureValueString = featureValueString.replaceFirst(",", "");
		
		String hql = "from FeatureValuePair where featureName in ('LinkToPhraseType') "+
					" AND featureValue ='"+pFeatureValue+"' ";
		
		
		List<FeatureValuePair> featurev_list = 
			(List<FeatureValuePair>) HibernateUtil.executeReader(hql);
		if (featurev_list.size() !=0)
		{
			feature_value_id = featurev_list.get(0).getFeatureValuePairId();
		}
		
		return feature_value_id;
	}
	public static List<FeatureValuePair> getInstancesBulk(
			List<String> vector_values, String featurePrefix) {
		
		String valuesOfInterest = "";
		for (Integer d=0;d<vector_values.size();d++)
		{
			String curValue = vector_values.get(d);
			valuesOfInterest+="'"+curValue+"',";
		}
		valuesOfInterest+="''";
		String hql = "from FeatureValuePair where featureName like :featurePrefix and featureValue in ("+valuesOfInterest+")";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("featurePrefix", featurePrefix+"%");
		
		List<FeatureValuePair> featurev_list = 
			(List<FeatureValuePair>) HibernateUtil.executeReader(hql, params);
		HibernateUtil.startTransaction();;
		Session session = HibernateUtil.saverSession;
		List<FeatureValuePair> fvpsResult = new ArrayList<FeatureValuePair>();
		
		for (Integer d=0;d<vector_values.size();d++)
		{
			FeatureValuePair feature_value = null;
			String curValue = vector_values.get(d);
			boolean found = false;
			for(FeatureValuePair fvp : featurev_list){
				if(fvp.getFeatureName().equals(featurePrefix+d) && 
						fvp.getFeatureValue().equals(curValue)){
					feature_value = fvp;
					found = true;
					break;
				}
			}
			
			if(!found){
				feature_value = new FeatureValuePair();
				feature_value.setFeatureName(featurePrefix+d);
				feature_value.setFeatureValue(curValue);
				session.save(feature_value);
			}
			
			fvpsResult.add(feature_value);
		}
		
		HibernateUtil.endTransaction(); 
		
		
		return fvpsResult;
	}
	
	
} public enum FeatureName {
	// Document Features
	JournalTitle,
	CompletedYear,
	CreatedYear,
	RevisedYear,
	MESHHeading,
		

	// Paragraph Features
	// PositionInDoc,

	// Sentence Features
	ProteinCountInSentence,
	SentenceTFIDF,

	// Words Features
	POS,
	POSNext1,
	POSNext2,
	POSPre1,
	POSPre2,
		
	PorterStem, 
	WordnetStem, 
	OriginalWord, 
	NameEntity, 
	StartWithUppercase, 
	AllUppercase, 
	AllLowercase, 
	HasSpecialChars, 
	HasDigit, 
		
	CommaLeftCount,
	CommaRightCount,
	QuoteLeftCount,
	QuoteRightCount,
	ProteinCountInWindow,
		
	SimilarityToGene_expression, 
	SimilarityToTranscription, 
	SimilarityToProtein_catabolism, 
	SimilarityToLocalization, 
	SimilarityToBinding, 
	SimilarityToPhosphorylation, 
	SimilarityToRegulation, 
	SimilarityToPositive_regulation, 
	SimilarityToNegative_regulation,
	PositionInDoc,
		
	//coreference features
	AnaphoraIsSubject,
	AntecedentInFirstSubject,
	AntecedentInHeader,
	AntecedentIsSubject,
	Appositive,
	NumberAgreement,
	SentenceDistance, TWOGram, TWOGramBackward, ThreeGram, ThreeGramBackward, NellLink, 
	ProblemCountInSentence, TestCountInSentence, TreatmentCountInSentence, TestsBeforeWord, 
	TreatmentsBeforeWord, ProblemsBeforeWord, ProblemPossibleCountInSentence, ProblemHypoCountInSentence, 
	ProblemConditionalCountInSentence, ProblemAWSECountInSentence, ProblemAbsentCountInSentence, ProblemPresentCountInSentence, EdgeType,
	WordWindowNext, WordWindowPre, EdgeParsePath, EdgeParseDistance, DependencyLinkedTokens,
		
	TimexCount, ClinicalEventsCount, LinkWordBetween, LinkArgumentType, LinkFromPhrasePolarity, LinkFromPhraseModality, LinkFromPhraseType, LinkToPhraseModality, LinkToPhraseType, 
	LinkToPhrasePolarity, LinkToPhraseTimexMod, LinkFromPhraseTimexMod, 
	InterMentionLocationType, AreDirectlyConnected, HaveCommonGovernors, AreConjunctedAnd,
	//NGrams
	NonNormalizedNGram2, NonNormalizedNGram3, NorBetweenNGram2, NorBetweenNGram3, Link2GramBetween, 
	Link2GramFrom,Link2GramTo,
		
	//Link Args basic features
	FromPhraseContent, ToPhraseContent, FromPhrasePOS, ToPhrasePOS,
		
	LinkBetweenWordCount, LinkBetweenPhraseCount, 
		
	//ParseDependency features
	FromPhraseRelPrep, ToPhraseRelPrep, FromPhraseGovVerb, ToPhraseGovVerb,  FromPhraseGovVerbTense,
	FromPhraseGovVerbAux, toPhraseGovVerbAux, areGovVerbsConnected,
	normalizedDependencies,
	//pattern statistics
	POverlapGivenPattern, PBeforeGivenPattern, PAfterGivenPattern, PNoLinkGivenPattern, hasFeasibleLink, 
	POverlapGivenPatternTTO, PBeforeGivenPatternTTO, PAfterGivenPatternTTO, PNoLinkGivenPatternTTO,
	maxProbClassByPattern,
		
	ParseTreePath, ParseTreePathSize,
		
		
	//sectime features
	relatedSectionInDoc, AdmissionOrDischarge, 
	//normalized 
	fromPrepArg, toPrepArg, isToPhDirectPrepArgOfFromPh, isEventAfterProblem, norToTypeDep,
	fromToToPathExist, toToFromPathExist, fromToToPathSize, toToFromPathSize, customGraphPath,
	//custom graph

	LabeledGraphNorDepPath,  customGraphIndividualPath, customGraphPathString,
		

	TemporalSignal, FromPhrasePOS1By1, ToPhrasePOS1By1, 
	FromPhrasePOSWindowBefore, ToPhrasePOSWindowBefore,  
	FromPhrasePOSWindowAfter, ToPhrasePOSWindowAfter, 
	ToPhrasePOSBigramAfter, FromPhrasePOSBigramBefore,
	ToPhrasePOSBigramBefore, FromToPhrasePOSBigram, LinkFromToWordDistance, LinkPOSBetween,
		
		
	betweenChunck, 
		
	//for the deext project
	chunkContent, embeddings,
		
	//For Twitter
	content, hasTermInADRDic, 
		
	//for conceptExtraction
	TokenContent, PrevTokenContent, NextTokenContent, SecondPrevTokenContent, SecondNextTokenContent, PositiveTokensInChunk, NegativeTokensInChunk, 
	PositiveTokensInNbrChunk, NegativeTokensInNbrChunk, MaxLexiconSim, isInLexiconEntry, GovVerb, MaxPhraseLexSim, BinaryLexiconSim, DeepClassNumber, PrevDeepClassNumber, SecondPrevDeepClassNumber, NextDeepClassNumber, SecondNextDeepClassNumber, GovVerbDeepClassNumber,
	isTokenInLexicon,isTokenNegated, TokenContentProcessed, isTokenNegatedExpanded,ThirdPrevTokenContent,ThirdNextTokenContent,ThirdPrevDeepClassNumber,
	ThirdNextDeepClassNumber,isPrevTokenInLexicon,isSPrevTokenInLexicon, isNextTokenInLexicon, isSNextTokenInLexicon,
		
	//for SVM classifier
	SVMCandTokenContent, SVMCandPrevTokenContent, SVMCandSecondPrevTokenContent, SVMCandNextTokenContent, SVMCandSecondNextTokenContent, SVMCandGovVerb, Token_IDF, Token_Lexicon_IDF, 
	SVMCandParSentToken, SVMCandParSentTokenIdf, SVMCandBigrams, SVMCandParSenDeepClassNumber, SVMCandThirdNextTokenContent, SVMCandThirdPrevTokenContent, 
	SVMIsPhraseNegatedExpanded, SVMIsPhraseNegated, 
	//hierachical clustering bitstring
	HClusterBitStringCurrent, PrevHClusterBitString, SecondPrevHClusterBitString, ThirdPrevHClusterBitString, NextHClusterBitString, SecondNextHClusterBitString, ThirdNextHClusterBitString,
	HClusterBitStringCurrent8, PrevHClusterBitString8, SecondPrevHClusterBitString8, ThirdPrevHClusterBitString8, NextHClusterBitString8, SecondNextHClusterBitString8, ThirdNextHClusterBitString8,
	HClusterBitStringCurrent4, PrevHClusterBitString4, SecondPrevHClusterBitString4, ThirdPrevHClusterBitString4, NextHClusterBitString4, SecondNextHClusterBitString4, ThirdNextHClusterBitString4, 
		
	Word2vecADRSimilarity, Word2vecIndicationSimilarity,  
		

		
} public interface IFeatureCalculator {
	public void calculateFeatures(MLExample exampleToProcess) throws SQLException, Exception;
} @Entity
@Table( name = "MLExample" )
public class MLExample {
	int exampleId;


	int predictedClass;
	int expectedClass;
	boolean forTrain;
	String corpusName;
	String predictionEngine;
	Artifact relatedArtifact;
	PhraseLink relatedPhraseLink;
	private String associatedFilePath;
	private double predictionWeight;
	private int expectedReal;
	private int expectedClosure;
	private int expectedIntegrated;
	
	private Phrase relatedPhrase;
	
	@Transient
	List<MLExampleFeature> exampleFeatures;
	static public Session hibernateSession; 
	
	String relatedDrug;
	@Transient
	public List<MLExampleFeature> getExampleFeatures()
	{
		if(exampleFeatures==null)
		{
			if(hibernateSession == null)
				hibernateSession = HibernateUtil.sessionFactory.openSession();
			String hql = "from MLExampleFeature where relatedExample = "+
			 getExampleId()+ 
			 " order by featureValuePair.tempFeatureIndex";
//			 " order by featureValuePair.featureValuePairId";
			exampleFeatures = (List<MLExampleFeature>) HibernateUtil.executeReader(hql, null,null, hibernateSession);
		}
		return exampleFeatures;
	}
	@Transient
	public List<MLExampleFeature> getNonEmbeddingsExampleFeatures()
	{
		if(exampleFeatures==null)
		{
			if(hibernateSession == null)
				hibernateSession = HibernateUtil.sessionFactory.openSession();
			String hql = "from MLExampleFeature  where relatedExample = "+
			 getExampleId()+ 
			 " and featureValuePair. featureName not like '%Embeddings%' order by featureValuePair.tempFeatureIndex";
//			 " order by featureValuePair.featureValuePairId";
			exampleFeatures = (List<MLExampleFeature>) HibernateUtil.executeReader(hql, null,null, hibernateSession);
		}
		return exampleFeatures;
	}
	@Transient
	//TODO: improve this, this is a quick way to get a specific one value String feature
	public String getFeatureValueForExample(MLExample e, String featureName)
	{
		
		if(hibernateSession == null || !hibernateSession.isOpen())
			hibernateSession = HibernateUtil.sessionFactory.openSession();
		String hql = "from MLExampleFeature  where relatedExample = "+
		 e.getExampleId()+ 
		 " and featureValuePair. featureName ='"+featureName+"' order by featureValuePair.tempFeatureIndex";
//			 " order by featureValuePair.featureValuePairId";
		List<MLExampleFeature> eFeatures = (List<MLExampleFeature>) HibernateUtil.executeReader(hql, null,null, hibernateSession);
		
		if (eFeatures.isEmpty())
		{
			FileUtil.logLine("/tmp/featureErrors.txt", "no feature for example id:  "+e.getExampleId());
			return "UNK";
		}

		MLExampleFeature feature = eFeatures.get(0);
		FeatureValuePair fvp = feature.getFeatureValuePair();
		
		return fvp.getFeatureValue();
	}
	public String getPredictionEngine() {
		return predictionEngine;
	}

	public void setPredictionEngine(String pPredictionEngine) {
		predictionEngine = pPredictionEngine;
	}


	
	
	public String getCorpusName() {
		return corpusName;
	}
	
	public void setCorpusName(String pCorpusName) {
		corpusName = pCorpusName;
	}
	
	public boolean getForTrain() {
		return forTrain;
	}
//	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE }, fetch=FetchType.LAZY )	
	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE } )
    @JoinColumn(name="relatedArtifact")
	public Artifact getRelatedArtifact() {
		return relatedArtifact;
	}

	public void setRelatedArtifact(Artifact relatedArtifact) {
		this.relatedArtifact = relatedArtifact;
	}

	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} , fetch=FetchType.LAZY )
    @JoinColumn(name="relatedPhraseLink")
	public PhraseLink getRelatedPhraseLink() {
		return relatedPhraseLink;
	}

	public void setRelatedPhraseLink(PhraseLink relatedPhraseLink) {
		this.relatedPhraseLink = relatedPhraseLink;
	}

	public void setForTrain(boolean isForTrain) {
		forTrain = isForTrain;
	}
	
	public int getPredictedClass() {
		return predictedClass;
	}
	
	public void setPredictedClass(int pPredictedClass) {
		predictedClass = pPredictedClass;
	}
	
	public int getExpectedClass() {
		return expectedClass;
	}
	
	public void setExpectedClass(int pExpectedClass) {
		expectedClass = pExpectedClass;
	}
	@Id
	@GeneratedValue(generator="increment")
	@GenericGenerator(name="increment", strategy = "increment")
	public int getExampleId() {
		return exampleId;
	}

	public void setExampleId(int exampleId) {
		this.exampleId = exampleId;
	}
	@Temporal(TemporalType.TIMESTAMP)
    Date updateTime;
	
	
	@PrePersist
    protected void onCreate() {
		updateTime = new Date();
    }

    @PreUpdate
    protected void onUpdate() {
    	updateTime = new Date();
    }

	public static MLExample getInstanceForArtifact(Artifact artifact,
			String experimentgroup) {
		String hql = "from MLExample where relatedArtifact = "+
				artifact.getArtifactId() + " and corpusName = '"+
				experimentgroup+"'";
			List<MLExample> example_objects = 
					getExamplesList(hql);
		    
			MLExample example_obj;
		    if(example_objects.size()==0)
		    {
		    	example_obj = new MLExample();
		  
		    	
		    	example_obj.setCorpusName(experimentgroup);
		    	example_obj.setRelatedArtifact(artifact);
		    	
		    	if(Setting.SaveInGetInstance)
			    	saveExample(example_obj);
		    }else
		    {
		    	example_obj = 
		    			example_objects.get(0);
		    }
		    return example_obj;
	}
	public static MLExample findInstanceForArtifact(Artifact artifact,
			String experimentgroup) {
		String hql = "from MLExample where relatedArtifact = "+
				artifact.getArtifactId() + " and corpusName = '"+
				experimentgroup+"'";
			List<MLExample> example_objects = 
					getExamplesList(hql);
		    
			MLExample example_obj;
		    if(example_objects.size()==0)
		    {
		    	example_obj= null;
		    }else
		    {
		    	example_obj = 
		    			example_objects.get(0);
		    }
		    return example_obj;
	}

	public void calculateFeatures(
			List<IFeatureCalculator> featureCalculators) throws Exception {
		for(IFeatureCalculator feature_calculator : featureCalculators)
		{
			feature_calculator.calculateFeatures(this);
			HibernateUtil.clearLoaderSession();
		}
		
	}

	public static MLExample getInstanceForLink(PhraseLink phrase_link,
			String experimentgroup) {
		String hql = "from MLExample where relatedPhraseLink = "+
				phrase_link.getPhraseLinkId() + " and corpusName = '"+
						experimentgroup+"'";
			List<MLExample> example_objects = 
					getExamplesList(hql);
		    
			MLExample example_obj;
		    if(example_objects.size()==0)
		    {
		    	example_obj = new MLExample();
		  
		    	
		    	example_obj.setCorpusName(experimentgroup);
		    	example_obj.setRelatedPhraseLink(phrase_link);
		    	example_obj.setAssociatedFilePath(phrase_link.getFromPhrase().getStartArtifact().getAssociatedFilePath());
		    	
		    	if(Setting.SaveInGetInstance)
		    		saveExample(example_obj);
		    }else
		    {
		    	example_obj = 
		    			example_objects.get(0);
		    }
		    return example_obj;
	}


	public static void saveExample(MLExample example)
	{
		if(hibernateSession == null)
			hibernateSession = HibernateUtil.loaderSession;
		
		HibernateUtil.save(example, hibernateSession);
	}
	static List<MLExample> getExamplesList(String hql)
	{
		List<MLExample> examples;
		if(hibernateSession == null || !hibernateSession.isOpen())
			hibernateSession = HibernateUtil.loaderSession;
		examples = 
			(List<MLExample>) HibernateUtil.executeReader(hql, null, null, hibernateSession);
		return examples;
	}
	static List<MLExample> getExamplesList(String hql, Integer limit)
	{
		List<MLExample> examples;
		if(hibernateSession == null || !hibernateSession.isOpen())
			hibernateSession = HibernateUtil.loaderSession;
		examples = 
			(List<MLExample>) HibernateUtil.executeReader(hql, null, limit, hibernateSession);
		return examples;
	}
	static List<MLExample> getExamplesList(String hql, HashMap<String, Object> params)
	{
		List<MLExample> examples;
		if(hibernateSession == null || !hibernateSession.isOpen())
			hibernateSession = HibernateUtil.loaderSession;
		
		examples = 
			(List<MLExample>) HibernateUtil.executeReader(hql, params, null, hibernateSession);
		return examples;
	}
	public static List<MLExample> temptGetAllExamplesFotTrain(String experimentgroup, boolean for_train)
	{
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"' and forTrain="+(for_train?1:0)
						+"  and exampleId < 37406 "
						+ " order by relatedArtifact";
		
		return getExamplesList(hql);
	}
	public static List<MLExample> getAllExamples(String experimentgroup, boolean for_train)
	{
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"' and forTrain="+(for_train?1:0)
						+ " order by relatedArtifact";
		
		return getExamplesList(hql);
	}
	//get limited selected examples 
	public static List<MLExample> getLimitedPreSelectedExamples(String experimentgroup, boolean for_train,String condition)
	{
//		tempIsSelectedForTrain=1
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"' and forTrain="+(for_train?1:0)
						+ " and "+condition+" order by relatedArtifact";
		
		return getExamplesList(hql);
	}
	public static List<MLExample> getAllExamplesTwitter(String experimentgroup, boolean for_train)
	{
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"' and forTrain="+(for_train?1:0)
						+" and expectedClosure=1 order by exampleId";
		
		return getExamplesList(hql);
	}
	public static List<MLExample> getAllExamples(boolean for_train)
	{
		String hql = "from MLExample where  forTrain="+(for_train?1:0)
						+"  order by exampleId";
		
		return getExamplesList(hql);
	}
	public static List<MLExample> getExampleById(int example_id, String experimentgroup)
	{
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"' and exampleId="+example_id
						+" order by exampleId";
		return getExamplesList(hql);
	}
	public static MLExample getExampleById(int example_id)
	{
		String hql = "from MLExample where exampleId="+example_id;
		List<MLExample> example_objects = 
			(List<MLExample>) HibernateUtil.executeReader(hql);
    
		MLExample example_obj=null;
	    if(example_objects.size()!=0)
	    {
	    	example_obj = 
	    		example_objects.get(0);
	    }
	    return example_obj;
	}
	public static List<MLExample> getAllExamples(String experimentgroup, boolean for_train, int limit)
	{
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"'  and forTrain="+(for_train?1:0)+" order by exampleId";
		return getExamplesList(hql,limit);
	}
	public static List<MLExample> getLastExamples(String experimentgroup, boolean for_train, int limit)
	{
		String hql = "from MLExample where corpusName = '"+
			experimentgroup+"'  and forTrain="+(for_train?1:0)+
				"order by exampleId desc";
		return getExamplesList(hql);
	}
	
	public static List<MLExample> getExampleByExpectedClass(String experimentgroup,boolean for_train, LinkType type)
	{
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"' and expectedClass="+type.ordinal()
						+" and  forTrain="+(for_train?1:0)+" order by exampleId";
		return getExamplesList(hql);
	}
	
	public static List<MLExample> getExamplesInDocument(String experimentgroup, 
			String doc_path)
	{
		
		String hql = "FROM MLExample "  +
				"where corpusName =:corpusName " +
				" and associatedFilePath = '" +
				doc_path + "' " +
						"order by exampleId desc";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("corpusName", experimentgroup);
		
		return getExamplesList(hql, params);
	}

	public static List<MLExample> getExamplesByDocument(String experimentgroup, 
			boolean for_train, int num_of_documents)
	{
		List<Artifact> docs = Artifact.listByType(Type.Document, for_train);
		if(docs.size()<num_of_documents)
			num_of_documents = docs.size();
		
		String docPaths = "";
		for(int i=0;i<num_of_documents;i++)
			docPaths = docPaths.concat(", '"+docs.get(i).getAssociatedFilePath()+"'");
		docPaths = docPaths.replaceFirst(",", "");
		
		String hql = "FROM MLExample "  +
				"where corpusName =:corpusName " +
				" and forTrain="+(for_train?1:0) +" and associatedFilePath in (" +
						docPaths + ") " +
						"order by associatedFilePath desc";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("corpusName", experimentgroup);
		
		return getExamplesList(hql, params);
	}
	public static List<MLExample> getTokenExamplesBySent(Artifact sent,String experimentgroup, 
			boolean for_train)
	{	
//		String hql = "FROM MLExample "  +
//				"where corpusName =:corpusName " +
//				" and forTrain="+(for_train?1:0) +" and relatedArtifact.parentArtifact=" +sent.getArtifactId()+
//						
//						" order by associatedFilePath desc";
		String hql = "FROM MLExample "  +
				"where corpusName =:corpusName " +
				"  and relatedArtifact.parentArtifact=" +sent.getArtifactId()+
						
						" order by exampleID asc";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("corpusName", experimentgroup);
		
		return getExamplesList(hql, params);
	}
	
	public static List<MLExample> getExamplesByEventTypeByDocument(String experimentgroup, 
			boolean for_train, int num_of_documents, String type1,
			String type2,String order)
	{
		List<Artifact> docs = Artifact.listByType(Type.Document,for_train);
		if(docs.size()<num_of_documents)
			num_of_documents = docs.size();
		
		String docPaths = "";
		if (order.equals("top"))
		{
			for(int i=0;i<num_of_documents;i++)
				docPaths = docPaths.concat(", '"+docs.get(i).getAssociatedFilePath()+"'");
			docPaths = docPaths.replaceFirst(",", "");
		}
		else if(order.equals("last"))
		{
			for(int i=docs.size()-1;i>docs.size()-num_of_documents-1;i--)
				docPaths = docPaths.concat(", '"+docs.get(i).getAssociatedFilePath()+"'");
			docPaths = docPaths.replaceFirst(",", "");
		}
		Integer type1_from_fvpIds = FeatureValuePair.getRelatedFromEventTypeFValuePairIds(type1);
		Integer type1_to_fvpIds = FeatureValuePair.getRelatedToEventTypeFValuePairIds(type1);
		Integer type2_from_fvpIds = FeatureValuePair.getRelatedFromEventTypeFValuePairIds(type2);
		Integer type2_to_fvpIds = FeatureValuePair.getRelatedToEventTypeFValuePairIds(type2);
		
//		String from_fvpIds = "";
//		for(Integer id: fromFeatureValuePairIds)
//		{
//			from_fvpIds = from_fvpIds.concat(", '"+id+"'");
//		}
//		from_fvpIds = from_fvpIds.replaceFirst(",", "");
//		
//
//		String to_fvpIds = "";
//		for(Integer id: toTeatureValuePairIds)
//		{
//			to_fvpIds = to_fvpIds.concat(", '"+id+"'");
//		}
//		to_fvpIds = to_fvpIds.replaceFirst(",", "");
		
		String hql = " FROM MLExample m  "  +
	    "where (( exists (from MLExampleFeature f where m.exampleId =f.relatedExample and featureValuePair in ("+type1_from_fvpIds+"))" +
	    " and exists (from MLExampleFeature f where m.exampleId =f.relatedExample and featureValuePair in ("+type2_to_fvpIds+"))) or" +
	    " (exists (from MLExampleFeature f where m.exampleId =f.relatedExample and featureValuePair in ("+type2_from_fvpIds+")) and " +
	    		"exists (from MLExampleFeature f where m.exampleId =f.relatedExample and featureValuePair in ("+type1_to_fvpIds+")))) " +
	    " and corpusName =:corpusName " +
	    " and forTrain="+(for_train?1:0) +" and " +
	      "associatedFilePath in (" +
	      docPaths + ") " +
      "order by associatedFilePath desc";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("corpusName", experimentgroup);
		
		return getExamplesList(hql, params);
	}
	public static List<MLExample> getLastExamplesByDocument(String experimentgroup, 
			boolean for_train, int num_of_documents)
	{
		List<Artifact> docs = Artifact.listByType(Type.Document,for_train);
		if(docs.size()<num_of_documents)
			num_of_documents = docs.size();
		
		String docPaths = "";
		for(int i=docs.size()-1;i>docs.size()-num_of_documents-1;i--)
			docPaths = docPaths.concat(", '"+docs.get(i).getAssociatedFilePath()+"'");
		docPaths = docPaths.replaceFirst(",", "");
		
		String hql = "FROM MLExample "  +
				"where corpusName =:corpusName " +
				" and forTrain="+(for_train?1:0) +" and associatedFilePath in (" +
						docPaths + ") " +
						"order by associatedFilePath desc";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("corpusName", experimentgroup);
		
		return getExamplesList(hql, params);
	}

	@Override
	public MLExample clone()
	{
		if(relatedArtifact!=null)
			return getInstanceForArtifact(relatedArtifact, corpusName);
		else
			return getInstanceForLink(relatedPhraseLink, corpusName);	}


	
	public static void resetExamplesPredicted(String experimentgroup, boolean for_train) {
		String hql = "update MLExample set predictedClass = -1 where corpusName = '"+
				experimentgroup+"' and forTrain="+(for_train?1:0);
		HibernateUtil.executeNonReader(hql);
	}
	public static void setExamplePredictedClass(int example_id, int predicted) {
		String hql = "update MLExample set predictedClass = "+predicted+" where exampleId="+example_id;
		HibernateUtil.executeNonReader(hql);
	}
	public static void resetExamplesPredictedToDefault(String experimentgroup, boolean for_train, int default_predicted) {
		String hql = "update MLExample set predictedClass = "+default_predicted+" where corpusName = '"+
				experimentgroup+"' and forTrain="+(for_train?1:0);
		HibernateUtil.executeNonReader(hql);
	}

	public void setAssociatedFilePath(String associatedFilePath) {
		this.associatedFilePath = associatedFilePath;
	}

	public String getAssociatedFilePath() {
		return associatedFilePath;
	}
	public static void updateAssociatedFilePath() {
		String hql = "from MLExample ";
		Session tempSession = HibernateUtil.sessionFactory.openSession();
		List<MLExample> examples = 
				(List<MLExample>) HibernateUtil.executeReader(hql, null,null, tempSession);
		
		for (MLExample example: examples)
		{
			PhraseLink related_phrase_link = example.getRelatedPhraseLink();
			Phrase from_phrase = related_phrase_link.getFromPhrase();
			Artifact start_artifact = from_phrase.getStartArtifact();
			
			String file_path = start_artifact.getAssociatedFilePath();
			example.setAssociatedFilePath(file_path);
			HibernateUtil.save(example, tempSession);
			
		}
		tempSession.clear();
		tempSession.close();
	}

	public static MLExample findInstance(PhraseLink phrase_link,
			String experimentgroup) {
		String hql = "from MLExample where relatedPhraseLink = "+
				phrase_link.getPhraseLinkId() + " and corpusName = '"+
						experimentgroup+"'";
			List<MLExample> example_objects = 
					getExamplesList(hql);
		    
			MLExample example_obj=null;
		    if(example_objects.size()!=0)
		    {
		    	example_obj = 
		    			example_objects.get(0);
		    }
		    return example_obj;
	}
	public static MLExample findInstance(PhraseLink phrase_link) {
		String hql = "from MLExample where relatedPhraseLink = "+
				phrase_link.getPhraseLinkId();
		
			List<MLExample> example_objects = 
					getExamplesList(hql);
		    
			MLExample example_obj=null;
		    if(example_objects.size()!=0)
		    {
		    	example_obj = 
		    			example_objects.get(0);
		    }
		    return example_obj;
	}

	public void setPredictionWeight(double predictionWeight) {
		this.predictionWeight = predictionWeight;
	}

	public double getPredictionWeight() {
		return predictionWeight;
	}

	public static List<MLExample> getDecidedExamplesForGraph(Artifact p_sentence) {
		String hql = "from MLExample where predictedClass <> -1 " +
				"and relatedPhraseLink.fromPhrase.startArtifact.parentArtifact = "+
		p_sentence.getArtifactId()+" and relatedPhraseLink.toPhrase.startArtifact.parentArtifact = "+
		p_sentence.getArtifactId();

		List<MLExample> example_objects = 
				getExamplesList(hql);
	    
	    return example_objects;
	}
	 public static List<MLExample> getPhraseExamplesByPredictedClass(Artifact p_sentence, String corpusName, boolean forTrain, Integer predictedClass) {
		 

//		String hql = "from MLExample where (predictedClass = 1 or predictedClass=2 ) and forTrain="+(forTrain?1:0)
		String hql = "from MLExample where (predictedClass = "+predictedClass+" ) and forTrain="+(forTrain?1:0)
					+
				" and corpusName = '"+corpusName+"' and relatedPhrase.startArtifact.parentArtifact = "+p_sentence.getArtifactId();

		List<MLExample> example_objects = 
				getExamplesList(hql);
	    
	    return example_objects;
	}
	 public static List<MLExample> getPhraseExamplesByCorpus(Artifact p_sentence, String corpusName, boolean forTrain) {
		 
		String hql = "from MLExample where forTrain="+(forTrain?1:0)
					+
				" and corpusName = '"+corpusName+"' and relatedPhrase.startArtifact.parentArtifact = "+p_sentence.getArtifactId();

		List<MLExample> example_objects = 
				getExamplesList(hql);
	    
	    return example_objects;
	}

	public void setExpectedReal(int expectedReal) {
		this.expectedReal = expectedReal;
	}

	public int getExpectedReal() {
		return expectedReal;
	}

	public void setExpectedClosure(int expectedClosure) {
		this.expectedClosure = expectedClosure;
	}

	public int getExpectedClosure() {
		return expectedClosure;
	}

	public void setExpectedIntegrated(int expectedIntegrated) {
		this.expectedIntegrated = expectedIntegrated;
	}

	public int getExpectedIntegrated() {
		return expectedIntegrated;
	}

	public static MLExample getInstanceForPhrase(Phrase relatedPhrase,
			String experimentgroup) {
		String hql = "from MLExample where relatedPhrase = "+
				relatedPhrase.getPhraseId() + " and corpusName = '"+
				experimentgroup+"'";
			List<MLExample> example_objects = 
					getExamplesList(hql);
		    
			MLExample example_obj;
		    if(example_objects.size()==0)
		    {
		    	example_obj = new MLExample();
		    	example_obj.setCorpusName(experimentgroup);
		    	example_obj.setRelatedPhrase(relatedPhrase);
		    	
		    	if(Setting.SaveInGetInstance)
			    	saveExample(example_obj);
		    }else
		    {
		    	example_obj = 
		    			example_objects.get(0);
		    }
		    return example_obj;
	}

	@Transient
	public String getRelatedDrug() {
		return relatedDrug;
	}

	public void setRelatedDrug(String relatedDrug) {
		this.relatedDrug = relatedDrug;
	}
//	@OneToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} , fetch=FetchType.LAZY )
	@OneToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} )
    @JoinColumn(name="relatedPhrase")
	public Phrase getRelatedPhrase() {
		return relatedPhrase;
	}
	public void setRelatedPhrase(Phrase relatedPhrase) {
		this.relatedPhrase = relatedPhrase;
	}
} @Entity
@Table( name = "MLExampleFeature" )
public class MLExampleFeature {
	private Integer exampleFeatureId;
	
	private MLExample relatedExample;
	
	private FeatureValuePair featureValuePair;
	
	public MLExampleFeature()
	{
		
	}
	
	public void setExampleFeatureId(Integer _artifactFeatureId) {
		this.exampleFeatureId = _artifactFeatureId;
	}
	@Id
	@GeneratedValue(generator="increment")
	@GenericGenerator(name="increment", strategy = "increment")
	public Integer getExampleFeatureId() {
		return exampleFeatureId;
	}
	
	
	public void setRelatedExample(MLExample relatedExample) {
		this.relatedExample = relatedExample;
	}
	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} )
    @JoinColumn(name="relatedExample")
	public MLExample getRelatedExample() {
		return relatedExample;
	}
	
	
	public void setFeatureValuePair(FeatureValuePair _featureValuePair) {
		featureValuePair = _featureValuePair;
	}
	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} )
    @JoinColumn(name="featureValuePair")
	public FeatureValuePair getFeatureValuePair() {
		return featureValuePair;
	}

	public static MLExampleFeature setFeatureExample(MLExample pExample,
			FeatureValuePair pNewFeature) {
		return setFeatureExample(pExample, pNewFeature, false);
	}
	public static MLExampleFeature setFeatureExample(MLExample pExample,
			FeatureValuePair pNewFeature, boolean isDenseFeature) {

//		if(!isDenseFeature){
		if(pNewFeature.getFeatureValueAuxiliary()!=null)
			//is multi value
			deleteExampleFeatures(pExample, pNewFeature.getFeatureName()
					,pNewFeature.getFeatureValue());
		else
			deleteExampleFeatures(pExample, pNewFeature.getFeatureName());
//		}
		
		MLExampleFeature artifact_feature = new MLExampleFeature();
		artifact_feature.setFeatureValuePair(pNewFeature);
		artifact_feature.setRelatedExample(pExample);
		
		
		Session session = HibernateUtil.sessionFactory.openSession();
		
		HibernateUtil.save(artifact_feature, session);
		
		session.clear();
		session.close();
		
		return artifact_feature;
	}
	private static void deleteExampleFeatures(MLExample pExample,
			String featureName, String featureValue) {
		
		Session session = HibernateUtil.sessionFactory.openSession();
		session.beginTransaction();
		String hql = "from MLExampleFeature af where af.relatedExample = " +
				pExample.getExampleId()+" AND "+
				"af.featureValuePair.featureName = :featureName"+
				" AND af.featureValuePair.featureValue = :featureValue";
		
		List<MLExampleFeature> artifact_feature_list = 
			session.createQuery(hql).setParameter("featureValue", featureValue)
			.setParameter("featureName", featureName).list();
		
		
		for(MLExampleFeature af : artifact_feature_list)
			session.delete(af);
		
		session.getTransaction().commit();
		session.close();
	}

	/*
	 * delete all attribute name for an specific artifact
	 */
	public static void deleteExampleFeatures(MLExample pExample,
			String featureName) {
		String hql = "from MLExampleFeature as af inner join fetch af.featureValuePair " +
				"where af.relatedExample = " +
				pExample.getExampleId()+" AND "+
				"af.featureValuePair.featureName = :featureName";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("featureName", featureName);
		Session session = HibernateUtil.sessionFactory.openSession();
		
		List<MLExampleFeature> artifact_feature_list = 
			(List<MLExampleFeature>) HibernateUtil.executeReader(hql, params,null, session);
		
		if(artifact_feature_list.size()>0)
		{
			String example_feature_ids = "";
			for(MLExampleFeature af : artifact_feature_list)
				example_feature_ids=example_feature_ids.concat(af.getExampleFeatureId()+",");
			example_feature_ids = example_feature_ids.replaceAll(",$", "");
			HibernateUtil.executeNonReader("delete from MLExampleFeature where exampleFeatureId in ("+
					example_feature_ids+")");
		}
		
		session.close();
	}

	@Override public String toString()
	{
		return " Example = "+relatedExample.toString()+
			" / Feature = "+
			featureValuePair.toString();
	}
	
	@Transient
	public static void truncateTable()
	{
		String hql = "delete from MLExampleFeature";
		HibernateUtil.executeNonReader(hql);
	}

	public static void setBulkFeaturesExample(MLExample exampleToProcess,
			List<FeatureValuePair> features, String featurePrefix) {
//		HibernateUtil.executeNonReader("delete from MLExampleFeature where relatedExample = "+
//				exampleToProcess.getExampleId()+" and featureValuePair in ()");
		
		HibernateUtil.startTransaction();;
		Session session = HibernateUtil.saverSession;
		
		for (FeatureValuePair fvp : features)
		{
			MLExampleFeature artifact_feature = new MLExampleFeature();
			artifact_feature.setFeatureValuePair(fvp);
			artifact_feature.setRelatedExample(exampleToProcess);
			session.save(artifact_feature);
		}
		
		HibernateUtil.endTransaction(); 
		
		
	}

} public class HibernateUtil {
	//private static SessionFactory sessionFactory;
	private static ServiceRegistry serviceRegistry;
	// configures settings from hibernate.cfg.xml
	public static SessionFactory sessionFactory;      
	public static Session loaderSession;
	public static Session saverSession;
	public static boolean changeDB=true;
	public static String db;
	public static String user;
	public static String pass;
	
	static{
		
		sessionFactory = configureSessionFactory();
		loaderSession = sessionFactory.openSession();
		saverSession = sessionFactory.openSession();
	}
	static boolean inTransaction = false;
	
	
	private static SessionFactory configureSessionFactory() throws HibernateException {
		Configuration configuration = new Configuration();
	    configuration.configure();
	    serviceRegistry = new ServiceRegistryBuilder().applySettings(configuration.getProperties()).buildServiceRegistry();        
	    sessionFactory = configuration.buildSessionFactory(serviceRegistry);
	    
	    return sessionFactory;
	}

	public static void changeConfigurationDatabase(String databaseURL, String user, String pass){
	   	    
	    Configuration configuration = new Configuration();
		configuration.configure();
//		serviceRegistry = new ServiceRegistryBuilder().applySettings(configuration.getProperties()).buildServiceRegistry();   
//		configuration.setProperty("hibernate.connection.url", databaseURL);
//		configuration.setProperty("hibernate.connection.username", user);
//		configuration.setProperty("hibernate.connection.password", pass);
//	    configuration.configure();
		configuration.setProperty("connection.url", databaseURL);
		configuration.setProperty("hibernate.connection.url", databaseURL);
		configuration.setProperty("hibernate.connection.username", user);
		configuration.setProperty("hibernate.connection.password", pass);
		
		configuration.getProperty("hibernate.connection.url");
	    serviceRegistry = new ServiceRegistryBuilder().applySettings(configuration.getProperties()).buildServiceRegistry(); 
		
	  
	    sessionFactory = configuration.buildSessionFactory(serviceRegistry);
	    loaderSession = sessionFactory.openSession();
		saverSession = sessionFactory.openSession();
		MLExample.hibernateSession=loaderSession;
	}
	/**
	 * save object status
	 */
	public static void save(Object _object, Session session) {
		try {
			session.beginTransaction();
			session.saveOrUpdate(_object);
			session.flush();
			session.getTransaction().commit();
		} catch (HibernateException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	public static void save(Object _object) {
		save(_object, loaderSession);
	}
	
	public static List<?> executeReader(String hql)
	{
		return executeReader(hql, null, null, loaderSession);
	}
	
	public static List<?> executeReader(String hql, HashMap<String, Object> params)
	{
		return executeReader(hql, params, null, loaderSession);
	}
	
	public static List<?> executeReader(String hql, HashMap<String, Object> params, Integer limit)
	{
		return executeReader(hql, params, limit, loaderSession);
	}
	public static List<?> executeReader(String hql, HashMap<String, Object> params, Integer limit, Session session)
	{
		Query q = session.createQuery(hql);
		
		if(params!=null)
			for(String key :params.keySet())
			{
				Object val = params.get(key);
				if(val instanceof String)
					q.setString(key, (String)val);
				else
					q.setInteger(key, (Integer)val);
			}
		if(limit!=null)
			q.setMaxResults(limit);
		
		List<?> result_list = 
			q.list();
		
		return result_list;
	}
	
	
	
	public static void executeNonReader(String hql, HashMap<String, Object> params)
	{
		if(!inTransaction)
		{
//			session = sessionFactory.openSession();
			saverSession = clearSession(saverSession);
			saverSession.beginTransaction();
		}
		
		Query qr = saverSession.createQuery(hql);
		if (params!=null)
		for(String key :params.keySet())
		{
			Object val = params.get(key);
			qr.setParameter(key, val);
		}
		
		qr.executeUpdate();
		
		if(!inTransaction)
		{
			saverSession.flush();
			saverSession.getTransaction().commit();
//			session.close();
			
		}
	}
	
	
	public static Object getHibernateTemplate() {
		// TODO Auto-generated method stub
		return null;
	}
	public static void executeNonReader(String hql) {
		executeNonReader(hql, null);
		
	}
	public static void startTransaction() {
		saverSession = sessionFactory.openSession();
		saverSession.beginTransaction();
		inTransaction = true;
	}
	
		  
		
	public static void endTransaction() {
		saverSession.flush();
		if(!saverSession.getTransaction().wasCommitted())
			saverSession.getTransaction().commit();
		saverSession.close();
		inTransaction = false;
	}
	public static void clearLoaderSession()
	{
		loaderSession = clearSession(loaderSession);
	}
	public static Session clearSession(Session session)
	{
		if(session!=null && session.isOpen()){
			session.clear();
			session.close();
		}
		session= sessionFactory.openSession();
		return session;
	}
	
	
	public static void flushTransaction() {
		saverSession.flush();
		saverSession.getTransaction().commit();
		
	}
	public static Object executeGetOneValue(String sql,
			HashMap<String, Object> params) {
		Session session = sessionFactory.openSession();
		Query query = session.createSQLQuery(sql);
		if (params!=null)
			for(String key :params.keySet())
			{
				Object val = params.get(key);
				query.setParameter(key, val);
			}
		Object oneVal = query.uniqueResult();
		session.clear();
		session.close();
		return oneVal;
	}
} public class StringUtil {
	static String lemmatizerResourceFolder ="lemmatiser";
	static EngLemmatiser lemmatiser = null;
	static{
		initialize();
	}
	public static void initialize()
	{	
		try {
			System.out.println("start loading the lemmatiser...");
			System.out.println(System.getProperty("user.dir"));
			lemmatiser = new EngLemmatiser(System.getProperty("user.dir")+"/lemmatiser", true, false);
			System.out.println("Lemmatizer loaded");
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
//	static EngLemmatiser lemmatiser = new EngLemmatiser("/rnlp-stb/nlpdata/lemmatiser",	true, false);

	/**
	 * 
	 * @param inputString
	 * @return MD5 hash of given string
	 * @throws UnsupportedEncodingException
	 * @throws NoSuchAlgorithmException
	 */
	public static String getStringDigest(String inputString)
			throws UnsupportedEncodingException, NoSuchAlgorithmException {
		MessageDigest md = MessageDigest.getInstance("MD5");
		md.update(inputString.getBytes(), 0, inputString.length());

		return new BigInteger(1, md.digest()).toString(16);
	}

	/**
	 * Customized definition of stop words for word
	 * @param word
	 * @return
	 */
	public static List<String> customGoogleCommonWord = FileUtil.loadLineByLine
			("/home/azadeh/projects/drug-effect-ext/data/abbvie-experiments/google-comon-words.txt");
	public static boolean isStopWord(String word){
		boolean isStopWord = false;
		
		if(word.length() < 2
				||	Stopwords.isStopword(word)
				|| customGoogleCommonWord.contains(word)
				|| word.matches("\\W+")
				)
			isStopWord = true;
		
		return isStopWord;
	}
	public static String removeStopWords(String text){
		boolean isStopWord = false;
		String[] tokens = text.split(" ");
		String new_text = "";
		
		for (String token:tokens)
		{
			if(token.length() < 2
					||	Stopwords.isStopword(token)
					|| customGoogleCommonWord.contains(token)
					|| token.matches("\\W+")
					)
				continue;
			else
				new_text=new_text+" "+token;
			
		}
		
		return new_text.trim();
	}
	

	
	/**
	 * Porter stem
	 * @param word
	 * @return stemmed word
	 */
	public static String getWordPorterStem(String word)
	{
		PorterStemmer stemmer = new PorterStemmer();
		String stemmed_word = stemmer.stem(word).toLowerCase();
		return stemmed_word;
	}
	public static String prepareSQLString(String sqlString) {
		sqlString = sqlString.replace("\\", "\\\\").
			replace("'", "''").
			replace("%", "\\%").
			replace("_", "\\_");
		return sqlString;
	}

public static String castForRegex(String textContent) {
		
		return textContent.replace("\\","\\\\").replace("/","\\/").replace("*", "\\*").replace("+", "\\+").replace(".", "\\.").replace("?", "\\?")
			.replace(")", "\\)").replace("{", "\\{").replace("}", "\\}")
			.replace("(", "\\(").replace("[", "\\[").replace("]", "\\]").replace("%", "\\%");
	}
public static String decastRegex(String textContent) {
		
		return textContent.replace("\\\\","\\").replace("\\/","/").replace("\\*", "*").replace("\\+", "+").replace("\\.", ".").replace("\\?", "?")
			.replace("\\)", ")").replace("\\_", "_")
			.replace("\\{", "{").replace("\\}", "}").replace("\\(", "(").
			replace("\\[", "[").replace("\\]", "]").replace("\\%", "%");
	}
	
	public static String getTermByTermPorter(String phrase)
	{
		String[] words = phrase.split(" ");
		String rootString = "";
		for(int i=0;i<words.length;i++){
			rootString += StringUtil.getWordPorterStem(words[i])+" ";
		}
		return rootString.trim();
	}
	
	public static String compress(String text) {
		return text.replace(" ", "").replace(" ", "");
	}
	static HashMap<String, String> lemmaCache = new HashMap<String, String>();

	public static String getTermByTermWordnet(String phrase)
	{
		String[] words = phrase.split(" ");
		String rootString = "";
		for(int i=0;i<words.length;i++)
		{
			String lemma = lemmaCache.get(words[i]);
			if (words[i].equals(lemma)
					&& lemma.matches(".*ioning$"))
			{
				lemma=lemma.replaceAll("ioning$", "ion");
			}
			if(lemma == null)
			{
				lemma = lemmatiser.stem(words[i]);
				lemmaCache.put(words[i], lemma);
			}
			rootString = rootString.concat(lemma+" ");
		}
		
		return rootString.trim();
	}
	public static String getWordLemma(String word)
	{
		String word_lemma= word;

		word_lemma= lemmatiser.stem(word);
		return word_lemma;
	}

	public static String orderStringAlphabetically(String text) {
		String words[] =text.split(" ");
        String t;
        int n = words.length;
        int i,j,c;
        for (i=0; i<n-1; i++) 
        {
            for (j=i+1; j<n; j++) 
            {
                c = words[i].compareTo(words[j]);
                if (c >0) 
                {
                    t = words[i];
                    words[i] = words[j];
                    words[j] = t;       
                }   
            }
        }
        String ordered_words = "";
        for (i=0; i<n ;i++) 
        {
        	ordered_words +=words[i]+" ";
        }
		return ordered_words.trim();
	}

	public static String cleanString(String text) {
		String clean = text.replaceAll("[\\d[^\\w\\s]]+", " ").replaceAll("(\\s{2,})", " ").toLowerCase();
		return clean.trim();
	}
} public class DidYouMeanIndexer {
	String correctText;
	public HashMap<String, String> lemma_correct_wrong_spell_map = new HashMap<>();

    public static void main (String[] args) throws IOException, ParseException
    {
    	DidYouMeanIndexer sc = new DidYouMeanIndexer();
    	String correct = sc.getTermByTermCorrectSpell("diarrhea");
    	System.out.println(correct);
    }
    public static void createSpellIndex(String spellIndexDirectoryPath,String sourceDirectoryPath) throws IOException {

        Directory sourceDirectory = FSDirectory.open(new File(sourceDirectoryPath));
        
        Directory spellIndexDirectory = FSDirectory.open(new File(spellIndexDirectoryPath));
        
        SpellChecker spellChecker = new SpellChecker(spellIndexDirectory);
        IndexReader reader = IndexReader.open(sourceDirectory);
        
       
        LuceneDictionary dictionary = new LuceneDictionary(reader, "content");
        
        IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LUCENE_36, new StandardAnalyzer(Version.LUCENE_36,CharArraySet.EMPTY_SET));
        spellChecker.indexDictionary(dictionary, indexWriterConfig, true);
           
        spellChecker.close();
    }

    public static void search(String queryString) throws IOException, ParseException {
        long startTime = System.currentTimeMillis();
        IndexSearcher is = null;
        try {
        	
        	Directory indexDirectory = FSDirectory.open(new File(Setting.getValue("LuceneIndexFile")+"/adrProjectLexicon"));
			
			
			QueryParser qp = new QueryParser(Version.LUCENE_36, "content", new StandardAnalyzer(Version.LUCENE_36));
			
			Query query = qp.parse("content:"+queryString);
			 
			IndexReader reader = IndexReader.open(indexDirectory);
			
			IndexSearcher searcher = new IndexSearcher(reader);
           
            qp.setDefaultOperator(QueryParser.AND_OPERATOR);
            
            ScoreDoc[] hits = searcher.search(query,1).scoreDocs;
           
//            Hits hits = searcher.search(query);
            long endTime = System.currentTimeMillis();
            
        } finally {
            if (is != null) {
                is.close();
            }
        }
    }
    static String luceneDir = System.getProperty("user.dir")+"/LuceneIndexes/EnglishSpellIndex";
    static Directory spell_directory;
    static IndexReader reader;
    static SpellChecker spellChecker;
    static{
    	 try {
			spell_directory = FSDirectory.open(new File(luceneDir));
			reader = IndexReader.open(spell_directory);
			spellChecker= new SpellChecker(spell_directory);
	        spellChecker.setStringDistance(new JaroWinklerDistance());
		} catch (IOException e) {
			//TODO
			System.out.println(e.getMessage());
		}
    }
	
	
    public static String getCorrectSpell(String wordForSuggestions ) throws IOException
    {
    	String correct_spell = wordForSuggestions;
    	
    	if (StringUtil.isStopWord(wordForSuggestions))
    	{
    		return correct_spell;
    	}
  
 
        
        if (!spellChecker.exist(wordForSuggestions))
        {
//        	String[] suggestions = spellChecker.suggestSimilar(wordForSuggestions, 1,reader,"content" ,SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX);
        	String[] suggestions = spellChecker.suggestSimilar(wordForSuggestions, 5,0.7f);
        	if (suggestions!=null && suggestions.length>0) {
                 for (String word : suggestions) {
                	 if (!(word.startsWith(String.valueOf(wordForSuggestions.charAt(0)))) )
                	 {
                		 continue;
                	 }
                	 correct_spell = word;
                	 break;
//                	 System.out.println("suggestions found for word:"+correct_spell);
                 }
             }
             else {
                 System.out.println("No suggestions found for word:"+wordForSuggestions);
             }
        }
       
    	return correct_spell;
    }
    public String getTermByTermCorrectSpell(String phrase) throws IOException
    {
    	
    	phrase = phrase.toLowerCase();
    	
    	String[] words = phrase.split(" ");
		String correctSpellString = "";
		for(int i=0;i<words.length;i++)
		{
			
			String correct = getCorrectSpell(words[i]);
			lemma_correct_wrong_spell_map.put(StringUtil.getTermByTermWordnet(correct), StringUtil.getTermByTermWordnet(words[i]));
			correctSpellString = correctSpellString.concat(correct+" ");
		}
		
		correctText = correctSpellString.trim();
		//%%% This is an special handling for n't
		if (phrase.matches("n't"))
			return phrase;
		return correctText;
    }
    

    

  
}
import java.util.List;
import LuceneManagerADR.ADRLuceneSearcher;
import LuceneManagerADR.DidYouMeanIndexer;
import rainbownlp.core.Artifact;
import rainbownlp.core.FeatureValuePair;
import rainbownlp.core.FeatureValuePair.FeatureName;
import rainbownlp.machineLearning.IFeatureCalculator;
import rainbownlp.machineLearning.MLExample;
import rainbownlp.machineLearning.MLExampleFeature;
import rainbownlp.util.HibernateUtil;
import rainbownlp.util.StringUtil;
import LuceneManagerADR.DidYouMeanIndexer;
/**
 * 
 */
package edu.asu.diego.adrmine.features;





/**
 * @author Azadeh
 * 
 */
public class TokenBasicFeatures implements IFeatureCalculator {
	
	
	public static void main (String[] args) throws Exception
	{
		

	}
	static DidYouMeanIndexer SpellSearcher;
	static{
		System.out.println("****** initializing spell corrector****");
		SpellSearcher= new DidYouMeanIndexer();
	}
	@Override
	public void calculateFeatures(MLExample exampleToProcess) throws Exception {
		
		
		
		//get related artifact
		Artifact relatedArtifact = exampleToProcess.getRelatedArtifact();
		String content = relatedArtifact.getContent();
		
		
		String lemma = content;
		
		if (lemma.matches("\\w+"))
		{
			lemma = StringUtil.getTermByTermWordnet(lemma);