public class ADRLuceneSearcher {
	static Directory indexDirectory;
	private static  String lucenePath;

	public static int topN=60;
	static{  
		
		setLucenePath(Setting.getValue("LuceneIndexFile")+"/adrProjectLexicon");
//		setLucenePath(Setting.getValue("LuceneIndexFile")+"/adrProjectLexiconSpell");
//		lucenePath = Setting.getValue("LuceneIndexFile")+"/adrProjectLexiconExactMatch";


	}
	
	
	public static void main(String[] args ) throws IOException, ParseException
	{	
		String test_sent =  "I gained an two cardiorespiratory arrests";
		DidYouMeanIndexer si = new DidYouMeanIndexer();
		test_sent = si.getTermByTermCorrectSpell(test_sent);
		
		test_sent= StringUtil.cleanString(test_sent);
		
			test_sent = StringUtil.removeStopWords(test_sent);
		
		
		
		String test = StringUtil.getTermByTermWordnet(test_sent);
//		ADRLexiconEntry ADR_le = getLexiconEntryByContent(test_sent);
		ArrayList<ADRLexiconEntry> ret = ADRLuceneSearcher.getTopRatedADRFromLuceneIndex(test,topN,Setting.getValue("LuceneIndexFile")+"/adrProjectLexicon");
//		int doc = getDocFreq("at",ADRCorpusIndex.getLucenePath());
//		System.out.println(doc);
	}

	public static ADRLexiconEntry getLexiconEntryByContent(String queryContent) throws ParseException {
		String adr_content= null;
		queryContent = queryContent.replaceAll("\\W", "");
		if (!queryContent.matches("\\w+")
				|| queryContent.isEmpty())
		{
			return null;
		}
		try {
			indexDirectory = FSDirectory.open(new File(getLucenePath()));
			
			
			QueryParser qp = new QueryParser(Version.LUCENE_36, "content", new StandardAnalyzer(Version.LUCENE_36));
			Query query = qp.parse("content:"+queryContent);
//			FuzzyQuery query = new FuzzyQuery(new Term("content", queryContent),0,2);
			 
			IndexReader reader = IndexReader.open(indexDirectory);
			
			IndexSearcher searcher = new IndexSearcher(reader);
			ScoreDoc[] hits = searcher.search(query, null, 1).scoreDocs;
//			TopScoreDocCollector collector = TopScoreDocCollector.create(1, true);
//			searcher.search(query,collector);
//			ScoreDoc[] hits = collector.topDocs().scoreDocs;
			if(hits.length>0)
			{
				int i=0;
				for (ScoreDoc h:hits)
				{
					float score = h.score;
					Document hitDoc = searcher.doc(hits[i].doc);
					
					adr_content = hitDoc.get("content");
					
					System.out.println(adr_content);
					System.out.println(score);
					String umls_id = hitDoc.get("umls_id");
//					if (score>=9.0)
//					{
						
						System.out.println(adr_content);
//					}
					i++;
				}
				Document hitDoc = searcher.doc(hits[0].doc);
				
				adr_content = hitDoc.get("content");
				String umls_id = hitDoc.get("umls_id");
				
				ADRLexiconEntry lexicon_entry = new ADRLexiconEntry(adr_content, umls_id);
				
				return lexicon_entry;
			}
				
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} 
//		catch (ParseException e) {
//			// TODO Auto-generated catch block
//			e.printStackTrace();
//		} 
		catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return null;
	}

	public static int getDocFreq(String token,String lecene_path) throws IOException
	{
		indexDirectory = FSDirectory.open(new File(lecene_path));
		
		token =StringUtil.getTermByTermWordnet(token);
		 
		IndexReader reader = IndexReader.open(indexDirectory);
		TermEnum termEnum = reader.terms(new Term("content", token));
		
		int doc_freq = termEnum.docFreq();
		
		return doc_freq;
	}

	public static String getLucenePath() {
		return lucenePath;
	}

	public static void setLucenePath(String lucenePath) {
		ADRLuceneSearcher.lucenePath = lucenePath;
	}
	
	public static ArrayList<ADRLexiconEntry>  getTopRatedADRFromLuceneIndex(String querystr, int hitsPerPage,
			String luceneIndexPath) { 
		ArrayList<ADRLexiconEntry> search_result = new ArrayList<>();
	    try { 
	    	
	      // Instantiate a query parser 
			QueryParser qp = new QueryParser(Version.LUCENE_36, "content", new StandardAnalyzer(Version.LUCENE_36,CharArraySet.EMPTY_SET ));
		  // Parse 
			Query query = qp.parse("content:"+querystr);
//			indexDirectory = FSDirectory.open(new File(Setting.getValue("LuceneIndexFile")+"/adrProjectLexicon"));
			indexDirectory = FSDirectory.open(new File(luceneIndexPath));
	      // Instantiate a searcher 
	  	 IndexReader reader = IndexReader.open(indexDirectory);
		
		 IndexSearcher searcher = new IndexSearcher(reader); 
	      // Ranker 
	      TopScoreDocCollector collector = TopScoreDocCollector.create( 
	          hitsPerPage, true); 
	      // Search! 
	      searcher.search(query, collector); 
	      // Retrieve the top-n documents 
	      ScoreDoc[] hits = collector.topDocs().scoreDocs; 
	 
	      // Display results 
//	      System.out.println("Found " + hits.length + " hits."); 
	      for (int i = 0; i < hits.length; ++i) { 
	        int docId = hits[i].doc; 
	        Document d = searcher.doc(docId); 
	        

			String adr_content = d.get("content");
			String umls_id = d.get("umls_id");
			ADRLexiconEntry lexicon_entry = new ADRLexiconEntry(adr_content, umls_id);
			search_result.add(lexicon_entry);
	        
	        System.out.println((i + 1) + ". " + d.get("content")); 
	      } 
	 
	      // Close the searcher 
	      searcher.close(); 
	    } catch (Exception e) { 
	      System.out.println("Got an Exception: " + e.getMessage()); 
	    } 
	    return search_result;
	  }
	public static boolean isPhraseInIndex(String querystr, int hitsPerPage, String indexPath) { 
		boolean is_in_index=false;
		
		
	    try { 
	    	
	      // Instantiate a query parser 
			QueryParser qp = new QueryParser(Version.LUCENE_36, "content", new StandardAnalyzer(Version.LUCENE_36));
		  // Parse 
			Query query = qp.parse("content:"+querystr);
			indexDirectory = FSDirectory.open(new File(indexPath));
		      // Instantiate a searcher 
		  	 IndexReader reader = IndexReader.open(indexDirectory);
			
			 IndexSearcher searcher = new IndexSearcher(reader); 
			 // Ranker 
			 TopScoreDocCollector collector = TopScoreDocCollector.create( 
	          hitsPerPage, true); 
			 // Search! 
			 searcher.search(query, collector); 
	      // Retrieve the top-n documents 
			 ScoreDoc[] hits = collector.topDocs().scoreDocs; 

		     for (int i = 0; i < hits.length; ++i) { 
		        int docId = hits[i].doc; 
		        Document d = searcher.doc(docId); 
		        String adr_content = d.get("content");
				String umls_id = d.get("umls_id");
				
		        is_in_index =true;

	//	        System.out.println((i + 1) + ". " + d.get("content")); 
		        break;
	      } 
	 
	      // Close the searcher 
	      searcher.close();
	      reader.close();
	      indexDirectory.close();
	      
	    } catch (Exception e) { 
	      System.out.println("Got an Exception: " + e.getMessage()); 
	    } 
	    return is_in_index;
	  }
	public static boolean isPhraseInIndexVerbatim(String querystr, int hitsPerPage, String indexPath) { 
		boolean is_in_index=false;
		
		
	    try { 
	    	
	      // Instantiate a query parser 
			QueryParser qp = new QueryParser(Version.LUCENE_36, "content", new StandardAnalyzer(Version.LUCENE_36));
		  // Parse 
			Query query = qp.parse("content:"+querystr);
			indexDirectory = FSDirectory.open(new File(indexPath));
	      // Instantiate a searcher 
	  	 IndexReader reader = IndexReader.open(indexDirectory);
		
		 IndexSearcher searcher = new IndexSearcher(reader); 
	      // Ranker 
	      TopScoreDocCollector collector = TopScoreDocCollector.create( 
	          hitsPerPage, true); 
	      // Search! 
	      searcher.search(query, collector); 
	      // Retrieve the top-n documents 
	      ScoreDoc[] hits = collector.topDocs().scoreDocs; 
	 
	      // Display results 
	      System.out.println("Found " + hits.length + " hits."); 
	      boolean is_in_index_with_all_tokens = true;
	      for (int i = 0; i < hits.length; ++i) { 
	        int docId = hits[i].doc; 
	        Document d = searcher.doc(docId); 
	        String adr_content = d.get("content");
	        String[] adr_content_tokens =adr_content.split(" ");
	        for (String token:adr_content_tokens)
	        {
	        	if (!querystr.matches(".*"+token+".*"))
	        	{
	        		is_in_index_with_all_tokens=false;
	        		break;
	        	}
	        }
			if (!is_in_index_with_all_tokens)
			{
				continue;
			}
	        is_in_index =true;

	        
	        System.out.println((i + 1) + ". " + d.get("content")); 
	        break;

	      } 
	 
	      // Close the searcher 
	      searcher.close(); 
	    } catch (Exception e) { 
	      System.out.println("Got an Exception: " + e.getMessage()); 
	    } 
	    return is_in_index;
	  }
	
} public class TokenSequenceExampleBuilder {
	//This classifies a chunk as having ADR or not
	public static String ExperimentGroupADRConcepts = "ADRConceptTokenSeq";

	public enum TokenLabel {
		OUT(0),
		BADR(1),
		IADR(2),
		BIND(3),
		IIND(4),
		BBEN(5),
		IBEN(6);
		private static final Map<Integer,TokenLabel> lookup = 
			new HashMap<Integer,TokenLabel>();
		
		static {
	          for(TokenLabel l : EnumSet.allOf(TokenLabel.class))
	               lookup.put(l.getCode(), l);
	     }
		
		private int code;

	     private TokenLabel(int code) {
	          this.code = code;
	     }

	     public int getCode() { return code; }

	     public static TokenLabel getEnum(int code) { 
	          return lookup.get(code); 
	     }	
	     
	}
	public static void createTokenSequenceExamples(boolean forTrain,String corpusName)
	{
		
		//get all child artifacts and check if they are in ADR concepts
		List<Artifact> all_sentences = Artifact.listByTypeByForTrainByCorpus(Type.Sentence,forTrain,corpusName);
		
		List<String> crf_format = new ArrayList<String>();
		int count=0;
		int all =all_sentences.size();
		for (Artifact sent:all_sentences)
		{
			// build one example for each
			HashMap<Artifact, Integer> token_label_map = new HashMap<Artifact, Integer>();
			
			List<Artifact> tokens = sent.getChildsArtifact();
			
			for (Artifact token:tokens)
			{
				token_label_map.put(token, TokenLabel.OUT.ordinal());
			}

			for (Artifact token: token_label_map.keySet())
			{
				MLExample token_seq_example = 
						MLExample.getInstanceForArtifact(token, corpusName);
				Integer label = token_label_map.get(token);
				
				token_seq_example.setExpectedClass(label);
					
				token_seq_example.setPredictedClass(-1);
				
				token_seq_example.setForTrain(forTrain);
				token_seq_example.setAssociatedFilePath(token.getAssociatedFilePath());
				
				MLExample.saveExample(token_seq_example);
			}
			count++;
			System.out.println("Building classification candidates... count: "+count+"/"+all);
			HibernateUtil.clearLoaderSession();
		}

	
	}	
} public enum TokenLabel {
	OUT(0),
	BADR(1),
	IADR(2),
	BIND(3),
	IIND(4),
	BBEN(5),
	IBEN(6);
	private static final Map<Integer,TokenLabel> lookup = 
		new HashMap<Integer,TokenLabel>();
		
	static {
          for(TokenLabel l : EnumSet.allOf(TokenLabel.class))
               lookup.put(l.getCode(), l);
     }
		
	private int code;

     private TokenLabel(int code) {
          this.code = code;
     }

     public int getCode() { return code; }

     public static TokenLabel getEnum(int code) { 
          return lookup.get(code); 
     }	
	     
} public enum TargetSemanticType{
	ADR,
	Indication,
	//healthRelated
	ADR_Indication
} public class StanfordParser {
	
	static LexicalizedParser lp;
	static TreebankLanguagePack tlp;
	
	static{
		tlp = new PennTreebankLanguagePack();
	}
	
	public String tagged_sentence;
	public Tree bufferTree = null;
	
	public static void main(String[] args)
	{
		StanfordParser parser = new StanfordParser();
		parser.parse("this is a test");
		String tagged = parser.getTagged();
		String dependencies = parser.getDependencies();
		String penn_tree = parser.getPenn();
		
		System.out.println(tagged);
		System.out.println(penn_tree);
		System.out.println(dependencies);
		
		String tokenized=getTokenizedSentence("Currently not on b/c of cost .");
	}
	public StanfordParser()
	{
		if(lp == null)
			lp = LexicalizedParser.loadModel("edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz");  
	}
	
	public String getTagged() {

		TreePrint tp = new TreePrint("wordsAndTags");
		Writer parse_string = new StringWriter();
		PrintWriter printWriter = new PrintWriter(parse_string);
		
		tp.printTree(bufferTree, printWriter); // print tree
	
		return parse_string.toString();
		
	}
	
	
	
	public void parse(String sentence)
	{
		if(sentence.equals("")) return;
		// prepare Parser, Tokenizer and Tree printer:
		if(lp == null)
			lp = LexicalizedParser.loadModel("nlpdata/englishFactored.ser.gz"); 

		
		
		// print sentence:
		System.out.println ("\n\n\n\nORIGINAL:\n\n" + sentence);
		
		
		// put tokens in a list:
		Tokenizer<? extends HasWord> toke = 
	        tlp.getTokenizerFactory().getTokenizer(new StringReader(sentence));
	    List<? extends HasWord> sentence_tokenized = toke.tokenize();
	      
		if(sentence_tokenized.size() ==0 || (sentence_tokenized.size()==1 && sentence_tokenized.get(0).equals(""))) return;
		
		bufferTree = lp.apply(sentence_tokenized);
	}
	
	public String getPenn()
	{
		if(bufferTree==null) return "";
		TreePrint tp = new TreePrint("penn");
		
		Writer parse_string = new StringWriter();
		PrintWriter printWriter = new PrintWriter(parse_string);
		
		tp.printTree(bufferTree, printWriter); // print tree
		
		return parse_string.toString();
	}
	
	public Tree getParseTree(String sentence)
	{
		TokenizerFactory tf = PTBTokenizer.factory(false, new WordTokenFactory());
		TreePrint tp = new TreePrint("penn,typedDependenciesCollapsed");
		

		List tokens = tf.getTokenizer(new StringReader(sentence)).tokenize();
		Tree parse_tree = (Tree) lp.apply(tokens);
		return parse_tree;
	}
	
	public static String getTokenizedSentence(String sentence)
	{
		 String tokenizeSentence="";
			
	      PTBTokenizer ptbt = new PTBTokenizer(new StringReader(sentence),
	              new CoreLabelTokenFactory(), "");
	      for (CoreLabel label; ptbt.hasNext(); ) {
	        label = (CoreLabel) ptbt.next();
	        tokenizeSentence+=" "+label.originalText();
	        
	      }
		return tokenizeSentence.trim();
//		TokenizerFactory tf = PTBTokenizer.factory(false, new WordTokenFactory());
//		TreePrint tp = new TreePrint("penn,typedDependenciesCollapsed");
//		
//
//		List<Word> tokens = tf.getTokenizer(new StringReader(sentence)).tokenize();
//		String tokenized = "";
//		for (Word token:tokens)
//		{
//			tokenized+=" "+token.toString();
//		}
//		System.out.println(tokenized.trim());
//		return tokenized.trim();
	}
	public String getDependencies()
	{
		TreePrint tp = new TreePrint("typedDependenciesCollapsed");
		Writer parse_string = new StringWriter();
		PrintWriter printWriter = new PrintWriter(parse_string);
		
		tp.printTree(bufferTree, printWriter); // print tree
		
		return parse_string.toString();
	}
	
	
	
	public void load(String penn) throws IOException
	{
	    
	    Reader in = new StringReader(penn);
	    PennTreeReader tr = new PennTreeReader(in, new LabeledScoredTreeFactory(),
                new NPTmpRetainingTreeNormalizer());
	    bufferTree = tr.readTree();
	}
	
	public void load(String penn,String tagged) throws IOException
	{
	    
	    Reader in = new StringReader(penn);
	    PennTreeReader tr = new PennTreeReader(in, new LabeledScoredTreeFactory(),
                new NPTmpRetainingTreeNormalizer());
	    bufferTree = tr.readTree();
	    
	    tagged_sentence = tagged;
	}
	
} @Entity
@Table( name = "Artifact" )
public class Artifact {
	String content;
	String generalized_content;
	List<Artifact> childsArtifact;
	Artifact parentArtifact;
	Artifact nextArtifact;
	Artifact previousArtifact;
	Type artifactType;
	private boolean forTrain;
	private boolean forDemo=false;
	
	
	
	@Transient
	boolean isNew;
	@Transient
//	List<ArtifactFeature> features = new ArrayList<ArtifactFeature>();
	
	String POS;
	
	// These two variable are lazy and will be initialized in getter
	Integer startIndex = null;
	Integer endIndex = null;

	Integer lineOffset = null;
	Integer wordOffset = null;
	
	String stanDependency =null;
	private String stanPennTree = null;
	
	int artifactId = -1;
	/*
	 * Physical address of file which artifact loaded from that
	 */
	String associatedFilePath;
	String corpusName;

	public enum Type {
		Document, Sentence, Phrase, Word
	}

	
	public Artifact()
	{
		
	}
	/**
	 * Loads Artifact by id
	 * @param pArtifactID
	 * @return
	 */
	public static Artifact getInstance(int pArtifactID) {
		String hql = "from Artifact where artifactId = "+pArtifactID;
		Artifact artifact_obj = 
			(Artifact)HibernateUtil.executeReader(hql).get(0);
		return artifact_obj;
	}
	/**
	 * Creates completely empty Artifact object
	 * @return
	 */
	public static Artifact getInstance(Type pArtifactType) {
		Artifact artifact_obj = new Artifact();
		artifact_obj.setArtifactType(pArtifactType);
		HibernateUtil.save(artifact_obj);
		return artifact_obj;
	}
	
	/**
	 * Loads or creates the Artifact
	 * @param pArtifactType
	 * @param pFilePath
	 * @param pStartIndex
	 * @return
	 */
	public static Artifact getInstance(Type pArtifactType, String pFilePath, 
			int pStartIndex){
		String hql = "from Artifact where artifactType = "+
			pArtifactType.ordinal()+" and associatedFilePath ='" +
			pFilePath + "' and startIndex="+pStartIndex;
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	artifact_obj.setStartIndex(pStartIndex);
	    	artifact_obj.setAssociatedFilePath(pFilePath);
	    	artifact_obj.setArtifactType(pArtifactType);
	    	artifact_obj.setForTrain(Setting.TrainingMode);
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact getInstance(Type pArtifactType, String pFilePath, 
			int pStartIndex,String corpusName){
		String hql = "from Artifact where artifactType = "+
			pArtifactType.ordinal()+" and associatedFilePath ='" +
			pFilePath + "' and corpusName= '"+corpusName+"' and startIndex="+pStartIndex;
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	artifact_obj.setStartIndex(pStartIndex);
	    	artifact_obj.setAssociatedFilePath(pFilePath);
	    	artifact_obj.setArtifactType(pArtifactType);
	    	artifact_obj.setForTrain(Setting.TrainingMode);
	    	artifact_obj.setCorpusName(corpusName);
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact getInstance(Type pArtifactType, String pFilePath, 
			int pStartIndex,boolean forDemo){
		String hql = "from Artifact where artifactType = "+
			pArtifactType.ordinal()+" and associatedFilePath ='" +
			pFilePath + "' and startIndex="+pStartIndex+" and forDemo="+forDemo;
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	artifact_obj.setStartIndex(pStartIndex);
	    	artifact_obj.setAssociatedFilePath(pFilePath);
	    	artifact_obj.setArtifactType(pArtifactType);
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact getInstance(int pWordIndex,Type pArtifactType, String pFilePath
			){
		String hql = "from Artifact where artifactType = "+
			pArtifactType.ordinal()+" and associatedFilePath ='" +
			pFilePath + "' and wordIndex="+pWordIndex;
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	artifact_obj.setWordIndex(pWordIndex);
	    	artifact_obj.setAssociatedFilePath(pFilePath);
	    	artifact_obj.setArtifactType(pArtifactType);
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact getInstance(Type pArtifactType, String unique_id, 
			String content){
		String hql = "from Artifact where artifactType = "+
			pArtifactType.ordinal()+" and associatedFilePath ='" +
			unique_id+"'";
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	
	    	artifact_obj.setAssociatedFilePath(unique_id);
	    	artifact_obj.setContent(content);
	    	artifact_obj.setArtifactType(pArtifactType);
	    	
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact getInstanceByEndChar(Type pArtifactType, String pFilePath, 
			int pEndIndex){
		String hql = "from Artifact where artifactType = "+
			pArtifactType.ordinal()+" and associatedFilePath ='" +
			pFilePath + "' and endIndex="+pEndIndex;
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	artifact_obj.setEndIndex(pEndIndex);
	    	artifact_obj.setAssociatedFilePath(pFilePath);
	    	artifact_obj.setArtifactType(pArtifactType);
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact getMadeUpInstance(String content){
		String hql = "from Artifact where content = '"+content+"'";
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj;
	    if(artifact_objects.size()==0)
	    {
	    	artifact_obj = new Artifact();
	    	artifact_obj.setContent(content);
	    	HibernateUtil.save(artifact_obj);
	    }else
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE})
//	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE}, fetch=FetchType.LAZY )
//	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} )
    @JoinColumn(name="parentArtifact")
	public Artifact getParentArtifact()  {
		return parentArtifact;
	}
	public void setParentArtifact(Artifact _parentArtifact) {
		parentArtifact = _parentArtifact;
	}
	
//	@OneToMany(mappedBy="parentArtifact")
//    @OrderBy("startIndex")
	@Transient
	public List<Artifact> getChildsArtifact() {
		if(childsArtifact==null && artifactType != Type.Word)
			childsArtifact = 
				(List<Artifact>) HibernateUtil.executeReader( "from Artifact where parentArtifact = "+artifactId+" order by artifactId");

		return childsArtifact;
	}
	

	
	
//	@OneToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE}, fetch=FetchType.LAZY )
	@OneToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} )
	@JoinColumn(name="nextArtifact")
    public Artifact getNextArtifact() {
		return nextArtifact;
	}
	public void setNextArtifact(Artifact pNextArtifact){
		nextArtifact = pNextArtifact;
	}
	

//	@OneToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE}, fetch=FetchType.LAZY )
	@OneToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} )
	@JoinColumn(name="previousArtifact")
    public Artifact getPreviousArtifact() {
		return previousArtifact;
	}
	public void setPreviousArtifact(Artifact pPreviousArtifact){
		previousArtifact = pPreviousArtifact;
	}
	/**
	 * Return oldest parent of this Artifact which is usually a Document
	 */
	@Transient
	public Artifact grandFather() throws SQLException {
		Artifact grandFather = this;

		while (grandFather.getParentArtifact() != null) {
			grandFather = grandFather.getParentArtifact();
		}

		return grandFather;
	}

	@Index(name = "index_start_index")
	public Integer getStartIndex() {
		if (this.getArtifactType() == Type.Document) {
			startIndex = 0;
		} else if (startIndex == null) {
			int _previousArtifactsLength = 0;
			Artifact previous = this.getPreviousArtifact();
			while (previous != null) {
				_previousArtifactsLength += previous.getContent().length() + 1;
				previous = previous.getPreviousArtifact();
			}
			if(getParentArtifact()!=null)
				startIndex = parentArtifact.getStartIndex()
						+ _previousArtifactsLength + 1;
			
		}
		return startIndex;
	}

	public void setEndIndex(Integer _endIndex) {
		if (_endIndex == null && !this.getContent().isEmpty()) {
			_endIndex = startIndex + this.getContent().length();
		}
		endIndex = _endIndex;
	}

	@Index(name = "index_end_index")
	public Integer getEndIndex() {
		if (endIndex == null)
		{
			if(getStartIndex()!=null)
				endIndex = startIndex + getContent().length();
		}
		
		return endIndex;
	}

	

	public String getPOS()
	{
//		if(POS==null){
////			try {
////				POS = 
////					ArtifactAttributeTable.getAttributeValue(artifactId, 
////							AttributeValuePair.AttributeType.POS.name());
////			} catch (SQLException e) {
////				// TODO Auto-generated catch block
////				e.printStackTrace();
////			}
//			if(POS==null && artifactType == Type.Word){
//				String contentRegexCasted = StringUtil.castForRegex(getContent());
//				lineOffset = getLineIndex();
//				if(lineOffset==null) return "";
//				String tagged = 
//					StanfordParser.getTagged(associatedFilePath, lineOffset);
//				String[] taggedTokens = 
//					tagged.split(" ");
//				wordOffset = getWordIndex();
//				boolean index_invalid = false;
//				if(wordOffset!=null && 
//						taggedTokens.length>wordOffset)
//				{
//					String[] parts = taggedTokens[wordOffset].replace("\\/", "/"). // '/' is tagger splitter; decast to match					
//							split(contentRegexCasted+"/");
//					if(parts.length==2)
//						POS = parts[1];
//					else
//						index_invalid = true;
//				}
//				else
//					index_invalid= true;
//
//				if(index_invalid)
//				{
//					
//					nextArtifact = getNextArtifact();
//					previousArtifact = getPreviousArtifact();
//					if(previousArtifact != null && nextArtifact!=null)
//					{
//						String preContent=previousArtifact.getContent();
//						String nextContent=nextArtifact.getContent();
//						if(nextContent.equals("-"))
//						{
//							Artifact nextnext = nextArtifact.getNextArtifact();
//							if(nextnext!=null)
//							{
//								contentRegexCasted = 
//									contentRegexCasted+
//									nextContent+
//									nextnext.getContent();
//							}
//						}
//						// to make sure not casted before
////							contentRegexCasted = 
////								Util.castForRegex(Util.decastRegex(contentRegexCasted));
//						
//						Pattern p = Pattern.compile("(.* )?"+contentRegexCasted+"/(\\S+)( .*)?");
//						Pattern p2 = Pattern.compile("(.* )?\\S*"+contentRegexCasted+"\\S*/(\\S+)( .*)?");
//						Matcher m = p.matcher(tagged);
//						if(m.matches())
//						{
//							POS=m.group(2);
////								Util.log("_POS found: "+_POS+
////										" for "+contentRegexCasted, Level.INFO);
//						}else
//						{
//							m = p2.matcher(tagged);
//							if(m.matches())
//							{
//								POS=m.group(2);
////									Util.log("_POS found: "+_POS+
////											" for "+contentRegexCasted, Level.INFO);
//							}
//						}
//						
//					}
//					
//					if(POS==null &&  previousArtifact != null)
//					{
//						POS = previousArtifact.getPOS();
//					}
//					if(POS==null)
//					{
//						POS = getContent();
////						Util.log("Error in _POS: word index = "+_wordOffset+
////							", tagged:"+tagged+", tagged token:"+
////							" "+contentRegexCasted+" -> " 
////							+getContent(), Level.WARNING);
//					}
//				}
//			}
//		}
//		if(getContent().equals("-"))
//			POS = "-";

		
		return POS;
	}
	public void setPOS(String pPOS) {
		POS = pPOS;
	}
	/**
	 * starts from 0
	 * @return
	 */
	@Index(name = "index_word_index")
	public Integer getWordIndex() {
		
		
		if(wordOffset==null)
		{
			
			if(artifactType==Type.Word)
			{
			
				wordOffset=0;
				Artifact preWord = getPreviousArtifact();
				while(preWord!=null)
				{
					wordOffset++;
					preWord = preWord.getPreviousArtifact();
					previousArtifact = null;
				}
			}
			
		}
		return wordOffset;
	}
	
	public void setWordIndex(Integer _wordOffset) {
		wordOffset = _wordOffset;
	}
	public void setLineIndex(Integer _lineIndex) {
		lineOffset = _lineIndex;
	}
	public void setStartIndex(Integer _startIndex) {
		startIndex = _startIndex;
	}
	
	/**
	 * sentence index, starts from 1
	 * @return
	 */
	@Index(name = "index_line_index")
	public Integer getLineIndex() {
		if(lineOffset==null)
		{
			if(artifactType==Type.Word)
			{
				Artifact sentence = getParentArtifact();
				if(sentence==null) 
				{
					nextArtifact = getNextArtifact();
					if(nextArtifact!=null)
						sentence = nextArtifact.getParentArtifact();
				}
				if(sentence==null) 
				{
					previousArtifact = getPreviousArtifact();
					if(previousArtifact!=null)
						sentence = previousArtifact.getParentArtifact();
				}
				if(sentence==null) 
				{
//						Util.log("Error in getLineIndex: word '"+ _artifactId 
//								+"' doesn't have parent!", Level.SEVERE);
						
					return null;
				}
				lineOffset = sentence.getLineIndex();
			}
			if(artifactType==Type.Sentence)
			{
				lineOffset=1;
				Artifact preSentence = getPreviousArtifact();
				while(preSentence!=null)
				{
					lineOffset++;
					preSentence = preSentence.getPreviousArtifact();
				}
			}
			
		}
		return lineOffset;
	}

	public void setContent(String _content) {
		this.content = _content;
	}
	@Column(length = 2000)
	@Index(name = "index_content")
	public String getContent() {
		if(content==null) return "";
		return content;
	}

	public void setAssociatedFilePath(String _associatedFilePath) {
		this.associatedFilePath = _associatedFilePath;
	}

	public String getAssociatedFilePath() {
		return associatedFilePath;
	}
	
	@Column(nullable = false)
	public void setArtifactType(Type _artifactType) {
		this.artifactType = _artifactType;
	}

	public Type getArtifactType() {
		return artifactType;
	}

	public void setArtifactId(int _artifactId) {
		this.artifactId = _artifactId;
	}
	@Id
	@GeneratedValue(generator="increment")
	@GenericGenerator(name="increment", strategy = "increment")
	public int getArtifactId() {
		return artifactId;
	}
	
	@Override public String toString()
	{
		return associatedFilePath.substring(associatedFilePath.length()-15, 
				associatedFilePath.length()-1) + ":"+
			artifactId+"-"+artifactType.toString()+"-"
			+parentArtifact.artifactId+"-"+
			nextArtifact.artifactId+"-"+
			previousArtifact.artifactId;
	}
	
	@Override public boolean equals(Object pArtifact)
	{
		if(!(pArtifact instanceof Artifact))
			return false;
		Artifact a = (Artifact)pArtifact;
		return (a.getArtifactId() == artifactId) ||
				(a.getStartIndex() == getStartIndex() &&
				a.getContent().equals(getContent()) &&
				a.getAssociatedFilePath().equals(getAssociatedFilePath()) &&
				a.getArtifactType().equals(getArtifactType()));
	}
	@Override public int hashCode()
	{
		return artifactId;
	}
	public static Artifact findInstance(Type pArtifactType, String pFilePath, 
			int pWordIndex, int pLineIndex){
		String hql = "from Artifact where artifactType =:artifactType  and" +
				" associatedFilePath like :associatedFilePath and wordIndex=:wordIndex " +
				" and lineIndex= :lineIndex";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("associatedFilePath", '%'+pFilePath);
		params.put("wordIndex", pWordIndex);
		params.put("artifactType", pArtifactType.ordinal());
		params.put("lineIndex",pLineIndex);
		
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
	    
	    
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstance(Artifact pSentence,int pWordIndex){
		String hql = "from Artifact where parentArtifact = "+pSentence.getArtifactId()+" and wordIndex =" +
		pWordIndex;
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	    
	    
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstance(Type pArtifactType, String pFilePath, 
			int pLine, int pWordIndex, String pContent, String pSecondWord){
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		String hql = "from Artifact where artifactType =:artifactType  and" +
				" associatedFilePath like :associatedFilePath and content like:content " +
				" and lineIndex= :lineIndex ";
		if (pSecondWord != null)
		{
			hql += "and nextArtifact.content = :nextContent";
			params.put("nextContent", pSecondWord);
		}
		else
		{
			hql += " and ABS(wordIndex - :pWordIndex) <=5";
			params.put("pWordIndex",pWordIndex);
		}
		
		params.put("associatedFilePath", '%'+pFilePath);
		params.put("content", pContent+'%');
		params.put("artifactType", pArtifactType.ordinal());
		params.put("lineIndex",pLine);
		
		
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
	    
	    
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstanceByContent(Type pArtifactType, String pFilePath, 
			String pContent ){
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		String hql = "from Artifact where artifactType =:artifactType  and" +
				" associatedFilePath like :associatedFilePath and content like:content ";
		
		
		params.put("associatedFilePath", '%'+pFilePath+'%');
		params.put("content", '%'+pContent+'%');
		params.put("artifactType", pArtifactType.ordinal());
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
	    
	    
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstanceByContentEnd(Type pArtifactType, String pFilePath, 
			String pContent ){
		HashMap<String, Object> params = new HashMap<String, Object>();
		String hql = "from Artifact where artifactType =:artifactType  and" +
				" associatedFilePath like :associatedFilePath and content like:content ";
		
		params.put("associatedFilePath", '%'+pFilePath+'%');
		params.put("content", '%'+pContent);
		params.put("artifactType", pArtifactType.ordinal());
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstanceByExactContent(Type pArtifactType, String pFilePath, 
			String pContent ){
		HashMap<String, Object> params = new HashMap<String, Object>();
		String hql = "from Artifact where artifactType =:artifactType  and" +
				" associatedFilePath like :associatedFilePath and content like:content ";
		
		params.put("associatedFilePath", '%'+pFilePath+'%');
		params.put("content", pContent);
		params.put("artifactType", pArtifactType.ordinal());
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstance(Type pArtifactType, String pFilePath, 
			int pStartChar, String pContent, String pSecondWord){
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		String hql = "from Artifact where artifactType =:artifactType  and" +
				" associatedFilePath like :associatedFilePath and content like:content " +
				" and ABS(startIndex - :pStartIndex) <45";
		if (pSecondWord != null)
		{
			hql += "and nextArtifact.content = :nextContent";
			params.put("nextContent", pSecondWord);
		}
	
		
		params.put("associatedFilePath", '%'+pFilePath);
		params.put("content", pContent+'%');
		params.put("artifactType", pArtifactType.ordinal());
		params.put("pStartIndex",pStartChar);
		
		
		
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
	    
	    
	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstanceByStartIndex(String pFilePath,int pStartIndex){
		String hql = "from Artifact where associatedFilePath like :associatedFilePath  and  artifactType=:artifactType and startIndex=:startIndex ";

		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("associatedFilePath", '%'+pFilePath);
		params.put("startIndex", pStartIndex);
		params.put("artifactType", 3);
		

		List<Artifact> artifact_objects = 
					(List<Artifact>) HibernateUtil.executeReader(hql, params);


	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static Artifact findInstanceByEndIndex(String pFilePath,int pEndIndex){
		String hql = "from Artifact where associatedFilePath like :associatedFilePath  and  artifactType=:artifactType and endIndex=:endIndex ";

		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("associatedFilePath", '%'+pFilePath);
		params.put("endIndex", pEndIndex);
		params.put("artifactType", 3);
		

		List<Artifact> artifact_objects = 
					(List<Artifact>) HibernateUtil.executeReader(hql, params);


	    Artifact artifact_obj=null;
	    if(artifact_objects.size()!=0)
	    {
	    	artifact_obj = 
				artifact_objects.get(0);
	    }
	    return artifact_obj;
	}
	public static List<Artifact> listByType(Type pSentenceType) {
		String hql = "from Artifact where artifactType = "+pSentenceType.ordinal()
				+" order by associatedFilePath";
			
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
		return artifact_objects;
	}
	//TODO: move this method to the right class
	public static List<Artifact> listTweets(Type pSentenceType) {
		String hql = "from Artifact where artifactType = "+pSentenceType.ordinal()
				+" and associatedFilePath not like '%host%' order by associatedFilePath";
			
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
		return artifact_objects;
	}
	public static List<Artifact> listByTypeByForTrain(Type pSentenceType, boolean forTrain) {
		String hql = "from Artifact where artifactType = "+pSentenceType.ordinal()
				+" and forTrain= "+forTrain+" order by associatedFilePath";
			
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
		return artifact_objects;
	}
	public static List<Artifact> listByTypeByForTrainByCorpus(Type pSentenceType, boolean forTrain,String corpusName) {
		String hql = "from Artifact where artifactType = "+pSentenceType.ordinal()
				+" and forTrain= "+forTrain+" and corpusName= '"+corpusName+"' order by associatedFilePath";
			
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
		return artifact_objects;
	}

	public static List<Artifact> listByTypeLimited(Type pSentenceType,boolean for_train, Integer count) {
		
		String hql = "from Artifact where artifactType = "+pSentenceType.ordinal()
				+"  and forTrain = "+for_train+" order by rand()";
		HashMap<String, Object> params = new HashMap<String, Object>();
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params,count);
		return artifact_objects;
	}
	public static List<Artifact> listByType(Type pSentenceType,boolean for_train) {
		String file_path_pattern = "/train/";
		if (!for_train)
		{
			file_path_pattern = "/test/";
		}
		String hql = "from Artifact where artifactType = "+pSentenceType.ordinal()
				+"  and associatedFilePath like :pPattern order by associatedFilePath";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("pPattern", '%'+file_path_pattern+'%');
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql,params);
		return artifact_objects;
	}

	public void setStanDependency(String stanDependency) {
		this.stanDependency = stanDependency;
	}
	@Column(length = 2000)
	public String getStanDependency() {
		return stanDependency;
	}
	
	public void setStanPennTree(String stanPennTree) {
		this.stanPennTree = stanPennTree;
	}
	@Column(length = 2000)
	public String getStanPennTree() {
		return stanPennTree;
	}
	@Transient
	List<Artifact> children = null;
	@Transient
	public Artifact getChildByWordIndex(int word_index) {
		if(children==null)
			loadChildren();
		Artifact foundArtifact = null;
		for(Artifact art : children)
			if(art.getWordIndex() == word_index)
			{
				foundArtifact = art;
				break;
			}
		return foundArtifact;
	}
	private void loadChildren() {
		String hql = "from Artifact where parentArtifact = "+getArtifactId();
		
		children = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
	}
	
	public static List<Artifact> listByTypeAndForTrain(Type pSentenceType,boolean for_train) {

		String hql = "from Artifact where forTrain ="+for_train+" and artifactType = "+pSentenceType.ordinal()
				+"  order by associatedFilePath";
		HashMap<String, Object> params = new HashMap<String, Object>();
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
		return artifact_objects;
	}
	public static List<Artifact> listByTypeAndForTrain(Type pSentenceType,boolean for_train,String corpus) {

		String hql = "from Artifact where forTrain ="+for_train+" and artifactType = "+pSentenceType.ordinal()
				+"  and corpusName='"+corpus+"' order by associatedFilePath";
		HashMap<String, Object> params = new HashMap<String, Object>();
		List<Artifact> artifact_objects = 
				(List<Artifact>) HibernateUtil.executeReader(hql);
		return artifact_objects;
	}
	public boolean isForTrain() {
		return forTrain;
	}
	public void setForTrain(boolean forTrain) {
		this.forTrain = forTrain;
	}
	public boolean isForDemo() {
		return forDemo;
	}
	public void setForDemo(boolean forDemo) {
		this.forDemo = forDemo;
	}
	public String getCorpusName() {
		return corpusName;
	}
	public void setCorpusName(String name) {
		this.corpusName = name;
	}
	
} @Entity
@Table( name = "Phrase" )
@Inheritance(strategy=InheritanceType.JOINED)
public class Phrase {
	private String phraseContent;
	
	private Artifact startArtifact;
	private Artifact endArtifact;
	
	private int phraseId;
	
	private String phraseEntityType;
	private String altID;
	private int startCharOffset;
	private int endCharOffset;
	private Artifact headArtifact;
	
	private Integer altLineIndex;
	private Integer altStartWordIndex;
	private Integer altEndWordIndex;
	private String normalizedHead;
	private Integer normalOffset;
	private Artifact govVerb;
	
	//remove these
//	private String umlsId;
//	private String drug;
	
	public Phrase()
	{
		
	}
	/**
	 * Loads Phrase by id
	 * @param pPhraseID
	 * @return
	 */
	public static Phrase getInstance(int pPhraseID) {
		String hql = "from Phrase where phraseId = "+pPhraseID;
		Phrase phrase_obj = 
			(Phrase)HibernateUtil.executeReader(hql).get(0);
		return phrase_obj;
	}

	
	
	/**
	 * Loads or creates the Phrase
	 * @param pPhraseContent
	 * @param pFilePath
	 * @param pStartIndex
	 * @return
	 */
	public static Phrase getInstance(String pPhraseContent, 
			Artifact pStartArtifact, Artifact pEndArtifact){
		String hql = "from Phrase where phraseContent = :phraseContent "+
			" and startArtifact= :startArtifact and endArtifact = :endArtifact ";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("phraseContent", pPhraseContent);
		params.put("startArtifact", pStartArtifact.getArtifactId());
		params.put("endArtifact", pEndArtifact.getArtifactId());
		
		List<Phrase> phrase_objects = 
				(List<Phrase>) HibernateUtil.executeReader(hql, params);
	    
		Phrase phrase_obj;
	    if(phrase_objects.size()==0)
	    {
	    	phrase_obj = new Phrase();
	    	phrase_obj.setPhraseContent(pPhraseContent);
	    	phrase_obj.setStartArtifact(pStartArtifact);
	    	phrase_obj.setEndArtifact(pEndArtifact);
	    	phrase_obj.setStartCharOffset(pStartArtifact.getStartIndex());
	    	phrase_obj.setEndCharOffset(pEndArtifact.getEndIndex());
	    	
	    	HibernateUtil.save(phrase_obj);
	    	FileUtil.logLine("/tmp/NonExistingPhrase.txt","pPhraseContent"+pPhraseContent
	    			+" pStartArtifact "+pStartArtifact.getArtifactId()+ " pEndArtifact "+pEndArtifact.getArtifactId());
	    }else
	    {
	    	phrase_obj = 
	    		phrase_objects.get(0);
	    }
	    return phrase_obj;
	}

	public static Phrase getInstanceWithoutSavingIndb(String pPhraseContent, 
			Artifact pStartArtifact, Artifact pEndArtifact){
		String hql = "from Phrase where phraseContent = :phraseContent "+
			" and startArtifact= :startArtifact and endArtifact = :endArtifact ";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("phraseContent", pPhraseContent);
		params.put("startArtifact", pStartArtifact.getArtifactId());
		params.put("endArtifact", pEndArtifact.getArtifactId());
		
		List<Phrase> phrase_objects = 
				(List<Phrase>) HibernateUtil.executeReader(hql, params);
	    
		Phrase phrase_obj;
	    if(phrase_objects.size()==0)
	    {
	    	phrase_obj = new Phrase();
	    	phrase_obj.setPhraseContent(pPhraseContent);
	    	phrase_obj.setStartArtifact(pStartArtifact);
	    	phrase_obj.setEndArtifact(pEndArtifact);
	    	phrase_obj.setStartCharOffset(pStartArtifact.getStartIndex());
	    	phrase_obj.setEndCharOffset(pEndArtifact.getEndIndex());
	    }else
	    {
	    	phrase_obj = 
	    		phrase_objects.get(0);
	    }
	    return phrase_obj;
	}
	//This is for the MADEUP discharge times NOT TO BE USED dor other purpose
	public static Phrase getMadeUpInstance(String pPhraseContent, String altId,String PhraseType){
		String hql = "from Phrase where phraseContent = :phraseContent "+
			" and altID = :paltID and phraseEntityType =:phraseEntityType ";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("phraseContent", pPhraseContent);
		params.put("paltID",altId);
		params.put("phraseEntityType",PhraseType);
		
		List<Phrase> phrase_objects = 
				(List<Phrase>) HibernateUtil.executeReader(hql, params);
	    
		Phrase phrase_obj;
	    if(phrase_objects.size()==0)
	    {
	    	phrase_obj = new Phrase();
	    	Artifact start = Artifact.getMadeUpInstance(pPhraseContent);
	    	phrase_obj.setStartArtifact(start);
	    	phrase_obj.setEndArtifact(start);
	    	phrase_obj.setPhraseContent(pPhraseContent);
	    	phrase_obj.setAltID(altId);
	    	phrase_obj.setPhraseEntityType(PhraseType);
	    	
	    	HibernateUtil.save(phrase_obj);
	    	
	    }else
	    {
	    	phrase_obj = 
	    		phrase_objects.get(0);
	    }
	    return phrase_obj;
	}
	public static Phrase getInstance(String pPhraseContent, 
			Artifact pStartArtifact, Artifact pEndArtifact, String pPhraseMentionType,
			String pAlt_id){
		String hql = "from Phrase where phraseContent = :phraseContent "+
			" and startArtifact= :startArtifact and endArtifact = :endArtifact " +
			" and phraseEntityType =:phraseEntityType and altID = :altId";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("phraseContent", pPhraseContent);
		params.put("startArtifact", pStartArtifact.getArtifactId());
		params.put("endArtifact", pEndArtifact.getArtifactId());
		params.put("phraseEntityType", pPhraseMentionType);
		params.put("altId", pAlt_id);
		
		List<Phrase> phrase_objects = 
				(List<Phrase>) HibernateUtil.executeReader(hql, params);
	    
		Phrase phrase_obj;
	    if(phrase_objects.size()==0)
	    {
	    	phrase_obj = new Phrase();
	    	phrase_obj.setPhraseContent(pPhraseContent);
	    	phrase_obj.setStartArtifact(pStartArtifact);
	    	phrase_obj.setEndArtifact(pEndArtifact);
	    	phrase_obj.setPhraseEntityType(pPhraseMentionType);
	    	phrase_obj.setStartCharOffset(pStartArtifact.getStartIndex());
	    	phrase_obj.setEndCharOffset(pEndArtifact.getEndIndex());
	    	HibernateUtil.save(phrase_obj);
	    	FileUtil.logLine("/tmp/NonExistingPhrase.txt","pPhraseContent"+pPhraseContent
	    			+" pStartArtifact "+pStartArtifact.getArtifactId()+ " pEndArtifact "+pEndArtifact.getArtifactId());
	    }else
	    {
	    	phrase_obj = 
	    		phrase_objects.get(0);
	    }
	    return phrase_obj;
	}
	public static Phrase getInstance(String pPhraseContent, 
			Artifact pStartArtifact, Artifact pEndArtifact, String pPhraseMentionType){
		String hql = "from Phrase where phraseContent = :phraseContent "+
			" and startArtifact= :startArtifact and endArtifact = :endArtifact " +
			" and phraseEntityType =:phraseEntityType ";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("phraseContent", pPhraseContent);
		params.put("startArtifact", pStartArtifact.getArtifactId());
		params.put("endArtifact", pEndArtifact.getArtifactId());
		params.put("phraseEntityType", pPhraseMentionType);
		
		List<Phrase> phrase_objects = 
				(List<Phrase>) HibernateUtil.executeReader(hql, params);
	    
		Phrase phrase_obj;
	    if(phrase_objects.size()==0)
	    {
	    	phrase_obj = new Phrase();
	    	phrase_obj.setPhraseContent(pPhraseContent);
	    	phrase_obj.setStartArtifact(pStartArtifact);
	    	phrase_obj.setEndArtifact(pEndArtifact);
	    	phrase_obj.setPhraseEntityType(pPhraseMentionType);
	    	phrase_obj.setStartCharOffset(pStartArtifact.getStartIndex());
	    	phrase_obj.setEndCharOffset(pEndArtifact.getEndIndex());
	    	
	    	HibernateUtil.save(phrase_obj);
//	    	FileUtil.logLine("/tmp/NonExistingPhrase.txt","pPhraseContent"+pPhraseContent
//	    			+" pStartArtifact "+pStartArtifact.getArtifactId()+ " pEndArtifact "+pEndArtifact.getArtifactId());
	    }else
	    {
	    	phrase_obj = 
	    		phrase_objects.get(0);
	    }
	    return phrase_obj;
	}
	public void setPhraseContent(String content) {
		this.phraseContent = content;
	}
	@Index(name = "phraseContent")
	@Column(name = "phraseContent", nullable = false, length = 1000)
	public String getPhraseContent() {
		if(phraseContent==null) return "";
		return phraseContent;
	}
	
	
	
	public void setStartArtifact(Artifact startArtifact) {
		this.startArtifact = startArtifact;
	}
	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE})
	@JoinColumn(name="startArtifact")
	public Artifact getStartArtifact() {
		return startArtifact;
	}
	public void setEndArtifact(Artifact endArtifactId) {
		this.endArtifact = endArtifactId;
	}
	
	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} )
	@JoinColumn(name="endArtifact")
	public Artifact getEndArtifact() {
		return endArtifact;
	}
	
	public void setPhraseId(int phraseId) {
		this.phraseId = phraseId;
	}
	
	@Id
	@GeneratedValue(generator="increment")
	@GenericGenerator(name="increment", strategy = "increment")
	public int getPhraseId() {
		return phraseId;
	}
//	public void setAssociatedFilePath(String _associatedFilePath) {
//		this.associatedFilePath = _associatedFilePath;
//	}
//
//	public String getAssociatedFilePath() {
//		return associatedFilePath;
//	}
	public void setPhraseEntityType(String phraseEntityType) {
		this.phraseEntityType = phraseEntityType;
	}
	public String getPhraseEntityType() {
		return phraseEntityType;
	}
	public static Phrase findInstance(String pFilePath, 
			int pStartLineOffset, int pStartWordOffset,
			int pEndLineOffset, int pEndWordOffset){
		Artifact startArtifcat =
				Artifact.findInstance(Artifact.Type.Word, pFilePath, pStartWordOffset, pStartLineOffset);
		
		Artifact endArtifcat =
			Artifact.findInstance(Artifact.Type.Word, pFilePath, pEndWordOffset, pEndLineOffset);
		
		String hql = "from Phrase where startArtifact =" +
			":startArtifact and endArtifact= :endArtifact";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("startArtifact", startArtifcat.getArtifactId());
		params.put("endArtifact", endArtifcat.getArtifactId());
		
		List<Phrase> phrase_objects = 
				(List<Phrase>) HibernateUtil.executeReader(hql, params);
	    
	    
		Phrase phrase_obj=null;
	    if(phrase_objects.size()!=0)
	    {
	    	phrase_obj = 
	    		phrase_objects.get(0);
	    }
	    return phrase_obj;
	}
	public static Phrase findInstance(String pPartialFileName, 
			String altId){
		
		String hql = "from Phrase where startArtifact.associatedFilePath like" +
			":filePath and altID = :altID";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("filePath", '%'+pPartialFileName+'%');
		params.put("altID", altId);
		
		List<Phrase> phrase_objects = 
				(List<Phrase>) HibernateUtil.executeReader(hql, params);
	    
	    
		Phrase phrase_obj=null;
	    if(phrase_objects.size()!=0)
	    {
	    	phrase_obj = 
	    		phrase_objects.get(0);
	    }
	    return phrase_obj;
	}
	public static Phrase findInstance(Artifact startArtifact, Artifact endArtifact,
			String phraseEntityType){
		
		String hql = "from Phrase where startArtifact = :start" +
			" and endArtifact= :end and PhraseEntityType = :phraseEntityType ";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("start", startArtifact.getArtifactId());
		params.put("end", endArtifact.getArtifactId());
		params.put("phraseEntityType", phraseEntityType);
		
		List<Phrase> phrase_objects = 
				(List<Phrase>) HibernateUtil.executeReader(hql, params);
	    
	    
		Phrase phrase_obj=null;
	    if(phrase_objects.size()!=0)
	    {
	    	phrase_obj = 
	    		phrase_objects.get(0);
	    }
	    return phrase_obj;
	}
	
//	public static Phrase findInstance(String pFilePath, 
//			int pStartCharIndex,String pContent){
//		
//		String hql = "from Phrase where startArtifact.startIndex =" +
//			":startCharIndex and startArtifact.associatedFilePath = "+
//			":filePath and phraseContent= :phraseContent";
//		HashMap<String, Object> params = new HashMap<String, Object>();
//		params.put("startCharIndex", pStartCharIndex);
//		params.put("phraseContent", pContent);
//		params.put("filePath", pFilePath);
//		
//		
//		List<Phrase> phrase_objects = 
//				(List<Phrase>) HibernateUtil.executeReader(hql, params);
//	    
//	    
//		Phrase phrase_obj=null;
//	    if(phrase_objects.size()!=0)
//	    {
//	    	phrase_obj = 
//	    		phrase_objects.get(0);
//	    }
//	    return phrase_obj;
//	}
	public static Phrase findInstance(Artifact pStartArtifact,String pContent){
		
		String hql = "from Phrase where startArtifact =" +
			":startArtifact and phraseContent= :phraseContent";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("startArtifact", pStartArtifact.getArtifactId());
		params.put("phraseContent", pContent);
		
		
		
		List<Phrase> phrase_objects = 
				(List<Phrase>) HibernateUtil.executeReader(hql, params);
	    
	    
		Phrase phrase_obj=null;
	    if(phrase_objects.size()!=0)
	    {
	    	phrase_obj = 
	    		phrase_objects.get(0);
	    }
	    return phrase_obj;
	}
	public void setAltID(String altID) {
		this.altID = altID;
	}
	public String getAltID() {
		return altID;
	}
	public void setStartCharOffset(int startCharOffset) {
		this.startCharOffset = startCharOffset;
	}
	public int getStartCharOffset() {
		return startCharOffset;
	}
	public void setEndCharOffset(int endCharOffset) {
		this.endCharOffset = endCharOffset;
	}
	public int getEndCharOffset() {
		return endCharOffset;
	}
	
	//This method returns all artifacts that are annotated
	@Transient
	public static ArrayList<Artifact> getAnnotatedWordsInSentence(Artifact sentence)
	{
		List<Phrase> phrase_objects = getPhrasesInSentence(sentence);
		
		ArrayList<Artifact> annotated_artifacts = new ArrayList<Artifact>();
		
		for (Phrase p:phrase_objects)
		{
			Artifact start_artifact = p.getStartArtifact();
			Artifact end_artifact =  p.getEndArtifact();
//			annotated_artifacts.add(start_artifact);
			
			Artifact next_artifact =start_artifact;
			while(next_artifact!=null &&
					!next_artifact.equals(end_artifact))
			{
				annotated_artifacts.add(next_artifact);
				next_artifact = next_artifact.getNextArtifact();
			}
			annotated_artifacts.add(end_artifact);
		}
    
	    return annotated_artifacts;
	}
	
	public static List<Phrase> getPhrasesInSentence(Artifact sentence)
	{
		String hql = "from Phrase p where p.startArtifact.parentArtifact =" +
			":sentId and p.endArtifact.parentArtifact =:sentId and p.phraseEntityType<>'Extracted'";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("sentId", sentence.getArtifactId());
		
		List<Phrase> phrase_objects = 
			(List<Phrase>) HibernateUtil.executeReader(hql, params);
		
		return phrase_objects;
	}
	public static List<Phrase> getOrderedPhrasesInSentence(Artifact sentence)
	{	
		String hql = "from Phrase p where p.startArtifact.parentArtifact =" +
		":sentId and p.endArtifact.parentArtifact =:sentId and phraseEntityType <> 'SECTIME' and p.phraseEntityType<>'Extracted'";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("sentId", sentence.getArtifactId());
		
		List<Phrase> phrase_objects = (List<Phrase>) HibernateUtil.executeReader(hql, params);
		Collections.sort(phrase_objects, new phraseComparator());
		
		
		return phrase_objects;
	}
	public static Phrase getNextPhraseInSentence(Phrase p,Artifact sentence)
	{	
		Phrase next_Phrase = null;
		List<Phrase> ordered =getOrderedPhrasesInSentence(sentence);
		
		int index_of_p = ordered.indexOf(p);
		if (index_of_p+1< ordered.size())
		{
			next_Phrase = ordered.get(index_of_p+1);
			
		}
		
		
		return next_Phrase;
	}
	public static Phrase getPrevPhraseInSentence(Phrase p,Artifact sentence)
	{	
		Phrase prev_Phrase = null;
		List<Phrase> ordered =getOrderedPhrasesInSentence(sentence);
		
		int index_of_p = ordered.indexOf(p);
		if (index_of_p-1>=0)
		{
			prev_Phrase = ordered.get(index_of_p-1);
			
		}
		
		
		return prev_Phrase;
	}
	public static Artifact getFirstNounInPhrase(Phrase p,Artifact sentence)
	{	
		Artifact first_noun = null;
		Artifact cur_artifact = p.getStartArtifact();
		while (cur_artifact != p.getEndArtifact())
		{
			if(cur_artifact.getPOS().startsWith("NN"))
			{
				first_noun = cur_artifact;
				break;
			}
			cur_artifact = cur_artifact.getNextArtifact();
		}
		if (first_noun ==null)
		{
			first_noun = p.getEndArtifact();
		}

		return first_noun;
	}
	public static Artifact getLastNounInPhrase(Phrase p,Artifact sentence)
	{	
		Artifact last_noun = null;
		Artifact cur_artifact = p.getEndArtifact();
		while (cur_artifact != p.getStartArtifact())
		{
			if(cur_artifact.getPOS().startsWith("NN"))
			{
				last_noun = cur_artifact;
				break;
			}
			cur_artifact = cur_artifact.getPreviousArtifact();
		}
		if (last_noun ==null)
		{
			last_noun = p.getEndArtifact();
		}

		return last_noun;
	}
	public static boolean isNestedPhrase(Phrase p,Artifact sentence)
	{	
		boolean is_nested = false;
		Phrase next_p = getNextPhraseInSentence(p, sentence);
		if(next_p != null && p.getEndArtifact().getWordIndex() >= next_p.getStartArtifact().getWordIndex() )
		{
			is_nested = true;
		}
		if (is_nested==false)
		{
			Phrase prev_p = getPrevPhraseInSentence(p, sentence);
			if(prev_p != null && prev_p.getEndArtifact().getWordIndex() >= p.getStartArtifact().getWordIndex() )
			{
				is_nested = true;
			}
		}
		
		return is_nested;
	}
	@Transient
	public String getPOS() {
		
		Artifact start_artifact = getStartArtifact();
		Artifact end_Artifact  = getEndArtifact();
		Artifact cur_artifact = start_artifact;
		
		String pos = "";
		while(!cur_artifact.equals(end_Artifact))
		{
			pos += cur_artifact.getPOS()+"-";
			cur_artifact = cur_artifact.getNextArtifact();
		}
		pos += cur_artifact.getPOS();
		
		return pos;
		
	}
	//this returns the offsets included in this phrase
	public List<Integer> listWordOffsetsInPhrase()
	{
		List<Integer> included_offsets =  new ArrayList();
		
		Artifact start_artifact = getStartArtifact();
		Artifact end_Artifact  = getEndArtifact();
		Artifact cur_artifact = start_artifact;
		
		while(!cur_artifact.equals(end_Artifact))
		{
			
			included_offsets.add(cur_artifact.getWordIndex());
			cur_artifact = cur_artifact.getNextArtifact();
		}
		included_offsets.add(cur_artifact.getWordIndex());
		return included_offsets;
	}
	//this is somehow duplicate with normalizedhead...
	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE } , fetch=FetchType.LAZY  )
    @JoinColumn(name="headArtifact")
	public Artifact getHeadArtifact() throws SQLException {
		
		if (headArtifact!=null)
		{
			return headArtifact;
		}
		else
		{
			Artifact h= calculateHeadWord();
			setHeadArtifact(h);
//			HibernateUtil.save(this);
			return h;
		}
	}
//	TODO: improve and make it a solid approach
	private Artifact calculateHeadWord() throws SQLException
	{
		Artifact head = getEndArtifact();
		if (getStartArtifact().getArtifactId() == getEndArtifact().getArtifactId())
		{
			return head;
		}
		String phrase_pos= getPOS();
		String type = getPhraseEntityType();
		
		if(type != null && type.equals("EVENT"))
		{	
			if (phrase_pos != null && (phrase_pos.equals("VB-RP") || phrase_pos.equals("VBG-RP")
					|| phrase_pos.equals("VBN-JJ") || phrase_pos.equals("VB-IN") || phrase_pos.equals("VBN-IN"))
					||  phrase_pos.equals("JJ-TO-VB") ||  phrase_pos.startsWith("VBG-TO"))
			{
				head = getStartArtifact();
			}
			else if(!getEndArtifact().getContent().matches(".*\\W$"))
			{
				head = getEndArtifact();
			}
			else
			{
				Artifact cur_artifact = getEndArtifact();
				while(cur_artifact != null && cur_artifact != getStartArtifact() && (cur_artifact.getContent().matches("\\W") ||
						cur_artifact.getContent().matches("\\W.*")))
				{
					cur_artifact = cur_artifact.getPreviousArtifact();
				}
				head = cur_artifact;
			}
//			
		}
		else if (type != null && type.equals("TIMEX3"))
		{
			//get the type of the timex
//			TimexPhrase timex_obj = TimexPhrase.getRelatedTimexFromPhrase(this);
//			TimexPhrase.TimexType  timex_type = timex_obj.getTimexType();
			Artifact sentence = getStartArtifact().getParentArtifact();
//			Phrase next_p= Phrase.getNextPhraseInSentence(this, sentence);
//		
//			if(timex_type != null && timex_type.equals(TimexPhrase.TimexType.DATE))
//			{
				if (getPhraseContent().startsWith("\\d+\\/") || getPhraseContent().startsWith("\\d+-"))
				{
					head = getStartArtifact();
				}
				else
				{
					head = Phrase.getFirstNounInPhrase(this, sentence);
				}
				
//			}
//			else
//			{
//				
//				head = Phrase.getFirstNounInPhrase(this, sentence);
//			}
			
		}
		return head;
	}
	
	public void setHeadArtifact(Artifact headArtifact) {
		this.headArtifact = headArtifact;
	}
	public static List<Phrase> getPhrasesInDocument(String file_path) {
		String hql = "from Phrase p where p.startArtifact.associatedFilePath = :filePath " +
				"and p.phraseEntityType is not null";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("filePath", file_path);
		
		List<Phrase> phrase_objects = 
			(List<Phrase>) HibernateUtil.executeReader(hql, params);
		
		return phrase_objects;
	}
	@Override
	public String toString()
	{
		return "Id="+getPhraseId()+"/Content="+
			getPhraseContent();
	}
	public void setAltLineIndex(Integer altLineIndex) {
		this.altLineIndex = altLineIndex;
	}
	public Integer getAltLineIndex() {
		return altLineIndex;
	}
	public void setAltStartWordIndex(Integer altStartWordIndex) {
		this.altStartWordIndex = altStartWordIndex;
	}
	public Integer getAltStartWordIndex() {
		return altStartWordIndex;
	}
	public void setAltEndWordIndex(Integer altEndWordIndex) {
		this.altEndWordIndex = altEndWordIndex;
	}
	public Integer getAltEndWordIndex() {
		return altEndWordIndex;
	}
	
	public static boolean isSecTime(Phrase p)
	{
		boolean is_sectime = false;
		Artifact startArtifact = p.getStartArtifact();
		String hql = "from Phrase where startArtifact = "+
		startArtifact.getArtifactId()+" and endArtifact="+
		p.getEndArtifact().getArtifactId()+
		" and phraseEntityType = 'SECTIME'";
		List<Phrase> phrase_obj = 
			(List<Phrase>)HibernateUtil.executeReader(hql);
		if (phrase_obj.size()!=0)
		{
			is_sectime = true;
		}
			
		return is_sectime;
	}
	public void setNormalizedHead(String normalizedHead) {
		this.normalizedHead = normalizedHead;
	}
	public String getNormalizedHead() {
		return normalizedHead;
	}
	public void setNormalOffset(Integer normalOffset) {
		this.normalOffset = normalOffset;
	}
	public Integer getNormalOffset() {
		return normalOffset;
	}
	
	@Override public boolean equals(Object pPhrase)
	{
		if(!(pPhrase instanceof Phrase))
			return false;
		Phrase p = (Phrase)pPhrase;
		return (p.getPhraseId() == phraseId) || 
				(p.getPhraseContent()==phraseContent && 
				p.getStartArtifact().equals(getStartArtifact()) &&
				p.getEndArtifact().equals(getEndArtifact()));
	}
	@Override public int hashCode()
	{
		return phraseId;
	}
	public static class phraseComparator implements Comparator<Phrase>{
	    public int compare(Phrase p1, Phrase p2) {
	        return ((Integer)p1.getStartCharOffset()).compareTo((Integer)(p2.getStartCharOffset()));
	    }
	}
	@Transient	
	public static List<Phrase> getPhrasesBetweenPhrases(Phrase phrase1,Phrase phrase2, String filePath) {
		String hql = "FROM Phrase where startCharOffset > :endphrase1 and endCharOffset<:startphrase2 and startArtifact.associatedFilePath=:filePath";
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("endphrase1", phrase1.getEndCharOffset());
		params.put("startphrase2", phrase2.getStartCharOffset());
		params.put("filePath", filePath);
		return (List<Phrase>) HibernateUtil.executeReader(hql, params);
	}

		
	public Artifact calclateGovVerb() {
		Artifact gov_verb = null;
		String pos = getPOS();
		if (pos==null) return null;
		if (pos != null && (pos.matches("VB|VBD|VBN|VBP|VBZ") || pos.equals("VB-RP") || pos.equals("VBG-RP")|| pos.equals("VBN-JJ") || pos.equals("VB-IN") || pos.equals("VBN-IN"))
				||  pos.equals("JJ-TO-VB") ||  pos.startsWith("VBG-TO"))
		{
			gov_verb = this.getStartArtifact();
		}
		else
		{
			Artifact next = this.getEndArtifact().getNextArtifact();
			if (next != null && next.getPOS()!= null && next.getPOS().startsWith("VB"))
			{
				Artifact next_verb = next.getNextArtifact();
				if (next_verb != null && 
						next_verb.getPOS().matches("VBD"))
				{
					gov_verb = next_verb;
				}
				else
				{
					gov_verb = next;
				}
			}
			else
			{
				Artifact prev = this.getStartArtifact().getPreviousArtifact();
				while (prev != null  && prev.getPOS()!= null &&
						!prev.getPOS().matches("VB|VBD|VBN|VBP|VBZ") )
				{
					prev = prev.getPreviousArtifact();
				}
				if (prev != null && prev.getPOS()!= null && prev.getPOS().startsWith("VB"))
				{
					gov_verb = prev;
				}
			}
			//if still null
			if (gov_verb == null)
			{
				while (next != null  &&
						next.getPOS()!= null && !next.getPOS().matches("VB|VBD|VBN|VBP|VBZ") )
				{
					next = next.getNextArtifact();
				}
				if (next != null && next.getPOS()!= null && next.getPOS().startsWith("VB"))
				{
					Artifact next_verb = next.getNextArtifact();
					if (next_verb.getPOS().matches("VBD"))
					{
						gov_verb = next_verb;
					}
					else
					{
						gov_verb = next;
					}
					
				}
			}
		}
		
		return gov_verb;
	}
	public Phrase getPreviousPhrase(List<Phrase> oregered_list)
	{
		Phrase prev = null;
		int index = oregered_list.indexOf(this)-1;
		if (index !=-1)
		{
			prev = oregered_list.get(index);
		}
		return prev;
		
	}
	@Transient
	public List<Artifact> getAllTokenArtifacts()
	{
		List<Artifact> all_tokens = new ArrayList<>();
		Artifact current =startArtifact;
		
		while (current != null
				&& !current.equals(endArtifact))
		{
			all_tokens.add(current);
			current=current.getNextArtifact();
		}
		
		all_tokens.add(endArtifact);
		
		return all_tokens;
		
	}
	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} )
    @JoinColumn(name="govVerb")
	public Artifact getGovVerb() {
		if (govVerb==null)
		{
			govVerb = calclateGovVerb();
		}
		return govVerb;
	}
	public void setGovVerb(Artifact govVerb) {
		this.govVerb = govVerb;
	}
//	public String getUmlsId() {
//		return umlsId;
//	}
//	public void setUmlsId(String string) {
//		this.umlsId = string;
//	}
//	public String getDrug() {
//		return drug;
//	}
//	public void setDrug(String drug) {
//		this.drug = drug;
//	}
} @Entity
@Table( name = "MLExample" )
public class MLExample {
	int exampleId;


	int predictedClass;
	int expectedClass;
	boolean forTrain;
	String corpusName;
	String predictionEngine;
	Artifact relatedArtifact;
	PhraseLink relatedPhraseLink;
	private String associatedFilePath;
	private double predictionWeight;
	private int expectedReal;
	private int expectedClosure;
	private int expectedIntegrated;
	
	private Phrase relatedPhrase;
	
	@Transient
	List<MLExampleFeature> exampleFeatures;
	static public Session hibernateSession; 
	
	String relatedDrug;
	@Transient
	public List<MLExampleFeature> getExampleFeatures()
	{
		if(exampleFeatures==null)
		{
			if(hibernateSession == null)
				hibernateSession = HibernateUtil.sessionFactory.openSession();
			String hql = "from MLExampleFeature where relatedExample = "+
			 getExampleId()+ 
			 " order by featureValuePair.tempFeatureIndex";
//			 " order by featureValuePair.featureValuePairId";
			exampleFeatures = (List<MLExampleFeature>) HibernateUtil.executeReader(hql, null,null, hibernateSession);
		}
		return exampleFeatures;
	}
	@Transient
	public List<MLExampleFeature> getNonEmbeddingsExampleFeatures()
	{
		if(exampleFeatures==null)
		{
			if(hibernateSession == null)
				hibernateSession = HibernateUtil.sessionFactory.openSession();
			String hql = "from MLExampleFeature  where relatedExample = "+
			 getExampleId()+ 
			 " and featureValuePair. featureName not like '%Embeddings%' order by featureValuePair.tempFeatureIndex";
//			 " order by featureValuePair.featureValuePairId";
			exampleFeatures = (List<MLExampleFeature>) HibernateUtil.executeReader(hql, null,null, hibernateSession);
		}
		return exampleFeatures;
	}
	@Transient
	//TODO: improve this, this is a quick way to get a specific one value String feature
	public String getFeatureValueForExample(MLExample e, String featureName)
	{
		
		if(hibernateSession == null || !hibernateSession.isOpen())
			hibernateSession = HibernateUtil.sessionFactory.openSession();
		String hql = "from MLExampleFeature  where relatedExample = "+
		 e.getExampleId()+ 
		 " and featureValuePair. featureName ='"+featureName+"' order by featureValuePair.tempFeatureIndex";
//			 " order by featureValuePair.featureValuePairId";
		List<MLExampleFeature> eFeatures = (List<MLExampleFeature>) HibernateUtil.executeReader(hql, null,null, hibernateSession);
		
		if (eFeatures.isEmpty())
		{
			FileUtil.logLine("/tmp/featureErrors.txt", "no feature for example id:  "+e.getExampleId());
			return "UNK";
		}

		MLExampleFeature feature = eFeatures.get(0);
		FeatureValuePair fvp = feature.getFeatureValuePair();
		
		return fvp.getFeatureValue();
	}
	public String getPredictionEngine() {
		return predictionEngine;
	}

	public void setPredictionEngine(String pPredictionEngine) {
		predictionEngine = pPredictionEngine;
	}


	
	
	public String getCorpusName() {
		return corpusName;
	}
	
	public void setCorpusName(String pCorpusName) {
		corpusName = pCorpusName;
	}
	
	public boolean getForTrain() {
		return forTrain;
	}
//	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE }, fetch=FetchType.LAZY )	
	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE } )
    @JoinColumn(name="relatedArtifact")
	public Artifact getRelatedArtifact() {
		return relatedArtifact;
	}

	public void setRelatedArtifact(Artifact relatedArtifact) {
		this.relatedArtifact = relatedArtifact;
	}

	@ManyToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} , fetch=FetchType.LAZY )
    @JoinColumn(name="relatedPhraseLink")
	public PhraseLink getRelatedPhraseLink() {
		return relatedPhraseLink;
	}

	public void setRelatedPhraseLink(PhraseLink relatedPhraseLink) {
		this.relatedPhraseLink = relatedPhraseLink;
	}

	public void setForTrain(boolean isForTrain) {
		forTrain = isForTrain;
	}
	
	public int getPredictedClass() {
		return predictedClass;
	}
	
	public void setPredictedClass(int pPredictedClass) {
		predictedClass = pPredictedClass;
	}
	
	public int getExpectedClass() {
		return expectedClass;
	}
	
	public void setExpectedClass(int pExpectedClass) {
		expectedClass = pExpectedClass;
	}
	@Id
	@GeneratedValue(generator="increment")
	@GenericGenerator(name="increment", strategy = "increment")
	public int getExampleId() {
		return exampleId;
	}

	public void setExampleId(int exampleId) {
		this.exampleId = exampleId;
	}
	@Temporal(TemporalType.TIMESTAMP)
    Date updateTime;
	
	
	@PrePersist
    protected void onCreate() {
		updateTime = new Date();
    }

    @PreUpdate
    protected void onUpdate() {
    	updateTime = new Date();
    }

	public static MLExample getInstanceForArtifact(Artifact artifact,
			String experimentgroup) {
		String hql = "from MLExample where relatedArtifact = "+
				artifact.getArtifactId() + " and corpusName = '"+
				experimentgroup+"'";
			List<MLExample> example_objects = 
					getExamplesList(hql);
		    
			MLExample example_obj;
		    if(example_objects.size()==0)
		    {
		    	example_obj = new MLExample();
		  
		    	
		    	example_obj.setCorpusName(experimentgroup);
		    	example_obj.setRelatedArtifact(artifact);
		    	
		    	if(Setting.SaveInGetInstance)
			    	saveExample(example_obj);
		    }else
		    {
		    	example_obj = 
		    			example_objects.get(0);
		    }
		    return example_obj;
	}
	public static MLExample findInstanceForArtifact(Artifact artifact,
			String experimentgroup) {
		String hql = "from MLExample where relatedArtifact = "+
				artifact.getArtifactId() + " and corpusName = '"+
				experimentgroup+"'";
			List<MLExample> example_objects = 
					getExamplesList(hql);
		    
			MLExample example_obj;
		    if(example_objects.size()==0)
		    {
		    	example_obj= null;
		    }else
		    {
		    	example_obj = 
		    			example_objects.get(0);
		    }
		    return example_obj;
	}

	public void calculateFeatures(
			List<IFeatureCalculator> featureCalculators) throws Exception {
		for(IFeatureCalculator feature_calculator : featureCalculators)
		{
			feature_calculator.calculateFeatures(this);
			HibernateUtil.clearLoaderSession();
		}
		
	}

	public static MLExample getInstanceForLink(PhraseLink phrase_link,
			String experimentgroup) {
		String hql = "from MLExample where relatedPhraseLink = "+
				phrase_link.getPhraseLinkId() + " and corpusName = '"+
						experimentgroup+"'";
			List<MLExample> example_objects = 
					getExamplesList(hql);
		    
			MLExample example_obj;
		    if(example_objects.size()==0)
		    {
		    	example_obj = new MLExample();
		  
		    	
		    	example_obj.setCorpusName(experimentgroup);
		    	example_obj.setRelatedPhraseLink(phrase_link);
		    	example_obj.setAssociatedFilePath(phrase_link.getFromPhrase().getStartArtifact().getAssociatedFilePath());
		    	
		    	if(Setting.SaveInGetInstance)
		    		saveExample(example_obj);
		    }else
		    {
		    	example_obj = 
		    			example_objects.get(0);
		    }
		    return example_obj;
	}


	public static void saveExample(MLExample example)
	{
		if(hibernateSession == null)
			hibernateSession = HibernateUtil.loaderSession;
		
		HibernateUtil.save(example, hibernateSession);
	}
	static List<MLExample> getExamplesList(String hql)
	{
		List<MLExample> examples;
		if(hibernateSession == null || !hibernateSession.isOpen())
			hibernateSession = HibernateUtil.loaderSession;
		examples = 
			(List<MLExample>) HibernateUtil.executeReader(hql, null, null, hibernateSession);
		return examples;
	}
	static List<MLExample> getExamplesList(String hql, Integer limit)
	{
		List<MLExample> examples;
		if(hibernateSession == null || !hibernateSession.isOpen())
			hibernateSession = HibernateUtil.loaderSession;
		examples = 
			(List<MLExample>) HibernateUtil.executeReader(hql, null, limit, hibernateSession);
		return examples;
	}
	static List<MLExample> getExamplesList(String hql, HashMap<String, Object> params)
	{
		List<MLExample> examples;
		if(hibernateSession == null || !hibernateSession.isOpen())
			hibernateSession = HibernateUtil.loaderSession;
		
		examples = 
			(List<MLExample>) HibernateUtil.executeReader(hql, params, null, hibernateSession);
		return examples;
	}
	public static List<MLExample> temptGetAllExamplesFotTrain(String experimentgroup, boolean for_train)
	{
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"' and forTrain="+(for_train?1:0)
						+"  and exampleId < 37406 "
						+ " order by relatedArtifact";
		
		return getExamplesList(hql);
	}
	public static List<MLExample> getAllExamples(String experimentgroup, boolean for_train)
	{
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"' and forTrain="+(for_train?1:0)
						+ " order by relatedArtifact";
		
		return getExamplesList(hql);
	}
	//get limited selected examples 
	public static List<MLExample> getLimitedPreSelectedExamples(String experimentgroup, boolean for_train,String condition)
	{
//		tempIsSelectedForTrain=1
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"' and forTrain="+(for_train?1:0)
						+ " and "+condition+" order by relatedArtifact";
		
		return getExamplesList(hql);
	}
	public static List<MLExample> getAllExamplesTwitter(String experimentgroup, boolean for_train)
	{
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"' and forTrain="+(for_train?1:0)
						+" and expectedClosure=1 order by exampleId";
		
		return getExamplesList(hql);
	}
	public static List<MLExample> getAllExamples(boolean for_train)
	{
		String hql = "from MLExample where  forTrain="+(for_train?1:0)
						+"  order by exampleId";
		
		return getExamplesList(hql);
	}
	public static List<MLExample> getExampleById(int example_id, String experimentgroup)
	{
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"' and exampleId="+example_id
						+" order by exampleId";
		return getExamplesList(hql);
	}
	public static MLExample getExampleById(int example_id)
	{
		String hql = "from MLExample where exampleId="+example_id;
		List<MLExample> example_objects = 
			(List<MLExample>) HibernateUtil.executeReader(hql);
    
		MLExample example_obj=null;
	    if(example_objects.size()!=0)
	    {
	    	example_obj = 
	    		example_objects.get(0);
	    }
	    return example_obj;
	}
	public static List<MLExample> getAllExamples(String experimentgroup, boolean for_train, int limit)
	{
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"'  and forTrain="+(for_train?1:0)+" order by exampleId";
		return getExamplesList(hql,limit);
	}
	public static List<MLExample> getLastExamples(String experimentgroup, boolean for_train, int limit)
	{
		String hql = "from MLExample where corpusName = '"+
			experimentgroup+"'  and forTrain="+(for_train?1:0)+
				"order by exampleId desc";
		return getExamplesList(hql);
	}
	
	public static List<MLExample> getExampleByExpectedClass(String experimentgroup,boolean for_train, LinkType type)
	{
		String hql = "from MLExample where corpusName = '"+
						experimentgroup+"' and expectedClass="+type.ordinal()
						+" and  forTrain="+(for_train?1:0)+" order by exampleId";
		return getExamplesList(hql);
	}
	
	public static List<MLExample> getExamplesInDocument(String experimentgroup, 
			String doc_path)
	{
		
		String hql = "FROM MLExample "  +
				"where corpusName =:corpusName " +
				" and associatedFilePath = '" +
				doc_path + "' " +
						"order by exampleId desc";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("corpusName", experimentgroup);
		
		return getExamplesList(hql, params);
	}

	public static List<MLExample> getExamplesByDocument(String experimentgroup, 
			boolean for_train, int num_of_documents)
	{
		List<Artifact> docs = Artifact.listByType(Type.Document, for_train);
		if(docs.size()<num_of_documents)
			num_of_documents = docs.size();
		
		String docPaths = "";
		for(int i=0;i<num_of_documents;i++)
			docPaths = docPaths.concat(", '"+docs.get(i).getAssociatedFilePath()+"'");
		docPaths = docPaths.replaceFirst(",", "");
		
		String hql = "FROM MLExample "  +
				"where corpusName =:corpusName " +
				" and forTrain="+(for_train?1:0) +" and associatedFilePath in (" +
						docPaths + ") " +
						"order by associatedFilePath desc";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("corpusName", experimentgroup);
		
		return getExamplesList(hql, params);
	}
	public static List<MLExample> getTokenExamplesBySent(Artifact sent,String experimentgroup, 
			boolean for_train)
	{	
//		String hql = "FROM MLExample "  +
//				"where corpusName =:corpusName " +
//				" and forTrain="+(for_train?1:0) +" and relatedArtifact.parentArtifact=" +sent.getArtifactId()+
//						
//						" order by associatedFilePath desc";
		String hql = "FROM MLExample "  +
				"where corpusName =:corpusName " +
				"  and relatedArtifact.parentArtifact=" +sent.getArtifactId()+
						
						" order by exampleID asc";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("corpusName", experimentgroup);
		
		return getExamplesList(hql, params);
	}
	
	public static List<MLExample> getExamplesByEventTypeByDocument(String experimentgroup, 
			boolean for_train, int num_of_documents, String type1,
			String type2,String order)
	{
		List<Artifact> docs = Artifact.listByType(Type.Document,for_train);
		if(docs.size()<num_of_documents)
			num_of_documents = docs.size();
		
		String docPaths = "";
		if (order.equals("top"))
		{
			for(int i=0;i<num_of_documents;i++)
				docPaths = docPaths.concat(", '"+docs.get(i).getAssociatedFilePath()+"'");
			docPaths = docPaths.replaceFirst(",", "");
		}
		else if(order.equals("last"))
		{
			for(int i=docs.size()-1;i>docs.size()-num_of_documents-1;i--)
				docPaths = docPaths.concat(", '"+docs.get(i).getAssociatedFilePath()+"'");
			docPaths = docPaths.replaceFirst(",", "");
		}
		Integer type1_from_fvpIds = FeatureValuePair.getRelatedFromEventTypeFValuePairIds(type1);
		Integer type1_to_fvpIds = FeatureValuePair.getRelatedToEventTypeFValuePairIds(type1);
		Integer type2_from_fvpIds = FeatureValuePair.getRelatedFromEventTypeFValuePairIds(type2);
		Integer type2_to_fvpIds = FeatureValuePair.getRelatedToEventTypeFValuePairIds(type2);
		
//		String from_fvpIds = "";
//		for(Integer id: fromFeatureValuePairIds)
//		{
//			from_fvpIds = from_fvpIds.concat(", '"+id+"'");
//		}
//		from_fvpIds = from_fvpIds.replaceFirst(",", "");
//		
//
//		String to_fvpIds = "";
//		for(Integer id: toTeatureValuePairIds)
//		{
//			to_fvpIds = to_fvpIds.concat(", '"+id+"'");
//		}
//		to_fvpIds = to_fvpIds.replaceFirst(",", "");
		
		String hql = " FROM MLExample m  "  +
	    "where (( exists (from MLExampleFeature f where m.exampleId =f.relatedExample and featureValuePair in ("+type1_from_fvpIds+"))" +
	    " and exists (from MLExampleFeature f where m.exampleId =f.relatedExample and featureValuePair in ("+type2_to_fvpIds+"))) or" +
	    " (exists (from MLExampleFeature f where m.exampleId =f.relatedExample and featureValuePair in ("+type2_from_fvpIds+")) and " +
	    		"exists (from MLExampleFeature f where m.exampleId =f.relatedExample and featureValuePair in ("+type1_to_fvpIds+")))) " +
	    " and corpusName =:corpusName " +
	    " and forTrain="+(for_train?1:0) +" and " +
	      "associatedFilePath in (" +
	      docPaths + ") " +
      "order by associatedFilePath desc";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("corpusName", experimentgroup);
		
		return getExamplesList(hql, params);
	}
	public static List<MLExample> getLastExamplesByDocument(String experimentgroup, 
			boolean for_train, int num_of_documents)
	{
		List<Artifact> docs = Artifact.listByType(Type.Document,for_train);
		if(docs.size()<num_of_documents)
			num_of_documents = docs.size();
		
		String docPaths = "";
		for(int i=docs.size()-1;i>docs.size()-num_of_documents-1;i--)
			docPaths = docPaths.concat(", '"+docs.get(i).getAssociatedFilePath()+"'");
		docPaths = docPaths.replaceFirst(",", "");
		
		String hql = "FROM MLExample "  +
				"where corpusName =:corpusName " +
				" and forTrain="+(for_train?1:0) +" and associatedFilePath in (" +
						docPaths + ") " +
						"order by associatedFilePath desc";
		
		HashMap<String, Object> params = new HashMap<String, Object>();
		params.put("corpusName", experimentgroup);
		
		return getExamplesList(hql, params);
	}

	@Override
	public MLExample clone()
	{
		if(relatedArtifact!=null)
			return getInstanceForArtifact(relatedArtifact, corpusName);
		else
			return getInstanceForLink(relatedPhraseLink, corpusName);	}


	
	public static void resetExamplesPredicted(String experimentgroup, boolean for_train) {
		String hql = "update MLExample set predictedClass = -1 where corpusName = '"+
				experimentgroup+"' and forTrain="+(for_train?1:0);
		HibernateUtil.executeNonReader(hql);
	}
	public static void setExamplePredictedClass(int example_id, int predicted) {
		String hql = "update MLExample set predictedClass = "+predicted+" where exampleId="+example_id;
		HibernateUtil.executeNonReader(hql);
	}
	public static void resetExamplesPredictedToDefault(String experimentgroup, boolean for_train, int default_predicted) {
		String hql = "update MLExample set predictedClass = "+default_predicted+" where corpusName = '"+
				experimentgroup+"' and forTrain="+(for_train?1:0);
		HibernateUtil.executeNonReader(hql);
	}

	public void setAssociatedFilePath(String associatedFilePath) {
		this.associatedFilePath = associatedFilePath;
	}

	public String getAssociatedFilePath() {
		return associatedFilePath;
	}
	public static void updateAssociatedFilePath() {
		String hql = "from MLExample ";
		Session tempSession = HibernateUtil.sessionFactory.openSession();
		List<MLExample> examples = 
				(List<MLExample>) HibernateUtil.executeReader(hql, null,null, tempSession);
		
		for (MLExample example: examples)
		{
			PhraseLink related_phrase_link = example.getRelatedPhraseLink();
			Phrase from_phrase = related_phrase_link.getFromPhrase();
			Artifact start_artifact = from_phrase.getStartArtifact();
			
			String file_path = start_artifact.getAssociatedFilePath();
			example.setAssociatedFilePath(file_path);
			HibernateUtil.save(example, tempSession);
			
		}
		tempSession.clear();
		tempSession.close();
	}

	public static MLExample findInstance(PhraseLink phrase_link,
			String experimentgroup) {
		String hql = "from MLExample where relatedPhraseLink = "+
				phrase_link.getPhraseLinkId() + " and corpusName = '"+
						experimentgroup+"'";
			List<MLExample> example_objects = 
					getExamplesList(hql);
		    
			MLExample example_obj=null;
		    if(example_objects.size()!=0)
		    {
		    	example_obj = 
		    			example_objects.get(0);
		    }
		    return example_obj;
	}
	public static MLExample findInstance(PhraseLink phrase_link) {
		String hql = "from MLExample where relatedPhraseLink = "+
				phrase_link.getPhraseLinkId();
		
			List<MLExample> example_objects = 
					getExamplesList(hql);
		    
			MLExample example_obj=null;
		    if(example_objects.size()!=0)
		    {
		    	example_obj = 
		    			example_objects.get(0);
		    }
		    return example_obj;
	}

	public void setPredictionWeight(double predictionWeight) {
		this.predictionWeight = predictionWeight;
	}

	public double getPredictionWeight() {
		return predictionWeight;
	}

	public static List<MLExample> getDecidedExamplesForGraph(Artifact p_sentence) {
		String hql = "from MLExample where predictedClass <> -1 " +
				"and relatedPhraseLink.fromPhrase.startArtifact.parentArtifact = "+
		p_sentence.getArtifactId()+" and relatedPhraseLink.toPhrase.startArtifact.parentArtifact = "+
		p_sentence.getArtifactId();

		List<MLExample> example_objects = 
				getExamplesList(hql);
	    
	    return example_objects;
	}
	 public static List<MLExample> getPhraseExamplesByPredictedClass(Artifact p_sentence, String corpusName, boolean forTrain, Integer predictedClass) {
		 

//		String hql = "from MLExample where (predictedClass = 1 or predictedClass=2 ) and forTrain="+(forTrain?1:0)
		String hql = "from MLExample where (predictedClass = "+predictedClass+" ) and forTrain="+(forTrain?1:0)
					+
				" and corpusName = '"+corpusName+"' and relatedPhrase.startArtifact.parentArtifact = "+p_sentence.getArtifactId();

		List<MLExample> example_objects = 
				getExamplesList(hql);
	    
	    return example_objects;
	}
	 public static List<MLExample> getPhraseExamplesByCorpus(Artifact p_sentence, String corpusName, boolean forTrain) {
		 
		String hql = "from MLExample where forTrain="+(forTrain?1:0)
					+
				" and corpusName = '"+corpusName+"' and relatedPhrase.startArtifact.parentArtifact = "+p_sentence.getArtifactId();

		List<MLExample> example_objects = 
				getExamplesList(hql);
	    
	    return example_objects;
	}

	public void setExpectedReal(int expectedReal) {
		this.expectedReal = expectedReal;
	}

	public int getExpectedReal() {
		return expectedReal;
	}

	public void setExpectedClosure(int expectedClosure) {
		this.expectedClosure = expectedClosure;
	}

	public int getExpectedClosure() {
		return expectedClosure;
	}

	public void setExpectedIntegrated(int expectedIntegrated) {
		this.expectedIntegrated = expectedIntegrated;
	}

	public int getExpectedIntegrated() {
		return expectedIntegrated;
	}

	public static MLExample getInstanceForPhrase(Phrase relatedPhrase,
			String experimentgroup) {
		String hql = "from MLExample where relatedPhrase = "+
				relatedPhrase.getPhraseId() + " and corpusName = '"+
				experimentgroup+"'";
			List<MLExample> example_objects = 
					getExamplesList(hql);
		    
			MLExample example_obj;
		    if(example_objects.size()==0)
		    {
		    	example_obj = new MLExample();
		    	example_obj.setCorpusName(experimentgroup);
		    	example_obj.setRelatedPhrase(relatedPhrase);
		    	
		    	if(Setting.SaveInGetInstance)
			    	saveExample(example_obj);
		    }else
		    {
		    	example_obj = 
		    			example_objects.get(0);
		    }
		    return example_obj;
	}

	@Transient
	public String getRelatedDrug() {
		return relatedDrug;
	}

	public void setRelatedDrug(String relatedDrug) {
		this.relatedDrug = relatedDrug;
	}
//	@OneToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} , fetch=FetchType.LAZY )
	@OneToOne( cascade = {CascadeType.PERSIST, CascadeType.MERGE} )
    @JoinColumn(name="relatedPhrase")
	public Phrase getRelatedPhrase() {
		return relatedPhrase;
	}
	public void setRelatedPhrase(Phrase relatedPhrase) {
		this.relatedPhrase = relatedPhrase;
	}
} public class FileUtil {
	/**
	 * Create new file if not exists
	 * @param path
	 * @return true if new file created
	 */
	public static boolean createFileIfNotExists(String path) {
		boolean result = false;
		File modelFile = new File(path);
		if (!modelFile.exists()) {
			try {
				result = modelFile.createNewFile();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		return result;
	}
	 
	 public static void appendLine(String path, String line) throws UnsupportedEncodingException, FileNotFoundException
	 {
		 File file = new File(path);
		 Writer writer = new BufferedWriter(new OutputStreamWriter(
			        new FileOutputStream(file, true), "UTF-8"));
			
			try {
				writer.write(line+"\n");
				writer.flush();
				writer.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
	 }
	public static void createFilewithFormat(String path,
				List<String> contentLines,String format) throws UnsupportedEncodingException, FileNotFoundException {

			File file = new File(path);
			Writer out = new BufferedWriter(new OutputStreamWriter(
					new FileOutputStream(file), format));
			
			try {
				
				for(int i=0;i<contentLines.size();i++)
				{
					String line = contentLines.get(i);
					out.write(line+"\n");
				}
				out.flush();
				out.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
			
			
		}

	
	public static void createFileIfNotExists(String path,
			List<String> contentLines) {

		File file = new File(path);
		if (!file.exists()) {
			try {
				file.createNewFile();
				BufferedWriter writer = new BufferedWriter(new FileWriter(file));
				for(int i=0;i<contentLines.size();i++)
				{
					String line = contentLines.get(i);
					writer.write(line+"\n");
				}
				writer.flush();
				writer.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		
	}
	
	public static void createFile(String path,
			List<String> contentLines) {

		File file = new File(path);
		
		try {
			file.createNewFile();
			BufferedWriter writer = new BufferedWriter(new FileWriter(file));
			for(int i=0;i<contentLines.size();i++)
			{
				String line = contentLines.get(i);
				writer.write(line+"\n");
			}
			writer.flush();
			writer.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		
	}
	public static void writeToExistingFile(File file,
			List<String> contentLines) {

		
		try {
			BufferedWriter writer = new BufferedWriter(new FileWriter(file));
			for(int i=0;i<contentLines.size();i++)
			{
				String line = contentLines.get(i);
				writer.write(line+"\n");
			}
			writer.flush();
			writer.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		
	}
	public static void writeToFile(File file,
			List<String> contentLines) {
		
		try {
			file.createNewFile();
			BufferedWriter writer = new BufferedWriter(new FileWriter(file));
			for(int i=0;i<contentLines.size();i++)
			{
				String line = contentLines.get(i);
				writer.write(line+"\n");
			}
			writer.flush();
			writer.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		
	}
	public static void createFile(String path,
			String content) {

		File file = new File(path);
		
		try {
			file.createNewFile();
			BufferedWriter writer = new BufferedWriter(new FileWriter(file));
			writer.write(content+"\n");
			writer.flush();
			writer.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		
	}
	
	/**
	 * 
	 * @param path
	 * @return true if created
	 */
	public static boolean createFolderIfNotExists(String path) {
		boolean result = false;
		File modelFolder = new File(path);
		if (!modelFolder.exists() || !modelFolder.isDirectory()) {
			result = modelFolder.mkdir();
		}

		return result;
	}
	public static String createFolderAndGetPath(String path) {
		boolean result = false;
		File modelFolder = new File(path);
		if (!modelFolder.exists() || !modelFolder.isDirectory()) {
			result = modelFolder.mkdir();
		}
		if (result == true || modelFolder.exists())
			return path;
		else
			return null;
	}

	public static String deleteAndCreateFolder(String path) throws IOException {
		
		boolean success_del;
		File modelFolder = new File(path);
		success_del = deleteDirectory(modelFolder);
		if (!success_del) {
		    // Deletion failed
			throw new IOException();
		}
		File new_modelFolder = new File(path);

		if (new_modelFolder.mkdir()==true)
		{
			return path;
		}
		else{
			throw new IOException();
		}

	}
  static  boolean deleteDirectory(File path) {
	    if( path.exists() ) {
	      File[] files = path.listFiles();
	      for(int i=0; i<files.length; i++) {
	         if(files[i].isDirectory()) {
	           deleteDirectory(files[i]);
	         }
	         else {
	           files[i].delete();
	         }
	      }
	    }
	    return( path.delete() );
	  }
	
	public static void CopyFileToDirectory(String file_name, String from_directory, String to_directory)
	{
	  	InputStream inStream = null;
		OutputStream outStream = null;
	 
    	try{
 
    	    File afile =new File(from_directory+"/"+file_name);
    	    File bfile =new File(to_directory+"/"+file_name);
 
    	    inStream = new FileInputStream(afile);
    	    outStream = new FileOutputStream(bfile);
 
    	    byte[] buffer = new byte[1024];
 
    	    int length;
    	    //copy the file content in bytes 
    	    while ((length = inStream.read(buffer)) > 0){
 
    	    	outStream.write(buffer, 0, length);
 
    	    }
 
    	    inStream.close();
    	    outStream.close();
 
    	    System.out.println("File is copied successful!");
 
    	}catch(IOException e){
    	    e.printStackTrace();
    	}
	    
	}
	
	/**
	 * Read whole file content into a string
	 * @param filePath
	 * @return The File content
	 * @throws IOException
	 */
	public static String readWholeFile(String filePath) throws IOException
	{
		StringBuffer fileData = new StringBuffer(1000);
		FileReader fr = new FileReader(filePath);
        BufferedReader reader = new BufferedReader(fr);
        char[] buf = new char[1024];
        int numRead=0;
        while((numRead=reader.read(buf)) != -1){
            String readData = String.valueOf(buf, 0, numRead);
            fileData.append(readData);
            buf = new char[1024];
        }
        reader.close();
        fr.close();
        return fileData.toString();
	}
	
	public static FileIterator getFilesInDirectory(String root)
	{
		ExtensionFilter filter = new ExtensionFilter(".txt");
		FileIterator file_iterator = new FileIterator(new File(root),
				filter, FileIterator.LAST_DIRECTORY);
		return file_iterator;
	}
	public static FileIterator getallFilesInDirectory(String root)
	{
		FileIterator file_iterator = new FileIterator(new File(root),
				null, FileIterator.LAST_DIRECTORY);
		return file_iterator;
	}
	public static FileIterator getFilesInDirectory(String root,String extension)
	{
		ExtensionFilter filter = new ExtensionFilter("."+extension);
		FileIterator file_iterator = new FileIterator(new File(root),
				filter, FileIterator.LAST_DIRECTORY);
		return file_iterator;
	}
	
	public static List<String> loadLineByLine(String file_path) {
		List<String> lines = new ArrayList<String>();
		try {
			File f = new File(file_path);
			if(!f.exists()) return lines;
			BufferedReader br1 = new BufferedReader(
					new FileReader(f));
			while (br1.ready()) {
				String line = br1.readLine();
				lines.add(line);
			}
			br1.close();
		} catch (Exception e) {
			e.printStackTrace();
		}
		return lines;
	}
	public static List<String> loadLineByLineAndTrim(String file_path) {
		List<String> lines = new ArrayList<String>();
		try {
			File f = new File(file_path);
			if(!f.exists()) return lines;
			BufferedReader br1 = new BufferedReader(
					new FileReader(f));
			while (br1.ready()) {
				String line = br1.readLine();
				lines.add(line.trim());
			}
			br1.close();
		} catch (Exception e) {
			e.printStackTrace();
		}
		return lines;
	}


	public static boolean fileExists(String path) {
		File modelFile = new File(path);
		return modelFile.exists();
	}
	public static String getFilePathInDirectory(String directory, String fileName)
	{
		String file_path = directory;
		file_path += "/"+fileName;
		return file_path;
	}
	public static void logLine(String filePath, String line)
	{
		if(filePath==null)
		{
			String log = (new Date())+" : "+line+"\n";
			System.out.print(log);
		}else
		{
			try {
				
				BufferedWriter writer = new BufferedWriter( new FileWriter( filePath , true ) );
//				writer.write((new Date())+" : "+line+"\n");
				writer.write(line+"\n");
				writer.flush();
				writer.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}
	public static String ReadFileInToString(String filePath) throws IOException {
	       
        //create file object
        File file = new File(filePath);
        BufferedInputStream bin = null;
        String strFileContents = null;
        StringBuffer fileData = new StringBuffer();
        BufferedReader reader = new BufferedReader(
                new FileReader(filePath));
        char[] buf = new char[1024];
        int numRead=0;
        while((numRead=reader.read(buf)) != -1){
            String readData = String.valueOf(buf, 0, numRead);
            fileData.append(readData);
        }
        reader.close();
        return fileData.toString();
	}
	
	

 

} public class StringUtil {
	static String lemmatizerResourceFolder ="lemmatiser";
	static EngLemmatiser lemmatiser = null;
	static{
		initialize();
	}
	public static void initialize()
	{	
		try {
			System.out.println("start loading the lemmatiser...");
			System.out.println(System.getProperty("user.dir"));
			lemmatiser = new EngLemmatiser(System.getProperty("user.dir")+"/lemmatiser", true, false);
			System.out.println("Lemmatizer loaded");
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
//	static EngLemmatiser lemmatiser = new EngLemmatiser("/rnlp-stb/nlpdata/lemmatiser",	true, false);

	/**
	 * 
	 * @param inputString
	 * @return MD5 hash of given string
	 * @throws UnsupportedEncodingException
	 * @throws NoSuchAlgorithmException
	 */
	public static String getStringDigest(String inputString)
			throws UnsupportedEncodingException, NoSuchAlgorithmException {
		MessageDigest md = MessageDigest.getInstance("MD5");
		md.update(inputString.getBytes(), 0, inputString.length());

		return new BigInteger(1, md.digest()).toString(16);
	}

	/**
	 * Customized definition of stop words for word
	 * @param word
	 * @return
	 */
	public static List<String> customGoogleCommonWord = FileUtil.loadLineByLine
			("/home/azadeh/projects/drug-effect-ext/data/abbvie-experiments/google-comon-words.txt");
	public static boolean isStopWord(String word){
		boolean isStopWord = false;
		
		if(word.length() < 2
				||	Stopwords.isStopword(word)
				|| customGoogleCommonWord.contains(word)
				|| word.matches("\\W+")
				)
			isStopWord = true;
		
		return isStopWord;
	}
	public static String removeStopWords(String text){
		boolean isStopWord = false;
		String[] tokens = text.split(" ");
		String new_text = "";
		
		for (String token:tokens)
		{
			if(token.length() < 2
					||	Stopwords.isStopword(token)
					|| customGoogleCommonWord.contains(token)
					|| token.matches("\\W+")
					)
				continue;
			else
				new_text=new_text+" "+token;
			
		}
		
		return new_text.trim();
	}
	

	
	/**
	 * Porter stem
	 * @param word
	 * @return stemmed word
	 */
	public static String getWordPorterStem(String word)
	{
		PorterStemmer stemmer = new PorterStemmer();
		String stemmed_word = stemmer.stem(word).toLowerCase();
		return stemmed_word;
	}
	public static String prepareSQLString(String sqlString) {
		sqlString = sqlString.replace("\\", "\\\\").
			replace("'", "''").
			replace("%", "\\%").
			replace("_", "\\_");
		return sqlString;
	}

public static String castForRegex(String textContent) {
		
		return textContent.replace("\\","\\\\").replace("/","\\/").replace("*", "\\*").replace("+", "\\+").replace(".", "\\.").replace("?", "\\?")
			.replace(")", "\\)").replace("{", "\\{").replace("}", "\\}")
			.replace("(", "\\(").replace("[", "\\[").replace("]", "\\]").replace("%", "\\%");
	}
public static String decastRegex(String textContent) {
		
		return textContent.replace("\\\\","\\").replace("\\/","/").replace("\\*", "*").replace("\\+", "+").replace("\\.", ".").replace("\\?", "?")
			.replace("\\)", ")").replace("\\_", "_")
			.replace("\\{", "{").replace("\\}", "}").replace("\\(", "(").
			replace("\\[", "[").replace("\\]", "]").replace("\\%", "%");
	}
	
	public static String getTermByTermPorter(String phrase)
	{
		String[] words = phrase.split(" ");
		String rootString = "";
		for(int i=0;i<words.length;i++){
			rootString += StringUtil.getWordPorterStem(words[i])+" ";
		}
		return rootString.trim();
	}
	
	public static String compress(String text) {
		return text.replace(" ", "").replace(" ", "");
	}
	static HashMap<String, String> lemmaCache = new HashMap<String, String>();

	public static String getTermByTermWordnet(String phrase)
	{
		String[] words = phrase.split(" ");
		String rootString = "";
		for(int i=0;i<words.length;i++)
		{
			String lemma = lemmaCache.get(words[i]);
			if (words[i].equals(lemma)
					&& lemma.matches(".*ioning$"))
			{
				lemma=lemma.replaceAll("ioning$", "ion");
			}
			if(lemma == null)
			{
				lemma = lemmatiser.stem(words[i]);
				lemmaCache.put(words[i], lemma);
			}
			rootString = rootString.concat(lemma+" ");
		}
		
		return rootString.trim();
	}
	public static String getWordLemma(String word)
	{
		String word_lemma= word;

		word_lemma= lemmatiser.stem(word);
		return word_lemma;
	}

	public static String orderStringAlphabetically(String text) {
		String words[] =text.split(" ");
        String t;
        int n = words.length;
        int i,j,c;
        for (i=0; i<n-1; i++) 
        {
            for (j=i+1; j<n; j++) 
            {
                c = words[i].compareTo(words[j]);
                if (c >0) 
                {
                    t = words[i];
                    words[i] = words[j];
                    words[j] = t;       
                }   
            }
        }
        String ordered_words = "";
        for (i=0; i<n ;i++) 
        {
        	ordered_words +=words[i]+" ";
        }
		return ordered_words.trim();
	}

	public static String cleanString(String text) {
		String clean = text.replaceAll("[\\d[^\\w\\s]]+", " ").replaceAll("(\\s{2,})", " ").toLowerCase();
		return clean.trim();
	}
}
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import javax.print.attribute.HashAttributeSet;
import org.apache.lucene.queryParser.ParseException;
import LuceneManagerADR.ADRLuceneSearcher;
import edu.asu.diego.adrmine.classification.TokenSequenceExampleBuilder;
import edu.asu.diego.adrmine.classification.TokenSequenceExampleBuilder.TokenLabel;
import edu.asu.diego.adrmine.evaluation.CRFBasedExtractionUtils.TargetSemanticType;
import preprocessutils.StanfordParser;
import rainbownlp.core.Artifact;
import rainbownlp.core.Phrase;
import rainbownlp.machineLearning.MLExample;
import rainbownlp.util.FileUtil;
import rainbownlp.util.StringUtil;
package edu.asu.diego.adrmine.evaluation;







public class MainConceptExtractionEvaluation {
	public static String experimentgroup = TokenSequenceExampleBuilder.ExperimentGroupADRConcepts;
	private boolean conbineWithLexiconCandClasses = false;
	public static  int  countFP=0;
	public static int  countFN=0;
	public static boolean applyPostProcessingRules= false;
	public static boolean tokenizeExpectedADRs = true;
	public static StanfordParser parser = new StanfordParser();