public class JATEException extends Exception {
	private static final long serialVersionUID = -4511571544868986419L;

	public JATEException(String exception){
		super(exception);
	}

	public JATEException(Exception e){
		super(e);
	}

	public JATEException(String exception, Throwable cause){
		super(exception, cause);
	}
}
 public class JATEProperties {
    private static final Logger _logger = Logger.getLogger(JATEProperties.class);

    private final Properties _properties = new Properties();
    private static JATEProperties _ref = null;

    //public static final String NP_FILTER_PATTERN = "[^a-zA-Z0-9\\-]";
    //replaced by the following var:
    public static final String TERM_CLEAN_PATTERN = "[^a-zA-Z0-9\\-]";
    public static final String REGEX_QUOTES="[\"]";
    

    public static final String NLP_PATH = "jate.system.nlp";
    public static final String TERM_MAX_WORDS = "jate.system.term.maxwords";
    public static final String TERM_IGNORE_DIGITS = "jate.system.term.ignore_digits";
    public static final String MULTITHREAD_COUNTER_NUMBERS="jate.system.term.frequency.counter.multithread";
    public static final String CONTEXT_PERCENTAGE = "jate.system.NCValue.percent";
    //Ankit: Adding separate input parameter for ChiSquareAlgorithm
    public static final String TOP_FREQUENT_PERCENTAGE = "jate.system.ChiSquare.percent";
    
    public static final String CORPUS_PATH = "jate.system.corpus_path";
    public static final String RESULT_PATH="jate.system.result_path";
    public static final String IGNORE_SINGLE_WORDS = "jate.system.term.ignore_singleWords";
    public static final String TERM_MIN_FREQ = "jate.system.term.minFreq";
    
    /*code modification RAKE : starts*/
    
    public static final String TERM_CLEAN_PATTERN_RAKE = "[^a-zA-Z0-9\\-.'&_]";
    
    /*code modification RAKE : ends*/

    private JATEProperties() {
        read();
    }

    public static JATEProperties getInstance() {
        if (_ref == null) {
            _ref = new JATEProperties();
        }
        return _ref;
    }

    private void read() {
        InputStream in = null;
        try {
            /*InputStream x= getClass().getResourceAsStream("/indexing.properties");*/
            in = getClass().getResourceAsStream("/jate.properties");
            _properties.load(in);
        } catch (IOException e) {
            _logger.error( "Error reading properites", e );
        } finally {
            if (in != null) try {
                in.close();
                in = null;
            } catch (IOException e) {
                _logger.error( "Error while closing file", e );
            }
        }
    }

    private String getProperty(String key) {
        return _properties.getProperty(key);
    }

    public void setProperty( String key, String value ) {
        _properties.setProperty( key, value );
    }

    public String getWorkPath() {
        return System.getProperty( "java.io.tmpdir" );
    }

    public String getNLPPath() {
        return getProperty(NLP_PATH);
    }

    public InputStream getNLPInputStream( String resource ) throws IOException {
        String path = getNLPPath();
        while ( path.endsWith( "/" ) || path.endsWith( "\\" ) ) {
            path = path.substring( 0, path.length() - 1 );
        }
        while ( resource.startsWith("/") || resource.startsWith( "\\" ) ) {
            resource = resource.substring( 1 );
        }
        if ( path.startsWith( "jar:" ) ) {
            return getClass().getResourceAsStream( File.separator + path.substring( "jar:".length() )  + File.separator + resource );
        }
        return new FileInputStream( path + File.separator + resource );
    }

    public int getMaxMultipleWords() {
        try {
            return Integer.valueOf(getProperty(TERM_MAX_WORDS));
        } catch (NumberFormatException e) {
            return 5;
        }
    }

    public int getMultithreadCounterNumbers() {
        try {
            return Integer.valueOf(getProperty(MULTITHREAD_COUNTER_NUMBERS));
        } catch (NumberFormatException e) {
            return 5;
        }
    }

    public boolean isIgnoringDigits() {
        try {
            return Boolean.valueOf(getProperty(TERM_IGNORE_DIGITS));
        } catch (Exception e) {
            return true;
        }
    }
    
    //Ankit: Getter for ChiSquare top frequent percentage parameter
    public int getTopFrequentPercentage(){
    	try{
    		return Integer.valueOf(getProperty(TOP_FREQUENT_PERCENTAGE));
    	} catch (Exception e){
    		return 10;
    	}
    }
    
    //code modification NCValue begins
    
    public int getPercentage() {
        try {
            return Integer.valueOf(getProperty(CONTEXT_PERCENTAGE));
        } catch (Exception e) {
            return 30;
        }
    }

    // Only use this for testing
    public String getCorpusPath() {
        return getProperty(CORPUS_PATH);
    }

    // Only use this for testing
    public String getResultPath() {
        return getProperty(RESULT_PATH);
    }
    
    public boolean isIgnoringSingleWords() {
        try {
            return Boolean.valueOf(getProperty(IGNORE_SINGLE_WORDS));
        } catch (Exception e) {
            return true;
        }
    }
    
    //code modification NCValue ends
     
    //code modification chi Square begins
    public int getMinFreq() {
        try {
            return Integer.valueOf(getProperty(TERM_MIN_FREQ));
        } catch (NumberFormatException e) {
            return 4;
        }
    }
    
    //code modification chi Square ends
    

}
 public class CValueAlgorithm implements Algorithm {

	/**
	 *
	 * @param store
	 * @return
	 * @throws uk.ac.shef.dcs.oak.jate.JATEException
	 */
	public Term[] execute(AbstractFeatureWrapper store) throws JATEException {
		if (!(store instanceof CValueFeatureWrapper)) throw new JATEException("" +
				"Required: CValueFeatureWrapper");
		CValueFeatureWrapper cFeatureStore = (CValueFeatureWrapper) store;
		Set<Term> result = new HashSet<Term>();

		for(String s:cFeatureStore.getTerms()){
			double score;
			double log2a=Math.log((double)s.split(" ").length  + 0.1)/Math.log(2.0); //Anurag mods for log (a), log(a + 0.1)
			double freqa=(double)cFeatureStore.getTermFreq(s);
			int[] nest = cFeatureStore.getNestsOf(s);
			double pTa = (double)nest.length;
			double sumFreqb=0.0;
			
			/* code modification begins */
			
			for(Integer i:nest){
				if(i!=null)
				{
					sumFreqb+=(double)cFeatureStore.getTermFreq(i);		
					//String x= cFeatureStore.getTerm(i);
					int[] nest_i = cFeatureStore.getNestsOf(cFeatureStore.getTerm(i));
					List<Integer> i_already_considered = new ArrayList<Integer>();
					for(Integer j:nest_i)
					{
						if (j!=null && (!i_already_considered.contains(j)|| i_already_considered.size()==0))
						{
							sumFreqb-=(double)cFeatureStore.getTermFreq(j);
							int[] nest_j = cFeatureStore.getNestsOf(cFeatureStore.getTerm(j));
							for(int x1: nest_j)
								i_already_considered.add(x1);
						}
						
					}		
				}				
			}
			
		
			//Commented LOCs in original version
			
			/*
			for(Integer i:nest){
				if(i!=null) sumFreqb+=(double)cFeatureStore.getTermFreq(i);
			}
			*/
			
			/*code modification ends*/

			score = pTa==0?log2a*freqa:log2a*(freqa-(sumFreqb/pTa));

			result.add(new Term(s,score));
		}

		Term[] all = result.toArray(new Term[result.size()]);
		Arrays.sort(all);
		return all;
	}
	
	

	public String toString(){
		return "CValue_ALGORITHM";
	}
} public class CValueFeatureWrapper extends AbstractFeatureWrapper {

	private final FeatureCorpusTermFrequency _termFreq;
	private final FeatureTermNest _termNest;

	/**
	 * Creates an instance. CValueFeatureWrapper wraps an instance of FeatureCorpusTermFrequency and an instance of FeatureTermNest
	 * @param termFreq
	 * @param termNest
	 */
	public CValueFeatureWrapper(FeatureCorpusTermFrequency termFreq, FeatureTermNest termNest){
		_termFreq=termFreq;
		_termNest=termNest;
	}

	/**
	 * @param term
	 * @return number of occurrences of a term in a corpus. If a term cannot be found it always returns 1.
	 */
	public int getTermFreq(String term){
		int freq= _termFreq.getTermFreq(term);
		return freq==0?1:freq;
	}

	/**
	 * @param id - the id of the term in question
	 * @return number of occurrences of a term in a corpus. If a term cannot be found it always returns 1.
	 */
	public int getTermFreq(int id){
		int freq= _termFreq.getTermFreq(id);
		return freq==0?1:freq;
	}

	/**
	 * @param nested
	 * @return ids of terms in which the provided term nests in
	 */
	public int[] getNestsOf(String nested){
		Set<Integer> res = _termNest.getNestIdsOf(nested);
		if (res==null)
            return new int[0];
        int[] result = new int[res.size()];
        int c=0;
        for(int i: res){
            result[c]=i;
            c++;
        }
        return result;
	}
	
	//modified code for NC-Value begins
	
	public String getTerm(int id){
		return _termFreq.getGlobalIndex().retrieveTermCanonical(id);
	}
	
	//modified code ends

	public Set<String> getTerms(){
		return _termFreq.getGlobalIndex().getTermsCanonical();
	}

}
 public class FeatureBuilderCorpusTermFrequency extends AbstractFeatureBuilder {

	private static final Logger _logger = Logger.getLogger(FeatureBuilderCorpusTermFrequency.class);

	/**
	 * Creates an instance
	 * @param counter1 candidate term counter, counting distributions of candidate terms
	 * @param counter2 word counter, counting number of words in documents
	 * @param normaliser a normaliser for returning terms to their canonical forms
	 * over the corpus and add up to the total frequencies of the lemma.
	 */
	public FeatureBuilderCorpusTermFrequency(TermFreqCounter counter1, WordCounter counter2, Normalizer normaliser) {
		super(counter1, counter2, normaliser);
	}

	/**
	 * Build an instance of FeatureCorpusTermFrequency
	 * @param index the global resource index
	 * @return
	 * @throws uk.ac.shef.dcs.oak.jate.JATEException
	 */
	public FeatureCorpusTermFrequency build(GlobalIndex index) throws JATEException {
		FeatureCorpusTermFrequency _feature = new FeatureCorpusTermFrequency(index);
		if (index.getTermsCanonical().size() == 0 || index.getDocuments().size() == 0) throw new
                JATEException("No resource indexed!");

		_logger.info("About to build FeatureCorpusTermFrequency...");
		int totalCorpusTermFreq = 0;
		//for (Document d : index.getDocuments()) {
		//	_logger.info("For Document " + d);
			//String context = CandidateTermExtractor.applyCharacterReplacement(d.getContent(), JATEProperties.TERM_CLEAN_PATTERN);
		/*File file = new File ("C:\\Users\\aniya_agarwal\\Desktop\\reqtsent\\sample.txt");
		Document doc=null;
		try {
			doc = new DocumentImpl(file.toURI().toURL());
		} catch (MalformedURLException e) {
			_logger.error( "Error processing " + file, e );
		}
		*/
		
		
		for (Document d : index.getDocuments()) {
			_logger.info("For Document " + d);
			String context = CandidateTermExtractor.applyCharacterReplacement(d.getContent(), JATEProperties.TERM_CLEAN_PATTERN);
			
			totalCorpusTermFreq += _wordCounter.countWords(d);

			Set<String> candidates = index.retrieveTermsCanonicalInDoc(d);
            //counting by single-threaded termcounter
			for (String np : candidates) {
				//Set<String> va = index.retrieveVariantsOfTermCanonical(np);
				int freq = _termFreqCounter.count(context, index.retrieveVariantsOfTermCanonical(np));
				_feature.addToTermFreq(np, freq);
			}
		}
		_feature.setTotalCorpusTermFreq(totalCorpusTermFreq);

		return _feature;
	}

	
}
 public class FeatureBuilderTermNest extends AbstractFeatureBuilder {

    private static final Logger _logger = Logger.getLogger(FeatureBuilderTermNest.class);

    /**
     * Default constructor
     */
    public FeatureBuilderTermNest() {
        super(null, null, null);
    }

    /**
     * Build an instance of FeatureTermNest from an instance of GlobalIndexMem
     *
     * @param index
     * @return
     * @throws uk.ac.shef.dcs.oak.jate.JATEException
     *
     */
    public FeatureTermNest build(GlobalIndex index) throws JATEException {
        FeatureTermNest _feature = new FeatureTermNest(index);
        if (index.getTermsCanonical().size() == 0 || index.getDocuments().size() == 0) throw new
                JATEException("No resource indexed!");

        _logger.info("About to build FeatureTermNest...");
        int counter = 0;
        for (String np : index.getTermsCanonical()) {
          
        	/*for (String anp : index.getTermsCanonical()) {
                   if (anp.length() <= np.length()) continue;
                   if (anp.indexOf(" " + np) != -1 || anp.indexOf(np + " ") != -1) //e.g., np=male, anp=the male
                       _feature.termNestIn(np, anp);
               }
               counter++;

            Expensive string operations pulled out for performance of the inner loop
            final String spacePrefixed = " " + np;
            final String spaceSuffixed = np + " "; */
        	
        	
        	/*modified code begins */
        	
        	Set<String> variants_np = index.retrieveVariantsOfTermCanonical(np);
        	variants_np.add(np);
        	
        	 StringBuilder pattern = new StringBuilder("\\b(");
             Iterator<String> it = variants_np.iterator();
             while(it.hasNext())
             {
             	pattern.append(it.next()).append("|");
             }
             
             pattern.deleteCharAt(pattern.length()-1);
             pattern.append(")\\b");
             
             Pattern p = Pattern.compile(pattern.toString());
        	 
        	 
        	/*modified code ends*/ 
            
        	
        	for (String anp : index.getTermsCanonical()) {
                if (anp.length() <= np.length()) continue;
                //modified code begins              
               
                Matcher m = p.matcher(anp);

                if (m.find()) {
                	_feature.termNestIn(np, anp);                	
                }
                
        	}
            counter++;


            if (counter % 500 == 0) _logger.info("Batch done" + counter + " end: " + np);
        }
        return _feature;
    }
}
 public class FeatureCorpusTermFrequency extends AbstractFeature {

    private final Map<Integer, Integer> _termFreqMap = new ConcurrentHashMap<Integer, Integer>();
    private int _totalCorpusTermFreq = 0;

    protected FeatureCorpusTermFrequency(GlobalIndex index) {
        _index = index;
    }

    public int getTotalCorpusTermFreq() {
        return _totalCorpusTermFreq;
    }

    public void setTotalCorpusTermFreq(int i) {
        _totalCorpusTermFreq = i;
    }

    /**
     * increment the number of occurrences of term by i
     *
     * @param term
     * @param i
     */
    public void addToTermFreq(String term, int i) {
        int termId = _index.retrieveTermCanonical(term);
        if (termId == -1) {
            //_logger.info("Term (" + term + ") has not been indexed!");
        } else {
            addToTermFreq(termId, i);
        }
    }

    /**
     * increment the number of occurrences of term with id t by i
     *
     * @param t
     * @param i
     */
    public void addToTermFreq(int t, int i) {
        Integer freq = _termFreqMap.get(t);
        if (freq == null) freq = 0;
        _termFreqMap.put(t, freq + i);
    }

    /**
     * Set the number of occurrences of a term
     *
     * @param term
     * @param freq
     */
    public void setTermFreq(String term, int freq) {
        int termId = _index.retrieveTermCanonical(term);
        if (termId == -1) {
            //_logger.info("Term (" + term + ") has not been indexed!");
        } else {
            setTermFreq(termId, freq);
        }
    }

    /**
     * Set the number of occurrences of a term with id t
     *
     * @param t
     * @param freq
     */
    public void setTermFreq(int t, int freq) {
        _termFreqMap.put(t, freq);
    }

    /**
     * Get the number of occurrences of a term in the corpus
     *
     * @param term
     * @return
     */
    public int getTermFreq(String term) {
        int termId = _index.retrieveTermCanonical(term);
        if (termId == -1) {
            //_logger.info("Term (" + term + ") has not been indexed!");
            return 0;
        } else {
            return getTermFreq(termId);
        }
    }

    /**
     * Get the number of occurrences of a term in the corpus
     *
     * @param t the id of the term
     * @return
     */
    public int getTermFreq(int t) {
        Integer freq = _termFreqMap.get(t);
        if (freq == null) {
            freq = 0;
        }
        return freq;
    }

    /**
     * Get the normalised frequency of a term in the corpus, which is the number of occurrences of that term as a fraction
     * of the total number of occurrences of all terms in the corpus.
     *
     * @param term
     * @return
     */
    public double getNormalizedTermFreq(String term) {
        int termId = _index.retrieveTermCanonical(term);
        if (termId == -1) {
            //_logger.info("Term (" + term + ") has not been indexed!");
            return 0.0;
        } else {
            return getNormalizedTermFreq(termId);
        }
    }

    /**
     * Get the normalised frequency of a term in the corpus, which is the number of occurrences of that term as a fraction
     * of the total number of occurrences of all terms in the corpus.
     *
     * @param t the id of the term
     * @return
     */
    public double getNormalizedTermFreq(int t) {
        return (double) getTermFreq(t) / ((double) getTotalCorpusTermFreq() + 1);
    }
}
 public class FeatureTermNest extends AbstractFeature {

    private final Map<Integer, Set<Integer>> _termNested = new ConcurrentHashMap<Integer, Set<Integer>>();

    protected FeatureTermNest(GlobalIndex index) {
        _index = index;
    }

    /**
     * @param nested
     * @return the ids of terms in which the provided term is nested
     */
    public Set<Integer> getNestIdsOf(String nested) {
        int termId = _index.retrieveTermCanonical(nested);
        if (termId == -1) {
            //_logger.info("Term (" + nested + ") has not been indexed! Ignored.");
            return new HashSet<Integer>();
        } else {
            return getNestIdsOf(termId);
        }
    }

    /**
     * @param nested
     * @return the ids of terms in which the provided term with id "nested" is nested
     */
    public Set<Integer> getNestIdsOf(int nested) {
        Set<Integer> res = _termNested.get(nested);
        return res == null ? new HashSet<Integer>() : res;
    }

    /**
     * Index a term "nested" as nested in a term "nest"
     *
     * @param nested
     * @param nest
     */
    public void termNestIn(String nested, String nest) {
        int termId = _index.retrieveTermCanonical(nested);
        int nestId = _index.retrieveTermCanonical(nest);
        if (termId == -1) {
            //_logger.info("Term (" + nested + ") has not been indexed! Ignored.");
        } else if (nestId == -1) {
            //_logger.info("Term (" + nest + ") has not been indexed! Ignored.");
        } else {
            termNestIn(termId, nestId);
        }
    }

    /**
     * Index a term with id "nested" as nested in a term with id "nest"
     *
     * @param nested
     * @param nest
     */
    public void termNestIn(int nested, int nest) {
        Set<Integer> res = _termNested.get(nested);
        res = res == null ? new HashSet<Integer>() : res;
        res.add(nest);
        _termNested.put(nested, res);
    }


}
 public class TermVariantsUpdater{
	private final GlobalIndexMem _index;
	
	private final Map<String, Set<String>> additionalVariantsMap;
	
	public TermVariantsUpdater(GlobalIndexMem termDocIndex) {
		// TODO Auto-generated constructor stub
		_index = termDocIndex;
		additionalVariantsMap = new HashMap<String, Set<String>>();
	}

	/*no need to use lemmatizer to lemmatize the last word because 'canonical terms' are picked up from the global index itself which are already lemmatized.*/
	public GlobalIndexMem updateVariants(){
		for(String term:_index.getTermsCanonical()){
			//Set<String> v = _index.retrieveVariantsOfTermCanonical(term);
			boolean flag = false;
			Set<String> variants_term_additional = null;
			for(String lemmatized_term : _index.getTermsCanonical()){
				//lemmatized_term = lemmatized_term.toLowerCase();
				if(lemmatized_term.equals(term))
					continue;
				 Pattern p = Pattern.compile("\\b"+term+"\\b");
				 Matcher m = p.matcher(lemmatized_term);
	             if(!m.find()) {
	                	continue;                	
	             }
				else{
					int idx = lemmatized_term.indexOf(term);					
					int term_len = term.split(" ").length;
					Set<String> variants_term = _index.retrieveVariantsOfTermCanonical(term);					
					Set<String> variants_lemmatized_term = _index.retrieveVariantsOfTermCanonical(lemmatized_term);
					
					for(String variant:variants_lemmatized_term){
						String subString = variant.substring(idx);
						String[] split_subString = subString.split(" ");
						StringBuilder sb_substring = new StringBuilder();
						for(int i=0; i<term_len ;i++){
							sb_substring.append(split_subString[i]);
							sb_substring.append(" ");
						}
						if(variants_term.contains(sb_substring.toString().trim())){
							continue;
						}
						else{
							flag = true;
							if(variants_term_additional == null){
								variants_term_additional = new HashSet<String>();
							}
							variants_term_additional.add(sb_substring.toString().trim());
						}				
						
					}
				}
					
			}
			if(flag){
				additionalVariantsMap.put(term, variants_term_additional);
			}			
			
		}		
		if(additionalVariantsMap.size() > 0)
			_index.updateIndexTermWithVariant(additionalVariantsMap);
	
		return _index;
		
		
	}
	
} public class GlobalIndexBuilderMem implements GlobalIndexBuilder{

	private static final Logger _logger = Logger.getLogger(GlobalIndexBuilderMem.class);

	private boolean BUILD_TERM_TO_DOC_MAP = true;
	private boolean BUILD_DOC_TO_TERM_MAP = true;



	public GlobalIndexBuilderMem() {
	}

	/**
	 * creates an instance
	 * @param textunit2DocMap true if should create information of textunit-document binary relation
	 * @param doc2TextunitMap true if should create information of document-textunit binary relation
	 */
	public GlobalIndexBuilderMem(boolean textunit2DocMap, boolean doc2TextunitMap) {
		BUILD_TERM_TO_DOC_MAP = textunit2DocMap;
		BUILD_DOC_TO_TERM_MAP = doc2TextunitMap;
	}

	/**
	 * Build a GlobalIndexMem from a corpus, using the specified text unit extractor
	 * @param c
	 * @param extractor
	 * @return
	 * @throws uk.ac.shef.dcs.oak.jate.JATEException
	 */
	
	
	
	
	public GlobalIndexMem build(Corpus c, CandidateTermExtractor extractor) throws JATEException {
		GlobalIndexMem _index = new GlobalIndexMem();
		for (Document d : c) {
			_logger.info("For Document " + d);
			Map<String,Set<String>> nps = extractor.extract(d);
			_index.indexTermWithVariant(nps);
			if (BUILD_DOC_TO_TERM_MAP) _index.indexDocWithTermsCanonical(d, nps.keySet());
			if (BUILD_TERM_TO_DOC_MAP) {
				for (String t : nps.keySet()) _index.indexTermCanonicalInDoc(t, d);
			}
		}
		return _index;
	}
	
	
	
	//newly added for RAKE : code modified begins
	public List<String> build(Corpus c, PhraseExtractor extractor) throws JATEException, IOException {		
		return extractor.extract(c);	
		
	}
	//code modified ends
}
 public class NounPhraseExtractorOpenNLP extends CandidateTermExtractor {

	
	
	private static final Logger _logger = Logger.getLogger(NounPhraseExtractorOpenNLP.class);
 
    /**
     * Creates an instance with specified stopwords list and norm
     *
     * @param stop
     * @param normaliser
     * @throws IOException
     */
    public NounPhraseExtractorOpenNLP(IStopList stop, Normalizer normaliser) throws IOException {
        _stoplist = stop;
        _normaliser = normaliser;
        
    }

    public Map<String, Set<String>> extract(Corpus c) throws JATEException {
        Map<String, Set<String>> res = new HashMap<String, Set<String>>();
        for (Document d : c) {
            _logger.info("Extracting candidate NP... From Document " + d);
            for (Map.Entry<String, Set<String>> e : extract(d).entrySet()) {
                Set<String> variants = res.get(e.getKey());
                variants = variants == null ? new HashSet<String>() : variants;
                variants.addAll(e.getValue());
                res.put(e.getKey(), variants);
            }
        }
        return res;
    }
    
    
    //modified part begins..the first function only commented and the rest are new implementations

    public Map<String, Set<String>> extract(Document d) throws JATEException {
        Map<String, Set<String>> res = new HashMap<String, Set<String>>();
        try {
            for (String s : NLPToolsControllerOpenNLP.getInstance().getSentenceSplitter().sentDetect(d.getContent())) {
                for (Map.Entry<String, Set<String>> e : extract(s).entrySet()) {
                    Set<String> variants = res.get(e.getKey());
                    variants = variants == null ? new HashSet<String>() : variants;
                    variants.addAll(e.getValue());
                    res.put(e.getKey(), variants);
                }
            }
        } catch (IOException e) {
            throw new JATEException(e);
        }
       
        return res;
    }
    
    
    

   
    /*
    public Map<String, Set<String>> extract(Document d) throws JATEException {
        Map<String, Set<String>> res = new HashMap<String, Set<String>>();
        
        String[] candidateTerms = d.getContent().split("\\r?\\n"); 
        for (Map.Entry<String, Set<String>> e : extract(candidateTerms).entrySet()) {
        	Set<String> variants = res.get(e.getKey());
            variants = variants == null ? new HashSet<String>() : variants;
            variants.addAll(e.getValue());
            res.put(e.getKey(), variants);
        }       
        return res;
    }
    
    */
   
    public Map<String, Set<String>> extract(String[] candidates) throws JATEException {
        Map<String, Set<String>> nouns = new HashMap<String, Set<String>>(); 
        for (String c : candidates) {
        	
        	//modified code begins
        	c = applyCharacterReplacement(c,JATEProperties.REGEX_QUOTES);
            /*    c = applyCharacterReplacement(c, JATEProperties.TERM_CLEAN_PATTERN);
                String[] e = applySplitList(c);

                for (String str : e) {
                String str=c;
                   String stopremoved = applyTrimStopwords(str, _stoplist, _normaliser);
                    if (stopremoved == null) continue;
                */
        	String original = c;
            c = _normaliser.normalize(c.toLowerCase()).trim();
            if( c.equals(""))
            	continue;

                   /* String[] nelements = str.split("\\s+");
                    if (nelements.length < 1 || nelements.length >
                            Integer.valueOf(JATEProperties.getInstance().getMaxMultipleWords()))
                        continue;
                    if (JATEProperties.getInstance().isIgnoringDigits() &&
                            containsDigit(str))
                        continue;
                    if (!containsLetter(str)) continue;
                    if (!hasReasonableNumChars(str)) continue;				//needed or not?
*/
            //modified code ends
            if (c.toLowerCase().indexOf(c) != -1) {
            	Set<String> variants = nouns.get(c);
                variants = variants == null ? new HashSet<String>() : variants;
                variants.add(original);
                nouns.put(c, variants);
            }              
        }        
        return nouns;
    }
    
    
    
    
    
    //modified part ends
    
    


    public Map<String, Set<String>> extract(String content) throws JATEException {
      //  _logger.info(content+ "||" );
    	Map<String, Set<String>> nouns = new HashMap<String, Set<String>>();
        try {
            String[] tokens = NLPToolsControllerOpenNLP.getInstance().getTokeniser().tokenize(content);
            String[] pos = NLPToolsControllerOpenNLP.getInstance().getPosTagger().tag(tokens);
            String[] candidates = chunkNPs(tokens, pos);
            for (String c : candidates) {
                c = applyCharacterReplacement(c, JATEProperties.TERM_CLEAN_PATTERN);
                String[] e = applySplitList(c);

                for (String str : e) {
                    String stopremoved = applyTrimStopwords(str, _stoplist, _normaliser);
                    if (stopremoved == null) continue;
                    String original = stopremoved;
                    str = _normaliser.normalize(stopremoved.toLowerCase()).trim();

                    String[] nelements = str.split("\\s+");
                    if (nelements.length < 1 || nelements.length >
                            JATEProperties.getInstance().getMaxMultipleWords())
                        continue;
                    if (JATEProperties.getInstance().isIgnoringDigits() &&
                            containsDigit(str))
                        continue;
                    if (!containsLetter(str)) continue;
                    if (!hasReasonableNumChars(str)) continue;

                    if (c.toLowerCase().indexOf(str) != -1) {
                    	//System.out.print(original+"|");
                    	
                        Set<String> variants = nouns.get(str);
                        variants = variants == null ? new HashSet<String>() : variants;
                        variants.add(original);
                        nouns.put(str, variants);
                    }
                }
            }
          //  _logger.info("\n");
        } catch (IOException wte) {
            throw new JATEException(wte);
        }
       
        return nouns;
    }

    private String[] chunkNPs(String[] tokens, String[] pos) throws IOException {
        String[] phrases = NLPToolsControllerOpenNLP.getInstance().getPhraseChunker().chunk(tokens, pos);
       /* for(int i=0;i<phrases.length;i++){
        	_logger.info(tokens[i]+ " "+ phrases[i]);
        }
        */
        List<String> candidates = new ArrayList<String>();
        for (int n = 0; n < tokens.length; n++) {
            if (phrases[n].equals("B-NP")) {
                String phrase = tokens[n];
                for (int m = n + 1; m < tokens.length; m++) {
                    if (phrases[m].equals("I-NP")) {
                        phrase = phrase + " " + tokens[m];
                    } else {
                        n = m;
                        break;
                    }
                }
                phrase = phrase.replaceAll("\\s+", " ").trim();
                if (phrase.length() > 0)
                    candidates.add(phrase);

            }
        }
        return candidates.toArray(new String[candidates.size()]);
    }

   

} public class ResultWriter2File {

	private final GlobalIndex _index;

	/*modified code begins:RAKE*/
	
	public ResultWriter2File() {
		_index=null;
	}
	
	/*modified code ends:RAKE*/


	/**
	 * @param index an instance of GlobalIndexMem. The writer will read mapping data from term canonical from to variant
	 * forms and output the result
	 */
	public ResultWriter2File(GlobalIndex index) {
		this._index = index;
	}

	/**
	 * Output the result. This writes one term on a line in the format of:
	 * <p>
	 * canonical |variant1 |variant2 |variantX       confidence
	 * </p>
	 * @param result term recognition result sorted descendingly by confidence
	 * @param path output file path
	 * @throws uk.ac.shef.dcs.oak.jate.JATEException
	 */
	public void output(Term[] result, String path) throws JATEException {
		try {
			PrintWriter pw = new PrintWriter(new FileWriter(path));
			if (_index == null) {
				for (Term c : result) {
					pw.println(c.getConcept() + "\t\t\t" + c.getConfidence());
				}
			}
			else{
				for (Term c : result) {
					Set<String> originals = _index.retrieveVariantsOfTermCanonical(c.getConcept());
					if(originals==null || originals.size() == 0)
					{
						pw.println(c.getConcept() + "\t\t\t" + c.getConfidence());
					}
					else{
						pw.println(c.getConcept()+" |"+writeToString(originals) + "\t\t\t" + c.getConfidence());
					}
				}
			}
			pw.close();
		}
		catch (IOException ioe) {
			throw new JATEException(ioe);
		}
	}

	private String writeToString(Set<String> container){
		StringBuilder sb = new StringBuilder();
		for(String s:container){
			sb.append(s).append(" |");
		}
		return sb.toString().substring(0,sb.lastIndexOf("|"));
	}
}
 public interface IStopList {
    /**
     * @return true if the word is a stop word, false if otherwise
     */
    boolean isStopWord( String word );
} public class StopList implements IStopList {

	private final Set<String> words = new HashSet<String>();

	private final boolean _caseSensitive;

	/**
	 * Creates an instance of stop word list
	 * @param caseSensitive whether the list should ignore cases
	 * @throws IOException
	 */
	public StopList (final boolean caseSensitive) throws IOException {
		super();
		_caseSensitive =caseSensitive;
		loadStopList( JATEProperties.getInstance().getNLPInputStream( "stoplist.txt" ),_caseSensitive);
	}

	/**
	 * @param word
	 * @return true if the word is a stop word, false if otherwise
	 */
	public boolean isStopWord(String word){
		if(!_caseSensitive) return words.contains(word.toLowerCase());
		return words.contains(word);
	}

	private void loadStopList(final InputStream stream, final boolean lowercase) throws IOException {
      final BufferedReader reader = new BufferedReader(new InputStreamReader( stream ));
	  try {
		  String line;
		  while ((line = reader.readLine()) != null) {
			  line = line.trim();
			  if (line.equals("") || line.startsWith("//")) continue;
			  if (lowercase) words.add(line.toLowerCase());
			  else words.add(line);
		  }
	  } finally {
		  reader.close();
	  }
   }

}
 public class WordCounter {

	public WordCounter(){
	}

	/**
	 * Count number of words in a corpus, delimited by a space character
	 * @param c
	 * @return
	 */
	public int countWords(final Corpus c){
		int total=0;
		for(Document doc:c){
			StringTokenizer tokenizer = new StringTokenizer(doc.getContent().replaceAll("\\s+"," ")
					.replaceAll("\\s+"," ")," ");
			total+=tokenizer.countTokens();
		}
		return total;
	}

	/**
	 * Count number of words in a document, delimited by a space character
	 * @param d
	 * @return
	 */
	public int countWords(final Document d){
		
		
		/*StringTokenizer tokenizer = new StringTokenizer(d.getContent().replaceAll("\\s+"," ")
					.replaceAll("\\s+"," ")," ");
		*/
		//modified part
		
		StringTokenizer tokenizer = new StringTokenizer(d.getContent(), "\r\n|\n");
		
		return tokenizer.countTokens();			
	}

	/**
	 * Main testing method.
	 * @param args args[0] is the path to directory containing files
	 */
	public static void main(String[] args) {
		File targetFolder = new File(args[0]);
      	File[] files = targetFolder.listFiles();
		if ( files == null ) {
			System.err.println( "Unable to find files in " + args[0] );
			return;
		}
		Corpus c = new CorpusImpl();
      	for (File f : files) {
	      try {
		      c.add(new DocumentImpl(f.toURI().toURL()));
	      } catch (MalformedURLException e) {
		      e.printStackTrace();
	      }
      	}//for

		System.out.println(new WordCounter().countWords(c));
	}
}
 public class GlobalIndexMem extends GlobalIndex {

    /**
     * Term to TermId map
     */
    protected final Map<String, Integer> _termIdMap = new HashMap<String, Integer>();

    /**
     * TermId to Term map (array index is TermID)
     */
    protected final  List<String> _terms = new ArrayList<String>();

    /**
     * Variant to VariantId map
     */
    protected final Map<String, Integer> _variantIdMap = new HashMap<String, Integer>();

    /**
     * VariantId to Variant map (array index is VariantID)
     */
    protected final List<String> _variants = new ArrayList<String>();

    /**
     * Document to DocumentId map
     */
    protected final Map<Document, Integer> _docIdMap = new HashMap<Document, Integer>();

    /**
     * DocumentId to Document map (array index is DocumentID)
     */
    protected final List<Document> _documents = new ArrayList<Document>();


    protected final Map<Integer, Set<Integer>> _term2Docs = new HashMap<Integer, Set<Integer>>();
    protected final Map<Integer, Set<Integer>> _doc2Terms = new HashMap<Integer, Set<Integer>>();
    protected final Map<Integer, Set<Integer>> _term2Variants = new HashMap<Integer, Set<Integer>>();
    protected final Map<Integer, Integer> _variant2term = new HashMap<Integer, Integer>();

    protected GlobalIndexMem() {
    }

    public Map<String, Integer> getTermIdMap() {
        return _termIdMap;
    }

    public Map<String, Integer> getVariantIdMap() {
        return _variantIdMap;
    }

    public Map<Document, Integer> getDocIdMap() {
        return _docIdMap;
    }

    public Map<Integer, Set<Integer>> getTerm2Docs() {
        return _term2Docs;
    }

    public Map<Integer, Set<Integer>> getDoc2Terms() {
        return _doc2Terms;
    }

    public Map<Integer, Set<Integer>> getTerm2Variants() {
        return _term2Variants;
    }

    public Map<Integer, Integer> getVariant2Term() {
        return _variant2term;
    }

    /**
     * Given a candidate term's canonical form, return its id. If the term has not been indexed, it will be indexed
     * with a new id and that id will be returned
     *
     * @param term
     * @return the id
     */
    protected int indexTermCanonical(String term) {
        Integer index = _termIdMap.get(term);
        //if (index == null) _termIdMap.put(term, index = _termCounter++);
        if (index == null) {
            _termIdMap.put(term, index = _termCounter++);
            _terms.add(term);
        }
        return index;
    }

    /**
     * Given a candidate term's canonical form, return its id. (If the term has not been index, -1 will be returned
     *
     * @param term
     * @return the id
     */
    public int retrieveTermCanonical(String term) {
        Integer index = _termIdMap.get(term);
        if (index == null) return -1;
        return index;
    }

    /**
     * Given an id, retrieve the candidate term's canonical form
     *
     * @param id
     * @return
     */
    public String retrieveTermCanonical(int id) {
        /*for (Map.Entry<String, Integer> en : _termIdMap.entrySet()) {
            if (en.getValue() == id) return en.getKey();
        }
        return null;*/

        return _terms.get(id);
    }

    /**
     * @return all indexed candidate term canonical form ids
     */
    public Set<Integer> getTermCanonicalIds() {
        return new HashSet<Integer>(_termIdMap.values());
    }

    /**
     * @return all candidate term canonical forms
     */
    public Set<String> getTermsCanonical() {
        return _termIdMap.keySet();
    }

    /**
     * Given a candidate term variant, index it and return its id.
     *
     * @param termV
     * @return the id
     */
    protected int indexTermVariant(String termV) {
        Integer index = _variantIdMap.get(termV);

        //if (index == null) _variantIdMap.put(termV, index = _variantCounter++);
        if (index == null) {
            index = _variantCounter++;
            _variantIdMap.put(termV, index);
            _variants.add(termV);
        }
        return index;
    }

    /**
     * Given an id of a candidate term variant, retrieve the text
     *
     * @param id
     * @return
     */
    protected String retrieveTermVariant(int id) {
        /*for (Map.Entry<String, Integer> en : _variantIdMap.entrySet()) {
            if (en.getValue() == id) return en.getKey();
        }
        return null;*/
        return _variants.get(id);
    }

    public Set<Integer> getTermVariantIds() {
        return new HashSet<Integer>(_variantIdMap.values());
    }

    public Set<String> getTermVariants() {
        return _variantIdMap.keySet();
    }

    /**
     * Given a document, return its id. If the document has not been indexed, index it and return the new id
     *
     * @param d
     * @return the id
     */
    protected int indexDocument(Document d) {
        Integer index = _docIdMap.get(d);
        //if (index == null) _docIdMap.put(d, index = _docCounter++);
        if (index == null) {
            _docIdMap.put(d, index = _docCounter++);
            _documents.add(d);
        }
        return index;
    }

    /**
     * Given a document, return its id. If the document has not been indexed, return -1
     *
     * @param d
     * @return the id
     */
    public int retrieveDocument(Document d) {
        Integer index = _docIdMap.get(d);
        if (index == null) return -1;
        return index;
    }

    /**
     * Given a document id return the document
     *
     * @param id
     * @return
     */
    public Document retrieveDocument(int id) {
        //          for (Map.Entry<Document, Integer> en : _docIdMap.entrySet()) {
        //        if (en.getValue() == id) return en.getKey();
        //      }
//        return null;
        return _documents.get(id);
    }

    /**
     * @return all indexed documents
     */
    public Set<Document> getDocuments() {
        return _docIdMap.keySet();
    }

    /**
     * @return return all indexed document ids
     */
    public Set<Integer> getDocumentIds() {
        return new HashSet<Integer>(_docIdMap.values());
    }

    /*
     term - - variant
      */

    /**
     * Given a map containing [term canonical form - term variant forms], index the mapping, plus the mapping from
     * term variant to term canonical
     *
     * @param map
     */
    protected void indexTermWithVariant(Map<String, Set<String>> map) {
        for (Map.Entry<String, Set<String>> e : map.entrySet()) {
            int termid = indexTermCanonical(e.getKey());

            Set<Integer> variants = _term2Variants.get(termid);
            if (variants == null) variants = new HashSet<Integer>();
            for (String v : e.getValue()) {
                int varid = indexTermVariant(v);
                variants.add(varid);
                _variant2term.put(varid, termid);
            }
            _term2Variants.put(termid, variants);
        }
    }

    /**
     * Given a term canonical form, retrieve its variant forms found in the corpus
     *
     * @param term
     * @return
     */
    public Set<String> retrieveVariantsOfTermCanonical(String term) {
        Set<String> variants = new HashSet<String>();
        Set<Integer> vIds = _term2Variants.get(retrieveTermCanonical(term));
        if (vIds == null) return variants;

        Iterator<Integer> it = vIds.iterator();
        while (it.hasNext())
            variants.add(retrieveTermVariant(it.next()));
        return variants;
    }

    /**
     * Given a term variant form, retrieve its canonical form
     *
     * @param termVar
     * @return
     */
    public int retrieveCanonicalOfTermVariant(String termVar) {
        Integer varId = _variantIdMap.get(termVar);
        if (varId == null)
            return -1;
        Integer termId = _variant2term.get(varId);
        if (termId == null)
            return -1;
        return termId;
    }

    /*
         term - - docs
          */

    /**
     * Given a candidate term's canonical form t found in document d, index the binary relation "t found_in d"
     *
     * @param t
     * @param d
     */
    protected void indexTermCanonicalInDoc(String t, Document d) {
        indexTermCanonicalInDoc(indexTermCanonical(t), indexDocument(d));
    }

    /**
     * Given a candidate term's canonical form id t found in document with id d, index the binary relation "t found_in d"
     *
     * @param t
     * @param d
     */
    protected void indexTermCanonicalInDoc(int t, int d) {
        Set<Integer> docs = _term2Docs.get(t);
        if (docs == null) docs = new HashSet<Integer>();
        docs.add(d);
        _term2Docs.put(t, docs);
    }

    /**
     * @param t the candidate term's canonical form in question
     * @return the document ids of which documents contain the candidate term t
     */
    public Set<Integer> retrieveDocIdsContainingTermCanonical(String t) {
        return retrieveDocIdsContainingTermCanonical(indexTermCanonical(t));
    }

    /**
     * @param id the candidate term's canonical form in questoin
     * @return the document ids of which documents contain the candidate term t
     */
    public Set<Integer> retrieveDocIdsContainingTermCanonical(int id) {
        Set<Integer> res = _term2Docs.get(id);
        return res == null ? new HashSet<Integer>() : res;
    }

    /**
     * @param t the candidate term's canonical form in question
     * @return the documents which contain the candidate term t
     */
    public Set<Document> retrieveDocsContainingTermCanonical(String t) {
        return retrieveDocsContainingTermCanonical(indexTermCanonical(t));
    }

    /**
     * @param t the candidate term's canonical form id in question
     * @return the documents which contain the candidate term t
     */
    public Set<Document> retrieveDocsContainingTermCanonical(int t) {
        Set<Document> docs = new HashSet<Document>();
        Set<Integer> docids = retrieveDocIdsContainingTermCanonical(t);
        Iterator<Integer> it = docids.iterator();
        while (it.hasNext()) docs.add(retrieveDocument(it.next()));
        return docs;
    }

    /**
     * @param t the candidate term's canonical form
     * @return number of documents that contain the candidate term (any variants)
     */
    public int sizeTermInDocs(String t) {
        return retrieveDocIdsContainingTermCanonical(t).size();
    }

    /**
     * @param t the id of candidate term's canonical form
     * @return number of documents that contain the candidate term (any variants)
     */
    public int sizeTermInDocs(int t) {
        return retrieveDocIdsContainingTermCanonical(t).size();
    }

    /*
         doc - - terms
          */

    /**
     * Given a document d which contains a set of terms (canonical form), index the binary relation "document contains term canonical"
     *
     * @param d
     * @param terms canonical forms of candidate terms found in document d
     */
    protected void indexDocWithTermsCanonical(Document d, Set<String> terms) {
        Set<Integer> termids = new HashSet<Integer>();
        for (String t : terms) termids.add(indexTermCanonical(t));
        indexDocWithTermsCanonical(indexDocument(d), termids);
    }

    /**
     * Given a document with id d which contains a set of terms (canonical form), index the binary relation "document contains term canonical"
     *
     * @param d     id of document
     * @param terms canonical forms of candidate terms found in document d
     */
    protected void indexDocWithTermsCanonical(int d, Set<Integer> terms) {
        Set<Integer> termids = _doc2Terms.get(d);
        if (termids != null) termids.addAll(terms);
        _doc2Terms.put(d, terms);
    }

    /**
     * @param d
     * @return the ids of canonical forms of terms found in the document d
     */
    public Set<Integer> retrieveTermCanonicalIdsInDoc(Document d) {
        return retrieveTermCanonicalIdsInDoc(indexDocument(d));
    }

    /**
     * @param d
     * @return the ids of canonical forms of terms found in the document d
     */
    public Set<Integer> retrieveTermCanonicalIdsInDoc(int d) {
        Set<Integer> res = _doc2Terms.get(d);
        return res == null ? new HashSet<Integer>() : res;
    }

    /**
     * @param d
     * @return the canonical form of terms found in the document d
     */
    public Set<String> retrieveTermsCanonicalInDoc(Document d) {
        return retrieveTermCanonicalInDoc(indexDocument(d));
    }

    /**
     * @param d
     * @return the canonical form of terms found in the document d
     */
    public Set<String> retrieveTermCanonicalInDoc(int d) {
        Set<String> res = new HashSet<String>();
        Set<Integer> termids = retrieveTermCanonicalIdsInDoc(d);
        Iterator<Integer> it = termids.iterator();
        while (it.hasNext()) {
            res.add(retrieveTermCanonical(it.next()));
        }
        return res;
    }

    /**
     * @param d
     * @return number of unique candidate terms (canonical form) found in document d
     */
    public int sizeDocHasTerms(Document d) {
        return retrieveTermsCanonicalInDoc(d).size();
    }

    /**
     * @param d
     * @return number of unique candidate terms (canonical form) found in document d
     */
    public int sizeDocHasTerms(int d) {
        return retrieveTermCanonicalInDoc(d).size();
    }
    
  /*  public void updateVariantsOfTermCanonical(String termCanonical, Set<String> variants_term_additional){
    	Set<String> variants = retrieveVariantsOfTermCanonical(termCanonical);
    	if(variants_term_additional.size()>0){
    		variants.addAll(variants_term_additional);
    	}
    	
    }
    */
    
    public void updateIndexTermWithVariant(Map<String, Set<String>> additional_variants_map) {
        for (Map.Entry<String, Set<String>> e : additional_variants_map.entrySet()) {
            int termid = indexTermCanonical(e.getKey());

            Set<Integer> variants = _term2Variants.get(termid);
            
            for (String v : e.getValue()) {
                int varid = indexTermVariant(v);
                variants.add(varid);
                _variant2term.put(varid, termid);
            }
            _term2Variants.put(termid, variants);
        }
    }



}
 public abstract class CandidateTermExtractor {
	protected IStopList _stoplist;
	protected Normalizer _normaliser;

	/**
	 * @param c corpus
	 * @return a map containing mappings from term canonical form to its variants found in the corpus
	 * @throws uk.ac.shef.dcs.oak.jate.JATEException
	 */
	public abstract Map<String, Set<String>> extract(Corpus c) throws JATEException;

	/**
	 * @param d document
	 * @return a map containing mappings from term canonical form to its variants found in the document
	 * @throws uk.ac.shef.dcs.oak.jate.JATEException
	 */
	public abstract Map<String, Set<String>> extract(Document d) throws JATEException;

	/**
	 * @param content a string
	 * @return a map containing mappings from term canonical form to its variants found in the string
	 * @throws uk.ac.shef.dcs.oak.jate.JATEException
	 */
	public abstract Map<String, Set<String>> extract(String content) throws JATEException;

    /**
     *
     * @param string a string
     * @return true if the string contains letter; false otherwise
     */
	public static boolean containsLetter(String string) {
		char[] chars = string.toCharArray();
		for (char c : chars) {
			if (Character.isLetter(c)) return true;
		}
		return false;
	}

    /**
     *
     * @param string
     * @return true if the string contains digit; false otherwise
     */
	public static boolean containsDigit(String string) {
		for (char c : string.toCharArray()) {
			if (Character.isDigit(c)) return true;
		}
		return false;
	}

    /**
     * Replaces [^a-zA-Z\-] characters with " "(white space). Used to normalize texts and candidate terms before counting
     * frequencies
     * @param string input string to be processed
     * @param pattern regular expression pattern defining characters to be replaced by " " (white space)
     * @return
     */
    public static String applyCharacterReplacement(String string, String pattern) {
        return string.replaceAll(pattern, " ").replaceAll("\\s+", " ").trim();
    }

    /**
     * If the input string contains "and", "or"  and "," , it is assumed to contain multiple candidate terms, and is split.
     * This method is used to process extracted candidate terms. Due to imperfections of NLP tools, some times candidate
     * terms can be noisy and need to be further processed/normalized.
     *
     * @param string input string
     * @return individual strings seperated by "and", "or" and ","
     */
    
    //Code modification begins : the second and third 'if' changed to 'else if'
    
    public static String[] applySplitList(String string) {
        StringBuilder sb = new StringBuilder();
        if (string.indexOf(" and ") != -1) {
            String[] parts = string.split("\\band\\b");
            for (String s : parts) sb.append(s.trim()).append("|");
        }
        else if (string.indexOf(" or ") != -1) {
            String[] parts = string.split("\\bor\\b");
            for (String s : parts) sb.append(s.trim()).append("|");
        }
        else if (string.indexOf(",") != -1) {
            if (!containsDigit(string)) {
                String[] parts = string.split("\\,");
                for (String s : parts) sb.append(s.trim()).append("|");
            }
        } else {
            sb.append(string);
        }
        String v = sb.toString();
        if (v.endsWith("|")) v = v.substring(0, v.lastIndexOf("|"));
        return v.split("\\|");
    }
    
    //code modification ends


    /**
     * If a string beings or ends with a stop word (e.g., "the"), the stop word is removed.
     *
     * This method is used to further process/normalize extracted candidate terms. Due to imperfections of NLP tools,
     * sometimes candidate terms can be noisy and needs further normalization.
     *
     * @param string
     * @param stop
     * @return null if the string is a stopword; otherwise the normalized string
     */
    public static String applyTrimStopwords(String string, IStopList stop, Normalizer normalizer) {
        //check the entire string first (e.g., "e. g. " and "i. e. " which will fail the following checks
       // String s = normalizer.normalize(string);
    	
    	if(stop.isStopWord(normalizer.normalize(string).replaceAll("\\s+","").trim()))
            return null;

        String[] e = string.split("\\s+");
        if (e == null || e.length < 1) return string;

        int head = e.length;
        int end = -1;
        for (int i = 0; i < e.length; i++) {
            if (!stop.isStopWord(e[i])) {
                head = i;
                break;
            }
        }

        for (int i = e.length - 1; i > -1; i--) {
            if (!stop.isStopWord(e[i])) {
                end = i;
                break;
            }
        }

        if (head <= end) {
            String trimmed = "";
            for (int i = head; i <= end; i++) {
                trimmed += e[i] + " ";
            }
            return trimmed.trim();
        }
        return null;
    }

    /**
     * This method is used to check if candidate term is a possible noisy symbol or non-term.
     *
     * @param string
     * @return true if the string contains at least 2 letters or digits; false otherwise
     */
    public static boolean hasReasonableNumChars(String string) {
        int len = string.length();
        if (len < 2) return false;
        if (len < 5) {
            char[] chars = string.toCharArray();
            int num = 0;
            for (int i = 0; i < chars.length; i++) {
                if (Character.isLetter(chars[i]) || Character.isDigit(chars[i]))
                    num++;
                if (num > 2) break;
            }
            if (num > 2) return true;
            return false;
        }
        return true;
    }
}
 public class CorpusImpl implements Corpus {
	private static final Logger _logger = Logger.getLogger(CorpusImpl.class);

	private final Set<Document> _docs;

	public CorpusImpl(){
		_docs = new HashSet<Document>();
	}

	public Iterator<Document> iterator(){
		return _docs.iterator();
	}

	public CorpusImpl(String path){
		_docs=new HashSet<Document>();
		File targetFolder = new File(path);
		
		File[] files = targetFolder.listFiles();
		if ( files != null ) {
			for (File f : files) {
				try {
					this.add(new DocumentImpl(f.toURI().toURL()));
				} catch (MalformedURLException e) {
					_logger.error( "Error processing " + f, e );
				}
			}
		}
	}

   public boolean add(Document document){
	   return _docs.add(document);
   }

   public boolean contains(Document document){
	   return _docs.contains(document);
   }

   public int size() {
      return _docs.size();
   }

   public boolean remove(Document doc){
	   return _docs.remove(doc);
   }

}
 public class Lemmatizer extends Normalizer {

	private EngLemmatiser lemmatizer;
	private final Map<String, Integer> tagLookUp = new HashMap<String, Integer>();

	public Lemmatizer() {
		init();
	}

	/**
	 * @param value original word
	 * @param pos the part of speech of the last word
	 * @return the lemma of original word
	 */
	public String getLemma(String value, String pos) {
		int POS = tagLookUp.get(pos);
		if (POS == 0)
			return lemmatizer.lemmatize(value);
		else
			return lemmatizer.lemmatize(value, POS);
	}

	/**
	 * Lemmatise a phrase or word. If a phrase, only lemmatise the most RHS word.
	 * @param value
	 * @return
	 */
	public String normalize(String value) {
		if(value.indexOf(" ")==-1||value.endsWith(" s")||value.endsWith("'s")) //if string is a single word, or it is in "XYZ's" form where the ' char has been removed
			return lemmatizer.lemmatize(value,1).trim();

		String part1 = value.substring(0,value.lastIndexOf(" "));
		String part2 = lemmatizer.lemmatize(value.substring(value.lastIndexOf(" ")+1),1);
		return part1+" "+part2.trim();

	}

	/**
	 * Lemmatise every word in the input string
	 * @param in
	 * @return the lemmatised string
	 */
	public String normalizeContent(String in){
		StringBuilder sb = new StringBuilder();
		StringTokenizer tokenizer = new StringTokenizer(in);
		while(tokenizer.hasMoreTokens()){
			String tok=tokenizer.nextToken();
			sb.append(normalize(tok)).append(" ");
		}
		return sb.toString().trim();
	}


	private void init() {
		// Initializes the temporary directory if required.
		final JATEProperties instance = JATEProperties.getInstance();
		String path = instance.getNLPPath();
		if ( path.startsWith( "jar:" ) ) {
			String dir = instance.getWorkPath() + File.separator + "lemmatizer";
			path = dir;
			File directory = new File(dir);
			if ( !directory.exists() ) {
				if ( !directory.mkdirs() ) {
					throw new UnsupportedOperationException("Could not initialize " + directory);
				}
				for ( String file : new String[]{ "adj.exc", "adj.index", "adv.exc", "adv.index", "noun.exc", "stopwordexc.list", "umlserror.list", "verb.exc", "verb.index" } ) {
					try {
						InputStream input = null;
						OutputStream output = null;
						try {
							input = instance.getNLPInputStream( "lemmatizer" + File.separator + file );
							if ( input == null ) {
								throw new IOException( "Unable to read: " + file );
							}
							output = new FileOutputStream( dir + File.separator + file );
							byte [] bytes = new byte[ 4096 ];
							while ( true ) {
								int len = input.read( bytes );
								if ( len == -1 ) {
									break;
								}
								output.write( bytes, 0, len );
							}
						} finally {
							try {
								if ( input != null ) {
									input.close();
								}
							} finally {
								if ( output != null ) {
									output.close();
								}
							}
						}
					} catch ( IOException ioe ) {
						throw new UnsupportedOperationException( "Unable to copy data", ioe );
					}
				}
			}
		}

		lemmatizer = new EngLemmatiser( path, false, true );
		tagLookUp.put("NN", 1);
		tagLookUp.put("NNS", 1);
		tagLookUp.put("NNP", 1);
		tagLookUp.put("NNPS", 1);
		tagLookUp.put("VB", 2);
		tagLookUp.put("VBG", 2);
		tagLookUp.put("VBD", 2);
		tagLookUp.put("VBN", 2);
		tagLookUp.put("VBP", 2);
		tagLookUp.put("VBZ", 2);
		tagLookUp.put("JJ", 3);
		tagLookUp.put("JJR", 3);
		tagLookUp.put("JJS", 3);
		tagLookUp.put("RB", 4);
		tagLookUp.put("RBR", 4);
		tagLookUp.put("RBS", 4);
	}

}

import org.apache.log4j.Logger;
import uk.ac.shef.dcs.oak.jate.JATEException;
import uk.ac.shef.dcs.oak.jate.JATEProperties;
import uk.ac.shef.dcs.oak.jate.core.algorithm.AbstractFeatureWrapper;
import uk.ac.shef.dcs.oak.jate.core.algorithm.Algorithm;
import uk.ac.shef.dcs.oak.jate.core.algorithm.CValueAlgorithm;
import uk.ac.shef.dcs.oak.jate.core.algorithm.CValueFeatureWrapper;
import uk.ac.shef.dcs.oak.jate.core.feature.FeatureBuilderCorpusTermFrequency;
import uk.ac.shef.dcs.oak.jate.core.feature.FeatureBuilderTermNest;
import uk.ac.shef.dcs.oak.jate.core.feature.FeatureCorpusTermFrequency;
import uk.ac.shef.dcs.oak.jate.core.feature.FeatureTermNest;
import uk.ac.shef.dcs.oak.jate.core.feature.TermVariantsUpdater;
import uk.ac.shef.dcs.oak.jate.core.feature.indexer.GlobalIndexBuilderMem;
import uk.ac.shef.dcs.oak.jate.core.npextractor.NounPhraseExtractorOpenNLP;
import uk.ac.shef.dcs.oak.jate.io.ResultWriter2File;
import uk.ac.shef.dcs.oak.jate.model.Term;
import uk.ac.shef.dcs.oak.jate.util.control.IStopList;
import uk.ac.shef.dcs.oak.jate.util.control.StopList;
import uk.ac.shef.dcs.oak.jate.util.counter.TermFreqCounter;
import uk.ac.shef.dcs.oak.jate.util.counter.WordCounter;
import uk.ac.shef.dcs.oak.jate.core.feature.indexer.GlobalIndexMem;
import uk.ac.shef.dcs.oak.jate.core.npextractor.CandidateTermExtractor;
import uk.ac.shef.dcs.oak.jate.model.CorpusImpl;
import uk.ac.shef.dcs.oak.jate.util.control.Lemmatizer;
import java.io.IOException;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;

package uk.ac.shef.dcs.oak.jate.test;




public class TestCValue {
	private static final Logger _logger = Logger.getLogger(AlgorithmTester.class);

	private final Map<Algorithm, AbstractFeatureWrapper> _algregistry = new HashMap<Algorithm, AbstractFeatureWrapper>();

	public void registerAlgorithm(Algorithm a, AbstractFeatureWrapper f) {
		_algregistry.put(a, f);
	}

	public void execute(GlobalIndexMem index) throws JATEException, IOException {
		_logger.info("Initializing outputter, loading NP mappings...");
		ResultWriter2File writer = new ResultWriter2File(index);
		if (_algregistry.size() == 0) throw new JATEException("No algorithm registered!");
		_logger.info("Running NP recognition...");

		/*.extractNP(c);*/
		for (Map.Entry<Algorithm, AbstractFeatureWrapper> en : _algregistry.entrySet()) {
			_logger.info("Running feature store builder and ATR..." + en.getKey().toString());
			Term[] result = en.getKey().execute(en.getValue());
			writer.output(result, en.getKey().toString() + ".txt");
		}
	}

	public static void main(String[] args) throws IOException, JATEException{
		String[] argument = new String[]{JATEProperties.getInstance().getCorpusPath(), JATEProperties.getInstance().getResultPath()};
		TestCValue.CValueTester(argument);
	}
	
	public static AlgorithmTester CValueTester(String[] args) throws IOException, JATEException {
		
		
		String path_to_corpus= args[0];
		String path_to_output=args[1];
		if (args.length < 2) {
			System.out.println("Usage: java TestCValue [path_to_corpus] [output_folder]");
		}
		else
		{
			System.out.println("Started "+ TestCValue.class+"at: " + new Date() + "... For detailed progress see log file jate.log.");

			//creates instances of required processors and resources

			//stop word list
			IStopList stop = new StopList(true);

			//lemmatizer
			Lemmatizer lemmatizer = new Lemmatizer();

			//noun phrase extractor
			CandidateTermExtractor npextractor = new NounPhraseExtractorOpenNLP(stop, lemmatizer);

			//counters
			TermFreqCounter npcounter = new TermFreqCounter();
			WordCounter wordcounter = new WordCounter();

			//create global resource index builder, which indexes global resources, such as documents and terms and their
			//relations
			GlobalIndexBuilderMem builder = new GlobalIndexBuilderMem();
			//build the global resource index
			GlobalIndexMem termDocIndex = builder.build(new CorpusImpl(path_to_corpus), npextractor);

			/*newly added for improving frequency count calculation: begins*/
			
			
			TermVariantsUpdater update = new TermVariantsUpdater(termDocIndex);
			 
			
			GlobalIndexMem termIndex = update.updateVariants();
			
			FeatureCorpusTermFrequency termCorpusFreq =

							new FeatureBuilderCorpusTermFrequency(npcounter, wordcounter, lemmatizer).build(termIndex);
