public class MySqlDB
{
	//String url, usr, passwd;
	Connection con=null;
	Statement st=null;

	public MySqlDB(String url, String user, String passwd) {
		try {
			Class.forName("com.mysql.jdbc.Driver").newInstance();
		}
		catch (Exception E) {
			System.err.println("Unable to load driver.");
			E.printStackTrace();
			System.exit(1);
		}
		try {
			Connection con2 = DriverManager.getConnection(url, user, passwd);
			//System.out.println(url+" "+user+" "+passwd);
			con = con2;
		}
		catch (SQLException E) {
			System.out.println("SQLException: " + E.getMessage());
			System.out.println("SQLState:     " + E.getSQLState());
			System.out.println("VendorError:  " + E.getErrorCode());
			E.printStackTrace();
			con=null;
			System.exit(1);
		}



	}

	public ResultSet executeQuery(String query) throws java.lang.Exception {
		return con.createStatement().executeQuery(query);
	}

	public DatabaseMetaData getMetaData() throws java.lang.Exception {
		return con.getMetaData();
	}

	public int executeUpdate(String query) throws java.lang.Exception {
		return con.createStatement().executeUpdate(query);
	}

	public PreparedStatement prepareStatement(String query)
		throws java.lang.Exception {
			return con.prepareStatement(query);
		}

	public void close() throws Exception {
		if(con != null)
			con.close();
	}
}  public class AccuracyMeasure {

	/**
	 *  Calculates precision at the recall levels 0.1, 0.2, ... ,.09 , 1 and
	 *  returns all these precision values in a ordered array where first element 
	 *  is the precision at the greatest recall-level less than than 0.1; second 
	 *  element is the precision at the greatest recall-level >= 0.1 and < 0.2 and so on .
	 * 
	 * @param orderedList 
	 * @param expectedRecords 
	 * @return double[] 
	 * 
	 */
	public static double[] elevenPointPreciosn(int[] orderedList, int expectedRecords){
		double[] precisionArray = {0,0,0,0,0,0,0,0,0,0,0};
		double[] allOnes = {1,1,1,1,1,1,1,1,1,1,1};
		if(expectedRecords <= 0){
			return allOnes;
		}else if(orderedList == null){
			return precisionArray;
		}else if(orderedList.length == 0){
			return precisionArray;
		}else{
			double currentPrecisionValue = 0.0;
			int lastUpdatedIndex = -1;
			int indexToUpdate = 0;
			int relevantRecordsTillNow = 0;
			int recordsTillNow = 0;
			for(int i: orderedList){
				recordsTillNow++;
				if(i==1){
					relevantRecordsTillNow++;
					indexToUpdate = (int) (relevantRecordsTillNow*10.0)/(expectedRecords);
					currentPrecisionValue = (relevantRecordsTillNow*1.0)/recordsTillNow;
					for(int k=lastUpdatedIndex+1; k< indexToUpdate; k++){
						precisionArray[k] = currentPrecisionValue;
					}
					precisionArray[indexToUpdate] = currentPrecisionValue;
					lastUpdatedIndex = indexToUpdate;
				}
			}
			return precisionArray;
		}
	}
	
	/**
	 * 
	 *  Calculates precision at the recall levels 0.1, 0.2, ... ,.09 , 1 and
	 *  returns all these precision values in a ordered array where first element 
	 *  is the precision at recall-level equal or greater than 0.1 and so on.
	 *  It just finds the expected number of relevant records (expectedRecords) 
	 *  and calls the function elevenPointPreciosn(orderedList, expectedRecords).
	 *  
	 * @param orderedList  
	 * @return double[] 
	 * 
	 */
	public static double[] elevenPointPreciosn(int[] orderedList){
		double[] precisionArray = {1,1,1,1,1,1,1,1,1,1,1};
		if(orderedList == null){
			return precisionArray;
		}else if(orderedList.length == 0){
			return precisionArray;
		}else{
			int expectedRecords = 0;
			for(int i: orderedList){
				if(i==1){
					expectedRecords++;
				}
			}
			return elevenPointPreciosn(orderedList, expectedRecords);
		}
	}

	/**
	 *  To find the precision at the positionK. If the argument positionK is 0, it returns 0.
	 *  if the list is empty, it returns 0. positionK = 1 means the first element of the orderedList.
	 * 
	 * @param orderedList
	 * @param positionK
	 * @return
	 */
	public static double precisionAtTopK(int[] orderedList, int positionK){
		double precision=0.0;
		int relevantRecords = 0;
		if((orderedList != null) && (orderedList.length > 0) && (positionK > 0)){
			int finalPosition = Math.min(positionK, orderedList.length);
			for(int i=0;i < finalPosition; i++){
				if(orderedList[i] == 1){
					relevantRecords++;
				}
			}
			return (relevantRecords*1.0)/positionK;
		}else{
			return precision;
		}
	}
	/**
	 * 
	 *  Calculates precision at each point a relevant record is found and then 
	 *  takes an average over all the precision values and return the average.
	 *  expectedRecord argument is just required for efficiency so that it can 
	 *  stop iterating over the array once it has found expectedRecords relevant
	 *  records.
	 *  
	 * @param orderedList 
	 * @param expectedRecords 
	 * @return double 
	 * 
	 */	
	public static double meanAveragePrecision(int[] orderedList, int expectedRecords){
		if(orderedList == null){
			return 0.0;
		}		
		double sumPrecision = 0.0;
		int relevantRecords = 0;
		int records=0;
		for(int i:orderedList){
			records++;
			if(i==1){
				relevantRecords++;
				sumPrecision += ( relevantRecords*1.0 )/records;
				if(relevantRecords >= expectedRecords){
					break;
				}
			}
		}
		if(relevantRecords > 0){
			return sumPrecision/expectedRecords;
		}else{
			return 0.0;
		}
	}

	/**
	 * 
	 *  Calculates precision at each point a relevant record is found and then 
	 *  takes an average over all the precision vlaues and return the average.
	 *  It just calls the function 
	 *  meanAveragePrecision(orderedList, orderedList.length).
	 *  
	 *  
	 * @param orderedList 
	 * @param expectedRecords 
	 * @return double 
	 * @see meanAveragePrecision(int[] orderedList)
	 * 
	 */		
	/* fix this to find exppectedRecords
	 * public static double meanAveragePrecision(int[] orderedList){
		if(orderedList == null){
			return 0.0;
		}
		return meanAveragePrecision(orderedList, orderedList.length);
	}
	*/
	/**
	 *  Finds the position (k) of the first occurrence of a relevant record 
	 *  and return (1/k) i.e. the reciprocal rank. if no relevant record 
	 *  occur then it returns 0.
	 * 
	 * @param orderedList
	 * @return double
	 * 
	 */
	public static double reciprocalRank(int[] orderedList){
		if(orderedList == null){
			return 0.0;
		}
		double rank = 0.0;
		int position =0;
		boolean relevantRecordFound=false;
		for(int i: orderedList){
			position++;
			if(i==1){
				relevantRecordFound=true;
				break;
			}
		}
		if(relevantRecordFound){
			rank = 1.0/position;
		}
		return rank;
	}
	
	/**
	 * 
	 * To print the integer array.
	 * 
	 * @param list
	 * 
	 */
	public static void printList(int[] list){
		System.out.print("[");
		if(list != null){
			for(int i: list){
				System.out.print(i+", ");
			}
		}
		System.out.println("]");
	}
	
	/**
	 * 
	 * To print the double array.
	 * 
	 * @param list
	 * 
	 */
	public static void printList(double[] list){
		System.out.print("[");
		if(list != null){
			for(double i: list){
				System.out.print(i+", ");
			}
		}
		System.out.println("]");
	}
	
	/*
	 *  Just to check the implementation of the above functions.
	*/ 
	public static void main(String[] args){
		int myList1[] = {1,0,1,0,1,1,1,1,1,0,1,1,1};
		int[] myList2 = {0,0,1,0};
		double rank=0.0, map=0.0, precitionAtTopK=0.0;
		double[] elevenPtPrecision = null;
		System.out.print("Ordered List:  ");
		AccuracyMeasure.printList(myList1);
		rank = AccuracyMeasure.reciprocalRank(myList1);
		System.out.println("Reciprocal Rank: "+rank);
		//map = AccuracyMeasure.meanAveragePrecision(myList1);
		System.out.println("MAP: "+map);
		elevenPtPrecision = AccuracyMeasure.elevenPointPreciosn(myList1);
		AccuracyMeasure.printList(elevenPtPrecision);
		for(int j=0; j< myList1.length; j++){
			precitionAtTopK = AccuracyMeasure.precisionAtTopK(myList1, j+1);
			System.out.println("Precision at Top "+(j+1)+" : "+precitionAtTopK);
		}
		
		System.out.println();
		
		System.out.print("Ordered List:  ");
		AccuracyMeasure.printList(myList2);
		rank = AccuracyMeasure.reciprocalRank(myList2);
		System.out.println("Reciprocal Rank: "+rank);
		//map = AccuracyMeasure.meanAveragePrecision(myList2);
		System.out.println("MAP: "+map);
		elevenPtPrecision = AccuracyMeasure.elevenPointPreciosn(myList2);
		AccuracyMeasure.printList(elevenPtPrecision);
	}
	
} public class BM25 extends Preprocess {

	static double k1 = 1.5, b = 0.675, k3 = 8;

	// Tokenizer based on whitespace; evry token is a word
	public HashMap<String, Double> getTF(String str) {
	//	System.out.println("BM25-getTF::  ["+str+"] "+str.length());
		HashMap<String, Double> tokenTF = getTokenTFMultiple(str);
	//	System.out.println("BM25-getTF:: "+tokenTF+"\n"+ tokenTF.size() + " sum_tf:"+ getTotalTokens(tokenTF));
		return tokenTF;
	}

	public void convertDFtoIDF(HashMap<String, Double> qgramIDF, int size) {
		for (String qgram : qgramIDF.keySet()) {
			qgramIDF.put(qgram, Math.log((size - qgramIDF.get(qgram) + 0.5) / (qgramIDF.get(qgram) + 0.5)));
		}
	}

	public HashMap<String, Double> getWeights(HashMap<String, Double> tokenTF, HashMap<String, Double> qgramIDF,
			double averageLength) {
		double bm25Weight, recordLength = 0, normFactor;
		recordLength = getTotalTokens(tokenTF);
		normFactor = k1 * ((1 - b) + b * recordLength / averageLength);
		for (String qgram : tokenTF.keySet()) {
			if (qgramIDF.containsKey(qgram)) {
				bm25Weight = tokenTF.get(qgram);
				bm25Weight = (1 + k1) * bm25Weight / (normFactor + bm25Weight);
				tokenTF.put(qgram, bm25Weight * qgramIDF.get(qgram));
			}
		}
		return tokenTF;
	}

	public HashMap<String, Double> getQueryWeights(String str, HashMap<String, Double> qgramIDF) {
		HashMap<String, Double> tokenTF = getTF(str);
		double bm25Weight;
		for (String qgram : tokenTF.keySet()) {
			bm25Weight = tokenTF.get(qgram);
			bm25Weight = (1 + k3) * bm25Weight / (k3 + bm25Weight);
			tokenTF.put(qgram, bm25Weight);
		}
		return tokenTF;
	}
} public class CondBM25 extends BM25 {

	//static double k1 = 1.5, b = 0.675, k3 = 8;

	// Tokenizer based on whitespace; evry token is a word
	public HashMap<String, Double> getTF(String str) {
		return getConditionalTF(str);
	}

	public void convertDFtoIDF(HashMap<String, Double> qgramIDF, int size) {
		//We will convert the DF to conditional IDF
		/*   t_0 = size
		 *   idf( t_n ) =  log(  [ df(t_{n-1}) + df(t_n) + 0.5 ]/ [df(t_n) + 0.5 ])
		 */
		HashMap<String, Double> qgramFrequency = new HashMap<String, Double>();
		qgramFrequency.putAll(qgramIDF);
		
		for (String qgram : qgramIDF.keySet()) {
			if(qgram.endsWith("_1")){
				qgramIDF.put(qgram, Math.log((size - qgramFrequency.get(qgram) + 0.5) / (qgramFrequency.get(qgram) + 0.5)));
			}else{
				int j=2;
				while(! qgram.endsWith("_"+j)){
					j++;
				}
				String[] tokens = qgram.split("_");
				qgramIDF.put(qgram, Math.log(( qgramFrequency.get(tokens[0]+"_"+(j-1)) - qgramFrequency.get(qgram) + 0.5) / (qgramFrequency.get(qgram) + 0.5)));
			}
		}
		qgramFrequency.clear();
	}

} public class CondHMM extends HMM {

	public HashMap<String, Double> getTF(String str) {
		return getConditionalTF(str);
	}

	public void convertDFtoIDF(HashMap<String, Double> qgramIDF, int size, Vector<HashMap<String, Double>> recordTokenWeights) {
		//We will convert the DF to conditional IDF
		/*   t_0 = size
		 *   idf( t_n ) =  log( 1 + df(t_{n-1}) / df(t_n))
		 */
		double numTokens = 0;
		for (String qgram : qgramIDF.keySet()) {
			numTokens += qgramIDF.get(qgram);
		}
		HashMap<String, Double> qgramFrequency = new HashMap<String, Double>();
		qgramFrequency.putAll(qgramIDF);
		
		for (String qgram : qgramIDF.keySet()) {
			if(qgram.endsWith("_1")){
				qgramIDF.put(qgram, qgramFrequency.get(qgram)/numTokens);
			}else{
				int j=2;
				while(! qgram.endsWith("_"+j)){
					j++;
				}
				String[] tokens = qgram.split("_");
				String baseToken = tokens[0]+"_"+(j-1);
				double countTokenWithBaseToken = 0;
				for(int k=0; k < recordTokenWeights.size(); k++){
					HashMap<String, Double> tokenTF = recordTokenWeights.get(k);
					if(tokenTF.containsKey(baseToken)){
						countTokenWithBaseToken += getTotalTokens(tokenTF);
					}
				}
				qgramIDF.put(qgram, qgramFrequency.get(qgram) / countTokenWithBaseToken);
			}
		}
	}
	
	
	
	
} public class CondIdf extends TfIdf {

	// Tokenizer based on conditional occurance; evry token is a word
	public HashMap<String, Double> getTF(String str) {
		return getConditionalTF(str);
	}

	public void convertDFtoIDF(HashMap<String, Double> qgramIDF, int size) {
		//We will convert the DF to conditional IDF
		/*   t_0 = size
		 *   idf( t_n ) =  log( 1 + df(t_{n-1}) / df(t_n))
		 */
		HashMap<String, Double> qgramFrequency = new HashMap<String, Double>();
		qgramFrequency.putAll(qgramIDF);
		
		for (String qgram : qgramIDF.keySet()) {
			if(qgram.endsWith("_1")){
				qgramIDF.put(qgram, Math.log(1 + size / qgramFrequency.get(qgram)));
			}else{
				int j=2;
				while(! qgram.endsWith("_"+j)){
					j++;
				}
				String[] tokens = qgram.split("_");
				qgramIDF.put(qgram, Math.log(1 + qgramFrequency.get(tokens[0]+"_"+(j-1))/ qgramFrequency.get(qgram)));
			}
		}
	}
} public class DistinctBM25 extends BM25 {

	//static double k1 = 1.5, b = 0.675, k3 = 8;

	// Tokenizer based on whitespace; evry token is a word
	public HashMap<String, Double> getTF(String str) {
		return getConditionalTF(str);
	}

} public class DistinctHMM extends HMM {

	public HashMap<String, Double> getTF(String str) {
		return getConditionalTF(str);
	}

} public class DistinctTfIdf extends TfIdf {

	// Tokenizer based on conditional occurance; evry token is a word
	public HashMap<String, Double> getTF(String str) {
		return getConditionalTF(str);
	}

} public class HMM extends Preprocess {

	private static double a0 = 0.8;

	private static double a1 = 0.2;

	public HashMap<String, Double> getTF(String str) {
		return getTokenTFMultiple(str);
	}

	public void incrementDFCount(String qgram, HashMap<String, Double> tokenCount, double qgramTF) {
		if (tokenCount.containsKey(qgram)) {
			tokenCount.put(qgram, tokenCount.get(qgram) + qgramTF);
		} else {
			tokenCount.put(qgram, qgramTF);
		}
	}

	public void convertDFtoIDF(HashMap<String, Double> qgramIDF, int size) {
		double numTokens = 0;
		for (String qgram : qgramIDF.keySet()) {
			numTokens += qgramIDF.get(qgram);
		}
		for (String qgram : qgramIDF.keySet()) {
			qgramIDF.put(qgram, qgramIDF.get(qgram) / numTokens);
		}
	}

	public HashMap<String, Double> getWeights(HashMap<String, Double> tokenTF, HashMap<String, Double> qgramIDF) {
		double sum = 0;
		double weight=0;
		for (String qgram : tokenTF.keySet()) {
			sum += tokenTF.get(qgram);
		}
		for (String qgram : tokenTF.keySet()) {
			weight = ( a1 * tokenTF.get(qgram) / sum )/( a0 * qgramIDF.get(qgram));
			tokenTF.put(qgram, Math.log( 1 +  weight ) );
		}
		return tokenTF;
	}

	public HashMap<String, Double> getQueryWeights(String str, HashMap<String, Double> qgramIDF) {
		HashMap<String, Double> tokenTF = getTF(str);
		for (String qgram : tokenTF.keySet()) {
			tokenTF.put(qgram, 1.0);
		}
		return tokenTF;
	}

	public static void main(String[] args) {
		double[] mapArray;
		int maxQueries = 100;
		HMM hmm = new HMM();
		long t1, t4, t2, t3, prepTime = 0, queryTime = 0;
		Vector<String> files = new Vector<String>();
		Vector<HashMap<String, Double>> baseTableTokenWeights = new Vector<HashMap<String, Double>>();
		HashMap<String, Double> qgramIDF = new HashMap<String, Double>();

		t1 = System.currentTimeMillis();
		// Toggle this variable to log the preprocessing info to DB
		Preprocess.setLogToDB(false);
		hmm.preprocessTable(files, qgramIDF, baseTableTokenWeights, args[0]);
		t2 = System.currentTimeMillis();
		prepTime = t2 - t1;
		System.err.println("Preprocessing Done: took " + prepTime + " ms");

		// System.gc();
		// For each query, order the filenames by their jaccard score
		int i = 1;
		String query;
		t1 = System.currentTimeMillis();
		HashMap<Integer, String> queries = SimilarityMatch.testQueries();
		t2 = System.currentTimeMillis();
		System.err.println("Collected Random Queries: took " + (t2 - t1) + " ms");

		
		//If due to some reason we cant get maxQueries qeuries then readjust this number
		maxQueries = queries.size();

		mapArray = new double[maxQueries];
		for (int a = 0; a < maxQueries; a++) {
			mapArray[a] = 0;
		}

		Vector<String> queryVector = new Vector<String>();
		Vector<Integer> queryIdVector = new Vector<Integer>();
		String baseDir = "/home/oktie/dcp-workspace/log/res/" + args[0] + "/";
		File baseDirPath = new File(baseDir);
		baseDirPath.mkdirs();

		for (Integer queryId : queries.keySet()) {
			query = queries.get(queryId);
			HashSet<Integer> actualResult = SimilarityMatch.getAllTidsHavingId(queryId, args[1]);
			queryVector.add(query);
			queryIdVector.add(queryId);

			t3 = System.currentTimeMillis();
			// The tids in the scoreList is 1 less than the actual tid e.g.
			// tid=1 willl be having id=0 in IdScore object
			List<IdScore> scoreList = hmm.getSimilarRecords(query, qgramIDF, baseTableTokenWeights);
			SimilarityMatch.logOutput(baseDir + i + "." + hmm.getClass().getName() + ".txt", query, files, scoreList);
			t4 = System.currentTimeMillis();
			queryTime += (t4 - t3);
			// Compute Accuracy
			int[] booleanList = SimilarityMatch.generateBooleanList(actualResult, scoreList);
			double map = AccuracyMeasure.meanAveragePrecision(booleanList, actualResult.size());
			mapArray[i - 1] = map;
			// System.err.println("Query Time: " + (t4-t1));
			System.err.println("query: " + i + "  metric: " + hmm.getClass().getName() + "      MAP: " + map);

			System.err.println("DONE QUERY: " + i);
			i++;
		}

		System.err.println("\nQuery Time: " + queryTime + " ms     Preprocess Time : " + prepTime + " ms ");
	}

} public class Idf extends TfIdf {

	// Tokenizer based on whitespace; evry token is a word
	public HashMap<String, Double> getTF(String str) {
		return gettokenTFSingle(str);
	}

} public class ModBM25 extends BM25 {

	// Tokenizer based on whitespace; every token is a word
	public HashMap<String, Double> getTF(String str) {
		return gettokenTFSingle(str);
	}

} public class ModHMM extends HMM {

	// Tokenizer based on whitespace; every token is a word
	public HashMap<String, Double> getTF(String str) {
		return gettokenTFSingle(str);
	}

} public class Preprocess {

	public static boolean logToDB = false;
	
	public boolean tokenizeUsingQgrams = true;
	
	public int qgramSize = 2;
	
	public static boolean debugModeOn = true;
	
	public static boolean replaceSpaceWithSpecialCharacter = true; 
	
	public static void setLogToDB(boolean logToDB) {
		Preprocess.logToDB = logToDB;
	}

	public static boolean logToDB() {
		return logToDB;
	}
	
	public boolean tokenizeUsingQgrams(){
		return tokenizeUsingQgrams;
	}
	
	public static String generateSpecialCharacters(int qgSize){
		String str="";
		for(int i=0; i < qgSize-1; i++){
			str += "#";
		}
		//System.out.println(str);
		return str;
	}
	
	public static String extractMetricName(String fullMetricName){
		String str[] = fullMetricName.toLowerCase().split("\\.");
		return str[str.length-1];
	}
	
	public void incrementCount(String qgram, HashMap<String, Double> tokenCount) {
		if (tokenCount.containsKey(qgram)) {
			tokenCount.put(qgram, tokenCount.get(qgram) + 1);
		} else {
			tokenCount.put(qgram, 1.0);
		}
	}

	public void incrementDFCount(String qgram, HashMap<String, Double> tokenCount, double qgramTF) {
		incrementCount(qgram, tokenCount);
	}

	// Tokenization function
	public HashMap<String, Double> getTF(String str) {
		HashMap<String, Double> tokenTF = new HashMap<String, Double>();
		/*
		 * String[] tokens = str.split("\\s+"); for (int i = 0; i <
		 * tokens.length; i++) { incrementCount(tokens[i], tokenTF); }
		 */
		str = str.toLowerCase();
		if(replaceSpaceWithSpecialCharacter){
			str = str.replaceAll(" ", generateSpecialCharacters(qgramSize));
			str = generateSpecialCharacters(qgramSize) + str + generateSpecialCharacters(qgramSize); 
		}
		for (int i = 0; i <= str.length() - qgramSize; i++) {
			String qgram = str.substring(i, i + qgramSize);
			incrementCount(qgram, tokenTF);
		}
		return tokenTF;
	}


	// Tokenizer based on whitespace; every token is a word
	public HashMap<String, Double> gettokenTFSingle(String str) {
		HashMap<String, Double> tokenTF = new HashMap<String, Double>();
		if (!tokenizeUsingQgrams ) {
			String[] tokens = str.toLowerCase().split(" ");
			for (int i = 0; i < tokens.length; i++) {
				// incrementCount(tokens[i], tokenTF);
				tokenTF.put(tokens[i], 1.0);
			}
		} else {
			str = str.toLowerCase();
			if(replaceSpaceWithSpecialCharacter){
				str = str.replaceAll(" ", generateSpecialCharacters(qgramSize));
				str = generateSpecialCharacters(qgramSize) + str + generateSpecialCharacters(qgramSize); 
			}
			for (int i = 0; i <= str.length() - qgramSize; i++) {
				String qgram = str.substring(i, i + qgramSize);
				tokenTF.put(qgram, 1.0);
			}
		}
		return tokenTF;
	}

	

	// Tokenizer based on whitespace; every token is a word
	public HashMap<String, Double> getTFWsign(String str, Long signature) {
		HashMap<String, Double> tokenTF = new HashMap<String, Double>();
		if (!tokenizeUsingQgrams ) {
			String[] tokens = str.toLowerCase().split(" ");
			for (int i = 0; i < tokens.length; i++) {
				// incrementCount(tokens[i], tokenTF);
				tokenTF.put(tokens[i], 1.0);
			}
		} else {
			str = str.toLowerCase();
			if(replaceSpaceWithSpecialCharacter){
				str = str.replaceAll(" ", generateSpecialCharacters(qgramSize));
				str = generateSpecialCharacters(qgramSize) + str + generateSpecialCharacters(qgramSize); 
			}
			for (int i = 0; i <= str.length() - qgramSize; i++) {
				String qgram = str.substring(i, i + qgramSize);
				tokenTF.put(qgram, 1.0);
			}
		}
		return tokenTF;
	}
	
	
	
	// Tokenizer based on whitespace; evry token is a word
	public HashMap<String, Double> getTokenTFMultiple(String str) {
		HashMap<String, Double> tokenTF = new HashMap<String, Double>();
		if (! tokenizeUsingQgrams) {
			String[] tokens = str.toLowerCase().split(" ");
			for (int i = 0; i < tokens.length; i++) {
				incrementCount(tokens[i], tokenTF);
			}
		} else {
			str = str.toLowerCase();
			if(replaceSpaceWithSpecialCharacter){
				str = str.replaceAll(" ", generateSpecialCharacters(qgramSize));
				str = generateSpecialCharacters(qgramSize) + str + generateSpecialCharacters(qgramSize); 
			}
			for (int i = 0; i <= str.length() - qgramSize; i++) {
				String qgram = str.substring(i, i + qgramSize);
				incrementCount(qgram, tokenTF);
			}
		}
		return tokenTF;
	}

	// Tokenizer based on conditional occurance; evry token is a word
	public HashMap<String, Double> getConditionalTF(String str) {
		HashMap<String, Double> tokenTF = new HashMap<String, Double>();
		if (! tokenizeUsingQgrams ) {
			String[] tokens = str.toLowerCase().split(" ");
			for (int i = 0; i < tokens.length; i++) {
				// incrementCount(tokens[i], tokenTF);
				int j=1;
				String modToken = tokens[i]+"_"+j;
				while(tokenTF.containsKey(modToken)){
					j++;
					modToken = tokens[i]+"_"+j;
				}
				tokenTF.put(modToken, 1.0);
			}
		} else {
			str = str.toLowerCase();
			if(replaceSpaceWithSpecialCharacter){
				str = str.replaceAll(" ", generateSpecialCharacters(qgramSize));
				str = generateSpecialCharacters(qgramSize) + str + generateSpecialCharacters(qgramSize); 
			}
			for (int i = 0; i <= str.length() - qgramSize; i++) {
				String qgram = str.substring(i, i + qgramSize);
				int j=1;
				String modToken = qgram+"_"+j;
				while(tokenTF.containsKey(modToken)){
					j++;
					modToken = qgram+"_"+j;
				}
				tokenTF.put(modToken, 1.0);
			}
		}
		return tokenTF;
	}

	// This is an implementation for tfidf weights
	public HashMap<String, Double> getWeights(HashMap<String, Double> tokenTF, HashMap<String, Double> qgramIDF) {
		double sum = 0, tfidf;
		for (String qgram : tokenTF.keySet()) {
			if (qgramIDF.containsKey(qgram)) {
				// This is a slight variant of TfIdf
				// w(u,C) = log(1 + tf)*idf
            /*
               Lets change this formula to w(u,c) = tf*idf , tf is raw tf
            */
				//tfidf =(1 + Math.log( tokenTF.get(qgram)) ) * qgramIDF.get(qgram);
				tfidf =  tokenTF.get(qgram) * qgramIDF.get(qgram);
				sum += tfidf * tfidf;
				tokenTF.put(qgram, tfidf);
			}
		}
		sum = Math.sqrt(sum);
		for (String qgram : tokenTF.keySet()) {
			tokenTF.put(qgram, tokenTF.get(qgram) / sum);
		}
		return tokenTF;
	}

	public HashMap<String, Double> getWeights(HashMap<String, Double> tokenTF, HashMap<String, Double> qgramIDF,
			double averageLength) {
		return getWeights(tokenTF, qgramIDF);
	}

	public HashMap<String, Double> getWeights(String str, HashMap<String, Double> qgramIDF) {
		return getWeights(getTF(str), qgramIDF);
	}

	public HashMap<String, Double> getQueryWeights(String str, HashMap<String, Double> qgramIDF) {
		return getWeights(getTF(str), qgramIDF);
	}

	public void convertDFtoIDF(HashMap<String, Double> qgramIDF, int size) {
		for (String qgram : qgramIDF.keySet()) {
         // changing idf to log(N/n)
			//qgramIDF.put(qgram, Math.log(1 + size / qgramIDF.get(qgram)));
			qgramIDF.put(qgram, Math.log( size / qgramIDF.get(qgram)));
		}
	}

	public void convertDFtoIDF(HashMap<String, Double> qgramIDF, int size, Vector<HashMap<String, Double>> recordTokenWeights) {
		convertDFtoIDF(qgramIDF, size);
	}

	
	public void getDFandTFweight(int recordId, String str, HashMap<String, Double> qgramIDF,
			Vector<HashMap<String, Double>> recordTokenWeights) {
		HashMap<String, Double> tokenTF = getTF(str);
		recordTokenWeights.insertElementAt(tokenTF, recordId);
		// Set df's
		for (String qgram : tokenTF.keySet()) {
			incrementDFCount(qgram, qgramIDF, tokenTF.get(qgram));
		}
	}

	public double getTotalTokens(HashMap<String, Double> tokenTf) {
		double totalTF = 0;
		for (String str : tokenTf.keySet()) {
			totalTF += tokenTf.get(str);
		}
		return totalTF;
	}

	public double getAverageRecordLength(Vector<HashMap<String, Double>> recordTokenWeights) {
		double aveLength = 0;
		for (int i = 0; i < recordTokenWeights.size(); i++) {
			HashMap<String, Double> tokenTf = recordTokenWeights.get(i);
			aveLength += getTotalTokens(tokenTf);
		}
		return aveLength/recordTokenWeights.size();
	}

	public void convertTFtoWeights(Vector<String> records, HashMap<String, Double> qgramIDF,
			Vector<HashMap<String, Double>> recordTokenWeights) {
		double averageLength = 0;
		averageLength = getAverageRecordLength(recordTokenWeights);
		//System.out.println(" avgDL: "+averageLength);
		for (int i = 0; i < records.size(); i++) {
			HashMap<String, Double> tokenTfidf = recordTokenWeights.get(i);
			tokenTfidf = getWeights(tokenTfidf, qgramIDF, averageLength);
			recordTokenWeights.setElementAt(tokenTfidf, i);
		}
	}

	public void preprocessTable(HashMap<Integer,String> records, HashMap<String, Double> qgramIDF,
			Vector<HashMap<String, Double>> recordTokenWeights, String tableName) {
	
	}

	public void preprocessTable(Vector<String> records, HashMap<String, Double> qgramIDF,
			Vector<HashMap<String, Double>> recordTokenWeights, String tableName) {
		Config config = new Config();
		MySqlDB mysqlDB = new MySqlDB(config.returnURL(), config.user, config.passwd);
		int numberOfRecords = 0, k = 0;
		try {
			String query = "select tid, "+config.preprocessingColumn+" from " + config.dbName + "." + tableName + " order by tid asc";
			ResultSet rs = mysqlDB.executeQuery(query);
			if (rs != null) {
				while (rs.next()) {
					String str = rs.getString(config.preprocessingColumn);
					k = rs.getInt("tid");
					if ((str != null) && (!str.equals(""))) {
						//System.out.println(k+":  ["+str+"]  :"+str.length() +"   {"+str.trim()+"} "+str.trim().length());
						//str = str.replaceAll("'", "''");
						str = str.toLowerCase();
						records.insertElementAt(str, k - 1);
						// Find the tf's of all the qgrams
						getDFandTFweight(k - 1, str, qgramIDF, recordTokenWeights);
					}
					numberOfRecords++;
				}
			}
			mysqlDB.close();
		} catch (Exception e) {
			System.out.println("database error: cannot read table");
			e.printStackTrace();
		}
		
		if(logToDB){
			storePreprocessedDataToDB(tableName, recordTokenWeights, extractMetricName(this.getClass().getName())+"_tf" );
		}
		// convert the df into IDF
		//System.out.println("N: " + numberOfRecords);
		convertDFtoIDF(qgramIDF, numberOfRecords, recordTokenWeights);
		if(logToDB){
			storePreprocessedIDF(tableName, qgramIDF, extractMetricName(this.getClass().getName())+"_idf" );
		}
		// Now convert the tf in fileQrmanWeight to tfidf weights
		convertTFtoWeights(records, qgramIDF, recordTokenWeights);
		if(logToDB){
			storePreprocessedDataToDB(tableName, recordTokenWeights, extractMetricName(this.getClass().getName())+"_wt" );
		}
		//System.exit(0);
	}

	public void preprocessTableWSign(Vector<String> records, HashMap<String, Double> qgramIDF,
			Vector<HashMap<String, Double>> recordTokenWeights, 
			HashMap<Integer, Long> signature, 
			Integer par1,
			Integer par2,
			Integer par3,
			String tableName) {
	}
	
	
	
	
	
	public List<IdScore> getSimilarRecords(String query, HashMap<String, Double> qgramIDF,
			Vector<HashMap<String, Double>> recordTokenWeights) {
		//query = query.replaceAll("'", "''");
		List<IdScore> scoreList = new ArrayList<IdScore>();
		HashMap<String, Double> queryWeights = getQueryWeights(query, qgramIDF);
		for (int k = 0; k < recordTokenWeights.size(); k++) {
			double score = 0;
			boolean haveCommonToken = false;
			HashMap<String, Double> fileWeights = recordTokenWeights.get(k);
			for (String qgram : queryWeights.keySet()) {
				if (fileWeights.containsKey(qgram)) {
					score += fileWeights.get(qgram) * queryWeights.get(qgram);
					haveCommonToken = true;
				}
			}
			if (haveCommonToken) {
				scoreList.add(new IdScore(k, score));
			}
		}
		// // System.gc();
		Collections.sort(scoreList);
		
		return scoreList;
	}

	// This function is for the metrics which has to go iteratively through all the records to get the final scoreList
	public List<IdScore> getSimilarRecords(String query, HashMap<String, Double> qgramIDF,
			Vector<HashMap<String, Double>> recordTokenWeights, Vector<String> recordVector) {
		return getSimilarRecords(query, qgramIDF, recordTokenWeights);
	}
	
	// TODO: Check if the threshold version works
	public List<IdScore> getSimilarRecordsTH(String query, HashMap<String, Double> qgramIDF,
			Vector<HashMap<String, Double>> recordTokenWeights, Vector<String> recordVector, Double thr, int begin) {
		List<IdScore> scoreList = new ArrayList<IdScore>();
		HashMap<String, Double> queryWeights = getQueryWeights(query, qgramIDF);
		for (int k = (begin-1); k < recordTokenWeights.size(); k++) {
			double score = 0;
			//boolean haveCommonToken = false;
			HashMap<String, Double> fileWeights = recordTokenWeights.get(k);
			for (String qgram : queryWeights.keySet()) {
				if (fileWeights.containsKey(qgram)) {
					score += fileWeights.get(qgram) * queryWeights.get(qgram);
					//haveCommonToken = true;
				}
			}
			if (score >= thr) {
				scoreList.add(new IdScore(k, score));
			}
		}
		// // System.gc();
		Collections.sort(scoreList);
		
		return scoreList;	
	}

	// TODO: Check if the threshold version works
	public List<IdScore> getSimilarRecordsTH2(String query, HashMap<String, Double> qgramIDF,
			HashMap<Integer, String> records, Double thr) {
		return null;
	}

	public IdScore getMaxSimilarRecord(String query, HashMap<String, Double> qgramIDF,
			HashMap<Integer, String> records, Double thr) {
		return null;		
	}	
	
	
	public IdScore getMaxSimilarRecord2(String query, HashMap<String, Double> qgramIDF,
			HashMap<Integer, String> records, Vector<HashMap<String, Double>> recordTokenWeights, Double thr) {
		IdScore max = null;
		HashMap<String, Double> queryWeights = getQueryWeights(query, qgramIDF);
		//double meanIDF = Util.getMeanIDF(qgramIDF);
		double maxscore = thr;
		for (int k:records.keySet()) {
			double score = 0;
			HashMap<String, Double> fileWeights = recordTokenWeights.get(k);
			for (String qgram : queryWeights.keySet()) {
				if (fileWeights.containsKey(qgram)) {
					score += fileWeights.get(qgram) * queryWeights.get(qgram);
					//haveCommonToken = true;
				}
			}
			//System.out.println(score);
			if(score >= maxscore){
				max = new IdScore(k, score);
				maxscore = score;
			}
		}
		// // System.gc();
		
		//if (scoreList.size()!=0) max = Collections.max(scoreList);
		return max;		
	}	
	

	
	
	// Returns single score
	// TODO: check
	public double getSimilarityScore(String query, HashMap<String, Double> qgramIDF,
			Vector<HashMap<String, Double>> recordTokenWeights, int tid, String record) {
		HashMap<String, Double> queryWeights = getQueryWeights(query, qgramIDF);
		double score = 0;
		HashMap<String, Double> fileWeights = recordTokenWeights.get(tid-1);
		for (String qgram : queryWeights.keySet()) {
			if (fileWeights.containsKey(qgram)) {
				score += fileWeights.get(qgram) * queryWeights.get(qgram);
			}
		}
		return score;
	}
	
	public double getSimilarityScore(String query, HashMap<String, Double> qgramIDF, double meanIDF,
			Vector<HashMap<String, Double>> recordTokenWeights, int tid, String record) {
		HashMap<String, Double> queryWeights = getQueryWeights(query, qgramIDF);
		double score = 0;
		HashMap<String, Double> fileWeights = recordTokenWeights.get(tid-1);
		for (String qgram : queryWeights.keySet()) {
			if (fileWeights.containsKey(qgram)) {
				score += fileWeights.get(qgram) * queryWeights.get(qgram);
			}
		}
		return score;
	}
	
	public static void storePreprocessedDataToDB(String tableName, Vector<HashMap<String, Double>> recordTokenWeights,
			String metric) {
		Config config = new Config();
		MySqlDB mysqlDB = new MySqlDB(config.returnURL(), config.user, config.passwd);
		String query = "";
		try {
			query = "drop table if exists " + config.dbName + ".PP_" + tableName + "_" + metric;
			mysqlDB.executeUpdate(query);

			query = "create table " + config.dbName + ".PP_" + tableName + "_" + metric
					+ " (rid int, token varchar(255), weight double)";
			mysqlDB.executeUpdate(query);

			System.err.println("Executing " + query);
			for (int a = 0; a < recordTokenWeights.size(); a++) {
				HashMap<String, Double> tokenWeights = recordTokenWeights.get(a);
				for (String token : tokenWeights.keySet()) {
					query = "insert into " + config.dbName + ".PP_" + tableName + "_" + metric + " values (" + (a + 1)
							+ " ,'" + token.replaceAll("'", "''") + "', "+ tokenWeights.get(token) + ")";
					mysqlDB.executeUpdate(query);
				}
			}
			mysqlDB.close();
		} catch (Exception e) {
			System.err.println("Can't execute Query: " + query);
			e.printStackTrace();
		}
	}

	public static void storePreprocessedIDF(String tableName, HashMap<String, Double> qgramIDF, String metric) {
		Config config = new Config();
		MySqlDB mysqlDB = new MySqlDB(config.returnURL(), config.user, config.passwd);
		String query = "";
		try {
			query = "drop table if exists " + config.dbName + ".PP_" + tableName + "_" + metric;
			mysqlDB.executeUpdate(query);

			query = "create table " + config.dbName + ".PP_" + tableName + "_" + metric
					+ " (token varchar(255), idf double)";
			mysqlDB.executeUpdate(query);

			System.err.println("Executing " + query);
			for (String token: qgramIDF.keySet()) {
					query = "insert into " + config.dbName + ".PP_" + tableName + "_" + metric + " values ('" 
							 + token.replaceAll("'", "''") + "', "+ qgramIDF.get(token) + ")";
					mysqlDB.executeUpdate(query);
			}
			mysqlDB.close();
		} catch (Exception e) {
			System.err.println("Can't execute Query: " + query);
			e.printStackTrace();
		}
	}
} public class TfIdf extends Preprocess {

	// Tokenizer based on whitespace; evry token is a word
	public HashMap<String, Double> getTF(String str) {
		return getTokenTFMultiple(str);
	}

} public class Config 
{
	public String cfgFileName;
	public String dbName;
	public String host ;
	public String user ;
	public String passwd;
	public String port ;
	public String url ;
	public boolean useFile;
	public String fileName;
	public String dbServer;
	public String outputFile;
    public String preprocessingColumn;
	public boolean debugMode;
	public int dbOption;
	
	public static String storeResultDirectory = "c:/Users/admin/dcp-workspace/log/";
    public static String queryFileName = "c:/Users/admin/dcp-workspace/data/_fixedQueries_";
    public static String expResultTablePrefix = "_ExpRes_"; 
	public static String queryClass = "dcp.clean";
    
	public static boolean tokenizeUsingQgrams = true;
    public int numHeaders;
    

	public String bm25WeightTable = "$Aux$_BM253_0_cu1_string_BMBaseWeights";
	public String tfidfWeightTable = "$Aux$_TfIdf3_0_cu1_string_Weights";
	// Class constructor 
	public Config() {
		outputFile = "geneRatios.txt";
		
		host = "localhost";
		user = "root";
		passwd = "";
		port = "3306";
		url = "";
		useFile = false;
	    fileName = "tmpFile";
	    dbServer = "mysql";
	    numHeaders = 7;
	    dbName = "midas";
	    preprocessingColumn = "name";
	    
	    //dbName = "probtest";
	    //preprocessingColumn = "title";
	}
	
	public String returnURL(){
		if(dbOption == 1)
			url = "jdbc:postgres://"+ host + ":"+ port +"/"+ dbName;
		else
			url = "jdbc:mysql://"+host+":"+port+"/"+dbName ;
		return url;
	}
	
	public void setDebugMode(boolean val){
		debugMode = val;
	}

	public void setDbName(String name){
		dbName = name;
	}
	
}
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.sql.ResultSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Vector;
import dbdriver.MySqlDB;
import evaluation.AccuracyMeasure;
import simfunctions.BM25;
import simfunctions.CondBM25;
import simfunctions.CondHMM;
import simfunctions.CondIdf;
import simfunctions.DistinctBM25;
import simfunctions.DistinctHMM;
import simfunctions.DistinctTfIdf;
import simfunctions.HMM;
import simfunctions.Idf;
import simfunctions.ModBM25;
import simfunctions.ModHMM;
import simfunctions.Preprocess;
import simfunctions.TfIdf;
import utility.Config;
			}
		}
		return queries;
	}

	public static HashSet<Integer> getAllTidsHavingId(int id, String tableName) {
		HashSet<Integer> tidsHavingThisID = new HashSet<Integer>();
		Config config = new Config();
		MySqlDB mysqlDB = new MySqlDB(config.returnURL(), config.user, config.passwd);
		try {
			String query = "SELECT tid FROM " + config.dbName + "." + tableName + " where id=" + id;
			System.err.println("Executing " + query);
			ResultSet rs = mysqlDB.executeQuery(query);
			if (rs != null) {
				while (rs.next()) {
					tidsHavingThisID.add(rs.getInt("tid"));
				}
			}
			mysqlDB.close();
		} catch (Exception e) {
			System.err.println("Can't generate Random Queries");
			e.printStackTrace();
		}
		return tidsHavingThisID;
	}

	public static int[] generateBooleanList(HashSet<Integer> actualResult, List<IdScore> scoreList) {
		int[] booleanList = new int[scoreList.size()];
		for (int i = 0; i < scoreList.size(); i++) {
			if (actualResult.contains(scoreList.get(i).id + 1)) {
				booleanList[i] = 1;
			} else {
				booleanList[i] = 0;
			}
		}
		return booleanList;
	}

	public static double[] logMapArray(String filename, Vector<Preprocess> preprocessVector, double[][] mapArray) {
		double[] meanMAP = new double[preprocessVector.size()];
		try {
			BufferedWriter out = new BufferedWriter(new FileWriter(filename));
			for (int a = 0; a < preprocessVector.size(); a++) {
				out.write(preprocessVector.get(a).getClass().getName() + "\t\t");
				meanMAP[a] = 0;
			}
			out.write("\n");
			for (int a = 0; a < mapArray.length; a++) {
				for (int b = 0; b < preprocessVector.size(); b++) {
					out.write(mapArray[a][b] + "\t\t");
					meanMAP[b] += mapArray[a][b];
				}
				out.write("\n");
			}
			out.close();
		} catch (IOException e) {
			System.err.println("Error while writing to  " + filename);
		}
		if (mapArray.length > 0) {
			for (int a = 0; a < preprocessVector.size(); a++) {
				meanMAP[a] = meanMAP[a] / mapArray.length;
			}
		}
		return meanMAP;
	}

	public static void logMapArrayToDB(String tableName, double[][] mapArray, Vector<Preprocess> preprocessVector,
			Vector<String> queryVector, Vector<Integer> queryIdVector) {
		Config config = new Config();
		MySqlDB mysqlDB = new MySqlDB(config.returnURL(), config.user, config.passwd);
		String query = "";
		try {
			query = "drop table if exists " + config.dbName + "." + Config.expResultTablePrefix + tableName + queryTokenLength;
			mysqlDB.executeUpdate(query);

			query = "create table " + config.dbName + "." + Config.expResultTablePrefix + tableName + queryTokenLength
					+ " (qid int, query text, id int, metric varchar(255), map double)";
			mysqlDB.executeUpdate(query);

			System.err.println("Executing " + query);
			for (int a = 0; a < mapArray.length; a++) {
				for (int b = 0; b < preprocessVector.size(); b++) {
					query = "insert into " + config.dbName + "." + Config.expResultTablePrefix + tableName + queryTokenLength + " values ("
							+ (a + 1) + ",'" + queryVector.get(a) + "'," + queryIdVector.get(a) + ",'"
							+ preprocessVector.get(b).getClass().getName() + "', " + mapArray[a][b] + ")";
					mysqlDB.executeUpdate(query);
				}
				// mysqlDB.executeUpdate(query);
			}
			mysqlDB.close();
		} catch (Exception e) {
			System.err.println("Can't execute Query: " + query);
			e.printStackTrace();
		}
	}

	public static void main(String[] args) {
		// TfIdf tfidf = new TfIdf();
		Preprocess metric = null;
		// Vector<Preprocess> preprocessVector = new Vector<Preprocess>();
		Vector<Preprocess> preprocessVector = new Vector<Preprocess>();
		Vector<Vector<HashMap<String, Double>>> basetableTokenWeightsVector = new Vector<Vector<HashMap<String, Double>>>();
		Vector<HashMap<String, Double>> qgramIDFVector = new Vector<HashMap<String, Double>>();
		double[][] mapArray;
		int maxQueries = 100;
		if (args.length > 2) {
			queryTokenLength = Integer.parseInt(args[2]);
			System.err.println("Setting queryTokenLength to " + queryTokenLength);
		}

		Preprocess tfidf = new TfIdf();
		Preprocess idf = new Idf();
		Preprocess condidf = new CondIdf();
		Preprocess disttfidf = new DistinctTfIdf();
		
		Preprocess bm25 = new BM25();
		Preprocess modbm25 = new ModBM25();
		Preprocess condbm25 = new CondBM25();
		Preprocess distbm25 = new DistinctBM25();
		
		Preprocess hmm = new HMM();