public class AlchemyClassification {

	private String category;
	private double score;
	
	public AlchemyClassification() {
		category = null;
		score = -1.0;
	}
	
	public AlchemyClassification(String category, double score) {
		this.category = category;
		this.score = score;
	}
	
	public String getCategory() {
		return category;
	}
	
	public double getScore() {
		return score;
	}
	
	public void print() {
		System.out.println(category+": "+score);
	}

}
 public class AlchemyClassifier {
	
	private static String[] keys;
	private static int keyIndex;
	private static AlchemyAPI alchemyAPI;
	private static int requestCount;

	public AlchemyClassifier() {
		init(); //NOTE: needs to be called from elsewhere too
	}
	
	public static void init() {
		keys = new String[35];
		keys[0] = "c497c0fc6dd00dd6fb13a7cb29c3311709d8c344";
		keys[1] = "9dcb52bec38f96f4d455467453858e44cbeccdaf";
		keys[2] = "5821b26a00557cf60289b03cb1be2257dc3cf6eb";
		keys[3] = "8aa54e2962747f2653cd1d848a63d2a463e4d41f";
		keys[4] = "c0b97edd81b8f4bef113f8d78b4c2755768746b6";
		keys[5] = "f45c15726375f4fa78644d75dbc2eef2e3528d87";
		keys[6] = "c3811e9bf08a209569d1c335e576e8b4ac0df0bc";
		keys[7] = "358855bc94a0fbb2a7fd5ac574099cc574a63932";
		keys[8] = "73173512d654109b54694292fc87fdf51998d408";
		keys[9] = "2225a47a67142a25643ac926bd6e79a0b6250dd1";
		keys[10] = "1601aaa431917f4e3ba62c7a3e1ba6afd3190d10";
		keys[11] = "b6e21a7ded427e2d9bf0f080edadb7cd0e0f31fe";
		keys[12] = "31b8d6989ec914b4b36f72eff8df6ac20ee37263";
		keys[13] = "5eb99cbc7756802962c891b7843f4f6160037c31";
		keys[14] = "8d442ee0195e3e5343a97e1d4c2b478bd91b8899";
		keys[15] = "02e3f0ab5fd43d85178ed501e9917227c24a1be7";
		keys[16] = "e4f32b39fa8ca88ef12f488895797695a9c6f902";
		keys[17] = "a10efc9508044afdbd3415341e1dfb12ef27fc92";
		keys[18] = "f809b9795a4761084f22e07b339e293c5556d308";
		keys[19] = "b0470e00a30d6e9fbf94716baca9c4174d2b9f12";
		keys[20] = "e7f90d7158b2168479ae944af07f56bad6bc2247";
		keys[21] = "88bd58ed3df9d77eff384092e62d0788363242cd";
		keys[22] = "3254e7332a9262a475236563b9984eb8b18fc97e";
		keys[23] = "4af5c0b33f491ea00499b8dfae827c8eaa3ed128";
		keys[24] = "15a9800bd191aad9bce2e65b7040eee74c36a110";
		keys[25] = "51c2e1771680357dbd34bb8352bda62a6b661316";
		keys[26] = "e21a8e0e597e5b9d3278e9d361080909481d20c9";
		keys[27] = "371ac1ba6dc6b6809a4c2bb15bc37b78879d2714";
		keys[27] = "81d9efc046063f6508acaf5052083c2188605817";
		keys[28] = "b44c8b62149e0d27edc51e7df2feddd397178145";
		keys[29] = "7431ba2ecc7703222a09dd1d86642837af5697eb";
		keys[30] = "f80d64cc89038a918f42e95c02b6f7fe6b9308c1";
		keys[31] = "ca3a625d7c429ace4b8033d4cc9ec3b08acaa878";
		keys[32] = "eaab19db33996b5404a9be6ba00bc5e8c3851561";
		keys[33] = "97b981de050c18582c39e37f9340f4c57e2e1500";
		keys[34] = "084385760b5c9b55f8c8c91b26d5195ba3bfa6dd";
		
		
		keyIndex = 0;
		alchemyAPI = AlchemyAPI.GetInstanceFromString(keys[keyIndex]);
		requestCount = 0;
	}
	
	//Either returns a valid AlchemyClassification, or it returns an empty one. Never null...
	public static AlchemyClassification classifyURL(String url) {
		updateRequestCount();
		try {
			org.w3c.dom.Document d = alchemyAPI.URLGetCategory(url);
			String category = d.getChildNodes().item(0).getChildNodes().item(9).getTextContent();
			double score = Double.parseDouble(d.getChildNodes().item(0).getChildNodes().item(11).getTextContent());
			if(category.isEmpty() || category == null) {
				return new AlchemyClassification();
			} else {
				return new AlchemyClassification(category,score);
			}
		} catch(NullPointerException e) {
			System.err.println("NullPointerException trying to classify a URL:");
			System.err.println(url);
		} catch (IllegalArgumentException e) {
			//URL not well formatted - might be a truncated retweet. This is fine.		
		} catch (XPathExpressionException e) {
			e.printStackTrace();
		} catch (IOException e) {
			//Page not HTML - image or something. Or - daily limit reached!!!
			if(e.getMessage().contains("daily-transaction-limit-exceeded")) {
				System.err.println("Transaction limit alert");
				/*
				if(updateKey()) {
					return classifyText(text);
				} else {
					return null; 
				}
				*/
				return null;
			}
		} catch (SAXException e) {
			e.printStackTrace();
		} catch (ParserConfigurationException e) {
			e.printStackTrace();
		}
		return new AlchemyClassification();
	}
	
	public static AlchemyClassification classifyText(String text) {
		updateRequestCount();
		try {
			org.w3c.dom.Document d = alchemyAPI.TextGetCategory(text);	
			//Tools.prettyPrintDocument(d);
			String category = d.getChildNodes().item(0).getChildNodes().item(9).getTextContent();
			double score = Double.parseDouble(d.getChildNodes().item(0).getChildNodes().item(11).getTextContent());
			if(category == null || category.isEmpty()) {
				return new AlchemyClassification();
			} else {
				return new AlchemyClassification(category,score);
			}
		} catch(NullPointerException e) {
			System.err.println("NullPointerException trying to classify some text:");
			System.err.println(text);
		} catch (IllegalArgumentException e) {
			//text not well formatted - might be an empty tweet? This is (probably) fine.		
		} catch (XPathExpressionException e) {
			e.printStackTrace();
		} catch (IOException e) {
			//Page not HTML - image or something. Or - daily limit reached!!!
			if(e.getMessage().contains("daily-transaction-limit-exceeded")) {
				System.err.println("Transaction limit alert");
				if(updateKey()) {
					return classifyText(text);
				} else {
					return null; 
				}
			}
		} catch (SAXException e) {
			e.printStackTrace();
		} catch (ParserConfigurationException e) {
			e.printStackTrace();
		}
		return new AlchemyClassification();
	}
	
	private static void updateRequestCount() {
		requestCount++;
		if(requestCount % 50 == 0) {
			//System.out.println("Alchemy Requests Made: "+requestCount);
			
			
			//Experimental key-change -it works !!!
			if(requestCount % 29900 == 0) {
				updateKey();
			}
		}
	}
	
	private static boolean updateKey() {
		if(keyIndex < keys.length - 1) {
			System.out.println("Trying another alchemy key...");
			keyIndex++;
			System.out.println("Using key "+keyIndex);
			alchemyAPI = AlchemyAPI.GetInstanceFromString(keys[keyIndex]);
			return true;
		} else {
			System.out.println("Out of alchemy keys to try");
			keyIndex=0;
			return true;
			//return false;
		}
	}
	
}
 public class FullAlchemyClassification {
	
	private long userID;
	private Map<String,Double> classifications;
	
	public FullAlchemyClassification(long userID) {
		this.userID = userID;
		classifications = new HashMap<String,Double>();
		
		try {
			FileInputStream fstream = new FileInputStream("classifications/alchemy/"+userID+".csv");
			DataInputStream in = new DataInputStream(fstream);
			BufferedReader br = new BufferedReader(new InputStreamReader(in));
			String strLine;
			br.readLine(); //Skp the CSV descriptor
			while ((strLine = br.readLine()) != null)   {
				String[] split = strLine.split(",");
				String cat = split[0].substring(1,split[0].length()-1);
				
				double score = 0.0;
				//tweet scores
				if(Integer.parseInt(split[2]) != 0) {
					score += Double.parseDouble(split[1]) / Double.parseDouble(split[2]);
				}
				//url scores
				if(Integer.parseInt(split[4]) != 0) {
					score += Double.parseDouble(split[3]) / Double.parseDouble(split[4]);
				}
				
				classifications.put(cat,score);
			}
			in.close();
		} catch (Exception e) {
			System.err.println("Error: " + e.getMessage());
		}
		
		//need to normalise the alchemy classifcations, durr
		double scoreSum = 0.0;
		for(String cat : classifications.keySet()) {
			scoreSum += classifications.get(cat);
		}
		for(String cat : classifications.keySet()) {
			classifications.put(cat, classifications.get(cat)/scoreSum);
		}
		
		classifications = Tools.sortMapByValueDesc(classifications);
	}
	
	public void print() {
		for(String cat : classifications.keySet()) {
			System.out.println(cat+": "+classifications.get(cat));
		}
	}
	
	public Set<String> getCategorySet() {
		return classifications.keySet();
	}
	
	public boolean hasCategory(String cat) {
		if(classifications.keySet().contains(cat)) return true;
		else return false;
	}
	
	public double getScore(String cat) {
		return classifications.get(cat);
	}
	
	public double magnitude() {
		double sum = 0.0;
		for(String cat : classifications.keySet()) {
			sum += (classifications.get(cat)*classifications.get(cat));
		}
		sum = Math.sqrt(sum);
		return sum;
	}

	public double cosineSimilarity(FullAlchemyClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		
		double score = 0.0;
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				score += (this.getScore(cat) * c.getScore(cat));
			}
		}
		
		//normalise by magnitudes
		Double magnitudes = (this.magnitude() * c.magnitude());
		score /= magnitudes;
		if(Double.isNaN(score)) {
			return 0.0; //NaN caused by zero vectors ie no classifications!
		}
		else return score;
	}
	
	public double jsDivergence(FullAlchemyClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		Map<String,Double> M = new HashMap<String,Double>();
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				M.put(cat, (this.getScore(cat)+c.getScore(cat))/2.0);
			}
		}
		double d1 = 0.0;
		for(String cat : M.keySet()) {
			if(this.getCategorySet().contains(cat) ) {
				d1 += this.getScore(cat) * Math.log(this.getScore(cat)/M.get(cat));
			}
		}	
		double d2 = 0.0;
		for(String cat : M.keySet()) {
			if(c.getCategorySet().contains(cat)) {
				d1 += c.getScore(cat) * Math.log(c.getScore(cat)/M.get(cat));
			}
		}		
		double score = d1/2.0 + d2/2.0;
		return score;
	}
	
	
}
 public class FullCalaisClassification {
	
	private long userID;
	private Map<String,Double> classifications;
	
	public FullCalaisClassification(long userID) {
		this.userID = userID;
		classifications = new HashMap<String,Double>();
		
		try {
			FileInputStream fstream = new FileInputStream("classifications/calais/"+userID+".csv");
			DataInputStream in = new DataInputStream(fstream);
			BufferedReader br = new BufferedReader(new InputStreamReader(in));
			String strLine;
			br.readLine(); //Skp the CSV descriptor
			while ((strLine = br.readLine()) != null)   {
				String[] split = strLine.split(",");
				String cat = split[0].substring(1,split[0].length()-1);
				
				double score = Double.parseDouble(split[1]) / Double.parseDouble(split[2]);
				
				classifications.put(cat,score);
			}
			in.close();
		} catch (Exception e) {
			System.err.println("Error: " + e.getMessage());
		}
		
		//need to normalise the classifcations, durr
		double scoreSum = 0.0;
		for(String cat : classifications.keySet()) {
			scoreSum += classifications.get(cat);
		}
		for(String cat : classifications.keySet()) {
			classifications.put(cat, classifications.get(cat)/scoreSum);
		}
		
		classifications = Tools.sortMapByValueDesc(classifications);
	}
	
	public void print() {
		for(String cat : classifications.keySet()) {
			System.out.println(cat+": "+classifications.get(cat));
		}
	}
	
	public Set<String> getCategorySet() {
		return classifications.keySet();
	}
	
	public boolean hasCategory(String cat) {
		if(classifications.keySet().contains(cat)) return true;
		else return false;
	}
	
	public double getScore(String cat) {
		return classifications.get(cat);
	}
	
	public double magnitude() {
		double sum = 0.0;
		for(String cat : classifications.keySet()) {
			sum += (classifications.get(cat)*classifications.get(cat));
		}
		sum = Math.sqrt(sum);
		return sum;
	}

	public double cosineSimilarity(FullCalaisClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		
		double score = 0.0;
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				score += (this.getScore(cat) * c.getScore(cat));
			}
		}
		
		//normalise by magnitudes
		Double magnitudes = (this.magnitude() * c.magnitude());
		score /= magnitudes;
		if(Double.isNaN(score)) {
			return 0.0; //NaN caused by zero vectors ie no classifications!
		}
		else return score;
	}
	
	public double jsDivergence(FullCalaisClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		Map<String,Double> M = new HashMap<String,Double>();
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				M.put(cat, (this.getScore(cat)+c.getScore(cat))/2.0);
			}
		}
		double d1 = 0.0;
		for(String cat : M.keySet()) {
			if(this.getCategorySet().contains(cat) ) {
				d1 += this.getScore(cat) * Math.log(this.getScore(cat)/M.get(cat));
			}
		}	
		double d2 = 0.0;
		for(String cat : M.keySet()) {
			if(c.getCategorySet().contains(cat)) {
				d1 += c.getScore(cat) * Math.log(c.getScore(cat)/M.get(cat));
			}
		}		
		double score = d1/2.0 + d2/2.0;
		return score;
	}
	
} public class FullTextwiseClassification {
	
	private long userID;
	private Map<String,Double> classifications;
	
	public FullTextwiseClassification(long userID, boolean proper) {
		this.userID = userID;
		classifications = new HashMap<String,Double>();
		
		try {
			FileInputStream fstream;
			if(proper) fstream = new FileInputStream("classifications/textwiseproper/"+userID+".csv");
			else fstream = new FileInputStream("classifications/textwise/"+userID+".csv");
			DataInputStream in = new DataInputStream(fstream);
			BufferedReader br = new BufferedReader(new InputStreamReader(in));
			String strLine;
			br.readLine(); //Skp the CSV descriptor
			while ((strLine = br.readLine()) != null)   {
				//typical, some categories have commas in them!
				String[] initSplit = strLine.split("\"");
				String cat = initSplit[1];
				String[] split = initSplit[2].split(",");
				
				double score = 0.0;
					if(!proper) {
					//tweet scores
					if(Integer.parseInt(split[2]) != 0) {
						score += Double.parseDouble(split[1]) / Double.parseDouble(split[2]);
					}
					//url scores
					if(Integer.parseInt(split[4]) != 0) {
						score += Double.parseDouble(split[3]) / Double.parseDouble(split[4]);
					}
				} else {
					score = Double.parseDouble(split[1])/Double.parseDouble(split[2]);
				}
				
				if(proper) cat = cat.substring(0,cat.indexOf("/")); //take only the textwise root
				classifications.put(cat,score);
			}
			in.close();
		} catch (Exception e) {
			System.err.println("Error: " + e.getMessage());
			System.err.println(userID);
		}
		
		//need to normalise the classifcations, durr
		double scoreSum = 0.0;
		for(String cat : classifications.keySet()) {
			scoreSum += classifications.get(cat);
		}
		for(String cat : classifications.keySet()) {
			classifications.put(cat, classifications.get(cat)/scoreSum);
		}
		
		classifications = Tools.sortMapByValueDesc(classifications);
	}
	
	public void print() {
		for(String cat : classifications.keySet()) {
			System.out.println(cat+": "+classifications.get(cat));
		}
	}
	
	public Set<String> getCategorySet() {
		return classifications.keySet();
	}
	
	public boolean hasCategory(String cat) {
		if(classifications.keySet().contains(cat)) return true;
		else return false;
	}
	
	public double getScore(String cat) {
		return classifications.get(cat);
	}
	
	public double magnitude() {
		double sum = 0.0;
		for(String cat : classifications.keySet()) {
			sum += (classifications.get(cat)*classifications.get(cat));
		}
		sum = Math.sqrt(sum);
		return sum;
	}

	public double cosineSimilarity(FullTextwiseClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		
		double score = 0.0;
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				score += (this.getScore(cat) * c.getScore(cat));
			}
		}
		
		//normalise by magnitudes
		Double magnitudes = (this.magnitude() * c.magnitude());
		score /= magnitudes;
		if(Double.isNaN(score)) {
			return 0.0; //NaN caused by zero vectors ie no classifications!
		}
		else return score;
	}
	
	public double jsDivergence(FullTextwiseClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		Map<String,Double> M = new HashMap<String,Double>();
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				M.put(cat, (this.getScore(cat)+c.getScore(cat))/2.0);
			}
		}
		double d1 = 0.0;
		for(String cat : M.keySet()) {
			if(this.getCategorySet().contains(cat) ) {
				d1 += this.getScore(cat) * Math.log(this.getScore(cat)/M.get(cat));
			}
		}	
		double d2 = 0.0;
		for(String cat : M.keySet()) {
			if(c.getCategorySet().contains(cat)) {
				d1 += c.getScore(cat) * Math.log(c.getScore(cat)/M.get(cat));
			}
		}		
		double score = d1/2.0 + d2/2.0;
		return score;
	}
	
} public class Pearson {

	public static double pmcc(SimilarityMatrix sm1, SimilarityMatrix sm2) {
		if(sm1.dimension() != sm2.dimension()) {
			System.err.println("Similarity Matrices not of same dimension");
			return -1.0;
		}
		List<Long> uids = Tools.getCSVUserIDs();
		
		//create baseline similarity pairs
		List<SimilarityPair> sps1 = new ArrayList<SimilarityPair>();
		List<SimilarityPair> sps2 = new ArrayList<SimilarityPair>();
		for(int uid1=0; uid1<uids.size(); uid1++) {
			for(int uid2=uid1; uid2<uids.size();uid2++) {
				SimilarityPair sp1 = new SimilarityPair(sm1,sm1.lookupID(uid1),sm2.lookupID(uid2));
				SimilarityPair sp2 = new SimilarityPair(sm2,sm1.lookupID(uid1),sm2.lookupID(uid2));
				if(sp1 == null || sp2 == null) {
					//no categories for one of them
					continue;
				} else {
					sps1.add(sp1);
					sps2.add(sp2);
				}
			}
		}

		//pmcc correlation
		int n = sps1.size();
		double sum1 = 0.0;
		double sum2 = 0.0;
		for(int i=0; i<n; i++) {
			sum1 += sps1.get(i).similarity();
			sum2 += sps2.get(i).similarity();
		}
		double mean1 = sum1/n;
		double mean2 = sum2/n;
		double sumMinusMean1 = 0.0;
		double sumMinusMean2 = 0.0;
		for(int i=0; i<n; i++) {
			sumMinusMean1 += Math.pow(sps1.get(i).similarity()-mean1,2.0);
			sumMinusMean2 += Math.pow(sps2.get(i).similarity()-mean2,2.0);
		}
		double sd1 = Math.sqrt(sumMinusMean1/(n-1));
		double sd2 = Math.sqrt(sumMinusMean2/(n-1));
		double pmccSum = 0.0;
		for(int i=0; i<n; i++) {
			pmccSum += ((sps1.get(i).similarity()-mean1)/sd1)*((sps2.get(i).similarity()-mean2)/sd2);
		}
		double pmcc = pmccSum/(n-1);
		
		return pmcc;
	}
	
} public class SimilarityMatrix implements Serializable {

	private static final long serialVersionUID = 8293810632053185977L;
	
	private double[][] sim;
	private long[] userIDLookup;
	private Map<Long,Integer> indexLookup;
	private int d; //dimensions
	private boolean verbose = false;
	
	public SimilarityMatrix() {
		if(verbose) System.out.println("Creating similarity matrix");
		d = 2506;
		sim = new double[d][d];
		userIDLookup = new long[d];
		indexLookup = new HashMap<Long,Integer>();
		fillLookups();
	}
	
	public SimilarityMatrix(int d) {
		if(verbose) System.out.println("Creating similarity matrix");
		this.d = d;
		sim = new double[d][d];
		userIDLookup = new long[d];
		indexLookup = new HashMap<Long,Integer>();
		fillLookups();
	}
	
	public Double getSimilarity(long uid1, long uid2) {
		try {
			int index1 = indexLookup.get(uid1);
			int index2 = indexLookup.get(uid2);
			return sim[index1][index2];
		} catch (NullPointerException e) {
			//Occurs when a uid had no LLDA topics - so no classifications - so no indexLookup
			return null;
		}
	}
	
	public void fillLookups() {
		if(verbose) System.out.println("Filling the lookup tables");
		int indexCount = 0;
		for(long id : Tools.getCSVUserIDs()) {
			userIDLookup[indexCount] = id;
			indexLookup.put(id, indexCount);
			indexCount++;
		}
	}
	
	public int dimension() {
		return d;
	}
	
	public long lookupID(int index) {
		return userIDLookup[index];
	}

	public void fillAlchemy() {
		//get clasifications
		System.out.println("Filling from Alchemy classifications");
		FullAlchemyClassification[] classifications = new FullAlchemyClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullAlchemyClassification(id); 
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = classifications[m].cosineSimilarity(classifications[n]);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}
	
	public void fillAlchemyJS() {
		//get clasifications
		System.out.println("Filling from Alchemy classifications");
		FullAlchemyClassification[] classifications = new FullAlchemyClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullAlchemyClassification(id); 
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = classifications[m].jsDivergence(classifications[n]);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}
	
	public void fillCalais() {
		//get clasifications
		System.out.println("Filling from OpenCalais classifications");
		FullCalaisClassification[] classifications = new FullCalaisClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullCalaisClassification(id); 
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = classifications[m].cosineSimilarity(classifications[n]);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}
	
	public void fillCalaisJS() {
		//get clasifications
		System.out.println("Filling from OpenCalais classifications");
		FullCalaisClassification[] classifications = new FullCalaisClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullCalaisClassification(id); 
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = classifications[m].jsDivergence(classifications[n]);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}
	
	public void fillTextwise() {
		//get clasifications
		System.out.println("Filling from Textwise classifications");
		FullTextwiseClassification[] classifications = new FullTextwiseClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullTextwiseClassification(id,true); 
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = classifications[m].cosineSimilarity(classifications[n]);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}	
	
	public void fillTextwiseJS() {
		//get clasifications
		System.out.println("Filling from Textwise classifications");
		FullTextwiseClassification[] classifications = new FullTextwiseClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullTextwiseClassification(id,true); 
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = classifications[m].jsDivergence(classifications[n]);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}	
	
	public void fillLIWC(boolean naiveBayes) {
		System.out.println("Filling from LIWC classifications, NB="+naiveBayes);
		FullLIWCClassification[] classifications = new FullLIWCClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullLIWCClassification(naiveBayes,id);
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			System.out.println("On row "+m);
			for(int n=m; n<d; n++) {
				Double cos = classifications[m].cosineSimilarity(classifications[n]);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}
	
	public void fillLLDA(String topicType, double alpha) {
		System.out.println("Filling from LLDA-inferred "+topicType+" classifications");
		FullLLDAClassification[] classifications = new FullLLDAClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullLLDAClassification(topicType,alpha,id);
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = classifications[m].cosineSimilarity(classifications[n]);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}
	
	public void fillLLDA(String topicType, double alpha, boolean fewerProfiles, int reduction) {
		//System.out.println("Filling from LLDA-inferred "+topicType+" classifications");
		FullLLDAClassification[] classifications = new FullLLDAClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullLLDAClassification(topicType,alpha,fewerProfiles,reduction,id);
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			//System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = classifications[m].cosineSimilarity(classifications[n]);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}
	
	public void fillKLLDA(String topicType, double alpha, int k) {
		//System.out.println("Filling from LLDA-inferred "+topicType+" classifications");
		FullLLDAClassification[] classifications = new FullLLDAClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullLLDAClassification(topicType,alpha,id);
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			//System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = CosineManager.cosineKSimilarity(classifications[m],classifications[n],k);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}
	
	public void fillLLDAJS(String topicType, double alpha) {
		System.out.println("Filling from LLDA-inferred "+topicType+" classifications");
		FullLLDAClassification[] classifications = new FullLLDAClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullLLDAClassification(topicType,alpha,id);
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = classifications[m].jsDivergence(classifications[n]);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}
	
	public void fillSVM(String topicType) {
		System.out.println("Filling from SVM "+topicType+" classifications");
		FullSVMClassification[] classifications = new FullSVMClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullSVMClassification(topicType,id);
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = classifications[m].cosineSimilarity(classifications[n]);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}
	
	public static void lldaMatrixCreation() {
		//String[] topicTypes = {"alchemy","calais","textwise"};
		String[] topicTypes = {"textwise"};
		double[] alphas = {0.25,0.5,0.75,1.0,1.25,1.5,1.75,2.0};
		for(String topicType : topicTypes) {
			for(double alpha : alphas) {
				System.out.println("Creating SM for "+topicType+" "+alpha);
				SimilarityMatrix sm = new SimilarityMatrix(2506);
				sm.fillLLDA(topicType, alpha);
				sm.save("llda-"+topicType+"-"+alpha);
			}
		}
	}
	
	public static void ldaMatrixCreation() {
		double[] alphas = {0.25,0.5,0.75,1.0,1.25,1.5,1.75,2.0};
		for(double alpha : alphas) {
			System.out.println("Creating SM for lda "+alpha);
			SimilarityMatrix sm = new SimilarityMatrix(2506);
			sm.fillLDAAndSave(50,1000,100,alpha);
		}
	}
	
	public void fillLDAAndSave(int numTopics, int burn, int sample, double alpha) {
		//get clasifications
		System.out.println("Filling from LDA classifications");
		FullLDAClassification[] classifications = new FullLDAClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullLDAClassification(id,numTopics,burn,sample,alpha); 
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = classifications[m].cosineSimilarity(classifications[n]);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
		
		//save with params
		String saveString = "lda-"+numTopics+"-"+burn+"-"+sample+"-"+alpha;
		save(saveString);
	}
	
	public void fillLDAJS(int numTopics, int burn, int sample, double alpha) {
		//get clasifications
		System.out.println("Filling from LDA classifications");
		FullLDAClassification[] classifications = new FullLDAClassification[d];
		for(long id : Tools.getCSVUserIDs()) {
			classifications[indexLookup.get(id)] = new FullLDAClassification(id,numTopics,burn,sample,alpha); 
		}
		
		//cosine similarities!
		for(int m=0; m<d; m++) {
			System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = classifications[m].jsDivergence(classifications[n]);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}
	
	public void fillRestricted(boolean baseline, String topicType, int topTopics, double alpha) {
		List<Map<String,Double>> classifications = new ArrayList<Map<String,Double>>();
		if(baseline) {
			if(topicType.equals("alchemy")) {
				for(long id : Tools.getCSVUserIDs()) {
					FullAlchemyClassification c = new FullAlchemyClassification(id);
					Map<String,Double> classification = new HashMap<String,Double>();
					int topicCount = 0;
					for(String topic : c.getCategorySet()) {
						if(topicCount == topTopics) break;
						classification.put(topic, c.getScore(topic));
						topicCount++;
					}
					classifications.add(classification);
				}
			} else if(topicType.equals("calais")) {
				for(long id : Tools.getCSVUserIDs()) {
					FullCalaisClassification c = new FullCalaisClassification(id);
					Map<String,Double> classification = new HashMap<String,Double>();
					int topicCount = 0;
					for(String topic : c.getCategorySet()) {
						if(topicCount == topTopics) break;
						if(topic.equals("Other")) continue;
						classification.put(topic, c.getScore(topic));
						topicCount++;
					}
					classifications.add(classification);
				}
			} else if(topicType.equals("textwise")) {
				for(long id : Tools.getCSVUserIDs()) {
					FullTextwiseClassification c = new FullTextwiseClassification(id,true);
					Map<String,Double> classification = new HashMap<String,Double>();
					int topicCount = 0;
					for(String topic : c.getCategorySet()) {
						if(topicCount == topTopics) break;
						classification.put(topic, c.getScore(topic));
						topicCount++;
					}
					classifications.add(classification);
				}
			} 
		} else {
			for(long id : Tools.getCSVUserIDs()) {
				FullLLDAClassification c = new FullLLDAClassification(topicType,alpha,id);
				Map<String,Double> classification = new HashMap<String,Double>();
				int topicCount = 0;
				for(String topic : c.getCategorySet()) {
					if(topicCount == topTopics) break;
					if(topic.equals("Other")) continue;
					classification.put(topic, c.getScore(topic));
					topicCount++;
				}
				classifications.add(classification);
			}
		}
		
		for(int m=0; m<d; m++) {
			//System.out.println("On row "+m);
			for(int n=m; n<d; n++) { //no point working eveyrthing out twice!
				Double cos = CosineManager.cosineSimilarity(classifications.get(m), classifications.get(n));
				//System.out.println("Similarity Found: "+cos);
				sim[m][n] = cos;
				sim[n][m] = cos;
			}
		}
	}
	
	public void print() {
		System.out.println("Printing similarity matrix");
		for(int m=0; m<d; m++) {
			for(int n=0; n<d; n++) {
				Tools.dpPrint(sim[m][n], 2);
				System.out.print(" ");
			}
			System.out.println();
		}
	}
	
    public void save(String name) {
		try {
			String filename = "smatrices/"+name+".smatrix";
			FileOutputStream fileOut = new FileOutputStream(filename);
			ObjectOutputStream objectOut = new ObjectOutputStream(fileOut);
			objectOut.writeObject(this);
			objectOut.close();
			System.out.println("Saved smatrix "+name);
		} catch (FileNotFoundException e) {
			System.out.println("Couldn't save smatrix "+name);
			e.printStackTrace();
		} catch (IOException e) {
			System.out.println("Couldn't save smatrix "+name);
			e.printStackTrace();			
		}    	
    }
    
    public static SimilarityMatrix load(String name) {
		try {
			String filename = "smatrices/"+name+".smatrix";
			FileInputStream fileIn = new FileInputStream(filename);
			ObjectInputStream objectIn = new ObjectInputStream(fileIn);
			SimilarityMatrix smatrix = (SimilarityMatrix)objectIn.readObject();
			objectIn.close();
			//System.out.println("Loaded smatrix "+name);
			return smatrix;
		} catch (FileNotFoundException e) {
			System.out.println("Couldn't load smatrix "+name);
			e.printStackTrace();
		} catch (IOException e) {
			System.out.println("Couldn't load smatrix "+name);
			e.printStackTrace();			
		} catch (ClassNotFoundException e) {
			System.out.println("Couldn't load smatrix "+name);
			e.printStackTrace();			
		}
		return null;
    }
    
    public double getID(long m, long n) {
    	return sim[indexLookup.get(m)][indexLookup.get(n)];
    }
    
    public double getIndex(int m, int n) {
    	return sim[m][n];
    }

}
 public class SpearmanRank {
	
	public static double srcc(SimilarityMatrix sm1, SimilarityMatrix sm2) {
		if(sm1.dimension() != sm2.dimension()) {
			System.err.println("Similarity Matrices not of same dimension");
			return -1.0;
		}
		List<Long> uids = Tools.getCSVUserIDs();
		
		//create baseline similarity pairs
		List<SimilarityPair> sps1 = new ArrayList<SimilarityPair>();
		List<SimilarityPair> sps2 = new ArrayList<SimilarityPair>();
		for(int uid1=0; uid1<uids.size(); uid1++) {
			for(int uid2=uid1; uid2<uids.size();uid2++) {
				SimilarityPair sp1 = new SimilarityPair(sm1,sm1.lookupID(uid1),sm2.lookupID(uid2));
				SimilarityPair sp2 = new SimilarityPair(sm2,sm1.lookupID(uid1),sm2.lookupID(uid2));
				if(sp1 == null || sp2 == null) {
					//no categories for one of them
					continue;
				} else {
					sps1.add(sp1);
					sps2.add(sp2);
				}
			}
		}
		
		//sort both pairing lists
		Collections.sort(sps1);
		Collections.sort(sps2);
		Collections.reverse(sps1);
		Collections.reverse(sps2);
		
		//For efficiency: need a map of pairs to ranks
		//this makes it O(n), rather than O(n^2)
		Map<SimilarityPair,Integer> rankMap = new HashMap<SimilarityPair,Integer>();
		for(int i=0; i<sps1.size(); i++) {
			//lose the 0-sims, they lend nothing
			if(sps1.get(i).similarity()>0.0) {
				rankMap.put(sps1.get(i),i);
			}
		}
		
		double n = rankMap.keySet().size();
		double rhoInc = 6.0 / (n*(n*n-1));
		double rho = 1.0;

		for(int i=0; i<sps2.size(); i++) {
			//if(i%10000==0) System.out.println("i="+i+", rho="+rho);
			if(rankMap.containsKey(sps2.get(i))) {
				int diff = rankMap.get(sps2.get(i))-i;
				rho -= rhoInc*diff*diff;
				//System.out.println("Pair "+sps2.get(i).uid1()+","+sps2.get(i).uid2()+":"+sps1.get(i).similarity()+" at rank "+i+" appears at rank "+(diff+i)+" in other list");
			}
		}
		
		return rho;
	}
	
	public static double jscSRCC(SimilarityMatrix sm1, SimilarityMatrix sm2) {
		if(sm1.dimension() != sm2.dimension()) {
			System.err.println("Similarity Matrices not of same dimension");
			return -1.0;
		}
		List<Long> uids = Tools.getCSVUserIDs();
		
		int numPairings = (sm1.dimension()*(sm1.dimension()+1)/2);
		double[] pairedDataX = new double[numPairings];
		double[] pairedDataY = new double[numPairings];
		int pairingIndex = 0;
		for(int i=0; i<uids.size(); i++) {
			for(int j=i; j<uids.size(); j++) {
				pairedDataX[pairingIndex] = sm1.getID(uids.get(i), uids.get(j));
				pairedDataY[pairingIndex] = sm2.getID(uids.get(i), uids.get(j));
				pairingIndex++;
			}
		}
		PairedData pairedData = new PairedData(pairedDataX,pairedDataY);
		//System.out.println("now running sc");
		//SpearmanCorrelation sc = new SpearmanCorrelation(pairedData);
		//return sc.getR();
		return LinearCorrelation.correlationCoeff(pairedData);
		
		/*
		//create baseline similarity pairs
		List<SimilarityPair> sps1 = new ArrayList<SimilarityPair>();
		List<SimilarityPair> sps2 = new ArrayList<SimilarityPair>();
		for(int uid1=0; uid1<uids.size(); uid1++) {
			for(int uid2=uid1; uid2<uids.size();uid2++) {
				SimilarityPair sp1 = new SimilarityPair(sm1,sm1.lookupID(uid1),sm2.lookupID(uid2));
				SimilarityPair sp2 = new SimilarityPair(sm2,sm1.lookupID(uid1),sm2.lookupID(uid2));
				if(sp1 == null || sp2 == null) {
					//no categories for one of them
					continue;
				} else {
					sps1.add(sp1);
					sps2.add(sp2);
				}
			}
		}
		
		//sort both pairing lists
		Collections.sort(sps1);
		Collections.sort(sps2);
		Collections.reverse(sps1);
		Collections.reverse(sps2);
		
		Map<SimilarityPair,Double> rankMap1 = new HashMap<SimilarityPair,Double>();
		Map<SimilarityPair,Double> rankMap2 = new HashMap<SimilarityPair,Double>();
		//rank first list
		for(int i=0; i<sps1.size(); i++) {
			double similarity = sps1.get(i).similarity();
			
			//check for tied values
			int j = i+1;
			int count = 0;
			while(j < sps1.size() && sps1.get(j).similarity()==similarity) {
				count++;
				j++;
			}
			
			if(count == 0) {
				rankMap1.put(sps1.get(i),(double)i);
			} else {
				//work out the average rank, give it to all tied values
				double newRank = (i+j)/2.0;
				for(int k=i; k<j; k++) {
					rankMap1.put(sps1.get(k), newRank);
				}
				i=j-1;
			}
		}
		//rank second list
		for(int i=0; i<sps2.size(); i++) {
			double similarity = sps2.get(i).similarity();
			
			//check for tied values
			int j = i+1;
			int count = 0;
			while(j < sps2.size() && sps2.get(j).similarity()==similarity) {
				count++;
				j++;
			}
			
			if(count == 0) {
				rankMap2.put(sps2.get(i),(double)i);
			} else {
				//work out the average rank, give it to all tied values
				double newRank = (i+j)/2.0;
				for(int k=i; k<j; k++) {
					rankMap2.put(sps2.get(k), newRank);
				}
				i=j-1;
			}
		}
		*/
	}

} public class FullLLDAClassification {

	private long userID;
	private Map<String,Double> classifications; 
	
	public FullLLDAClassification(String topicType, double alpha, long userID) {
		this.userID = userID;
		//Construct the path to the classification file
		String path = "classifications/llda/"+topicType+"/1000-100-"+alpha;
		path += "/"+userID+".csv";
		
		classifications = new HashMap<String,Double>();
		
		try {
			FileInputStream fstream = new FileInputStream(path);
			DataInputStream in = new DataInputStream(fstream);
			BufferedReader br = new BufferedReader(new InputStreamReader(in));
			String strLine;
			br.readLine(); //Skp the CSV descriptor
			while ((strLine = br.readLine()) != null)   {
				//no quotes, damn
				int splitIndex = strLine.lastIndexOf(",");
				String cat = strLine.substring(0,splitIndex);
				double score = Double.parseDouble(strLine.substring(splitIndex+1,strLine.length()));
				classifications.put(cat,score);
			}
			in.close();
		} catch (Exception e) {
			//System.err.println("Error: " + e.getMessage());
		}
		classifications = Tools.sortMapByValueDesc(classifications);
	}
	
	public FullLLDAClassification(String topicType, double alpha, boolean fewerProfiles, int reduction, long userID) {
		this.userID = userID;
		//Construct the path to the classification file
		String path = "";
		if(fewerProfiles) path = "classifications/fewerprofiles/"+reduction+"/llda/"+topicType+"/1000-100-"+alpha;
		else path = "classifications/fewertweets/"+reduction+"/llda/"+topicType+"/1000-100-"+alpha;
		path += "/"+userID+".csv";
		
		classifications = new HashMap<String,Double>();
		
		try {
			FileInputStream fstream = new FileInputStream(path);
			DataInputStream in = new DataInputStream(fstream);
			BufferedReader br = new BufferedReader(new InputStreamReader(in));
			String strLine;
			br.readLine(); //Skp the CSV descriptor
			while ((strLine = br.readLine()) != null)   {
				//no quotes, damn
				int splitIndex = strLine.lastIndexOf(",");
				String cat = strLine.substring(0,splitIndex);
				double score = Double.parseDouble(strLine.substring(splitIndex+1,strLine.length()));
				classifications.put(cat,score);
			}
			in.close();
		} catch (Exception e) {
			//System.err.println("Error: " + e.getMessage());
		}
		classifications = Tools.sortMapByValueDesc(classifications);
	}
	
	public void print() {
		for(String cat : classifications.keySet()) {
			System.out.println(cat+": "+classifications.get(cat));
		}
	}
	
	public Set<String> getCategorySet() {
		return classifications.keySet();
	}

	
	public boolean hasCategory(String cat) {
		if(classifications.keySet().contains(cat)) return true;
		else return false;
	}
	
	public double getScore(String cat) {
		return classifications.get(cat);
	}
	
	public double magnitude() {
		double sum = 0.0;
		for(String cat : classifications.keySet()) {
			sum += (classifications.get(cat)*classifications.get(cat));
		}
		sum = Math.sqrt(sum);
		return sum;
	}

	public double cosineSimilarity(FullLLDAClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		
		double score = 0.0;
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				score += (this.getScore(cat) * c.getScore(cat));
			}
		}
		
		//normalise by magnitudes
		Double magnitudes = (this.magnitude() * c.magnitude());
		score /= magnitudes;
		if(Double.isNaN(score)) {
			return 0.0; //NaN caused by zero vectors ie no classifications!
		}
		else return score;
	}
	
	//horrible hacky duplicates coming up
	public double cosineSimilarity(FullAlchemyClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		
		double score = 0.0;
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				score += (this.getScore(cat) * c.getScore(cat));
			}
		}
		
		//normalise by magnitudes
		Double magnitudes = (this.magnitude() * c.magnitude());
		score /= magnitudes;
		if(Double.isNaN(score)) {
			return 0.0; //NaN caused by zero vectors ie no classifications!
		}
		else return score;
	}
	
	public double cosineSimilarity(FullCalaisClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		
		double score = 0.0;
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				score += (this.getScore(cat) * c.getScore(cat));
			}
		}
		
		//normalise by magnitudes
		Double magnitudes = (this.magnitude() * c.magnitude());
		score /= magnitudes;
		if(Double.isNaN(score)) {
			return 0.0; //NaN caused by zero vectors ie no classifications!
		}
		else return score;
	}
	
	public double cosineSimilarity(FullTextwiseClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		
		double score = 0.0;
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				score += (this.getScore(cat) * c.getScore(cat));
			}
		}
		
		//normalise by magnitudes
		Double magnitudes = (this.magnitude() * c.magnitude());
		score /= magnitudes;
		if(Double.isNaN(score)) {
			return 0.0; //NaN caused by zero vectors ie no classifications!
		}
		else return score;
	}
	//end of hacky duplications
	
	public double jsDivergence(FullLLDAClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		Map<String,Double> M = new HashMap<String,Double>();
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				M.put(cat, (this.getScore(cat)+c.getScore(cat))/2.0);
			}
		}
		double d1 = 0.0;
		for(String cat : M.keySet()) {
			if(this.getCategorySet().contains(cat) ) {
				d1 += this.getScore(cat) * Math.log(this.getScore(cat)/M.get(cat));
			}
		}	
		double d2 = 0.0;
		for(String cat : M.keySet()) {
			if(c.getCategorySet().contains(cat)) {
				d1 += c.getScore(cat) * Math.log(c.getScore(cat)/M.get(cat));
			}
		}		
		double score = d1/2.0 + d2/2.0;
		return score;
	}
	
	public double jsDivergence(FullAlchemyClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		Map<String,Double> M = new HashMap<String,Double>();
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				M.put(cat, (this.getScore(cat)+c.getScore(cat))/2.0);
			}
		}
		double d1 = 0.0;
		for(String cat : M.keySet()) {
			if(this.getCategorySet().contains(cat) ) {
				d1 += this.getScore(cat) * Math.log(this.getScore(cat)/M.get(cat));
			}
		}	
		double d2 = 0.0;
		for(String cat : M.keySet()) {
			if(c.getCategorySet().contains(cat)) {
				d1 += c.getScore(cat) * Math.log(c.getScore(cat)/M.get(cat));
			}
		}		
		double score = d1/2.0 + d2/2.0;
		return score;
	}
	
	public double jsDivergence(FullCalaisClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		Map<String,Double> M = new HashMap<String,Double>();
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				M.put(cat, (this.getScore(cat)+c.getScore(cat))/2.0);
			}
		}
		double d1 = 0.0;
		for(String cat : M.keySet()) {
			if(this.getCategorySet().contains(cat) ) {
				d1 += this.getScore(cat) * Math.log(this.getScore(cat)/M.get(cat));
			}
		}	
		double d2 = 0.0;
		for(String cat : M.keySet()) {
			if(c.getCategorySet().contains(cat)) {
				d1 += c.getScore(cat) * Math.log(c.getScore(cat)/M.get(cat));
			}
		}		
		double score = d1/2.0 + d2/2.0;
		return score;
	}
	
	public double jsDivergence(FullTextwiseClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		Map<String,Double> M = new HashMap<String,Double>();
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				M.put(cat, (this.getScore(cat)+c.getScore(cat))/2.0);
			}
		}
		double d1 = 0.0;
		for(String cat : M.keySet()) {
			if(this.getCategorySet().contains(cat) ) {
				d1 += this.getScore(cat) * Math.log(this.getScore(cat)/M.get(cat));
			}
		}	
		double d2 = 0.0;
		for(String cat : M.keySet()) {
			if(c.getCategorySet().contains(cat)) {
				d1 += c.getScore(cat) * Math.log(c.getScore(cat)/M.get(cat));
			}
		}		
		double score = d1/2.0 + d2/2.0;
		return score;
	}
	
}
 public class FullSVMClassification {
	private long userID;
	private Map<String,Double> classifications; 
	
	public FullSVMClassification(String topicType, long userID) {
		this.userID = userID;
		//Construct the path to the classification file
		String path = "classifications/svm/"+topicType+"/"+userID+".csv";
		
		classifications = new HashMap<String,Double>();
		
		try {
			FileInputStream fstream = new FileInputStream(path);
			DataInputStream in = new DataInputStream(fstream);
			BufferedReader br = new BufferedReader(new InputStreamReader(in));
			String strLine;
			br.readLine(); //Skp the CSV descriptor
			while ((strLine = br.readLine()) != null)   {
				//no quotes, damn
				int splitIndex = strLine.lastIndexOf(",");
				String cat = strLine.substring(0,splitIndex);
				double score = Double.parseDouble(strLine.substring(splitIndex+1,strLine.length()));
				classifications.put(cat,score);
			}
			in.close();
		} catch (Exception e) {
			//System.err.println("Error: " + e.getMessage());
		}
		classifications = Tools.sortMapByValueDesc(classifications);
		
		//normalisation - make it so that after correction, lowest topic will be 0.0
		//is this fair?
		//could only normalise positive values...
		double magSum = 0.0;
		for(String topic : classifications.keySet()) {
			magSum += Math.abs(classifications.get(topic));
		}
		double min = Double.MAX_VALUE;
		for(String topic : classifications.keySet()) {
			double newScore = classifications.get(topic)/magSum;
			if(newScore<min) min = newScore;
			classifications.put(topic, newScore);
		}
		if(min<0) {
			for(String topic : classifications.keySet()) {
				classifications.put(topic, classifications.get(topic)+Math.abs(min));
			}
		}
	}
	
	public void print() {
		for(String cat : classifications.keySet()) {
			System.out.println(cat+": "+classifications.get(cat));
		}
	}
	
	public Set<String> getCategorySet() {
		return classifications.keySet();
	}

	
	public boolean hasCategory(String cat) {
		if(classifications.keySet().contains(cat)) return true;
		else return false;
	}
	
	public double getScore(String cat) {
		return classifications.get(cat);
	}
	
	public double magnitude() {
		double sum = 0.0;
		for(String cat : classifications.keySet()) {
			sum += (classifications.get(cat)*classifications.get(cat));
		}
		sum = Math.sqrt(sum);
		return sum;
	}

	public double cosineSimilarity(FullSVMClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		
		double score = 0.0;
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				score += (this.getScore(cat) * c.getScore(cat));
			}
		}
		
		//normalise by magnitudes
		Double magnitudes = (this.magnitude() * c.magnitude());
		score /= magnitudes;
		if(Double.isNaN(score)) {
			return 0.0; //NaN caused by zero vectors ie no classifications!
		}
		else return score;
	}
	
	public double cosineSimilarity(FullLLDAClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		
		double score = 0.0;
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				score += (this.getScore(cat) * c.getScore(cat));
			}
		}
		
		//normalise by magnitudes
		Double magnitudes = (this.magnitude() * c.magnitude());
		score /= magnitudes;
		if(Double.isNaN(score)) {
			return 0.0; //NaN caused by zero vectors ie no classifications!
		}
		else return score;
	}
	
	//horrible hacky duplicates coming up
	public double cosineSimilarity(FullAlchemyClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		
		double score = 0.0;
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				score += (this.getScore(cat) * c.getScore(cat));
			}
		}
		
		//normalise by magnitudes
		Double magnitudes = (this.magnitude() * c.magnitude());
		score /= magnitudes;
		if(Double.isNaN(score)) {
			return 0.0; //NaN caused by zero vectors ie no classifications!
		}
		else return score;
	}
	
	public double cosineSimilarity(FullCalaisClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		
		double score = 0.0;
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				score += (this.getScore(cat) * c.getScore(cat));
			}
		}
		
		//normalise by magnitudes
		Double magnitudes = (this.magnitude() * c.magnitude());
		score /= magnitudes;
		if(Double.isNaN(score)) {
			return 0.0; //NaN caused by zero vectors ie no classifications!
		}
		else return score;
	}
	
	public double cosineSimilarity(FullTextwiseClassification c) {
		Set<String> catSet = new HashSet<String>();
		catSet.addAll(classifications.keySet());
		catSet.addAll(c.getCategorySet());
		
		double score = 0.0;
		for(String cat : catSet) {
			if(this.hasCategory(cat) && c.hasCategory(cat)) {
				score += (this.getScore(cat) * c.getScore(cat));
			}
		}
		
		//normalise by magnitudes
		Double magnitudes = (this.magnitude() * c.magnitude());
		score /= magnitudes;
		if(Double.isNaN(score)) {
			return 0.0; //NaN caused by zero vectors ie no classifications!
		}
		else return score;
	}
	//end of hacky duplications
	
} public class LDATopicModel implements Serializable {

	private static final long serialVersionUID = 7094157759480964151L;
	
	private Corpus corpus;
	private boolean hasRun;
	private int[][] documents;
	private int numTopics;
	private int numIterations;
	private int numSamplingIterations;
	private int numTotalIterations;
	private int samplingLag;
	private Map<String,Integer> wordIDs;
	private ArrayList<String> idLookup;
	private ArrayList<Document> docIDLookup;
	private int numWords;
	private int numDocs;
	private int numStats;
	private double alpha;
	private double beta;
	private int[][] wordTopicAssignments;
	private int[][] docTopicAssignments;
	private int[] numWordsAssignedToTopic;
	private int[] numWordsInDocument;
	private double[][] thetaSum;
	private double[][] phiSum;
	private boolean phiSumNormalised;
	private int[][] topicAssignments; //z
	private boolean printEachIteration;
	
	public LDATopicModel(Corpus corpus, int numTopics, int numIterations, int numSamplingIterations, int samplingLag, double alpha, double beta) {
		this.corpus = corpus;
		this.numTopics = numTopics;
		this.numIterations = numIterations;
		this.numSamplingIterations = numSamplingIterations;
		this.samplingLag = samplingLag;
		this.alpha = alpha;
		this.beta = beta;
		printEachIteration = false;
		numTotalIterations = numIterations;
		if(samplingLag > 0) numTotalIterations += numSamplingIterations * samplingLag;
		else numTotalIterations += numSamplingIterations;
		init();
	}
	
	public void printEachIteration() {
		printEachIteration = true;
	}
	
	//Here, we could remove the least used words instead of giving them an ID - lengths would need consideration
	public void init() {
		hasRun = false;
		Set<Document> documentSet = corpus.getDocuments();
		numDocs = documentSet.size();	
		documents = new int[numDocs][];
		numWordsInDocument = new int[numDocs];
		wordIDs = new HashMap<String,Integer>();
		idLookup = new ArrayList<String>();
		docIDLookup = new ArrayList<Document>();
		int docID = 0;
		for(Document document : documentSet) {
			docIDLookup.add(document);
			String[] tokens = document.getDocumentString().split("\\s+");
			documents[docID] = new int[tokens.length];
			numWordsInDocument[docID] = tokens.length;
			for(int i=0; i<documents[docID].length; i++) {
				//Add the token's ID to the documents array
				int wordID;
				if(wordIDs.containsKey(tokens[i])) { 
					wordID = wordIDs.get(tokens[i]);
				} else {
					wordID = wordIDs.keySet().size();
					wordIDs.put(tokens[i], wordID);
					idLookup.add(tokens[i]);
				}
				documents[docID][i] = wordID; 
			}
			docID++;
		}
		numWords = wordIDs.keySet().size();

		//Now for random initial topic assignment
		wordTopicAssignments = new int[numWords][numTopics];
		docTopicAssignments = new int[numDocs][numTopics];
		numWordsAssignedToTopic = new int[numTopics];
		topicAssignments = new int[numDocs][];
		for(int m=0; m<documents.length; m++) {		
			topicAssignments[m] = new int[documents[m].length];
			for(int n=0; n<documents[m].length; n++) {
				//Generate a random topic and update arrays
				int topicID = (int)(Math.random()*numTopics);
				topicAssignments[m][n] = topicID;
				wordTopicAssignments[documents[m][n]][topicID]++;
				docTopicAssignments[m][topicID]++;
				numWordsAssignedToTopic[topicID]++;	
			}
		}
		
		//Initialise topic and word distributions for later		
		thetaSum = new double[numDocs][numTopics];
		phiSum = new double[numTopics][numWords];
		phiSumNormalised = false;
		numStats = 0;
	}
	
	public void runGibbsSampling() {	
		hasRun = true;
		System.out.println("Starting Gibbs sampling");
		System.out.println("Documents: "+numDocs+" docs");
		System.out.println("Words: "+numWords+" unique words");
		System.out.println("Topics: "+numTopics+" topics");
		System.out.println("Iterations: "+numTotalIterations);
		
		for(int i=0; i<numTotalIterations; i++) {
			//if(i % 50 == 0) 
			System.out.println("Starting iteration "+i);
			for(int m=0; m<topicAssignments.length; m++) { //m is document index
				for(int n=0; n<topicAssignments[m].length; n++) { //n is word index
					//Get an updated topic sample for this word
					topicAssignments[m][n] = sampleTopic(m, n);
				}
			}
					
			if(i >= numIterations && (samplingLag == 0 || i % samplingLag == 0)) {
				updatePhiThetaSums();
				
				if(printEachIteration) {
					if(i % 20 == 0)	iterationPrintTopicsNew(5);
				}
				
				//try{System.in.read();}catch(IOException e){}
			}
		}
	}
	
	//Sample a new topic from the multinomial topic distribution
	private int sampleTopic(int m, int n) {
		//Removes this iteration's topicAssignment from the counting variables
		int topicID = topicAssignments[m][n]; //Get the currently assigned topic
		wordTopicAssignments[documents[m][n]][topicID]--; //Decrement the topic count for the current word
		docTopicAssignments[m][topicID]--; //Decrement the topic count for the current document
		numWordsAssignedToTopic[topicID]--; //Decrement the number of words the current topic has
		numWordsInDocument[m]--;
		
		//Cumulative multinomial sampling
		double[] p = new double[numTopics];
		for(int k=0; k<numTopics; k++) {
			p[k] = (wordTopicAssignments[documents[m][n]][k] + beta) / (numWordsAssignedToTopic[k] + numWords * beta) * (docTopicAssignments[m][k] + alpha) / (numWordsInDocument[m] + numTopics * alpha);
		}
		
		//Cumulative part - could be combined into above part cleverly
		for(int k=1; k<numTopics; k++) {
			p[k] += p[k-1];
		}
		
		//Sampling part - scaled because we haven't normalised
		double topicThreshold = Math.random() * p[numTopics-1];
		int sampledTopicID = 0;
		for(sampledTopicID=0; sampledTopicID<numTopics; sampledTopicID++) {
			if(topicThreshold < p[sampledTopicID]) {
				break;
			}
		}
		
		//Maybe a fix needed by faulty double arithmetic
		if(sampledTopicID >= numTopics) {
			sampledTopicID = numTopics-1;
		}
		
		//Finally, increment the relevant count variables
		wordTopicAssignments[documents[m][n]][sampledTopicID]++;
		docTopicAssignments[m][sampledTopicID]++;
		numWordsAssignedToTopic[sampledTopicID]++;
		numWordsInDocument[m]++;
		
		return sampledTopicID;
	}
	
	//By eqs 82 and 83 of paper mentioned in topmost comment
	private void updatePhiThetaSums() {
		
		for(int docID=0; docID<numDocs; docID++) {
			for(int topicID=0; topicID<numTopics; topicID++) {
				thetaSum[docID][topicID] += (docTopicAssignments[docID][topicID] + alpha) / (numWordsInDocument[docID] + numTopics * alpha);
			}
		}
		
		for(int topicID=0; topicID<numTopics; topicID++) {
			for(int wordID=0; wordID<numWords; wordID++) {
				phiSum[topicID][wordID] += (wordTopicAssignments[wordID][topicID] + beta) / (numWordsAssignedToTopic[topicID] + numWords * beta) ;
			}
		}
		numStats++; //Used if we want to take the mean of many stats
	}
	
	private double[][] getTheta() {
		double[][] theta = new double[numDocs][numTopics];
		for(int docID=0; docID<numDocs; docID++) {
			for(int topicID=0; topicID<numTopics; topicID++) {
				if(numStats > 0) theta[docID][topicID] = thetaSum[docID][topicID] / numStats; 
				else theta[docID][topicID] = (docTopicAssignments[docID][topicID] + alpha) / (numWordsInDocument[docID] + numTopics * alpha);
			}
		}
		return theta;
	}
	
	private List<Map<Integer,Double>> getThetaMap() {
		List<Map<Integer,Double>> docList = new LinkedList<Map<Integer,Double>>();
		for(int docID=0; docID<numDocs; docID++) {
			Map<Integer,Double> topicMap = new HashMap<Integer,Double>();
			for(int topicID=0; topicID<numTopics; topicID++) {
				topicMap.put(topicID, thetaSum[docID][topicID] / numStats);  
			}
			docList.add(Tools.sortMapByValue(topicMap));
		}
		return docList;
	}
	
	private double[][] getPhi() {
		double[][] phi = new double[numTopics][numWords];
		for(int topicID=0; topicID<numTopics; topicID++) {
			for(int wordID=0; wordID<numWords; wordID++) {
				if(numStats > 0) phi[topicID][wordID] = phiSum[topicID][wordID] / numStats; 
				else phi[topicID][wordID] += (wordTopicAssignments[wordID][topicID] + beta) / (numWordsAssignedToTopic[topicID] + numWords * beta) ;
			}
		}
		return phi;
	}
	
	private List<Map<Integer,Double>> getPhiMap() {
		List<Map<Integer,Double>> topicList = new LinkedList<Map<Integer,Double>>();
		for(int topicID=0; topicID<numTopics; topicID++) {
			Map<Integer,Double> wordMap = new HashMap<Integer,Double>();
			for(int wordID=0; wordID<numWords; wordID++) {
				wordMap.put(wordID, phiSum[topicID][wordID] / numStats);  
			}
			topicList.add(Tools.sortMapByValue(wordMap));
		}
		return topicList;
	}
	
	private double[][] normalisePhiSum() {
		if(phiSumNormalised) return phiSum;
		phiSumNormalised = true;
		for(int topicID=0; topicID<numTopics; topicID++) {
			for(int wordID=0; wordID<numWords; wordID++) {
				phiSum[topicID][wordID] /= numStats; 
			}
		}
		return phiSum;
	}
	
	public List<List<WordScore>> getTopics() {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return null;
		}
		
		List<List<WordScore>> topics = new LinkedList<List<WordScore>>();
		double[][] phi = getPhi();
		for(int topicID=0; topicID<numTopics; topicID++) {
			double[] wordProbs = phi[topicID];
			List<WordScore> wordScores = new LinkedList<WordScore>();
			for(int wordID=0; wordID<numWords; wordID++) {
				wordScores.add(new WordScore(idLookup.get(wordID), wordProbs[wordID]));
			}
			Collections.sort(wordScores);
			Collections.reverse(wordScores);
			topics.add(wordScores);
		}
		
		return topics;
	}
		
	public List<Map<Integer,Double>> getTopicsNew() {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return null;
		}
		
		List<Map<Integer,Double>> topics = new ArrayList<Map<Integer,Double>>();
		double[][] phi = normalisePhiSum();
		for(int topicID=0; topicID<numTopics; topicID++) {
			double[] wordProbs = phi[topicID];
			Map<Integer,Double> wordScores = new HashMap<Integer,Double>();
			for(int wordID=0; wordID<numWords; wordID++) {
				wordScores.put(wordID, wordProbs[wordID]);
			}
			Tools.sortMapByValueDesc(wordScores);
			topics.add(wordScores);
		}
		return topics;
	}
	
	public double[][] getTopicsUnsorted() {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return null;
		}
		
		double[][] phi = normalisePhiSum();
		return phi; //Hmmm...
	}
	
	public List<List<WordScore>> getDocuments() {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return null;
		}
		
		List<List<WordScore>> topics = new LinkedList<List<WordScore>>();
		double[][] theta = getTheta();
		for(int docID=0; docID<numDocs; docID++) {
			double[] topicProbs = theta[docID];
			List<WordScore> wordScores = new LinkedList<WordScore>();
			for(int topicID=0; topicID<numTopics; topicID++) {
				wordScores.add(new WordScore("Topic"+topicID, topicProbs[topicID]));
			}
			Collections.sort(wordScores);
			Collections.reverse(wordScores);
			topics.add(wordScores);
		}
		
		return topics;
	}
	
	public void printTopics(int topWords) {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return;
		}
		
		List<List<WordScore>> topics = getTopics();
		for(int k=0; k<numTopics; k++) {
			System.out.print("Topic "+k+":");
			List<WordScore> words = topics.get(k);
			for(int n=0; n<topWords; n++) {
				if(n == numWords) break; //incase topWords is huge or Corpus is tiny...
				//System.out.println(words.get(n).getWord()+" = "+words.get(n).getScore());
				System.out.print(words.get(n).getWord()+" ");
			}
			System.out.println();
		}
	}
	
	//Note - this is destructive as it calls normalisePhiSum
	public void printTopicsNew(int topWords) {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return;
		}
		
		double[][] phi;
		if(phiSumNormalised) phi = phiSum;
		else phi = normalisePhiSum();
		for(int topicID=0; topicID<numTopics; topicID++) {
			double[] wordProbs = phi[topicID];
			
			List<WordScore> wordScores = new LinkedList<WordScore>();
			for(int wordID=0; wordID<numWords; wordID++) {
				wordScores.add(new WordScore(idLookup.get(wordID), wordProbs[wordID]));
			}
			Collections.sort(wordScores);
			Collections.reverse(wordScores);
			
			System.out.print("Topic "+topicID+": ");
			int count=0;
			for(WordScore score : wordScores) {
				if(count==topWords) break;
				System.out.print(score.getWord()+" ");
				count++;
			}
			System.out.println();
		}
	}
	
	public boolean isPhiSumNormalised() {
		return phiSumNormalised;
	}
	
	private void iterationPrintTopics(int topWords) {
		List<List<WordScore>> topics = getTopics();
		for(int k=0; k<numTopics; k++) {
			System.out.print("Topic "+k+": ");
			List<WordScore> words = topics.get(k);
			for(int n=0; n<topWords; n++) {
				if(n == numWords) break; //incase topWords is huge or Corpus is tiny...
				System.out.print(words.get(n).getWord()+" ");
			}
			System.out.println();
		}
	}
	
	private void iterationPrintTopicsNew(int topWords) {
		List<Map<Integer,Double>> phiMap = getPhiMap();
		int topicID = 0;
		for(Map<Integer,Double> words : phiMap) {
			System.out.print("Topic "+topicID+": ");
			int count = 0;
			for(int wordID : words.keySet()) {
				if(count == topWords) break;
				System.out.print(idLookup.get(wordID)+" ");
				count++;
			}
			System.out.println();
			topicID++;
		}
	}
	
	public void printDocuments(int topTopics) {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return;
		}
		
		List<List<WordScore>> docs = getDocuments();
		for(int d=0; d<numDocs; d++) {
			System.out.println("Document "+d+":");
			List<WordScore> topics = docs.get(d);
			for(int n=0; n<topTopics; n++) {
				if(n == numWords) break;
				System.out.println(topics.get(n).getWord()+" = "+topics.get(n).getScore());
			}
			try {System.in.read();} catch (IOException e) {}
		}
	}
	
	public void printDocumentsVerbose(int topTopics) {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return;
		}
		
		List<List<WordScore>> docs = getDocuments();
		for(int d=0; d<numDocs; d++) {
			List<WordScore> topics = docs.get(d);
			System.out.print("Document "+d+": ");
			for(int n=0; n<documents[d].length; n++) {
				System.out.print(idLookup.get(documents[d][n])+"["+topicAssignments[d][n]+"] ");
			}
			System.out.println();
			for(int n=0; n<topTopics; n++) {
				if(n == numWords) break;
				System.out.println(topics.get(n).getWord()+" = "+topics.get(n).getScore());
			}
			try {System.in.read();} catch (IOException e) {}
		}
	}
	
	public void print() {
		//TODO
	}

    public void save(String name) {
		try {
			String filename = "models/lda/"+name+".model";
			FileOutputStream fileOut = new FileOutputStream(filename);
			ObjectOutputStream objectOut = new ObjectOutputStream(fileOut);
			objectOut.writeObject(this);
			objectOut.close();
			System.out.println("Saved LDA topic model "+name);
		} catch (FileNotFoundException e) {
			System.out.println("Couldn't save LDA topic model "+name);
			e.printStackTrace();
		} catch (IOException e) {
			System.out.println("Couldn't save LDA topic model "+name);
			e.printStackTrace();			
		}    	
    }
    
    public static LDATopicModel load(String name) {
		try {
			String filename = "models/lda/"+name+".model";
			FileInputStream fileIn = new FileInputStream(filename);
			ObjectInputStream objectIn = new ObjectInputStream(fileIn);
			LDATopicModel model = (LDATopicModel)objectIn.readObject();
			objectIn.close();
			System.out.println("Loaded LDA topic model "+name);
			return model;
		} catch (FileNotFoundException e) {
			System.out.println("Couldn't load LDA topic model "+name);
			e.printStackTrace();
		} catch (IOException e) {
			System.out.println("Couldn't load LDA topic model "+name);
			e.printStackTrace();			
		} catch (ClassNotFoundException e) {
			System.out.println("Couldn't load LDA topic model "+name);
			e.printStackTrace();			
		}
		return null;
    }
    
    public long getDocIDFromIndex(int m) {
    	return docIDLookup.get(m).getId();
    }
    
    public Map<String,Integer> getVocab() {
    	return wordIDs;
    }
	
}
 public class LLDATopicModel implements Serializable{

	private static final long serialVersionUID = 6251334729243904933L;
	
	String topicType;
	private Corpus corpus;
	private boolean hasRun;
	private int[][] documents;
	private int numTopics;
	private int numIterations;
	private int numSamplingIterations;
	private int numTotalIterations;
	private int samplingLag;
	private Map<String,Integer> wordIDs;
	private Map<String,Integer> topicIDs;
	private ArrayList<String> idLookup;
	private ArrayList<String> topicLookup;
	private ArrayList<Document> docIDLookup;
	private ArrayList<ArrayList<Integer>> docLabels;
	private int numWords;
	private int numDocs;
	private int numStats;
	private double alpha;
	private double beta;
	private int[][] wordTopicAssignments;
	private int[][] docTopicAssignments;
	private int[] numWordsAssignedToTopic;
	private int[] numWordsInDocument;
	private int[] numTopicsInDocument;
	private double[][] thetaSum;
	private double[][] phiSum;
	private boolean phiSumNormalised;
	private int[][] topicAssignments; //z
	private boolean printEachIteration;
	
	private int threadNum; //hacky hacky hacky
	public LLDATopicModel(Corpus corpus, int numIterations, int numSamplingIterations, int samplingLag, double alpha, double beta, int threadNum) {
		this.threadNum = threadNum;
		topicType = corpus.getTopicType();
		this.corpus = corpus;
		this.numIterations = numIterations;
		this.numSamplingIterations = numSamplingIterations;
		this.samplingLag = samplingLag;
		this.alpha = alpha;
		this.beta = beta;
		printEachIteration = false;
		numTotalIterations = numIterations;
		if(samplingLag > 0) numTotalIterations += numSamplingIterations * samplingLag;
		else numTotalIterations += numSamplingIterations;
		init();
	}
	
	public LLDATopicModel(Corpus corpus, int numIterations, int numSamplingIterations, int samplingLag, double alpha, double beta) {
		topicType = corpus.getTopicType();
		this.corpus = corpus;
		this.numIterations = numIterations;
		this.numSamplingIterations = numSamplingIterations;
		this.samplingLag = samplingLag;
		this.alpha = alpha;
		this.beta = beta;
		printEachIteration = false;
		numTotalIterations = numIterations;
		if(samplingLag > 0) numTotalIterations += numSamplingIterations * samplingLag;
		else numTotalIterations += numSamplingIterations;
		init();
	}
	
	public void printEachIteration() {
		printEachIteration = true;
	}
	
	//Here, we could remove the least used words instead of giving them an ID - lengths would need consideration
	public void init() {
		hasRun = false;
		Set<Document> documentSet = corpus.getDocuments();
		
		//Remove all no-topic documents to avoid breaking LLDA - use an iterator to avoid exceptions when removing
		for(Iterator<Document> iter = documentSet.iterator(); iter.hasNext();) {
			Document document = iter.next();
			if(document.getTopics().isEmpty()) {
				iter.remove();
			}
		}
		
		numDocs = documentSet.size();	
		System.out.println("THREAD "+threadNum+": numDocs = "+numDocs);
		documents = new int[numDocs][];
		numWordsInDocument = new int[numDocs];
		numTopicsInDocument = new int[numDocs];
		wordIDs = new HashMap<String,Integer>();
		topicIDs = new HashMap<String,Integer>();
		idLookup = new ArrayList<String>();
		topicLookup = new ArrayList<String>();
		docIDLookup = new ArrayList<Document>();
		docLabels = new ArrayList<ArrayList<Integer>>();
		int docID = 0;
		for(Document document : documentSet) {
			docIDLookup.add(document);
			ArrayList<Integer> labels = new ArrayList<Integer>();
			for(String topic : document.getTopics()) {
				int topicID;
				if(topicIDs.containsKey(topic)) {
					topicID = topicIDs.get(topic); 
				} else {
					topicID = topicIDs.keySet().size();
					topicIDs.put(topic, topicID);
					topicLookup.add(topic);
				}
				labels.add(topicID);
			}
			docLabels.add(labels); //In correct position docID...
			numTopicsInDocument[docID] = document.getTopics().size();
			String[] tokens = document.getDocumentString().split("\\s+");
			documents[docID] = new int[tokens.length];
			numWordsInDocument[docID] = tokens.length;
			for(int i=0; i<documents[docID].length; i++) {
				//Add the token's ID to the documents array
				int wordID;
				if(wordIDs.containsKey(tokens[i])) { 
					wordID = wordIDs.get(tokens[i]);
				} else {
					wordID = wordIDs.keySet().size();
					wordIDs.put(tokens[i], wordID);
					idLookup.add(tokens[i]);
				}
				documents[docID][i] = wordID; 
			}
			docID++;
		}
		numWords = wordIDs.keySet().size();
		numTopics = topicIDs.keySet().size();
		
		//Now for random initial topic assignment
		wordTopicAssignments = new int[numWords][numTopics];
		docTopicAssignments = new int[numDocs][numTopics];
		numWordsAssignedToTopic = new int[numTopics];
		topicAssignments = new int[numDocs][];
		for(int m=0; m<documents.length; m++) {		
			topicAssignments[m] = new int[documents[m].length];
			for(int n=0; n<documents[m].length; n++) {
				//Generate a random topic and update arrays
				//NOTE: This is now constrained to only those topics in the document
				int topicIDIndex = (int)(Math.random()*numTopicsInDocument[m]);
				int topicID = -1;
				int labelCount = 0;
				for(Integer k : docLabels.get(m)) {
					if(labelCount == topicIDIndex) {
						topicID = k;
						break;
					} else {
						labelCount++;
					}
				}
				if(topicID == -1) {
					System.out.println("Something went wrong when choosing a random topic from the document's topic set - or, no topics");
					System.out.println(numTopicsInDocument[m]);
				}
				
				topicAssignments[m][n] = topicID;
				wordTopicAssignments[documents[m][n]][topicID]++;
				docTopicAssignments[m][topicID]++;
				numWordsAssignedToTopic[topicID]++;	
			}
		}
		
		//Initialise topic and word distributions for later		
		thetaSum = new double[numDocs][numTopics];
		phiSum = new double[numTopics][numWords];
		phiSumNormalised = false;
		numStats = 0;
		
		System.out.println("LLDA Topic Model successfully initialised");
	}
	
	public void runGibbsSampling() {	
		hasRun = true;
		System.out.println("Starting Gibbs sampling");
		System.out.println("Documents: "+numDocs+" docs");
		System.out.println("Words: "+numWords+" unique words");
		System.out.println("Topics: "+numTopics+" topics");
		for(int i=0; i<numTotalIterations; i++) {
			//if(i % 50 == 0) System.out.println("Starting iteration "+i);
			System.out.println("Starting iteration "+i);
			for(int m=0; m<topicAssignments.length; m++) { //m is document index
				for(int n=0; n<topicAssignments[m].length; n++) { //n is word index
					//Get an updated topic sample for this word
					//topicAssignments[m][n] = sampleTopicExperimental(m, n);
					topicAssignments[m][n] = sampleTopic(m, n);
					
				}
			}
			
			if(i >= numIterations && (samplingLag == 0 || i % samplingLag == 0)) {
				updatePhiThetaSums();
				
				if(printEachIteration) {
					iterationPrintTopics(10);
				}
			}
		}
	}
	
	public void runCVGibbsSampling(int startDoc, int endDoc) {	
		hasRun = true;
		int segmentSize = endDoc-startDoc+1;
		
		numDocs -= segmentSize;
		//need to get an updated numWords
		Set<Integer> tempWordSet = new HashSet<Integer>();
		for(int m=0; m<numDocs; m++) {
			if(m >=startDoc && m <= endDoc) {
				continue;
			}
			for(int n=0; n<numWordsInDocument[m]; n++) {
				tempWordSet.add(documents[m][n]);
			}
		}
		numWords = tempWordSet.size();
		
		System.out.println("Starting CV Gibbs sampling");
		System.out.println("Documents: "+numDocs+" docs");
		System.out.println("Words: "+numWords+" unique words");
		System.out.println("Topics: "+numTopics+" topics");
		System.out.println("CV Segment: "+segmentSize+" docs");

		
		for(int i=0; i<numIterations; i++) {
			//if(i % 50 == 0) System.out.println("Starting iteration "+i);
			System.out.println("Starting iteration "+i);
			for(int m=0; m<topicAssignments.length; m++) { //m is document index
				if(m >= startDoc && m <= endDoc) {
					//this is one of the segment docs - abort
					continue;
				}
				//otherwise, normal LLDA sampling please
				for(int n=0; n<topicAssignments[m].length; n++) { //n is word index
					//Get an updated topic sample for this word
					topicAssignments[m][n] = sampleTopic(m, n);
				}
			}
		}
		
		for(int m=startDoc; m<=endDoc; m++) {
			thetaSum[m] = getUnseenDocTheta(m);
		}
		
		saveOut(startDoc, endDoc, 0);
	}	
	
	private void saveOut(int startDoc, int endDoc, int reduction) {
		String dir = "";
		double dReduction = reduction/10.0;
		if(reduction == 0) {
			dir = "classifications/llda/"+topicType+"/"+numIterations+"-"+numSamplingIterations+"-"+alpha;
		} else if(reduction < 0) {
			dir = "classifications/fewertweets/"+(reduction*-1)+"/llda/"+topicType+"/"+numIterations+"-"+numSamplingIterations+"-"+alpha;
		} else {
			dir = "classifications/fewerprofiles/"+reduction+"/llda/"+topicType+"/"+numIterations+"-"+numSamplingIterations+"-"+alpha;
		}
		java.io.File dirFile = new java.io.File(dir);
		if(!dirFile.exists()) System.out.println(dirFile.mkdirs());
		
		for(int m=startDoc; m<=endDoc; m++) {
			long userID = docIDLookup.get(m).getId();
			try {
				FileOutputStream fileOut;
				fileOut = new FileOutputStream(dir+"/"+userID+".csv");
				PrintWriter writeOut = new PrintWriter(fileOut);
				writeOut.println("\"topic\",\"probability\"");
				for(int k=0; k<numTopics; k++) {
					writeOut.println(topicLookup.get(k)+","+thetaSum[m][k]);
				}
				writeOut.close();	
			} catch (FileNotFoundException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		System.out.println("Successfully saved LLDA classifications for "+topicType+" "+startDoc+"-"+endDoc);
	}
	
	public void runQuickCVGibbsSampling(int reduction) {
		hasRun = true;
		
		int segmentSize = numDocs/10; //emulates CV
		numDocs -= segmentSize;
		//need to get an updated numWords
		numWords *= 0.9;
		
		System.out.println("Starting CV Gibbs sampling");
		System.out.println("Documents: "+numDocs+" docs");
		System.out.println("Words: "+numWords+" unique words");
		System.out.println("Topics: "+numTopics+" topics");
		System.out.println("CV Segment: "+segmentSize+" docs");

		for(int i=0; i<numIterations; i++) {
			System.out.println("THREAD "+threadNum+": "+"Starting iteration "+i);
			for(int m=0; m<topicAssignments.length; m++) { //m is document index
				for(int n=0; n<topicAssignments[m].length; n++) { //n is word index
					//Get an updated topic sample for this word
					topicAssignments[m][n] = sampleTopic(m, n);
				}
			}
		}
		
		for(int m=0; m<topicAssignments.length; m++) {
			System.out.println("THREAD "+threadNum+": "+"Inferring for document "+m);
			thetaSum[m] = getUnseenDocTheta(m);
		}
		
		//take the values back to normal
		numDocs += segmentSize;
		numWords /= 0.9;
		saveOut(0, numDocs-1, reduction);
	}
	
	//Sample a new topic from the multinomial topic distribution
	private int sampleTopic(int m, int n) {
		//Removes this iteration's topicAssignment from the counting variables
		int topicID = topicAssignments[m][n]; //Get the currently assigned topic
		wordTopicAssignments[documents[m][n]][topicID]--; //Decrement the topic count for the current word
		docTopicAssignments[m][topicID]--; //Decrement the topic count for the current document
		numWordsAssignedToTopic[topicID]--; //Decrement the number of words the current topic has
		numWordsInDocument[m]--;
		
		//THIS BIT PUTS THE L IN LLDA - by restricting topics to only those in doc m, we get LLDA
		//Cumulative multinomial sampling
		Map<Integer,Double> p = new HashMap<Integer,Double>();
		for(int k : docLabels.get(m)) {
			double sample = (wordTopicAssignments[documents[m][n]][k] + beta) / (numWordsAssignedToTopic[k] + numWords * beta) * (docTopicAssignments[m][k] + alpha) / (numWordsInDocument[m] + numTopics * alpha);
			p.put(k, sample);
		}
		
		//Cumulative part - could be combined into above part cleverly
		double pSum = 0;
		for(int k : p.keySet()) {
			pSum += p.get(k);
		}

		//Sampling part - scaled because we haven't normalised
		double topicThreshold = Math.random() * pSum;
		double accumulator = 0.0; //used to see if the threshold lies in this mmultinomial segment
		int sampledTopicID = -1;
		for(int topic : p.keySet()) {
			accumulator += p.get(topic);
			if(topicThreshold < accumulator) {
				sampledTopicID = topic;
				break;
			}
		}
		
		//Maybe a fix needed by faulty double arithmetic
		if(sampledTopicID == -1) {
			System.err.println("Couldn't sample a topic, scaled sample failed");
		}
		
		//Finally, increment the relevant count variables
		wordTopicAssignments[documents[m][n]][sampledTopicID]++;
		docTopicAssignments[m][sampledTopicID]++;
		numWordsAssignedToTopic[sampledTopicID]++;
		numWordsInDocument[m]++;
		
		return sampledTopicID;
	}
	
	//like sampleTopic, but doesn't sample from one segment
	private int sampleTopicCV(int m, int n) {
		//Removes this iteration's topicAssignment from the counting variables
		int topicID = topicAssignments[m][n]; //Get the currently assigned topic
		wordTopicAssignments[documents[m][n]][topicID]--; //Decrement the topic count for the current word
		docTopicAssignments[m][topicID]--; //Decrement the topic count for the current document
		numWordsAssignedToTopic[topicID]--; //Decrement the number of words the current topic has
		numWordsInDocument[m]--;
		
		//Note: we allow sampling from everything now...
		//Cumulative multinomial sampling
		Map<Integer,Double> p = new HashMap<Integer,Double>();
		for(int k=0; k<numTopics; k++) {
			double sample = (wordTopicAssignments[documents[m][n]][k] + beta) / (numWordsAssignedToTopic[k] + numWords * beta) * (docTopicAssignments[m][k] + alpha) / (numWordsInDocument[m] + numTopics * alpha);
			p.put(k, sample);
		}
		
		//Cumulative part - could be combined into above part cleverly
		double pSum = 0;
		for(int k : p.keySet()) {
			pSum += p.get(k);
		}

		//Sampling part - scaled because we haven't normalised
		double topicThreshold = Math.random() * pSum;
		double accumulator = 0.0; //used to see if the threshold lies in this mmultinomial segment
		int sampledTopicID = -1;
		for(int topic : p.keySet()) {
			accumulator += p.get(topic);
			if(topicThreshold < accumulator) {
				sampledTopicID = topic;
				break;
			}
		}
		
		//Maybe a fix needed by faulty double arithmetic
		if(sampledTopicID == -1) {
			System.err.println("Couldn't sample a topic, scaled sample failed");
		}
		
		//Finally, increment the relevant count variables
		wordTopicAssignments[documents[m][n]][sampledTopicID]++;
		docTopicAssignments[m][sampledTopicID]++;
		numWordsAssignedToTopic[sampledTopicID]++;
		numWordsInDocument[m]++;
		
		return sampledTopicID;
	}
	
	//By eqs 82 and 83 of paper mentioned in topmost comment
	private void updatePhiThetaSums() {
		for(int docID=0; docID<numDocs; docID++) {
			for(int topicID=0; topicID<numTopics; topicID++) {
				thetaSum[docID][topicID] += (docTopicAssignments[docID][topicID] + alpha) / (numWordsInDocument[docID] + numTopics * alpha);
			}
		}
		
		for(int topicID=0; topicID<numTopics; topicID++) {
			for(int wordID=0; wordID<numWords; wordID++) {
				phiSum[topicID][wordID] += (wordTopicAssignments[wordID][topicID] + beta) / (numWordsAssignedToTopic[topicID] + numWords * beta) ;
			}
		}
		numStats++; //Used if we want to take the mean of many stats
	}
	
	//used for unseen inference
	private double[] getUnseenDocTheta(int m) {
		double[] topicDistribution = new double[numTopics];
		
		//model is complete, so we only have to sample over the unseen document
		for(int i=0; i<numSamplingIterations; i++) {
			//iterate over all words in the document
			for(int n=0; n<numWordsInDocument[m]; n++) {
				topicAssignments[m][n] = sampleTopicCV(m,n);
			}
			
			//now take a sample
			for(int topicID=0; topicID<numTopics; topicID++) {
				topicDistribution[topicID] += (docTopicAssignments[m][topicID] + alpha) / (numWordsInDocument[m] + numTopics * alpha);
			}
		}
		
		//now to normalise over samples
		for(int topicID=0; topicID<numTopics; topicID++) {
			topicDistribution[topicID] /= numSamplingIterations;
		}
		
		return topicDistribution;
	}
	
	private double[][] getTheta() {
		double[][] theta = new double[numDocs][numTopics];
		for(int docID=0; docID<numDocs; docID++) {
			for(int topicID=0; topicID<numTopics; topicID++) {
				if(numStats > 0) theta[docID][topicID] = thetaSum[docID][topicID] / numStats; 
				else theta[docID][topicID] = (docTopicAssignments[docID][topicID] + alpha) / (numWordsInDocument[docID] + numTopics * alpha);
			}
		}
		return theta;
	}
	
	private double[][] getPhi() {
		double[][] phi = new double[numTopics][numWords];
		for(int topicID=0; topicID<numTopics; topicID++) {
			for(int wordID=0; wordID<numWords; wordID++) {
				if(numStats > 0) phi[topicID][wordID] = phiSum[topicID][wordID] / numStats; 
				else phi[topicID][wordID] += (wordTopicAssignments[wordID][topicID] + beta) / (numWordsAssignedToTopic[topicID] + numWords * beta) ;
			}
		}
		return phi;
	}
	
	private double[][] normalisePhiSum() {
		if(phiSumNormalised) return phiSum;
		phiSumNormalised = true;
		for(int topicID=0; topicID<numTopics; topicID++) {
			for(int wordID=0; wordID<numWords; wordID++) {
				phiSum[topicID][wordID] /= numStats; 
			}
		}
		return phiSum;
	}
	
	public List<List<WordScore>> getTopics() {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return null;
		}
		
		List<List<WordScore>> topics = new LinkedList<List<WordScore>>();
		double[][] phi = getPhi();
		for(int topicID=0; topicID<numTopics; topicID++) {
			double[] wordProbs = phi[topicID];
			List<WordScore> wordScores = new LinkedList<WordScore>();
			for(int wordID=0; wordID<numWords; wordID++) {
				wordScores.add(new WordScore(idLookup.get(wordID), wordProbs[wordID]));
			}
			Collections.sort(wordScores);
			Collections.reverse(wordScores);
			topics.add(wordScores);
		}
		
		return topics;
	}
	
	public List<Map<Integer,Double>> getTopicsNew() {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return null;
		}
		
		List<Map<Integer,Double>> topics = new ArrayList<Map<Integer,Double>>();
		double[][] phi = normalisePhiSum();
		for(int topicID=0; topicID<numTopics; topicID++) {
			double[] wordProbs = phi[topicID];
			Map<Integer,Double> wordScores = new HashMap<Integer,Double>();
			for(int wordID=0; wordID<numWords; wordID++) {
				wordScores.put(wordID, wordProbs[wordID]);
			}
			Tools.sortMapByValueDesc(wordScores);
			topics.add(wordScores);
		}
		return topics;
	}
	
	public double[][] getTopicsUnsorted() {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return null;
		}
		
		double[][] phi = normalisePhiSum();
		return phi; //Hmmm...
	}
	
	public void printTopicsNew(int topWords) {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return;
		}
		
		double[][] phi = normalisePhiSum();
		for(int topicID=0; topicID<numTopics; topicID++) {
			double[] wordProbs = phi[topicID];
			
			List<WordScore> wordScores = new LinkedList<WordScore>();
			for(int wordID=0; wordID<numWords; wordID++) {
				wordScores.add(new WordScore(idLookup.get(wordID), wordProbs[wordID]));
			}
			Collections.sort(wordScores);
			Collections.reverse(wordScores);
			
			System.out.print("Topic "+topicLookup.get(topicID)+": ");
			int count=0;
			for(WordScore score : wordScores) {
				if(count==topWords) break;
				System.out.print(score.getWord()+" ");
				count++;
			}
			System.out.println();
		}
	}
	
	public boolean isPhiSumNormalised() {
		return phiSumNormalised;
	}
	
	public List<List<WordScore>> getDocuments() {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return null;
		}
		
		List<List<WordScore>> topics = new LinkedList<List<WordScore>>();
		double[][] theta = getTheta();
		for(int docID=0; docID<numDocs; docID++) {
			double[] topicProbs = theta[docID];
			List<WordScore> wordScores = new LinkedList<WordScore>();
			for(int topicID=0; topicID<numTopics; topicID++) {
				wordScores.add(new WordScore(topicLookup.get(topicID), topicProbs[topicID]));
			}
			Collections.sort(wordScores);
			Collections.reverse(wordScores);
			topics.add(wordScores);
		}
		
		return topics;
	}
	
	public void printTopics(int topWords) {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return;
		}
		
		List<List<WordScore>> topics = getTopics();
		for(int k=0; k<numTopics; k++) {
			System.out.println("Topic "+topicLookup.get(k)+":");
			List<WordScore> words = topics.get(k);
			for(int n=0; n<topWords; n++) {
				if(n == numWords) break; //incase topWords is huge or Corpus is tiny...
				System.out.println(words.get(n).getWord()+" = "+words.get(n).getScore());
			}
			try {System.in.read();} catch (IOException e) {}
		}
	}
	
	private void iterationPrintTopics(int topWords) {
		List<List<WordScore>> topics = getTopics();
		for(int k=0; k<numTopics; k++) {
			System.out.print("Topic "+topicLookup.get(k)+": ");
			List<WordScore> words = topics.get(k);
			for(int n=0; n<topWords; n++) {
				if(n == numWords) break; //incase topWords is huge or Corpus is tiny...
				System.out.print(words.get(n).getWord()+"(");
				Tools.dpPrint(words.get(n).getScore(),3);
				System.out.print(") ");
			}
			System.out.println();
		}
	}
	
	public void printDocuments(int topTopics) {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return;
		}
		
		List<List<WordScore>> docs = getDocuments();
		for(int d=0; d<numDocs; d++) {
			System.out.println("Document "+d+":");
			List<WordScore> topics = docs.get(d);
			for(int n=0; n<topTopics; n++) {
				if(n == numWords) break;
				System.out.println(topics.get(n).getWord()+" = "+topics.get(n).getScore());
			}
			try {System.in.read();} catch (IOException e) {}
		}
	}
	
	public void printDocumentsVerbose(int topTopics) {
		if(!hasRun) {
			System.err.println("Gibbs sampler has not yet run");
			return;
		}
		
		
		List<List<WordScore>> docs = getDocuments();
		for(int d=0; d<numDocs; d++) {
			List<WordScore> topics = docs.get(d);
			System.out.print("Document "+d+" (uid: "+docIDLookup.get(d).getId()+"): ");
			for(int n=0; n<documents[d].length; n++) {
				System.out.print(idLookup.get(documents[d][n])+"["+topicLookup.get(topicAssignments[d][n])+"] ");
			}
			System.out.println();
			
			System.out.print("Initial topic set: {");
			for(int n=0; n<docLabels.get(d).size(); n++) {	
				System.out.print(topicLookup.get(docLabels.get(d).get(n)));
				if(docLabels.get(d).size() > 1 && n < docLabels.get(d).size()-1) System.out.print(", ");
			}
			System.out.println("}");
			
			for(int n=0; n<topTopics; n++) {
				if(n == numWords) break;
				System.out.println(topics.get(n).getWord()+" = "+topics.get(n).getScore());
			}
			try {System.in.read();} catch (IOException e) {}
		}
	}
	
	public void print() {
		//TODO
	}
	
    public void save(String name) {
		try {
			String filename = "models/llda/"+topicType+"/"+name+".model";
			FileOutputStream fileOut = new FileOutputStream(filename);
			ObjectOutputStream objectOut = new ObjectOutputStream(fileOut);
			objectOut.writeObject(this);
			objectOut.close();
			System.out.println("Saved LLDA topic model "+name);
		} catch (FileNotFoundException e) {
			System.out.println("Couldn't save LLDA topic model "+name);
			e.printStackTrace();
		} catch (IOException e) {
			System.out.println("Couldn't save LLDA topic model "+name);
			e.printStackTrace();			
		}    	
    }
    
    public static LLDATopicModel load(String topicType, String name) {
		try {
			String filename = "models/llda/"+topicType+"/"+name+".model";
			FileInputStream fileIn = new FileInputStream(filename);
			ObjectInputStream objectIn = new ObjectInputStream(fileIn);
			LLDATopicModel model = (LLDATopicModel)objectIn.readObject();
			objectIn.close();
			System.out.println("Loaded LLDA topic model "+name);
			return model;
		} catch (FileNotFoundException e) {
			System.out.println("Couldn't load LLDA topic model "+name);
			e.printStackTrace();
		} catch (IOException e) {
			System.out.println("Couldn't load LLDA topic model "+name);
			e.printStackTrace();			
		} catch (ClassNotFoundException e) {
			System.out.println("Couldn't load LLDA topic model "+name);
			e.printStackTrace();			
		}
		return null;
    }
    
    public static LLDATopicModel loadFromPath(String topicType, String path) {
		try {
			String filename = path;
			FileInputStream fileIn = new FileInputStream(filename);
			ObjectInputStream objectIn = new ObjectInputStream(fileIn);
			LLDATopicModel model = (LLDATopicModel)objectIn.readObject();
			objectIn.close();
			System.out.println("Loaded LLDA topic model "+path);
			return model;
		} catch (FileNotFoundException e) {
			System.out.println("Couldn't load LLDA topic model "+path);
			e.printStackTrace();
		} catch (IOException e) {
			System.out.println("Couldn't load LLDA topic model "+path);
			e.printStackTrace();			
		} catch (ClassNotFoundException e) {
			System.out.println("Couldn't load LLDA topic model "+path);
			e.printStackTrace();			
		}
		return null;
    }
    
    public long getDocIDFromIndex(int m) {
    	return docIDLookup.get(m).getId();
    }
    
    public Map<String,Integer> getVocab() {
    	return wordIDs;
    }
    
    public ArrayList<String> getTopicsIDList() {
    	return topicLookup;
    }

    public Map<String,Double> inferTopicDistribution(SimpleProfile sp, int burnIn, int sampling, double alpha, double beta) {
    	//Get FV from SP
    	Document d = sp.asDocument();
		String[] tokens = d.getDocumentString().split("\\s+");
		ArrayList<Integer> fv = new ArrayList<Integer>();
		int numExistingWords = 0;
		for(int i=0; i<tokens.length; i++) {
			if(wordIDs.containsKey(tokens[i])) { 
				fv.add(wordIDs.get(tokens[i]));
			}
		}
		
		//Run Gibbs Sampler again
		int[] z = new int[fv.size()];
		int[] zCounts = new int[numTopics];
		for(int n=0; n<z.length; n++) {
			//Random topic assignments
			z[n] = (int)(Math.random()*numTopics);
			zCounts[z[n]]++;
		}
		double[] thetam = new double[numTopics];
		for(int i=0; i<burnIn + sampling; i++) {
			System.out.print(".");
			for(int n=0; n<fv.size(); n++) {
				//Cumulative multinomial sampling
				double[] p = new double[numTopics];
				for(int k=0; k<numTopics; k++) {
					p[k] = (wordTopicAssignments[fv.get(n)][k] + beta) / (numWordsAssignedToTopic[k] + numWords * beta) * (zCounts[k] + alpha) / (fv.size() + numTopics * alpha);
				}
				
				//Cumulative part - could be combined into above part cleverly
				for(int k=1; k<numTopics; k++) {
					p[k] += p[k-1];
				}
				
				//Sampling part - scaled because we haven't normalised
				double topicThreshold = Math.random() * p[numTopics-1];
				int sampledTopicID = 0;
				for(sampledTopicID=0; sampledTopicID<numTopics; sampledTopicID++) {
					if(topicThreshold < p[sampledTopicID]) {
						break;
					}
				}
				
				//Maybe a fix needed by faulty double arithmetic
				if(sampledTopicID >= numTopics) {
					sampledTopicID = numTopics-1;
				}
				
				zCounts[z[n]]--;
				z[n] = sampledTopicID;
				zCounts[z[n]]++;
			}
			
			if(i >= burnIn) {
				for(int topicID=0; topicID<numTopics; topicID++) {
					thetam[topicID] += (zCounts[topicID] + alpha) / (fv.size() + numTopics * alpha);
				}
			}
		}
		System.out.println();
		//normalise theta and store
		Map<String,Double> results = new HashMap<String,Double>();
		for(int k=0; k<numTopics; k++) {
			thetam[k] /= sampling;
			results.put(topicLookup.get(k),thetam[k]);
		}
		
		return Tools.sortMapByValueDesc(results);
    }
    
    public LightweightLLDA asLightweightLLDA() {
    	LightweightLLDA lllda = new LightweightLLDA(wordIDs, topicLookup, numTopics, numWords, wordTopicAssignments, numWordsAssignedToTopic);
    	return lllda;
    }
    
    public void printStats() {
    	System.out.println("num words = "+wordIDs.size()+" and "+numWords);
		System.out.println("num topics = "+topicLookup.size()+" and "+numTopics);
    }
} public class Profiler {
	
	Set<Long> existingProfiles;
	Twitter twitter;
	String accessTokenFilename = "WEB-INF/tokens/accessToken.obj";
	AccessToken accessToken;
	private String consumerKey = "ovSLNjZKYbYu3kcrDyVuXg";
	private String consumerSecret = "YzfRt93vUrPvO7ZKHnqcNslrsy7oa5fIQTiY0ARno";
	
	public Profiler() {
		//Figure out which profiles we've already saved so we don't have to worry about them later
		existingProfiles = new HashSet<Long>();		
		
		//Either get a new authentication token, or load the persistent one
		registerOAuth();
	}
	
	public void findExistingProfileNames() {
		File profileDir = new File("profiles/simple");
		ArrayList<String> profiles = new ArrayList<String>(Arrays.asList(profileDir.list()));
		profiles.remove(".svn");
		if(profiles == null) return;
		//System.out.println("Existing Profiles: ");
		for(int i=0; i<profiles.size(); i++) { //Set to 1 to skip .svn...
			String username = profiles.get(i).split("\\.")[0];
			existingProfiles.add(Long.parseLong(username));
			//System.out.println(username);
		}
	}
	
	public Set<Long> getExistingProfileNames() {
		return existingProfiles;
	}
	
	@SuppressWarnings("deprecation")
	public void registerOAuth() {
		twitter = new TwitterFactory().getInstance();	
		twitter.setOAuthConsumer(consumerKey, consumerSecret);		
		
		try {
			FileInputStream fileIn = new FileInputStream(accessTokenFilename);
			ObjectInputStream objectIn = new ObjectInputStream(fileIn);
			accessToken = (AccessToken)objectIn.readObject();
			if(accessToken != null) {
				System.out.println("Successfully read accessToken");
				System.out.println(accessToken.getToken()+", "+accessToken.getTokenSecret());
				twitter.setOAuthAccessToken(accessToken);
				return;
			}
		} catch (IOException e){
			System.out.println("Couldn't read access token");
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		}		
		
	    RequestToken requestToken = null;
		try {
			requestToken = twitter.getOAuthRequestToken();
		} catch (TwitterException e1) {
			System.err.println("Couldn't get request token for OAuth");
			e1.printStackTrace();
			return;
		}
	    BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
	    while (null == accessToken) {
	    	System.out.println("Open the following URL and grant access to your account:");
	    	System.out.println(requestToken.getAuthorizationURL());
	    	System.out.print("Enter the PIN(if aviailable) or just hit enter.[PIN]:");
	    	String pin;
	    	try{
				pin = br.readLine();
	    		if(pin.length() > 0){
	    			accessToken = twitter.getOAuthAccessToken(requestToken, pin);
	    		}else{
	    			accessToken = twitter.getOAuthAccessToken();
	    		}
	    	} catch (TwitterException te) {
	    		if(401 == te.getStatusCode()){
	    			System.out.println("Unable to get the access token.");
	    		}else{
	    			te.printStackTrace();
	    		}
	    		return;
	    	} catch (IOException e) {
	    		e.printStackTrace();
	    		return;
	    	}
	    }
	    
	    //persist to the accessToken for future reference.
	    try {
	    	FileOutputStream fileOut = new FileOutputStream(accessTokenFilename);
	    	ObjectOutputStream objectOut = new ObjectOutputStream(fileOut);
	    	objectOut.writeObject(accessToken);
	    	System.out.println("Access token aquired successfully");
	    } catch (IOException e) {
	    	System.err.println("Couldn't store access token");
	    	e.printStackTrace();
	    }
	}
	
	//Note: this can't update a profile
	//TODO: Create a version that synchronises profiles without redownloading
	public RawProfile getRawProfile(String username, int tweets) {
		
		//Download Profile as a list of Status objects
		List<Status> timeline = null;
		int statusesCount;
		try {
			User user = twitter.showUser(username);
			statusesCount = user.getStatusesCount();
			System.out.println(username+" has "+statusesCount+" tweets, attempting download...");
		} catch (TwitterException e) {
			System.err.println("Couldn't get "+username+"'s number of tweets");
			e.printStackTrace();
			return null;			
		}
		
		RawProfile rawProfile = new RawProfile(username);
		try {
			rawProfile.setUserID(twitter.showUser(username).getId());
		} catch (TwitterException e) {
			System.err.println("Couldn't get UID");
			e.printStackTrace();
		}
		
		int pageCount = 1;
		do {
			try {
				System.out.println("Attempting to obtain page "+pageCount+" of "+username+"'s tweets");
				//Paging paging = new Paging(pageCount,statusesCount);
				Paging paging = new Paging(pageCount,200); //200 is max but gives http error 502
				timeline = twitter.getUserTimeline(username, paging);
				if(timeline.isEmpty()) {
					break; //To avoid the case of the phantom missing tweet...
				}
				rawProfile.addTweets(timeline);
				if(rawProfile.getTweets().size() >= tweets) break;
				pageCount++;				
			} catch (TwitterException e) {
				System.err.println("Couldn't get "+username+"'s profile, retrying...");
				//e.printStackTrace();
			}
			System.out.println("Downloaded "+rawProfile.getTweets().size()+" tweets so far");
		} while(rawProfile.getTweets().size() != statusesCount);
		
		//rawProfile.print();
		
		//save raw profile to a profile file and CSV, as well as a simple profile and a simple CSV
		//rawProfile.save();
		//rawProfile.saveCSV();
		//rawProfile.saveAsSimpleProfile();
		
		return rawProfile;
	}	
		
	public static RawProfile loadRawProfile(String username) {
		RawProfile profile = null;
		try {
			FileInputStream fileIn = new FileInputStream("profiles/raw/"+username+".profile");
			ObjectInputStream objectIn = new ObjectInputStream(fileIn);
			profile = (RawProfile)objectIn.readObject();
			if(profile != null) {
				System.out.println("Successfully read profile "+username);
			}
		} catch (IOException e){
			System.err.println("Couldn't read profile "+username);
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		}			
		return profile;
	}
	
	public SimpleProfile loadSimpleProfile(String username) {
		try {
			return loadSimpleProfile(twitter.showUser(username).getId());
		} catch (TwitterException e) {
			e.printStackTrace();
			return null;
		}
	}
	
	public SimpleProfile loadSimpleProfileWeb(String username) {
		try {
			return loadSimpleProfileWeb(twitter.showUser(username).getId());
		} catch (TwitterException e) {
			e.printStackTrace();
			return null;
		}
	}
	
	public static SimpleProfile loadSimpleProfile(long userID) {
		SimpleProfile profile = new SimpleProfile(userID);
		try {
			FileInputStream fileIn = new FileInputStream("profiles/simple/"+userID+".sprofile");
			ObjectInputStream objectIn = new ObjectInputStream(fileIn);
			profile = (SimpleProfile)objectIn.readObject();
			if(profile != null) {
				System.out.println("Successfully read simple profile "+userID);
			}
		} catch (IOException e){
			System.err.println("Couldn't read simple profile "+userID);
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		}			
		return profile;
	}
	
	public static SimpleProfile loadSimpleProfileWeb(long userID) {
		SimpleProfile profile = new SimpleProfile(userID);
		try {
			FileInputStream fileIn = new FileInputStream("WEB-INF/profiles/"+userID+".sprofile");
			ObjectInputStream objectIn = new ObjectInputStream(fileIn);
			profile = (SimpleProfile)objectIn.readObject();
			if(profile != null) {
				System.out.println("Successfully read simple profile "+userID);
			}
		} catch (IOException e){
			System.err.println("Couldn't read simple profile "+userID);
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		}			
		return profile;
	}
	
	public SimpleProfile loadCSVProfile(String username) {
		try {
			return loadCSVProfile(twitter.showUser(username).getId());
		} catch (TwitterException e) {
			e.printStackTrace();
			return null;
		}
	}	
	
	public static SimpleProfile loadCSVProfile(long userID) {
		SimpleProfile profile = new SimpleProfile(userID);
		try {
			FileInputStream fileIn = new FileInputStream("profiles/csv/"+userID+".csv");
			BufferedReader buffer = new BufferedReader(new InputStreamReader(fileIn));
			String nextLine = "";
			buffer.readLine(); //skip past the CSV descriptor
			while(true) {
				nextLine = buffer.readLine();
				if(nextLine == null) break;
				String[] splitTweet = nextLine.split(",");
				long tweetID = Long.parseLong(splitTweet[1]);
				String text = nextLine.substring(nextLine.indexOf("\"")+1,nextLine.length()-1); //to avoid comma tweeting errors
				SimpleTweet newTweet = new SimpleTweet(userID,tweetID,text);
				profile.addTweet(newTweet);
			}
		} catch (IOException e){
			System.err.println("Couldn't read profile "+userID);
			return null;
		} 
		return profile;
	}
	
	public void parseCSVDataset() {
		try {
			FileInputStream fileIn = new FileInputStream("dataset/users_with_location.csv");
			BufferedReader buffer = new BufferedReader(new InputStreamReader(fileIn));
			String nextLine = "";
			buffer.readLine(); //skip past the CSV descriptor
			SimpleProfile currentProfile = null;
			int lineCount = 1;
			while(true) {
				
				lineCount++;
				if(lineCount % 1000 == 0) System.out.println("Currently on line "+lineCount);
				
				nextLine = buffer.readLine();
				//If nextLine is null, we still have to save the final profile!
				if(nextLine == null) {
					System.out.println("Profile "+currentProfile.getUserID()+" is the final profile, saving:");
					currentProfile.save();
					currentProfile.saveCSV();		
					break;
				}
				
				String[] splitTweet = nextLine.split(",");
				long currentTweetUserID = Long.parseLong(splitTweet[0]);
				long currentTweetID = Long.parseLong(splitTweet[1]);
				String currentTweetText = nextLine.substring(nextLine.indexOf("\"")+1,nextLine.length()-1);  //to avoid comma tweeting errors
				
				//Creating a SimpleTweet out of the CSV entry
				SimpleTweet currentTweet = new SimpleTweet(currentTweetUserID,currentTweetID,currentTweetText);
				
				//If we are at the first profile in the list, simply create it, add the tweet and continue
				if(currentProfile == null) {
					System.out.println("Found first profile, ID: "+currentTweetUserID);
					currentProfile = new SimpleProfile(currentTweetUserID);
					currentProfile.addTweet(currentTweet);
					continue;
				}
				
				//Else, check if we've hit a new user - if so, we need to save the simple profile and make a new one
				if(currentProfile.getUserID() != currentTweetUserID) {
					System.out.println("Profile "+currentProfile.getUserID()+" finished, starting on "+currentTweetUserID);
					currentProfile.save();
					currentProfile.saveCSV();
					currentProfile = new SimpleProfile(currentTweetUserID);
					currentProfile.addTweet(currentTweet);
					continue;
				}
				
				//Else, we're simply on the same user's next tweet - so add it and carry on.
				currentProfile.addTweet(currentTweet);
			}
		} catch(IOException e) {
			System.err.println("Couldn't open dataset");
			e.printStackTrace();
		}

	}
}
 public class SimpleProfile implements Serializable {

	private static final long serialVersionUID = 4610549868654867470L;
	
	//private static LIWCDictionary liwc = LIWCDictionary.loadDictionaryFromFile("liwc/dictionary.obj");
	private long userID;
	private List<SimpleTweet> tweets;
	
	public SimpleProfile(long userID) {
		this.userID = userID;
		tweets = new ArrayList<SimpleTweet>();
	}
	
	public long getUserID() {
		return userID;
	}
	
	public void addTweet(SimpleTweet tweet) {
		tweets.add(tweet);
	}
	
	public void addTweets(List<SimpleTweet> tweetList) {
		tweets.addAll(tweetList);
	}
	
	public List<SimpleTweet> getTweets() {
		return tweets;
	}
	
	public void print() {
		System.out.println("Twitter Profile for "+userID);
		for(SimpleTweet tweet : tweets) {
			tweet.print();
		}
	}
	
	public void printUrls() {
		List<String> urls = new ArrayList<String>();
		for(SimpleTweet tweet : tweets) {
			urls.addAll(tweet.getUrls());
		}
		System.out.println("Urls Used: ");
		for(String url : urls) {
			System.out.println(url);
		}
	}
	
	public void printHashtags() {
		List<String> hashtags = new ArrayList<String>();
		for(SimpleTweet tweet : tweets) {
			hashtags.addAll(tweet.getHashtags());			
		}
		System.out.println("Hashtags Used: ");
		for(String hashtag : hashtags) {
			System.out.println(hashtag);
		}
	}
	
	public void printStripped() {
		System.out.println("Twitter Profile for "+userID);
		for(SimpleTweet tweet : tweets) {
			tweet.printStripped();
		}
	}
	
	public void save() {
		try {
			String profileFilename = "profiles/simple/"+userID+".sprofile";
			FileOutputStream fileOut = new FileOutputStream(profileFilename);
			ObjectOutputStream objectOut = new ObjectOutputStream(fileOut);
			objectOut.writeObject(this);
			objectOut.close();
			System.out.println("Saved simple profile for "+userID);
		} catch (FileNotFoundException e) {
			System.out.println("Couldn't save simple profile for "+userID);
			e.printStackTrace();
		} catch (IOException e) {
			System.out.println("Couldn't save simple profile for "+userID);
			e.printStackTrace();			
		}
	}
	
	public void saveCSV() {
		try {
			String profileFilename = "profiles/csv/"+userID+".csv";
			FileOutputStream fileOut = new FileOutputStream(profileFilename);
			PrintWriter writeOut = new PrintWriter(fileOut);
			writeOut.println("\"userid\",\"tweetid\",\"text\"");
			for(SimpleTweet tweet : tweets) {
				writeOut.print(tweet.getUserID()+",");
				writeOut.print(tweet.getTweetID()+",");
				writeOut.print("\""+tweet.getText()+"\"");
				writeOut.println();
			}
			writeOut.close();
			System.out.println("Saved CSV profile for "+userID);
		} catch (IOException e) {
			System.out.println("Couldn't save CSV profile for "+userID);
			e.printStackTrace();
		}
	}
	
	/*
	//NOTE: We perform stripping while converting a profile to a Document Set
	//NOTE: NEED TO MEMOISE CLASSIFICATIONS
	public Set<Document> asDocumentSet(String topicType) {
		System.out.println("Converting profile to Document set...");
		Set<Document> documents = new HashSet<Document>();
		Map<Long,List<CategoryScore>> scores = null; //for LIWC
		int count = 0;
		for(SimpleTweet tweet : tweets) {
			if(count % 50 == 0) System.out.println(count+"/"+tweets.size()+" tweets converted...");
			count++;
			String strippedText = Tools.stripTweet(tweet.getText());
			//String strippedText = Tools.stripTweetVerbose(tweet.getText());
			Set<String> topics = new HashSet<String>();
			Document document = null;
			if(topicType != null) {
				//NOTE: We classify on the TWEET, not on the stemmed/stripped version!
				if(topicType.equals("liwc")) {
					if(scores == null) scores = liwc.getLIWCCategoryScores(userID);
					//List<CategoryScore> liwcClassifications = liwc.classifyTweet(tweet); //NOTE: this is already sorted by LIWCDictionary
					List<CategoryScore> liwcClassifications = scores.get(tweet.getTweetID());
					//Take the top 3... note that this is of limited use, I just need some topics to get started
					int getTop = 3;
					if(getTop > liwcClassifications.size()) getTop = liwcClassifications.size();
					for(int i=0; i<getTop; i++) {
						topics.add(liwcClassifications.get(i).getCategory().getTitle());
					}
					document = new Document(strippedText,tweet.getTweetID(),topics);		
				}
				
				else if(topicType.equals("liwcnb")) {
					if(scores == null) scores = liwc.getLIWCCategoryScoresNB(userID);
					//List<CategoryScore> liwcClassifications = liwc.classifyTweetNaiveBayes(tweet); //NOTE: this is already sorted by LIWCDictionary
					List<CategoryScore> liwcClassifications = scores.get(tweet.getTweetID());
					//Take the top 3... note that this is of limited use, I just need some topics to get started
					int getTop = 3;
					if(getTop > liwcClassifications.size()) getTop = liwcClassifications.size();
					for(int i=0; i<getTop; i++) {
						topics.add(liwcClassifications.get(i).getCategory().getTitle());
					}
					document = new Document(strippedText,tweet.getTweetID(),topics);	
				}
			} else {
				document = new Document(strippedText,tweet.getTweetID());
			}
			documents.add(document);
		}
		return documents;
	}
	
	public Set<Document> asDocumentSet() {
		return asDocumentSet(null);
	}
	
	
	public Corpus asCorpus(String topicType) {
		Corpus corpus = new Corpus(this.asDocumentSet(topicType),topicType);
		return corpus;
	}
	
	public Corpus asCorpus() {
		Corpus corpus = new Corpus(this.asDocumentSet(null));
		return corpus;
	}
	*/

	/* NOTE Details of weighted alchemy classification here:
	 * Rather than counting instances of each different classification
	 * by adding 1 every time a category appears, instead the score
	 * of that category (between 0.0 and 1.0) will be added to that
	 * category's count - this way a weighted count can be achieved.
	 * Two seperate counts are found - text count (based on tweet content)
	 * and URL count (based on the classifications of all the URLs of a
	 * profile. String categories therefore form the keys of the count
	 * hashmaps.  
	 * 
	 * Also NOTE: this method can take AGES
	 * 
	 * Also NOTE: this method ONLY SAVES A CATEGORY into teh csv if it appears
	 * in the profile or its URLS. it doesn't have a comprehensive list of all alchemy cats.
	 */
	public boolean classifyAlchemy() {
		
		System.out.println("Alchemy classifying profile "+userID);
		
		//Check if classification already exists, to save time and allow resuming classification if something goes wrong
		String classificationFilename = "classifications/alchemy/"+userID+".csv";
		File classificationFile = new File(classificationFilename);
		if(classificationFile.exists()) {
			System.out.println("Alchemy classification already exists for "+userID+", aborting");
			return true; //or do we want to return false...
		}
		
		FileOutputStream fileOutTweets = null;
		PrintWriter writeOutTweets = null;
		try {
			String tweetsFilename = "classifications/alchemy/tweets/"+userID+".csv";
			fileOutTweets = new FileOutputStream(tweetsFilename);
			writeOutTweets = new PrintWriter(fileOutTweets);
			writeOutTweets.println("\"tweetID\",\"category\",\"score\",\"urlcategories\",\"urlscores\"");
		} catch(IOException e) {
			System.err.println("Couldn't open tweets file to save to");
		}
		
		Map<String,Double> textCategoryScores = new HashMap<String,Double>();
		Map<String,Integer> textCategoryCounts = new HashMap<String,Integer>();
		Map<String,Double> urlCategoryScores = new HashMap<String,Double>();
		Map<String,Integer> urlCategoryCounts = new HashMap<String,Integer>();
		
		for(SimpleTweet tweet : tweets) {
			//Handle tweet content classification
			AlchemyClassification textClassification = AlchemyClassifier.classifyText(tweet.getText());
			if(textClassification == null) {
				System.err.println("Daily Transaction Limit has been reached (or a null classification was given weirdly)");
				return false;
			}
			
			writeOutTweets.print(tweet.getTweetID()+","+textClassification.getCategory()+","+textClassification.getScore());
			System.err.println("JUST WROTE TWEEET OUT");
			
			if(textCategoryScores.containsKey(textClassification.getCategory())) {
				//Add the new value, since the key already exists
				textCategoryScores.put(textClassification.getCategory(), textCategoryScores.get(textClassification.getCategory()) + textClassification.getScore());
				textCategoryCounts.put(textClassification.getCategory(), textCategoryCounts.get(textClassification.getCategory()) + 1);
			} else {
				//Insert the category into the map
				textCategoryScores.put(textClassification.getCategory(), textClassification.getScore());
				textCategoryCounts.put(textClassification.getCategory(), 1);
			}
			
			//Handle URL classification
			for(String url : tweet.getUrls()) {
				AlchemyClassification urlClassification = AlchemyClassifier.classifyURL(url);
				if(urlClassification == null) {
					System.err.println("Daily Transaction Limit has been reached (or a null classification was given weirdly)");
					return false;
				}
				
				writeOutTweets.print(","+urlClassification.getCategory()+","+urlClassification.getScore());
				
				if(urlCategoryScores.containsKey(urlClassification.getCategory())) {
					//Add the new value, since the key already exists
					urlCategoryScores.put(urlClassification.getCategory(), urlCategoryScores.get(urlClassification.getCategory()) + urlClassification.getScore());
					urlCategoryCounts.put(urlClassification.getCategory(), urlCategoryCounts.get(urlClassification.getCategory()) + 1);
				} else {
					//Insert the category into the map
					urlCategoryScores.put(urlClassification.getCategory(), urlClassification.getScore());
					urlCategoryCounts.put(urlClassification.getCategory(), 1);
				}
			}

			//writeOutTweets.println();
		}
		
		writeOutTweets.close();
		
		//Now save as a CSV Alchemy classification file containing the scores
		try {
			FileOutputStream fileOut = new FileOutputStream(classificationFilename);
			PrintWriter writeOut = new PrintWriter(fileOut);
			writeOut.println("\"category\",\"textscore\",\"textcount\",\"urlscore\",\"urlcount\"");
			
			//Need to merge the category sets from both text and URL classifications
			//in case one has a category the other does not have
	
			Set<String> categorySet = new HashSet<String>(textCategoryScores.keySet());
			categorySet.addAll(urlCategoryScores.keySet());
			
			//remove the useless categories
			categorySet.remove(null); //you can have a null key? weird
			categorySet.remove("unknown");
			
			for(String category : categorySet) {				
				Double textscore = textCategoryScores.get(category);
				Integer textcount = textCategoryCounts.get(category);
				Double urlscore = urlCategoryScores.get(category);
				Integer urlcount = urlCategoryCounts.get(category);
				
				//If no score exists for a category, it has not appeared in a profile
				if(textscore == null) {
					textscore = new Double(0.0);
					textcount = new Integer(0);
				}
				if(urlscore == null) {
					urlscore = new Double(0.0);
					urlcount = new Integer(0);
				}
				
				writeOut.print("\""+category+"\",");
				writeOut.print(textscore+",");
				writeOut.print(textcount+",");
				writeOut.print(urlscore+",");
				writeOut.print(urlcount);
				writeOut.println();
			}
			writeOut.close();
			System.out.println("Saved CSV Alchemy classification for "+userID);
			return true;
		} catch (IOException e) {
			System.out.println("Couldn't save CSV Alchemy classification for "+userID);
			e.printStackTrace();
			return false;
		}
	}
	
	/*
	//NOTE: Incorporates both naive frequency counting and naive bayes, stores both
	public boolean classifyLIWC() {
		System.out.println("LIWC classifying profile "+userID);
		
		//Check if classification already exists, to save time and allow resuming classification if something goes wrong
		String classificationFilename = "classifications/liwc/"+userID+".csv";
		String classificationFilenameCount = "classifications/liwc/count/"+userID+".csv";
		String classificationFilenameNB = "classifications/liwc/nb/"+userID+".csv";
		File classificationFileCount = new File(classificationFilenameCount);
		File classificationFileNB = new File(classificationFilenameNB);
		if(classificationFileCount.exists() || classificationFileNB.exists()) {
			System.out.println("LIWC classification already exists for "+userID+", aborting");
			return true; //or do we want to return false...
		}
							
		//Now save as a CSV LIWC classification file containing the scores
		try {
			FileOutputStream fileOut = new FileOutputStream(classificationFilename);
			FileOutputStream fileOutCount = new FileOutputStream(classificationFilenameCount);
			FileOutputStream fileOutNB = new FileOutputStream(classificationFilenameNB);
			PrintWriter writeOut= new PrintWriter(fileOut);
			PrintWriter writeOutCount = new PrintWriter(fileOutCount);
			PrintWriter writeOutNB = new PrintWriter(fileOutNB);
			writeOut.println("\"category\",\"countscore\",\"nbscore\"");
			writeOutCount.println("\"tweetid\",\"category\",\"countscore\"");
			writeOutNB.println("\"tweetid\",\"category\",\"nbscore\"");
			
			//To speed up full-profile classification
			Map<Category,Double> scoresNormal = new HashMap<Category,Double>();
			Map<Category,Double> scoresNaiveBayes = new HashMap<Category,Double>();
		
			//Individual tweet classifications
			int averageCount = 0;
			for(SimpleTweet tweet : tweets) {
				List<CategoryScore> scoresCount = liwc.classifyTweet(tweet);
				List<CategoryScore> scoresNB = liwc.classifyTweetNaiveBayes(tweet);
				Collections.sort(scoresCount);
				Collections.sort(scoresNB);
				Collections.reverse(scoresCount);
				Collections.reverse(scoresNB);
				
				writeOutCount.print(tweet.getTweetID()+",");
				String outString = "";
				for(CategoryScore score : scoresCount) {
					//update counts for full-profile
					if(scoresNormal.containsKey(score.getCategory())) {
						scoresNormal.put(score.getCategory(),scoresNormal.get(score.getCategory())+score.getScore());
					} else {
						scoresNormal.put(score.getCategory(),score.getScore());
					}
					
					outString += (score.getCategory().getTitle()+","+score.getScore()+",");	
				}
				if(!outString.isEmpty()) outString.subSequence(0, outString.length()-1);
				writeOutCount.print(outString);
				writeOutCount.println();
				
				writeOutNB.print(tweet.getTweetID()+",");
				outString = "";
				for(CategoryScore score : scoresNB) {
					//update counts for full-profile
					if(scoresNaiveBayes.containsKey(score.getCategory())) {
						//Moving average - hacky...
						scoresNaiveBayes.put(score.getCategory(),((scoresNaiveBayes.get(score.getCategory())*averageCount)+score.getScore()) / (averageCount+1));
					} else {
						scoresNaiveBayes.put(score.getCategory(),score.getScore());
					}
					
					outString += (score.getCategory().getTitle()+","+score.getScore()+",");	
				}
				if(!outString.isEmpty()) outString.subSequence(0, outString.length()-1);
				writeOutNB.print(outString);
				writeOutNB.println();
				averageCount++;
			}
			writeOutCount.close();
			writeOutNB.close();
			
			Set<Category> fullCatSet = new HashSet<Category>();
			fullCatSet.addAll(scoresNormal.keySet());
			fullCatSet.addAll(scoresNaiveBayes.keySet());
			for(Category cat : fullCatSet) {
				Double normalScore = scoresNormal.get(cat);
				if(normalScore == null) normalScore = 0.0;
				writeOut.println(cat.getTitle()+","+normalScore+","+scoresNaiveBayes.get(cat));
			}
			writeOut.close();
			
			System.out.println("Saved CSV LIWC classification for "+userID);
			return true;
		} catch (IOException e) {
			System.out.println("Couldn't save CSV LIWC classification for "+userID);
			e.printStackTrace();
			return false;
		}
	}
	*/
	
	//Performs the same weighted sum as the Alchemy classification does
	//ie sums the scores for all categories over all tweets.
	public boolean classifyCalais() {
		System.out.println("Calais classifying profile "+userID);
		
		//Check if classification already exists, to save time and allow resuming classification if something goes wrong
		String classificationFilename = "classifications/calais/"+userID+".csv";
		File classificationFile = new File(classificationFilename);
		if(classificationFile.exists()) {
			System.out.println("Calais classification already exists for "+userID+", aborting");
			return true; //or do we want to return false...
		}
		
		Map<String,Double> resultsMap = new HashMap<String,Double>();
		Map<String,Integer> countsMap = new HashMap<String,Integer>();
		
		for(SimpleTweet tweet : tweets) {
			CalaisClassification classification = CalaisClassifier.classifyText(tweet.getText());
			if(classification == null) {
				//Normally this is because of a <100 char tweet
				continue;
			}
			for(String category : classification.getCategories()) {
				double score = classification.lookupScore(category);			
				if(resultsMap.containsKey(category)) {
					resultsMap.put(category, resultsMap.get(category) + score);
					countsMap.put(category, countsMap.get(category) + 1);
				} else {
					resultsMap.put(category, score);
					countsMap.put(category, 1);
				}
			}
		}
							
		//Now save as a CSV Calais classification file containing the scores
		try {
			FileOutputStream fileOut = new FileOutputStream(classificationFilename);
			PrintWriter writeOut = new PrintWriter(fileOut);
			writeOut.println("\"category\",\"score\",\"countscore\"");
		
			for(String category : resultsMap.keySet()) {
				writeOut.print("\""+category+"\",");
				writeOut.print(resultsMap.get(category)+",");
				writeOut.print(countsMap.get(category));
				writeOut.println();
			} 
			writeOut.close();
			System.out.println("Saved CSV Calais classification for "+userID);
			return true;
		} catch (IOException e) {
			System.out.println("Couldn't save CSV Calais classification for "+userID);
			e.printStackTrace();
			return false;
		}
	}
	
	public boolean classifyTextwise() {
		
		System.out.println("Textwise classifying profile "+userID);
		
		//Check if classification already exists, to save time and allow resuming classification if something goes wrong
		String classificationFilename = "classifications/textwise/"+userID+".csv";
		File classificationFile = new File(classificationFilename);
		if(classificationFile.exists()) {
			System.out.println("Textwise classification already exists for "+userID+", aborting");
			return true; //or do we want to return false...
		}
		
		Map<String,Double> textCategoryScores = new HashMap<String,Double>();
		Map<String,Integer> textCategoryCounts = new HashMap<String,Integer>();
		Map<String,Double> urlCategoryScores = new HashMap<String,Double>();
		Map<String,Integer> urlCategoryCounts = new HashMap<String,Integer>();
		
		for(SimpleTweet tweet : tweets) {
			//System.err.println(tweet.getText());
			
			TextwiseClassification textClassification = TextwiseClassifier.classify(tweet.getText(), false);
			
			textClassification.print();
			
			for(String category : textClassification.getCategories()) {
				if(textCategoryScores.containsKey(category)) {
					textCategoryScores.put(category, textCategoryScores.get(category) + textClassification.lookupScore(category));
					textCategoryCounts.put(category, textCategoryCounts.get(category) + 1);
				} else {
					textCategoryScores.put(category, textClassification.lookupScore(category));
					textCategoryCounts.put(category, 1);
				}
			}
			
			for(String url : tweet.getUrls()) {
				
				//System.err.println(url);
				
				TextwiseClassification urlClassification = TextwiseClassifier.classify(url, true);
				
				urlClassification.print();
				
				for(String category : urlClassification.getCategories()) {
					if(urlCategoryScores.containsKey(category)) {
						urlCategoryScores.put(category, urlCategoryScores.get(category) + urlClassification.lookupScore(category));
						urlCategoryCounts.put(category, urlCategoryCounts.get(category) + 1);
					} else {
						urlCategoryScores.put(category, urlClassification.lookupScore(category));
						urlCategoryCounts.put(category, 1);
					}
				}
			}
			
		}
		
		//Now save as a CSV Textwise classification file containing the scores
		try {
			FileOutputStream fileOut = new FileOutputStream(classificationFilename);
			PrintWriter writeOut = new PrintWriter(fileOut);
			writeOut.println("\"category\",\"textscore\",\"textcount\",\"urlscore\",\"urlcount\"");
			
			//Need to merge the category sets from both text and URL classifications
			//in case one has a category the other does not have
	
			Set<String> categorySet = new HashSet<String>(textCategoryScores.keySet());
			categorySet.addAll(urlCategoryScores.keySet());
			
			//remove the useless categories
			//categorySet.remove(null); //you can have a null key? weird
			
			for(String category : categorySet) {				
				Double textscore = textCategoryScores.get(category);
				Integer textcount = textCategoryCounts.get(category);
				Double urlscore = urlCategoryScores.get(category);
				Integer urlcount = urlCategoryCounts.get(category);
				
				//If no score exists for a category, it has not appeared in a profile
				if(textscore == null) {
					textscore = new Double(0.0);
					textcount = new Integer(0);
				}
				if(urlscore == null) {
					urlscore = new Double(0.0);
					urlcount = new Integer(0);
				}
				
				writeOut.print("\""+category+"\",");
				writeOut.print(textscore+",");
				writeOut.print(textcount+",");
				writeOut.print(urlscore+",");
				writeOut.print(urlcount);
				writeOut.println();
			}
			writeOut.close();
			System.out.println("Saved CSV Textwise classification for "+userID);
			return true;
		} catch (IOException e) {
			System.out.println("Couldn't save CSV Textwise classification for "+userID);
			e.printStackTrace();
			return false;
		}
		
	}
	
	public boolean classifyTextwiseProper() {
		
		System.out.println("Textwise classifying profile "+userID);
		
		//Check if classification already exists, to save time and allow resuming classification if something goes wrong
		String classificationFilename = "classifications/textwiseproper/"+userID+".csv";
		File classificationFile = new File(classificationFilename);
		if(classificationFile.exists()) {
			System.out.println("Textwise classification already exists for "+userID+", aborting");
			return true; //or do we want to return false...
		}
		
		//Concatenate tweets
		String concat = "";
		for(SimpleTweet tweet : tweets) {
			concat += tweet.getText();
		}
		
		Map<String,Double> classifications = new HashMap<String,Double>();
		Map<String,Integer> classificationsCount = new HashMap<String,Integer>();
		for(int i=0; i<concat.length()-500; i+=500) {
			TextwiseClassification textClassification;
			if(i+500>=concat.length()) textClassification = TextwiseClassifier.classify(concat.substring(i,concat.length()), false);
			else textClassification = TextwiseClassifier.classify(concat.substring(i,i+500), false);	
			Map<String,Double> scores = textClassification.getCategoryScores();
			for(String cat : textClassification.getCategories()) {
				if(classifications.containsKey(cat)) {
					classifications.put(cat,classifications.get(cat)+scores.get(cat));
					classificationsCount.put(cat,classificationsCount.get(cat)+1);
				} else {
					classifications.put(cat,scores.get(cat));
					classificationsCount.put(cat,1);
				}
			}
		}
		
		//Now save as a CSV Textwise classification file containing the scores
		try {
			FileOutputStream fileOut = new FileOutputStream(classificationFilename);
			PrintWriter writeOut = new PrintWriter(fileOut);
			writeOut.println("\"category\",\"score\",\"count\"");
			
			//remove the useless categories
			//categorySet.remove(null); //you can have a null key? weird
			for(String category : classifications.keySet()) {				
				writeOut.println("\""+category+"\","+classifications.get(category)+","+classificationsCount.get(category));
			}
			writeOut.close();
			System.out.println("Saved CSV Textwise classification for "+userID);
			return true;
		} catch (IOException e) {
			System.out.println("Couldn't save CSV Textwise classification for "+userID);
			e.printStackTrace();
			return false;
		}
		
	}
	
	//Concatenate all the tweets and topics
	//NOTE: This basically is only for LIWC - can be used later if we want individual tweet stuff
	public Document asDocument(String topicType) {
		if(topicType != null) {
			if(topicType.equals("alchemy")) { //TODO: Hacky...
				return asLLDADocument("alchemy");
			}
			if(topicType.equals("calais")) {
				return asLLDADocument("calais");
			}
			if(topicType.equals("textwise")) {
				return asLLDADocument("textwise");
			}
			if(topicType.equals("textwiseproper")) {
				return asLLDADocument("textwiseproper");
			}
			if(topicType.equals("liwc")) {
				return asLLDADocument("liwc");
			}
			if(topicType.equals("liwcnb")) {
				return asLLDADocument("liwcnb");
			}
		}
		
		
		//Concatenate all tweets
		String concat = "";
		for(SimpleTweet tweet : tweets) {
			concat += tweet.getText()+" ";
		}
		String strippedText = Tools.stripTweet(concat);
		Document document = new Document(strippedText, userID);
		return document;
	}
	
	public void reduceBy(int reduction) {
		if(reduction > 9) return;
		int newSize = (int)Math.ceil(tweets.size() * (1.0 - reduction/10.0));
		List<SimpleTweet> newTweets = new ArrayList<SimpleTweet>();
		Collections.shuffle(tweets);
		for(int i=0; i<newSize; i++) {
			newTweets.add(tweets.get(i));
		}
		tweets = newTweets;
	}
	
	//For use in Corpus' getFullProfileCorpus(topicType) method
	public Document asLLDADocument(String topicType) {
		Set<String> topics = new HashSet<String>();
		//Note: if not LIWC or LIWCNB, we have no topics yet!
		if(topicType.equals("alchemy")) {
			FullAlchemyClassification fac = new FullAlchemyClassification(userID);
			int topTopics = 3;
			//alchemy too sparse to threshold
			int count = 0;
			for(String topic : fac.getCategorySet()) {
				if(count == topTopics) break; //stop getting more than 3 topics
				//if(fac.getScore(topic) < scoreThreshold) break; //stop getting low-prob topics
				topics.add(topic);
				count++;
			}
		} else if(topicType.equals("calais")) {
			FullCalaisClassification fcc = new FullCalaisClassification(userID);
			int topTopics = 3;
			int count = 0;
			for(String topic : fcc.getCategorySet()) {
				if(topic.equals("Other")) continue; //really prominent...
				if(count == topTopics) break; //stop getting more than 3 topics
				//if(fac.getScore(topic) < scoreThreshold) break; //stop getting low-prob topics
				topics.add(topic);
				count++;
			}
		} else if(topicType.equals("textwise")) {
			FullTextwiseClassification ftc = new FullTextwiseClassification(userID,false);
			int topTopics = 3;
			int count = 0;
			for(String topic : ftc.getCategorySet()) {
				if(count == topTopics) break; //stop getting more than 3 topics
				//if(fac.getScore(topic) < scoreThreshold) break; //stop getting low-prob topics
				topics.add(topic);
				count++;
			}
		} else if(topicType.equals("textwiseproper")) {
			FullTextwiseClassification ftc = new FullTextwiseClassification(userID,true);
			int topTopics = 3;
			int count = 0;
			for(String topic : ftc.getCategorySet()) {
				if(count == topTopics) break; //stop getting more than 3 topics
				//if(fac.getScore(topic) < scoreThreshold) break; //stop getting low-prob topics
				topics.add(topic);
				count++;
			}
		}else if(topicType.equals("liwc")) {
			FullLIWCClassification flc = new FullLIWCClassification(false,userID);
			int topTopics = 3;
			int count = 0;
			for(String topic : flc.getCategorySet()) {
				if(count == topTopics) break;
				topics.add(topic);
				count++;
			}
		} else if(topicType.equals("liwcnb")) {
			FullLIWCClassification flc = new FullLIWCClassification(true,userID);
			int topTopics = 3;
			double threshold = 0.2;
			int count = 0;
			for(String topic : flc.getCategorySet()) {
				if(count == topTopics) break;
				if(flc.getScore(topic) < threshold) break;
				topics.add(topic);
				count++;
			}
		} else {
			System.out.println("invalid topic type");
			return null;
		}
		
		//Concatenate all tweets
		String concat = "";
		for(SimpleTweet tweet : tweets) {
			concat += tweet.getText()+" ";
		}
		String strippedText = Tools.stripTweet(concat);
		Document document = new Document(strippedText, userID, topics);
		return document;
	}
	
	public Document asDocument() {
		return asDocument(null);
	}
}
 public class Corpus implements Serializable {

	private static final long serialVersionUID = 2305142967278892458L;

	private Set<Document> documents;
	private String topicType;
	
	public Corpus() {
		documents = new HashSet<Document>();
		topicType = null;
	}
	
	public Corpus(Set<Document> documents) {
		this.documents = documents;
		topicType = null;
	}
	
	public Corpus(String topicType) {
		documents = new HashSet<Document>();
		this.topicType = topicType;
	}
	
	public Corpus(Set<Document> documents, String topicType) {
		this.documents = documents;
		this.topicType = topicType;
	}
	
	public Set<Document> getDocuments() {
		return documents;
	}
	
	public String getTopicType() {
		return topicType;
	}
	
	public List<Document> asDocumentList() {
		return new ArrayList<Document>(documents);
	}
	
	public void addDocument(Document document) {
		documents.add(document);
	}

    public void print() {
    	System.out.println("Printing a Corpus - Documents:");
    	for(Document document : documents) {
    		document.print();
    		System.out.println();
    	}
    }
    
    public void removeLeastCommonWords(double percentage, double numDocThreshold) {
    	Set<Document> newDocuments = new HashSet<Document>();
    	
    	//Discover the frequencies of each word
    	Map<String,Integer> frequencyCounts = new HashMap<String,Integer>();
    	Map<String,Integer> numDocsContaining = new HashMap<String,Integer>();
    	for(Document doc : documents) {
    		Set<String> wordsFoundInDoc = new HashSet<String>();
    		for(String word : doc.getDocumentString().split("\\s+")) {
    			if(frequencyCounts.containsKey(word)) {
    				frequencyCounts.put(word, frequencyCounts.get(word)+1);
    			} else {
    				frequencyCounts.put(word, 1);
    			}
    			
    			//If it's the first time we've seen the word in this doc, increase doc-containing count
    			if(!wordsFoundInDoc.contains(word)) {
    				wordsFoundInDoc.add(word);
        			if(numDocsContaining.containsKey(word)) {
        				numDocsContaining.put(word, numDocsContaining.get(word)+1);
        			} else {
        				numDocsContaining.put(word, 1);
        			}
    			}
    		}
    	}
    	
    	//Sort them and make a set of the least-used words
    	frequencyCounts = Tools.sortMapByValue(frequencyCounts);
    	numDocsContaining = Tools.sortMapByValue(numDocsContaining);
    	System.out.println("Initial vocab size = "+frequencyCounts.keySet().size());
    	int numToRemove = (int)((frequencyCounts.keySet().size()*percentage) / 100.0);
    	System.out.println("Removing the "+numToRemove+" least-used words");
    	int count = 0;
    	Set<String> toRemove = new HashSet<String>();
    	for(String word : frequencyCounts.keySet()) {
    		if(count >= numToRemove) break;
    		toRemove.add(word);
    		//System.out.println("Added "+word+" to the least-common set");
    		count++;
    	}
    	
    	System.out.println("Removing words that appear in "+numDocThreshold+" documents or fewer");
    	int numDocCount=0;
    	for(String word : frequencyCounts.keySet()) {
    		if(numDocsContaining.get(word) > numDocThreshold) break;
    		toRemove.add(word);
    		numDocCount++;
    		//System.out.println("Added "+word+" to the too-few-documents set");
    	}
    	System.out.println(numDocCount+" such profiles");

    	//Create a new document set and remove these words from it
    	for(Document doc : documents) {
    		Document newDoc = new Document();
;    		newDoc.setId(doc.getId());
    		newDoc.setTopics(doc.getTopics());
    		String newDocumentString = "";
    		for(String word : doc.getDocumentString().split("\\s+")) {
    			if(toRemove.contains(word)) continue;
    			else newDocumentString += (word+" ");
    		}
    		//if(newDocumentString.isEmpty()) continue; //allowing empty documents...
    		newDoc.setDocumentString(newDocumentString);
    		newDocuments.add(newDoc);
    	}
    	
    	//Finally, update the document set pointer
    	documents = newDocuments;
    }
    
    public static Corpus getFullProfileCorpus(String topicType, int reduction) {
    	Corpus corpus = new Corpus(topicType);
    	int count = 0;
    	for(Long userID : Tools.getCSVUserIDs()) {
    		if(count % 50 == 0) System.out.println("Added "+count+" profiles to Corpus");
    		count++;
    		SimpleProfile profile = Profiler.loadCSVProfile(userID);
    		profile.reduceBy(reduction);
    		corpus.addDocument(profile.asDocument(topicType));
    	}
    	return corpus;	
    }
    
    public static Corpus getFullProfileCorpus(String topicType) {
    	Corpus corpus = new Corpus(topicType);
    	int count = 0;
    	for(Long userID : Tools.getCSVUserIDs()) {
    		if(count % 50 == 0) System.out.println("Added "+count+" profiles to Corpus");
    		count++;
    		SimpleProfile profile = Profiler.loadCSVProfile(userID);
    		corpus.addDocument(profile.asDocument(topicType));
    	}
    	return corpus;
    }
    
    public static Corpus getFullProfileCorpus() {
    	return getFullProfileCorpus(null);
    }
    
    public void save(String name) {
		try {
			String filename;
			if(topicType != null) {
				filename = "corpora/"+topicType+"/"+name+".corpus";	
			} else {
				filename = "corpora/"+name+".corpus";
			}
			FileOutputStream fileOut = new FileOutputStream(filename);
			ObjectOutputStream objectOut = new ObjectOutputStream(fileOut);
			objectOut.writeObject(this);
			objectOut.close();
			System.out.println("Saved corpus "+name);
		} catch (FileNotFoundException e) {
			System.out.println("Couldn't save corpus "+name);
			e.printStackTrace();
		} catch (IOException e) {
			System.out.println("Couldn't save corpus "+name);
			e.printStackTrace();			
		}    	
    }
    
    public static Corpus load(String name) {
		try {
			String filename = "corpora/"+name+".corpus";
			FileInputStream fileIn = new FileInputStream(filename);
			ObjectInputStream objectIn = new ObjectInputStream(fileIn);
			Corpus corpus = (Corpus)objectIn.readObject();
			objectIn.close();
			System.out.println("Loaded corpus "+name);
			return corpus;
		} catch (FileNotFoundException e) {
			System.out.println("Couldn't load corpus "+name);
			e.printStackTrace();
		} catch (IOException e) {
			System.out.println("Couldn't load corpus "+name);
			e.printStackTrace();			
		} catch (ClassNotFoundException e) {
			System.out.println("Couldn't load corpus "+name);
			e.printStackTrace();			
		}
		return null;
    }
    
    public static Corpus loadLabelled(String topicType,String name) {
		try {
			String filename = "corpora/"+topicType+"/"+name+".corpus";
			FileInputStream fileIn = new FileInputStream(filename);
			ObjectInputStream objectIn = new ObjectInputStream(fileIn);
			Corpus corpus = (Corpus)objectIn.readObject();
			objectIn.close();
			System.out.println("Loaded corpus "+name);
			return corpus;
		} catch (FileNotFoundException e) {
			System.out.println("Couldn't load corpus "+name);
			e.printStackTrace();
		} catch (IOException e) {
			System.out.println("Couldn't load corpus "+name);
			e.printStackTrace();			
		} catch (ClassNotFoundException e) {
			System.out.println("Couldn't load corpus "+name);
			e.printStackTrace();			
		}
		return null;
    }
    
    public int size() {
    	return documents.size();
    }
    
    public Corpus[] split(Set<Long> ids1, Set<Long> ids2) {
    	Corpus c[] = new Corpus[2];
    	c[0] = new Corpus(topicType);
    	c[1] = new Corpus(topicType);
    	for(Document document : documents) {
    		if(ids1.contains(document.getId())) {
    			c[0].addDocument(document);
    		} else if(ids2.contains(document.getId())) {
    			c[1].addDocument(document);
    		}
    	}
    	return c;
    }
    
    public Corpus randomlyRemove(double fraction) {
    	if(fraction > 1.0) return this;
    	int toKeep = (int)(documents.size()*(1.0-fraction));
    	Corpus newCorpus = new Corpus(topicType);
    	List<Document> oldDocumentsList = new ArrayList<Document>(documents);
    	Collections.shuffle(oldDocumentsList);
    	for(int i=0; i<toKeep; i++) {
        	newCorpus.addDocument(oldDocumentsList.get(i));
    	}
    	return newCorpus;
    }
	
}
 public class WordScore implements Serializable, Comparable<WordScore> {

	private static final long serialVersionUID = -8651433241587027607L;
	
	private String word;
	private double score;
	
	public WordScore(String word, double score) {
		this.word = word;
		this.score = score;
	}
	
	public String getWord() {
		return word;
	}
	
	public double getScore() {
		return score;
	}

	public int compareTo(WordScore newScore) {
		if(score < newScore.getScore()) return -1;
		if(score > newScore.getScore()) return 1;
		return 0;
	}

}
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.ObjectOutputStream;
import java.io.PrintWriter;
import java.io.StringReader;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import uk.ac.cam.ha293.tweetlabel.classify.AlchemyClassification;
import uk.ac.cam.ha293.tweetlabel.classify.AlchemyClassifier;
import uk.ac.cam.ha293.tweetlabel.classify.FullAlchemyClassification;
import uk.ac.cam.ha293.tweetlabel.classify.FullCalaisClassification;
import uk.ac.cam.ha293.tweetlabel.classify.FullTextwiseClassification;
import uk.ac.cam.ha293.tweetlabel.eval.Pearson;
import uk.ac.cam.ha293.tweetlabel.eval.SimilarityMatrix;
import uk.ac.cam.ha293.tweetlabel.eval.SpearmanRank;
import uk.ac.cam.ha293.tweetlabel.topics.FullLLDAClassification;
import uk.ac.cam.ha293.tweetlabel.topics.FullSVMClassification;
import uk.ac.cam.ha293.tweetlabel.topics.LDATopicModel;
import uk.ac.cam.ha293.tweetlabel.topics.LLDATopicModel;
import uk.ac.cam.ha293.tweetlabel.twitter.Profiler;
import uk.ac.cam.ha293.tweetlabel.twitter.SimpleProfile;
import uk.ac.cam.ha293.tweetlabel.types.Corpus;
import uk.ac.cam.ha293.tweetlabel.types.WordScore;

	}
	
	//this one gets profiles or tweets removed vs similarity to baseline
	public static void dataGatherer() {
		double[] alphas = {1.25,1.5,1.75,2.0};
		String[] topicTypes = {"alchemy","calais","textwiseproper"};
		List<Long> uids = Tools.getCSVUserIDs();
		for(double alpha : alphas) {
			for(int reduction = 1; reduction <= 9; reduction++) {
				for(String topicType : topicTypes) {
					//get average cosine similarity to baseline
					double cosineSum = 0.0;
					int cosineCount = 0;
					for(Long uid : uids) {
						if(topicType.equals("alchemy")) {
							FullAlchemyClassification baseline = new FullAlchemyClassification(uid);
							FullLLDAClassification llda = new FullLLDAClassification(topicType,alpha,false,reduction,uid);
							double sim = llda.cosineSimilarity(baseline);
							cosineSum += sim;
							cosineCount++;
						} else if(topicType.equals("calais")) {
							FullCalaisClassification baseline = new FullCalaisClassification(uid);
							FullLLDAClassification llda = new FullLLDAClassification(topicType,alpha,false,reduction,uid);
							double sim = llda.cosineSimilarity(baseline);
							cosineSum += sim;
							cosineCount++;
						} else if(topicType.equals("textwiseproper")) {
							FullTextwiseClassification baseline = new FullTextwiseClassification(uid,true);
							FullLLDAClassification llda = new FullLLDAClassification(topicType,alpha,false,reduction,uid);
							double sim = llda.cosineSimilarity(baseline);
							cosineSum += sim;
							cosineCount++;
						}
					}
					double avgCosine = cosineSum/cosineCount;
					System.out.println(alpha+","+reduction+","+topicType+","+avgCosine);
				}
			}
		}
	}
	
	public static void kFinder(String topicType, double alpha) {
		System.out.println("Finding optimal K for "+topicType+" "+alpha);
		String[] topics = Tools.getTopics(topicType);
		int maxK = topics.length;
		for(int k=1; k<maxK; k++) {
			int totalCount = 0;
			int correctCount = 0;
			for(Long uid : Tools.getCSVUserIDs()) {
				//System.out.println(totalCount);
				totalCount++;
				Set<String> lldaTopicSet = new HashSet<String>();
				Set<String> baselineTopicSet = new HashSet<String>();
				String modTopic = topicType;
				if(modTopic.equals("textwise")) modTopic = "textwiseproper";
				FullLLDAClassification llda = new FullLLDAClassification(modTopic,alpha,uid);
				int kCount=0;
				for(String topic : llda.getCategorySet()) {
					if(kCount == k) break;
					kCount++;
					lldaTopicSet.add(topic);
				}
				if(topicType.equals("alchemy")) {
					FullAlchemyClassification baseline = new FullAlchemyClassification(uid);
					kCount=0;
					for(String topic : baseline.getCategorySet()) {
						if(kCount == k) break;
						kCount++;
						baselineTopicSet.add(topic);
					}
				} else if(topicType.equals("calais")) {
					FullCalaisClassification baseline = new FullCalaisClassification(uid);
					kCount=0;
					for(String topic : baseline.getCategorySet()) {
						if(kCount == k) break;
						if(topic.equals("Other")) continue;
						kCount++;
						baselineTopicSet.add(topic);
					}
				} else if(topicType.equals("textwise")) {
					FullTextwiseClassification baseline = new FullTextwiseClassification(uid,true);
					kCount=0;
					for(String topic : baseline.getCategorySet()) {
						if(kCount == k) break;
						kCount++;
						baselineTopicSet.add(topic);
					}
				}

				if(lldaTopicSet.equals(baselineTopicSet)) {
					correctCount++;
				}
			}
			double correctFraction = (double)correctCount/(double)totalCount;
			System.out.println(correctFraction);
		}
	}
	
	public static void bigKFinder() {
		String[] topicTypes = {"alchemy","calais","textwise"};
		double[] alphas = {0.25,0.5,0.75,1.0,1.25,1.5,1.75,2.0};
		for(double alpha : alphas) {
			for(String topicType : topicTypes) {
				kFinder(topicType,alpha);
			}
		}
	}
	
	public static void svmKFinder(String topicType) {
		System.out.println("Finding optimal K for svm "+topicType);
		String[] topics = Tools.getTopics(topicType);
		int maxK = topics.length;
		for(int k=1; k<maxK; k++) {
			int totalCount = 0;
			int correctCount = 0;
			for(Long uid : Tools.getCSVUserIDs()) {
				//System.out.println(totalCount);
				totalCount++;
				Set<String> svmTopicSet = new HashSet<String>();
				Set<String> baselineTopicSet = new HashSet<String>();

				FullSVMClassification svm = new FullSVMClassification(topicType,uid);
