public class MarketMatrixTensor implements NormalizedVectorEmbedding {
	Vocabulary vocab; // May change to share one vocabulary for all method
	Vocabulary secondvocab;
	Map<Integer, double[]> vectors;
	double[] T;
	double[] TSim;

	int dim;
	static final String rootPath = "data/bptf/";
	
	public MarketMatrixTensor() throws IOException {
		this(58);
	}
	
	public MarketMatrixTensor(int idx) throws IOException {
		String uPath = "roget_mm-"+idx+"_U.mm";
		String vPath = "roget_mm-"+idx+"_V.mm";
		vocab = new SimpleVocab(new File(rootPath+"roget_mm-voc"), new File(rootPath+"gre_oov.txt.voc"), -1);
//		secondvocab = new SimpleVocab(new File(rootPath+"oov-sense.voc"), -1);
//		vocab = new SimpleVocab(new File(rootPath+"roget_mm-"+idx+"_voc"));
		vectors = new HashMap<Integer, double[]>();

		Scanner s = new Scanner(new File(rootPath+vPath));
		s.nextLine();s.nextLine();
		String line = s.nextLine();
		String[] param = line.split("\\s");
		int N = Integer.parseInt(param[0]);
		//assert N == vocab.getVocabSize();
		dim = Integer.parseInt(param[1]);
		
		int index = 0; // assume vocabulary index equals to line number
		line = s.nextLine();
		String[] svec = line.trim().split("\\s");
		assert svec.length == dim;
		T = new double[dim];
		for (int i = 0; i < dim; i++) {
			T[i] = Float.parseFloat(svec[i]);
		}
		line = s.nextLine();
		svec = line.trim().split("\\s");
		assert svec.length == dim;
		TSim = new double[dim];
		for (int i = 0; i < dim; i++) {
			TSim[i] = Float.parseFloat(svec[i]);
		}		
		
		s = new Scanner(new File(rootPath+uPath));
		s.nextLine();s.nextLine();
		line = s.nextLine();		
		
		while (s.hasNextLine()) {
			line = s.nextLine();
			svec = line.trim().split("\\s");
			assert svec.length == dim;
			double[] vec = new double[dim];
			for (int i = 0; i < dim; i++) {
				vec[i] = Float.parseFloat(svec[i]);
			}
			vectors.put(index, vec);
			index++;
		}
		
		s = new Scanner(new File(rootPath+"gre_oov.txt"));
		
		while (s.hasNextLine()) {
			line = s.nextLine();
			svec = line.trim().split("\\s");
			assert svec.length == dim;
			double[] vec = new double[dim];
			for (int i = 0; i < dim; i++) {
			vec[i] = Float.parseFloat(svec[i]);
			}
			vectors.put(index, vec);
			index++;
		}
		
	}
	
	/*private void GREOOV() throws IOException {

		Scanner w = new Scanner(new File("data/test-data/gre_oov.txt.voc"));
		Scanner e = new Scanner(new File("data/test-data/gre_oov.txt"));
		String line;
		while (w.hasNextLine() && e.hasNextLine()) {
			line = w.nextLine().trim();
			assert vocab.lookupWord(line) == vocab.OOVindex();
			int idx = vocab.addWord(line);
			line = e.nextLine();
			String[] svec = line.trim().split("\\s");
			assert svec.length == dim;
			double[] vec = new double[dim];
			for (int i = 0; i < dim; i++) {
				vec[i] = Float.parseFloat(svec[i]);
			}
			vectors.put(idx, vec);
		}
		
	}*/
	@Override
	public int getDimension() {
		return dim;
	}

	@Override
	public Vocabulary getVocab() {
		return vocab;
	}
	
	public Vocabulary getSecondVocab() {
		return secondvocab;
	}

	@Override
	public double[] getVectorRep(int word) {
		return vectors.get(word);
	}

	@Override
	public int findClosestWord(double[] vector) {
		// TODO Auto-generated method stub
		return 0;
	}
	

	@Override
	public double similarity(int word1, int word2) {
		double sim = Util.dotProd(Util.elemWiseProd(T, vectors.get(word1)), vectors.get(word2));
		//sim = sim > 1.0 ? 1.0 : sim;
		//sim = sim < -1.0 ? -1.0 : sim;
		return sim;
		//return Util.cosineSimilarity(vectors.get(word1), vectors.get(word2));
	}
	public double getT(int i) {
		return T[i];
	}
} public class NormalizedTextFileEmbedding extends TextFileEmbedding implements NormalizedVectorEmbedding {

	public NormalizedTextFileEmbedding(File f, Vocabulary vocab)
			throws FileNotFoundException {
		super(f, vocab);
		
		//Normalize embeddings
		for (int i = 0; i < embeddings.length; i++) {
			MatrixOps.twoNormalize(embeddings[i]);
		}
	}

} public class SimpleThesaurus implements Thesaurus {
	
	//entries are sorted by word1, then by isAntonym, then by w2. This makes all the antonyms of a word contiguous, and all the synonyms of a word continuous
	static class ThesaurusEntry implements Comparable<ThesaurusEntry>, Entry {
		public int word1;
		public int word2;

		boolean isAntonym;

		public ThesaurusEntry() {
		}
		

		public ThesaurusEntry(int word1, int word2, boolean isAntonym) {
			super();
			this.word1 = word1;
			this.word2 = word2;
			this.isAntonym = isAntonym;
		}


		@Override
		public int compareTo(ThesaurusEntry o) {
			int compare = Integer.compare(this.word1, o.word1);
			if (compare == 0) {
				compare = Boolean.compare(this.isAntonym, o.isAntonym);
			}
			if (compare == 0) {
				compare = Integer.compare(this.word2, o.word2);
			}
			return compare;
		}

		@Override
		public boolean equals(Object obj) {
			if (obj instanceof ThesaurusEntry) {
				ThesaurusEntry other = ((ThesaurusEntry) obj);
				return word1 == other.word1 && isAntonym == other.isAntonym
						&& this.word2 == other.word2;
			}
			return false;
		}

		@Override
		public int word1() {
			return word1;
		}

		@Override
		public int word2() {
			return word2;
		}

		@Override
		public boolean isAntonym() {
			return isAntonym;
		}

	}

	Vocabulary vocab;

	ThesaurusEntry[] entries;

	int numSynonyms;
	public SimpleThesaurus(Vocabulary vocab, File synonyms, File antonyms)
			throws FileNotFoundException {
		this.vocab = vocab;
		this.numSynonyms=0;
		List<ThesaurusEntry> ent =new ArrayList<ThesaurusEntry>();
		Scanner s = new Scanner(synonyms);
		while (s.hasNextLine()) {
			String[] wordz = s.nextLine().split("\\s");
			int w1 = vocab.lookupWord(wordz[0]);
			if (w1 == vocab.OOVindex()) {
				continue;
			}

			for (int i = 1; i < wordz.length; i++) {
				int w2 = vocab.lookupWord(wordz[i]);
				if (w2 == vocab.OOVindex()) {
					continue;
				}
				ent.add(new ThesaurusEntry(w1,w2,false));
				numSynonyms++;
			}

		}

		s = new Scanner(antonyms);
		while (s.hasNext()) {
			String[] wordz = s.nextLine().split("\\s");
			int w1 = vocab.lookupWord(wordz[0]);
			if (w1 == vocab.OOVindex()) {
				continue;
			}

			for (int i = 1; i < wordz.length; i++) {
				int w2 = vocab.lookupWord(wordz[i]);
				if (w2 == vocab.OOVindex()) {
					continue;
				}
				ent.add(new ThesaurusEntry(w1,w2,true));
			}
		}
		entries = ent.toArray(new ThesaurusEntry[ent.size()]);
		Arrays.sort(entries);
	}

	@Override
	public Vocabulary getVocab() {
		return vocab;
	}

	@Override
	public int numEntries() {
		return entries.length;
	}
	public int numSynonyms() {
		return numSynonyms;
	}
	public int numAntonyms() {
		return numEntries()-numSynonyms();
	}

	@Override
	public Entry getEntry(int entryn) {
		return entries[entryn];
	}
	
	public int entryStart(int word1, int word2, boolean isant) {
		int ind=lookupEntry(word1, word2, isant);
		if (ind < 0) {
			return -(ind+1); //the insertion point according to the binary search api
		} else {
			return ind;
		}
	}
	
	public int lookupEntry(int word1, int word2, boolean isant) {
		ThesaurusEntry key = new ThesaurusEntry();
		key.word1 = word1;
		key.word2 = word2;
		key.isAntonym = isant;
		return Arrays.binarySearch(entries, key);
	}
	
	List<Integer> lookupentrylist(int word, boolean isant) {
		int first = entryStart(word, 0, isant);
		int last;
		for (last = first; last < entries.length; last++) {
			ThesaurusEntry e = entries[last];
			if (e.word1 != word || e.isAntonym != isant) {
				break;
			}
		}
		List<Integer> result = new ArrayList<Integer>(last - first);
		for (int i = first; i < last; i++) {
			result.add(entries[i].word2);
		}
		return result;
	}

	public List<Integer> getAntonyms(int word) {
		return lookupentrylist(word, true);
	}

	public List<Integer> getSynonyms(int word) {
		return lookupentrylist(word, false);
	}


	@Override
	public boolean isAntonym(int word1, int word2) {
		return lookupEntry(word1, word2, true)>=0;
	}

	@Override
	public boolean isSynonym(int word1, int word2) {
		return lookupEntry(word1, word2, false)>=0;
	}

} public class SimpleVocab implements Vocabulary {

	List<String> words;
	Map<String, Integer> ids;
	
	int OOVindex;
	

	public SimpleVocab(File f) throws FileNotFoundException {
		this(f,0);
	}
	public SimpleVocab(InputStream stream) {
		this(stream,0);
	}
	public SimpleVocab(File f,int OOVindex) throws FileNotFoundException {
		this(new FileInputStream(f),OOVindex);
	}
	/**
	 * Creates vocabulary from newline delimited text stream
	 * @param stream
	 */
	
	public SimpleVocab(InputStream stream, int OOVindex) {
		words = new ArrayList<String>();
		ids = new HashMap<String, Integer>();
		Scanner scan=new Scanner(stream);
		int index=0;
		while(scan.hasNextLine()) {
			String word=scan.nextLine();
			words.add(word);
			ids.put(word,index++);
		}
		scan.close();
		this.OOVindex=OOVindex;
	}
	
	public SimpleVocab(File file, File file2, int i) throws FileNotFoundException {
		words = new ArrayList<String>();
		ids = new HashMap<String, Integer>();
		Scanner scan=new Scanner(file);
		int index=0;
		while(scan.hasNextLine()) {
			String word=scan.nextLine();
			words.add(word);
			ids.put(word,index++);
		}
		scan=new Scanner(file2);
		while(scan.hasNextLine()) {
			String word=scan.nextLine();
			words.add(word);
			ids.put(word,index++);
		}
		scan.close();
		this.OOVindex=i;
	}
	
	@Override
	public int size() {
		return words.size();
	}
/**
 * Returns the index in the dictionary for out-of-vocabulary words.
 */
	@Override
	public int OOVindex() {
		return OOVindex;
	}

	@Override
	public String lookupIndex(int index) {
		if (index == -1) 
			return "__OUT_OF_VOCABULARY__";
		return words.get(index);
	}

	@Override
	public int lookupWord(String word) {
		if (ids.containsKey(word))
			return ids.get(word);
		return OOVindex;
	}

	public int getVocabSize() {
		return words.size();
	}
} public class Util {
	static void saveVectors(VectorEmbedding embed, Vocabulary vocab, File out) throws FileNotFoundException {
		int dim=embed.getDimension();
		
		PrintStream ps=new PrintStream(out);
		ps.println(dim);
		for(int i=0; i<vocab.size(); i++) {
			double[] vect=embed.getVectorRep(i);
			for(int j=0; j<dim; j++) {
				ps.print(vect[j]);
				ps.print(' ');
			}
			ps.println();
		}
		ps.close();
	}
	
	public static double cosineSimilarity(double[] vector1, double[] vector2) {
		double dotProduct = 0.0;
		double v1 = 0.0;
		double v2 = 0.0;
		double cosineSimilarity = 0.0;

        for (int i = 0; i < vector1.length; i++) //vector1 and vector2 must be of same length
        {
            dotProduct += vector1[i] * vector2[i];  
            v1 += Math.pow(vector1[i], 2);  
            v2 += Math.pow(vector2[i], 2); 
        }

        v1 = Math.sqrt(v1);	
        v2 = Math.sqrt(v2);	

        if (v1 != 0.0 && v2 != 0.0) {
            cosineSimilarity = dotProduct / (v1 * v2);
        } else {
            return 0;
        }
        return cosineSimilarity;
    }
	public static Random r=new Random();
	
	public static double dotProd(double[] vector1, double[] vector2) {
		double dotProduct = 0.0;
        for (int i = 0; i < vector1.length; i++) //vector1 and vector2 must be of same length
        {
            dotProduct += vector1[i] * vector2[i];  
        }
        return dotProduct;
    }
	
	public static double[] elemWiseProd(double[] vector1, double[] vector2) {
		double[] Product = new double[vector1.length];
        for (int i = 0; i < vector1.length; i++) //vector1 and vector2 must be of same length
        {
            Product[i] = vector1[i] * vector2[i];  
        }
        return Product;
    }
		
	public static int hashInteger(int i, int max) {
		r.setSeed(i);
		return r.nextInt(max);
	}

	public static int hashIntegerPair(int i, int j, int max) {
		return hashInteger(31*i+j, max);
	}
	
	static class Pair {
		int a;
		int b;
		static int max;
		public Pair(int a, int b) {
			this.a = a;
			this.b = b;
		}
		@Override
	    public int hashCode() {
			int aa = a > b ? a:b;
			int bb = a > b ? b:a;	
			return aa * max + bb;
		}
		@Override
		public boolean equals(Object o) {       
	        Pair c = (Pair) o;	        
	        return a == c.a && b == c.b || a == c.b && b == c.a;
		}
	}
	
	
	public static void main(String[] args) throws IOException{
		Thesaurus t = new ThesaurusImp(new File("data/Rogets/antonym.txt"), new File("data/Rogets/synonym.txt"), new File("data/Rogets/vocabulary.txt"));
	}
} public class LinearVectorMetric implements Optimizable.ByGradientValue,
		WordMetric {
	Thesaurus th;
	VectorEmbedding orig;
	// lower diagonal half of symmetric matrix
	// 0
	// 1 2
	// 3 4 5
	// 6 7 8 9
	double[] parameters;

	int vocabSize;

	double regularizationStrength = 1;

	// cached optimization values;
	boolean cacheDirty;
	double cachedValue;
	double[] cachedGradient;

	public LinearVectorMetric(NormalizedVectorEmbedding orig) {
		this.orig = orig;
		parameters = new double[(orig.getDimension() * (orig.getDimension() + 1)) / 2];
		this.vocabSize = orig.getVocab().getVocabSize();

		// Initialize the embedding to the identity matrix
		int ind = 0;
		for (int i = 0; i < orig.getDimension(); i++) {
			parameters[ind] = 1.0d;
			ind += (i + 2);
		}

		cacheDirty = true;
		cachedGradient = new double[parameters.length];
	}

	public void setThesaurus(Thesaurus th) {
		this.th = th;
		cacheDirty = true;
	}

	@Override
	public int getNumParameters() {
		return parameters.length;
	}

	@Override
	public double getParameter(int arg0) {
		return parameters[arg0];
	}

	@Override
	public void getParameters(double[] arg0) {
		System.arraycopy(parameters, 0, arg0, 0, parameters.length);
	}

	@Override
	public void setParameter(int arg0, double arg1) {
		parameters[arg0] = arg1;
		cacheDirty = true;
	}

	@Override
	public void setParameters(double[] arg0) {
		System.arraycopy(arg0, 0, parameters, 0, parameters.length);
		cacheDirty = true;
	}

	public double innerProd(double[] vec1, double[] vec2) {

		double tot = 0;

		int ind = 0;
		for (int i = 0; i < orig.getDimension(); i++) {
			for (int j = 0; j < i; j++) {
				// Calculate the contribution from this entry and the reflected
				// entry in the upper half part
				tot += vec1[i] * parameters[ind] * vec2[j];
				tot += vec1[j] * parameters[ind] * vec2[i];
				ind++;
			}
			tot += vec1[i] * parameters[ind] * vec2[i];
			ind++;
		}
		return tot;
	}

	double cosineSimGradAndValue(int w1ind, int w2ind, double[] gradOut) {
		double[] vec1 = orig.getVectorRep(w1ind);
		double[] vec2 = orig.getVectorRep(w2ind);

		double n = innerProd(vec1, vec1);
		double m = innerProd(vec2, vec2);
		double l = innerProd(vec1, vec2);

		// Value of cosine similarity
		double value = l;// / Math.sqrt(n * m);

		// System.out.println("l="+l+" n="+n+" m="+m+" cosinesim="+value);

		// Now we calculate the gradient
		// See doc/linearembeddingtheory for the math behind this

		double denom = Math.pow(n * m, 3 / 2);

		int ind = 0;
		for (int i = 0; i < orig.getDimension(); i++) {
			for (int j = 0; j < i; j++) {
				// Off diagonal elements
				gradOut[ind] = n * m * (vec1[i] * vec2[j] + vec1[j] * vec2[i])
						- l * m * vec1[i] * vec1[j] - l * n * vec2[i] * vec2[j];
				gradOut[ind] /= denom;

				gradOut[ind] = vec1[i] * vec2[j] + vec1[j] * vec2[i];
				ind++;
			}
			// on diagonal elements
			gradOut[ind] = n * m * vec1[i] * vec2[i] - 0.5 * l * m * vec1[i]
					* vec1[i] - 0.5 * l * n * vec2[i] * vec2[i];
			gradOut[ind] /= denom;

			gradOut[ind] = vec1[i] * vec2[i];

			ind++;
		}
		return value;
	}

	void computeGradientAndValue() {
		cacheDirty = false;
		cachedValue = 0;
		Arrays.fill(cachedGradient, 0.0d);

		double[] gradTmp1 = new double[parameters.length];
		double[] gradTmp2 = new double[parameters.length];

		for (int i = 0; i < th.numEntries(); i++) {
			Entry ent = th.getEntry(i);

			int randSample = Util.r.nextInt(vocabSize);

			double entsim = cosineSimGradAndValue(ent.word1(), ent.word2(),
					gradTmp1);

			if (ent.isAntonym()) {
				cachedValue -= (entsim + 1) * (entsim + 1)/th.numAntonyms();
				for (int p = 0; p < parameters.length; p++) {
					cachedGradient[p] -= 2 * (entsim + 1) * gradTmp1[p]/th.numAntonyms();
				}
			} else {
				cachedValue -= (entsim - 1) * (entsim - 1)/th.numSynonyms();
				for (int p = 0; p < parameters.length; p++) {
					cachedGradient[p] -= 2 * (entsim - 1) * gradTmp1[p]/th.numSynonyms();
				}
			}

		}

		cacheDirty = false;
	}

	@Override
	public void getValueGradient(double[] arg0) {
		if (cacheDirty) {
			computeGradientAndValue();
		}
		System.arraycopy(cachedGradient, 0, arg0, 0, cachedGradient.length);
	}

	@Override
	public double getValue() {
		if (cacheDirty) {
			computeGradientAndValue();
		}
		return cachedValue;

	}

	@Override
	public Vocabulary getVocab() {
		return orig.getVocab();
	}

	@Override
	public double similarity(int word1, int word2) {
		double[] d1 = orig.getVectorRep(word1);
		double[] d2 = orig.getVectorRep(word2);
		return innerProd(d1, d2);
	}
} public interface NormalizedVectorEmbedding extends VectorEmbedding {

} public class SubThesaurus implements Thesaurus {

	Thesaurus parent;
	List<Integer> indices;
	int numSynonyms;
	int numAntonyms;

	public SubThesaurus(Thesaurus parent, List<Integer> indices) {
		super();
		this.parent = parent;
		this.indices = indices;
		Collections.sort(this.indices);
		this.numSynonyms=0;
		this.numAntonyms=0;
		for(Integer i:this.indices) {
			if(parent.getEntry(i).isAntonym()) {
				numAntonyms++;
			} else {
				numSynonyms++;
			}
		}
	}

	public void addEntry(int entry) {
		indices.add(entry);
	}

	@Override
	public Vocabulary getVocab() {
		return parent.getVocab();
	}

	@Override
	public int numEntries() {
		return indices.size();
	}

	@Override
	public Entry getEntry(int entryn) {
		return parent.getEntry(indices.get(entryn));
	}

	@Override
	public boolean isAntonym(int word1, int word2) {
		return lookupEntry(word1, word2, true)>=0;
	}

	@Override
	public boolean isSynonym(int word1, int word2) {
		return lookupEntry(word1, word2, false)>=0;
	}

	@Override
	public int lookupEntry(int word1, int word2, boolean isAnt) {
		int ent=parent.lookupEntry(word1, word2, isAnt);
		return Collections.binarySearch(indices, ent);
	}
	
//Samples a subthesaurus with n entries
	public static Thesaurus SampleThesaurus(Thesaurus th, int n) {
		int thsize=th.numEntries();
		List<Integer> sample=new ArrayList<Integer>(n);
		for(int i=0; i<n; i++) {
			sample.add(Util.r.nextInt(thsize));
		}
		return new SubThesaurus(th, sample);
	}
	public static Thesaurus[] evenSplitThesaurus(Thesaurus th, int folds) {
		double[] probs = new double[folds];
		Arrays.fill(probs, 1 / folds);
		return splitThesaurus(th, probs);
	}

	// This just randomly divvies up the entries of the thesaurus, however, it
	// makes sure to put an entry (word1,word2) in the same partition as the
	// entry
	// (word2,word1).
	public static Thesaurus[] splitThesaurus(Thesaurus th,
			double[] probabilities) {
		class Key {

			private final int x;
			private final int y;

			public Key(int x, int y) {
				this.x = x;
				this.y = y;
			}

			@Override
			public boolean equals(Object o) {
				if (this == o)
					return true;
				if (!(o instanceof Key))
					return false;
				Key key = (Key) o;
				return x == key.x && y == key.y;
			}

			@Override
			public int hashCode() {
				int result = x;
				result = 31 * result + y;
				return result;
			}

		}
		Map<Key, Integer> entrymap = new HashMap<>();
		SubThesaurus[] result = new SubThesaurus[probabilities.length];
		for (int i = 0; i < probabilities.length; i++) {
			result[i] = new SubThesaurus(th, new ArrayList<Integer>());
		}
		for (int i = 0; i < th.numEntries(); i++) {
			Entry ent = th.getEntry(i);
			Integer assignment = entrymap
					.get(new Key(ent.word2(), ent.word1()));
			// If we haven't assigned the reversal of this entry yet, we put it
			// in a random partition
			if (assignment == null) {
				double r = Util.r.nextDouble();
				double tot = 0;
				for (int j = 0; j < probabilities.length; j++) {
					tot += probabilities[j];
					if (tot > r) {
						assignment = j;
						break;
					}
				}
				if (assignment == null) {
					assignment = probabilities.length - 1;
				}
			}
			result[assignment].indices.add(i);
			entrymap.put(new Key(ent.word1(), ent.word2()), assignment);
		}
		return result;
	}

	@Override
	public int numSynonyms() {
		return numSynonyms;
	}

	@Override
	public int numAntonyms() {
		return numAntonyms;
	}

} public interface Thesaurus {
	public interface Entry {
		public int word1();
		public int word2();
		public boolean isAntonym();
	}
	public Vocabulary getVocab();
	
	//Returns the words included in the Thesaurus
	public int numEntries();
	public int numSynonyms();
	public int numAntonyms();
	public Entry getEntry(int entryn);
	
	public int lookupEntry(int word1, int word2, boolean isAnt);

	
	public boolean isAntonym(int word1, int word2);
	public boolean isSynonym(int word1, int word2);
} public interface VectorEmbedding extends WordMetric {

	int getDimension();
	public double[] getVectorRep(int word);
	public int findClosestWord(double[] vector);
	Vocabulary getVocab();
} public interface Vocabulary {
	public int size();
	public int OOVindex();
	public String lookupIndex(int index);
	public int lookupWord(String word);
	public int getVocabSize();
} public class RandomizedTestCase implements MetricEvaluator {

	public Thesaurus th;

	public RandomizedTestCase(Thesaurus th) {
		this.th = th;
	}

	@Override
	public double score(WordMetric metric) throws IOException {

		String result_folder = "data/result-data/";
		new File(result_folder).mkdirs();

		double acc = 0.0d;

		Vocabulary vocab = metric.getVocab();

		int ant_num = 0;
		int syn_num = 0;
		double mse_ant = 0.0;
		double mse_syn = 0.0;
		FileWriter out = new FileWriter(result_folder + "ant.txt");

		for (int i = 0; i < th.numEntries(); i++) {
			Entry ent = th.getEntry(i);
			out.append(vocab.lookupIndex(ent.word1()) + " "
					+ vocab.lookupIndex(ent.word2()) + " ");
			double cs = metric.similarity(ent.word1(), ent.word2());
			if (ent.isAntonym()) {
				ant_num++;
				mse_ant+=(-1-cs)*(-1-cs);
				out.append(Double.toString(cs));
			} else {
				syn_num++;
				mse_syn+=(1-cs)*(1-cs);
				out.append(Double.toString(cs));
			}
			out.append("\n");
		}
		mse_ant = mse_ant / ant_num;
		mse_syn = mse_syn / syn_num;
		acc -= mse_ant;
		acc -= mse_syn;
		out.append("\n");
		out.append("Mean squared error for antonym is: " + mse_ant + "\n");
		System.out
				.println("[SIMPLE ANT TEST] Mean squared error for antonym is: "
						+ Double.toString(mse_ant));
		if (syn_num != 0) {
			out.append("Mean squared error for synonym is: " + mse_syn + "\n");
			System.out
					.println("[SIMPLE ANT TEST] Mean squared error for synonym is: "
							+ Double.toString(mse_syn));
		}
		out.close();

		return acc;
	}

} public class TestCase implements MetricEvaluator{
	
	/*	
	 * 	Input arguments:
	 *  Usage: 
 	 *	-a    test files with antonyms
 	 *	-s    test files with synonyms
 	 *	-h    print out usage
 	 *	
	 *  example:
	 *  	String[] input = {"-s", "data/test-data/syn0.txt", "data/test-data/syn1.txt"};
	 *		TestCase n = new TestCase(input);
	 *		double score = n.score(metric);
	 *
	 *		or 
	 *		String[] input = {"-a", "data/test-data/ant0.txt"};
	 *		TestCase n = new TestCase(input);
	 *		double score = n.score(metric);
	 *
	 *		or 
	 *		String[] input = {"-a", "data/test-data/ant0.txt", "-s", "data/test-data/syn0.txt",};
	 *		TestCase n = new TestCase(input);
	 *		double score = n.score(metric);
	 *		Note: 
	 *		You can still see standard output results for both ant and syn
	 *			, but only the last score would be returned
	 *
	 */
	public ArrayList<String> antfiles = new ArrayList<String>();
	public ArrayList<String> synfiles = new ArrayList<String>();
	
	@SuppressWarnings("static-access")
	private static Options createOptions() {
	    Options options = new Options();
	    options.addOption(OptionBuilder.hasArgs().withDescription("followed by test files with antonyms").isRequired(false).create("a"));
	    options.addOption(OptionBuilder.hasArgs().withDescription("followed by test files with synonyms").isRequired(false).create("s"));
	    options.addOption("h", false, "print out usage");
	    return options;
	  }

	private static void showHelp(Options options) {
	    HelpFormatter h = new HelpFormatter();
	    h.printHelp("help", options);
	    System.exit(-1);
	  }
	  
	public TestCase(String[]  args){
		Options options = createOptions();
	    try {
	      CommandLineParser parser = new PosixParser();
	      CommandLine cmd = parser.parse(options, args);

	      if(cmd.hasOption("s")){
	    	  String path[] = cmd.getOptionValues("s"); 
	    	  for(int j=0;j<path.length;j++){
	    		  try{
	    			  FileInputStream f = new FileInputStream(path[j]);
		    		  synfiles.add(path[j]);
	    		  }catch(FileNotFoundException e){
	    			  System.out.println("File not found, exit");
	    			  System.exit(0);
	    		  }
	    		  
	    	  }  
	      }
	      if(cmd.hasOption("a")){
	    	  String path[] = cmd.getOptionValues("a"); 
	    	  for(int j=0;j<path.length;j++){
	    		  try{
	    			  FileInputStream f = new FileInputStream(path[j]);
		    		  antfiles.add(path[j]);
	    		  }catch(FileNotFoundException e){
	    			  System.out.println("File not found, exit");
	    			  System.exit(0);
	    		  }
	    	  }  
	      }
	      if(cmd.hasOption("h")){
	    	  showHelp(options);
	      }
	      if((!cmd.hasOption("h"))&&(!cmd.hasOption("a"))&&(!cmd.hasOption("s"))){
	          showHelp(options);
	      }
	    } catch (Exception e) {
	      e.printStackTrace();
	      showHelp(options);
	    }
		
	}
	
	@Override
	public double score(WordMetric metric) throws IOException {

		String result_folder = "data/result-data/";
		new File(result_folder).mkdirs();

		double acc = 0.0d;
		
		if(!antfiles.isEmpty()){
			
			Vocabulary vocab = metric.getVocab();
			int OOV = vocab.OOVindex();
			
			for(int i=0;i<antfiles.size();i++){
				int ant_num = 0;
				int syn_num = 0;
				double mse_ant = 0.0;
				double mse_syn = 0.0;
				
				FileWriter out = new FileWriter(result_folder+ "ant"+i+".txt"); 
				BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(antfiles.get(i))));
				String line = br.readLine();
				while(line!=null){
					String[] words = line.split("\\s+");
					ArrayList<Integer> wids = new ArrayList<Integer>();
					for(int j=0;j<words.length;j++){
						int wordID = vocab.lookupWord(words[j]);
						if (wordID != OOV) {
							wids.add(wordID);
							out.append(words[j]+" ");
						}
						else {
							out.append("No such word: " + words[j]+ " ");
						}
					}	
					if(wids.size()==2 && words.length==2){
						double cs = metric.similarity(wids.get(0), wids.get(1));
						ant_num++;
						mse_ant+= Math.pow(-1-cs,2);
						out.append(Double.toString(cs));
					}
					else if(wids.size()==3){
						double cs = metric.similarity(wids.get(0), wids.get(1));
						ant_num++;
						mse_ant+= Math.pow(-1-cs,2);
						out.append(Double.toString(cs)+" ");
						cs =metric.similarity(wids.get(0), wids.get(2));
						ant_num++;
						mse_ant+= Math.pow(-1-cs,2);
						out.append(Double.toString(cs)+" ");
						cs = metric.similarity(wids.get(1), wids.get(2));
						syn_num++;
						mse_syn+= Math.pow(1-cs,2);
						out.append(Double.toString(cs)+" ");
					}
								
					out.append("\n");
					line = br.readLine();
				}
				mse_ant = mse_ant/ant_num;
				mse_syn = mse_syn/syn_num;
				acc-=mse_ant;
				out.append("\n");
				out.append("Mean squared error for antonym is: " + mse_ant +"\n");
				System.out.println("[SIMPLE ANT TEST" + i + "] Mean squared error for antonym is: " + Double.toString(mse_ant));
				if(syn_num!=0){
					out.append("Mean squared error for synonym is: " + mse_syn +"\n");
					System.out.println("[SIMPLE ANT TEST" + i + "] Mean squared error for synonym is: " + Double.toString(mse_syn));
				}
				out.close();
				br.close();
			}
			System.out.println("Overall accuracy is: "+ acc);
		}
		if(!synfiles.isEmpty()){
			Vocabulary vocab = metric.getVocab();
			int OOV = vocab.OOVindex();
			
			acc = 0.0d;
			for (int i = 0; i < synfiles.size(); i++) {
				int syn_num = 0;
				double mse_syn = 0.0;

				FileWriter out = new FileWriter(result_folder + "syn"+i+".txt");
				BufferedReader br = new BufferedReader(new InputStreamReader(
						new FileInputStream(synfiles.get(i))));
				String line = br.readLine();

				while (line != null) {
					String[] words = line.split("\\s+");
					ArrayList<Integer> vectors = new ArrayList<Integer>();
					for (int j = 0; j < words.length; j++) {
						int wordID = vocab.lookupWord(words[j]);
						if (wordID != OOV) {
							vectors.add(wordID);
							out.append(words[j] + " ");
						} else {
							out.append("No such word: " + words[j] + " ");
						}
					}

					for (int k = 0; k < vectors.size(); k++) {
						double cs = metric.similarity(
								vectors.get(k % vectors.size()),
								vectors.get((k + 1) % vectors.size()));
						syn_num++;
						mse_syn += Math.pow(1 - cs, 2);
						out.append(Double.toString(cs) + " ");
					}

					out.append("\n");
					line = br.readLine();
				}
				mse_syn = mse_syn / syn_num;
				acc -= mse_syn;
				out.append("\n");
				out.append("Mean squared error for synonym is: " + mse_syn + "\n");
				System.out.println("[SIMPLE SNY TEST" + i
						+ "] Mean squared error for synonym is: "
						+ Double.toString(mse_syn));
				out.close();
				br.close();

			}
			System.out.println("Overall accuracy is: "+ acc);
		}
		
		return acc;
	}


} public class TestCaseGRE implements MetricEvaluator {
	
	@Override
	public double score(WordMetric metric) throws IOException {
		return score(metric, "testset.txt");
	}
	
	public double score(WordMetric metric, String test0) throws IOException {
		
		String test_folder = "data/test-data/";
		String result_folder = "data/result-data/";
		new File(result_folder).mkdirs();		
		
		
		ArrayList<String> gres = new ArrayList<String>();
		double acc = 0.0d;
		gres.add(test0);	//add more file if needed
		
		//Vocabulary smallvocab = new SimpleVocab(new File("/home/jingwei/workspace/pmf/roget_mm-voc"), -1);
		Vocabulary vocab = metric.getVocab();
		int OOV = vocab.OOVindex();
		
		for(int i=0;i<gres.size();i++){
			int accuracy_pos = 0;
			int accuracy_neg = 0;
			int all = 0;
			
			FileWriter out = new FileWriter(result_folder+ gres.get(i) ); 
			BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(test_folder+gres.get(i))));
			String line = br.readLine();
			while(line!=null){
				all++;
				int index_target = line.indexOf(":");
				int index_answer = line.indexOf("::");
				String target = line.substring(0,index_target);
				String choice = line.substring(index_target+2,index_answer);
				String answer = line.substring(index_answer+3,line.length());
				String[] choices = choice.split("\\s+");
				
				int wordID = vocab.lookupWord(target);
				if (wordID != OOV) {
					if (vocab.lookupWord(answer) == OOV) {
						line = br.readLine();
						continue;
					}
					
					out.append(target+" ");
				}
				else {
					out.append("Cannot find target: " + target);
					out.append(" WRONG ");
					accuracy_neg++;
					//all--;
					out.append("\n");

					line = br.readLine();
					continue;
				}
				if(vocab.lookupWord(answer)==OOV) {
					line=br.readLine();
					continue;
				}
				
				// calculate dist to target from each choice
				double closest = Double.MAX_VALUE;
				String test_answer = null;
				
				for(int j=0;j<choices.length;j++){
					if(vocab.lookupWord(choices[j])==OOV) {
					continue;
					}
					int choiceID = vocab.lookupWord(choices[j]);
					if (choiceID != OOV) {
						out.append(choices[j]+" ");
						double cs = metric.similarity(wordID, choiceID);
						if(cs<closest){
							closest = cs;
							test_answer = choices[j];
						}
					}
					else {
						out.append("Cannot find: " + choices[j]+ " ");
					}
				}	
				
				out.append("result: " +test_answer);
				out.append("("+Double.toString(closest)+") ");
				out.append("true: "+answer);
				if (vocab.lookupWord(answer) != OOV)
					out.append("("+metric.similarity(wordID, vocab.lookupWord(answer))+") ");
				
				//accuracy
				if(test_answer.equals(answer)){
					out.append(" RIGHT ");
					accuracy_pos++;
				}
				else{
					out.append(" WRONG ");
					accuracy_neg++;
				}
				out.append("\n");
				
				line = br.readLine();
			}
			
			out.append("\n");
			double accuracy = 0.0;
			double recall = 0.0;
			if((accuracy_pos+accuracy_neg)!=0){
				accuracy = ((double)accuracy_pos)/(double)(accuracy_pos+accuracy_neg);
				recall = ((double)accuracy_pos)/(double)(all);
			}
			double f1 = 2*accuracy*recall/(accuracy+recall);
			acc+=recall;
			out.append("Final accuracy = "+ Double.toString(accuracy));
			System.out.println("[GRE TEST] Final accuracy = " + Double.toString(accuracy) + "\t" + Double.toString(recall) + "\t" + Double.toString(f1));
			System.out.println(""+accuracy_pos+"\t"+(accuracy_pos+accuracy_neg));
			br.close();
			out.close();
		}
		
		return acc;
	}
	
	
}
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import cc.mallet.optimize.ConjugateGradient;
import cc.mallet.optimize.GradientAscent;
import cc.mallet.optimize.LimitedMemoryBFGS;
import cc.mallet.optimize.Optimizable;
import cc.mallet.optimize.OptimizationException;
import cc.mallet.optimize.Optimizer;
import cc.mallet.optimize.tests.TestOptimizable;
import edu.antonym.MarketMatrixTensor;
import edu.antonym.NormalizedTextFileEmbedding;
import edu.antonym.SimpleThesaurus;
import edu.antonym.SimpleVocab;
import edu.antonym.Util;
import edu.antonym.metric.LinearVectorMetric;
import edu.antonym.prototype.NormalizedVectorEmbedding;
import edu.antonym.prototype.SubThesaurus;
import edu.antonym.prototype.Thesaurus;
import edu.antonym.prototype.VectorEmbedding;
import edu.antonym.prototype.Vocabulary;
import edu.antonym.test.RandomizedTestCase;
import edu.antonym.test.TestCase;
import edu.antonym.test.TestCaseGRE;
package edu.antonym.nn.training;



public class LinearMetricTrainer {

	Optimizer cachedOptimizer;
	LinearVectorMetric embedding;

	int batchsize;
	int numiterations = Integer.MAX_VALUE;

	double regweight;
	boolean LBFGS;

	public LinearMetricTrainer(NormalizedVectorEmbedding orig) {
		this(orig, 1000, 0.1, false);
	}

	public LinearMetricTrainer(NormalizedVectorEmbedding orig, int batchsize,
			double regweight, boolean LBFGS) {
		this.embedding = new LinearVectorMetric(orig);
		this.batchsize = batchsize;
		this.regweight = regweight;
		this.LBFGS = LBFGS;
	}

	public void setNumIterations(int n) {
		this.numiterations = n;
	}

	public Optimizer getOptimizer() {
		if (cachedOptimizer == null) {
			Optimizable.ByGradientValue o =embedding;
					//new RegularizedOptimizable(					embedding, Math.sqrt(embedding.getNumParameters()), regweight);
			if (LBFGS) {
				cachedOptimizer = new LimitedMemoryBFGS(o);
			} else {
				GradientAscent a=new GradientAscent(o);
				cachedOptimizer = a;
			}
		}
		return cachedOptimizer;
	}

	public LinearVectorMetric train(Thesaurus th) {
		return train(th, numiterations);
	}

	public LinearVectorMetric train(Thesaurus th, int iterations) {
		embedding.setThesaurus(th);
		Optimizer optim = getOptimizer();
		System.out.println("Beginning Batch training. Objective value "
				+ embedding.getValue());
		training_loop: for (int i = 0; i < iterations; i++) {
			boolean converged = true;
			for (int j = 0; j < th.numEntries() / batchsize; j++) {
				try {
					embedding.setThesaurus(SubThesaurus.SampleThesaurus(th,
							batchsize));
					if (!optim.optimize(1)) { // if convergence
						converged=false;
					} else {
						break training_loop;
					}
				} catch (OptimizationException e) {
				}
				System.out.println("One minibatch complete. Batch value "
						+ embedding.getValue());
			}
			embedding.setThesaurus(th);
			System.out
					.println("One pass of dataset complete.  Objective value "
							+ embedding.getValue());
			if (converged) {
				System.out.println("Converged!");
				break training_loop;
			}
		}

		return embedding;
	}

	public static void main(String[] args) throws IOException {
	MarketMatrixTensor origEmbed=new MarketMatrixTensor(58);
		Vocabulary vocab = origEmbed.getVocab();
		//Vocabulary vocab = new SimpleVocab(new File("data/huangvocab.txt"));
		Thesaurus th = new SimpleThesaurus(vocab, new File(
				"data/combine/synonym.txt"), new File("data/combine/antonym.txt"));
		//NormalizedVectorEmbedding origEmbed = new NormalizedTextFileEmbedding(
		//		new File("data/huangvectors.txt"), vocab);
		
		
		Thesaurus[] split = SubThesaurus.splitThesaurus(th, new double[] { 0.8,
				0.2 }); // train/test split
		LinearMetricTrainer t = new LinearMetricTrainer(origEmbed, 1000, 1,
				true);

		int ind = 0;
		for (int i = 0; i < origEmbed.getDimension(); i++) {
			t.embedding.setParameter(ind, origEmbed.getT(i));
			ind += (i + 2);
		}
		LinearVectorMetric m = t.train(split[0],100);


		RandomizedTestCase n = new RandomizedTestCase(split[1]);
		double score = n.score(m);
		System.out.println("score = " + score);

		score = n.score(origEmbed);
		System.out.println("origscore = " + score);

	
		n = new RandomizedTestCase(split[0]);
		score = n.score(m);
		System.out.println("trainscore = " + score);

		score = n.score(origEmbed);
		System.out.println("origtrainscore = " + score);

		TestCaseGRE g=new TestCaseGRE();