/FileExtraction/Java_unseen/slavpetrov_berkeleyparser/src/edu/berkeley/nlp/PCFGLA/HierarchicalFullyConnectedLexicon.java
/**
 * 
 */
package edu.berkeley.nlp.PCFGLA;

import java.util.Arrays;
import java.util.List;

import edu.berkeley.nlp.PCFGLA.smoothing.Smoother;
import edu.berkeley.nlp.syntax.StateSet;
import edu.berkeley.nlp.syntax.Tree;

/**
 * @author petrov
 * 
 */
public class HierarchicalFullyConnectedLexicon extends HierarchicalLexicon {

	private static final long serialVersionUID = 1L;
	protected int knownWordCount;

	/**
	 * @param numSubStates
	 * @param threshold
	 */
	public HierarchicalFullyConnectedLexicon(short[] numSubStates,
			int knownWordCount) {
		super(numSubStates, 0);
		this.knownWordCount = knownWordCount;
	}

	public HierarchicalFullyConnectedLexicon(short[] numSubStates,
			int smoothingCutoff, double[] smoothParam, Smoother smoother,
			StateSetTreeList trainTrees, int knownWordCount) {
		this(numSubStates, knownWordCount);
		init(trainTrees);
	}

	/**
	 * @param previousLexicon
	 */
	public HierarchicalFullyConnectedLexicon(SimpleLexicon previousLexicon,
			int knownWordCount) {
		super(previousLexicon);
		this.knownWordCount = knownWordCount;
	}

	@Override
	public HierarchicalFullyConnectedLexicon newInstance() {
		return new HierarchicalFullyConnectedLexicon(this.numSubStates,
				this.knownWordCount);
	}

	@Override
	public void init(StateSetTreeList trainTrees) {
		for (Tree<StateSet> tree : trainTrees) {
			List<StateSet> words = tree.getYield();
			for (StateSet word : words) {
				String sig = word.getWord();
				wordIndexer.add(sig);
			}
		}
		wordCounter = new int[wordIndexer.size()];
		for (Tree<StateSet> tree : trainTrees) {
			List<StateSet> words = tree.getYield();
			int ind = 0;
			for (StateSet word : words) {
				String wordString = word.getWord();
				wordCounter[wordIndexer.indexOf(wordString)]++;

				String sig = getSignature(word.getWord(), ind++);
				wordIndexer.add(sig);
			}
		}

		tagWordIndexer = new IntegerIndexer[numStates];
		for (int tag = 0; tag < numStates; tag++) {
			tagWordIndexer[tag] = new IntegerIndexer(wordIndexer.size());
		}

		labelTrees(trainTrees);

		boolean[] lexTag = new boolean[numStates];
		for (Tree<StateSet> tree : trainTrees) {
			List<StateSet> words = tree.getYield();
			List<StateSet> tags = tree.getPreTerminalYield();
			int ind = 0;
			for (StateSet word : words) {
				int tag = tags.get(ind).getState();
				tagWordIndexer[tag].add(new Integer(word.wordIndex));
				tagWordIndexer[tag].add(new Integer(word.sigIndex));
				lexTag[tag] = true;
				ind++;
			}
		}

		expectedCounts = new double[numStates][][];
		scores = new double[numStates][][];
		for (int tag = 0; tag < numStates; tag++) {
			if (!lexTag[tag]) {
				tagWordIndexer[tag] = null;
				continue;
			}
			// else tagWordIndexer[tag] = tagIndexer;
			// expectedCounts[tag] = new
			// double[numSubStates[tag]][tagWordIndexer[tag].size()];
			scores[tag] = new double[numSubStates[tag]][tagWordIndexer[tag]
					.size()];
		}
		nWords = wordIndexer.size();
	}

	public double[] score(int globalWordIndex, int globalSigIndex, short tag,
			int loc, boolean noSmoothing, boolean isSignature) {
		double[] res = new double[numSubStates[tag]];
		if (globalWordIndex != -1) {
			int tagSpecificWordIndex = tagWordIndexer[tag]
					.indexOf(globalWordIndex);
			if (tagSpecificWordIndex != -1) {
				for (int i = 0; i < numSubStates[tag]; i++) {
					res[i] = scores[tag][i][tagSpecificWordIndex];
				}
			} else {
				Arrays.fill(res, 1.0);
			}
		} else {
			Arrays.fill(res, 1.0);
		}
		if (globalWordIndex >= 0
				&& (wordCounter[globalWordIndex] > knownWordCount)) {
			// if (globalSigIndex!=-1)
			// System.out.println("Problem: frequent word has signature!");
			return res;
		}
		if (globalSigIndex != -1) {
			int tagSpecificWordIndex = tagWordIndexer[tag]
					.indexOf(globalSigIndex);
			if (tagSpecificWordIndex != -1) {
				for (int i = 0; i < numSubStates[tag]; i++) {
					res[i] *= scores[tag][i][tagSpecificWordIndex];
				}
				// } else{
				// System.out.println("unseen sig-tag pair");
			}
			// } else{
			// System.out.println("unseen sig");
		}
		// if (smoother!=null) smoother.smooth(tag,res);
		return res;
	}

	@Override
	public double[] score(StateSet stateSet, short tag, boolean noSmoothing,
			boolean isSignature) {
		if (stateSet.wordIndex == -2) {
			String word = stateSet.getWord();
			if (isSignature) {
				stateSet.wordIndex = -1;
				stateSet.sigIndex = wordIndexer.indexOf(word);
			} else {
				stateSet.wordIndex = wordIndexer.indexOf(word);
				// if (stateSet.wordIndex > wordCounter.length){
				// System.out.println("no count for this word: "+(String)wordIndexer.get(tagWordIndexer[tag].get(stateSet.wordIndex)));
				// stateSet.sigIndex = -1;
				// } else {
				if ((stateSet.wordIndex >= 0 && (wordCounter[stateSet.wordIndex] > knownWordCount))
						|| noSmoothing)
					stateSet.sigIndex = -1;
				else if (knownWordCount > 0)
					stateSet.sigIndex = wordIndexer.indexOf(getSignature(word,
							stateSet.from));
				else
					stateSet.wordIndex = wordIndexer.indexOf(getSignature(word,
							stateSet.from));
			}
			// }
		}
		return score(stateSet.wordIndex, stateSet.sigIndex, tag, stateSet.from,
				noSmoothing, isSignature);
	}

	@Override
	public void labelTrees(StateSetTreeList trainTrees) {
		for (Tree<StateSet> tree : trainTrees) {
			List<StateSet> words = tree.getYield();
			List<StateSet> tags = tree.getPreTerminalYield();
			int ind = 0;
			for (StateSet word : words) {
				word.wordIndex = wordIndexer.indexOf(word.getWord());
				if (word.wordIndex < 0 || word.wordIndex >= wordCounter.length) {
					System.out.println("Have never seen this word before: "
							+ word.getWord() + " " + word.wordIndex);
					System.out.println(tree);
				} else if (wordCounter[word.wordIndex] <= knownWordCount) {
					short tag = tags.get(ind).getState();
					String sig = getSignature(word.getWord(), ind);
					wordIndexer.add(sig);
					word.sigIndex = wordIndexer.indexOf(sig);
					tagWordIndexer[tag].add(wordIndexer.indexOf(sig));
				} else
					word.sigIndex = -1;
				ind++;
			}
		}
	}

}
