/FileExtraction/Java_unseen/slavpetrov_berkeleyparser/src/edu/berkeley/nlp/discPCFG/HiearchicalAdaptiveLinearizer.java
/**
 * 
 */
package edu.berkeley.nlp.discPCFG;

import java.util.List;

import edu.berkeley.nlp.PCFGLA.BinaryRule;
import edu.berkeley.nlp.PCFGLA.Grammar;
import edu.berkeley.nlp.PCFGLA.HierarchicalAdaptiveBinaryRule;
import edu.berkeley.nlp.PCFGLA.HierarchicalAdaptiveGrammar;
import edu.berkeley.nlp.PCFGLA.HierarchicalAdaptiveLexicalRule;
import edu.berkeley.nlp.PCFGLA.HierarchicalAdaptiveUnaryRule;
import edu.berkeley.nlp.PCFGLA.HierarchicalFullyConnectedAdaptiveLexicon;
import edu.berkeley.nlp.PCFGLA.SimpleLexicon;
import edu.berkeley.nlp.PCFGLA.SpanPredictor;
import edu.berkeley.nlp.PCFGLA.UnaryRule;
import edu.berkeley.nlp.syntax.StateSet;
import edu.berkeley.nlp.syntax.StateSetWithFeatures;
import edu.berkeley.nlp.util.ArrayUtil;

/**
 * @author petrov
 * 
 */
public class HiearchicalAdaptiveLinearizer extends HierarchicalLinearizer {
	private static final long serialVersionUID = 1L;

	HierarchicalAdaptiveGrammar grammar;
	HierarchicalFullyConnectedAdaptiveLexicon lexicon;

	public HiearchicalAdaptiveLinearizer(Grammar grammar,
			SimpleLexicon lexicon, SpanPredictor sp, int fLevel) {
		this.grammar = (HierarchicalAdaptiveGrammar) grammar;
		lexicon.explicitlyComputeScores(fLevel);
		grammar.closedSumRulesWithParent = grammar.closedViterbiRulesWithParent = grammar.unaryRulesWithParent;
		grammar.closedSumRulesWithChild = grammar.closedViterbiRulesWithChild = grammar.unaryRulesWithC;
		grammar.clearUnaryIntermediates();
		grammar.makeCRArrays();

		this.lexicon = (HierarchicalFullyConnectedAdaptiveLexicon) lexicon;
		this.spanPredictor = sp;
		this.finalLevel = fLevel;
		this.nSubstates = (int) ArrayUtil.max(grammar.numSubStates);
		init();
		computeMappings();

	}

	@Override
	public SimpleLexicon getLexicon() {
		return lexicon;
	}

	@Override
	public Grammar getGrammar() {
		return grammar;
	}

	@Override
	public double[] getLinearizedLexicon(boolean update) {
		if (update) {
			nLexiconWeights = 0;
			for (short tag = 0; tag < lexicon.rules.length; tag++) {
				for (int word = 0; word < lexicon.rules[tag].length; word++) {
					lexicon.rules[tag][word].identifier = nLexiconWeights
							+ nGrammarWeights;
					nLexiconWeights += lexicon.rules[tag][word].getFinalLevel()
							.size(); // lexicon.rules[tag][word].nParam;
				}
			}
		}
		double[] logProbs = new double[nLexiconWeights];
		// if (update) linearIndex = new int[lexicon.rules.length][];

		int index = 0;
		for (short tag = 0; tag < lexicon.rules.length; tag++) {
			// if (update) linearIndex[tag] = new
			// int[lexicon.rules[tag].length];
			for (int word = 0; word < lexicon.rules[tag].length; word++) {
				// if (update) linearIndex[tag][word] = index + nGrammarWeights;
				List<Double> vals = lexicon.rules[tag][word].getFinalLevel();
				for (Double val : vals) {
					logProbs[index++] = val;
				}
			}
		}
		if (index != logProbs.length)
			System.out.println("unequal length in lexicon");

		return logProbs;
	}

	public void delinearizeLexicon(double[] logProbs, boolean usingOnlyLastLevel) {
		for (short tag = 0; tag < lexicon.rules.length; tag++) {
			for (int word = 0; word < lexicon.rules[tag].length; word++) {
				lexicon.rules[tag][word].updateScores(logProbs);
				lexicon.rules[tag][word].explicitlyComputeScores(finalLevel,
						usingOnlyLastLevel);
			}
		}
	}

	@Override
	public void delinearizeLexicon(double[] logProbs) {
		for (short tag = 0; tag < lexicon.rules.length; tag++) {
			for (int word = 0; word < lexicon.rules[tag].length; word++) {
				lexicon.rules[tag][word].updateScores(logProbs);
				lexicon.rules[tag][word].explicitlyComputeScores(finalLevel,
						false);
			}
		}
	}

	@Override
	public void increment(double[] counts, StateSet stateSet, int tag,
			double[] weights, boolean isGold) {
		if (!(stateSet instanceof StateSetWithFeatures)) {
			int globalSigIndex = stateSet.sigIndex;
			if (globalSigIndex != -1) {
				int tagSpecificWordIndex = lexicon.tagWordIndexer[tag]
						.indexOf(globalSigIndex);
				if (tagSpecificWordIndex >= 0) {
					HierarchicalAdaptiveLexicalRule rule = lexicon.rules[tag][tagSpecificWordIndex];
					int startIndexWord = rule.identifier;
					short[] mapping = rule.mapping;
					for (int i = 0; i < nSubstates; i++) {
						if (isGold)
							counts[startIndexWord + mapping[i]] += weights[i];
						else
							counts[startIndexWord + mapping[i]] -= weights[i];
					}
				}
			}
			int globalWordIndex = stateSet.wordIndex;
			int tagSpecificWordIndex = lexicon.tagWordIndexer[tag]
					.indexOf(globalWordIndex);
			if (tagSpecificWordIndex < 0) {
				for (int i = 0; i < nSubstates; i++) {
					weights[i] = 0;
				}
			} else {
				HierarchicalAdaptiveLexicalRule rule = lexicon.rules[tag][tagSpecificWordIndex];
				int startIndexWord = rule.identifier;
				short[] mapping = rule.mapping;
				for (int i = 0; i < nSubstates; i++) {
					if (isGold)
						counts[startIndexWord + mapping[i]] += weights[i];
					else
						counts[startIndexWord + mapping[i]] -= weights[i];
					weights[i] = 0;
				}
			}
		} else {
			StateSetWithFeatures stateSetF = (StateSetWithFeatures) stateSet;
			for (int f : stateSetF.features) {
				if (f < 0)
					continue;
				int tagF = lexicon.tagWordIndexer[tag].indexOf(f);
				if (tagF < 0)
					continue;

				HierarchicalAdaptiveLexicalRule rule = lexicon.rules[tag][tagF];
				int startIndexWord = rule.identifier;
				short[] mapping = rule.mapping;
				for (int i = 0; i < nSubstates; i++) {
					if (isGold)
						counts[startIndexWord + mapping[i]] += weights[i];
					else
						counts[startIndexWord + mapping[i]] -= weights[i];
				}
			}
			for (int i = 0; i < nSubstates; i++) {
				weights[i] = 0;
			}
		}
	}

	@Override
	public void increment(double[] counts, BinaryRule rule, double[] weights,
			boolean isGold) {
		HierarchicalAdaptiveBinaryRule hr = (HierarchicalAdaptiveBinaryRule) rule;
		int thisStartIndex = hr.identifier;
		if (true) {
			for (int curInd = 0; curInd < hr.nParam; curInd++) {
				double val = weights[curInd];
				if (val > 0) {
					weights[curInd] = 0;
					if (isGold)
						counts[thisStartIndex + curInd] += val;
					else
						counts[thisStartIndex + curInd] -= val;
				}
				// System.out.println(counts[thisStartIndex + curInd]);
			}
		} else {
			int curInd = 0;
			for (int lp = 0; lp < nSubstates; lp++) {
				for (int rp = 0; rp < nSubstates; rp++) {
					// if (scores[cp]==null) continue;
					for (int np = 0; np < nSubstates; np++) {
						double val = weights[curInd];
						short mapping[][][] = hr.mapping;
						if (val > 0) {
							counts[thisStartIndex + mapping[lp][rp][np]] += val;
							weights[curInd] = 0;
						}
						curInd++;
					}
				}
			}
		}
	}

	@Override
	public void increment(double[] counts, UnaryRule rule, double[] weights,
			boolean isGold) {
		HierarchicalAdaptiveUnaryRule hr = (HierarchicalAdaptiveUnaryRule) rule;
		int thisStartIndex = hr.identifier;
		if (true) {
			// if (hr.parentState==0)
			// System.out.println("letss ee");
			for (int curInd = 0; curInd < hr.nParam; curInd++) {
				double val = weights[curInd];
				if (val > 0) {
					weights[curInd] = 0;
					if (isGold)
						counts[thisStartIndex + curInd] += val;
					else
						counts[thisStartIndex + curInd] -= val;
				}
				// System.out.println(counts[thisStartIndex + curInd]);
			}
		} else {
			int curInd = 0;
			if (rule.parentState == -1) {
				for (int cp = 0; cp < nSubstates; cp++) {
					double val = weights[curInd];
					short[][] mapping = hr.mapping;
					if (val > 0) {
						if (isGold)
							counts[thisStartIndex + mapping[cp][0]] += val;
						else
							counts[thisStartIndex + mapping[cp][0]] -= val;
						weights[curInd] = 0;
					}
					curInd++;
				}
				return;
			}

			for (int cp = 0; cp < nSubstates; cp++) {
				// if (scores[cp]==null) continue;
				for (int np = 0; np < nSubstates; np++) {
					double val = weights[curInd];
					short[][] mapping = hr.mapping;
					if (val > 0) {
						if (isGold)
							counts[thisStartIndex + mapping[cp][np]] += val;
						else
							counts[thisStartIndex + mapping[cp][np]] -= val;
						weights[curInd] = 0;
					}
					curInd++;
				}
			}
		}
	}

	@Override
	public void delinearizeGrammar(double[] probs) {
		int nDangerous = 0;
		for (BinaryRule bRule : grammar.binaryRuleMap.keySet()) {
			HierarchicalAdaptiveBinaryRule hRule = (HierarchicalAdaptiveBinaryRule) bRule;
			hRule.updateScores(probs);
		}
		if (nDangerous > 0)
			System.out
					.println("Left "
							+ nDangerous
							+ " binary rule weights unchanged since the proposed weight was dangerous.");

		nDangerous = 0;
		for (UnaryRule uRule : grammar.unaryRuleMap.keySet()) {
			HierarchicalAdaptiveUnaryRule hRule = (HierarchicalAdaptiveUnaryRule) uRule;
			hRule.updateScores(probs);
		}
		if (nDangerous > 0)
			System.out
					.println("Left "
							+ nDangerous
							+ " unary rule weights unchanged since the proposed weight was dangerous.");

		grammar.explicitlyComputeScores(finalLevel);
		grammar.closedSumRulesWithParent = grammar.closedViterbiRulesWithParent = grammar.unaryRulesWithParent;
		grammar.closedSumRulesWithChild = grammar.closedViterbiRulesWithChild = grammar.unaryRulesWithC;
		// computePairsOfUnaries();
		grammar.clearUnaryIntermediates();
		grammar.makeCRArrays();
		// return grammar;
	}

	@Override
	public double[] getLinearizedGrammar(boolean update) {
		if (update) {
			// int nRules = grammar.binaryRuleMap.size() +
			// grammar.unaryRuleMap.size();
			// startIndex = new int[nRules];

			nGrammarWeights = 0;
			for (BinaryRule bRule : grammar.binaryRuleMap.keySet()) {
				HierarchicalAdaptiveBinaryRule hRule = (HierarchicalAdaptiveBinaryRule) bRule;
				if (!grammar.isGrammarTag[bRule.parentState]) {
					System.out.println("Incorrect grammar tag");
				}
				bRule.identifier = nGrammarWeights;
				nGrammarWeights += hRule.nParam;
			}
			for (UnaryRule uRule : grammar.unaryRuleMap.keySet()) {
				HierarchicalAdaptiveUnaryRule hRule = (HierarchicalAdaptiveUnaryRule) uRule;
				uRule.identifier = nGrammarWeights;
				nGrammarWeights += hRule.nParam;
			}
		}
		double[] logProbs = new double[nGrammarWeights];

		for (BinaryRule bRule : grammar.binaryRuleMap.keySet()) {
			HierarchicalAdaptiveBinaryRule hRule = (HierarchicalAdaptiveBinaryRule) bRule;
			int ind = hRule.identifier;// startIndex[ruleIndexer.indexOf(hRule)];
			List<Double> vals = hRule.getFinalLevel();
			for (Double val : vals) {
				logProbs[ind++] = val;
			}
		}

		for (UnaryRule uRule : grammar.unaryRuleMap.keySet()) {
			HierarchicalAdaptiveUnaryRule hRule = (HierarchicalAdaptiveUnaryRule) uRule;
			int ind = hRule.identifier;// startIndex[ruleIndexer.indexOf(hRule)];
			if (uRule.childState == uRule.parentState)
				continue;
			List<Double> vals = hRule.getFinalLevel();
			for (Double val : vals) {
				logProbs[ind++] = val;
			}
		}
		return logProbs;
	}

	public void delinearizeLexiconWeights(double[] logWeights) {
		int nGrZ = 0, nLexZ = 0, nSpZ = 0;

		int tmpI = 0;
		for (int i = 0; i < nGrammarWeights; i++) {
			double val = logWeights[tmpI++];
			if (val == 0)
				nGrZ++;
		}

		for (int i = 0; i < nLexiconWeights; i++) {
			double val = logWeights[tmpI++];
			if (val == 0)
				nLexZ++;
		}
		delinearizeLexicon(logWeights, true);
	}

}
