public class FilledDependency implements Comparable<FilledDependency> {
	protected final short relID;
	protected final short headIndex; // position of the "head" word in the sentence
	protected final short fillerIndex; // position of the filler word in the sentence
	protected final short unaryRuleID; // if dependency has been created thro' a unary rule
	protected final short lrange; // if dependency has been created thro' the head-passing mechanism

	public final short conjFactor; // average divisor for multiple slot fillers in max-recall decoder

	public FilledDependency(short relID,
			short headIndex,
			short fillerIndex,
			short unaryRuleID,
			short lrange) {
		this.relID = relID;
		this.headIndex = headIndex;
		this.fillerIndex = fillerIndex;
		this.unaryRuleID = unaryRuleID;
		this.conjFactor = 1;
		this.lrange = lrange;

		if ( fillerIndex == 0 ) {
			throw new Error("expecting a non-zero filler index when constructing the filled dependency!");
		}
	}

	public FilledDependency(Dependency dep,
			short fillerIndex,
			short conjFactor,
			short lrange) {
		this.relID = dep.relID;
		this.headIndex = dep.headIndex;
		this.fillerIndex = fillerIndex;
		this.unaryRuleID = dep.unaryRuleID;
		this.conjFactor = conjFactor;
		this.lrange = (lrange != 0) ? lrange : dep.lrange;

		if ( fillerIndex == 0 ) {
			throw new Error("expecting a non-zero filler index when constructing the filled dependency!");
		}
	}

	public static ArrayList<FilledDependency> fromUnfilled(Dependency dep, Variable var, short lrange, ArrayList<FilledDependency> deps) {
		short conjFactor = var.countFillers();

		for ( int i = 0; i < var.fillers.length && var.fillers[i] != Variable.SENTINEL; i++ ) {
			deps.add(new FilledDependency(dep, var.fillers[i], conjFactor, lrange));
		}

		return deps;
	}

	@Override
	public String toString() {
		return headIndex + " " + relID + " " + fillerIndex + " " + unaryRuleID;
	}

	private String getPosTag(int index, Sentence sentence) {
		if ( index < 0 ) {
			return "START";
		} else if ( index >= sentence.words.size() ) {
			return "END";
		} else {
			return sentence.postags.get(index);
		}
	}

	public String[] getAttributes(Relations relations, Sentence sentence) {
		String[] output = new String[11];
		Relation relation = relations.getRelation(relID);
		output[0] = new Integer(headIndex - 1).toString();
		output[1] = relation.category;
		output[2] = String.valueOf(relation.jslot);
		output[3] = new Integer(fillerIndex - 1).toString();
		output[4] = String.valueOf(Math.abs(fillerIndex - headIndex));
		output[5] = new Integer(headIndex - 1).toString();
		output[6] = new Integer(fillerIndex - 1).toString();

		return output;
	}

	public void printFull(PrintWriter out, Relations relations, Sentence sentence) {
		String head = sentence.words.get(headIndex - 1);
		String filler = sentence.words.get(fillerIndex - 1);
		Relation relation = relations.getRelation(relID);
		String stringCat = relation.category;
		short slot = relation.slot;
		out.println(head + "_" + headIndex + " " + stringCat + " " + slot + " " + filler + "_" + fillerIndex + " " + unaryRuleID);
	}

	public void printFullJslot(PrintWriter out, Relations relations, Sentence sentence) {
		String head = sentence.words.get(headIndex - 1);
		String filler = sentence.words.get(fillerIndex - 1);
		Relation relation = relations.getRelation(relID);
		String stringCat = relation.category;
		short jslot = relation.jslot;
		out.println(head + "_" + headIndex + " " + stringCat + " " + jslot + " " + filler + "_" + fillerIndex + " " + unaryRuleID);
	}

	public void printForTraining(PrintWriter out, Categories categories, Sentence sentence) {
		Relations relations = categories.dependencyRelations;
		Relation relation = relations.getRelation(relID);
		String stringCat = relation.category;
		short jslot = relation.jslot;

		String plainCatString = categories.getPlainString(stringCat);
		if ( plainCatString == null ) {
			throw new Error("should have plain category string for all lexical categories! " + stringCat);
		}

		out.println(headIndex + " " + plainCatString + " " + jslot + " " + fillerIndex);
	}

	@Override
	public int compareTo(FilledDependency other) {
		int compare;
		if ( (compare = Short.compare(this.relID, other.relID)) != 0 ) { return compare; }
		if ( (compare = Short.compare(this.headIndex, other.headIndex)) != 0 ) { return compare; }
		if ( (compare = Short.compare(this.fillerIndex, other.fillerIndex)) != 0 ) { return compare; }

		return 0;
	}

	@Override
	public int hashCode() {
		/*
		 * ignore the unaryRuleID and lrange fields, since these are not part of the
		 * CCGbank dependencies that we use
		 */
		Hash h = new Hash(relID);
		h.plusEqual(headIndex);
		h.plusEqual(fillerIndex);
		return (int) (h.value());
	}

	@Override
	public boolean equals(Object other) {
		if ( other == null || getClass() != other.getClass() ) {
			return false;
		}

		FilledDependency cother = (FilledDependency) other;

		return compareTo(cother) == 0;
	}
} public class RuleInstancesParams {
	private boolean allRules;
	private boolean rightPunct;
	private boolean leftPunct;
	private boolean leftPunctConj;
	private boolean backwardComp;
	private boolean conj;
	private String directory;

	public RuleInstancesParams(boolean allRules, boolean rightPunct, boolean leftPunct, boolean leftPunctConj, boolean backwardComp, boolean conj, String directory) {
		this.allRules = allRules;
		this.rightPunct = rightPunct;
		this.leftPunct = leftPunct;
		this.leftPunctConj = leftPunctConj;
		this.backwardComp = backwardComp;
		this.conj = conj;
		this.directory = directory;
	}

	public boolean getAllRules() {
		return allRules;
	}

	public boolean getRightPunct() {
		return rightPunct;
	}

	public boolean getLeftPunct() {
		return leftPunct;
	}

	public boolean getLeftPunctConj() {
		return leftPunctConj;
	}

	public boolean getBackwardComp() {
		return backwardComp;
	}

	public boolean getConj() {
		return conj;
	}

	public String getDirectory() {
		return directory;
	}

	public void setAllRules(boolean allRules) {
		this.allRules = allRules;
	}

	public void setRightPunct(boolean rightPunct) {
		this.rightPunct = rightPunct;
	}

	public void setLeftPunct(boolean leftPunct) {
		this.leftPunct = leftPunct;
	}

	public void setLeftPunctConj(boolean leftPunctConj) {
		this.leftPunctConj = leftPunctConj;
	}

	public void setBackwardComp(boolean backwardComp) {
		this.backwardComp = backwardComp;
	}

	public void setConj(boolean conj) {
		this.conj = conj;
	}

	public void setDirectory(String directory) {
		this.directory = directory;
	}
} public class Rules {
	public Combinators combinators;
	public UnaryRules unaryRules;
	public PunctRules punctRules;
	public RuleInstances allRuleInstances;
	/*
	 * unaryRules and PunctRules use categories to get particular Category
	 * objects, eg the NP Category for the N => NP rule, as well as the lists of
	 * type-raised categories (for Unary)
	 */

	// global counts for how many times the rules are used:
	public static int forwardAppCount;
	public static int backwardAppCount;
	public static int forwardCompCount;
	public static int genForwardCompCount;
	public static int backwardCompCount;
	public static int genBackwardCompCount;
	public static int backwardCrossCount;
	public static int genBackwardCrossCount;
	public static int appositionCount;
	public static int rightPunctCount;
	public static int leftPunctCount;
	public static int leftPunctConjCount;
	public static int leftCommaTCCount;
	public static int rightCommaTCCount;
	public static int conjCount;
	public static int funnyConjCount;

	/*
	 * these flags are calculated by the static ruleFlags method below and used
	 * to determine which rules can apply to two categories
	 */
	private static final int NONE = 0;
	private static final int FORWARD = 1;
	private static final int BACKWARD = 2;
	private static final int CONJ = 3;
	private static final int PERIOD = 4;
	private static final int COLON = 5;
	private static final int COMMA = 6;
	private static final int BRACKET = 7;
	private static final int NFLAGS = 8;

	public static final Logger logger = LogManager.getLogger(Rules.class);

	public Rules(boolean eisnerNormalForm, Categories categories, RuleInstancesParams ruleInstancesParams) throws IOException {
		if (ruleInstancesParams.getAllRules()) {
			allRuleInstances = new RuleInstances(ruleInstancesParams.getDirectory() + "/all_rule_instances", categories);
		} else {
			allRuleInstances = null;
		}

		combinators = new Combinators(eisnerNormalForm, categories, ruleInstancesParams);
		unaryRules = new UnaryRules(categories);
		punctRules = new PunctRules(categories, ruleInstancesParams);
	}

	public boolean combine(SuperCategory leftSuperCat, SuperCategory rightSuperCat, ArrayList<SuperCategory> results,  Sentence sentence) {
		logger.trace("trying to combine two cats: ");
		logger.trace(leftSuperCat.cat);
		logger.trace(rightSuperCat.cat);

		if (allRuleInstances != null && !allRuleInstances.contains(leftSuperCat.cat, rightSuperCat.cat)) {
			/*
			 * PrintWriter systemOut = new PrintWriter(System.out, true);
			 * System.out.println("MISSING THESE: ");
			 * leftSuperCat.cat.print(systemOut); systemOut.flush();
			 * System.out.print(" "); rightSuperCat.cat.print(systemOut);
			 * systemOut.flush(); System.out.println();
			 */
			return false;
			// haven't got this to work yet - because of Category.equals, we need NP[nb] in the rules file
		}

		int leftFlags = ruleFlags(leftSuperCat.cat);
		int rightFlags = ruleFlags(rightSuperCat.cat);

		// not sure this is doing much, and may not be worth having
		// the extra field in Category; printing out categories only
		// appears to give cats with arguments up to 6
		if (leftSuperCat.cat.numArgs > 5 || rightSuperCat.cat.numArgs > 5) {
			return false;
		}

		/*
		 * this was the equivalent from C&C, using nvars: if(sc1->nvars > 9 ||
		 * sc2->nvars > 9){ ++rule_nvars; return false;
		 */

		/*
		 * each combination of left and right flags values results in a unique
		 * value for ruleFlagsPair:
		 */
		int ruleFlagsPair = leftFlags * NFLAGS + rightFlags;
		switch (ruleFlagsPair) {
			/*
			 * two atomic categories, neither punctuation nor conj:
			 */
			case NONE * NFLAGS + NONE:
				return combinators.apposition(leftSuperCat, rightSuperCat, results);
			/*
			 * atomic left, complex backslash right:
			 */
			case NONE * NFLAGS + BACKWARD:
				return combinators.backwardApplication(leftSuperCat, rightSuperCat, results, sentence);
			/*
			 * atomic or complex left, punct (not comma) right:
			 */
			case NONE * NFLAGS + PERIOD: // fall through
			case NONE * NFLAGS + COLON: // |
			case NONE * NFLAGS + BRACKET: // |
			case FORWARD * NFLAGS + PERIOD: // |
			case FORWARD * NFLAGS + COLON: // |
			case FORWARD * NFLAGS + BRACKET: // |
			case BACKWARD * NFLAGS + PERIOD: // |
			case BACKWARD * NFLAGS + COLON: // |
			case BACKWARD * NFLAGS + BRACKET: // |
				return punctRules.rightPunct(leftSuperCat, rightSuperCat, results);
			/*
			 * atomic or complex left, comma right:
			 */
			case NONE * NFLAGS + COMMA: // fall through
			case FORWARD * NFLAGS + COMMA: // |
			case BACKWARD * NFLAGS + COMMA: // |
				boolean typeChangeResult = punctRules.rightCommaTypeChange(leftSuperCat, rightSuperCat, results);
				boolean punctResult = punctRules.rightPunct(leftSuperCat, rightSuperCat, results);
				return typeChangeResult || punctResult;
			/*
			 * complex forward slash left, atomic right conj case is for
			 * conj/conj conj
			 */
			case FORWARD * NFLAGS + NONE: // fall through
			case FORWARD * NFLAGS + CONJ: // |
				return combinators.forwardApplication(leftSuperCat, rightSuperCat, results, sentence);
			/*
			 * complex left and right, both forward slash:
			 */
			case FORWARD * NFLAGS + FORWARD:
				if (combinators.forwardComposition(leftSuperCat, rightSuperCat, results, sentence)) {
					return true;
				} else {
					return combinators.forwardApplication(leftSuperCat, rightSuperCat, results, sentence);
				}
			/*
			 * complex left and right, forward slash left, backslash right:
			 */
			case FORWARD * NFLAGS + BACKWARD:
				if (combinators.backwardApplication(leftSuperCat, rightSuperCat, results, sentence)) {
					return true;
				} else if (combinators.forwardApplication(leftSuperCat, rightSuperCat, results, sentence)) {
					return true;
				} else {
					return combinators.backwardCrossComposition(leftSuperCat, rightSuperCat, results, sentence);
				}
			/*
			 * complex left and right, both backslash
			 */
			case BACKWARD * NFLAGS + BACKWARD:
				if (combinators.backwardApplication(leftSuperCat, rightSuperCat, results, sentence)) {
					return true;
				} else {
					return combinators.backwardComposition(leftSuperCat, rightSuperCat, results, sentence);
				}
			/*
			 * conj left, atomic right
			 */
			case CONJ * NFLAGS + NONE:
				boolean conjResult = combinators.coordination(leftSuperCat, rightSuperCat, results);
				boolean funnyConjResult = combinators.funnyConj(leftSuperCat, rightSuperCat, results);
				return conjResult || funnyConjResult;
			/*
			 * conj left, complex right; backwardApplication captures conj
			 * conj\conj - this was a bug in C&C since the 2nd case never gets
			 * called; fixed here by checking for conj\conj in
			 * coordinationReject
			 */
			case CONJ * NFLAGS + FORWARD: // fall through
			case CONJ * NFLAGS + BACKWARD: // |
				if (combinators.coordination(leftSuperCat, rightSuperCat, results)) {
					return true;
				} else {
					return combinators.backwardApplication(leftSuperCat, rightSuperCat, results, sentence);
				}
			/*
			 * colon or bracket left, atomic or complex right leftPunctConj case
			 * is for the semicolon (following C&C)
			 */
			case COLON * NFLAGS + NONE: // fall through
			case COLON * NFLAGS + FORWARD: // |
			case COLON * NFLAGS + BACKWARD: // |
			case BRACKET * NFLAGS + NONE: // |
			case BRACKET * NFLAGS + FORWARD: // |
			case BRACKET * NFLAGS + BACKWARD: // |
			case PERIOD * NFLAGS + NONE: // | // added these period cases since
			case PERIOD * NFLAGS + FORWARD: // | // we have more punct rules now
			case PERIOD * NFLAGS + BACKWARD: // |
				punctResult = punctRules.leftPunct(leftSuperCat, rightSuperCat, results);
				conjResult = punctRules.leftPunctConj(leftSuperCat, rightSuperCat, results);
				return punctResult || conjResult;
			/*
			 * comma left, atomic or complex right
			 */
			case COMMA * NFLAGS + NONE: // fall through
			case COMMA * NFLAGS + FORWARD: // |
			case COMMA * NFLAGS + BACKWARD: // |
				typeChangeResult = punctRules.leftCommaTypeChange(leftSuperCat, rightSuperCat, results);
				punctResult = punctRules.leftPunct(leftSuperCat, rightSuperCat, results);
				conjResult = punctRules.leftPunctConj(leftSuperCat, rightSuperCat, results);
				return typeChangeResult || punctResult || conjResult;
			default:
				return false;
		}
	}

	public void typeChange(ArrayList<SuperCategory> leftSuperCats, ArrayList<SuperCategory> results) {
		unaryRules.typeChange(leftSuperCats, results);
	}

	public void typeRaise(ArrayList<SuperCategory> leftSuperCats, ArrayList<SuperCategory> results) {
		unaryRules.typeRaise(leftSuperCats, results);
	}

	/*
	 * categories, for the purposes of deciding which rules to apply, are
	 * either: complex with forward slash complex with backward slash atomic
	 * conj atomic and one of 4 punctuation types atomic and none of the above
	 */
	private static int ruleFlags(Category cat) {
		switch (cat.atom.value()) {
			case Atom.NONE:
				if (cat.isFwd()) {
					return FORWARD;
				} else {
					return BACKWARD;
				}
			case Atom.CONJ:
				return CONJ;
			case Atom.PERIOD:
				return PERIOD;
			case Atom.COLON: // fall through
			case Atom.SEMICOLON:
				return COLON;
			case Atom.COMMA:
				return COMMA;
			case Atom.LQU: // fall through
			case Atom.RQU: // |
			case Atom.LRB: // |
			case Atom.RRB:
				return BRACKET;
			default:
				return NONE;
		}
	}
} public class SuperCategory {
	public final Category cat;

	public ArrayList<Dependency> unfilledDeps = null;
	public ArrayList<FilledDependency> filledDeps = null;

	/*
	 * used to record how the category was built (eg coordination) which then
	 * gets used by rule restrictions in the combinators
	 */
	public short flags;

	/*
	 * all the variables on this category; note that numVars and numActiveVars
	 * are both 1 more than the actual number of variables, since we assume a
	 * "NONE" Variable at the beginning; eg with X and Y variables, numVars = 3
	 * 
	 * the difference between numVars and numActiveVars is that numActiveVars
	 * comes from Unification (when applicable) which may have more variables
	 * than the resulting category (which is numVars);
	 */
	private final byte numVars;
	private final byte numActiveVars;
	public Variable[] vars;

	public final SuperCategory leftChild;
	public final SuperCategory rightChild;

	//linked list used to keep track of the equivalent SuperCategories in the packed chart
	public SuperCategory next = null;

	// used in the CategoryEquivalence hashMap
	private final Hash equivalenceHash;

	// all the scores and marked fields used by the various decoders:
	public double maxEquivScore;
	public SuperCategory maxEquivSuperCat = null;

	public double score = 0.0;
	public double inside = 0.0;
	public double outside = 0.0;
	public double disjInside = 0.0;

	public double logPScore = 0.0;
	public double depnnScore = 0.0;

	// used to mark active nodes
	public int marker = 0;
	public int goldMarker = 0;

	public boolean marked = false;

	/*
	 * perhaps not great design having this here, but need it for the oracle
	 * F-score decoder when filling the outside field which records the number
	 * of dependencies (correct or incorrect, but need to ignore those not taken
	 * into account in the evaluation)
	 */
	public static IgnoreDepsEval ignoreDeps;

	public static final Logger logger = LogManager.getLogger(SuperCategory.class);

	public static void setIgnoreDepsEval(IgnoreDepsEval i) {
		ignoreDeps = i;
	}

	public static IgnoreDepsEval getIgnoreDepsEval() {
		return ignoreDeps;
	}

	public SuperCategory(short headIndex, Category cat, short flags) {
		this.cat = cat;
		this.unfilledDeps = Dependency.getDependencies(headIndex, cat, (short) (0)); // last argument is ruleID
		this.filledDeps = new ArrayList<FilledDependency>();
		this.flags = flags;
		this.numVars = cat.getNumVars();
		this.numActiveVars = this.numVars; // same as numVars since lexical category
		this.vars = new Variable[this.numVars];

		for ( int i = 0; i < numVars; i++ ) {
			vars[i] = new Variable();
		}

		this.leftChild = null;
		this.rightChild = null;

		if ( cat.var != VarID.NONE ) {
			this.vars[cat.var] = new Variable(headIndex);
		} else {
			throw new Error("shouldn't we have a variable on a lexical category?!");
		}

		this.equivalenceHash = equivalenceHash();

		this.outside = 0.0;
	}

	public static SuperCategory Lexical(short headIndex, Category cat, short flags) {
		return new SuperCategory(headIndex, cat, flags);
	}

	/*
	 * sentence argument only used by numFilledDeps() and for the oracleFscore
	 * decoder - maybe there's a nicer solution
	 */
	public SuperCategory(Category cat,
			short flags,
			SuperCategory leftChild,
			SuperCategory rightChild,
			Unify unification,
			Sentence sentence) {
		this.cat = cat;
		this.unfilledDeps = new ArrayList<Dependency>();
		this.filledDeps = new ArrayList<FilledDependency>();
		this.flags = flags;
		this.numVars = unification.numVariables; // numActiveVars from Unification
		this.numActiveVars = cat.getNumVars(); // numVars from the resulting category
		this.vars = new Variable[this.numVars];
		this.leftChild = leftChild;
		this.rightChild = rightChild;
		this.next = null;

		// old sanity check from C&C; numActiveVars should never be greater than numVars (caught now below in the static call):
		if ( numActiveVars > numVars ) {
			throw new Error("numActiveVars > numVars!");
		}

		/*
		 * need to create new Variable objects based on the VarIDs present in
		 * the new Category (Unify never sees the Variables, only VarIDs)
		 * 
		 * note ignore i = 0, which corresponds to the "NONE" Variable
		 */
		this.vars[0] = new Variable(); // the other variables get created below

		for ( int i = 1; i < numVars; i++ ) {
			// old sanity check from C&C:
			if ( unification.old1[i] >= leftChild.numVars || unification.old2[i] >= rightChild.numVars ) {
				throw new Error("attempt to access variables outside range");
			}

			// chains together any filler chains on unifying variables
			vars[i] = new Variable(leftChild.vars[unification.old1[i]], rightChild.vars[unification.old2[i]]);
		}

		for ( Dependency dep : leftChild.unfilledDeps ) {
			byte var = unification.trans1[dep.var];

			if ( var == VarID.NONE ) {
				continue;
			}

			short lrange = 0;

			if ( vars[var].isFilled() ) {
				filledDeps = FilledDependency.fromUnfilled(dep, vars[var], lrange, filledDeps);
			} else {
				unfilledDeps.add(new Dependency(dep, var, lrange, true));
			}
		}

		for ( Dependency dep : rightChild.unfilledDeps ) {
			byte var = unification.trans2[dep.var];

			if ( var == VarID.NONE ) {
				continue;
			}

			short lrange = 0;

			if ( vars[var].isFilled() ) {
				filledDeps = FilledDependency.fromUnfilled(dep, vars[var], lrange, filledDeps);
			} else {
				unfilledDeps.add(new Dependency(dep, var, lrange, true));
			}
		}

		this.equivalenceHash = equivalenceHash();
		/*
		 * this gets used by the oracleFscore decoder to split equivalence
		 * classes by the number of dependencies produced
		 */
		this.outside = leftChild.outside + rightChild.outside + numFilledDeps(sentence);
	}

	public static SuperCategory BinaryCombinator(Category cat, short flags, SuperCategory leftChild, SuperCategory rightChild, Unify unification, Sentence sentence) {
		/*
		 * there are some pathological cases where this appears to happen, so
		 * just rule these out as ungrammatical
		 */
		if ( cat.getNumVars() > unification.numVariables ) {
			return null;
		}

		return new SuperCategory(cat, flags, leftChild, rightChild, unification, sentence);
	}

	public SuperCategory(Category cat,
			short flags,
			SuperCategory leftChild,
			SuperCategory rightChild) {
		this.cat = cat;
		this.unfilledDeps = new ArrayList<Dependency>(rightChild.unfilledDeps);
		this.filledDeps = new ArrayList<FilledDependency>();
		this.flags = flags;
		this.numVars = rightChild.numVars;
		this.numActiveVars = rightChild.numActiveVars;
		this.vars = new Variable[this.numVars];
		this.leftChild = leftChild;
		this.rightChild = rightChild;
		this.next = null;

		// old sanity check from C&C; numActiveVars should never be greater than numVars:
		if ( numActiveVars > numVars ) {
			throw new Error("numActiveVars > numVars!");
		}

		vars[0] = new Variable(); // the other variables get created below
		// copying Variables from non-conj category
		for ( int i = 1; i < numVars; i++ ) {
			vars[i] = new Variable(rightChild.vars[i]);
		}

		// need this check? just makes sure that conj cat has a head
		if ( !leftChild.vars[1].isFilled() ) {
			throw new Error("no head on conj cat!");
		}

		/*
		 * here we create the new dependency for the coordination; eg
		 * "and chips" will create a conj dependency; note that the var Variable
		 * will be an unfilled chain case, with "chips" already there as a
		 * filler (but not at index 0, which is UNFILLED); hence when
		 * "and chips" combines with "fish" then two dependencies will be
		 * created
		 */
		// "outer" variable on the non-conj cat
		Variable var = rightChild.vars[rightChild.cat.var];
		if ( var.isFilled() ) { // shouldn't this always be the case?
			// this is the "head" on the conj cat, eg "and"
			short head = leftChild.vars[1].getFiller();
			unfilledDeps.add(new Dependency(Relations.conj1, head, cat.argument.var, (short) (0)));
			// unfilledDeps is already set to those from rightChild; these become the next field
		}

		this.equivalenceHash = equivalenceHash();
		/*
		 * this gets used by the oracleFscore decoder to split equivalence
		 * classes by the number of dependencies produced
		 */
		this.outside = leftChild.outside + rightChild.outside;
	}

	public static SuperCategory Coordination(Category cat, short flags, SuperCategory leftSuperCat, SuperCategory rightSuperCat) {
		return new SuperCategory(cat, flags, leftSuperCat, rightSuperCat);
	}

	/*
	 * note we use some of the same objects from variablesSuperCat, eg
	 * unfilledDeps, vars (ie don't make copies); this is fine here since we
	 * always make copies when changes are made to the object; eg see the
	 * creation of FilledDeps above
	 */
	public SuperCategory(Category cat,
			short flags,
			SuperCategory leftSuperCat,
			SuperCategory rightSuperCat,
			SuperCategory variablesSuperCat) {
		this.cat = cat;
		this.unfilledDeps = new ArrayList<Dependency>(variablesSuperCat.unfilledDeps);
		this.filledDeps = new ArrayList<FilledDependency>();
		this.flags = flags;
		this.numVars = variablesSuperCat.numVars;
		this.numActiveVars = variablesSuperCat.numActiveVars;
		this.vars = variablesSuperCat.vars;
		this.leftChild = leftSuperCat;
		this.rightChild = rightSuperCat;
		this.equivalenceHash = variablesSuperCat.equivalenceHash;
		this.next = null;

		// old sanity check from C&C; numActiveVars should never be greater than numVars:
		if ( numActiveVars > numVars ) {
			throw new Error("numActiveVars > numVars!");
		}

		/*
		 * this gets used by the oracleFscore decoder to split equivalence
		 * classes by the number of dependencies produced
		 */
		this.outside = leftChild.outside + rightChild.outside;
	}

	public static SuperCategory Punct(Category cat, short flags, SuperCategory leftSuperCat, SuperCategory rightSuperCat, SuperCategory variablesSuperCat) {
		return new SuperCategory(cat, flags, leftSuperCat, rightSuperCat, variablesSuperCat);
	}

	public SuperCategory(Category cat,
			short flags,
			SuperCategory leftSuperCat,
			SuperCategory rightSuperCat,
			SuperCategory headSuperCat,
			boolean replace,
			short ruleID) {
		this.cat = cat;
		this.unfilledDeps = new ArrayList<Dependency>();
		this.filledDeps = new ArrayList<FilledDependency>();
		this.flags = flags;
		this.numVars = cat.getNumVars();
		this.numActiveVars = numVars;
		vars = new Variable[numVars];

		for ( int i = 0; i < numVars; i++ ) {
			vars[i] = new Variable();
		}

		this.leftChild = leftSuperCat;
		this.rightChild = rightSuperCat;
		this.next = null;

		// old sanity check from C&C; numActiveVars should never be greater than numVars:
		if ( numActiveVars > numVars ) {
			throw new Error("numActiveVars > numVars!");
		}

		Variable outerVariable = headSuperCat.vars[headSuperCat.cat.var];

		if ( replace ) {
			/*
			 * added this (not in the C&C code): (gets cases like
			 * "references would survive unamended", where there is a dep
			 * between references and unamended)
			 */
			if ( cat.isSbNPbSbNP() ) {
				unfilledDeps = Dependency.clone(headSuperCat.cat.argument.var, cat.argument.argument.var, ruleID, headSuperCat.unfilledDeps);
			} else if ( headSuperCat.cat.argument != null ) {
				unfilledDeps = Dependency.clone(headSuperCat.cat.argument.var, cat.argument.var, ruleID, headSuperCat.unfilledDeps);
			}
		} else {
			unfilledDeps = Dependency.getDependencies(outerVariable, cat, ruleID);
		}

		if ( cat.var != VarID.NONE ) {
			vars[cat.var] = outerVariable;
		}

		this.equivalenceHash = equivalenceHash();
		/*
		 * this gets used by the oracleFscore decoder to split equivalence
		 * classes by the number of dependencies produced
		 */
		this.outside = leftChild.outside;
		if ( rightChild != null ) {
			outside += rightChild.outside;
		}
	}

	/*
	 * TypeChanging is used for both unary and right comma type-changing rules,
	 * unlike in C&C where there are two constructors; the only difference is
	 * that in the binary case there is a rightSuperCat (so just pass in null
	 * for unary); this also combines the TypeChange and Special constructors
	 * from C&C (Special was only used for left comma type-changing)
	 */
	public static SuperCategory TypeChanging(Category cat, short flags, SuperCategory leftSuperCat, SuperCategory rightSuperCat, SuperCategory headSuperCat, boolean replace, short ruleID) {
		return new SuperCategory(cat, flags, leftSuperCat, rightSuperCat, headSuperCat, replace, ruleID);
	}

	public SuperCategory(TypeRaisedCategory trCat,
			short flags,
			SuperCategory leftSuperCat) {
		this.cat = trCat.cat;
		this.unfilledDeps = new ArrayList<Dependency>();
		this.filledDeps = new ArrayList<FilledDependency>();
		this.flags = flags;
		this.numVars = cat.getNumVars();
		this.numActiveVars = numVars;
		this.vars = new Variable[numVars];

		for ( int i = 0; i < numVars; i++ ) {
			vars[i] = new Variable();
		}

		this.leftChild = leftSuperCat;
		this.rightChild = null;
		this.next = null;

		// note can't be at the end of the defn because there is a return statement earlier!
		this.outside = leftChild.outside;

		// old sanity check from C&C; numActiveVars should never be greater than numVars:
		if ( numActiveVars > numVars ) {
			throw new Error("numActiveVars > numVars!");
		}

		// grab the Variable corresponding to the lexical item:
		if ( trCat.lexVar != VarID.NONE ) {
			vars[trCat.lexVar] = leftSuperCat.vars[leftSuperCat.cat.var];
		} else {
			// added this - not in C&C
			throw new Error("should always be a lexical item with a tr cat!");
		}

		if ( trCat.depVar == VarID.NONE ) { // no dependency in original category
			equivalenceHash = equivalenceHash();
			return;
			// TODO: did we forget to increment numSuperCategories++ here?
		}

		unfilledDeps.clear();

		/*
		 * essentially copying dependencies over from the original
		 * non-type-raised category (could be more than one if eg two APs have
		 * been coordinated), and replacing the var with trCat.depVar
		 */
		for ( Dependency dep : leftSuperCat.unfilledDeps ) {
			// boolean argument distinguishes the constructor from another with same signature
			unfilledDeps.add(new Dependency(dep, trCat.depVar, dep.lrange, true));
		}

		this.equivalenceHash = equivalenceHash();
	}

	public static SuperCategory TypeRaising(TypeRaisedCategory trCat, short flags, SuperCategory leftSuperCat) {
		return new SuperCategory(trCat, flags, leftSuperCat);
	}

	/*
	 * Apposition only called on NP and S[dcl] cases
	 */
	public SuperCategory(short flags,
			SuperCategory leftSuperCat,
			SuperCategory rightSuperCat) {
		this.cat = leftSuperCat.cat;
		this.unfilledDeps = new ArrayList<Dependency>();
		this.filledDeps = new ArrayList<FilledDependency>();
		this.flags = flags;
		this.numVars = 2;
		this.numActiveVars = numVars;
		this.vars = new Variable[numVars];

		for ( int i = 0; i < numVars; i++ ) {
			vars[i] = new Variable();
		}

		// calling this constructor directly is a little ugly (?)
		vars[1] = new Variable(leftSuperCat.vars[1], rightSuperCat.vars[1]);

		this.leftChild = leftSuperCat;
		this.rightChild = rightSuperCat;
		this.next = null;

		this.equivalenceHash = equivalenceHash();
		/*
		 * this gets used by the oracleFscore decoder to split equivalence
		 * classes by the number of dependencies produced
		 */
		this.outside = leftChild.outside + rightChild.outside;
	}

	public static SuperCategory Apposition(short flags, SuperCategory leftSuperCat, SuperCategory rightSuperCat) {
		return new SuperCategory(flags, leftSuperCat, rightSuperCat);
	}

	public int numFilledDeps(Sentence sentence) {
		if ( ignoreDeps == null ) {
			return 0;
		}

		int numDeps = 0;
		for ( FilledDependency dep : filledDeps ) {
			if ( !ignoreDeps.ignoreDependency(dep, sentence) ) {
				numDeps++;
			}
		}

		return numDeps;
	}

	/*
	 * categories are equivalent if they have the same syntactic type (got thro'
	 * the category's ehash) and have the same unfilled dependencies (got thro'
	 * both the relation and the variables)
	 */
	private Hash equivalenceHash() {
		Hash h = new Hash(cat.getEhash());

		for ( Dependency dep : unfilledDeps ) {
			h.plusEqual(dep.relID);
		}

		for ( int i = 1; i < numActiveVars; i++ ) {
			// numActiveVars is always 1 more than the real number
			for ( int j = 1; j != Variable.NUM_FILLERS; j++ ) {
				// note the start index is 1
				h.plusEqual(vars[i].fillers[j]);
			}
		}

		return h;
	}

	public long getEhash() {
		return equivalenceHash.value();
	}

	@Override
	public int hashCode() {
		return (int) (equivalenceHash.value());
	}

	@Override
	public boolean equals(Object other) {
		if ( other == null || getClass() != other.getClass() ) {
			return false;
		}

		SuperCategory cother = (SuperCategory) other;

		if ( this.equivalenceHash.value() != cother.equivalenceHash.value() ) {
			return false;
		}

		if ( this.numActiveVars != cother.numActiveVars ) {
			return false;
		}

		for ( int i = 1; i < this.numActiveVars; i++ ) {
			if ( !this.vars[i].equals(cother.vars[i]) ) {
				return false;
			}
		}

		// ensures cats created via unary rules get their own equiv classes
		if ( this.unary() != cother.unary() ) {
			return false;
		}

		if ( !this.cat.equalsWithVars(cother.cat) ) {
			return false;
		}

		if ( this.unfilledDeps.size() != cother.unfilledDeps.size() ) {
			return false;
		}

		Collections.sort(this.unfilledDeps);
		Collections.sort(cother.unfilledDeps);

		for ( int i = 0; i < this.unfilledDeps.size(); i++ ) {
			if ( !this.unfilledDeps.get(i).equals(cother.unfilledDeps.get(i)) ) {
				return false;
			}
		}

		return true;
	}

	public int compareToScore(SuperCategory other) {
		return Double.compare(other.score, this.score);
	}

	public static Comparator<SuperCategory> scoreComparator() {
		return new Comparator<SuperCategory>() {
			@Override
			public int compare(SuperCategory superCat1, SuperCategory superCat2) {
				return Double.compare(superCat2.score, superCat1.score);
			}
		};
	}

	public void markActive() {
		marker = 1;
	}

	public boolean isActive() {
		return marker != 0;
	}

	public void markActiveDisj(IntWrapper numDisjNodes) {
		numDisjNodes.value++;

		markActive();
		if ( leftChild != null && !leftChild.isActive() ) {
			leftChild.markActiveDisj(numDisjNodes);
		}

		if ( rightChild != null && !rightChild.isActive() ) {
			rightChild.markActiveDisj(numDisjNodes);
		}

		if ( next != null ) {
			next.markActiveConj(numDisjNodes);
		}
	}

	public void markActiveConj(IntWrapper numDisjNodes) {
		markActive();
		if ( leftChild != null && !leftChild.isActive() ) {
			leftChild.markActiveDisj(numDisjNodes);
		}

		if ( rightChild != null && !rightChild.isActive() ) {
			rightChild.markActiveDisj(numDisjNodes);
		}

		if ( next != null ) {
			next.markActiveConj(numDisjNodes);
		}
	}

	public void printFilledDeps(Relations relations) {
		logger.info("printing deps:");
		for ( FilledDependency dep : filledDeps ) {
			logger.info(dep);
		}
	}

	public int numEquivNodes() {
		return next != null ? 1 + next.numEquivNodes() : 1;
	}

	/*
	 * these are the same flags from C&C (with LEX renamed UNARY_TC)
	 */
	protected static final short CONJ = (short) (1 << 0);
	protected static final short TR = (short) (1 << 1);
	protected static final short UNARY_TC = (short) (1 << 2);

	protected static final short FWD_APP = (short) (1 << 3);
	protected static final short BWD_APP = (short) (1 << 4);
	protected static final short FWD_COMP = (short) (1 << 5);
	protected static final short BWD_COMP = (short) (1 << 6);
	protected static final short BWD_CROSS = (short) (1 << 7);

	protected static final short RECURSIVE = (short) (1 << 8);

	protected static final short FUNNY_CONJ = (short) (1 << 9);

	protected static final short LEFT_PUNCT = (short) (1 << 10);
	protected static final short RIGHT_PUNCT = (short) (1 << 11);

	protected static final short LEFT_TC = (short) (1 << 12);
	protected static final short RIGHT_TC = (short) (1 << 13);

	protected static final short APPO = (short) (1 << 14);
	protected static final short GEN_MISC = (short) (1 << 15);

	protected static final short CONJ_TR = (short) (CONJ | TR);
	protected static final short TR_UNARY = (short) (TR | UNARY_TC);
	protected static final short GEN_RULES = (short) (CONJ | TR | UNARY_TC | FWD_APP | BWD_APP | FWD_COMP | BWD_COMP | BWD_CROSS);

	public boolean coordinated() {
		return (flags & CONJ) != 0;
	}

	public boolean typeRaised() {
		return (flags & TR) != 0;
	}

	public boolean unary() {
		return (flags & TR_UNARY) != 0;
	}

	boolean coordinatedOrTypeRaised() {
		return (flags & CONJ_TR) != 0;
	}

	boolean forwardComp() {
		return (flags & FWD_COMP) != 0;
	}

	boolean backwardComp() {
		return (flags & BWD_COMP) != 0;
	}

	boolean backwardCrossComp() {
		return (flags & BWD_CROSS) != 0;
	}
} public class Variable {
	/*
	 * A Variable object is essentially an array which can store words an array
	 * is needed to deal with coordination cases, eg "apples and pears"
	 * 
	 * There are 2 cases: - unfilled, which means the variable is yet to be
	 * unified with a constant (word) - filled, which means it has unified with
	 * a word (represented as a position index)
	 * 
	 * There are also 2 subcases: - no chain, which means we haven't collected
	 * any constants - chain, which means we have
	 * 
	 * chaining occurs when we have eg "and pears", so that "pears" is stored on
	 * the array but we're still waiting to unify with the other conjunct long
	 * chains can occur with eg "and pears and bananas and coconuts"
	 */

	final static int NUM_FILLERS = 8; // maximum number of words allowed in
	// chains
	final static short UNFILLED = 0; // assumes sentence positions start at 1
	final static public short SENTINEL = 32767; // assumes sentence positions
	// never get this big

	public short[] fillers;

	// can we create a default constructor called by all these which
	// creates memory for the fillers array?

	/*
	 * unfilled variable the first position in the array indicates whether the
	 * variable is filled
	 */
	public Variable() {
		fillers = new short[NUM_FILLERS];
		Arrays.fill(fillers, 1, NUM_FILLERS - 1, SENTINEL);
		fillers[0] = UNFILLED;
	}

	/*
	 * filled variable the first position in the array stores the word's
	 * position in the sentence
	 */
	public Variable(short position) {
		fillers = new short[NUM_FILLERS];
		Arrays.fill(fillers, 1, NUM_FILLERS - 1, SENTINEL);
		fillers[0] = position;
	}

	/*
	 * "if" condition allows the other Variable to be filled, which creates the
	 * unfilled chained case
	 */
	public Variable(Variable other) {
		fillers = new short[NUM_FILLERS];
		if (other.isFilled()) {
			fillers[0] = UNFILLED;
			System.arraycopy(other.fillers, 0, fillers, 1, NUM_FILLERS - 1);
		} else {
			Arrays.fill(fillers, 1, NUM_FILLERS - 1, SENTINEL);
			fillers[0] = UNFILLED;
		}
	}

	/*
	 * unifying two variables to construct a new one
	 */
	public Variable(Variable v1, Variable v2) {
		fillers = new short[NUM_FILLERS];

		int v1Index = 0;
		int v2Index = 0;
		if (v1.fillers[0] == UNFILLED) {
			v1Index++;
		} else if (v2.fillers[0] == UNFILLED)
		{
			v2Index++;
			// } else
			// throw new
			// Error("trying to unify two constant variables and create a new one!");
			// also been checked earlier using unify(SuperCategory,
			// SuperCategory) method so Error should never get thrown (but
			// note this also gets called directly by the Apposition
			// SuperCategory constructor - in which case this Error bites)
		}

		/*
		 * shifts along both arrays until this array is full; "zero" elements
		 * are SENTINELS which have the *maximum* value
		 */
		for (int i = 0; i < fillers.length; i++) {
			if (v1.fillers[v1Index] < v2.fillers[v2Index]) {
				fillers[i] = v1.fillers[v1Index];
				v1Index++;
			} else {
				fillers[i] = v2.fillers[v2Index];
				v2Index++;
			}
		}
	}

	@Override
	public int hashCode() {
		return Arrays.hashCode(fillers);
	}

	@Override
	public boolean equals(Object other) {
		if ( other == null || getClass() != other.getClass() ) {
			return false;
		}

		Variable cother = (Variable) other;

		return Arrays.equals(this.fillers, cother.fillers);
	}

	public boolean isUnfilled() {
		return fillers[0] == UNFILLED;
	}

	public boolean isFilled() {
		return fillers[0] > UNFILLED;
	}

	public boolean isSingleConst() {
		return fillers[1] == SENTINEL && isFilled();
	}

	public Boolean isSet() {
		return fillers[1] != SENTINEL;
	}

	public Boolean isUnfilledNoChain() {
		return isUnfilled() & !isSet();
	}

	short getFiller() {
		return fillers[0];
	}

	short countFillers() {
		short count = 0;
		for (int i = 0; i < fillers.length; i++) {
			if (fillers[i] == SENTINEL) {
				break;
			} else {
				count++;
			}
		}
		return count;
	}
} public class Sentence {
	public ArrayList<String> words;
	public ArrayList<String> postags;
	public ArrayList<ArrayList<Supertag>> multiSupertags;
	public ArrayList<Integer> wordIDs;
	public ArrayList<Integer> postagIDs;
	public ArrayList<Category> outputSupertags;
	// stored before printing dependency structures

	public Sentence(int MAX_WORDS) {
		words = new ArrayList<String>(MAX_WORDS);
		postags = new ArrayList<String>(MAX_WORDS);
		multiSupertags = new ArrayList<ArrayList<Supertag>>(MAX_WORDS);
		wordIDs = new ArrayList<Integer>(MAX_WORDS);
		postagIDs = new ArrayList<Integer>(MAX_WORDS);
		outputSupertags = new ArrayList<Category>(MAX_WORDS);
	}

	public void addWord(String word) {
		words.add(word);
	}

	public void addPostag(String postag) {
		postags.add(postag);
	}

	public void addSupertags(ArrayList<Supertag> supertags) {
		multiSupertags.add(supertags);
	}

	public void addOutputSupertag(Category supertag) {
		outputSupertags.add(supertag);
	}

	/**
	 * Fills wordIDs and postagIDs with integer IDs obtained from lexicon.
	 * 
	 * @param lexicon lexicon
	 */
	public void addIDs(Lexicon lexicon) {
		for (int i = 0; i < words.size(); i++) {
			wordIDs.add(lexicon.getID(words.get(i)));
			postagIDs.add(lexicon.getID(postags.get(i)));
		}
	}

	public void clear() {
		words.clear();
		postags.clear();
		multiSupertags.clear();
		wordIDs.clear();
		postagIDs.clear();
		outputSupertags.clear();
	}

	public void printC_line(PrintWriter out) {
		out.print("<c>");

		for (int i = 0; i < words.size(); i++) {
			out.print(" " + words.get(i) + "|" + postags.get(i) + "|");
			out.print(outputSupertags.get(i).toStringNoOuterBracketsNoVars());
		}

		out.println();
	}

	public void printSupertags(PrintWriter out) {
		for (int i = 0; i < words.size(); i++) {
			if (i > 0) {
				out.print(" ");
			}

			out.print(outputSupertags.get(i).toStringNoOuterBracketsNoVars());
		}

		out.println();
	}
} public class Categories {
	// map from the plain category string to the markedup string:
	private HashMap<String, String> markedupStrings;

	// map from the markedup string to the plain category string:
	private HashMap<String, String> plainCategoryStrings;

	// map from the plain category string to the category object:
	private HashMap<String, Category> markedupCategories;

	// map from markedup category, slot pairs to a relation ID;
	// ID also indexes into an array of relation objects:
	public Relations dependencyRelations;

	// used in ruleInstances (and elsewhere) - use just one canonical
	// version of each category to prevent building the same category
	// over and over
	private HashMap<String, Category> seenCategories;
	private HashMap<Category, Category> canonicalCats;

	// lists of type-raised category objects of various types:
	public ArrayList<TypeRaisedCategory> trNPs;
	public ArrayList<TypeRaisedCategory> trPPs;
	public ArrayList<TypeRaisedCategory> trAPs;
	public ArrayList<TypeRaisedCategory> trVP_TOs;

	// ALT_MARKEDUP signals to use the alternative lines in the markedup file (marked ! in the file)
	public Categories(String grammarDir, boolean ALT_MARKEDUP) throws IOException {
		dependencyRelations = new Relations();
		readMarkedupFile(grammarDir, ALT_MARKEDUP);
		seenCategories = new HashMap<String, Category>();
		canonicalCats = new HashMap<Category, Category>();

		trNPs = new ArrayList<TypeRaisedCategory>();
		readTRcategories(grammarDir + "/trNP", trNPs);

		trPPs = new ArrayList<TypeRaisedCategory>();
		readTRcategories(grammarDir + "/trPP", trPPs);

		trAPs = new ArrayList<TypeRaisedCategory>();
		readTRcategories(grammarDir + "/trAP", trAPs);

		trVP_TOs = new ArrayList<TypeRaisedCategory>();
		readTRcategories(grammarDir + "/trVP_to", trVP_TOs);
	}

	public String getString(String plainCategoryString) {
		return markedupStrings.get(plainCategoryString);
	}

	public String getPlainString(String markedupString) {
		return plainCategoryStrings.get(markedupString);
	}

	public Category getCategory(String plainCategoryString) {
		return markedupCategories.get(plainCategoryString);
	}

	// 3 constants only used when reading the markedup file:
	private enum States {
		CAT, MARKEDUP, GRS
	}

	private void readMarkedupFile(String grammarDir, boolean ALT_MARKEDUP) throws IOException {
		markedupStrings = new HashMap<String, String>();
		plainCategoryStrings = new HashMap<String, String>();
		markedupCategories = new HashMap<String, Category>();

		// not sure when the sentinel is used for categories, but retaining from C&C:
		markedupStrings.put("+", "+");
		markedupCategories.put("+", null);

		String markedupFile = grammarDir + "/markedup";

		try ( BufferedReader in = new BufferedReader(new FileReader(markedupFile)) ) {
			Preface.readPreface(in);

			States state = States.CAT;

			String plainCatString = "";
			String markedupCatString = "";
			String line = "";

			while ( (line = in.readLine()) != null ) {
				if ( line.length() == 0 ) {
					if ( state != States.GRS ) {
						throw new Error("missing markedup or GRS");
					}
					state = States.CAT;
					continue;
				}

				if ( line.charAt(0) == '#' ) {
					continue;
				}

				if ( line.charAt(0) == '=' ) {
					String[] tokens = line.split("\\s");
					if ( tokens.length < 2 ) {
						throw new Error("error parsing constraints line in markedup");
					}
					continue;
				}

				Category cat;
				switch ( state ) {
					case CAT:
						String[] tokens = (line.trim()).split("\\s+");
						if ( tokens.length != 1 ) {
							throw new Error("error parsing plain cat line in markedup");
						}
						plainCatString = tokens[0];
						state = States.MARKEDUP;
						break;
					case MARKEDUP:
						tokens = (line.trim()).split("\\s+");

						if ( tokens.length != 2 ) {
							throw new Error("error parsing markedup cat line in markedup");
						}

						markedupCatString = tokens[1];
						cat = parse(markedupCatString);

						/*
						 * call reorder since C&C does, although this Java class
						 * does now check the variable ordering in the
						 * consumeSlot method
						 * 
						 * ensures categories have vars ordered according to the
						 * ordering in the VarID class:
						 */
						byte[] seenVariables = new byte[VarID.NUM_VARS];
						Arrays.fill(seenVariables, VarID.NONE); // not necessary
						// since VarID.NONE
						// is 0, but let's
						// be explicit

						cat.reorderVariables(seenVariables, new ByteWrapper((byte) (0)));

						markedupStrings.put(plainCatString, markedupCatString);
						plainCategoryStrings.put(markedupCatString, plainCatString);
						markedupCategories.put(plainCatString, cat);

						state = States.GRS;
						break;
					case GRS:
						tokens = line.split("\\s");
						if ( tokens.length < 2 ) {
							throw new Error("error parsing gr cat line in markedup");
						}
						String tmp = tokens[0];
						if ( tmp == "!" ) {
							if ( !ALT_MARKEDUP ) {
								continue;
							}

							markedupCatString = tokens[1];
							cat = parse(markedupCatString);

							seenVariables = new byte[VarID.NUM_VARS];
							Arrays.fill(seenVariables, VarID.NONE);

							cat.reorderVariables(seenVariables, new ByteWrapper((byte) (0)));

							markedupStrings.put(plainCatString, markedupCatString);
							plainCategoryStrings.put(markedupCatString, plainCatString);
							markedupCategories.put(plainCatString, cat);

							continue;
						}
						break;
				}
			}
		} catch ( IOException e ) {
			throw e;
		}
	}

	/*
	 * when parsing a markedup string, this will add new relations to
	 * dependencyRelations (so be careful when using this outside of parsing the
	 * markedup file)
	 */
	public Category parse(String catString) {
		ShortWrapper numJuliaSlots = new ShortWrapper((short) (0));
		ShortWrapper prevSlot = new ShortWrapper((short) (0));
		ShortWrapper index = new ShortWrapper((short) (0));

		Category cat = consumeCategory(catString, index, true, numJuliaSlots, prevSlot);

		if ( index.value != catString.length() ) {
			throw new Error("failed to parse category string: " + catString);
		}

		return cat;
	}

	private Category consumeCategory(String markedupString, ShortWrapper index, boolean inResult, ShortWrapper numJuliaSlots, ShortWrapper prevSlot) {
		switch ( markedupString.charAt(index.value) ) {
			case '(':
				index.value++;
				return consumeComplex(markedupString, index, inResult, numJuliaSlots, prevSlot);
			case ')': // fall through
			case '\\': // |
			case '/': // |
				throw new Error("failed to parse category string" + markedupString);
			default:
				return consumeBasic(markedupString, index, numJuliaSlots, prevSlot);
		}
	}

	private Category consumeBasic(String markedupString, ShortWrapper index, ShortWrapper numJuliaSlots, ShortWrapper prevSlot) {
		char currentChar = markedupString.charAt(index.value);

		if ( !Character.isLetter(currentChar) && currentChar != ',' && currentChar != '.' && currentChar != ';' && currentChar != ':' ) {
			throw new Error("unexpected token parsing category in consumeBasic");
		}

		StringBuilder atomStringBuilder = new StringBuilder();
		while ( Character.isLetter(currentChar) || currentChar == ',' || currentChar == '.' || currentChar == ';' || currentChar == ':' ) {
			atomStringBuilder.append(currentChar);
			index.value++;
			if ( index.value < markedupString.length() ) {
				currentChar = markedupString.charAt(index.value);
			} else {
				break;
			}
		}
		String atomString = atomStringBuilder.toString();
		Atom atom = new Atom(atomString);
		GrammaticalFeature feature = new GrammaticalFeature(GrammaticalFeature.NONE);

		if ( index.value < markedupString.length() && markedupString.charAt(index.value) == '[' ) {
			index.value++;
			feature = consumeFeature(markedupString, index);
		}

		byte var = VarID.NONE;
		ShortWrapper lrange = new ShortWrapper((short) (0));

		if ( index.value < markedupString.length() && markedupString.charAt(index.value) == '{' ) {
			index.value++;
			var = consumeVar(markedupString, index, lrange);
		}

		short slot = 0;

		if ( index.value < markedupString.length() && markedupString.charAt(index.value) == '<' ) {
			index.value++;
			slot = consumeSlot(markedupString, index, numJuliaSlots, prevSlot);
		}

		checkBracketChar(markedupString, index);

		return new Category(atom, feature, var, slot, lrange.value);
	}

	private Category consumeComplex(String markedupString, ShortWrapper index, boolean inResult, ShortWrapper numJuliaSlots, ShortWrapper prevSlot) {
		Category left = consumeCategory(markedupString, index, inResult, numJuliaSlots, prevSlot);

		char currentChar = markedupString.charAt(index.value);
		byte slash = 0;

		if ( currentChar == '/' ) {
			slash = Category.FWD_SLASH;
		} else if ( currentChar == '\\' ) {
			slash = Category.BWD_SLASH;
		} else {
			throw new Error("expected a forward or backward slash" + markedupString);
		}

		index.value++;

		if ( inResult ) {
			numJuliaSlots.value++;
		}

		Category right = consumeCategory(markedupString, index, false, numJuliaSlots, prevSlot);

		currentChar = markedupString.charAt(index.value);
		if ( currentChar != ')' ) {
			throw new Error("unbalanced parentheses in category string" + markedupString);
		}

		index.value++;

		byte var = VarID.NONE;
		ShortWrapper lrange = new ShortWrapper((short) (0));

		if ( index.value < markedupString.length() && markedupString.charAt(index.value) == '{' ) {
			index.value++;
			var = consumeVar(markedupString, index, lrange);
		}
		short slot = 0;

		if ( index.value < markedupString.length() && markedupString.charAt(index.value) == '<' ) {
			index.value++;
			slot = consumeSlot(markedupString, index, numJuliaSlots, prevSlot);
		}
		checkBracketChar(markedupString, index);

		return new Category(left, slash, right, var, slot, lrange.value);
	}

	private GrammaticalFeature consumeFeature(String markedupString, ShortWrapper index) {
		char currentChar = markedupString.charAt(index.value);

		StringBuilder featureStringBuilder = new StringBuilder();
		while ( Character.isLetter(currentChar) ) {
			featureStringBuilder.append(currentChar);
			currentChar = markedupString.charAt(++index.value);
		}
		String featureString = featureStringBuilder.toString();
		if ( currentChar != ']' ) {
			throw new Error("unexpected character parsing feature");
		}

		index.value++;

		return new GrammaticalFeature(featureString);
	}

	private byte consumeVar(String markedupString, ShortWrapper index, ShortWrapper lrange) {
		char currentChar = markedupString.charAt(index.value);

		StringBuilder varStringBuilder = new StringBuilder();
		while ( Character.isLetter(currentChar) || currentChar == '_' ) {
			varStringBuilder.append(currentChar);
			currentChar = markedupString.charAt(++index.value);
		}
		String varString = varStringBuilder.toString();
		if ( currentChar == '*' ) {
			/*
			 * from the C&C code: lrange.value = markedup.size(); this assumes
			 * that the markedup data structure is ordered and categories can be
			 * accessed also by an index (current size gives the index)
			 * 
			 * leave for later:
			 */
			lrange.value = 1;
			currentChar = markedupString.charAt(++index.value);
		}
		if ( currentChar != '}' ) {
			throw new Error("unexpected character parsing var: " + markedupString + " " + currentChar);
		}

		index.value++;

		return VarID.convert(varString);
	}

	private short consumeSlot(String markedupString, ShortWrapper index, ShortWrapper numJuliaSlots, ShortWrapper prevSlot) {
		char currentChar = markedupString.charAt(index.value);

		StringBuilder slotStringBuilder = new StringBuilder();
		while ( Character.isDigit(currentChar) ) {
			slotStringBuilder.append(currentChar);
			currentChar = markedupString.charAt(++index.value);
		}
		String slotString = slotStringBuilder.toString();
		if ( currentChar != '>' ) {
			throw new Error("unexpected character parsing dependency");
		}

		index.value++;

		/*
		 * the addRelation method from the Relations class assumes that the
		 * relations for a particular string are added consecutively and that
		 * the slots are ordered from 1, increasing by 1 each time:
		 */
		short slotShort = Short.parseShort(slotString);
		if ( slotShort != prevSlot.value + 1 ) {
			throw new Error("slots are not increasing by 1! " + markedupString + " " + slotShort + " " + prevSlot.value);
		}
		prevSlot.value++;

		return dependencyRelations.addRelation(markedupString, slotShort, numJuliaSlots.value);
	}

	private void checkBracketChar(String markedupString, ShortWrapper index) {
		if ( index.value < markedupString.length() ) {
			char currentChar = markedupString.charAt(index.value);

			if ( currentChar == '<' || currentChar == '(' || currentChar == '!' ) {
				throw new Error("unexpected nesting or dependency parsing basic category");
			}
		}
	}

	/*
	 * there are two fields to be read on each line: the markedup category and a
	 * variable. The variable is there because for some cases (AP, VP_TO) there
	 * is already a dependency in the category being type-raised; varString
	 * indicates what the variable is associated with the dependency, which
	 * needs carrying over
	 */
	private void readTRcategories(String trFile, ArrayList<TypeRaisedCategory> trList) throws IOException {
		try ( BufferedReader in = new BufferedReader(new FileReader(trFile)) ) {

			String markedupCatString = "";
			String varString = "";
			byte[] seenVariables = new byte[VarID.NUM_VARS];

			// Preface.readPreface(in);

			String line = "";
			while ( (line = in.readLine()) != null ) {
				String[] tokens = line.split("\\s");
				if ( tokens.length != 2 ) {
					throw new Error("expecting 2 items per line in the TR file!");
				}

				markedupCatString = tokens[0];
				varString = tokens[1];
				Category cat = parse(markedupCatString);
				/*
				 * typically the varString is the sentinel "+" - this gets
				 * converted to NONE by VarID.convert
				 */
				byte depVar = VarID.convert(varString);

				Arrays.fill(seenVariables, VarID.NONE);
				cat.reorderVariables(seenVariables, new ByteWrapper((byte) (0)));

				/*
				 * seenVariables[VarID.X] is the new variable corresponding to
				 * the lexical item - this is needed by the SuperCategory
				 * constructor when called later to copy over the corresponding
				 * Variable; seenVariables[depVar] is the new variable for the
				 * dependency variable (see TypeRaisedCategory.java)
				 */
				TypeRaisedCategory trCat = new TypeRaisedCategory(cat, seenVariables[VarID.X], seenVariables[depVar]);
				trList.add(trCat);
			}
		} catch ( IOException e ) {
			throw e;
		}
	}

	public Category canonize(String catString) {
		Category cat = seenCategories.get(catString);
		if ( cat == null ) {
			cat = parse(catString);
			if ( cat == null ) {
				throw new Error("failed to parse catString!" + catString);
			}
			canonicalCats.put(cat, cat);
		}
		return cat;
	}

	/*
	 * Returns the input if it's not found in the canonical map; otherwise
	 * returns the canonical version
	 */
	public Category canonize(Category checkCat) {
		Category cat = canonicalCats.get(checkCat);
		if ( cat == null ) {
			canonicalCats.put(checkCat, checkCat);
			return checkCat;
		} else {
			return cat;
		}
	}
} public class Relations {
	private ArrayList<Relation> relations;
	private HashMap<String, Short> relIDs;
	private HashMap<CategoryJSlotPair, Short> relIDsII;
	// from pairs of Category strings and Julia slots to RelIDs used in the oracle deps decoder
	public static short conj1 = 0;
	// the special conj relation for dealing with coordination

	public Relations() {
		relations = new ArrayList<Relation>();
		relIDs = new HashMap<String, Short>();
		relIDsII = new HashMap<CategoryJSlotPair, Short>();
		addRelation("", (short) (0), (short) (0));
		// first index of a relation needs to be 1
		initConj();
	}

	public Relation getRelation(short relID) {
		return relations.get(relID);
	}

	public short getRelID(String category, short slot) {
		return (short) (relIDs.get(category) + slot - 1);
	}

	/*
	 * note that we may get cases where the category, slot pair is not in the
	 * hashMap, eg if the slot is a Julia slot and it doesn't correspond to any
	 * of the relations in the markedup file
	 */
	public short getRelID_II(String category, short slot) {
		CategoryJSlotPair pair = new CategoryJSlotPair(category, slot);
		if ( relIDsII.containsKey(pair) ) {
		// if (relIDsII.get(pair) != null) {
			return relIDsII.get(pair);
		} else {
			return (short) (0); // not a relation in markedup
		}
	}

	public short addRelation(String categoryString, short slot, short juliaSlot) {
		Relation relation = new Relation(categoryString, slot, juliaSlot);
		short relID = (short) (relations.size());
		relations.add(relation);

		if ( slot == 1 ) {
			relIDs.put(categoryString, relID);
			// relID gets converted to an object thro' autoboxing
		}

		CategoryJSlotPair pair = new CategoryJSlotPair(categoryString, juliaSlot);
		relIDsII.put(pair, relID);

		return relID;
	}

	private void initConj() {
		conj1 = addRelation("conj", (short) (1), (short) (1));
	}
} public class Features {
	// FeatureCat
	final public static short catRoot = 1;
	final public static short catLex = 4;

	// FeatureCatHead
	final public static short catRootWord = 2;
	final public static short catRootPos = 3;
	final public static short catLexWord = 5;
	final public static short catLexPos = 6;

	// FeatureRule
	final public static short ruleBinary = 7;
	final public static short ruleUnary = 8;

	// FeatureRuleHead
	final public static short ruleBinaryWord = 9;
	final public static short ruleBinaryPos = 10;
	final public static short ruleUnaryWord = 11;
	final public static short ruleUnaryPos = 12;

	// FeatureRuleHeadHead
	final public static short ruleBinaryWordWord = 13;
	final public static short ruleBinaryWordPos = 14;
	final public static short ruleBinaryPosWord = 15;
	final public static short ruleBinaryPosPos = 16;

	// FeatureRuleHeadDist
	final public static short ruleBinaryWordDistAdj = 17; // adjacency distance measure
	final public static short ruleBinaryWordDistPunct = 18; // punctuation distance measure
	final public static short ruleBinaryWordDistVerb = 19; // verb distance measure
	final public static short ruleBinaryPosDistAdj = 20;
	final public static short ruleBinaryPosDistPunct = 21;
	final public static short ruleBinaryPosDistVerb = 22;

	// FeatureRuleHead
	// unary 1
	final public static short ruleUnaryWord2 = 300;
	final public static short ruleUnaryPos2 = 301;

	// FeatureRuleHeadHead
	// unary 1
	final public static short ruleBinaryWordWord2 = 310;
	final public static short ruleBinaryWordPos2 = 311;
	final public static short ruleBinaryPosWord2 = 312;
	final public static short ruleBinaryPosPos2 = 313;

	// binary 1
	final public static short ruleUnaryUnaryWordWord2 = 320;
	final public static short ruleUnaryUnaryWordPos2 = 321;
	final public static short ruleUnaryUnaryPosWord2 = 322;
	final public static short ruleUnaryUnaryPosPos2 = 323;

	final public static short ruleNullUnaryWordWord2 = 3200;
	final public static short ruleNullUnaryWordPos2 = 3210;
	final public static short ruleNullUnaryPosWord2 = 3220;
	final public static short ruleNullUnaryPosPos2 = 3230;

	final public static short ruleUnaryNullWordWord2 = 3201;
	final public static short ruleUnaryNullWordPos2 = 3211;
	final public static short ruleUnaryNullPosWord2 = 3221;
	final public static short ruleUnaryNullPosPos2 = 3231;

	final public static short ruleUnaryBinaryWordWord2 = 330;
	final public static short ruleUnaryBinaryWordPos2 = 331;
	final public static short ruleUnaryBinaryPosWord2 = 332;
	final public static short ruleUnaryBinaryPosPos2 = 333;

	final public static short ruleBinaryUnaryWordWord2 = 340;
	final public static short ruleBinaryUnaryWordPos2 = 341;
	final public static short ruleBinaryUnaryPosWord2 = 342;
	final public static short ruleBinaryUnaryPosPos2 = 343;

	final public static short ruleBinaryBinaryWordWord2 = 350;
	final public static short ruleBinaryBinaryWordPos2 = 351;
	final public static short ruleBinaryBinaryPosWord2 = 352;
	final public static short ruleBinaryBinaryPosPos2 = 353;

	final public static short ruleNullBinaryWordWord2 = 3500;
	final public static short ruleNullBinaryWordPos2 = 3510;
	final public static short ruleNullBinaryPosWord2 = 3520;
	final public static short ruleNullBinaryPosPos2 = 3530;

	final public static short ruleBinaryNullWordWord2 = 3501;
	final public static short ruleBinaryNullWordPos2 = 3511;
	final public static short ruleBinaryNullPosWord2 = 3521;
	final public static short ruleBinaryNullPosPos2 = 3531;

	final public static short ruleRuleUnaryLeftWord = 4010;
	final public static short ruleRuleUnaryLeftPos = 4020;

	final public static short ruleRuleUnaryRightWord = 4011;
	final public static short ruleRuleUnaryRightPos = 4021;

	final public static short ruleRuleBinaryLeftWordWord = 4110;
	final public static short ruleRuleBinaryLeftWordPos = 4120;
	final public static short ruleRuleBinaryLeftPosWord = 4130;
	final public static short ruleRuleBinaryLeftPosPos= 4140;
	
	final public static short ruleRuleBinaryRightWordWord = 4111;
	final public static short ruleRuleBinaryRightWordPos = 4121;
	final public static short ruleRuleBinaryRightPosWord = 4131;
	final public static short ruleRuleBinaryRightPosPos= 4141;

	public FeatureIDs<FeatureCat> featureCatIDs;
	public FeatureIDs<FeatureCatHead> featureCatHeadIDs;
	public FeatureIDs<FeatureRule> featureRuleIDs;
	public FeatureIDs<FeatureRuleHead> featureRuleHeadIDs;
	public FeatureIDs<FeatureRuleHeadHead> featureRuleHeadHeadIDs;
	public FeatureIDs<FeatureRuleHeadDist> featureRuleHeadDistIDs;
	public FeatureIDs<FeatureRuleRuleHead> featureRuleRuleHeadIDs;
	public FeatureIDs<FeatureRuleRuleHeadHead> featureRuleRuleHeadHeadIDs;

	public int numFeatures;

	private static boolean newFeatures;

	public static final Logger logger = LogManager.getLogger(Features.class);

	public static void setNewFeatures(boolean n) {
		newFeatures = n;
	}

	public static boolean getNewFeatures() {
		return newFeatures;
	}

	public Features(String featuresFile, Categories categories, boolean newFeatures) throws IOException {
		this.featureCatIDs = new FeatureIDs<FeatureCat>();
		this.featureCatHeadIDs = new FeatureIDs<FeatureCatHead>();
		this.featureRuleIDs = new FeatureIDs<FeatureRule>();
		this.featureRuleHeadIDs = new FeatureIDs<FeatureRuleHead>();
		this.featureRuleHeadHeadIDs = new FeatureIDs<FeatureRuleHeadHead>();
		this.featureRuleHeadDistIDs = new FeatureIDs<FeatureRuleHeadDist>();
		this.featureRuleRuleHeadIDs = new FeatureIDs<FeatureRuleRuleHead>();
		this.featureRuleRuleHeadHeadIDs = new FeatureIDs<FeatureRuleRuleHeadHead>();

		readFeatures(featuresFile, categories);

		Features.setNewFeatures(newFeatures);
	}

	public Features(String featuresFile, String weightsFile, Weights weights, Categories categories, boolean newFeatures) throws IOException {
		this.featureCatIDs = new FeatureIDs<FeatureCat>();
		this.featureCatHeadIDs = new FeatureIDs<FeatureCatHead>();
		this.featureRuleIDs = new FeatureIDs<FeatureRule>();
		this.featureRuleHeadIDs = new FeatureIDs<FeatureRuleHead>();
		this.featureRuleHeadHeadIDs = new FeatureIDs<FeatureRuleHeadHead>();
		this.featureRuleHeadDistIDs = new FeatureIDs<FeatureRuleHeadDist>();
		this.featureRuleRuleHeadIDs = new FeatureIDs<FeatureRuleRuleHead>();
		this.featureRuleRuleHeadHeadIDs = new FeatureIDs<FeatureRuleRuleHeadHead>();

		readFeaturesWeights(featuresFile, weightsFile, weights, categories);

		Features.setNewFeatures(newFeatures);
	}

	private void readFeatures(String featuresFile, Categories categories) throws IOException {
		try ( BufferedReader featuresIn = new BufferedReader(new FileReader(featuresFile)) ) {

			Preface.readPreface(featuresIn);

			String featuresLine = null;
			int ID = 0;

			while ((featuresLine = featuresIn.readLine()) != null) {
				readFeature(featuresLine, ID, categories);
				ID++;
			}

			numFeatures = ID;
			logger.info("Total number of features read in: " + numFeatures);
		}
	}

	private void readFeaturesWeights(String featuresFile, String weightsFile, Weights weights, Categories categories) throws IOException {
		ArrayList<Double> weightsList = new ArrayList<Double>();

		try ( BufferedReader featuresIn = new BufferedReader(new FileReader(featuresFile));
			  BufferedReader weightsIn = new BufferedReader(new FileReader(weightsFile)) ) {

			Preface.readPreface(featuresIn);
			Preface.readPreface(weightsIn);

			String featuresLine = null;
			String weightsLine = null;

			int ID = 0;
			Double weight;

			while ((weightsLine = weightsIn.readLine()) != null) {
				if ( weightsLine.startsWith("logp") ) {
					weights.setLogP(Double.parseDouble(weightsLine.split(":")[1]));
					continue;
				} else if ( weightsLine.startsWith("depnn") ) {
					weights.setDepNN(Double.parseDouble(weightsLine.split(":")[1]));
					continue;
				} else {
					weight = Double.parseDouble(weightsLine);
				}

				featuresLine = featuresIn.readLine();

				if ( weight == 0.0 ) {
					continue;
				} else {
					weightsList.add(weight);
				}

				readFeature(featuresLine, ID, categories);
				ID++;
			}

			double[] weightsListArray = new double[weightsList.size()];

			for ( int i = 0; i < weightsList.size(); i++ ) {
				weightsListArray[i] = weightsList.get(i);
			}

			weights.setWeights(weightsListArray);

			numFeatures = ID;
			logger.info("Total number of features read in: " + numFeatures);
		}
	}

	private void readFeature(String featuresLine, int ID, Categories categories) {
		String[] tokens = featuresLine.split("\\s");
		short featureType = Short.parseShort(tokens[0]);

		switch (featureType) {
			case Features.catRoot: // fall through
			case Features.catLex: // |
				if (tokens.length != 3) {
					throw new Error("catRoot and catLex features should have 3 fields!");
				}
				FeatureCat.readFeature(featureType, tokens, featureCatIDs, ID, categories);
				break;
			case Features.catRootWord: // fall through
			case Features.catRootPos: // |
			case Features.catLexWord: // |
			case Features.catLexPos: // |
				if (tokens.length != 4) {
					throw new Error("catRootWord and catLexWord features should have 4 fields!");
				}
				FeatureCatHead.readFeature(featureType, tokens, featureCatHeadIDs, ID, categories);
				break;
			case Features.ruleBinary: // fall through
			case Features.ruleUnary: // |
				if (tokens.length != 5) {
					throw new Error("ruleBinary and ruleUnary features should have 5 fields!");
				}
				FeatureRule.readFeature(featureType, tokens, featureRuleIDs, ID, categories);
				break;
			case Features.ruleBinaryWord: // fall through
			case Features.ruleBinaryPos: // |
			case Features.ruleUnaryWord: // |
			case Features.ruleUnaryPos: // |
				if (tokens.length != 6) {
					throw new Error("ruleBinaryWord and ruleUnaryWord features should have 6 fields!");
				}
				FeatureRuleHead.readFeature(featureType, tokens, featureRuleHeadIDs, ID, categories);
				break;
			case Features.ruleBinaryWordWord: // fall through
			case Features.ruleBinaryWordPos: // |
			case Features.ruleBinaryPosWord: // |
			case Features.ruleBinaryPosPos: // |
				if (tokens.length != 7) {
					throw new Error("ruleBinaryWordWord features should have 7 fields!");
				}
				FeatureRuleHeadHead.readFeature(featureType, tokens, featureRuleHeadHeadIDs, ID, categories);
				break;
			case Features.ruleBinaryWordDistAdj: // fall through
			case Features.ruleBinaryWordDistPunct: // |
			case Features.ruleBinaryWordDistVerb: // |
			case Features.ruleBinaryPosDistAdj: // |
			case Features.ruleBinaryPosDistPunct: // |
			case Features.ruleBinaryPosDistVerb: // |
				if (tokens.length != 7) {
					throw new Error("ruleBinaryWordDist features should have 7 fields!");
				}
				FeatureRuleHeadDist.readFeature(featureType, tokens, featureRuleHeadDistIDs, ID, categories);
				break;
			case Features.ruleUnaryWord2: // fall through
			case Features.ruleUnaryPos2: // |
				if (tokens.length != 6) {
					throw new Error("features should have 6 fields!");
				}
				FeatureRuleHead.readFeature(featureType, tokens, featureRuleHeadIDs, ID, categories);
				break;
			case Features.ruleBinaryWordWord2: // fall through
			case Features.ruleBinaryWordPos2: // |
			case Features.ruleBinaryPosWord2: // |
			case Features.ruleBinaryPosPos2: // |
			case Features.ruleUnaryUnaryWordWord2: // |
			case Features.ruleUnaryUnaryWordPos2 : // |
			case Features.ruleUnaryUnaryPosWord2: // |
			case Features.ruleUnaryUnaryPosPos2: // |
			case Features.ruleNullUnaryWordWord2: // |
			case Features.ruleNullUnaryWordPos2 : // |
			case Features.ruleNullUnaryPosWord2: // |
			case Features.ruleNullUnaryPosPos2: // |
			case Features.ruleUnaryNullWordWord2: // |
			case Features.ruleUnaryNullWordPos2 : // |
			case Features.ruleUnaryNullPosWord2: // |
			case Features.ruleUnaryNullPosPos2: // |
			case Features.ruleUnaryBinaryWordWord2: // |
			case Features.ruleUnaryBinaryWordPos2: // |
			case Features.ruleUnaryBinaryPosWord2: // |
			case Features.ruleUnaryBinaryPosPos2: // |
			case Features.ruleBinaryUnaryWordWord2: // |
			case Features.ruleBinaryUnaryWordPos2: // |
			case Features.ruleBinaryUnaryPosWord2: // |
			case Features.ruleBinaryUnaryPosPos2: // |
			case Features.ruleBinaryBinaryWordWord2: // |
			case Features.ruleBinaryBinaryWordPos2: // |
			case Features.ruleBinaryBinaryPosWord2: // |
			case Features.ruleBinaryBinaryPosPos2: // |
			case Features.ruleNullBinaryWordWord2: // |
			case Features.ruleNullBinaryWordPos2: // |
			case Features.ruleNullBinaryPosWord2: // |
			case Features.ruleNullBinaryPosPos2: // |
			case Features.ruleBinaryNullWordWord2: // |
			case Features.ruleBinaryNullWordPos2: // |
			case Features.ruleBinaryNullPosWord2: // |
			case Features.ruleBinaryNullPosPos2: // |
				if (tokens.length != 7) {
					throw new Error("features should have 7 fields!");
				}
				FeatureRuleHeadHead.readFeature(featureType, tokens, featureRuleHeadHeadIDs, ID, categories);
				break;
			case Features.ruleRuleUnaryLeftWord:
			case Features.ruleRuleUnaryLeftPos:
			case Features.ruleRuleUnaryRightWord:
			case Features.ruleRuleUnaryRightPos:
				if (tokens.length != 6) {
					throw new Error("features should have 6 fields!");
				}
				FeatureRuleRuleHead.readFeature(featureType, tokens, featureRuleRuleHeadIDs, ID, categories);
				break;
			case Features.ruleRuleBinaryLeftWordWord:
			case Features.ruleRuleBinaryLeftWordPos:
			case Features.ruleRuleBinaryLeftPosWord:
			case Features.ruleRuleBinaryLeftPosPos:
			case Features.ruleRuleBinaryRightWordWord:
			case Features.ruleRuleBinaryRightWordPos:
			case Features.ruleRuleBinaryRightPosWord:
			case Features.ruleRuleBinaryRightPosPos:
				if (tokens.length != 8) {
					throw new Error("features should have 8 fields!");
				}
				FeatureRuleRuleHeadHead.readFeature(featureType, tokens, featureRuleRuleHeadHeadIDs, ID, categories);
				break;
			default:
				throw new Error("run out of feature types!");
		}
	}

	public void collectLeafFeatures(SuperCategory superCat, Sentence sentence, ArrayList<Integer> featureIDs) {
		FeatureCat.collectFeatures(superCat, catLex, featureCatIDs, featureIDs);
		FeatureCatHead.collectFeatures(superCat, catLexWord, sentence.wordIDs, featureCatHeadIDs, featureIDs);
		FeatureCatHead.collectFeatures(superCat, catLexPos, sentence.postagIDs, featureCatHeadIDs, featureIDs);
	}

	public void collectUnaryFeatures(SuperCategory superCat, Sentence sentence, ArrayList<Integer> featureIDs) {
		FeatureRule.collectFeatures(superCat.leftChild, superCat.leftChild, superCat, ruleUnary, featureRuleIDs, featureIDs);
		FeatureRuleHead.collectFeatures(superCat.leftChild, superCat.leftChild, superCat, ruleUnaryWord, sentence.wordIDs, featureRuleHeadIDs, featureIDs);
		FeatureRuleHead.collectFeatures(superCat.leftChild, superCat.leftChild, superCat, ruleUnaryPos, sentence.postagIDs, featureRuleHeadIDs, featureIDs);

		if ( Features.newFeatures ) {
			if ( superCat.leftChild.leftChild!= null ) {
				if ( superCat.leftChild.rightChild != null ) {
					FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat, ruleBinaryWordWord2, sentence.wordIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
					FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat, ruleBinaryWordPos2, sentence.wordIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
					FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat, ruleBinaryPosWord2, sentence.postagIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
					FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat, ruleBinaryPosPos2, sentence.postagIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
	
					FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftWordWord, sentence.wordIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
					FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftWordPos, sentence.wordIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
					FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftPosWord, sentence.postagIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
					FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftPosPos, sentence.postagIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
	
				} else {
					FeatureRuleHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.leftChild, superCat, ruleUnaryWord2, sentence.wordIDs, featureRuleHeadIDs, featureIDs);
					FeatureRuleHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.leftChild, superCat, ruleUnaryPos2, sentence.postagIDs, featureRuleHeadIDs, featureIDs);
	
					FeatureRuleRuleHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild, superCat, ruleRuleUnaryLeftWord, sentence.wordIDs, featureRuleRuleHeadIDs, featureIDs);
					FeatureRuleRuleHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild, superCat, ruleRuleUnaryLeftPos, sentence.postagIDs, featureRuleRuleHeadIDs, featureIDs);
				}
			}
		}
	}

	public void collectBinaryFeatures(SuperCategory superCat, Sentence sentence, ArrayList<Integer> featureIDs) {
		// just the rule:
		FeatureRule.collectFeatures(superCat.leftChild, superCat.rightChild, superCat, ruleBinary, featureRuleIDs, featureIDs);

		// rule + result head:
		FeatureRuleHead.collectFeatures(superCat.leftChild, superCat.rightChild, superCat, ruleBinaryWord, sentence.wordIDs, featureRuleHeadIDs, featureIDs);
		FeatureRuleHead.collectFeatures(superCat.leftChild, superCat.rightChild, superCat, ruleBinaryPos, sentence.postagIDs, featureRuleHeadIDs, featureIDs);

		// rule + combining heads:
		FeatureRuleHeadHead.collectFeatures(superCat.leftChild, superCat.rightChild, superCat, ruleBinaryWordWord, sentence.wordIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
		FeatureRuleHeadHead.collectFeatures(superCat.leftChild, superCat.rightChild, superCat, ruleBinaryWordPos, sentence.wordIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
		FeatureRuleHeadHead.collectFeatures(superCat.leftChild, superCat.rightChild, superCat, ruleBinaryPosWord, sentence.postagIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
		FeatureRuleHeadHead.collectFeatures(superCat.leftChild, superCat.rightChild, superCat, ruleBinaryPosPos, sentence.postagIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);

		// rule + result head + distance:
		FeatureRuleHeadDist.collectFeatures(superCat.leftChild, superCat.rightChild, superCat, ruleBinaryWordDistAdj, sentence.wordIDs, sentence.postags, featureRuleHeadDistIDs, featureIDs);
		FeatureRuleHeadDist.collectFeatures(superCat.leftChild, superCat.rightChild, superCat, ruleBinaryWordDistPunct, sentence.wordIDs, sentence.postags, featureRuleHeadDistIDs, featureIDs);
		FeatureRuleHeadDist.collectFeatures(superCat.leftChild, superCat.rightChild, superCat, ruleBinaryWordDistVerb, sentence.wordIDs, sentence.postags, featureRuleHeadDistIDs, featureIDs);
		FeatureRuleHeadDist.collectFeatures(superCat.leftChild, superCat.rightChild, superCat, ruleBinaryPosDistAdj, sentence.wordIDs, sentence.postags, featureRuleHeadDistIDs, featureIDs);
		FeatureRuleHeadDist.collectFeatures(superCat.leftChild, superCat.rightChild, superCat, ruleBinaryPosDistPunct, sentence.wordIDs, sentence.postags, featureRuleHeadDistIDs, featureIDs);
		FeatureRuleHeadDist.collectFeatures(superCat.leftChild, superCat.rightChild, superCat, ruleBinaryPosDistVerb, sentence.wordIDs, sentence.postags, featureRuleHeadDistIDs, featureIDs);

		if ( Features.newFeatures ) {
			if ( superCat.leftChild.leftChild != null ) {
				if ( superCat.leftChild.rightChild != null ) {
					if ( superCat.rightChild.leftChild != null ) {
						if ( superCat.rightChild.rightChild != null ) {
							// case 8: binary, binary, binary
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.rightChild, superCat, ruleBinaryBinaryWordWord2, sentence.wordIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.rightChild, superCat, ruleBinaryBinaryWordPos2, sentence.wordIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.rightChild, superCat, ruleBinaryBinaryPosWord2, sentence.postagIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.rightChild, superCat, ruleBinaryBinaryPosPos2, sentence.postagIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
	
							FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftWordWord, sentence.wordIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
							FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftWordPos, sentence.wordIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
							FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftPosWord, sentence.postagIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
							FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftPosPos, sentence.postagIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
	
							FeatureRuleRuleHeadHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild.rightChild, superCat.rightChild, superCat, ruleRuleBinaryRightWordWord, sentence.wordIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
							FeatureRuleRuleHeadHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild.rightChild, superCat.rightChild, superCat, ruleRuleBinaryRightWordPos, sentence.wordIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
							FeatureRuleRuleHeadHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild.rightChild, superCat.rightChild, superCat, ruleRuleBinaryRightPosWord, sentence.postagIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
							FeatureRuleRuleHeadHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild.rightChild, superCat.rightChild, superCat, ruleRuleBinaryRightPosPos, sentence.postagIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
						} else {
							// case 7: binary, binary, unary
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.leftChild, superCat, ruleBinaryUnaryWordWord2, sentence.wordIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.leftChild, superCat, ruleBinaryUnaryWordPos2, sentence.wordIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.leftChild, superCat, ruleBinaryUnaryPosWord2, sentence.postagIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.leftChild, superCat, ruleBinaryUnaryPosPos2, sentence.postagIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
	
							FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftWordWord, sentence.wordIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
							FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftWordPos, sentence.wordIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
							FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftPosWord, sentence.postagIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
							FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftPosPos, sentence.postagIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
	
							FeatureRuleRuleHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild, superCat, ruleRuleUnaryRightWord, sentence.wordIDs, featureRuleRuleHeadIDs, featureIDs);
							FeatureRuleRuleHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild, superCat, ruleRuleUnaryRightPos, sentence.postagIDs, featureRuleRuleHeadIDs, featureIDs);
						}
					} else {
						// case 3: binary, binary, null
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild, superCat, ruleBinaryNullWordWord2, sentence.wordIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild, superCat, ruleBinaryNullWordPos2, sentence.wordIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild, superCat, ruleBinaryNullPosWord2, sentence.postagIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild, superCat, ruleBinaryNullPosPos2, sentence.postagIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
	
						FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftWordWord, sentence.wordIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
						FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftWordPos, sentence.wordIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
						FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftPosWord, sentence.postagIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
						FeatureRuleRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild.rightChild, superCat.leftChild, superCat, ruleRuleBinaryLeftPosPos, sentence.postagIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
					}
				} else {
					if ( superCat.rightChild.leftChild != null ) {
						if ( superCat.rightChild.rightChild != null ) {
							// case 6: binary, unary, binary
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.rightChild, superCat, ruleUnaryBinaryWordWord2, sentence.wordIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.rightChild, superCat, ruleUnaryBinaryWordPos2, sentence.wordIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.rightChild, superCat, ruleUnaryBinaryPosWord2, sentence.postagIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.rightChild, superCat, ruleUnaryBinaryPosPos2, sentence.postagIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
	
							FeatureRuleRuleHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild, superCat, ruleRuleUnaryLeftWord, sentence.wordIDs, featureRuleRuleHeadIDs, featureIDs);
							FeatureRuleRuleHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild, superCat, ruleRuleUnaryLeftPos, sentence.postagIDs, featureRuleRuleHeadIDs, featureIDs);
	
							FeatureRuleRuleHeadHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild.rightChild, superCat.rightChild, superCat, ruleRuleBinaryRightWordWord, sentence.wordIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
							FeatureRuleRuleHeadHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild.rightChild, superCat.rightChild, superCat, ruleRuleBinaryRightWordPos, sentence.wordIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
							FeatureRuleRuleHeadHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild.rightChild, superCat.rightChild, superCat, ruleRuleBinaryRightPosWord, sentence.postagIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
							FeatureRuleRuleHeadHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild.rightChild, superCat.rightChild, superCat, ruleRuleBinaryRightPosPos, sentence.postagIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
						} else {
							// case 5: binary, unary, unary
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.leftChild, superCat, ruleUnaryUnaryWordWord2, sentence.wordIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.leftChild, superCat, ruleUnaryUnaryWordPos2, sentence.wordIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.leftChild, superCat, ruleUnaryUnaryPosWord2, sentence.postagIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
							FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild.leftChild, superCat, ruleUnaryUnaryPosPos2, sentence.postagIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
	
							FeatureRuleRuleHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild, superCat, ruleRuleUnaryLeftWord, sentence.wordIDs, featureRuleRuleHeadIDs, featureIDs);
							FeatureRuleRuleHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild, superCat, ruleRuleUnaryLeftPos, sentence.postagIDs, featureRuleRuleHeadIDs, featureIDs);
	
							FeatureRuleRuleHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild, superCat, ruleRuleUnaryRightWord, sentence.wordIDs, featureRuleRuleHeadIDs, featureIDs);
							FeatureRuleRuleHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild, superCat, ruleRuleUnaryRightPos, sentence.postagIDs, featureRuleRuleHeadIDs, featureIDs);
						}
					} else {
						// case 1: binary, unary, null
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild, superCat, ruleUnaryNullWordWord2, sentence.wordIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild, superCat, ruleUnaryNullWordPos2, sentence.wordIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild, superCat, ruleUnaryNullPosWord2, sentence.postagIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild.leftChild, superCat.rightChild, superCat, ruleUnaryNullPosPos2, sentence.postagIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
	
						FeatureRuleRuleHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild, superCat, ruleRuleUnaryLeftWord, sentence.wordIDs, featureRuleRuleHeadIDs, featureIDs);
						FeatureRuleRuleHead.collectFeatures(superCat.leftChild.leftChild, superCat.leftChild, superCat, ruleRuleUnaryLeftPos, sentence.postagIDs, featureRuleRuleHeadIDs, featureIDs);
					}
				}
			} else {
				if ( superCat.rightChild.leftChild != null ) {
					if ( superCat.rightChild.rightChild != null ) {
						// case 4: binary, null, binary
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild, superCat.rightChild.rightChild, superCat, ruleNullBinaryWordWord2, sentence.wordIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild, superCat.rightChild.rightChild, superCat, ruleNullBinaryWordPos2, sentence.wordIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild, superCat.rightChild.rightChild, superCat, ruleNullBinaryPosWord2, sentence.postagIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild, superCat.rightChild.rightChild, superCat, ruleNullBinaryPosPos2, sentence.postagIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
	
						FeatureRuleRuleHeadHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild.rightChild, superCat.rightChild, superCat, ruleRuleBinaryRightWordWord, sentence.wordIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
						FeatureRuleRuleHeadHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild.rightChild, superCat.rightChild, superCat, ruleRuleBinaryRightWordPos, sentence.wordIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
						FeatureRuleRuleHeadHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild.rightChild, superCat.rightChild, superCat, ruleRuleBinaryRightPosWord, sentence.postagIDs, sentence.wordIDs, featureRuleRuleHeadHeadIDs, featureIDs);
						FeatureRuleRuleHeadHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild.rightChild, superCat.rightChild, superCat, ruleRuleBinaryRightPosPos, sentence.postagIDs, sentence.postagIDs, featureRuleRuleHeadHeadIDs, featureIDs);
					} else {
						// case 2: binary, null, unary
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild, superCat.rightChild.leftChild, superCat, ruleNullUnaryWordWord2, sentence.wordIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild, superCat.rightChild.leftChild, superCat, ruleNullUnaryWordPos2, sentence.wordIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild, superCat.rightChild.leftChild, superCat, ruleNullUnaryPosWord2, sentence.postagIDs, sentence.wordIDs, featureRuleHeadHeadIDs, featureIDs);
						FeatureRuleHeadHead.collectFeatures(superCat.leftChild, superCat.rightChild.leftChild, superCat, ruleNullUnaryPosPos2, sentence.postagIDs, sentence.postagIDs, featureRuleHeadHeadIDs, featureIDs);
	
						FeatureRuleRuleHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild, superCat, ruleRuleUnaryRightWord, sentence.wordIDs, featureRuleRuleHeadIDs, featureIDs);
						FeatureRuleRuleHead.collectFeatures(superCat.rightChild.leftChild, superCat.rightChild, superCat, ruleRuleUnaryRightPos, sentence.postagIDs, featureRuleRuleHeadIDs, featureIDs);
					}
				}
			}
		}
	}

	public void collectRootFeatures(SuperCategory superCat, Sentence sentence, ArrayList<Integer> featureIDs) {
		FeatureCat.collectFeatures(superCat, catRoot, featureCatIDs, featureIDs);
		FeatureCatHead.collectFeatures(superCat, catRootWord, sentence.wordIDs, featureCatHeadIDs, featureIDs);
		FeatureCatHead.collectFeatures(superCat, catRootPos, sentence.postagIDs, featureCatHeadIDs, featureIDs);
	}

	public void print(PrintWriter out) {
		featureCatIDs.print(out);
		featureCatHeadIDs.print(out);
		featureRuleIDs.print(out);
		featureRuleHeadIDs.print(out);
		featureRuleHeadHeadIDs.print(out);
		featureRuleHeadDistIDs.print(out);
		featureRuleRuleHeadIDs.print(out);
		featureRuleRuleHeadHeadIDs.print(out);
	}

	public void printWeights(Weights weights, PrintWriter out) {
		out.println("logp:" + weights.getLogP());
		out.println("depnn:" + weights.getDepNN());
		featureCatIDs.printWeights(weights, out);
		featureCatHeadIDs.printWeights(weights, out);
		featureRuleIDs.printWeights(weights, out);
		featureRuleHeadIDs.printWeights(weights, out);
		featureRuleHeadHeadIDs.printWeights(weights, out);
		featureRuleHeadDistIDs.printWeights(weights, out);
		featureRuleRuleHeadIDs.printWeights(weights, out);
		featureRuleRuleHeadHeadIDs.printWeights(weights, out);
	}
} public class Lexicon {
	private HashMap<String,Integer> lexicon;
	// map from words and pos as strings to IDs

	public Lexicon(String file) throws IOException {
		this.lexicon = new HashMap<String,Integer>();
		readWordPosFile(file);
	}

	public int getID(String word) {
		// returns -1 if word not in lexicon
		Integer ID = lexicon.get(word);
		if (ID == null) {
			return -1;
		} else {
			return ID;
		}
	}

	/*
	 * not assuming an empty intersection between words and pos (ie word and pos
	 * can get the same id eg punctuation); this is fine since the feature types
	 * distinguish word and pos
	 */
	private void readWordPosFile(String file) throws IOException {
		try ( BufferedReader in = new BufferedReader(new FileReader(file)) ) {
			Preface.readPreface(in);

			String wordPos = null;
			int ID = 0;

			while ((wordPos = in.readLine()) != null) {
				lexicon.put(wordPos, ID);
				ID++;
			}
		}
	}
} public class Weights {
	private double[] weights;
	private double logp;
	private double depnn;

	public Weights() {
	}

	public Weights(String weightsFile, int numFeatures) throws IOException {
		this.weights = new double[numFeatures];

		if (!weightsFile.equals("zero")) {
			readWeights(weightsFile);
		}
	}

	public void setWeights(double[] weights) {
		this.weights = weights.clone();
	}

	private void readWeights(String weightsFile) throws IOException {
		int ID = 0;

		try ( BufferedReader in = new BufferedReader(new FileReader(weightsFile)) ) {

			Preface.readPreface(in);

			String line = null;

			while ((line = in.readLine()) != null) {
				if ( line.startsWith("logp") ) {
					logp = Double.parseDouble(line.split(":")[1]);
				} else if ( line.startsWith("depnn") ) {
					depnn = Double.parseDouble(line.split(":")[1]);
				} else {
					double weight = Double.parseDouble(line);
					weights[ID] = weight;
					ID++;
				}
			}

			if (ID != weights.length) {
				throw new IllegalArgumentException("number of weights != number of features!");
			}
		}
	}

	public double getWeight(int ID) {
		return weights[ID];
	}

	public void setWeight(int ID, double weight) {
		weights[ID] = weight;
	}

	public double getLogP() {
		return logp;
	}

	public void setLogP(double logp) {
		this.logp = logp;
	}

	public double getDepNN() {
		return depnn;
	}

	public void setDepNN(double depnn) {
		this.depnn = depnn;
	}
}
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashSet;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import cat_combination.FilledDependency;
import cat_combination.RuleInstancesParams;
import cat_combination.Rules;
import cat_combination.SuperCategory;
import cat_combination.Variable;
import io.Sentence;
import lexicon.Categories;
import lexicon.Relations;
import model.Features;
import model.Lexicon;
import model.Weights;
import uk.ac.cam.cl.depnn.io.Feature;
package chart_parser;




public class ChartParser {
	public final int MAX_SUPERCATS;

	public boolean oracleFscore;
	public boolean adaptiveSupertagging;
	public boolean maxSuperCatsExceeded = false;
	public boolean maxWordsExceeded = false;

	public Categories categories;