public class BinaryDecider implements IClassifyDecider {

	private double bound;
	
	/**
	 * Constructor.
	 * 
	 * @param bound bound under which the second class is chosen instead of the first
	 */
	public BinaryDecider(double bound) {
		
		if( bound < 0 || bound > 1)
			throw new IllegalArgumentException("Error: the probability bound must be in [0,1]");
		
		this.bound = bound;
	}
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.IClassifyDecider#decide(CTBNToolkit.IClassificationResult)
	 */
	@Override
	public int decide(IClassificationResult<Double> results)
			throws RuntimeException {

		double[] p = results.getPDistribution();
		if( p.length > 2)
			throw new IllegalArgumentException("Only binary decision are managed");
		
		return (p[0] >= this.bound)?0:1;
	}

}
 public class CTBNCClassifyAlgorithm extends ClassifyAlgorithm<Double, CTDiscreteNode> {

	private boolean probabilities;
	private IClassifyDecider classDecider = null;
	
	/**
	 * Base constructor.
	 */
	public CTBNCClassifyAlgorithm() {
		
		this.setDefaultParameters();
	}
	
	@Override
	public void setParameters(Map<String, Object> params) throws IllegalArgumentException {
		
		super.setParameters(params);
		
		Boolean tmpBoolean;
		// probabilities
		tmpBoolean = (Boolean)params.get("probabilities");
		if( tmpBoolean != null)
			this.probabilities = tmpBoolean;
		// class decider
		this.classDecider = (IClassifyDecider)params.get("classifyDecider");
		
	}
	
	@Override
	public String helpParameters() {
		
		String helpStr = "";
		helpStr += "Parameters class CTBNCClassifyAlgorithm:\n";
		helpStr += "probabilities: true if we want to know the probability distribution of the class for each transition, false if we want to know the probability only at the end of the trajector. [Default value = false].\n";
		helpStr += "classifyDecider: the decider that can be used to choose the class from the probability distribution. If not defined the most likely class is chose. If defined the probability calculation is forced. [Default value = null].\n";
		return helpStr;  
	}
	
	@Override
	public void setDefaultParameters() {

		this.probabilities = false;
		this.classDecider = null;
		
		Map<String, Object> params = new TreeMap<String,Object>();
		params.put("probabilities", this.probabilities);
		super.setParameters(params);
		
	}

	@Override
	public IClassificationResult<Double> classify( ICTClassifier<Double, CTDiscreteNode> model, ITrajectory<Double> trajectory) throws RuntimeException {
		
		return this.classify(model, trajectory, new Vector<Double>(0));
	}

	@Override
	public IClassificationResult<Double> classify( ICTClassifier<Double, CTDiscreteNode> model, ITrajectory<Double> trajectory, Double samplingInterval) throws RuntimeException {
		
		Vector<Double> timeStream = new Vector<Double>();
		double t = trajectory.getTransitionTime(0) + samplingInterval;
		double tEnd = trajectory.getTransitionTime(trajectory.getTransitionsNumber() - 1);
		while(t < tEnd) {
			timeStream.add(t);
			t += samplingInterval;
		}
		
		return this.classify(model, trajectory, timeStream);
	}
	
	@Override
	public IClassificationResult<Double> classify( ICTClassifier<Double, CTDiscreteNode> model, ITrajectory<Double> trajectory, Vector<Double> timeStream) throws RuntimeException {

		if( trajectory == null || trajectory.getTransitionsNumber() == 0)
			throw new IllegalArgumentException("Error: empty trajectory to classify");
		
		// Data initialization
		NodeIndexing nodeIndexing = model.getNodeIndexing();
		CTDiscreteNode classNode = (CTDiscreteNode) model.getClassNode();		// class node
		int classNodeIndex = nodeIndexing.getClassIndex();						// class node index
		ClassificationResults<Double> results = new ClassificationResults<Double>(trajectory, timeStream); 	// the result trajectory
		Map<String,Integer> stateToIndex = new TreeMap<String,Integer>();		// to load the probability
		for(int iClass = 0; iClass < classNode.getStatesNumber(); ++iClass)
			stateToIndex.put(classNode.getStateName(iClass), iClass);
		 
		// Initialization at time 0
		double lastTime;
		for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {
			CTDiscreteNode cNode = model.getNode(iNode);
			int check = cNode.checkCIMs();
			if( check != -1)
				throw new RuntimeException("Error: in CIM validation for node " + cNode.getName() + " and parent entry " + check);
			if( iNode == classNodeIndex)
				continue;
			cNode.setEvidence(results.getNodeValue(0, iNode)); 					// set the evidence
		}
		lastTime = results.getTransitionTime(0);								// set the last changing time
	
		// Initialize log likelihood
		double[] ll = new double[classNode.getStatesNumber()];
		double[] llContribution = new double[classNode.getStatesNumber()];		// contribution of the last jump (calculated as ll(t) - ll(t-1))
		for(int iClass = 0; iClass < ll.length; ++iClass) {						// for each possible class value
			classNode.setEvidence(iClass); 										// set the class
			// Calculate the log likelihood from the class prior
			// and from the probability of all the static nodes
			llContribution[iClass] = 0.0;
			for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {		// calculate the log likelihood over all the static nodes (note that no static nodes can have continuous time parents)
				CTDiscreteNode node = model.getNode(iNode);
				if( node.isStaticNode())
					llContribution[iClass] += Math.log( node.getCIMValue(node.getCurrentParentsEntry(), 0, node.getCurrentStateIndex()));
			}
			ll[iClass] = llContribution[iClass];
		}
		if( this.probabilities || this.classDecider != null)
			results.setProbability(0, logPToP(llContribution, null), stateToIndex);
		
		// Classification algorithm
		for(int iJmp = 1; iJmp < results.getTransitionsNumber(); ++iJmp) {					// for each transition in the trajectory
			
			double deltaT = results.getTransitionTime( iJmp) - lastTime;					// delta time between two jumps
			
			for(int iClass = 0; iClass < classNode.getStatesNumber(); ++iClass) {  			// for each possible class
				llContribution[iClass] = 0.0;
				classNode.setEvidence(iClass); 												// set the class

				for(int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {		// for each node
					if(iNode == classNodeIndex)
						continue;
					
					CTDiscreteNode node = model.getNode(iNode);
					int previousStateIndex = node.getCurrentStateIndex();
					int nextStateIndex = node.getStateIndex( results.getNodeValue(iJmp, iNode));
					if( node.isStaticNode()) {												// ignore the static nodes
						if( previousStateIndex != nextStateIndex)							// if the node is static and it is changed value there is an error in the arguments
							throw new IllegalArgumentException("Error: static nodes can not change their value in the trajectory");
						else
							continue;
					}
					int pE = node.getCurrentParentsEntry();
					
					// Update log-likelihood with the "remain in the state" contribute
					// ll update (ll = ll - q*dT)
					llContribution[iClass] += node.getCIMValue(pE, previousStateIndex, previousStateIndex) * deltaT;
					
					// Update log-likelihood with the jump contribute
					if(previousStateIndex == nextStateIndex)								// if the node is not really changed (the class node must be ignored because the actual value can be different from the right one in the trajectory)
						continue;															// ignore the node					
					// ll update v3(ll = ll + log(q_xx'))
					llContribution[iClass] += Math.log( node.getCIMValue(pE, previousStateIndex, nextStateIndex));
				}
				ll[iClass] += llContribution[iClass];
			}
			
			// Set the new values for the changed nodes
			for(int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {		// for each node
				CTDiscreteNode node = model.getNode(iNode);
				if( node.isStaticNode())
					continue;
				
				int previousStateIndex = node.getCurrentStateIndex();
				int nextStateIndex = node.getStateIndex( results.getNodeValue(iJmp, iNode));
				if( previousStateIndex == nextStateIndex)
					continue;
				
				node.setEvidence( nextStateIndex);  									// set the evidence
			}
			// Time update
			lastTime = results.getTransitionTime(iJmp);
			
			// Update the ll in the results	
			if( this.probabilities || this.classDecider != null)
				results.setProbability(iJmp, logPToP(llContribution, results.getPDistribution(iJmp - 1)), stateToIndex);
		}
		
		// Find the best class
		if( this.classDecider == null) {
			int iBest = 0;
			for( int iClass = 1; iClass < ll.length; ++iClass)
				if( ll[iClass] > ll[iBest])
					iBest = iClass;
			results.setClassification(classNode.getStateName(iBest));
		}else{
			results.setClassification( classNode.getStateName( this.classDecider.decide(results)));
		}
		
		return results;
	}

	/**
	 * Transform the vector of log probabilities contribution
	 * in a distribution.
	 * 
	 * @param llContribution vector of log probability contribution in the last time slice
	 * @param pPrevious probability distribution in the previous jump
	 * @return the corresponding probability distribution
	 */
	private double[] logPToP(double[] llContribution, double[] pPrevious) {
	
		double[] p = new double[llContribution.length];
		
		double sum = 0.0;
		for( int i = 0; i < llContribution.length; ++i) {
			p[i] = Math.exp(llContribution[i]);
			sum += p[i];
		}
		// Normalization
		if( pPrevious != null)
			if( sum != 0.0) {
				double sum2 = 0.0;
				for( int i = 0; i < llContribution.length; ++i) {
					p[i] = (p[i] / sum) * pPrevious[i];
					sum2 += p[i];
				}
				for( int i = 0; i < llContribution.length; ++i)
					p[i] = (p[i] / sum2);
			} else
				for( int i = 0; i < llContribution.length; ++i)
					p[i] = pPrevious[i];
		else
			for( int i = 0; i < llContribution.length; ++i)
				p[i] = (p[i] / sum);							// the sum can not be zero because at the first time slice there is the class prior
		
		return p;
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.IClassifyAlgorithm#probabilityFlag()
	 */
	@Override
	public boolean probabilityFlag() {
		
		return this.probabilities;
	}
	
} public class CTBNCLocalStructuralLearning<K, T extends ILocalSearchIndividual<K,T>> extends LearningAlgorithm<Double, CTDiscreteNode> {

	private IElementFactory<T> elementsFactory;
	private boolean[][] adjMStructure;
	private int maxCacheEntries;

	/**
	 * Base constructor.
	 * Set the default parameters and a
	 * disconnected structure.
	 * 
	 * @param elementsFactory factory used to generate the individuals for the optimization
	 */
	public CTBNCLocalStructuralLearning(IElementFactory<T> elementsFactory) {
		
		this.elementsFactory = elementsFactory;
		this.adjMStructure = null;
		this.setDefaultParameters();
	}

	@Override
	public void setDefaultParameters() { 

		this.maxCacheEntries = 150;
		
		Map<String, Object> params = new TreeMap<String,Object>();
		params.put("maxCacheEntries", 150);
		
		super.setParameters(params);
	}
	
	@Override
	public void setParameters(Map<String, Object> params) throws IllegalArgumentException { 
		
		super.setParameters(params);
		
		Integer tmp;
		// maxCacheEntries
		tmp = (Integer)params.get("maxCacheEntries");
		if( tmp != null)
			this.maxCacheEntries = tmp;
	}

	@Override
	public void setStructure(boolean[][] adjMatrix) throws IllegalArgumentException { 

		this.adjMStructure = adjMatrix;
	}

	@Override
	public String helpParameters() {
		
		String helpStr = "";
		helpStr += "Parameters class CTBNCLocalStructuralLearning:\n";
		helpStr += "maxCacheEntries: Maximum number of entries in the LRU cache. [Default value = 150].\n";
		
		return helpStr;
	}

	@Override
	public ILearningResults learn(ICTClassifier<Double, CTDiscreteNode> model, Collection<ITrajectory<Double>> trainingSet) throws RuntimeException {
		
		NodeIndexing nodeIndexing = model.getNodeIndexing();
		
		// Initialize the structures
		boolean[][] learnedAdjMatrix = new boolean[nodeIndexing.getNodesNumber()][nodeIndexing.getNodesNumber()];	// used to save the learned structure
		boolean[][] localAdjMatrix = new boolean[nodeIndexing.getNodesNumber()][nodeIndexing.getNodesNumber()];	// used to learn a local structure (local = for a node)
		for( int i = 0; i < learnedAdjMatrix.length; ++i)
			for( int j = 0; j < learnedAdjMatrix.length; ++j) {
				learnedAdjMatrix[i][j] = false;
				localAdjMatrix[i][j] = false;
			}
		
		this.elementsFactory.setDataset(trainingSet);					// set the dataset to use in the evaluation
		
		// Hill climbing algorithm
		for(int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {	// for each node
			CTDiscreteNode node = model.getNode(iNode);
			if( node.isStaticNode())									// Ignore static nodes
				continue;
			
			// Create the cache
			ICache<K, T> cache = new LRUCache<K,T>( this.maxCacheEntries);
				
			// Initialize the new local structure:
			// use a disconnected structure except for the column
			// related to the node which we are going to learn
			// the parents. For this node we copy the suggested
			// structure. In this way we optimize the velocity
			// of the calculation of the sufficient statistics
			// in the learning algorithm.
			if(this.adjMStructure != null)
				for( int iNodeParent = 0; iNodeParent < localAdjMatrix.length; ++iNodeParent)
					localAdjMatrix[iNodeParent][iNode] = this.adjMStructure[iNodeParent][iNode];
			model.setStructure(localAdjMatrix);
			
			// Local Hill climbing algorithm
			// learn the node local structure
			T newInd;
			T currentInd = this.elementsFactory.newInstance(model, iNode);
			if( cache != null)
				cache.put(currentInd.getId(), currentInd);
			do {
				newInd = currentInd.getBestNeighbor(cache);
				if( newInd != null && newInd.evaluate() > currentInd.evaluate())
					currentInd = newInd;
			}while(currentInd == newInd);
			
			// Update structures
			IModel<Double, CTDiscreteNode> bestIndModel = currentInd.getModel();
			boolean[][] bestIndAdjMatrix = bestIndModel.getAdjMatrix();
			for(int iNodeParent = 0; iNodeParent < learnedAdjMatrix.length; ++iNodeParent) {
				// Save the learned local structure
				learnedAdjMatrix[iNodeParent][iNode] =  bestIndAdjMatrix[iNodeParent][iNode];
				// Disconnected the local matrix for the next node
				localAdjMatrix[iNodeParent][iNode] = false;					
			}
		}
		
		// Learn the model with the complete structure
		model.setStructure(learnedAdjMatrix);											// set the learned structure
		return this.elementsFactory.getParamsLearningAlg().learn(model, trainingSet);	// learn the complete model
	}
	
	@Override
	public boolean[][] getStructure() {
		return this.adjMStructure;
	}

} public class CTBNCParameterLLAlgorithm extends LearningAlgorithm<Double, CTDiscreteNode> {

	private boolean[][] adjMStructure;
	private double Mxx_prior;
	private double Tx_prior;
	private double Px_prior;

	/**
	 * Base constructor.
	 * Set the default parameters and a
	 * disconnected structure.
	 */
	public CTBNCParameterLLAlgorithm() {
		
		this.adjMStructure = null;
		this.setDefaultParameters();
	}

	@Override
	public void setDefaultParameters() {

		this.Mxx_prior = 0.0;
		this.Tx_prior = 0.0;
		this.Px_prior = 0.0;
		
		Map<String, Object> params = new TreeMap<String,Object>();
		params.put("Mxx_prior", this.Mxx_prior);
		params.put("Tx_prior", this.Tx_prior);
		params.put("Px_prior", this.Px_prior);
		super.setParameters(params);
		
	}
	
	@Override
	public void setParameters(Map<String, Object> params) throws IllegalArgumentException {

		super.setParameters(params);
		
		Double tmp;
		// Mxx_prior
		tmp = (Double)params.get("Mxx_prior");
		if( tmp != null)
			this.Mxx_prior = tmp;
			
		// Tx_prior
		tmp = (Double)params.get("Tx_prior");
		if( tmp != null)
			this.Tx_prior = tmp;
		
		// Px_prior
		tmp = (Double)params.get("Px_prior");
		if( tmp != null)
			this.Px_prior = tmp;
	}

	@Override
	public void setStructure(boolean[][] adjMatrix) throws IllegalArgumentException {

		this.adjMStructure = adjMatrix;
	}

	@Override
	public String helpParameters() {
		
		String helpStr = "";
		helpStr += "Parameters class CTBNCParameterLLAlgorithm:\n";
		helpStr += "Mxx_prior: prior for the value M[x,x'|u] (equal value for each x, x' and u). The sum over x' is the prior for M[x|u]. [Default value = 0].\n";
		helpStr += "Tx_prior: prior for the value T[x|u] (equal value for each x and u). [Default value = 0]. \n";
		helpStr += "Px_prior: prior for the value P[x|u] of the distribution of static nodes (equal value for each x and u). [Default value = 0]. \n";
		
		return helpStr;
	}
	
	/**
	 * Initialize the structure for the sufficient statistics
	 * calculation.
	 * 
	 * @param model the model to learn
	 * @return the sufficient statistics data initialized
	 * @throws RuntimeException in case of error during the process
	 */
	private SufficientStatistics[] learningInitialization(IModel<Double, CTDiscreteNode> model) throws RuntimeException{
		
		NodeIndexing nodeIndexing = model.getNodeIndexing();
		SufficientStatistics lData[] = new SufficientStatistics[nodeIndexing.getNodesNumber()];
		for(int i = 0; i < lData.length; ++i) {
			CTDiscreteNode node = model.getNode(i);
			lData[i] = new SufficientStatistics(node.getNumberParentsEntries(), node.getStatesNumber(), node.isStaticNode(),
					this.Px_prior, this.Mxx_prior, this.Tx_prior);
		}
		
		return lData;
	}
	
	/**
	 * Calculate the sufficient statistics for the
	 * parameters learning.
	 * 
	 * @param model the model to learn
	 * @param trainingSet the training set used to learn
	 * @return return the sufficient statistics learned
	 * @throws RuntimeException in case of some errors
	 */
	private SufficientStatistics[] calculateSufficientStatistics(IModel<Double, CTDiscreteNode> model, Collection<ITrajectory<Double>> trainingSet) throws RuntimeException {
		
		// Data initialization
		SufficientStatistics lData[] = learningInitialization(model);
		
		// Sufficient statistics calculation
		NodeIndexing nodeIndexing = model.getNodeIndexing();
		Iterator<ITrajectory<Double>> itTrj = trainingSet.iterator(); 
		while( itTrj.hasNext()) { 													// for each trajectory in the training set
			ITrajectory<Double> trj = itTrj.next();
			
			if( trj.getTransitionsNumber() == 0)
				continue;

			// Initialization at time 0
			double lastTime = trj.getTransitionTime(0);								// set the last changing time
			for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {
				CTDiscreteNode cNode = model.getNode(iNode);
				cNode.setEvidence(trj.getNodeValue(0, iNode)); 						// set the evidence
			}

			// Sufficient statistics for static nodes
			for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {
				CTDiscreteNode cNode = model.getNode(iNode);
				if( cNode.isStaticNode()) {
					int pE = cNode.getCurrentParentsEntry();						// because we need the parent entry we can not join together the two loops
					++lData[iNode].Px[pE][0][cNode.getCurrentStateIndex()];
					++lData[iNode].counts[pE];
				}
			}

			// Sufficient statistics for the continuous nodes
			// For each jump
			for( int iJmp = 1; iJmp < trj.getTransitionsNumber(); ++iJmp) {
				
				double deltaT = trj.getTransitionTime(iJmp) - lastTime;
				
				// For each node (update the sufficient statistics)
				for(int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {
					// Get the node information
					CTDiscreteNode node = model.getNode( iNode);
					int prevStIndex = node.getCurrentStateIndex();
					int nextStIndex = node.getStateIndex( trj.getNodeValue( iJmp, iNode));
					int pE = node.getCurrentParentsEntry();
					
					// Update the jump values
					if( !node.isStaticNode()) {
						lData[iNode].Tx[pE][prevStIndex] += deltaT;
						if(prevStIndex != nextStIndex) {								// if the node really jumped
							++lData[iNode].Mxx[pE][prevStIndex][nextStIndex];
							++lData[iNode].Mx[pE][prevStIndex];
						} 
					}else if(prevStIndex != nextStIndex) 
						throw new IllegalArgumentException("Error: static nodes (" + node.getName() + ") can not change their state during a trajectory (trj = " + trj.getName() + ")");
				}
	
				// For each changed node (set the new state)
				for(int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {
					// Get the node information
					CTDiscreteNode node = model.getNode( iNode);
					int prevStIndex = node.getCurrentStateIndex();
					int nextStIndex = node.getStateIndex( trj.getNodeValue( iJmp, iNode));
					if( prevStIndex == nextStIndex)
						continue;
					
					node.setEvidence(nextStIndex);
				}

				// Update the last transition time 
				lastTime = trj.getTransitionTime(iJmp);
				
			} // end of the for over the jumps in a trajectory
		} // end while over trajectories

		// Retract all the evidences
		for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode)
			model.getNode(iNode).retractEvidence();

		return lData;
	}
	
	/**
	 * Calculate the parameters from the sufficient statistics
	 * and set the CIMs in the model.
	 * 
	 * @param model the model to learn
	 * @param lData the sufficient statistics
	 * @throws RuntimeException if some error occurs
	 */
	private void setCIMs(IModel<Double, CTDiscreteNode> model, SufficientStatistics lData[]) throws RuntimeException {
		// Parameter calculation
		NodeIndexing nodeIndexing = model.getNodeIndexing();
		for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {
			CTDiscreteNode node = model.getNode(iNode);
			if( node.isStaticNode()) {
				for( int pE = 0; pE < node.getNumberParentsEntries(); ++pE) {
					// Probability distribution calculation
					double[][] prob = new double[1][node.getStatesNumber()];
					for( int sE = 0; sE < node.getStatesNumber(); ++sE)
						prob[0][sE] = lData[iNode].Px[pE][0][sE] / lData[iNode].counts[pE];
					node.setCIM( pE, prob);
				}
			}else {
				for( int pE = 0; pE < node.getNumberParentsEntries(); ++pE) {
					double[][] cim = new double[node.getStatesNumber()][node.getStatesNumber()];
					for( int fsE = 0; fsE < node.getStatesNumber(); ++fsE) {
						// CIM calculation
						cim[fsE][fsE] = -lData[iNode].Mx[pE][fsE] /  lData[iNode].Tx[pE][fsE];
						for( int ssE = 0; ssE < node.getStatesNumber(); ++ssE)
							if( fsE != ssE)
								cim[fsE][ssE] = lData[iNode].Mxx[pE][fsE][ssE] / lData[iNode].Tx[pE][fsE];
								// cim[fsE][ssE] = -cim[fsE][fsE] * (lData[iNode].Mxx[pE][fsE][ssE] / lData[iNode].Mx[pE][fsE]);
					}
					node.setCIM(pE, cim);
				}
			}
			
			int check = node.checkCIMs();
			if( check != -1)
				throw new RuntimeException("Error: unexpected error in CIM validation for node " + node.getName() + " and parent entry " + check + " (the inserting of the priors imaginary counts can solve this exception)");  
		}
	}
	
	@Override
	public GenericLearningResults learn(ICTClassifier<Double, CTDiscreteNode> model,  Collection<ITrajectory<Double>> trainingSet) throws RuntimeException {
		
		if( model == null)
			throw new IllegalArgumentException("Error: null model argument");
		
		// Set the structure
		if( this.adjMStructure != null)
			model.setStructure(this.adjMStructure);
		
		// Sufficient statistics calculation
		SufficientStatistics lData[] = calculateSufficientStatistics(model, trainingSet);
		
		// Calculate the parameter and update the CIMs of the model
		setCIMs(model, lData);
		
		return new GenericLearningResults(lData);
	}

	/**
	 * Return set prior over the sufficient
	 * statistic M[x,x'|u] for continuous time
	 * nodes.
	 * 
	 * @return the prior
	 */
	protected double getMxxPrior() {
		return this.Mxx_prior;
	}
	
	/**
	 * Return set prior over the sufficient
	 * statistic T[x|u] for continuous time
	 * nodes.
	 * 
	 * @return the prior
	 */
	protected double getTxPrior() {
		return this.Tx_prior;
	}
	
	/**
	 * Return set prior over the sufficient
	 * statistic P[x|u] for the static nodes.
	 * 
	 * @return the prior
	 */
	protected double getPxPrior() {
		return this.Px_prior;
	}
	
	@Override
	public boolean[][] getStructure() {
		return this.adjMStructure;
	}
	
}
 public class CTBNClassifier extends DiscreteModel<Double,CTDiscreteNode> implements ICTClassifier<Double, CTDiscreteNode> {
	
	private int nGeneratedTrajectories = 0;
	private String ext = ".csv";

	/**
	 * Base constructor
	 *  
	 * @param nodeIndexing global node indexing
	 * @param modelName model name
	 * @throws IllegalArgumentException in case of illegal arguments
	 */
	public CTBNClassifier(NodeIndexing nodeIndexing, String modelName) throws IllegalArgumentException {
		super(nodeIndexing, modelName);
	}

	/**
	 * Constructor that allow to set the nodes.
	 * 
	 * @param nodeIndexing global node indexing
	 * @param modelName model name
	 * @param nodesSet node set
	 * @throws IllegalArgumentException in case of illegal arguments
	 */
	public CTBNClassifier(NodeIndexing nodeIndexing, String modelName, Set<CTDiscreteNode> nodesSet) throws IllegalArgumentException {
		super(nodeIndexing, modelName, nodesSet);
	}

	@Override
	public ITrajectory<Double> generateTrajectory(double T) throws RuntimeException {

		if( T <= 0)
			throw new IllegalArgumentException("Error: illegal time value");
		if( this.getNodeIndexing().getClassIndex() == -1)
			throw new IllegalStateException("Error: class node doesn't set");
	
		List<Double> times = new Vector<Double>();
		List<String[]> values = new Vector<String[]>();
		// Initial state sampling
		generateInitialState(times, values);
		// Trajectory sampling
		generateTrajectory(times, values, T);
		
		CTTrajectory<Double> trj = new CTTrajectory<Double>(this.getNodeIndexing(), times, values);
		++nGeneratedTrajectories;
		trj.setName(this.getName() + "-trj" + nGeneratedTrajectories + this.ext);
		return trj;
	}

	/**
	 * Generate the initial state and update
	 * the trajectory information.
	 * The class is sampled using its prior,
	 * the other nodes are sampled using the
	 * uniform distribution.
	 * 
	 * @param times time stamp for the trajectory
	 * @param values list of values for the trajectory
	 */
	private void generateInitialState(List<Double> times, List<String[]> values) {
		
		NodeIndexing nodeIndexing = this.getNodeIndexing(); 
		String[] s0 = new String[nodeIndexing.getNodesNumber()];
		times.add(0.0);													// generate the time stamp
		
		// Class sampling
		int classIndex = nodeIndexing.getClassIndex();
		CTDiscreteNode clNode = this.getClassNode();
		clNode.setEvidence(clNode.sampleState());						// sample the class
		s0[classIndex] = clNode.getCurrentState();						// add the state to the trajectory
		
		// Uniform sampling for the not class nodes
		for( int i = 0; i < nodeIndexing.getNodesNumber(); ++i)
			if( i != classIndex) {
				CTDiscreteNode node = this.getNode(i);
				if( node == null)
					throw new RuntimeException("Error: not all the nodes defined in the general indexing are set");
				
				double[] pDistr = new double[node.getStatesNumber()];	// sample the initial state
				for(int j = 0; j < pDistr.length; ++j)
					pDistr[j] = 1.0 / pDistr.length;
				
				node.setEvidence(DiscreteModel.sample(pDistr));			// set the initial state
				s0[i] = node.getCurrentState();							// add the state to the trajectory
			}
			
		values.add(s0);													// add the first set of values
	}
	
	/**
	 * Generate a trajectory till the time T given
	 * the initial state and the class.
	 * 
	 * @param times list of time jumps for the generated trajectory
	 * @param values list of value map for the generated trajectory
	 * @param T the maximum length of the trajectory
	 */
	private void generateTrajectory(List<Double> times, List<String[]> values, double T) {
		
		NodeIndexing nodeIndexing = this.getNodeIndexing();
		int classIndex = nodeIndexing.getClassIndex();
		LinkedList<Integer> sortedNodes = new LinkedList<Integer>();
		LinkedList<Double> sortedTimes = new LinkedList<Double>();
		LinkedList<Integer> noTimeNodes = new LinkedList<Integer>();
		
		// Initialize the nodes without time
		for(int i = 0; i < nodeIndexing.getNodesNumber(); ++i) {
			if( i != classIndex)
				noTimeNodes.add( i);
		}
		
		CTDiscreteNode tmpNode;
		double currentTime = 0.0;
		while( currentTime < T) {
			// Sample the time for the nodes without time
			while( !noTimeNodes.isEmpty()) {
				// Calculate transition time
				int tmpNodeIndex = noTimeNodes.getFirst();
				tmpNode = this.getNode( tmpNodeIndex);
				double newTime = tmpNode.sampleTransitionTime() + currentTime;
				// Sort the node using the transition time
				int index;									// find the index
				for( index = sortedTimes.size() - 1; index >= 0 && newTime < sortedTimes.get(index); --index); ++index;
				
				// Lists update
				sortedNodes.add(index, tmpNodeIndex);
				sortedTimes.add(index, newTime);
				noTimeNodes.removeFirst();
			}

			currentTime = sortedTimes.getFirst();
			if( currentTime > T)
				break;
			
			// Sample the new transition
			int tmpNodeIndex = sortedNodes.getFirst();
			tmpNode = this.getNode(tmpNodeIndex);
			int nextState = tmpNode.sampleState();
			tmpNode.setEvidence(nextState);
			
			// Update the trajectory
			times.add(currentTime);
			String[] newValues = new String[nodeIndexing.getNodesNumber()];
			for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode)
				newValues[iNode] = this.getNode(iNode).getCurrentState();
			values.add(newValues);
			
			// Update the lists
			sortedTimes.removeFirst();							// node update
			sortedNodes.removeFirst();
			noTimeNodes.add(tmpNodeIndex);
			for(int i = 0; i < sortedNodes.size(); ++i) 		// child update
				if( tmpNode.getChildIndex( this.getNode(sortedNodes.get(i)).getName()) != null) {
					noTimeNodes.add(sortedNodes.get(i));
					sortedNodes.remove(i);
					sortedTimes.remove(i);
					--i;
				}
		}
		
		// Add the ending time
		String[] newValues = new String[nodeIndexing.getNodesNumber()];
		for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {
			newValues[iNode] = this.getNode(iNode).getCurrentState();
			tmpNode = this.getNode( iNode);
			tmpNode.retractEvidence();
		}
		times.add(T);
		values.add(newValues);
		
	}

	@Override
	public CTDiscreteNode getClassNode() {

		return this.getNode(this.getNodeIndexing().getClassIndex());
	}

	@Override
	public IClassificationResult<Double> classify(IClassifyAlgorithm<Double, CTDiscreteNode> algorithm, ITrajectory<Double> trajectory) throws Exception {

		return algorithm.classify(this, trajectory);
	}

	@Override
	public IClassificationResult<Double> classify(IClassifyAlgorithm<Double, CTDiscreteNode> algorithm, ITrajectory<Double> trajectory, Double samplingInterval) throws Exception {

		return algorithm.classify(this, trajectory, samplingInterval);
	}

	@Override
	public IClassificationResult<Double> classify(IClassifyAlgorithm<Double, CTDiscreteNode> algorithm, ITrajectory<Double> trajectory, Vector<Double> timeStream) throws Exception {

		return algorithm.classify(this, trajectory, timeStream);
	}
	
	@Override
	public IModel<Double, CTDiscreteNode> clone() {
		
		NodeIndexing nodeIndexing = this.getNodeIndexing();
		
		CTBNClassifier clonedModel = new CTBNClassifier(nodeIndexing, this.getName());
		for(int i = 0; i < nodeIndexing.getNodesNumber(); ++i)
			clonedModel.addNode((CTDiscreteNode) this.getNode(i).clone());
		
		clonedModel.setStructure(this.getAdjMatrix());
		
		for(int i = 0; i < nodeIndexing.getNodesNumber(); ++i) {
			CTDiscreteNode node = this.getNode(i);
			CTDiscreteNode clonedNode = clonedModel.getNode(i);
			// CIM update
			for(int pE = 0; pE < clonedNode.getNumberParentsEntries(); ++pE)
				clonedNode.setCIM(pE, node.getCIM(pE).clone());
			if( node.validatedCIMs())
				clonedNode.checkCIMs();
		}
		
		return clonedModel;
	}
	
	/**
	 * Print the model in string.
	 * 
	 * @return the representation of the model
	 */
	public String toString() {
		
		double[][] cim;
		CTDiscreteNode node;
		StringBuilder strBuilder = new StringBuilder(100000);
		NodeIndexing nodeIndexing = this.getNodeIndexing();
		
		// Introdution
		strBuilder.append( "-----------------------\n"); 
		strBuilder.append( "BAYESIAN NETWORK\n"); 
		strBuilder.append( "-----------------------\n"); 
		strBuilder.append( "BBNodes " + nodeIndexing.getNodesNumber() + "\n");
		
		// States
		strBuilder.append( "-----------------------\n");
		// class node
		int classIndex = nodeIndexing.getClassIndex();
		node = this.getNode(classIndex);
		strBuilder.append( node.getName() + "\t" + node.getStatesNumber() + "\n");
		// not class nodes
		for( int i = 0; i < nodeIndexing.getNodesNumber(); ++i)
			if( i != classIndex) {
				node = this.getNode(i);
				strBuilder.append( node.getName() + "\t" + node.getStatesNumber() + "\n");
			}
		
		// T=0 structure (disconnected)
		strBuilder.append( "-----------------------\n");
		// class node
		node = this.getNode(classIndex);
		strBuilder.append( node.getName() + " 0\n");
		// not class nodes
		for( int i = 0; i < nodeIndexing.getNodesNumber(); ++i)
			if( i != classIndex) {
				node = this.getNode(i);
				strBuilder.append( node.getName() + " 0\n");
			}
		
		// T=0 distribution (uniform distributed)
		strBuilder.append( "-----------------------\n");
		// class node
		node = this.getNode(classIndex);
		strBuilder.append( node.getName() + "\n");
		for(int s = 0; s < node.getStatesNumber(); ++s)
			strBuilder.append( 1/node.getStatesNumber() + " ");
		strBuilder.append( "\n");
		strBuilder.append( "-----------------------\n");
		// not class nodes
		for( int i = 0; i < nodeIndexing.getNodesNumber(); ++i)
			if( i != classIndex) {
				node = this.getNode(i);
				strBuilder.append( node.getName() + "\n");
				for(int s = 0; s < node.getStatesNumber(); ++s)
					strBuilder.append( 1.0/node.getStatesNumber() + " ");
				strBuilder.append( "\n");
				strBuilder.append( "-----------------------\n");
			}
		
		// Structure (parental relation)
		strBuilder.append( "-----------------------\n");
		strBuilder.append( "DIRECTED GRAPH \n");
		strBuilder.append( "-----------------------\n");
		// class node
		node = this.getNode(classIndex);
		strBuilder.append( node.getName() + "\t");
		for( int pi = 0; pi < node.getParentsNumber(); ++pi)
			strBuilder.append( node.getParent(pi).getName() + "\t");
		strBuilder.append( "0\n");
		// not class nodes
		for( int i = 0; i < nodeIndexing.getNodesNumber(); ++i)
			if( i != classIndex) {
				node = this.getNode(i);
				strBuilder.append( node.getName() + "\t");
				for( int pi = 0; pi < node.getParentsNumber(); ++pi)
					strBuilder.append( node.getParent(pi).getName() + "\t");
					strBuilder.append( "0\n");
			}
		strBuilder.append( "-----------------------\n");
		
		// CIMs
		strBuilder.append( "-----------------------\n");
		strBuilder.append( "CIMS\n");
		strBuilder.append( "-----------------------\n");
		// class node
		node = this.getNode(classIndex);
		strBuilder.append( node.getName() + "\n");
		for( int pE = 0; pE < node.getNumberParentsEntries(); ++pE) {
			cim = node.getCIM(pE);
			strBuilder.append( toStringCIM(cim) + "\n");
		}
		strBuilder.append( "-----------------------\n");
		// not class nodes
		for( int i = 0; i < nodeIndexing.getNodesNumber(); ++i)
			if( i != classIndex) {
				node = this.getNode(i);
				strBuilder.append( node.getName() + "\n");
				for( int pE = 0; pE < node.getNumberParentsEntries(); ++pE) {
					cim = node.getCIM(pE);
					strBuilder.append( toStringCIM(cim) + "\n");
				}
				strBuilder.append( "-----------------------\n");
			}	
		
		return strBuilder.toString();
	}
	 
	/**
	 * Print in a row the CIM.
	 * 
	 * @param cim the CIM to print
	 * @return string of the result
	 */
	private static String toStringCIM(double[][] cim) {
		
		StringBuilder strBuilder = new StringBuilder(500);
		
		for(int i = 0; i < cim.length; ++i) {
			for(int j = 0; j < cim[0].length; ++j)
				strBuilder.append( cim[i][j] + " ");
		}
		
		return strBuilder.toString();
	}

	
} public class CTDiscreteNode extends DiscreteNode {

	private boolean staticNode;
	private int nParentsEntries = 1;
	private boolean validatedCIMs = false;
	private List<double[][]> CIMs = null;
	
	/**
	 * Constructor of a node that evolves in continuous time.
	 * 
	 * @param nodeName node name
	 * @param states set of states of the node
	 * @exception IllegalStateException if a state contains character '@'
	 */
	public CTDiscreteNode(String nodeName, Set<String> states) throws IllegalStateException {
		this(nodeName, states, false);
	}
	
	/**
	 * Constructor
	 * 
	 * @param nodeName node name
	 * @param states set of states of the node
	 * @param staticNode true if the node is static, false if can evolve continuously in time
	 * @exception IllegalStateException if a state contains character '@'
	 */
	public CTDiscreteNode(String nodeName, Set<String> states, boolean staticNode) throws IllegalStateException {
		super(nodeName, states);
		
		Iterator<String> iter = states.iterator();
		while(iter.hasNext())
			if(iter.next().contains("@"))
				throw new IllegalStateException("Error: states names can not contain character '@'");
		
		this.staticNode = staticNode;
		this.CIMs = CTDiscreteNode.generateCIMsorCPTs(this.nParentsEntries, states.size(), this.staticNode);
		this.validatedCIMs = false;
	}
	
	/**
	 * Returns the entry value to get the right CIM
	 * given all the parents instanced.
	 * 
	 * @return the parents entry value
	 * @throws IllegalArgumentException if one parent is not instanced 
	 */
	public int getCurrentParentsEntry() throws IllegalArgumentException {

		int entry = 0;
		
		int parentMultiplier = 1;
		for(int i = 0; i < this.getParentsNumber(); ++i) {
			Integer stateIndex = this.getParent(i).getCurrentStateIndex();
			if( stateIndex == null)
				throw new IllegalArgumentException("Error: parent number " + i + " (" + this.getParent(i).getName() + ") is not instanced"); 
			entry += stateIndex * parentMultiplier;
			parentMultiplier *= this.getParent(i).getStatesNumber();
		}
		
		return entry;
	}
	
	/**
	 * Set the parents values given the parent
	 * entry.
	 * 
	 * @param pE parent entry (depend from the parents ordering)
	 * @throws IllegalArgumentException
	 */
	public void setParentsEntry(int pE) throws IllegalArgumentException {
	
		if( this.getParentsNumber() == 0)
			return;
		
		int parentMultiplier = this.getNumberParentsEntries();
		int i = this.getParentsNumber() - 1;
		do {
			CTDiscreteNode parent = this.getParent(i);
			parentMultiplier /= parent.getStatesNumber();
			
			int newState = pE / parentMultiplier;
			parent.setEvidence( newState);
			
			pE %= parentMultiplier;
			--i;
		}while(i >= 0);
		
	}
	
	/**
	 * Return the number of parents entries.
	 * 
	 * @return number of parents entries.
	 */
	public int getNumberParentsEntries() {
		
		return this.nParentsEntries;
	}
	
	/**
	 * Obtain the CIM value for a particular entry.
	 * 
	 * @param parentEntry parent entry that identify the parents values
	 * @param s0 state 0 (for static node always 0)
	 * @param s1 state 1 (for static node the index of the state for which get the probability)
	 * @return q value
	 * @throws RuntimeException can return exceptions if the indexes are out of bound or there aren't states
	 */
	public double getCIMValue(int parentEntry, int s0, int s1) throws RuntimeException {
		
		return this.CIMs.get(parentEntry)[s0][s1];
	}
	
	/**
	 * Return the CIM given the parent entry.
	 * 
	 * @param parentEntry parent entry that identify the parents values
	 * @return the CIM given the parents value
	 * @throws RuntimeException can return exceptions if the indexes are out of bound or there aren't states
	 */
	public double[][] getCIM(int parentEntry) throws RuntimeException {
		
		return this.CIMs.get(parentEntry);
	}
	
	/**
	 * Insert a value in the CIM for a particular
	 * parent entry.
	 * 
	 * @param parentEntry parent entry that identify the parents values
	 * @param s0 state 0
	 * @param s1 state 1
	 * @param value value to insert
	 * @throws RuntimeException can return exceptions if the indexes are out of bound or there aren't states
	 */
	public void setCIMValue(int parentEntry, int s0, int s1, double value) throws RuntimeException {
		
		this.CIMs.get(parentEntry)[s0][s1] = value;
		this.validatedCIMs = false;
	}
	
	/**
	 * Insert a complete CIM for a particular parent
	 * entry.
	 * 
	 * @param parentEntry parent entry that identify the parents values
	 * @param CIM CIM to insert
	 * @throws RuntimeException can return exceptions if the indexes are out of bound or there aren't states
	 */
	public void setCIM(int parentEntry, double[][] CIM) throws RuntimeException {
		
		if(CIM != null) {
			if( (!this.isStaticNode()) && CIM.length != CIM[0].length)
				throw new IllegalArgumentException("The CIM must be a squared matrix in continuous time nodes");
			else if( this.isStaticNode() && CIM.length != 1)
				throw new IllegalArgumentException("The CIM must be a vector in static nodes");
			
			if(CIM[0].length != this.getStatesNumber())
				throw new IllegalArgumentException("The CIM dimensions must correspond with number of states of the node");
		}
		
		this.CIMs.set(parentEntry, CIM);
		this.validatedCIMs = false;
	}
	
	/**
	 * Informs if the node is a static node or a
	 * continuous time node.
	 * The class is a static node.
	 * 
	 * @return true if it is a static node, false otherwise.
	 */
	public boolean isStaticNode() {
		return this.staticNode;
	}
	
	/**
	 * True if the CIMs were validated,
	 * false otherwise.
	 *
	 * @return if all the CIMs were validated.
	 */
	public boolean validatedCIMs() {
		
		return this.validatedCIMs;
	}
	
	/**
	 * Check the validity of all the CIMs
	 * and return the validation result.
	 * 
	 * @return -1 if all the CIMs are validated, otherwise the index of the first not valid parent entry CIM
	 */
	public int checkCIMs() {
		
		if( this.validatedCIMs)
			return -1;
		if( this.CIMs == null)
			return 0;
		
		for( int pE = 0; pE < this.CIMs.size(); ++pE) {
			double[][] cim = this.CIMs.get(pE);
			if(cim == null)
				return pE;
			
			for(int s0 = 0; s0 < cim.length; ++s0) {
				double sum = 0.0;
				for( int s1 = 0; s1 < cim[s0].length; ++s1) {
					if(this.staticNode) {
						if (cim[s0][s1] < 0 || cim[s0][s1] > 1)
							return pE;
					} else if((s0 == s1 && cim[s0][s1] > 0) || (s0 != s1 && cim[s0][s1] < 0))
						return pE;
					
					sum += cim[s0][s1];
				}
				if(((!this.staticNode) && (!Node.equalsDoubles(sum, 0.0, 0.0000001))) || (this.staticNode && (!Node.equalsDoubles(sum, 1.0, 0.0000001))))
					return pE; 
			}		
		}
		
		this.validatedCIMs = true;
		return -1;
	}

	/**
	 * Sample the next transition time given
	 * the current state and the current parents
	 * states.
	 * 
	 * @return the sampled time
	 * @throws RuntimeException if the system state doesn't allow to generate a transition time
	 */
	public double sampleTransitionTime() throws RuntimeException {
		
		if( this.isStaticNode())
			throw new IllegalStateException("Error: static nodes can not change state in time");
		if( !this.validatedCIMs())
			throw new IllegalStateException("Error: CIMs are not validated");
		if( !this.isInstanced())
			throw new IllegalStateException("Error: transition time can be calculated only for instanced nodes");
		
		double lambda = -this.getCIMValue(this.getCurrentParentsEntry(), this.getCurrentStateIndex(), this.getCurrentStateIndex());
		if( lambda == 0)						// if lambda is equal to zero the state is an absorbing state and can not change
			return Double.POSITIVE_INFINITY;
		
		return CTDiscreteNode.expSample(lambda);
	}
	
	/**
	 * Sample a state given the parents.
	 * If the node is static, sample a state
	 * using the prior. If the node is continuous
	 * sample the next jumping state.
	 * Note: sample a new state but doesn't set it.
	 * 
	 * @return index of the sampled state
	 * @throws RuntimeException if the system state doesn't allow to generate a new state
	 */
	public int sampleState() throws RuntimeException {
		
		if( !this.validatedCIMs())
			throw new IllegalStateException("Error: CIMs are not validated");
		
		if( this.isStaticNode())						// if the node is static
			return DiscreteModel.sample(this.getCIM(this.getCurrentParentsEntry())[0]);
		else if( !this.isInstanced())					// if the node is continuous but not instanced
			throw new IllegalStateException("Error: the next state can be calculated only for instanced continuous nodes");
		
		// If the node is continuous and it is instanced
		int stateIndex = this.getCurrentStateIndex();
		double[] pDistr = (this.getCIM(this.getCurrentParentsEntry())[stateIndex]).clone();
		if( pDistr.length < 2)
			throw new IllegalStateException("Error: in continuous node to jump to the next state you need at least 2 states");
		if( pDistr[stateIndex] == 0)
			throw new IllegalStateException("Error: the node it is in an absorbing state. New states can not be sampled");
		
		for(int i = 0; i < pDistr.length; ++i) 			// generate the probability distribution
			if( i != stateIndex)
				pDistr[i] /= -pDistr[stateIndex];		
		pDistr[stateIndex] = 0.0;

		return DiscreteModel.sample(pDistr);
	}
	
	@Override
	public boolean addParent(INode parent) throws RuntimeException {
		
		CTDiscreteNode ctParent = (CTDiscreteNode) parent;
		if( this.staticNode)
			throw new RuntimeException("Error: static nodes (class node) can not have parent");
		
		if( super.addParent(ctParent)) {
			this.nParentsEntries *= ctParent.getStatesNumber();
			this.CIMs = CTDiscreteNode.generateCIMsorCPTs( this.nParentsEntries, this.getStatesNumber(), this.staticNode);
			this.validatedCIMs = false;
			return true;
		}
		
		return false;
	}
	
	@Override
	public boolean removeParent(INode parent) throws RuntimeException {
		
		CTDiscreteNode ctParent = (CTDiscreteNode) parent;
		if( super.removeParent(ctParent)) {
			this.nParentsEntries /= ctParent.getStatesNumber();
			this.CIMs = CTDiscreteNode.generateCIMsorCPTs( this.nParentsEntries, this.getStatesNumber(), this.staticNode);
			this.validatedCIMs = false;
			return true;
		}
		
		return false;
	}
	
//	@Override
//	protected int addState(String stateName) throws IllegalStateException {
//		
//		if(stateName.contains("@"))
//			throw new IllegalStateException("Error: states names can not contain character '@'");
//		
//		int index = super.addState(stateName);
//		if( index != -1) {
//			this.CIMs = CTDiscreteNode.generateCIMsorCPTs( this.nParentsEntries, this.getStatesNumber(), this.staticNode);
//			this.validatedCIMs = false;
//		}
//		
//		return index;
//	}
	
	@Override
	public CTDiscreteNode getChild(int i) throws IllegalArgumentException {
		return (CTDiscreteNode) super.getChild(i);
	}
	
	@Override
	public CTDiscreteNode getChild(String childName) throws IllegalArgumentException {
		return (CTDiscreteNode) super.getChild(childName);
	}
	
	@Override
	public CTDiscreteNode getParent(int i) throws IllegalArgumentException {
		return (CTDiscreteNode) super.getParent(i); 
	}
	
	@Override
	public CTDiscreteNode getParent(String parentName) throws IllegalArgumentException {
		return (CTDiscreteNode) super.getParent(parentName);
	}
	
	/**
	 * Clone the node without coping the parental
	 * relation but only the name and the states.
	 * Also the CIMs are not copied.
	 * 
	 * @return the cloned node
	 */
	@Override
	public CTDiscreteNode clone() {
		
		TreeSet<String> states = new TreeSet<String>();
		for( int i = 0; i < this.getStatesNumber(); ++i)
			states.add(this.getStateName(i));
		
		CTDiscreteNode clonedNode = new CTDiscreteNode(this.getName(), states, this.staticNode);
		if( this.getCurrentState() != null)
			clonedNode.setEvidence(this.getCurrentState());
		
		return clonedNode;
	}
	
	/**
	 * Generate the CIMs or the CPTs for the node.
	 * 
	 * @param nParentsEntries number of entries for the parents
	 * @param nStates number of node states
	 * @param CPT true if the method has to generate CPTs, false if has to generate CIMs 
	 * @return the generated CIMs (null if there are no states)
	 */
	static private List<double[][]> generateCIMsorCPTs(int nParentsEntries, int nStates, boolean CPT) {
		
		if(nStates < 1)
			return null;
		
		List<double[][]> newCIMs = new Vector<double[][]>(nParentsEntries);
		for( int i = 0; i < nParentsEntries; ++i)
			if(!CPT)
				newCIMs.add(new double[nStates][nStates]);
			else
				newCIMs.add(new double[1][nStates]);
			
		return newCIMs;
	}
	
	/**
	 * Sample a value with exponential distribution
	 * with parameter lambda.
	 * 
	 * @param lambda lambda parameter of exponential distribution
	 * @return sampled time
	 */
	static public double expSample(double lambda) {

		return -Math.log(1 - Math.random()) / lambda ;
	}
	
} public interface ICTClassifier<TimeType extends Number & Comparable<TimeType>, NodeType extends INode> extends IModel<TimeType,NodeType>{

	/**
	 * Return the classification node.
	 * Null if it is not defined.
	 * 
	 * @return classification node
	 */
	public DiscreteNode getClassNode();
	
	/**
	 * Classify a trajectory.
	 * 
	 * @param algorithm classification algorithm
	 * @param trajectory input trajectory
	 * @return classification results
	 * @throws Exception in case of errors
	 */
	public IClassificationResult<TimeType> classify(IClassifyAlgorithm<TimeType,NodeType> algorithm, ITrajectory<TimeType> trajectory) throws Exception;
	
	/**
	 * Classify a trajectory.
	 * 
	 * @param algorithm classification algorithm
	 * @param trajectory input trajectory
	 * @param samplingInterval sampling interval to define a more granularity output trajectory
	 * @return classification results
	 * @throws Exception in case of errors
	 */
	public IClassificationResult<TimeType> classify(IClassifyAlgorithm<TimeType,NodeType> algorithm, ITrajectory<TimeType> trajectory, TimeType samplingInterval) throws Exception;
	
	/**
	 * Classify a trajectory.
	 * 
	 * @param algorithm classification algorithm
	 * @param trajectory input trajectory
	 * @param timeStream sequence of time interval in which calculate the probability distribution of the class
	 * @return classification results
	 * @throws Exception in case of errors
	 */
	public IClassificationResult<TimeType> classify(IClassifyAlgorithm<TimeType,NodeType> algorithm, ITrajectory<TimeType> trajectory, Vector<TimeType> timeStream) throws Exception;
} public interface IClassifyDecider {

	/**
	 * Choose one class give the probability
	 * distribution.
	 * 
	 * @param results results from which read the probability distribution for each class
	 * @return index of the chosen class
	 * @throws RuntimeException in case of errors
	 */
	public int decide(IClassificationResult<Double> results) throws RuntimeException;
} public interface ILearningAlgorithm<TimeType extends Number & Comparable<TimeType>, NodeType extends INode> {
	
	/**
	 * Set the default values for the algorithm
	 * parameters.
	 */
	public void setDefaultParameters();
	
	/**
	 * Set the algorithm parameters.
	 * 
	 * @param params maps of parameters represented as map of "parameter name" and "value"
	 * @throws IllegalArgumentException if there is some error with the parameters
	 */
	public void setParameters(Map<String,Object> params) throws IllegalArgumentException;

	/**
	 * Return a parameter value.
	 * 
	 * @param name name of the parameter
	 * @return the parameter value
	 * @throws IllegalArgumentException in case of some error
	 */
	public Object getParameter(String name) throws IllegalArgumentException;
	
	/**
	 * Set the structure of the model.
	 * Parameter learning algorithm must use the input structure
	 * to learn the parameters over it.
	 * Structural learning algorithm must use the input structure
	 * to start the learning from it or (where it doesn't make
	 * sense) they can ignore the loaded network.
	 * Set to null if you want to remove a previous inserted structure.
	 * 
	 * @param adjMatrix adjacency matrix that represent a graph structure (NxN where N is the number of node)
	 * @throws IllegalArgumentException if there is some error with the defined structure
	 */
	public void setStructure(boolean[][] adjMatrix) throws IllegalArgumentException;
	
	/**
	 * Return the loaded structure
	 * (null if it is not loaded).
	 * 
	 * @return the adjacency matrix that represent the structure (null if it is not loaded).
	 */
	public boolean[][] getStructure();

	/**
	 * Return the help description for the algorithm parameters.
	 * 
	 * @return parameters help
	 */
	public String helpParameters();
	
	/**
	 * Learn the model in input given the training set.
	 * 
	 * @param model model to learn
	 * @param trainingSet training set from which learn the model
	 * @return the learning result (a container for the learned sufficient statistics)
	 * @throws RuntimeException if there is some problem with the learning
	 */
	public ILearningResults learn(ICTClassifier<TimeType,NodeType> model, Collection<ITrajectory<TimeType>> trainingSet)  throws RuntimeException;

}
 public interface ITrajectory<TimeType extends Number> {

	/**
	 * Set an optional name for the trajectory
	 * 
	 * @param name the name
	 */
	public void setName(String name);
	
	/**
	 * Get name of the trajectory. Null if it is
	 * not set.
	 * 
	 * @return name of the trajectory
	 */
	public String getName();
	
	/**
	 * Return the number of transitions.
	 * 
	 * @return number of transition in the trajectory.
	 */
	 public int getTransitionsNumber();

	 /**
	  * Return the time related to the selected
	  * transition.
	  * In case of discrete trajectory it will be
	  * an integer. In case of continuous trajectory
	  * it will be a double.
	  * 
	  * @param iTransition index of the transition
	  * @return the time of the selected transition
	  * @throws IllegalArgumentException in case of illegal argument
	  */
	 public TimeType getTransitionTime(int iTransition) throws IllegalArgumentException;
	 
	 /**
	  * Return the node value for the specific node at the
	  * specific transition.
	  * 
	  * @param iTransition index of the transition
	  * @param nodeIndex node index
	  * @return value of the node at the specified transition
	  * @throws IllegalArgumentException if the is an illegal argument
	  */
	 public String getNodeValue(int iTransition, int nodeIndex) throws IllegalArgumentException;
	 
	 /**
	  * Return the selected transition
	  * 
	  * @param iTransition transition index
	  * @return the selected transition
	  * @throws IllegalArgumentException if the is an illegal argument
	  */
	 public ITransition<TimeType> getTransition(int iTransition) throws IllegalArgumentException;
	 
	 /**
	  * Return the global node indexing
	  * associate with the trajectory.
	  *  
	  * @return global node indexing
	  */
	 public NodeIndexing getNodeIndexing();
	 
} public class MultipleCTBNC<TimeType extends Number & Comparable<TimeType>, NodeType extends INode> implements ICTClassifier<TimeType,NodeType> {
	
	static public String otherStateName = "#";
	
	private String name;
	
	private ICTClassifier<TimeType,NodeType> baseModel;
	private List<ICTClassifier<TimeType,NodeType>> models;
	
	private boolean[][] adjMatrix;
	
	
	/**
	 * Base constructor.
	 * 
	 * @param name model name
	 * @param baseModel original model to use to generate the models experts
	 */
	public MultipleCTBNC(String name, ICTClassifier<TimeType,NodeType> baseModel) {
	
		this.name = name;
		this.baseModel = baseModel;
		
		DiscreteNode classNode = baseModel.getClassNode();
		this.models = new Vector<ICTClassifier<TimeType,NodeType>>( classNode.getStatesNumber());
		for( int i = 0; i < classNode.getStatesNumber(); ++i) {
			ICTClassifier<TimeType,NodeType> newModel = (ICTClassifier<TimeType,NodeType>) baseModel.clone();
			
			// Change the class of the new model in a binary class
			newModel.getClassNode().forceBinaryStates( newModel.getClassNode().getStateName( i));
			
			this.models.add( newModel);
		}
	}
	
	
	/**
	 * Constructor to insert directly the elements.
	 * 
	 * @param name model name
	 * @param baseModel original model used to generate the models experts
	 * @param models list of models experts
	 */
	private MultipleCTBNC(String name, ICTClassifier<TimeType,NodeType> baseModel, List<ICTClassifier<TimeType,NodeType>> models) {
		
		if( name == null || name.isEmpty())
			throw new IllegalArgumentException("Error: null or empty model name");
		
		if( baseModel == null)
			throw new IllegalArgumentException("Error: null base models arguments");
		
		if( models == null || models.isEmpty())
			throw new IllegalArgumentException("Error: null or empty list of experts models");
		
		this.name = name;
		this.baseModel = baseModel;
		this.models = models;
	}
	
	
	/**
	 * Get the name of the class with that index.
	 * 
	 * @param classIndex index of the class
	 * @return name of the class with index idx
	 */
	public String getClassName(int classIndex) {
		
		if( classIndex < 0 || classIndex >= models.size())
			throw new IllegalArgumentException("Error: class index out of range");
		
		return this.baseModel.getClassNode().getStateName( classIndex);
	}
	
	
	/**
	 * Return the number of model (it is equal
	 * to the number of classes).
	 * 
	 * @return number of models
	 */
	public int getNumberOfModel() {
		
		return this.models.size();
	}
	
	
	/**
	 * Get the model by index.
	 * 
	 * @param classIndex class index for which the model is an expert
	 * @throws IllegalArgumentException in case of illegal argument
	 */
	public ICTClassifier<TimeType,NodeType> getModel(int classIndex) throws IllegalArgumentException {
		
		if( classIndex < 0 || classIndex >= models.size())
			throw new IllegalArgumentException("Error: a model is defined for each class, so the range of value is {0,..., #classes}");
		
		return this.models.get( classIndex);
	}
	
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.IModel#getName()
	 */
	@Override
	public String getName() {
		
		return this.name;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.IModel#getNode(int)
	 */
	@Override
	public NodeType getNode(int iNode) throws IllegalArgumentException {

		return this.baseModel.getNode(iNode);
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.IModel#generateTrajectory(double)
	 */
	@Override
	public ITrajectory<TimeType> generateTrajectory(double T)
			throws RuntimeException {
		
		throw new RuntimeException("Error: MultipleCTBNC class does not support trajectories generation");
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.IModel#getAdjMatrix()
	 */
	@Override
	public boolean[][] getAdjMatrix() {
		
		return this.adjMatrix;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.IModel#setStructure(boolean[][])
	 */
	@Override
	public void setStructure(boolean[][] adjMatrix) throws RuntimeException {
		
		for(int i = 0; i < this.models.size(); ++i)
			this.models.get(i).setStructure(adjMatrix);
		
		this.adjMatrix = adjMatrix;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.IModel#getNodeIndexing()
	 */
	@Override
	public NodeIndexing getNodeIndexing() {

		return this.baseModel.getNodeIndexing();
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.ICTClassifier#getClassNode()
	 */
	@Override
	public DiscreteNode getClassNode() {

		return this.baseModel.getClassNode();
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.ICTClassifier#classify(CTBNToolkit.IClassifyAlgorithm, CTBNToolkit.ITrajectory)
	 */
	@Override
	public IClassificationResult<TimeType> classify(
			IClassifyAlgorithm<TimeType, NodeType> algorithm,
			ITrajectory<TimeType> trajectory) throws Exception {
		
		// Classify with all the models
		List<IClassificationResult<TimeType>> results = new Vector<IClassificationResult<TimeType>>(this.models.size());
		for( int i = 0; i < this.models.size(); ++i) {
			// Transform the trajectory
			ITrajectory<TimeType> trj = generateClassTrajectory(trajectory, i);
			
			// Classify the trajectory
			results.add( algorithm.classify( this.getModel(i), trj));
		}
		
		
		// Generate the result
		// Initialization
		DiscreteNode classNode = this.baseModel.getClassNode();
		Map<String,Integer> stateToIndex = new TreeMap<String,Integer>();
		for(int i = 0; i < classNode.getStatesNumber(); ++i)
			stateToIndex.put(classNode.getStateName(i), i);
		
		// Result initialization
		ClassificationResults<TimeType> finalResult = new ClassificationResults<TimeType>( trajectory);
		int iJmp;
		if( algorithm.probabilityFlag())
			iJmp = 0;
		else
			iJmp = finalResult.getTransitionsNumber() - 1;
		
		// Calculate the probabilities
		for(; iJmp < finalResult.getTransitionsNumber(); ++iJmp) {
			double[] p = new double[results.size()];
			
			// Get the probability for each class
			for(int iC = 0; iC < p.length; ++iC)
				p[iC] = results.get( iC).getProbability( classNode.getStateName( iC));
			
			// Normalize
			p = MultipleCTBNC.normalizeDistribution(p);
			
			// Set result
			finalResult.setProbability(iJmp, p, stateToIndex);
			if( iJmp == finalResult.getTransitionsNumber() - 1)
				finalResult.setClassification( classNode.getStateName( MultipleCTBNC.indexMax(p)));
		}
		
		return finalResult;
	}


	@Override
	public IClassificationResult<TimeType> classify(
			IClassifyAlgorithm<TimeType, NodeType> algorithm,
			ITrajectory<TimeType> trajectory, TimeType samplingInterval)
			throws Exception {
		
		// Classify with all the models
		List<IClassificationResult<TimeType>> results = new Vector<IClassificationResult<TimeType>>(this.models.size());
		for( int i = 0; i < this.models.size(); ++i) {
			// Transform the trajectory
			ITrajectory<TimeType> trj = generateClassTrajectory(trajectory, i);
			
			// Classify the trajectory
			results.add( algorithm.classify( this.getModel(i), trj, samplingInterval));
		}
		
		
		// Generate the result
		// Initialization
		DiscreteNode classNode = this.baseModel.getClassNode();
		Map<String,Integer> stateToIndex = new TreeMap<String,Integer>();
		for(int i = 0; i < classNode.getStatesNumber(); ++i)
			stateToIndex.put(classNode.getStateName(i), i);
		Vector<TimeType> timeStream = new Vector<TimeType>( results.get(0).getTransitionsNumber());
		for(int i = 0; i < results.get(0).getTransitionsNumber(); ++i)
			timeStream.add( results.get(0).getTransitionTime(i));
		
		// Result initialization
		ClassificationResults<TimeType> finalResult = new ClassificationResults<TimeType>( trajectory, timeStream);
		int iJmp;
		if( algorithm.probabilityFlag())
			iJmp = 0;
		else
			iJmp = finalResult.getTransitionsNumber() - 1;
		
		// Calculate the probabilities
		for(; iJmp < finalResult.getTransitionsNumber(); ++iJmp) {
			double[] p = new double[results.size()];
			
			// Get the probability for each class
			for(int iC = 0; iC < p.length; ++iC)
				p[iC] = results.get( iC).getProbability( classNode.getStateName( iC));
			
			// Normalize
			p = MultipleCTBNC.normalizeDistribution(p);
			
			// Set result
			finalResult.setProbability(iJmp, p, stateToIndex);
			if( iJmp == finalResult.getTransitionsNumber() - 1)
				finalResult.setClassification( classNode.getStateName( MultipleCTBNC.indexMax(p)));
		}
		
		return finalResult;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.ICTClassifier#classify(CTBNToolkit.IClassifyAlgorithm, CTBNToolkit.ITrajectory, java.util.Vector)
	 */
	@Override
	public IClassificationResult<TimeType> classify(
			IClassifyAlgorithm<TimeType, NodeType> algorithm,
			ITrajectory<TimeType> trajectory, Vector<TimeType> timeStream)
			throws Exception {
		
		// Classify with all the models
		List<IClassificationResult<TimeType>> results = new Vector<IClassificationResult<TimeType>>(this.models.size());
		for( int i = 0; i < this.models.size(); ++i) {
			// Transform the trajectory
			ITrajectory<TimeType> trj = generateClassTrajectory(trajectory, i);
			
			// Classify the trajectory
			results.add( algorithm.classify( this.getModel(i), trj, timeStream));
		}
		
		
		// Generate the result
		// Initialization
		DiscreteNode classNode = this.baseModel.getClassNode();
		Map<String,Integer> stateToIndex = new TreeMap<String,Integer>();
		for(int i = 0; i < classNode.getStatesNumber(); ++i)
			stateToIndex.put(classNode.getStateName(i), i);
		
		// Result initialization
		ClassificationResults<TimeType> finalResult = new ClassificationResults<TimeType>( trajectory, timeStream);
		int iJmp;
		if( algorithm.probabilityFlag())
			iJmp = 0;
		else
			iJmp = finalResult.getTransitionsNumber() - 1;
		
		// Calculate the probabilities
		for(; iJmp < finalResult.getTransitionsNumber(); ++iJmp) {
			double[] p = new double[results.size()];
			
			// Get the probability for each class
			for(int iC = 0; iC < p.length; ++iC)
				p[iC] = results.get( iC).getProbability( classNode.getStateName( iC));
			
			// Normalize
			p = MultipleCTBNC.normalizeDistribution(p);
			
			// Set result
			finalResult.setProbability(iJmp, p, stateToIndex);
			if( iJmp == finalResult.getTransitionsNumber() - 1)
				finalResult.setClassification( classNode.getStateName( MultipleCTBNC.indexMax(p)));
		}
		
		return finalResult;
	}

	@Override
	public IModel<TimeType,NodeType> clone() {
		
		List<ICTClassifier<TimeType,NodeType>> newModels = new Vector<ICTClassifier<TimeType,NodeType>>(this.models.size());
		for(int i = 0; i < this.models.size(); ++i)
			newModels.add(( ICTClassifier<TimeType,NodeType>) this.models.get( i).clone());
		
		MultipleCTBNC<TimeType,NodeType> clonedModel = new MultipleCTBNC<TimeType, NodeType>(this.name, this.baseModel, newModels);
		
		return clonedModel;
	}
	
	
	/**
	 * Print the model in string.
	 * 
	 * @return the representation of the model
	 */
	public String toString() {
		
		StringBuilder strBuilder = new StringBuilder();
		for( int i = 0; i < this.models.size(); ++i) {
			strBuilder.append("Model " + i + " Class " + this.baseModel.getClassNode().getStateName(i) + "\n");
			strBuilder.append(this.models.get(i).toString() + "\n\n");
		}
		
		return strBuilder.toString();
	}
	
	
	/**
	 * Transform the input trajectories in an
	 * equal trajectory where the class variable
	 * has only 2 states: the one associated to
	 * the class index and all the others.
	 * 
	 * @param trajectory trajectory to transform
	 * @param classIdx class index
	 * @return binary trajectory
	 */
	public ITrajectory<TimeType> generateClassTrajectory(ITrajectory<TimeType> trajectory, int classIdx) {
		
		List<TimeType> times = new Vector<TimeType>( trajectory.getTransitionsNumber());
		List<String[]> values = new Vector<String[]>( trajectory.getTransitionsNumber());
		
		int nNodes = trajectory.getNodeIndexing().getNodesNumber();
		int classNodeIndex = trajectory.getNodeIndexing().getClassIndex();
		for( int i = 0; i < trajectory.getTransitionsNumber(); ++i) {
			
			// Copy the time
			times.add( trajectory.getTransitionTime(i));
			
			// Copy the states
			String[] v = new String[nNodes];
			for(int j = 0; j < nNodes; ++j)
				v[j] = trajectory.getNodeValue(i, j);
			
			// Change the class state
			if( !v[classNodeIndex].equals( this.baseModel.getClassNode().getStateName( classIdx)))
				v[classNodeIndex] = MultipleCTBNC.otherStateName;
			
			// Update the values
			values.add(v);
		}
		
		return (ITrajectory<TimeType>) new CTTrajectory<TimeType>( trajectory.getNodeIndexing(), times, values);
	}
	

	/**
	 * Normalize a vector generating a distribution.
	 * 
	 * @param p vector to normalize
	 * @return a probability distribution
	 */
	private static double[] normalizeDistribution(double[] p) throws IllegalArgumentException {

		if( p == null || p.length < 1)
			throw new IllegalArgumentException("Error: empty vector");
		
		double s = 0;
		double[] newP = new double[p.length]; 
		for( int i = 0; i < p.length; ++i)
			s += p[i];
		for( int i = 0; i < p.length; ++i)
			newP[i] = p[i]/s;
		
		return newP;
	}


	/**
	 * Find the index of the maximum value.
	 * 
	 * @param p vector of values
	 * @return index of the maximum value
	 */
	private static int indexMax(double[] p) throws IllegalArgumentException {

		if( p == null || p.length < 1)
			throw new IllegalArgumentException("Error: empty vector");
		
		double max = p[0];
		int iMax = 0;
		for(int i = 1; i < p.length; ++i)
			if( p[i] > max) {
				max = p[i];
				iMax = i;
			}
		
		return iMax;
	}
} public class MultipleCTBNCLearningAlgorithm<TimeType extends Number & Comparable<TimeType>, NodeType extends INode> extends
		LearningAlgorithm<TimeType, NodeType> implements
		ILearningAlgorithm<TimeType, NodeType> {
	
	private boolean[][] adjMStructure;
	private ILearningAlgorithm<TimeType, NodeType> learningAlgorithm;
	
	
	/**
	 * Base constructor.
	 * 
	 * @param learningAlgorithm algorithm to use to learn all the model of the MultipleCTBNC
	 */
	public MultipleCTBNCLearningAlgorithm(ILearningAlgorithm<TimeType, NodeType> learningAlgorithm) {
		
		if( learningAlgorithm == null)
			throw new IllegalArgumentException("Error: null argument. A learning algorithm is required to learn the CTBNC models");
		
		this.learningAlgorithm = learningAlgorithm;
	}
	

	/* (non-Javadoc)
	 * @see CTBNToolkit.ILearningAlgorithm#setDefaultParameters()
	 */
	@Override
	public void setDefaultParameters() { }

	/* (non-Javadoc)
	 * @see CTBNToolkit.ILearningAlgorithm#setStructure(boolean[][])
	 */
	@Override
	public void setStructure(boolean[][] adjMatrix)
			throws IllegalArgumentException {

		this.adjMStructure = adjMatrix;		
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.ILearningAlgorithm#getStructure()
	 */
	@Override
	public boolean[][] getStructure() {
		
		return this.adjMStructure;
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.ILearningAlgorithm#helpParameters()
	 */
	@Override
	public String helpParameters() {
		
		String helpStr = "";
		helpStr += "Class MultipleCTBNCLearningAlgorithm does not require parameters\n";
		
		return helpStr;
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.ILearningAlgorithm#learn(CTBNToolkit.ICTClassifier, java.util.Collection)
	 */
	@Override
	public ILearningResults learn(ICTClassifier<TimeType, NodeType> model,
			Collection<ITrajectory<TimeType>> trainingSet)
			throws RuntimeException {

		if( !MultipleCTBNC.class.isAssignableFrom( model.getClass()))
			throw new RuntimeException("Error: MultipleCTBNCLearningAlgorithm can be used only over MultipleCTBNC models");
		
		MultipleCTBNC<TimeType, NodeType> mCTBNC = (MultipleCTBNC<TimeType, NodeType>) model; 
		List<ILearningResults> learningResults = new Vector<ILearningResults>( mCTBNC.getNumberOfModel());
		
		// Learn each model expert
		for(int i = 0; i < mCTBNC.getNumberOfModel(); ++i) {
			
			// Generate the transformed training set
			Collection<ITrajectory<TimeType>> newTrainingSet = new Vector<ITrajectory<TimeType>>(trainingSet.size());
			Iterator<ITrajectory<TimeType>> trjIter = trainingSet.iterator();
			while( trjIter.hasNext())
				newTrainingSet.add( mCTBNC.generateClassTrajectory( trjIter.next(), i));
			
			// Learn the model expert
			learningResults.add( this.learningAlgorithm.learn( mCTBNC.getModel(i), newTrainingSet));
		}		
		
		return new MultipleCTBNCLearningResults( learningResults);
	}

} public class CTBNClusteringParametersLLAlgorithm extends ClusteringAlgorithm<Double, CTDiscreteNode> {

	private boolean[][] adjMStructure;
	private double Mxx_prior;
	private double Tx_prior;
	private double Px_prior;
	private boolean hardClustering;
	
	
	/**
	 * Base constructor.
	 * Set the default parameters and a
	 * disconnected structure.
	 */
	public CTBNClusteringParametersLLAlgorithm() {

		this.adjMStructure = null;
		
		// Classification algorithm setting
		Map<String, Object> clAlgParams = new TreeMap<String,Object>();
		clAlgParams.put("probabilities", true);
		CTBNCClassifyAlgorithm classAlg = new CTBNCClassifyAlgorithm();
		classAlg.setParameters(clAlgParams);
		super.setClassificationAlgorithm( classAlg);
		
		// Stopping criterion setting
		super.setStopCriterion( new StandardStopCriterion(10, 1/100.0));
		
		// Setting default parameters
		this.setDefaultParameters();
	}
	
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.ILearningAlgorithm#setDefaultParameters()
	 */
	@Override
	public void setDefaultParameters() {
		
		this.Mxx_prior = 0.0;
		this.Tx_prior = 0.0;
		this.Px_prior = 0.0;
		this.hardClustering = false;
		
		Map<String, Object> params = new TreeMap<String,Object>();
		params.put("Mxx_prior", this.Mxx_prior);
		params.put("Tx_prior", this.Tx_prior);
		params.put("Px_prior", this.Px_prior);
		params.put("hardClustering", this.hardClustering);
		super.setParameters(params);
	}
	
	
	@Override
	public void setParameters(Map<String, Object> params) throws IllegalArgumentException {

		super.setParameters(params);
		
		Double tmpDouble;
		Boolean tmpBool;
		// Mxx_prior
		tmpDouble = (Double)params.get("Mxx_prior");
		if( tmpDouble != null)
			this.Mxx_prior = tmpDouble;
			
		// Tx_prior
		tmpDouble = (Double)params.get("Tx_prior");
		if( tmpDouble != null)
			this.Tx_prior = tmpDouble;
		
		// Px_prior
		tmpDouble = (Double)params.get("Px_prior");
		if( tmpDouble != null)
			this.Px_prior = tmpDouble;
		
		// hardClustering
		tmpBool = (Boolean)params.get("hardClustering");
		if( tmpBool != null)
			this.hardClustering = tmpBool;

	}

	
	/* (non-Javadoc)
	 * @see CTBNToolkit.ILearningAlgorithm#setStructure(boolean[][])
	 */
	@Override
	public void setStructure(boolean[][] adjMatrix) throws IllegalArgumentException {
		
		this.adjMStructure = adjMatrix;
		
	}

	
	/* (non-Javadoc)
	 * @see CTBNToolkit.ILearningAlgorithm#helpParameters()
	 */
	@Override
	public String helpParameters() {

		String helpStr = "";
		helpStr += "Parameters class CTBNClusteringParametersLLAlgorithm:\n";
		helpStr += "Mxx_prior: prior for the value M[x,x'|u] (equal value for each x, x' and u). The sum over x' is the prior for M[x|u]. [Default value = 0].\n";
		helpStr += "Tx_prior: prior for the value T[x|u] (equal value for each x and u). [Default value = 0]. \n";
		helpStr += "Px_prior: prior for the value P[x|u] of the distribution of static nodes (equal value for each x and u). [Default value = 0]. \n";
		helpStr += "hardClustering: true to use hard clustering algorithm, false to use soft clustering algorithm. [Default value = false]. \n";
		
		return helpStr;
	}

	
	/* (non-Javadoc)
	 * @see CTBNToolkit.ILearningAlgorithm#learn(CTBNToolkit.IModel, java.util.Collection)
	 */
	@Override
	public ClusteringResults<Double> learn(
			ICTClassifier<Double, CTDiscreteNode> model,
			Collection<ITrajectory<Double>> trainingSet)
			throws RuntimeException {

		
		// Parameters checking
		if( (!this.classificationAlgorithm.probabilityFlag()) && (!this.hardClustering))
			throw new IllegalArgumentException("Soft clustering algorithm needs a classification algorithm able to return the probability distribution of the class");
		if( trainingSet == null)
			throw new IllegalArgumentException("Error: null training set argument");
		if( trainingSet.size() == 0)
			throw new IllegalArgumentException("Error: empty training set");
		if( model == null)
			throw new IllegalArgumentException("Error: null model argument");
		
		NodeIndexing nodeIndexing = model.getNodeIndexing();
		int classNodeIndex = nodeIndexing.getClassIndex();
		if( classNodeIndex == -1)
			throw new IllegalArgumentException("Error: general indexing method doesn't contain " + classNodeIndex + " class node index");
		
		//Algorithm initialization
		// Set the structure
		if( this.adjMStructure != null)
			model.setStructure(this.adjMStructure);		

		// Generate the sufficient statistics divided by trajectory
		SufficientStatistics[][] trjSStats = generateTrajectoriesSufficientStatistics(model, trainingSet);
		// Generate initial sufficient statistics
		SufficientStatistics[] sStats = calculateInitialSufficientStatistics(model, trjSStats);
		// Set the parameters
		setCIMs(model, sStats);
		
		// Clustering algorithm
		double changedCluster = Double.MAX_VALUE;
		List<IClassificationResult<Double>> clusterizedTrajectories = initializeClusterizedTrajectories(trainingSet);
		for(int i = 0; true; ++i) {

			// Classification algorithm
			changedCluster = 0;
			for(int iTrj = 0; iTrj < clusterizedTrajectories.size(); ++iTrj) {
				IClassificationResult<Double> trjResult = this.classificationAlgorithm.classify(model, clusterizedTrajectories.get(iTrj));	// classify
				if( !trjResult.getClassification().equals(clusterizedTrajectories.get(iTrj).getClassification()))  // count the trajectories that changed value 
					++changedCluster;
				clusterizedTrajectories.set(iTrj, trjResult);						// add the new classified trajectory
			}
			
			// Stopping criteria evaluation
			if( stopCriterion.paramsStop(i, changedCluster/clusterizedTrajectories.size()))
				break;
			
			// New sufficient statistics calculation
			if( this.hardClustering)
				sStats = calculateNewHardSufficientStatistics(model, trjSStats, clusterizedTrajectories, sStats);
			else
				sStats = calculateNewSoftSufficientStatistics(model, trjSStats, clusterizedTrajectories, sStats);
			// Set the parameters
			setCIMs(model, sStats);
		}
		
		return new ClusteringResults<Double>(sStats, clusterizedTrajectories);
	}
	
	
	/**
	 * Calculate hard sufficient statistics.
	 * Make the calculation using the sufficient
	 * statistics for each trajectory (trjSStats) 
	 * and the initialized vector of sufficient
	 * statistics.
	 * If this vector will contain old sufficient
	 * statistics for the nodes (not the class) that
	 * have not the class as parent, these sufficient
	 * statistics are left unchanged. Instead, if
	 * the statistics of a node are null than the
	 * new statistics are normally calculated.
	 * 
	 * Note: the trajectories sufficient statistics
	 * are calculated supposing the assignment of
	 * state 0 for the class node.
	 * 
	 * @param model model for which we are calculating the sufficient statistics
	 * @param trjSStats sufficient statistics for each trajectories
	 * @param clusterizedTrajectories trajectories previous classificated
	 * @param stats initialized vector of sufficient statistics
	 * @return completed sufficient statistics over all trajectories
	 */
	private SufficientStatistics[] calculateNewHardSufficientStatistics(
			ICTClassifier<Double, CTDiscreteNode> model,
			SufficientStatistics[][] trjSStats,
			List<IClassificationResult<Double>> clusterizedTrajectories,
			SufficientStatistics[] stats) {
		
		// Initialization
		NodeIndexing nodeIndexing = model.getNodeIndexing();
		int iClassNode = nodeIndexing.getClassIndex();
		CTDiscreteNode classNode = model.getNode(iClassNode);
		stats[iClassNode] = new SufficientStatistics(classNode.getNumberParentsEntries(), classNode.getStatesNumber(), classNode.isStaticNode(), this.Px_prior, this.Mxx_prior, this.Tx_prior);
		//! Safety check. To extend the code it is necessary
		// define the soft sample for the instantiation of
		// the parents of the class node (see next for)
		if( stats[iClassNode].Px.length != 1)
			throw new RuntimeException("Error: clustering algorithm is developted for class node that have not parents");
		
		// Class node sufficient statistics
		int[] trjClassIdx = new int[clusterizedTrajectories.size()];
		for(int iTrj = 0; iTrj < clusterizedTrajectories.size(); ++iTrj) {
			trjClassIdx[iTrj] = classNode.getStateIndex( clusterizedTrajectories.get( iTrj).getClassification());

			++stats[iClassNode].Px[0][0][trjClassIdx[iTrj]];
			++stats[iClassNode].counts[0];
		}
				
		// Sufficient statistics calculation
		for(int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {			// for each node
			if( iNode == iClassNode)													// ignore the class node
				continue;
			
			CTDiscreteNode node = model.getNode(iNode);
			
			if( node.getParentIndex( classNode.getName()) == null) {					// if the class is not in the node parent set
				if( stats[iNode] != null)												// if the statistics are previously calculated
					continue;															// ignore the node and keep the old statistics
				
				stats[iNode] = new SufficientStatistics(node.getNumberParentsEntries(), node.getStatesNumber(), node.isStaticNode(), this.Px_prior, this.Mxx_prior, this.Tx_prior);
				if( node.isStaticNode()) {												// if the node is static
					for(int iTrj = 0; iTrj < trjSStats.length; ++iTrj)
						for( int pE = 0; pE < trjSStats[iTrj][iNode].Px.length; ++pE) {
							stats[iNode].counts[pE] += trjSStats[iTrj][iNode].counts[pE];
							for( int sE = 0; sE < trjSStats[iTrj][iNode].Px[0][0].length; ++sE)
								stats[iNode].Px[pE][0][sE] += trjSStats[iTrj][iNode].Px[pE][0][sE];
						}
				} else { 																// if the node is dynamic
					for(int iTrj = 0; iTrj < trjSStats.length; ++iTrj)
						for( int pE = 0; pE < trjSStats[iTrj][iNode].Mxx.length; ++pE)
							for( int fSE = 0; fSE < trjSStats[iTrj][iNode].Mxx[0].length; ++fSE) {
								stats[iNode].Tx[pE][fSE] += trjSStats[iTrj][iNode].Tx[pE][fSE];
								stats[iNode].Mx[pE][fSE] += trjSStats[iTrj][iNode].Mx[pE][fSE];
								for( int sSE = 0; sSE < trjSStats[iTrj][iNode].Mxx[0][0].length; ++sSE)
									stats[iNode].Mxx[pE][fSE][sSE] += trjSStats[iTrj][iNode].Mxx[pE][fSE][sSE];
							}
				}
			} else {																	// if the class node is a parent in the node parent set
				stats[iNode] = new SufficientStatistics(node.getNumberParentsEntries(), node.getStatesNumber(), node.isStaticNode(), this.Px_prior, this.Mxx_prior, this.Tx_prior);
				
				if( node.isStaticNode()) {												// if the node is static
					for(int iTrj = 0; iTrj < trjSStats.length; ++iTrj)
						for( int pE = 0; pE < trjSStats[iTrj][iNode].Px.length; ++pE) {
							// Get class state in the current parent entry
							node.setParentsEntry(pE);
							if( classNode.getCurrentStateIndex() != trjClassIdx[iTrj])	// if it is not the parent entry with the right class state
								continue;
							// Get the pE used to save the parameters for each trajectories
							classNode.setEvidence(0);
							int pETrj = node.getCurrentParentsEntry();
							
							stats[iNode].counts[pE] += trjSStats[iTrj][iNode].counts[pETrj];
							for( int sE = 0; sE < trjSStats[iTrj][iNode].Px[0][0].length; ++sE)
								stats[iNode].Px[pE][0][sE] += trjSStats[iTrj][iNode].Px[pETrj][0][sE];
						}
				} else { 																// if the node is dynamic
					for(int iTrj = 0; iTrj < trjSStats.length; ++iTrj)
						for( int pE = 0; pE < trjSStats[iTrj][iNode].Mxx.length; ++pE) {
							// Get class state in the current parent entry
							node.setParentsEntry(pE);
							if( classNode.getCurrentStateIndex() != trjClassIdx[iTrj])	// if it is not the parent entry with the right class state
								continue;
							// Get the pE used to save the parameters for each trajectories
							classNode.setEvidence(0);
							int pETrj = node.getCurrentParentsEntry();							

							for( int fSE = 0; fSE < trjSStats[iTrj][iNode].Mxx[0].length; ++fSE) {
								stats[iNode].Tx[pE][fSE] += trjSStats[iTrj][iNode].Tx[pETrj][fSE];
								stats[iNode].Mx[pE][fSE] += trjSStats[iTrj][iNode].Mx[pETrj][fSE];
								for( int sSE = 0; sSE < trjSStats[iTrj][iNode].Mxx[0][0].length; ++sSE)
									stats[iNode].Mxx[pE][fSE][sSE] += trjSStats[iTrj][iNode].Mxx[pETrj][fSE][sSE];
							}
						}
				}
			}
		}
		
		// Retract all the evidences
		for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode)
			model.getNode(iNode).retractEvidence();
		
		return stats;
	}


	/**
	 * Calculate soft sufficient statistics for
	 * all the nodes except the class node.
	 * Make the calculation using the sufficient
	 * statistics for each trajectory (trjSStats),
	 * the probability distribution of the class
	 * for each trajectory (trjPDist) and the
	 * initialized vector of sufficient statistics
	 * with the statistics of the class calculated
	 * previously.
	 * If this vector will contain old sufficient
	 * statistics for the nodes that have not the
	 * class as parent, these sufficient statistics
	 * are left unchanged. Instead, if the statistics
	 * of a node are null than the new statistics
	 * are normally calculated.
	 * 
	 * Note: the trajectories sufficient statistics
	 * are calculated supposing the assignment of
	 * state 0 for the class node.
	 * 
	 * @param model model for which we are calculating the sufficient statistics
	 * @param trjSStats sufficient statistics for each trajectories
	 * @param trjPDist probability distribution over the class for each trajectory
	 * @param stats initialized vector of sufficient statistics (class sufficient statistics must be calculated previously)
	 * @return completed sufficient statistics over all trajectories
	 */
	private SufficientStatistics[] calculateSoftSufficientStatistics(
			ICTClassifier<Double, CTDiscreteNode> model,
			SufficientStatistics[][] trjSStats,
			double[][] trjPDist,
			SufficientStatistics[] stats) {
		
		// Initialization
		NodeIndexing nodeIndexing = model.getNodeIndexing();
		int iClassNode = nodeIndexing.getClassIndex();
		CTDiscreteNode classNode = model.getNode(iClassNode);
		
		// Sufficient statistics calculation
		for(int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {		// for each node
			if( iNode == iClassNode)												// ignore the class node
				continue;
			
			CTDiscreteNode node = model.getNode(iNode);
			
			if( node.getParentIndex( classNode.getName()) == null) {				// if the class is not in the node parent set
				if( stats[iNode] != null)											// if the statistics are previously calculated
					continue;														// ignore the node and keep the old statistics
				
				stats[iNode] = new SufficientStatistics(node.getNumberParentsEntries(), node.getStatesNumber(), node.isStaticNode(), this.Px_prior, this.Mxx_prior, this.Tx_prior);
				if( node.isStaticNode()) {											// if the node is static
					for(int iTrj = 0; iTrj < trjSStats.length; ++iTrj)
						for( int pE = 0; pE < trjSStats[iTrj][iNode].Px.length; ++pE) {
							stats[iNode].counts[pE] += trjSStats[iTrj][iNode].counts[pE];
							for( int sE = 0; sE < trjSStats[iTrj][iNode].Px[0][0].length; ++sE)
								stats[iNode].Px[pE][0][sE] += trjSStats[iTrj][iNode].Px[pE][0][sE];
						}
				} else { 															// if the node is dynamic
					for(int iTrj = 0; iTrj < trjSStats.length; ++iTrj)
						for( int pE = 0; pE < trjSStats[iTrj][iNode].Mxx.length; ++pE)
							for( int fSE = 0; fSE < trjSStats[iTrj][iNode].Mxx[0].length; ++fSE) {
								stats[iNode].Tx[pE][fSE] += trjSStats[iTrj][iNode].Tx[pE][fSE];
								stats[iNode].Mx[pE][fSE] += trjSStats[iTrj][iNode].Mx[pE][fSE];
								for( int sSE = 0; sSE < trjSStats[iTrj][iNode].Mxx[0][0].length; ++sSE)
									stats[iNode].Mxx[pE][fSE][sSE] += trjSStats[iTrj][iNode].Mxx[pE][fSE][sSE];
							}
				}
			} else {																// if the class node is a parent in the node parent set
				stats[iNode] = new SufficientStatistics(node.getNumberParentsEntries(), node.getStatesNumber(), node.isStaticNode(), this.Px_prior, this.Mxx_prior, this.Tx_prior);
				
				if( node.isStaticNode()) {											// if the node is static
					for(int iTrj = 0; iTrj < trjSStats.length; ++iTrj)
						for( int pE = 0; pE < trjSStats[iTrj][iNode].Px.length; ++pE) {
							// Get class state in the current parent entry
							node.setParentsEntry(pE);
							int iClassState = classNode.getCurrentStateIndex();
							// Get the pE used to save the parameters for each trajectories
							classNode.setEvidence(0);
							int pETrj = node.getCurrentParentsEntry();
							
							stats[iNode].counts[pE] += ( trjSStats[iTrj][iNode].counts[pETrj] * trjPDist[iTrj][iClassState]);
							for( int sE = 0; sE < trjSStats[iTrj][iNode].Px[0][0].length; ++sE)
								stats[iNode].Px[pE][0][sE] += ( trjSStats[iTrj][iNode].Px[pETrj][0][sE] * trjPDist[iTrj][iClassState]);
						}
				} else { 															// if the node is dynamic
					for(int iTrj = 0; iTrj < trjSStats.length; ++iTrj)
						for( int pE = 0; pE < trjSStats[iTrj][iNode].Mxx.length; ++pE) {
							// Get class state in the current parent entry
							node.setParentsEntry(pE);
							int iClassState = classNode.getCurrentStateIndex();
							// Get the pE used to save the parameters for each trajectories
							classNode.setEvidence(0);
							int pETrj = node.getCurrentParentsEntry();							

							for( int fSE = 0; fSE < trjSStats[iTrj][iNode].Mxx[0].length; ++fSE) {
								stats[iNode].Tx[pE][fSE] += ( trjSStats[iTrj][iNode].Tx[pETrj][fSE] * trjPDist[iTrj][iClassState]);
								stats[iNode].Mx[pE][fSE] += ( trjSStats[iTrj][iNode].Mx[pETrj][fSE] * trjPDist[iTrj][iClassState]);
								for( int sSE = 0; sSE < trjSStats[iTrj][iNode].Mxx[0][0].length; ++sSE)
									stats[iNode].Mxx[pE][fSE][sSE] += ( trjSStats[iTrj][iNode].Mxx[pETrj][fSE][sSE] * trjPDist[iTrj][iClassState]);
							}
						}
				}
			}
		}
		
		// Retract all the evidences
		for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode)
			model.getNode(iNode).retractEvidence();
		
		return stats;
	}
	
	
	/**
	 * Calculate the new sufficient statistics
	 * using the sufficient statistics memorized
	 * for each trajectory and the previous
	 * classification.
	 * 
	 * @param model model for which we are calculating the sufficient statistics
	 * @param trjSStats sufficient statistics for each trajectories
	 * @param clusterizedTrajectories trajectories previous clusterized
	 * @param oldStats old sufficient statistics used to avoid to calculates statistics for the nodes that doesn't have the class as parent
	 * @return the results of the clustering (sufficient statistics and clusterized training set in the same order of the original training set)
	 */
	private SufficientStatistics[] calculateNewSoftSufficientStatistics(
			ICTClassifier<Double, CTDiscreteNode> model,
			SufficientStatistics[][] trjSStats,
			List<IClassificationResult<Double>> trajectories,
			SufficientStatistics[] oldStats) {

		// Initialization
		NodeIndexing nodeIndexing = model.getNodeIndexing();
		int iClassNode = nodeIndexing.getClassIndex();
		CTDiscreteNode classNode = model.getNode(iClassNode);
		oldStats[iClassNode] = new SufficientStatistics(classNode.getNumberParentsEntries(), classNode.getStatesNumber(), classNode.isStaticNode(), this.Px_prior, this.Mxx_prior, this.Tx_prior);
		//! Safety check. To extend the code it is necessary
		// define the soft sample for the instantiation of
		// the parents of the class node (see next for)
		if( oldStats[iClassNode].Px.length != 1)
			throw new RuntimeException("Error: clustering algorithm is developted for class node that have not parents");
		
		// Trajectories class initialization and class node sufficient statistics
		double[][] trjPDist = new double[trjSStats.length][classNode.getStatesNumber()];
		for(int iTrj = 0; iTrj < trjPDist.length; ++iTrj) {
			trjPDist[iTrj] = trajectories.get( iTrj).getPDistribution();
			
			for(int sE = 0; sE < trjPDist[0].length; ++sE) {
				oldStats[iClassNode].Px[0][0][sE] += trjPDist[iTrj][sE];
				oldStats[iClassNode].counts[0] += trjPDist[iTrj][sE];
			}
		}
		
		return calculateSoftSufficientStatistics(model, trjSStats, trjPDist, oldStats);
	}


	/**
	 * Calculate the initial sufficient statistics.
	 *  
	 * @param model model for which we are calculating the sufficient statistics
	 * @param trjSStats sufficient statistics for each trajectories
	 * @return initial sufficient statistics
	 */
	private SufficientStatistics[] calculateInitialSufficientStatistics(
			ICTClassifier<Double, CTDiscreteNode> model,
			SufficientStatistics[][] trjSStats) {

		// Initialization
		NodeIndexing nodeIndexing = model.getNodeIndexing();
		int iClassNode = nodeIndexing.getClassIndex();
		CTDiscreteNode classNode = model.getNode(iClassNode);
		SufficientStatistics[] stats = new SufficientStatistics[nodeIndexing.getNodesNumber()];
		stats[iClassNode] = new SufficientStatistics(classNode.getNumberParentsEntries(), classNode.getStatesNumber(), classNode.isStaticNode(), this.Px_prior, this.Mxx_prior, this.Tx_prior);
		//! Safety check. To extend the code it is necessary
		// define the soft sample for the instantiation of
		// the parents of the class node (see next for)
		if( stats[iClassNode].Px.length != 1)
			throw new RuntimeException("Error: clustering algorithm is developted for class node that have not parents");
		
		// Trajectories class initialization and class node sufficient statistics
		double[][] trjPDist = new double[trjSStats.length][classNode.getStatesNumber()];
		for(int iTrj = 0; iTrj < trjPDist.length; ++iTrj) {
			trjPDist[iTrj] = softSampling(classNode.getStatesNumber());
			
			for(int sE = 0; sE < trjPDist[0].length; ++sE) {
				stats[iClassNode].Px[0][0][sE] += trjPDist[iTrj][sE];
				stats[iClassNode].counts[0] += trjPDist[iTrj][sE];
			}
		}
		
		return calculateSoftSufficientStatistics(model, trjSStats, trjPDist, stats);
	}


	/**
	 * Generate a matrix of sufficient statistics.
	 * Rows represent the trajectory in the training
	 * set trajectories order. Column represent
	 * all the nodes in the model.
	 * 
	 * The generated sufficient statistics are
	 * generated supposing always the assignment
	 * of state zero for the class node.
	 * 
	 * @param model model used to generate the sufficient statistics
	 * @param trainingSet training set used to calculate the counts
	 * @return matrix of sufficient statistics for each trajectory supposing always the state zero value for the class node
	 */
	private SufficientStatistics[][] generateTrajectoriesSufficientStatistics(
			IModel<Double, CTDiscreteNode> model,
			Collection<ITrajectory<Double>> trainingSet) {
		
		NodeIndexing nodeIndexing = model.getNodeIndexing();
		int iClassNode = nodeIndexing.getClassIndex();

		// Initialize the sufficient statistics
		SufficientStatistics[][] stats = initializeTrajectoriesSufficientStatistics(model, trainingSet);
		
		// Sufficient statistics calculation
		Iterator<ITrajectory<Double>> itTrj = trainingSet.iterator();
		for( int iTrj = 0; itTrj.hasNext(); ++iTrj) {								// for each trajectory in the training set
			ITrajectory<Double> trj = itTrj.next();
			
			if( trj.getTransitionsNumber() == 0)
				continue;

			// Initialization at time 0
			double lastTime = trj.getTransitionTime(0);								// set the last changing time
			for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {
				CTDiscreteNode cNode = model.getNode(iNode);
				if( iNode == iClassNode)											// because the class is unknown we suppose the first
					cNode.setEvidence(0); 											// set the evidence
				else
					cNode.setEvidence(trj.getNodeValue(0, iNode)); 		// set the evidence
			}

			// Sufficient statistics for static nodes
			for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {
				CTDiscreteNode cNode = model.getNode(iNode);
				if( cNode.isStaticNode()) {
					int pE = cNode.getCurrentParentsEntry();						// Note: because we need the parent entry we can not join together the two loops
					++stats[iTrj][iNode].Px[pE][0][cNode.getCurrentStateIndex()];
					++stats[iTrj][iNode].counts[pE];
				}
			}

			// Sufficient statistics for the continuous nodes
			// For each jump
			for( int iJmp = 1; iJmp < trj.getTransitionsNumber(); ++iJmp) {
				
				double deltaT = trj.getTransitionTime(iJmp) - lastTime;
				
				// For each node (update the sufficient statistics)
				for(int chNodeIndex = 0; chNodeIndex < nodeIndexing.getNodesNumber(); ++chNodeIndex) {
					if( chNodeIndex == iClassNode)										// ignore the class node
						continue;
					
					// Get the node information
					CTDiscreteNode chNode = model.getNode( chNodeIndex);
					int prevStIndex = chNode.getCurrentStateIndex();
					int nextStIndex = chNode.getStateIndex( trj.getNodeValue( iJmp, chNodeIndex));
					int pE = chNode.getCurrentParentsEntry();
					
					// Update the jump values
					if( !chNode.isStaticNode()) {
						stats[iTrj][chNodeIndex].Tx[pE][prevStIndex] += deltaT;
						if(prevStIndex != nextStIndex) {								// if the node really jumped
							++stats[iTrj][chNodeIndex].Mxx[pE][prevStIndex][nextStIndex];
							++stats[iTrj][chNodeIndex].Mx[pE][prevStIndex];
						} 
					}else if(prevStIndex != nextStIndex) 
						throw new IllegalArgumentException("Error: static nodes (" + chNode.getName() + ") can not change their state during a trajectory (trj = " + trj.getName() + ")");
				}
	
				// For each changed node (set the new state)
				for(int chNodeIndex = 0; chNodeIndex < nodeIndexing.getNodesNumber(); ++chNodeIndex) {
					if( chNodeIndex == iClassNode)										// ignore the class node
						continue;
					
					// Get the node information
					CTDiscreteNode chNode = model.getNode( chNodeIndex);
					int prevStIndex = chNode.getCurrentStateIndex();
					int nextStIndex = chNode.getStateIndex( trj.getNodeValue( iJmp, chNodeIndex));
					if( prevStIndex == nextStIndex)
						continue;
					
					chNode.setEvidence(nextStIndex);
				}

				// Update the last transition time 
				lastTime = trj.getTransitionTime(iJmp);
				
			} // end of the for over the jumps in a trajectory
		} // end for over trajectories

		// Retract all the evidences
		for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode)
			model.getNode(iNode).retractEvidence();

		return stats;
	}


	/**
	 * Initialize the sufficient statistics
	 * for each trajectory and put to zero
	 * the statistics values.
	 * 
	 * @param model model used to generate the sufficient statistics
	 * @param trainingSet training set used to calculate the counts
	 * @return matrix of initialized sufficient statistics with zero values
	 */
	private SufficientStatistics[][] initializeTrajectoriesSufficientStatistics(
			IModel<Double, CTDiscreteNode> model,
			Collection<ITrajectory<Double>> trainingSet) {
		 
		SufficientStatistics[][] stats = new SufficientStatistics[trainingSet.size()][model.getNodeIndexing().getNodesNumber()];
		for(int iNode = 0; iNode < stats[0].length; ++iNode) {
			CTDiscreteNode node = model.getNode(iNode);
			for( int iTrj = 0; iTrj < stats.length; ++iTrj)
				stats[iTrj][iNode] = new SufficientStatistics(node.getNumberParentsEntries(), node.getStatesNumber(), node.isStaticNode(), 0.0, 0.0, 0.0);
		}
		
		return stats;
	}
	
	
	/**
	 * Initialize the clusterable trajectories
	 * of the training set.
	 * 
	 * @param trainingSet the training set from which generate the clusterable trajectories
	 * @return clusterable trajectories initialized
	 */
	private List<IClassificationResult<Double>> initializeClusterizedTrajectories(
			Collection<ITrajectory<Double>> trainingSet) {
		
		List<IClassificationResult<Double>> trajectories = new Vector<IClassificationResult<Double>>(trainingSet.size());
		Iterator<ITrajectory<Double>> itTrj = trainingSet.iterator();
		for( int i = 0; itTrj.hasNext(); ++i) {
			ITrajectory<Double> trj = itTrj.next();
			if( IClassificationResult.class.isAssignableFrom(trj.getClass()))
				trajectories.add(i, (IClassificationResult<Double>) trj);
			else
				trajectories.add(i, new ClassificationResults<Double>(trj));
		}
		
		return trajectories;
	}
	
	
	/**
	 * Calculate the parameters from the sufficient statistics
	 * and set the CIMs in the model.
	 * 
	 * @param model the model to learn
	 * @param sStats the sufficient statistics
	 * @throws RuntimeException if some error occurs
	 */
	private void setCIMs(IModel<Double, CTDiscreteNode> model, SufficientStatistics sStats[]) throws RuntimeException {
		// Parameter calculation
		NodeIndexing nodeIndexing = model.getNodeIndexing();
		for( int iNode = 0; iNode < nodeIndexing.getNodesNumber(); ++iNode) {
			CTDiscreteNode node = model.getNode(iNode);
			if( node.isStaticNode()) {
				for( int pE = 0; pE < node.getNumberParentsEntries(); ++pE) {
					// Probability distribution calculation
					double[][] prob = new double[1][node.getStatesNumber()];
					for( int sE = 0; sE < node.getStatesNumber(); ++sE)
						prob[0][sE] = sStats[iNode].Px[pE][0][sE] / sStats[iNode].counts[pE];
					node.setCIM( pE, prob);
				}
			}else {
				for( int pE = 0; pE < node.getNumberParentsEntries(); ++pE) {
					double[][] cim = new double[node.getStatesNumber()][node.getStatesNumber()];
					for( int fsE = 0; fsE < node.getStatesNumber(); ++fsE) {
						// CIM calculation
						cim[fsE][fsE] = -sStats[iNode].Mx[pE][fsE] /  sStats[iNode].Tx[pE][fsE];
						for( int ssE = 0; ssE < node.getStatesNumber(); ++ssE)
							if( fsE != ssE)
								cim[fsE][ssE] = sStats[iNode].Mxx[pE][fsE][ssE] / sStats[iNode].Tx[pE][fsE];
					}
					node.setCIM(pE, cim);
				}
			}
			
			int check = node.checkCIMs();
			if( check != -1)
				throw new RuntimeException("Error: unexpected error in CIM validation for node " + node.getName() + " and parent entry " + check);
		}
	}
	
	
	/**
	 * Sample a probability distribution
	 * over the states.
	 * 
	 * @param nStates number of states
	 * @return a probability distribution over the states
	 */
	static public double[] softSampling(int nStates) {
		
		double sum = 0.0;
		double[] p = new double[nStates];
		
		for(int i = 0; i < nStates; ++i) {
			p[i] = Math.random();
			sum += p[i];
		}
		for(int i = 0; i < nStates; ++i) {
			p[i] /= sum;
		}
		
		return p;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.ILearningAlgorithm#getStructure()
	 */
	@Override
	public boolean[][] getStructure() {

		return this.adjMStructure;
	}

} public class StandardStopCriterion implements IStopCriterion {

	private int maxIteration;
	private double changedBound;
	
	/**
	 * Constructor
	 * 
	 * @param maxIteration max number of iteration (valid range {2,..,+oo})
	 * @param changedBound bound of percentage of changed clusters under with stop the clustering (valid range [0,1])
	 * @throws IllegalArgumentException in case of illegal argument
	 */
	public StandardStopCriterion(int maxIteration, double changedBound) throws IllegalArgumentException {
		
		if( maxIteration <= 1)
			throw new IllegalArgumentException("Error: max iteration must be bigger then 0");
		if( changedBound < 0.0 || changedBound > 1.0)
			throw new IllegalArgumentException("Error: changed percentage bound must be in [0,1]");
		
		this.maxIteration = maxIteration;
		this.changedBound = changedBound;
	}
	
	
	/**
	 * Return the max iteration set.
	 * 
	 * @return maximum number of iteration
	 */
	public int getMaxIteration(){
		
		return this.maxIteration;
	}
	
	
	/**
	 * Return the percentage bound under with
	 * cut the clustering algorithm.
	 * 
	 * @return percentage bound
	 */
	public double getChangedBound() {
		
		return this.changedBound;
	}
	
		
	/* (non-Javadoc)
	 * @see CTBNToolkit.clustering.IStopCriterion#paramsStop(int, double)
	 */
	@Override
	public boolean paramsStop(int iIteration, double changedCluster)
			throws RuntimeException {

		return (iIteration > this.maxIteration || changedCluster <= this.changedBound);
	}

} public class CLLHillClimbingFactory implements ICTBNCHillClimbingFactory {

	private ILearningAlgorithm<Double, CTDiscreteNode> paramsLearningAlg;
	private Collection<ITrajectory<Double>> dataset;
	private int maxParents;
	private boolean dimensionPenalty;
	private boolean featureSelectionMode;
	private int expApprox;
	 
	/**
	 * Base constructor.
	 * 
	 * @param paramsLearningAlg learning algorithm used to learn the parameters during the structural optimization process
	 * @param maxParents maximum number of parents allowed class included
	 * @param dimensionPenalty true if the scoring of the individual has to give a penalty related to the dimension of the structure, false otherwise
	 * @param featureSelectionMode false to leave the class childhood relations as defined in the model. True to study the class childhood relations in order to select the best attributes	 for the class
	 */
	public CLLHillClimbingFactory(ILearningAlgorithm<Double, CTDiscreteNode> paramsLearningAlg, int maxParents, boolean dimensionPenalty, boolean featureSelectionMode) {
	
		this.paramsLearningAlg = paramsLearningAlg;
		this.maxParents = maxParents;
		this.dimensionPenalty = dimensionPenalty;
		this.featureSelectionMode = featureSelectionMode;
		this.dataset = null;
		this.expApprox = 10;
	}
	
	
	@Override
	public CTBNCHillClimbingIndividual newInstance(ICTClassifier<Double, CTDiscreteNode> model, int nodeIndex) throws IllegalArgumentException {

		return new CTBNCHillClimbingIndividual(this, model, nodeIndex);
	}

	@Override
	public CTBNCHillClimbingIndividual newInstance(ICTClassifier<Double, CTDiscreteNode> model, int nodeIndex, double score) throws IllegalArgumentException {
		
		return new CTBNCHillClimbingIndividual(this, model, nodeIndex, score);
	}
	
	/**
	 * Evaluate the model using the Conditional
	 * Log-Likelihood.
	 * The model must be a CTBNClassifier.
	 * 
	 * @param model the model to evaluate (CTBNClassifier instance)
	 * @param nodeIndex name of the node in evaluation
	 * @return scoring
	 */
	@Override
	public double evaluate(ICTClassifier<Double, CTDiscreteNode> model, int nodeIndex) throws RuntimeException {
		
		return StructuralLearningScoringFormulae.conditionalLogLikelihoodScore(model, nodeIndex, this.dataset, this.paramsLearningAlg, this.dimensionPenalty, this.expApprox);
	}
	
	
	@Override
	public void setDataset(Collection<ITrajectory<Double>> dataset) throws IllegalArgumentException {

		if( dataset == null)
			throw new IllegalArgumentException("Error: null argument. A dataset is not optional");
		
		this.dataset = dataset;		
	}

	@Override
	public Collection<ITrajectory<Double>> getDataset() {
		
		return this.dataset;
	}
	
	@Override
	public int getMaxParents() {
		
		return this.maxParents;
	}
	
	@Override
	public boolean getFeatureSelectionMode() {
		
		return this.featureSelectionMode;
	}
	
	/**
	 * Return the flag used to decide if gives
	 * a penalty to the dimension of the structure
	 * during the learning process.
	 * 
	 * @return  true if the scoring of the individual has to give a penalty related to the dimension of the structure, false otherwise
	 */
	public boolean getDimensionPenalty() {
		
		return this.dimensionPenalty;
	}

	/**
	 * Set the number of iteration to approximate
	 * the exponential in the scoring calculation.
	 * e^x = sum_n(x^n/n!)
	 * [Default value = 10]
	 * 
	 * @param N number of iteration used to approximate the exponential
	 * @throws IllegalArgumentException if N is less then 3
	 */
	public void setExpApproximation(int N) throws IllegalArgumentException {
		
		if( N < 3)
			throw new IllegalArgumentException("Error: the exponential approximation value must be at least 3");
		
		this.expApprox = N;
	}
	
	/**
	 * Get the number of iteration used to approximate
	 * the exponential in the scoring calculation.
	 * e^x = sum_n(x^n/n!)
	 * 
	 * @return number of iteration used to approximate the exponential
	 */
	public int getExpApproximation() {
		
		return this.expApprox;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.optimization.IElementFactory#getParamsLearningAlg()
	 */
	@Override
	public ILearningAlgorithm<Double, CTDiscreteNode> getParamsLearningAlg() {

		return this.paramsLearningAlg;
	}

}
 public class CTBNCHillClimbingIndividual implements ILocalSearchIndividual<String,CTBNCHillClimbingIndividual> {

	private ICTBNCHillClimbingFactory elementsFactory;
	private ICTClassifier<Double, CTDiscreteNode> model;
	private int nodeIndex;
	private Double value;
	private String id;
	
	/**
	 * Base constructor.
	 * 
	 * @param elementsFactory factory used to generate a new element
	 * @param model model used as starting individual
	 * @param nodeIndex index of the node who learn locally the structure
	 * @throws IllegalArgumentException in case of illegal arguments
	 */
	public CTBNCHillClimbingIndividual(ICTBNCHillClimbingFactory elementsFactory, ICTClassifier<Double, CTDiscreteNode> model, int nodeIndex) throws IllegalArgumentException {
		
		if( model.getNode( nodeIndex).isStaticNode())
			throw new IllegalArgumentException("Error: the scoring algorithm is defined only on continuous nodes");
		
		this.elementsFactory = elementsFactory;
		this.model = model;
		this.nodeIndex = nodeIndex;
		this.value = null;
		this.id = null;
	}
	
	/**
	 * Base constructor.
	 * 
	 * @param elementsFactory factory used to generate a new element
	 * @param model model used as starting individual
	 * @param nodeIndex index of the node who learn locally the structure
	 * @param value pre-calculated value of the individual
	 * @throws IllegalArgumentException in case of illegal arguments
	 */
	protected CTBNCHillClimbingIndividual(ICTBNCHillClimbingFactory elementsFactory, ICTClassifier<Double, CTDiscreteNode> model, int nodeIndex, double value) throws IllegalArgumentException {
		
		this(elementsFactory, model, nodeIndex);
		this.value = value;
		this.id = null;
	}
	
	@Override
	public double evaluate() {
 
		if( this.value == null)
			this.value = this.elementsFactory.evaluate(this.model, this.nodeIndex);
		
		return this.value;
	}
	
	@Override
	public CTBNCHillClimbingIndividual getBestNeighbor(ICache<String,CTBNCHillClimbingIndividual> cache) {
		
		CTBNCHillClimbingIndividual newInd;
		// Generate support structures
		NodeIndexing nodeIndexing = this.model.getNodeIndexing();
		// Indexes
		int classNodeIndex = -1;
		if( this.model instanceof CTBNClassifier)
			classNodeIndex = nodeIndexing.getClassIndex();
		// Structure data
		boolean[][] originalAdjMatrix = this.model.getAdjMatrix();
		int nParents = CTBNCHillClimbingIndividual.countParents(originalAdjMatrix, this.nodeIndex);
		// Best model data
		CTBNCHillClimbingIndividual bestInd = null;
		
		// Find the best neighbor
		for(int i = 0; i < originalAdjMatrix.length; ++i) {
			if( i == this.nodeIndex) 											// loops are ignored (because loops are defined implicitly in the CTBN)
				continue;
			if( i == classNodeIndex && (!this.elementsFactory.getFeatureSelectionMode()))
				continue;
			if( !originalAdjMatrix[i][this.nodeIndex] && nParents >= this.elementsFactory.getMaxParents()) 	// if we want to add a parent but the node have enough parents
				continue;
			
			// Generate the new structure to try
			boolean[][] newAdjMatrix = new boolean[originalAdjMatrix.length][originalAdjMatrix.length]; 
			for(int k = 0; k < newAdjMatrix.length; ++k)						// clone vector by vector
				newAdjMatrix[k] = originalAdjMatrix[k].clone();
			newAdjMatrix[i][this.nodeIndex] = !newAdjMatrix[i][this.nodeIndex];
			
			String key = CTBNCHillClimbingIndividual.getKey(newAdjMatrix, this.nodeIndex);
			if( cache != null && cache.contains(key))
				newInd = cache.get( key);
			else {
				ICTClassifier<Double, CTDiscreteNode> clonedModel = (ICTClassifier<Double, CTDiscreteNode>)this.model.clone();
				clonedModel.setStructure(newAdjMatrix);
				newInd = this.elementsFactory.newInstance(clonedModel, this.nodeIndex);
				if( cache != null)
					cache.put(key, newInd);
			}
			// Update the best element
			if( bestInd == null || newInd.evaluate() > bestInd.evaluate() || (newInd.evaluate() == bestInd.evaluate() && Math.random() > 0.5 )) {
				bestInd = newInd;
			}
		}
		
		return bestInd;
	}

	/**
	 * Get the model associated to the individual.
	 * 
	 * @return the model associated to the individual
	 */
	public IModel<Double, CTDiscreteNode> getModel() {
		
		return this.model;
	}
	
	/**
	 * Count the number of parents for a node using
	 * the adjacent matrix.
	 * 
	 * @param adjMatrix adjacent matrix of the structure
	 * @param nodeIndex node for which count the parents
	 * @return number of parents
	 */
	private static int countParents(boolean[][] adjMatrix, int nodeIndex) {
		
		int count = 0;
		
		for(int i = 0; i < adjMatrix.length; ++i)
			if( adjMatrix[i][nodeIndex])
				++count;
		
		return count;
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.optimization.ICacheElement#getId()
	 */
	@Override
	public String getId() {
		
		if( this.id != null)
			return this.id;

		this.id = CTBNCHillClimbingIndividual.getKey( this.model.getAdjMatrix(), this.nodeIndex);
		
		return this.id;
	}
	
	/**
	 * Get the unique key that represent the
	 * parent set for the couple adjacency
	 * matrix-node in input.
	 * 
	 * @param adjMatrix matrix to evaluate
	 * @param nodeIndex index of the node (column) to take into account
	 * @return key that identify the parents of the node of the given index
	 * @throws IllegalArgumentException in case of illegal arguments
	 */
	static public String getKey(boolean[][] adjMatrix, int nodeIndex) throws IllegalArgumentException {

		if( adjMatrix == null)
			throw new IllegalArgumentException("Error: null adjMatrix");
		if( adjMatrix.length == 0)
			throw new IllegalArgumentException("Error: empty matrix");
		if( adjMatrix[0].length <= nodeIndex)
			throw new IllegalArgumentException("Error: nodeIndex out of bound");
		
		StringBuilder strBuilder = new StringBuilder(adjMatrix.length);
		for(int i = 0; i < adjMatrix.length; ++i) {
			if( adjMatrix[i][nodeIndex])
				strBuilder.append("1");
			else
				strBuilder.append("0");
		}
		
		return strBuilder.toString();
		
		/*
		long id = (long) 0;
		long pow = 1;
		
		for(int i = 0; i < adjMatrix.length; ++i) {
			if( adjMatrix[i][nodeIndex])
				id += pow;
			pow *= 2;
		}
		
		return id;
		*/
	}

} public interface ICTBNCHillClimbingFactory extends IElementFactory<CTBNCHillClimbingIndividual> {

	/**
	 * Return the maximum number of parents
	 * allowed.
	 * 
	 * @return maximum number of allowed parents
	 */
	public int getMaxParents();

	/**
	 * Return the flag used to decide the execution
	 * mode.
	 * False to leave the class childhood relations
	 * as defined in the model.
	 * True to study the class childhood relations
	 * in order to select the best attributes
	 * for the class.
	 * 
	 * @return false to leave the class childhood relations as defined in the model. True to study the class childhood relations in order to select the best attributes	 for the class.
	 */
	public boolean getFeatureSelectionMode();
	
	/**
	 * Generate a new instance of an optimization
	 * element.
	 * 
	 * @param model model used as starting individual
	 * @param nodeIndex index of the node who learn locally the structure
	 * @param score pre-calculated score to assign to the new element
	 * @return new instance of an optimization element
	 * @throws IllegalArgumentException in case of illegal arguments
	 */
	public CTBNCHillClimbingIndividual newInstance(ICTClassifier<Double, CTDiscreteNode> model, int nodeIndex, double score);
	
}
 public class LLHillClimbingFactory implements ICTBNCHillClimbingFactory{

	private ILearningAlgorithm<Double, CTDiscreteNode> paramsLearningAlg;
	private Collection<ITrajectory<Double>> dataset;
	private int maxParents;
	private boolean dimensionPenalty;
	private boolean featureSelectionMode;
	 
	/**
	 * Base constructor.
	 * 
	 * @param paramsLearningAlg learning algorithm used to learn the parameters during the structural optimization process
	 * @param maxParents maximum number of parents allowed
	 * @param dimensionPenalty true if the scoring of the individual has to give a penalty related to the dimension of the structure, false otherwise
	 * @param featureSelectionMode false to leave the class childhood relations as defined in the model. True to study the class childhood relations in order to select the best attributes	 for the class
	 */
	public LLHillClimbingFactory(ILearningAlgorithm<Double, CTDiscreteNode> paramsLearningAlg, int maxParents, boolean dimensionPenalty, boolean featureSelectionMode) {
	
		this.paramsLearningAlg = paramsLearningAlg;
		this.maxParents = maxParents;
		this.dimensionPenalty = dimensionPenalty;
		this.featureSelectionMode = featureSelectionMode;
		this.dataset = null;
	}
	
	
	@Override
	public CTBNCHillClimbingIndividual newInstance(ICTClassifier<Double, CTDiscreteNode> model, int nodeIndex) throws IllegalArgumentException {

		return new CTBNCHillClimbingIndividual(this, model, nodeIndex);
	}

	@Override
	public CTBNCHillClimbingIndividual newInstance(ICTClassifier<Double, CTDiscreteNode> model, int nodeIndex, double score) throws IllegalArgumentException {
		
		return new CTBNCHillClimbingIndividual(this, model, nodeIndex, score);
	}
	
	/**
	 * Evaluate the model using the Nodelman
	 * formulae for the current node over the
	 * dataset inserted in the constructor.
	 * 
	 * @param model the model to evaluate
	 * @param nodeIndex index of the node in evaluation
	 * @return scoring
	 */
	@Override
	public double evaluate(ICTClassifier<Double, CTDiscreteNode> model, int nodeIndex) throws RuntimeException {
		
		return StructuralLearningScoringFormulae.logLikelihoodScore(model, nodeIndex, this.dataset, this.paramsLearningAlg, this.dimensionPenalty);
	}
	
	
	@Override
	public void setDataset(Collection<ITrajectory<Double>> dataset) throws IllegalArgumentException {

		if( dataset == null)
			throw new IllegalArgumentException("Error: null argument. A dataset is not optional");
		
		this.dataset = dataset;		
	}

	@Override
	public Collection<ITrajectory<Double>> getDataset() {
		
		return this.dataset;
	}
	
	@Override
	public int getMaxParents() {
		
		return this.maxParents;
	}
	
	@Override
	public boolean getFeatureSelectionMode() {
		
		return this.featureSelectionMode;
	}
	
	/**
	 * Return the flag used to decide if gives
	 * a penalty to the dimension of the structure
	 * during the learning process.
	 * 
	 * @return  true if the scoring of the individual has to give a penalty related to the dimension of the structure, false otherwise
	 */
	public boolean getDimensionPenalty() {
		
		return this.dimensionPenalty;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.optimization.IElementFactory#getParamsLearningAlg()
	 */
	@Override
	public ILearningAlgorithm<Double, CTDiscreteNode> getParamsLearningAlg() {

		return this.paramsLearningAlg;
	}

} public class ClassificationStandardPerformances<TimeType extends Number & Comparable<TimeType>> implements IClassificationSingleRunPerformances<TimeType> {
	
	private Map<String,Integer> valueToIndex;
	private Map<Integer,String> indexToValue;
	private ICTClassifier<TimeType, ?> learnedModel;
	
	private List<IClassificationResult<TimeType>> dataset;
	private int nTrajectories;
	private List<Double> inferenceTimes;
	private double[][] contingencyMatrix;
	
	private double learningTime;
	private double sumInferenceTimes;			// sum of inferences times
	private double avgInferenceTime;			// average inference time
	private double varInferenceTime;			// variance of inference time
	
	private double accuracy;					// accuracy
	private double[] precision;					// precision
	private double[] recall;					// recall
	private double[] fmeasure;					// f-measure
	private List<List<double[]>> precRec;		// Precision-recall curve
	private double[] AUCPrecRec;				// AUC Precision-recall curve
	private List<List<double[]>> ROC;			// ROC curve
	private double[] AUCROC;					// AUCROC
	private List<List<double[]>> cumulativeResp;// cumulative response
	private List<List<double[]>> liftChart;		// lift chart
	private double[] FPRate;					// false positive rate
	private double brier;						// brier value
	
	private double[] classPositiveNumber;		// number of positive in all the dataset (used only in ROC function, cumulative response and lift chart). Necessary to separate the contingency matrix from the ROC, Bier, cumulative responce, lift chart, and Precision-Recall curves calculation
	
	protected Map<String,Double> normalTable;
	
	/**
	 * Constructor.
	 * 
	 * @param classIndexToValue map index to class value
	 */
	public ClassificationStandardPerformances(Map<Integer,String> classIndexToValue) {
		
		this.valueToIndex = new TreeMap<String,Integer>();
		this.indexToValue = classIndexToValue;
		
		for( int i = 0; i < this.indexToValue.size(); ++i) {
			String val = this.indexToValue.get(i);
			if( val == null)
				throw new IllegalArgumentException("Error: key " + i + "doesn't included in the map index to value");
			
			this.valueToIndex.put(val, i);
		}
		
		this.initializeStructures();
	}	
	
	/**
	 * Constructor.
	 * 
	 * @param classIndexToValue map index to class value
	 * @param classValueToIndex map class value to index
	 */
	public ClassificationStandardPerformances(Map<Integer,String> classIndexToValue, Map<String,Integer> classValueToIndex) {
		
		if(classIndexToValue.size() != classValueToIndex.size())
			throw new IllegalArgumentException("Error: maps index to value and value to index have different dimension");
		for(int i = 0; i < classIndexToValue.size(); ++i)
			if( classIndexToValue.get(i) == null || classValueToIndex.get(classIndexToValue.get(i)) != i)
				throw new IllegalArgumentException("Error: maps index to value and value to index don't correpsond");

		this.valueToIndex = classValueToIndex;
		this.indexToValue = classIndexToValue;
		
		this.initializeStructures();
	}
	
	
	/**
	 * Initialize the structures
	 */
	protected void initializeStructures() {

		this.learnedModel = null;
		this.dataset = new LinkedList<IClassificationResult<TimeType>>();
		this.nTrajectories = 0;
		this.inferenceTimes = new LinkedList<Double>();
		
		this.classPositiveNumber = new double[this.indexToValue.size()];
		this.contingencyMatrix = new double[this.indexToValue.size()][this.indexToValue.size()];
		for( int i = 0; i < this.contingencyMatrix.length; ++i) {
			this.classPositiveNumber[i] = 0.0;
			for( int j = 0; j < this.contingencyMatrix[0].length; ++j)
				this.contingencyMatrix[i][j] = 0.0;
		}
		
		this.learningTime = -1.0;
		this.sumInferenceTimes = 0.0;
		
		this.resetPerformances();
		
		// Value of normal table for confidence interval
		// i.e.: 95% = 1.65 in normal table => this.normalTable.put("%90", 1.65); 
		this.normalTable = new TreeMap<String,Double>();
		this.normalTable.put("99.9%", 3.291);
		this.normalTable.put("99.8%", 3.09);
		this.normalTable.put("99%", 2.576);
		this.normalTable.put("98%", 2.326);
		this.normalTable.put("95%", 1.960);
		this.normalTable.put("90%", 1.645);
		this.normalTable.put("80%", 1.28);
	}


	/**
	 * Reset all the performances.
	 */
	protected void resetPerformances() {

		if( this.dataset == null)
			throw new RuntimeException("Error: it is impossible reset performances after the final calculation call");
		
		this.precision = null;
		this.recall = null;
		this.fmeasure = null;
		this.precRec = null;
		this.AUCPrecRec = null;
		this.ROC = null;
		this.AUCROC = null;
		this.cumulativeResp = null;
		this.liftChart = null;
		this.FPRate = null;
		this.brier = -1;
		
		this.avgInferenceTime = -1.0;
		this.varInferenceTime = -1.0;
	}
	

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformances#setLearnedModel(CTBNToolkit.ICTClassifier)
	 */
	@Override
	public void setLearnedModel(
			ICTClassifier<TimeType, ?> learnedModel) {

		this.learnedModel = learnedModel;
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformances#getLearnedModel()
	 */
	@Override
	public ICTClassifier<TimeType, ?> getLearnedModel() {
		
		return this.learnedModel;
	}
	

	/**
	 * Set directly the contingency matrix.
	 * 
	 * @param matrix matrix to set
	 * @throws IllegalArgumentException in case of errors in the argument
	 */
	protected void setContingencyMatrix(double[][] matrix) throws IllegalArgumentException {
		
		if(matrix == null)
			throw new IllegalArgumentException("Error: illegal argument");
		if(matrix.length != this.indexToValue.size() || matrix[0].length != this.indexToValue.size())
			throw new IllegalArgumentException("Error: illegal matrix dimension");
		for(int i = 0; i < matrix.length; ++i)
			for(int j = 0; j < matrix[0].length; ++j)
				if( matrix[i][j] < 0)
					throw new IllegalArgumentException("Error: contingency matrix can not have values smaller then 0");
		
		this.contingencyMatrix = matrix;
		this.resetPerformances();
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformances#setLearningTime(double)
	 */
	@Override
	public void setLearningTime(double time) throws IllegalArgumentException {

		if( time < 0)
			throw new IllegalArgumentException("Error: time can not be negative");
		
		this.learningTime = time;		
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformances#getLearningTime()
	 */
	@Override
	public double getLearningTime() throws RuntimeException {

		if( this.learningTime < 0)
			this.learningTime = Double.NaN;
		
		return this.learningTime;
	}
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformances#getInferenceTimes()
	 */
	@Override
	public List<Double> getInferenceTimes() {

		return this.inferenceTimes;
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformances#getAvgInferenceTime()
	 */
	@Override
	public double getAvgInferenceTime() {
		
		if( this.avgInferenceTime < 0) {
			if( this.inferenceTimes == null || this.inferenceTimes.size() == 0)
				this.avgInferenceTime = Double.NaN;
			else
				this.avgInferenceTime = this.sumInferenceTimes / (double)this.inferenceTimes.size();
		}
		
		return this.avgInferenceTime;
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformances#getVarianceInferenceTime()
	 */
	@Override
	public double getVarianceInferenceTime() {
		
		if( this.varInferenceTime < 0) {
			if( this.inferenceTimes == null || this.inferenceTimes.size() == 0) {
				
				this.varInferenceTime = Double.NaN;
			} else {
				this.getAvgInferenceTime();
		
				this.varInferenceTime = 0.0;
				for(int i = 0; i < this.inferenceTimes.size(); ++i)
					this.varInferenceTime += Math.pow(this.inferenceTimes.get(i) - this.avgInferenceTime, 2);
			
				this.varInferenceTime /= (double)this.inferenceTimes.size();
			}
		}
		
		return this.varInferenceTime;
	}
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.ISingleRunPerformances#getResultsTrajectories()
	 */
	@Override
	public List<IClassificationResult<TimeType>> getResultsTrajectories() {
		
		if( this.dataset == null)
			throw new RuntimeException("Error: impossible gets the trajectories after the final calculation call");
		
		return this.dataset;
	}
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformances#sortResultsTrajectoriesByNames()
	 */
	@Override
	public List<IClassificationResult<TimeType>> sortResultsTrajectoriesByNames() {
		
		if( this.dataset == null)
			throw new RuntimeException("Error: impossible sorts trajectories after the final calculation call");
		
		// Sort the dataset
		Collections.sort( this.dataset, new Comparator<IClassificationResult<TimeType>>() {
												public int compare(IClassificationResult<TimeType> trj1, IClassificationResult<TimeType> trj2) {
													return trj1.getName().compareTo( trj2.getName());
												}
											});
		
		return this.dataset;
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformances#datasetDimension()
	 */
	@Override
	public int datasetDimension() {

		return this.nTrajectories;
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformances#classesNumber()
	 */
	@Override
	public int classesNumber() {

		return this.indexToValue.size();
	}
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#indexToValue(int)
	 */
	@Override
	public String indexToValue(int index) {
		
		return this.indexToValue.get(index);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#valueToIndex(java.lang.Object)
	 */
	@Override
	public Integer valueToIndex(String value) {

		return this.valueToIndex.get( value);
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#addResults(java.util.Collection)
	 */
	@Override
	public void addResults(Collection<IClassificationResult<TimeType>> trjResults) {
		
		Iterator<IClassificationResult<TimeType>> iterTrj = trjResults.iterator();
		while( iterTrj.hasNext())
			this.addResult( iterTrj.next());
	}
	
	

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformances#addResults(java.util.List, java.util.List)
	 */
	@Override
	public void addResults(List<IClassificationResult<TimeType>> trjResults, List<Double> inferenceTimes) throws IllegalArgumentException {

		if( inferenceTimes == null) {
			this.addResults(trjResults);
			return;
		}
		
		if( trjResults.size() != inferenceTimes.size())
			throw new IllegalArgumentException("Error: the list have not the same length");
		
		for( int i = 0; i < trjResults.size(); ++i) {
			if( inferenceTimes.get(i) == null)
				throw new IllegalArgumentException("Error: null inference time");
			
			this.addResult( trjResults.get(i), inferenceTimes.get(i));
		}
		
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformances#addResult(CTBNToolkit.IClassificationResult, double)
	 */
	@Override
	public void addResult(IClassificationResult<TimeType> trj,
			double inferenceTime) throws IllegalArgumentException {
		
		if( inferenceTime < 0)
			throw new IllegalArgumentException("Error: negative inference time");

		String trueClass = trj.getNodeValue(0, trj.getNodeIndexing().getClassIndex());
		String classification = trj.getClassification();
		
		++this.contingencyMatrix[this.valueToIndex.get(trueClass)][this.valueToIndex.get(classification)];
		++this.classPositiveNumber[this.valueToIndex.get(trueClass)];
		this.dataset.add( trj);
		++this.nTrajectories;
		
		if( this.inferenceTimes != null) {
			this.sumInferenceTimes += inferenceTime;
			this.inferenceTimes.add( inferenceTime);
		}
		
		this.resetPerformances();
		
	}
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#addResult(CTBNToolkit.IClassificationResult)
	 */
	@Override
	public void addResult(IClassificationResult<TimeType> trj) {
		
		String trueClass = trj.getNodeValue(0, trj.getNodeIndexing().getClassIndex());
		String classification = trj.getClassification();
		
		++this.contingencyMatrix[this.valueToIndex.get(trueClass)][this.valueToIndex.get(classification)];
		++this.classPositiveNumber[this.valueToIndex.get(trueClass)];
		this.dataset.add( trj);	
		++this.nTrajectories;
		
		this.inferenceTimes = null;
		
		this.resetPerformances();
	}
	
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.ISingleRunPerformances#getContingencyMatrix()
	 */
	@Override
	public double[][] getContingencyMatrix() {

		return this.contingencyMatrix;
	}
	
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getAccuracy()
	 */
	@Override
	public double getAccuracy() {

		if( this.datasetDimension() == 0)
			return Double.NaN;
		
		if( this.precision == null)
			calculateContingencyPerformances();
		
		return this.accuracy;
	}
	

	/**
	 * Return the accuracy and in confidence
	 * interval in the following form:
	 * lower bound <= accuracy <= upper bound
	 * 
	 * @param confidenceLevel level of confidence (range of values {"99.9%", "99.8%", "99%", "98%", "95%", "90%", "80%"}
	 * @return accuracy confidence interval (lower bound, accuracy, upper bound)
	 * @throws IllegalArgumentException in case of illegal arguments
	 */
	@Override
	public double[] getAccuracy(String confidenceLevel) throws IllegalArgumentException {
		
		return  getAccuracy( confidenceLevel, this.datasetDimension());
	}
	
	/**
	 * Return the accuracy and in confidence
	 * interval in the following form:
	 * lower bound <= accuracy <= upper bound
	 * 
	 * @param confidenceLevel level of confidence (range of values {"99.9%", "99.8%", "99%", "98%", "95%", "90%", "80%"}
	 * @param N size of the sample
	 * @return accuracy confidence interval (lower bound, accuracy, upper bound)
	 * @throws IllegalArgumentException in case of illegal arguments
	 */
	protected double[] getAccuracy(String confidenceLevel, int N) throws IllegalArgumentException {
		
		if( confidenceLevel == null)
			throw new IllegalArgumentException("Error: null confidence level");
		
		if( this.datasetDimension() == 0) {
			double[] v = new double[3];
			v[0] = v[1] = v[2] = Double.NaN;
			return v;
		}
		
		if( this.precision == null)
			calculateContingencyPerformances();
		
		Double z = this.normalTable.get(confidenceLevel);
		if( z == null)
			throw new IllegalArgumentException("Error: confidence level didn't recognized");
		
		return ClassificationStandardPerformances.binomialInterval(this.accuracy, N, z);
	}

	
	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getError()
	 */
	@Override
	public double getError() {
		
		if( this.datasetDimension() == 0)
			return Double.NaN;
		
		if( this.precision == null)
			calculateContingencyPerformances();
		
		return 1.0 - this.accuracy;
	}

	
	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getPrecision(int)
	 */
	@Override
	public double getPrecision(String classValue) throws IllegalArgumentException {
		
		if( classValue == null)
			throw new IllegalArgumentException("Error: null class value");
		if( this.valueToIndex.get(classValue) == null)
			throw new IllegalArgumentException("Error: state = " + classValue + " didn't found");
		
		if( this.datasetDimension() == 0)
			return Double.NaN;
		
		if( this.precision == null)
			calculateContingencyPerformances();
		
		return this.precision[this.valueToIndex.get(classValue)];
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getRecall(int)
	 */
	@Override
	public double getRecall(String classValue) throws IllegalArgumentException {
		
		if( classValue == null)
			throw new IllegalArgumentException("Error: null class value");
		if( this.valueToIndex.get(classValue) == null)
			throw new IllegalArgumentException("Error: state = " + classValue + " didn't found");

		if( this.datasetDimension() == 0)
			return Double.NaN;
		
		if( this.recall == null)
			calculateContingencyPerformances();
		
		return this.recall[this.valueToIndex.get(classValue)];
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getFMeasure(int)
	 */
	@Override
	public double getFMeasure(String classValue) throws IllegalArgumentException {
		
		if( classValue == null)
			throw new IllegalArgumentException("Error: null class value");
		if( this.valueToIndex.get(classValue) == null)
			throw new IllegalArgumentException("Error: state = " + classValue + " didn't found");	
		
		if( this.datasetDimension() == 0)
			return Double.NaN;
		
		if( this.fmeasure == null)
			calculateContingencyPerformances();
		
		return this.fmeasure[this.valueToIndex.get(classValue)];
	}
	

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getPrecisionRecallAUC(java.lang.Object)
	 */
	@Override
	public double getPrecisionRecallAUC(String classValue)
			throws IllegalArgumentException {

		if( classValue == null)
			throw new IllegalArgumentException("Error: null class value");
		if( this.valueToIndex.get(classValue) == null)
			throw new IllegalArgumentException("Error: state = " + classValue + " didn't found");
		
		if( this.precRec == null)
			this.calculatePrecisionRecallCurves();
		
		return this.AUCPrecRec[this.valueToIndex.get(classValue)];
	}
	

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getPrecisionRecallCurve(java.lang.Object)
	 */
	@Override
	public List<double[]> getPrecisionRecallCurve(String classValue)
			throws IllegalArgumentException {

		if( classValue == null)
			throw new IllegalArgumentException("Error: null class value");
		if( this.valueToIndex.get(classValue) == null)
			throw new IllegalArgumentException("Error: state = " + classValue + " didn't found");
		
		if( this.precRec == null)
			this.calculatePrecisionRecallCurves();
		
		return this.precRec.get(this.valueToIndex.get(classValue));
	}
	

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getSensitivity(java.lang.Object)
	 */
	@Override
	public double getSensitivity(String classValue) throws IllegalArgumentException {
		
		if( classValue == null)
			throw new IllegalArgumentException("Error: null class value");
		if( this.valueToIndex.get(classValue) == null)
			throw new IllegalArgumentException("Error: state = " + classValue + " didn't found");
		
		if( this.datasetDimension() == 0)
			return Double.NaN;
		
		if( this.recall == null)
			calculateContingencyPerformances();
		
		return this.recall[this.valueToIndex.get(classValue)];
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getSpecificity(java.lang.Object)
	 */
	@Override
	public double getSpecificity(String classValue) throws IllegalArgumentException {
		
		if( classValue == null)
			throw new IllegalArgumentException("Error: null class value");
		if( this.valueToIndex.get(classValue) == null)
			throw new IllegalArgumentException("Error: state = " + classValue + " didn't found");
		
		if( this.datasetDimension() == 0)
			return Double.NaN;
		
		if( this.FPRate == null)
			calculateContingencyPerformances();
		
		return 1.0 - this.FPRate[this.valueToIndex.get(classValue)];
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getTruePositiveRate(java.lang.Object)
	 */
	@Override
	public double getTruePositiveRate(String classValue) throws IllegalArgumentException {
		
		if( classValue == null)
			throw new IllegalArgumentException("Error: null class value");
		if( this.valueToIndex.get(classValue) == null)
			throw new IllegalArgumentException("Error: state = " + classValue + " didn't found");
		
		if( this.datasetDimension() == 0)
			return Double.NaN;
		
		if( this.recall == null)
			calculateContingencyPerformances();
		
		return this.recall[this.valueToIndex.get(classValue)];
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getFalsePositiveRate(java.lang.Object)
	 */
	@Override
	public double getFalsePositiveRate(String classValue) throws IllegalArgumentException {
		
		if( classValue == null)
			throw new IllegalArgumentException("Error: null class value");
		if( this.valueToIndex.get(classValue) == null)
			throw new IllegalArgumentException("Error: state = " + classValue + " didn't found");
		
		if( this.datasetDimension() == 0)
			return Double.NaN;
		
		if( this.FPRate == null)
			calculateContingencyPerformances();
		
		return this.FPRate[this.valueToIndex.get(classValue)];
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getBrier(int)
	 */
	@Override
	public double getBrier() throws IllegalArgumentException {
		
		if( this.datasetDimension() == 0 || this.dataset.get(0).getProbability(this.dataset.get(0).getClassification()) == null)
			this.brier = Double.NaN;
		
		if( this.brier == -1)
			this.calculateBrier();
		
		return this.brier;
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getAUC(int)
	 */
	@Override
	public double getROCAUC(String classValue) throws IllegalArgumentException {
		
		if( classValue == null)
			throw new IllegalArgumentException("Error: null class value");
		if( this.valueToIndex.get(classValue) == null)
			throw new IllegalArgumentException("Error: state = " + classValue + " didn't found");
		
		if( this.ROC == null)
			this.calculateROCAndAUC();
		
		return this.AUCROC[this.valueToIndex.get(classValue)];
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.validation.IClassificationPerformance#getROC(int)
	 */
	@Override
	public List<double[]> getROC(String classValue) throws IllegalArgumentException {
		
		if( classValue == null)
			throw new IllegalArgumentException("Error: null class value");
		if( this.valueToIndex.get(classValue) == null)
			throw new IllegalArgumentException("Error: state = " + classValue + " didn't found");
		
		if( this.ROC == null)
			this.calculateROCAndAUC();
		
		return this.ROC.get(this.valueToIndex.get(classValue));
	}
	

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getCumulativeResponse(java.lang.String)
	 */
	@Override
	public List<double[]> getCumulativeResponse(String classValue)
			throws IllegalArgumentException {
		
		if( classValue == null)
			throw new IllegalArgumentException("Error: null class value");
		if( this.valueToIndex.get(classValue) == null)
			throw new IllegalArgumentException("Error: state = " + classValue + " didn't found");
		
		if( this.cumulativeResp == null)
			this.calculateCumulativeRespAndLiftChart();
		
		return this.cumulativeResp.get(this.valueToIndex.get(classValue));
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getLiftChart(java.lang.String)
	 */
	@Override
	public List<double[]> getLiftChart(String classValue)
			throws IllegalArgumentException {
		
		if( classValue == null)
			throw new IllegalArgumentException("Error: null class value");
		if( this.valueToIndex.get(classValue) == null)
			throw new IllegalArgumentException("Error: state = " + classValue + " didn't found");
		
		if( this.liftChart == null)
			this.calculateCumulativeRespAndLiftChart();
		
		return this.liftChart.get(this.valueToIndex.get(classValue));
	}

	
	/**
	 * Calculate all the performance from the 
	 * contingency matrix.
	 */
	protected void calculateContingencyPerformances() {
		
		if( this.dataset == null)
			throw new RuntimeException("Error: impossible sorts trajectories after the final calculation call");
		
		calculateAccuracy();		// accuracy calculation
		calculatePrecision(); 		// precision calculation
		calculateRecall();			// recall calculation
		calculateFMeasure();		// f-measure calculation
		calculateFPRate();			// false positive rate calculation
	}

	/**
	 * Calculate false positive rate
	 */
	protected void calculateFPRate() {
		
		this.FPRate = new double[this.indexToValue.size()];
		for( int i = 0; i < this.contingencyMatrix.length; ++i) {
			this.calculateFPRate(i);
		}
	}
	
	/**
	 * Calculate the false positive rate
	 * for the class in input.
	 * 
	 * @param iClass index of the class for which calculate the false positive rate
	 */
	protected void calculateFPRate(int iClass) {
		
		double n = 0;
		double fp = 0;
		for(int i = 0; i < this.contingencyMatrix.length; ++i)
			for( int j = 0; iClass != i && j < this.contingencyMatrix[0].length; ++j) {
				n += this.contingencyMatrix[i][j];
			}
		for( int i = 0; i < this.contingencyMatrix.length; ++i) {
				if( i != iClass)
					fp += this.contingencyMatrix[i][iClass];
			}
		
		this.FPRate[iClass] = fp/n;		// FP / N
		
	}

	/**
	 * Calculate f-measure
	 */
	protected void calculateFMeasure() {

		this.fmeasure = new double[this.indexToValue.size()];
		for( int i = 0; i < this.indexToValue.size(); ++i)
			this.fmeasure[i] = 2.0 / (1.0/this.precision[i] + 1.0/this.recall[i]);
	}
	
	/**
	 * Calculate recall
	 */
	protected void calculateRecall() {

		this.recall = new double[this.indexToValue.size()];
		for( int i = 0; i < this.contingencyMatrix.length; ++i) {
			double tot = 0;			
			for( int j = 0; j < this.contingencyMatrix[0].length; ++j)
				tot += this.contingencyMatrix[i][j];
			
			this.recall[i] = this.contingencyMatrix[i][i] / tot;
		}
	}

	/**
	 *  Calculate precision
	 */
	protected void calculatePrecision() {

		this.precision = new double[this.indexToValue.size()];
		for( int j = 0; j < this.contingencyMatrix[0].length; ++j) {
			double tot = 0;			
			for( int i = 0; i < this.contingencyMatrix.length; ++i)
				tot += this.contingencyMatrix[i][j];
			
			this.precision[j] = this.contingencyMatrix[j][j] / tot;
		}
	}
	
	/**
	 * Calculate accuracy
	 */
	protected void calculateAccuracy() {
		
		this.accuracy = 0;
		
		// Accuracy calculation
		double tot = 0.0;
		for( int i = 0; i < this.contingencyMatrix.length; ++i) {
			this.accuracy += this.contingencyMatrix[i][i];
			for( int j = 0; j < this.contingencyMatrix.length; ++j)
				tot += this.contingencyMatrix[i][j];
			}

		this.accuracy /= tot;
	}
	

	/**
	 * Calculate brier score for each class.
	 */
	protected void calculateBrier() {

		if( this.dataset == null)
			throw new RuntimeException("Error: impossible calculate Brier after the final calculation call");
		
		this.brier = 0;
		for( int i = 0; i < this.datasetDimension(); ++i) {
			IClassificationResult<TimeType> trj = this.dataset.get(i);
			
			String trueState = trj.getNodeValue(0, trj.getNodeIndexing().getClassIndex());
			String predictedState = trj.getClassification();
			double p = trj.getProbability(predictedState);
			
			double o;
			if( trueState.equals(predictedState))
				o = 1.0;
			else
				o = 0.0;
			
			
			this.brier += Math.pow(p - o, 2);
			
		}
		
		this.brier /= this.datasetDimension();
	}

	/**
	 * Calculate ROC curve, AUC and Brier score.
	 */
	protected void calculateROCAndAUC() {

		this.ROC = new Vector<List<double[]>>(this.indexToValue.size());
		for(int i = 0; i < this.indexToValue.size(); ++i)
			this.ROC.add(null);
		
		this.AUCROC = new double[this.indexToValue.size()];
		for( int i = 0; i < this.indexToValue.size(); ++i)
			calculateROCAndAUC(i);
		
	}
	
	/**
	 * Calculate ROC and AUC for the
	 * class in input.
	 * 
	 * @param index of the class
	 */
	protected void calculateROCAndAUC(int index) {

		if( this.dataset == null)
			throw new RuntimeException("Error: impossible calculate ROC and AUC after the final calculation call");
		
		if( this.datasetDimension() == 0) {
			this.AUCROC[index] = 0.0;
			return;
		}
		
		// Sort the dataset
		final String state = this.indexToValue.get(index);
		Collections.sort( this.dataset, new Comparator<IClassificationResult<TimeType>>() {
												public int compare(IClassificationResult<TimeType> trj1, IClassificationResult<TimeType> trj2) {
													Double p1 = trj1.getProbability(state);
													Double p2 = trj2.getProbability(state);
													if( Double.isNaN(p1))
														throw new IllegalArgumentException("Error: NaN probability value in trajectory " + trj1.getName());
													if( Double.isNaN(p2))
														throw new IllegalArgumentException("Error: NaN probability value in trajectory " + trj2.getName());
													
													return (int)Math.signum(p2 - p1);
												}
											});
		
		
		// Initialization
		double P = this.classPositiveNumber[index];
		double N = this.datasetDimension() - P;
		
		// Calculate ROC and AUC
		double FP = 0, FPPrev = 0;
		double TP = 0, TPPrev = 0;
		double pPrev = Double.NEGATIVE_INFINITY;
		this.AUCROC[index] = 0.0;
		List<double[]> roc = new LinkedList<double[]>();
		for( int i = 0; i < this.datasetDimension(); ++i) {
			IClassificationResult<TimeType> trj = this.dataset.get(i);
			
			double pCurr = trj.getProbability(state);
			if( pCurr != pPrev) {
				double[] data = new double[2];
				data[0] = FP/N; data[1] = TP/P;
				roc.add(data);
				
				this.AUCROC[index] += trapezoidArea(FP,FPPrev,TP,TPPrev);
				
				pPrev = pCurr;
				FPPrev = FP;
				TPPrev = TP;
			}
			
			if( trj.getNodeValue(0, trj.getNodeIndexing().getClassIndex()).equals(state))
				++TP;
			else
				++FP;
		}
		
		// Last ROC element
		double[] data = new double[2];
		data[0] = FP/N; data[1] = TP/P;
		roc.add(data);
		this.ROC.set(index, roc);
		
		// Last AUC trapezoid area
		this.AUCROC[index] += trapezoidArea(N,FPPrev,P,TPPrev);
		this.AUCROC[index] /= (P*N);			// normalization
		
	}

	
	/**
	 * Calculate the precision-recall
	 * curves for each class.
	 */
	protected void calculatePrecisionRecallCurves() {

		this.precRec = new Vector<List<double[]>>(this.indexToValue.size());
		for(int i = 0; i < this.indexToValue.size(); ++i)
			this.precRec.add(null);
		
		this.AUCPrecRec = new double[this.indexToValue.size()];
		for( int i = 0; i < this.indexToValue.size(); ++i)
			calculatePrecisionRecallCurve(i);
		
	}
	
	/**
	 * Calculate precision recall curve
	 * for a specific class.
	 * 
	 * @param index of the class which calculate precision-recall curve
	 */
	protected void calculatePrecisionRecallCurve(int index) {
		
		if( this.dataset == null)
			throw new RuntimeException("Error: impossible calculate precision-recall curves after the final calculation call");
		
		// Sort the dataset
		final String state = this.indexToValue.get(index);
		Collections.sort( this.dataset, new Comparator<IClassificationResult<TimeType>>() {
												public int compare(IClassificationResult<TimeType> trj1, IClassificationResult<TimeType> trj2) {
													Double p1 = trj1.getProbability(state);
													Double p2 = trj2.getProbability(state);
													if( Double.isNaN(p1))
														throw new IllegalArgumentException("Error: NaN probability value in trajectory " + trj1.getName());
													if( Double.isNaN(p2))
														throw new IllegalArgumentException("Error: NaN probability value in trajectory " + trj2.getName());
													
													return (int)Math.signum(p2 - p1);
												}
											});
		
		
		// Initialization
		double P = this.classPositiveNumber[index];
		
		// Calculate Precision-recall curve and AUC
		double p1, p2 = -1;
		double FP = 0.0, TP = 0.0;
		double prec = 1.0, precPrev = 1.0;
		double rec = 0.0, recPrev = 0.0;
		this.AUCPrecRec[index] = 0.0;
		List<double[]> pRCurve = new LinkedList<double[]>();
		IClassificationResult<TimeType> trj1 = null, trj2 = null;
		
		// Initialization
		double[] data = new double[2];
		data[0] = rec;	data[1] = prec;
		pRCurve.add(data);
		trj1 = this.dataset.get(0);
		for( int i = 1; i <= this.datasetDimension(); ++i) {
			
			p1 = trj1.getProbability(state);
			// Get trajectory
			if( i < this.datasetDimension()) {
				trj2 = this.dataset.get(i);
				p2 = trj2.getProbability(state);
			} else
				trj2 = null;
			
			// Update counts
			if( trj1.getNodeValue(0, trj1.getNodeIndexing().getClassIndex()).equals(state))
				++TP;
			else
				++FP;
			if( TP+FP == 0)				// precision
				prec = 1.0;
			else
				prec = TP/(TP+FP);
			rec = TP/P;					// recall
			
			if(  trj2 == null || p1 != p2) {
				data = new double[2];
				data[0] = rec;	data[1] = prec;
				pRCurve.add(data);
				this.AUCPrecRec[index] += trapezoidArea(rec,recPrev,prec,precPrev);
				precPrev = prec;
				recPrev = rec;
			}
			trj1 = trj2;
		}
		this.precRec.set(index, pRCurve);
	}

	
	
	/**
	 * Calculate cumulative response and lift chart curves.
	 */
	protected void calculateCumulativeRespAndLiftChart() {

		this.cumulativeResp = new Vector<List<double[]>>(this.indexToValue.size());
		for(int i = 0; i < this.indexToValue.size(); ++i)
			this.cumulativeResp.add(null);
		this.liftChart = new Vector<List<double[]>>(this.indexToValue.size());
		for(int i = 0; i < this.indexToValue.size(); ++i)
			this.liftChart.add(null);
		
		for( int i = 0; i < this.indexToValue.size(); ++i)
			calculateCumulativeRespAndLiftChart(i);
		
	}
	 
	/**
	 * Calculate cumulative response and lift chart curves
	 * for the class in input.
	 * 
	 * @param index of the class
	 */
	protected void calculateCumulativeRespAndLiftChart(int index) {

		if( this.dataset == null)
			throw new RuntimeException("Error: impossible calculate cumulative response and lift chart after the final calculation call");
		
		if( this.datasetDimension() == 0) {
			return;
		}
		
		// Sort the dataset
		final String state = this.indexToValue.get(index);
		Collections.sort( this.dataset, new Comparator<IClassificationResult<TimeType>>() {
												public int compare(IClassificationResult<TimeType> trj1, IClassificationResult<TimeType> trj2) {
													Double p1 = trj1.getProbability(state);
													Double p2 = trj2.getProbability(state);
													if( Double.isNaN(p1))
														throw new IllegalArgumentException("Error: NaN probability value in trajectory " + trj1.getName());
													if( Double.isNaN(p2))
														throw new IllegalArgumentException("Error: NaN probability value in trajectory " + trj2.getName());
													
													return (int)Math.signum(p2 - p1);
												}
											});
		
		
		// Initialization
		double P = this.classPositiveNumber[index];
		double positiveRatio = P/this.datasetDimension();
		
		// Calculate ROC and AUC
		double selectedP = 0;
		double pPrev = Double.NEGATIVE_INFINITY;
		List<double[]> cumulativeR = new LinkedList<double[]>();
		List<double[]> liftChart = new LinkedList<double[]>();
		for( int i = 0; i < this.datasetDimension(); ++i) {
			IClassificationResult<TimeType> trj = this.dataset.get(i);
			
			double pCurr = trj.getProbability(state);
			if( pCurr != pPrev) {
				double[] cumulativeData = new double[2];
				cumulativeData[0] = ((double)i)/this.datasetDimension(); cumulativeData[1] = selectedP/P;
				cumulativeR.add(cumulativeData);
				
				double[] liftData = new double[2];
				liftData[0] = ((double)i)/this.datasetDimension(); liftData[1] = (selectedP/i)/positiveRatio;
				liftChart.add(liftData);
				
				pPrev = pCurr;
			}
			
			if( trj.getNodeValue(0, trj.getNodeIndexing().getClassIndex()).equals(state))
				++selectedP;
		}
		
		// Last cumulative response element
		double[] cumulativeData = new double[2];
		cumulativeData[0] = 1.0; cumulativeData[1] = selectedP/P;
		cumulativeR.add(cumulativeData);
		this.cumulativeResp.set(index, cumulativeR);
		// Last Lift chart element
		double[] liftData = new double[2];
		liftData[0] = 1.0; liftData[1] = (selectedP/this.datasetDimension())/positiveRatio;
		liftChart.add(liftData);
		this.liftChart.set(index, liftChart);
		
	}
	
	
	/**
	 * Trapezoid area calculation used
	 * in AUC calculation.
	 * 
	 * @param X1 fist high of the trapezoid
	 * @param X2 second high of the trapezoid
	 * @param Y1 first base of the trapezoid
	 * @param Y2 second base of the trapezoid
	 * @return area of the trapezoid
	 */
	public static double trapezoidArea(double X1, double X2, double Y1, double Y2) {

		return ((Y1+Y2)/2) * Math.abs(X1-X2);
	}


	/**
	 * Calculate the values interval assuming
	 * a binomial distribution.
	 * 
	 * @param val value around which calculate the interval
	 * @param N number of samples
	 * @param z z value used for the confidence interval (i.e. Pr [ −z ≤ X ≤ z ] = c, Pr [ −1.65 ≤ X ≤ 1.65] = 90%)
	 * @return confidence interval in the form (val lower bound, val, val upper bound)
	 */
	public static double[] binomialInterval(double val, int N, double z) {
		
		double[] interval = new double[3];
		double z2 = Math.pow(z, 2);
		double f = val;
		double f2 = Math.pow(val, 2);
		double N2 = Math.pow(N, 2);
		double sqrtValue = z * Math.sqrt(f/N - f2/N + z2/(4*N2));
		
		interval[0] = (f + z2/(2*N) - sqrtValue) / (1 + z2/N);
		interval[1] = f;
		interval[2] = (f + z2/(2*N) + sqrtValue) / (1 + z2/N);
		
		return interval;
	}
	

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformances#calculateFinalResults(boolean)
	 */
	@Override
	public void calculateFinalResults(boolean deleteTrajectories) {

		// Calculates all the performances that requires
		// the trajectories
		this.calculateROCAndAUC();
		this.calculateBrier();
		this.calculatePrecisionRecallCurves();
		this.calculateCumulativeRespAndLiftChart();
		
		if( deleteTrajectories)
			this.dataset = null;
	}

	
} public class ClassificationStandardPerformancesFactory<TimeType extends Number & Comparable<TimeType>> implements
		ISingleRunPerformancesFactory<TimeType, ClassificationStandardPerformances<TimeType>> {

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformancesFactory#newInstance(java.util.Map)
	 */
	@Override
	public ClassificationStandardPerformances<TimeType> newInstance(
			Map<Integer, String> classIndexToValue) {

		return new ClassificationStandardPerformances<TimeType>(classIndexToValue);
	}

} public class ClusteringExternalPerformances<TimeType extends Number & Comparable<TimeType>> implements IExternalClusteringSingleRunPerformances<TimeType> {

	private IExternalClusteringPerformances inSamplePerformances;
	
	private Map<Integer,String> indexToTrueState;
	private Map<String,Integer> trueStateToIndex;
	private Map<Integer,String> indexToCluster;
	private Map<String,Integer> clusterToIndex;
	
	private ICTClassifier<TimeType, ?> learnedModel;
	
	private List<IClassificationResult<TimeType>> clusterizedTrajectories;
	private int nTrajectories;
	private List<Double> inferenceTimes;
	
	private double learningTime;
	private double sumInferenceTimes;			// sum of inferences times
	private double avgInferenceTime;			// average inference time
	private double varInferenceTime;			// variance of inference time
	
	private double[][] clustersPartitionsMatrix;
	private double[][] precisionM;
	private double[][] recallM;
	private double[][] fMeasureM;
	private double[][] associationMatrix;
	
	
	/**
	 * Constructor.
	 * 
	 * @param indexToTrueState index to true state association
	 * @param indexToCluster index to clustering association
	 */
	public ClusteringExternalPerformances(Map<Integer,String> indexToTrueState, Map<Integer,String> indexToCluster) {
		
		if( indexToTrueState == null)
			throw new IllegalArgumentException("Error: null indexes to true states map");
		if( indexToTrueState.size() < 1)
			throw new IllegalArgumentException("Error: indexes to true states map must contains at least a state");
		if( indexToCluster == null)
			throw new IllegalArgumentException("Error: null indexes to clusters map");
		if( indexToCluster.size() < 1)
			throw new IllegalArgumentException("Error: indexes to clusters map must contains at least a cluster");
		
		// True states maps
		this.trueStateToIndex = new TreeMap<String,Integer>();
		this.indexToTrueState = indexToTrueState;
		
		for( int i = 0; i < this.indexToTrueState.size(); ++i) {
			String val = this.indexToTrueState.get(i);
			if( val == null)
				throw new IllegalArgumentException("Error: key " + i + "doesn't included in the map index to value");
			
			this.trueStateToIndex.put(val, i);
		}
		
		// Clustering maps
		this.clusterToIndex = new TreeMap<String,Integer>();
		this.indexToCluster = indexToCluster;
		
		for( int i = 0; i < this.indexToCluster.size(); ++i) {
			String val = this.indexToCluster.get(i);
			if( val == null)
				throw new IllegalArgumentException("Error: key " + i + "doesn't included in the map index to value");
			
			this.clusterToIndex.put(val, i);
		}
		
		this.initializeStructures();
	}
	
	
	/**
	 * Initialize the structures
	 */
	protected void initializeStructures() {

		this.learnedModel = null;
		this.clusterizedTrajectories = new LinkedList<IClassificationResult<TimeType>>();
		this.nTrajectories = 0;
		
		this.inferenceTimes = new LinkedList<Double>();		
		this.learningTime = -1.0;
		this.sumInferenceTimes = 0.0;
		
		this.resetPerformances();
	}


	/**
	 * Reset all the performances.
	 */
	protected void resetPerformances() {

		if( this.clusterizedTrajectories == null)
			throw new RuntimeException("Error: it is impossible reset performances after the final calculation call");
		
		this.clustersPartitionsMatrix = null;
		this.precisionM = null;
		this.recallM = null;
		this.fMeasureM = null;
		this.associationMatrix = null;
		
		this.avgInferenceTime = -1.0;
		this.varInferenceTime = -1.0;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#datasetDimension()
	 */
	@Override
	public int datasetDimension() {
		
		return this.nTrajectories;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#classesNumber()
	 */
	@Override
	public int classesNumber() {
		
		return this.trueStateToIndex.size();
	}
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IExternalClusteringPerformances#clusterNumber()
	 */
	@Override
	public int clusterNumber() {

		return this.clusterToIndex.size();
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#indexToValue(int)
	 */
	@Override
	public String indexToValue(int index) {

		return this.indexToTrueState.get(index);
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#valueToIndex(java.lang.String)
	 */
	@Override
	public Integer valueToIndex(String value) {

		return this.trueStateToIndex.get( value);
	}
	
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IExternalClusteringPerformances#indexToCluster(int)
	 */
	@Override
	public String indexToCluster(int index) {

		return this.indexToCluster.get(index);
	}
	
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IExternalClusteringPerformances#clusterToIndex(java.lang.String)
	 */
	@Override
	public Integer clusterToIndex(String value) {

		return this.clusterToIndex.get( value);
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#setLearningTime(double)
	 */
	@Override
	public void setLearningTime(double time) throws IllegalArgumentException {

		if( time < 0)
			throw new IllegalArgumentException("Error: time can not be negative");
		
		this.learningTime = time;			
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#getLearningTime()
	 */
	@Override
	public double getLearningTime() {

		if( this.learningTime < 0)
			this.learningTime = Double.NaN;
		
		return this.learningTime;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#getAvgInferenceTime()
	 */
	@Override
	public double getAvgInferenceTime() {
		
		if( this.avgInferenceTime < 0) {
			if( this.inferenceTimes == null || this.inferenceTimes.size() == 0)
				this.avgInferenceTime = Double.NaN;
			else
				this.avgInferenceTime = this.sumInferenceTimes / (double)this.inferenceTimes.size();
		}
		
		return this.avgInferenceTime;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#getVarianceInferenceTime()
	 */
	@Override
	public double getVarianceInferenceTime() {
		
		if( this.varInferenceTime < 0) {
			if( this.inferenceTimes == null || this.inferenceTimes.size() == 0) {
				
				this.varInferenceTime = Double.NaN;
			} else {
				this.getAvgInferenceTime();
		
				this.varInferenceTime = 0.0;
				for(int i = 0; i < this.inferenceTimes.size(); ++i)
					this.varInferenceTime += Math.pow(this.inferenceTimes.get(i) - this.avgInferenceTime, 2);
			
				this.varInferenceTime /= (double)this.inferenceTimes.size();
			}
		}
		
		return this.varInferenceTime;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#getInferenceTimes()
	 */
	@Override
	public List<Double> getInferenceTimes() {
		
		return this.inferenceTimes;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#setLearnedModel(CTBNToolkit.ICTClassifier)
	 */
	@Override
	public void setLearnedModel(
			ICTClassifier<TimeType, ?> learnedModel) {

		this.learnedModel = learnedModel;
	}

	
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#getLearnedModel()
	 */
	@Override
	public ICTClassifier<TimeType, ?> getLearnedModel() {
		
		return this.learnedModel;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#addResults(java.util.Collection)
	 */
	@Override
	public void addResults(
			Collection<IClassificationResult<TimeType>> trjResults) {

		Iterator<IClassificationResult<TimeType>> iterTrj = trjResults.iterator();
		while( iterTrj.hasNext())
			this.addResult( iterTrj.next());
		
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#addResults(java.util.List, java.util.List)
	 */
	@Override
	public void addResults(List<IClassificationResult<TimeType>> trjResults,
			List<Double> inferenceTimes) throws IllegalArgumentException {
		
		if( inferenceTimes == null) {
			this.addResults(trjResults);
			return;
		}
		
		if( trjResults.size() != inferenceTimes.size())
			throw new IllegalArgumentException("Error: the list have not the same length");
		
		for( int i = 0; i < trjResults.size(); ++i) {
			if( inferenceTimes.get(i) == null)
				throw new IllegalArgumentException("Error: null inference time");
			
			this.addResult( trjResults.get(i), inferenceTimes.get(i));
		}
		
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#addResult(CTBNToolkit.IClassificationResult)
	 */
	@Override
	public void addResult(IClassificationResult<TimeType> trj) {
		
		if( this.clusterizedTrajectories == null)
			throw new RuntimeException("Error: impossible add results after the final calculation call");
		
		this.clusterizedTrajectories.add( trj);
		++this.nTrajectories;
		
		this.inferenceTimes = null;		
		this.resetPerformances();
		
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#addResult(CTBNToolkit.IClassificationResult, double)
	 */
	@Override
	public void addResult(IClassificationResult<TimeType> trj,
			double inferenceTime) throws IllegalArgumentException {
		
		if( inferenceTime < 0)
			throw new IllegalArgumentException("Error: negative inference time");
		if( this.clusterizedTrajectories == null)
			throw new RuntimeException("Error: impossible add results after the final calculation call");
		
		this.clusterizedTrajectories.add( trj);
		++this.nTrajectories;
		
		if( this.inferenceTimes != null) {
			this.sumInferenceTimes += inferenceTime;
			this.inferenceTimes.add( inferenceTime);
		}
		
		this.resetPerformances();
		
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IPerformances#getResultsTrajectories()
	 */
	@Override
	public List<IClassificationResult<TimeType>> getResultsTrajectories() {
		
		if( this.clusterizedTrajectories == null)
			throw new RuntimeException("Error: impossible gets the trajectories after the final calculation call");
		
		return this.clusterizedTrajectories;
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformances#sortResultsTrajectoriesByNames()
	 */
	@Override
	public List<IClassificationResult<TimeType>> sortResultsTrajectoriesByNames() {
		
		if( this.clusterizedTrajectories == null)
			throw new RuntimeException("Error: impossible sorts trajectories after the final calculation call");
		
		// Sort the dataset
		Collections.sort( this.clusterizedTrajectories, new Comparator<IClassificationResult<TimeType>>() {
			public int compare(IClassificationResult<TimeType> trj1, IClassificationResult<TimeType> trj2) {
				return trj1.getName().compareTo( trj2.getName());
			}
		});

		return this.clusterizedTrajectories;
	}
	
	
	/**
	 * Calculate the matrix of the relations
	 * between clusters and real partitions.
	 * It contains the count for each cluster
	 * and each true class in the original
	 * partitioning.
	 * 
	 * @return matrix of counts (cluster, true class)
	 * @throws RuntimeException if some error occurs (i.e. the class node is not find in the dataset)
	 */
	protected double[][] calculateClustersPartitionsMatrix() throws RuntimeException {
		
		if( this.clustersPartitionsMatrix != null)
			return this.clustersPartitionsMatrix;
		if( this.clusterizedTrajectories == null)
			throw new RuntimeException("Error: impossible calculates the cluster partition matrix after the final calculation call");
		
		// Get the global node indexing
		NodeIndexing nodeIndexing = this.clusterizedTrajectories.get(0).getNodeIndexing();
		
		// Table initialization
		this.clustersPartitionsMatrix = new double[clusterToIndex.size()][trueStateToIndex.size()];
		for(int i = 0; i < this.clustersPartitionsMatrix.length; ++i)
			for(int j = 0; j < this.clustersPartitionsMatrix[0].length; ++j)
				this.clustersPartitionsMatrix[i][j] = 0;
		
		// Table calculation
		for(int iTrj = 0; iTrj < this.clusterizedTrajectories.size(); ++iTrj) {
			IClassificationResult<TimeType> trj = this.clusterizedTrajectories.get( iTrj);
			++this.clustersPartitionsMatrix[this.clusterToIndex.get(trj.getClassification())][this.trueStateToIndex.get(trj.getNodeValue(0, nodeIndexing.getClassIndex()))];
		}
		
		return this.clustersPartitionsMatrix;			
	}	
	
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IExternalClusteringPerformances#getClustersPartitionsMatrix()
	 */
	@Override
	public double[][] getClustersPartitionsMatrix() {
		
		if( this.clustersPartitionsMatrix != null)
			return this.clustersPartitionsMatrix;
		
		return this.calculateClustersPartitionsMatrix();
	}
	
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IExternalClusteringPerformances#getPrecisionMatrix()
	 */
	@Override
	public double[][] getPrecisionMatrix() {
		
		if( this.precisionM != null)
			return this.precisionM;
		
		this.getClustersPartitionsMatrix();		
		this.precisionM = new double[this.clusterNumber()][this.classesNumber()];
		
		for( int iCl = 0; iCl < this.precisionM.length; ++iCl) {
			double den = 0;
			for( int iTS2 = 0; iTS2 < this.precisionM[0].length; ++iTS2)
				den += this.clustersPartitionsMatrix[iCl][iTS2];
			
			for( int iTS = 0; iTS < this.precisionM[0].length; ++iTS)
				this.precisionM[iCl][iTS] = this.clustersPartitionsMatrix[iCl][iTS] / den;
		}
		
		return this.precisionM;
	}
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IExternalClusteringPerformances#getRecallMatrix()
	 */
	@Override
	public double[][] getRecallMatrix() {
		
		if( this.recallM != null)
			return this.recallM;
		
		this.getClustersPartitionsMatrix();
		this.recallM = new double[this.clusterNumber()][this.classesNumber()]; 
		
		for( int iTS = 0; iTS < this.recallM[0].length; ++iTS) {
			double den = 0;
			for( int iCl2 = 0; iCl2 < this.recallM.length; ++iCl2)
				den += this.clustersPartitionsMatrix[iCl2][iTS];
			
			for( int iCl = 0; iCl < this.recallM.length; ++iCl)
				this.recallM[iCl][iTS] = this.clustersPartitionsMatrix[iCl][iTS] / den;
		}
		
		return this.recallM;
	}
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IExternalClusteringPerformances#getFMeasureMatrix()
	 */
	@Override
	public double[][] getFMeasureMatrix() {
		
		if( this.fMeasureM != null)
			return this.fMeasureM;
		
		this.getPrecisionMatrix();
		this.getRecallMatrix();
		
		this.fMeasureM = new double[this.clusterNumber()][this.classesNumber()];
		for( int iCl = 0; iCl < this.fMeasureM.length; ++iCl)
			for( int iTS = 0; iTS < this.fMeasureM[0].length; ++iTS) {
				
				double den = this.precisionM[iCl][iTS] + this.recallM[iCl][iTS];
				if( den == 0)
					this.fMeasureM[iCl][iTS] = 0.0;
				else
					this.fMeasureM[iCl][iTS] = 2 * this.precisionM[iCl][iTS] * this.recallM[iCl][iTS] / den;
			}
		
		return this.fMeasureM;
	}
	
	
	/**
	 * Calculate the 2x2 association matrix:
	 * SS SD
	 * DS DD
	 * where:
	 * - SS: is the number of data pairs that
	 *    are in the same cluster and in the
	 *    same original partition (true state)
	 * - SD: is the number of data pairs that
	 *    are in the same cluster but in
	 *    different partitions (true state)
	 * - DS: is the number of data pairs that
	 *    are in different clusters but in
	 *    the same partition (true state)
	 * - DD: is the number of data pairs that
	 *    are in different clusters and in
	 *    different partitions
	 * 
	 * @return return 2x2 association matrix
	 */
	protected double[][] calculateAssociationMatrix() {
		
		if( this.associationMatrix != null)
			return this.associationMatrix;
		if( this.clusterizedTrajectories == null)
			throw new RuntimeException("Error: impossible calculates the association matrix after the final calculation call");
		
		NodeIndexing nodeIndexing = this.clusterizedTrajectories.get(0).getNodeIndexing();
		
		// Association matrix initialization
		this.associationMatrix = new double[2][2];
		this.associationMatrix[0][0] = 0; this.associationMatrix[0][1] = 0; this.associationMatrix[1][0] = 0; this.associationMatrix[1][1] = 0;
		
		// Matrix calculation
		for(int iTrj1 = 0; iTrj1 < this.clusterizedTrajectories.size(); ++iTrj1) {
			IClassificationResult<TimeType> trj1 = this.clusterizedTrajectories.get( iTrj1);
			String cluster1 = trj1.getClassification();
			String tState1 = trj1.getNodeValue(0, nodeIndexing.getClassIndex());
			
			for(int iTrj2 = iTrj1 + 1; iTrj2 < this.clusterizedTrajectories.size(); ++iTrj2) {
				IClassificationResult<TimeType> trj2 = this.clusterizedTrajectories.get( iTrj2);
				boolean clEqual = cluster1.equals( trj2.getClassification());
				boolean tStateEqual = tState1.equals( trj2.getNodeValue(0, nodeIndexing.getClassIndex()));
				
				if( clEqual) {
					if( tStateEqual)
						++this.associationMatrix[0][0];	// SS
					else
						++this.associationMatrix[0][1];	// SD
				}else {
					if( tStateEqual)
						++this.associationMatrix[1][0];	// DS
					else
						++this.associationMatrix[1][1];	// DD
				}
			}
		}
		
		return this.associationMatrix;
	}	
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IExternalClusteringPerformances#getAssociationMatrix()
	 */
	@Override
	public double[][] getAssociationMatrix() {
		
		if( this.associationMatrix != null)
			return this.associationMatrix;
		
		return this.calculateAssociationMatrix();
	}
		
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IExternalClusteringPerformances#getRandStatistic()
	 */
	@Override
	public double getRandStatistic() {
		
		double m;
		double[][] mx = this.getAssociationMatrix();
		
		m = mx[0][0] + mx[0][1] + mx[1][0] + mx[1][1];
		
		return (mx[0][0] + mx[1][1]) / m;
	}
		
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IExternalClusteringPerformances#getJaccardCoefficient()
	 */
	@Override
	public double getJaccardCoefficient() {
		
		double dn;
		double[][] mx = this.getAssociationMatrix();
		
		dn = mx[0][0] + mx[0][1] + mx[1][0];
		
		return mx[0][0] / dn;
	}
		
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IExternalClusteringPerformances#getFolkesMallowsIndex()
	 */
	@Override
	public double getFolkesMallowsIndex() {
		
		double m1,m2;
		double[][] mx = this.getAssociationMatrix();
		
		m1 = mx[0][0] + mx[0][1];
		m2 = mx[0][0] + mx[1][0];
		
		return mx[0][0] / Math.sqrt(m1*m2);
	}	

	
	/**
	 * Set directly the cluster partition matrix.
	 * 
	 * @param matrix matrix to set
	 * @throws IllegalArgumentException in case of errors in the argument
	 */
	protected void setClustersPartitionsMatrix(double[][] matrix) throws IllegalArgumentException {
		
		if(matrix == null)
			throw new IllegalArgumentException("Error: illegal argument");
		if(matrix.length != this.clusterNumber() || matrix[0].length != this.classesNumber())
			throw new IllegalArgumentException("Error: illegal matrix dimension (right cardinality = |Cluster|x|TrueStates|)");
		for(int i = 0; i < matrix.length; ++i)
			for(int j = 0; j < matrix[0].length; ++j)
				if( matrix[i][j] < 0)
					throw new IllegalArgumentException("Error: contingency matrix can not have values smaller then 0");
	
		this.resetPerformances();
		this.clustersPartitionsMatrix = matrix;
	}
	
	/**
	 * Set directly the association matrix.
	 * 
	 * @param matrix matrix to set
	 * @throws IllegalArgumentException in case of errors in the argument
	 */
	protected void setAssociationMatrix(double[][] matrix) throws IllegalArgumentException {
		
		if(matrix == null)
			throw new IllegalArgumentException("Error: illegal argument");
		if(matrix.length != 2 || matrix[0].length != 2)
			throw new IllegalArgumentException("Error: illegal matrix dimension (it must be a 2x2 matrix)");
		for(int i = 0; i < matrix.length; ++i)
			for(int j = 0; j < matrix[0].length; ++j)
				if( matrix[i][j] < 0)
					throw new IllegalArgumentException("Error: contingency matrix can not have values smaller then 0");
		
		this.resetPerformances();
		this.associationMatrix = matrix;
	}
	

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IExternalClusteringPerformances#setInSamplePerformances(CTBNToolkit.performances.IExternalClusteringAggregatePerformances)
	 */
	@Override
	public void setInSamplePerformances(
			IExternalClusteringPerformances inSamplePerformances)
			throws IllegalArgumentException {
		
		this.inSamplePerformances = inSamplePerformances;
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IExternalClusteringPerformances#getInSamplePerformances()
	 */
	@Override
	public IExternalClusteringPerformances getInSamplePerformances() {

		return this.inSamplePerformances;
	}
	
		
	/**
	 * Print the performances.
	 * 
	 * @param performances performances to print
	 * @return performances in string
	 */
	@Override
	public String toString() {
		
		return ClusteringExternalPerformances.toString(this);
	}
	
	
	/**
	 * Print the performances in input.
	 * 
	 * @param performances performances to print
	 * @return performances in string
	 */
	static public String toString(IExternalClusteringPerformances performances) {
		
		double[][] mx;
		int fileColumns = performances.clusterNumber() + 1;
		StringBuilder strBuilder = new StringBuilder();
		
		strBuilder.append("\nGENERAL INFORMATION;" + printSemicolons(fileColumns - 1) + "\n");
		strBuilder.append("#clusters;" + performances.clusterNumber() + printSemicolons(fileColumns - 1) + "\n");
		strBuilder.append("#classes;" + performances.classesNumber() + printSemicolons(fileColumns - 1) + "\n");
		
		strBuilder.append("\nTIME;" + printSemicolons(fileColumns - 1) + "\n");
		strBuilder.append("Average inference time;" + performances.getAvgInferenceTime() + printSemicolons(fileColumns - 1) + "\n");
		strBuilder.append("Variance inference time;" + performances.getVarianceInferenceTime() + printSemicolons(fileColumns - 1) + "\n");
		strBuilder.append("Average learning time;" + performances.getLearningTime() + printSemicolons(fileColumns - 1) + "\n");
		if( IAggregatePerformances.class.isAssignableFrom(performances.getClass()))
			strBuilder.append("Variance leargning time;" + ((IAggregatePerformances<?,?>) performances).getVarianceLearningTime() + printSemicolons(fileColumns - 1) + "\n");
		
		strBuilder.append("\nSUMMARY STATISTICS;" + printSemicolons(fileColumns - 1) + "\n");
		strBuilder.append("Rand statistic;" + performances.getRandStatistic() + printSemicolons(fileColumns - 1) + "\n");
		strBuilder.append("Jaccard coefficient;" + performances.getJaccardCoefficient() + printSemicolons(fileColumns - 1) + "\n");
		strBuilder.append("Folkes & Mallows index;" + performances.getFolkesMallowsIndex() + printSemicolons(fileColumns - 1) + "\n");
		
		strBuilder.append("\nASSOCIATION MATRIX;" + printSemicolons(fileColumns - 1) + "\n");
		strBuilder.append(";SS;SD;" + printSemicolons(fileColumns - 3) + "\n");
		strBuilder.append("DS;" + printVector( performances.getAssociationMatrix()[0]) + printSemicolons(fileColumns - 3) + "\n");
		strBuilder.append("DD;" + printVector( performances.getAssociationMatrix()[1]) + printSemicolons(fileColumns - 3) + "\n");
		
		strBuilder.append("\nCLUSTERS PARTITION MATRIX;" + printSemicolons(fileColumns - 1) + "\n");
		mx =  performances.getClustersPartitionsMatrix();
		strBuilder.append(";" + printClusters(performances) + printSemicolons(fileColumns - (performances.clusterNumber() + 1)) + "\n");
		for(int i = 0; i < performances.classesNumber(); ++i) {
			strBuilder.append(performances.indexToValue(i) + ";" + printVector( mx[i]) + printSemicolons(fileColumns - (performances.clusterNumber() + 1)) + "\n");
		}
		
		strBuilder.append("\nPRECISION MATRIX;" + printSemicolons(fileColumns - 1) + "\n");
		mx =  performances.getPrecisionMatrix();
		strBuilder.append(";" + printClusters(performances) + printSemicolons(fileColumns - (performances.clusterNumber() + 1)) + "\n");
		for(int i = 0; i < performances.classesNumber(); ++i) {
			strBuilder.append(performances.indexToValue(i) + ";" + printVector( mx[i]) + printSemicolons(fileColumns - (performances.clusterNumber() + 1)) + "\n");
		}
		
		strBuilder.append("\nRECALL MATRIX;" + printSemicolons(fileColumns - 1) + "\n");
		mx =  performances.getRecallMatrix();
		strBuilder.append(";" + printClusters(performances) + printSemicolons(fileColumns - (performances.clusterNumber() + 1)) + "\n");
		for(int i = 0; i < performances.classesNumber(); ++i) {
			strBuilder.append(performances.indexToValue(i) + ";" + printVector( mx[i]) + printSemicolons(fileColumns - (performances.clusterNumber() + 1)) + "\n");
		}

		strBuilder.append("\nF-MEASURE MATRIX;" + printSemicolons(fileColumns - 1) + "\n");
		mx =  performances.getFMeasureMatrix();
		strBuilder.append(";" + printClusters(performances) + printSemicolons(fileColumns - (performances.clusterNumber() + 1)) + "\n");
		for(int i = 0; i < performances.classesNumber(); ++i) {
			strBuilder.append(performances.indexToValue(i) + ";" + printVector( mx[i]) + printSemicolons(fileColumns - (performances.clusterNumber() + 1)) + "\n");
		}
		
		return strBuilder.toString();
	}
	
	
	/**
	 * Print all the clusters in the
	 * order separating them by ';'.
	 * 
	 * @param performances performances to print
	 * @return clusters
	 */
	static private String printClusters(IExternalClusteringPerformances performances) {

		StringBuilder strBuilder = new StringBuilder();
		for(int i = 0; i < performances.clusterNumber(); ++i)
			strBuilder.append(performances.indexToCluster(i) + ";");
			
		return strBuilder.toString();
	}

	/**
	 * Print the vector calling for each object
	 * the toString method and adding a ';' after
	 * the call.
	 * 
	 * @param vect vector to print
	 * @return string printed
	 */
	static private String printVector(double[] vect) {
		
		StringBuilder strBuilder = new StringBuilder();
		
		for( int i = 0; i < vect.length; ++i)
			strBuilder.append(vect[i] + ";");
			
		return strBuilder.toString();
	}
	
	/**
	 * Print n semicolons.	 * 
	 * 
	 * @param n number of semicolons to print.
	 * @return sequence of n semicolons
	 */
	static private String printSemicolons(int n) {
		
		StringBuilder strBuilder = new StringBuilder();
		for(int i = 0; i < n; ++i)
			strBuilder.append(";");
			
		return strBuilder.toString();
	}


	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformances#calculateFinalResults(boolean)
	 */
	@Override
	public void calculateFinalResults(boolean deleteTrajectories) {

		// Calculate the structures used to calculate performances
		this.calculateClustersPartitionsMatrix();
		this.calculateAssociationMatrix();
		
		// Remove the results trajectories
		if( deleteTrajectories)
			this.clusterizedTrajectories = null;
	}
} public class ClusteringExternalPerformancesFactory<TimeType extends Number & Comparable<TimeType>> implements
		ISingleRunPerformancesFactory<TimeType, ClusteringExternalPerformances<TimeType>> {

	private Map<Integer,String> indexToCluster;
	
	/**
	 * Empty constructor.
	 * 
	 * In the new instance generation use
	 * the map indexes to states also as map
	 * indexes to clusters.
	 * The number of clusters MUST BE the same
	 * as the number of states!!!
	 */
	public ClusteringExternalPerformancesFactory() {
		
		this(null);
	}
	
	/**
	 * Constructor.
	 * 
	 * @param indexToCluster index to clustering association
	 * @throws IllegalArgumentException in case of illegal argument
	 */
	public ClusteringExternalPerformancesFactory(Map<Integer,String> indexToCluster) throws IllegalArgumentException {
	
		if( indexToCluster != null && indexToCluster.size() < 1)
			throw new IllegalArgumentException("Error: indexes to clusters map must contains at least a cluster");
		
		this.indexToCluster = indexToCluster;
	}
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.ISingleRunPerformancesFactory#newInstance(java.util.Map)
	 */
	@Override
	public ClusteringExternalPerformances<TimeType> newInstance(
			Map<Integer, String> classIndexToValue) {

		if( this.indexToCluster == null)
			return new ClusteringExternalPerformances<TimeType>(classIndexToValue, classIndexToValue);
		else
			return new ClusteringExternalPerformances<TimeType>(classIndexToValue, this.indexToCluster);
	}

} public interface IPerformances {

	/**
	 * Return the dimension of the used
	 * dataset.
	 * 
	 * @return dimension of the dataset
	 */
	public int datasetDimension();
	
	/**
	 * Return the number of classes.
	 * 
	 * @return number of classes
	 */
	public int classesNumber();
	
	/**
	 * Return the value type from the index.
	 * This index is the same used in the
	 * contingency matrix.
	 * Null if the index is not found.
	 * 
	 * @param index index which the value is given back
	 * @return value related to the index in input
	 */
	public String indexToValue(int index);
	
	/**
	 * Return the index from the value type.
	 * This index is the same used in the
	 * contingency matrix.
	 * Null if the value is not found.
	 * 
	 * @param value value which the index is given back
	 * @return index related to the value in input
	 */
	public Integer valueToIndex(String value);
	
	/**
	 * Return the model learning
	 * time.
	 * Return NaN if times are not
	 * set.
	 * 
	 * @return model leraning time
	 */
	public double getLearningTime();
	
	/**
	 * Return the average inference
	 * time.
	 * Return NaN if times are not
	 * set.
	 * 
	 * @return average inference time
	 */
	public double getAvgInferenceTime();
	
	/**
	 * Return the variance of the
	 * inference time.
	 * Return NaN if times are not
	 * set.
	 * 
	 * @return variance of the inference time
	 */
	public double getVarianceInferenceTime();
	
} public class MicroMacroClassificationPerformances<TimeType extends Number & Comparable<TimeType>,PType extends IClassificationSingleRunPerformances<TimeType>> implements
	IClassificationAggregatePerformances<TimeType,PType> {
	
	private MicroAvgAggregatePerformances<TimeType,PType> microAvg;
	private MacroAvgAggregatePerformances<TimeType,PType> macroAvg;
	
	boolean microFlag;
	
	/**
	 * Constructor.
	 * MicroFlag is set to true. All methods
	 * call will be turned on micro averaging
	 * performances.
	 * 
	 * @param classIndexToValue map index to class value
	 */
	public MicroMacroClassificationPerformances(Map<Integer,String> classIndexToValue) {
		
		Map<String,Integer> valueToIndex = new TreeMap<String,Integer>();
		
		for( int i = 0; i < classIndexToValue.size(); ++i) {
			String val = classIndexToValue.get(i);
			if( val == null)
				throw new IllegalArgumentException("Error: key " + i + "doesn't included in the map index to value");
			
			valueToIndex.put(val, i);
		}
		
		this.microAvg = new MicroAvgAggregatePerformances<TimeType,PType>(classIndexToValue, valueToIndex);
		this.macroAvg = new MacroAvgAggregatePerformances<TimeType,PType>(classIndexToValue, valueToIndex);
		this.microFlag = true;
	}
			
	/**
	 * Constructor.
	 * MicroFlag is set to true. All methods
	 * call will be turned on micro averaging
	 * performances.
	 * 
	 * @param classIndexToValue map index to class value
	 * @param classValueToIndex map class value to index
	 */
	public MicroMacroClassificationPerformances(Map<Integer, String> classIndexToValue, 
				Map<String, Integer> classValueToIndex) {
		
		this.microAvg = new MicroAvgAggregatePerformances<TimeType,PType>(classIndexToValue, classValueToIndex);
		this.macroAvg = new MacroAvgAggregatePerformances<TimeType,PType>(classIndexToValue, classValueToIndex);
		this.microFlag = true;
	}
	
	
	/**
	 * Return micro averaging performances.
	 * 
	 * @return micro averaging performances.
	 */
	public 	MicroAvgAggregatePerformances<TimeType,PType> getMicroAveraging() {
		
		return this.microAvg;
	}
	
	/**
	 * Return macro averaging performances.
	 * 
	 * @return macro averaging performances
	 */
	public MacroAvgAggregatePerformances<TimeType,PType> getMacroAveraging() {
		
		return this.macroAvg;
	}
	
	/**
	 * True to map all the methods on the
	 * micro averaging performances. False
	 * to map the methods on macro averaging
	 * performances.
	 * 
	 * @param microAvgFlag flag used to set the two modalities.
	 */
	public void setMicroFlag(boolean microAvgFlag) {
		
		this.microFlag = microAvgFlag;
	}

	
	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#datasetDimension()
	 */
	@Override
	public int datasetDimension() {

		if( this.microFlag)
			return this.microAvg.datasetDimension();
		else
			return this.macroAvg.datasetDimension();
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#classesNumber()
	 */
	@Override
	public int classesNumber() {
		
		if( this.microFlag)
			return this.microAvg.classesNumber();
		else
			return this.macroAvg.classesNumber();
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#indexToValue(int)
	 */
	@Override
	public String indexToValue(int index) {
		
		if( this.microFlag)
			return this.microAvg.indexToValue(index);
		else
			return this.macroAvg.indexToValue(index);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#valueToIndex(java.lang.Object)
	 */
	@Override
	public Integer valueToIndex(String value) {
		
		if( this.microFlag)
			return this.microAvg.valueToIndex(value);
		else
			return this.macroAvg.valueToIndex(value);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getLearningTime()
	 */
	@Override
	public double getLearningTime() {
		
		if( this.microFlag)
			return this.microAvg.getLearningTime();
		else
			return this.macroAvg.getLearningTime();
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getAvgInferenceTime()
	 */
	@Override
	public double getAvgInferenceTime() {
		
		if( this.microFlag)
			return this.microAvg.getAvgInferenceTime();
		else
			return this.macroAvg.getAvgInferenceTime();
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getVarianceInferenceTime()
	 */
	@Override
	public double getVarianceInferenceTime() {
		
		if( this.microFlag)
			return this.microAvg.getVarianceInferenceTime();
		else
			return this.macroAvg.getVarianceInferenceTime();
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getAccuracy()
	 */
	@Override
	public double getAccuracy() {
		
		if( this.microFlag)
			return this.microAvg.getAccuracy();
		else
			return this.macroAvg.getAccuracy();
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getAccuracy(java.lang.String)
	 */
	@Override
	public double[] getAccuracy(String confidenceLevel)
			throws IllegalArgumentException {
		
		if( this.microFlag)
			return this.microAvg.getAccuracy(confidenceLevel);
		else
			return this.macroAvg.getAccuracy(confidenceLevel);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getError()
	 */
	@Override
	public double getError() {
		
		if( this.microFlag)
			return this.microAvg.getError();
		else
			return this.macroAvg.getError();
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getPrecision(java.lang.Object)
	 */
	@Override
	public double getPrecision(String classValue)
			throws IllegalArgumentException {
		
		if( this.microFlag)
			return this.microAvg.getPrecision(classValue);
		else
			return this.macroAvg.getPrecision(classValue);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getRecall(java.lang.Object)
	 */
	@Override
	public double getRecall(String classValue)
			throws IllegalArgumentException {
		
		if( this.microFlag)
			return this.microAvg.getRecall(classValue);
		else
			return this.macroAvg.getRecall(classValue);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getFMeasure(java.lang.Object)
	 */
	@Override
	public double getFMeasure(String classValue)
			throws IllegalArgumentException {
		

		if( this.microFlag)
			return this.microAvg.getFMeasure(classValue);
		else
			return this.macroAvg.getFMeasure(classValue);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getPrecisionRecallAUC(java.lang.Object)
	 */
	@Override
	public double getPrecisionRecallAUC(String classValue)
			throws IllegalArgumentException {
		
		if( this.microFlag)
			return this.microAvg.getPrecisionRecallAUC(classValue);
		else
			return this.macroAvg.getPrecisionRecallAUC(classValue);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getPrecisionRecallCurve(java.lang.Object)
	 */
	@Override
	public List<double[]> getPrecisionRecallCurve(String classValue)
			throws IllegalArgumentException {
		
		if( this.microFlag)
			return this.microAvg.getPrecisionRecallCurve(classValue);
		else
			return this.macroAvg.getPrecisionRecallCurve(classValue);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getSensitivity(java.lang.Object)
	 */
	@Override
	public double getSensitivity(String classValue)
			throws IllegalArgumentException {
		
		if( this.microFlag)
			return this.microAvg.getSensitivity(classValue);
		else
			return this.macroAvg.getSensitivity(classValue);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getSpecificity(java.lang.Object)
	 */
	@Override
	public double getSpecificity(String classValue)
			throws IllegalArgumentException {

		if( this.microFlag)
			return this.microAvg.getSpecificity(classValue);
		else
			return this.macroAvg.getSpecificity(classValue);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getTruePositiveRate(java.lang.Object)
	 */
	@Override
	public double getTruePositiveRate(String classValue)
			throws IllegalArgumentException {
		
		if( this.microFlag)
			return this.microAvg.getTruePositiveRate(classValue);
		else
			return this.macroAvg.getTruePositiveRate(classValue);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getFalsePositiveRate(java.lang.Object)
	 */
	@Override
	public double getFalsePositiveRate(String classValue)
			throws IllegalArgumentException {
		
		if( this.microFlag)
			return this.microAvg.getFalsePositiveRate(classValue);
		else
			return this.macroAvg.getFalsePositiveRate(classValue);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getBrier()
	 */
	@Override
	public double getBrier() {
		
		if( this.microFlag)
			return this.microAvg.getBrier();
		else
			return this.macroAvg.getBrier();
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getROCAUC(java.lang.Object)
	 */
	@Override
	public double getROCAUC(String classValue)
			throws IllegalArgumentException {
		
		if( this.microFlag)
			return this.microAvg.getROCAUC(classValue);
		else
			return this.macroAvg.getROCAUC(classValue);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getROC(java.lang.Object)
	 */
	@Override
	public List<double[]> getROC(String classValue)
			throws IllegalArgumentException {
		
		if( this.microFlag)
			return this.microAvg.getROC(classValue);
		else
			return this.macroAvg.getROC(classValue);
	}
	

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getCumulativeResponse(java.lang.String)
	 */
	@Override
	public List<double[]> getCumulativeResponse(String classValue)
			throws IllegalArgumentException {

		if( this.microFlag)
			return this.microAvg.getCumulativeResponse(classValue);
		else
			return this.macroAvg.getCumulativeResponse(classValue);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IClassificationPerformances#getLiftChart(java.lang.String)
	 */
	@Override
	public List<double[]> getLiftChart(String classValue)
			throws IllegalArgumentException {

		if( this.microFlag)
			return this.microAvg.getLiftChart(classValue);
		else
			return this.macroAvg.getLiftChart(classValue);
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IAggregatePerformances#getPerformances()
	 */
	@Override
	public Collection<PType> getPerformances() {
		
		if( this.microFlag)
			return this.microAvg.getPerformances();
		else
			return this.macroAvg.getPerformances();
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IAggregatePerformances#addPerformances(java.util.Collection)
	 */
	@Override
	public void addPerformances(
			Collection<PType> runsPerformances) {

		this.microAvg.addPerformances(runsPerformances);
		this.macroAvg.addPerformances(runsPerformances);		
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IAggregatePerformances#addPerformances(CTBNToolkit.performances.ISingleRunPerformances)
	 */
	@Override
	public void addPerformances(
			PType performance) {

		this.microAvg.addPerformances(performance);
		this.macroAvg.addPerformances(performance);		
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IAggregatePerformances#getVarianceLearningTime()
	 */
	@Override
	public double getVarianceLearningTime() {
		
		if( this.microFlag)
			return this.microAvg.getVarianceLearningTime();
		else
			return this.macroAvg.getVarianceLearningTime();
	}

	/**
	 * Set the number of sampling to use
	 * in the vertical sampling of the
	 * ROC curve generation (zero sample
	 * is added by default). 
	 * 
	 * @param samplesNumber number of sample to generate the ROC curve.
	 */
	public void setMacroROCSamplesNumber(int samplesNumber) throws IllegalArgumentException {
		
		this.macroAvg.setROCSamplesNumber(samplesNumber);
		
	}
	
	/**
	 * Set the confidence interval
	 * for the ROC aggregate curve.
	 * 
	 * @param confidenceLevel level of confidence (range of values {"99.9%", "99.8%", "99%", "98%", "95%", "90%", "80%"}
	 * @throws IllegalArgumentException in case of illegal arguments
	 */
	public void setMacroROCConfidenceInterval(String confidenceInterval) throws IllegalArgumentException {
	
		this.macroAvg.setROCConfidenceInterval(confidenceInterval);
	}
	
	/**
	 * Set the number of sampling to use
	 * in the vertical sampling of the
	 * precision-recall curve generation
	 * (zero sample is added by default). 
	 * 
	 * @param samplesNumber number of sample to generate the precision-recall curve.
	 */
	public void setMacroPrecisionRecallSamplesNumber(int samplesNumber) throws IllegalArgumentException {
		
		this.macroAvg.setPrecisionRecallSamplesNumber(samplesNumber);
	}
	
	/**
	 * Set the confidence interval for the
	 * precision-recall aggregate curve.
	 * 
	 * @param confidenceLevel level of confidence (range of values {"99.9%", "99.8%", "99%", "98%", "95%", "90%", "80%"} [default "90%"]
	 * @throws IllegalArgumentException in case of illegal arguments
	 */
	public void setMacroPrecisionRecallConfidenceInterval(String confidenceInterval) throws IllegalArgumentException {
	
		this.macroAvg.setPrecisionRecallConfidenceInterval(confidenceInterval);
	}
	
} public class MicroMacroClassificationPerformancesFactory<TimeType extends Number & Comparable<TimeType>, SingPType extends IClassificationSingleRunPerformances<TimeType>> implements
		IAggregatePerformancesFactory<TimeType,MicroMacroClassificationPerformances<TimeType,SingPType>,SingPType> {

	public ISingleRunPerformancesFactory<TimeType,SingPType> singleRunFactory;
	private boolean microAvgFlag;

	
	/**
	 * Constructor.
	 * Default microAvgFlag value set to true.
	 * 
	 * @param singleRunFactory factory to use to generate the single run performance to add to the aggregate performances instances.
	 */
	public MicroMacroClassificationPerformancesFactory(ISingleRunPerformancesFactory<TimeType,SingPType> singleRunFactory) {
		
		this( singleRunFactory, true);
	}
	
	/**
	 * Constructor.
	 * 
	 * @param singleRunFactory factory to use to generate the single run performance to add to the aggregate performances instances.
	 * @param microAvgFlag microAvgFlag to set in the new instances.
	 */
	public MicroMacroClassificationPerformancesFactory(ISingleRunPerformancesFactory<TimeType,SingPType> singleRunFactory, boolean microAvgFlag) {
		
		if( singleRunFactory == null)
			throw new IllegalArgumentException("Error: null value of the single run fuctory parameter");
		
		this.singleRunFactory = singleRunFactory;
		this.microAvgFlag = microAvgFlag;
	}
	

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IAggregatePerformancesFactory#newAggregateInstance(java.util.Map)
	 */
	@Override
	public MicroMacroClassificationPerformances<TimeType,SingPType> newAggregateInstance(
			Map<Integer, String> classIndexToValue) {
		
		MicroMacroClassificationPerformances<TimeType,SingPType> microMacroPerformances = new MicroMacroClassificationPerformances<TimeType,SingPType>(classIndexToValue);
		microMacroPerformances.setMicroFlag( this.microAvgFlag);
		
		return microMacroPerformances;
	}

	/* (non-Javadoc)
	 * @see CTBNToolkit.performances.IAggregatePerformancesFactory#newSingleRunInstance(java.util.Map)
	 */
	@Override
	public SingPType newSingleRunInstance(
			Map<Integer, String> classIndexToValue) {

		return this.singleRunFactory.newInstance(classIndexToValue);
	}


} public class CTBNCTestFactory<TimeType extends Number & Comparable<TimeType>,NodeType extends INode,TM extends ICTClassifier<TimeType,NodeType>,PType extends IPerformances> implements ITestFactory<TimeType,NodeType,TM,PType> {
	
	private IModelFactory<TM,TimeType,NodeType> modelFactory;
	private IValidationMethod<TimeType,NodeType,TM,PType> validationMethod;
	private ILearningAlgorithm<TimeType, NodeType> learnAlgo;
	private IClassifyAlgorithm<TimeType, NodeType> infAlgo;
	
	/**
	 * Constructor without the model factory.
	 * The dataset can not be generated automatically.
	 * 
	 * @param validationMethod validation method to use during the tests
	 * @param learnAlgo learning algorithm
	 * @param infAlgo inference algorithm
	 * @throws IllegalArgumentException in case of illegal argument
	 */
	public CTBNCTestFactory( IValidationMethod<TimeType,NodeType,TM,PType>  validationMethod,
			ILearningAlgorithm<TimeType, NodeType> learnAlgo,
			IClassifyAlgorithm<TimeType, NodeType> infAlgo) throws IllegalArgumentException {
		
		this(null, validationMethod, learnAlgo, infAlgo);
	}
	
	/**
	 * Constructor.
	 * 
	 * @param modelFactory factory used to generate a model
	 * @param validationMethod validation method to use during the tests
	 * @param learnAlgo learning algorithm
	 * @param infAlgo inference algorithm
	 * @throws IllegalArgumentException in case of illegal argument
	 */
	public CTBNCTestFactory(IModelFactory<TM,TimeType,NodeType> modelFactory,
			IValidationMethod<TimeType,NodeType,TM,PType> validationMethod,
			ILearningAlgorithm<TimeType, NodeType> learnAlgo,
			IClassifyAlgorithm<TimeType, NodeType> infAlgo) throws IllegalArgumentException {
	
		if( validationMethod == null)
			throw new IllegalArgumentException("Error: null validatorFactory argument");
		if( learnAlgo == null)
			throw new IllegalArgumentException("Error: null learnAlgo argument");
		if( infAlgo == null)
			throw new IllegalArgumentException("Error: null infAlgo argument");
		
		this.modelFactory = modelFactory;
		this.validationMethod = validationMethod;
		this.learnAlgo = learnAlgo;
		this.infAlgo = infAlgo;
	}

	/* (non-Javadoc)
	 * @see test.ITestFactoty#newTest()
	 */
	@Override
	public GenericTestResults<TimeType,NodeType,TM,PType> newTest(String testName, int datasetDim) {
		
		if( this.modelFactory == null)
			throw new RuntimeException("Error: null modelFactory argument impossible generate a dataset");
		
		TM trueModel = modelFactory.newInstance();
		
		List<ITrajectory<TimeType>> dataset = new Vector<ITrajectory<TimeType>>(datasetDim);
		for(int i = 0; i < datasetDim; ++i)
			dataset.add( trueModel.generateTrajectory(this.modelFactory.getTrajectoryLength()));
		
		@SuppressWarnings("unchecked")
		PType performances = this.validationMethod.validate((TM) trueModel.clone(), this.learnAlgo, this.infAlgo, dataset);
		
		return new GenericTestResults<TimeType,NodeType,TM,PType>(testName, trueModel, dataset, performances);
	}

	/* (non-Javadoc)
	 * @see test.ITestFactoty#newTest(java.util.List)
	 */
	@Override
	public GenericTestResults<TimeType,NodeType,TM,PType> newTest(String testName, TM model, List<ITrajectory<TimeType>> dataset) {
		
		@SuppressWarnings("unchecked")
		PType performances = this.validationMethod.validate((TM) model.clone(), this.learnAlgo, this.infAlgo, dataset);
		
		return new GenericTestResults<TimeType,NodeType,TM,PType>(testName, null, dataset, performances);
	}
	
	
	/**
	 * Load a partition file and partitions
	 * the dataset in input.
	 * Partition files can be results files
	 * or text files where each test is divided
	 * by a line starting with the keyword
	 * "Test" and containing in the other
	 * lines the trajectories files for the
	 * tests.
	 * 
	 * @param filePath path of the partition file
	 * @param dataset dataset to divide
	 * @param filePrefixToRemove prefix to remove in the partition file (can be null if there is not)
	 * @param ext extension of the trajectories files
	 * @return partitioned input dataset
	 * @throws Exception in case of errors with the partitioning or the file loading
	 */
	public static List<List<IClassificationResult<Double>>> partitionResultDataset(String filePath, List<IClassificationResult<Double>> dataset, String filePrefixToRemove, String ext) throws Exception {
		
		List<List<IClassificationResult<Double>>> partitioning = new ArrayList<List<IClassificationResult<Double>>>();
		
		// Get the object of DataInputStream
		DataInputStream in = new DataInputStream(new FileInputStream(filePath));
		BufferedReader br = new BufferedReader(new InputStreamReader(in));
		String strLine;
		
		//Read File Line By Line
		while ((strLine = br.readLine()) != null) {
			
			// If line defines a new test
			if( strLine.startsWith("Test")) {
				partitioning.add(new ArrayList<IClassificationResult<Double>>());
				continue;
			}
			
			// Generate the string to seach in the loaded dataset
			int endIndex;
			if( (endIndex = strLine.indexOf(":")) != -1)	// cut the line to the ":" if presents
				strLine = strLine.substring( 0, endIndex);
			if( filePrefixToRemove != null && (!filePrefixToRemove.isEmpty()) && strLine.startsWith(filePrefixToRemove))	// remove the prefix if presents
				strLine = strLine.substring(filePrefixToRemove.length());
			if( !strLine.endsWith(ext))						// add the extension if not presents
				strLine += ext;
			
			// Partitions the dataset
			boolean added = false;
			for( int i = 0; i < dataset.size(); ++i) {
				if( dataset.get(i).getName().contains(strLine)) {
					partitioning.get(partitioning.size() - 1).add(dataset.get(i));
					added = true;
					break;
				}
			}
			if( !added) {
				br.close();
				throw new RuntimeException( "Error: " + strLine + " didn't found in the dataset's trajectories");
			}
		}
		in.close();
		br.close();
		
		return partitioning;
	}
	
	
	/**
	 * Load a partition file and partitions
	 * the dataset in input.
	 * Partition files can be results files
	 * or text files where each test is divided
	 * by a line starting with the keyword
	 * "Test" and containing in the other
	 * lines the trajectories files for the
	 * tests.
	 * 
	 * @param filePath path of the partition file
	 * @param dataset dataset to divide
	 * @param filePrefixToRemove prefix to remove in the partition file (can be null if there is not)
	 * @param ext extension of the trajectories files
	 * @return partitioned input dataset
	 * @throws Exception in case of errors with the partitioning or the file loading
	 */
	public static List<List<ITrajectory<Double>>> partitionDataset(String filePath, List<ITrajectory<Double>> dataset, String filePrefixToRemove, String ext) throws Exception {
		
		List<List<ITrajectory<Double>>> partitioning = new ArrayList<List<ITrajectory<Double>>>();
		
		// Get the object of DataInputStream
		DataInputStream in = new DataInputStream(new FileInputStream(filePath));
		BufferedReader br = new BufferedReader(new InputStreamReader(in));
		String strLine;
		
		//Read File Line By Line
		while ((strLine = br.readLine()) != null) {
			
			// If line defines a new test
			if( strLine.startsWith("Test")) {
				partitioning.add(new ArrayList<ITrajectory<Double>>());
				continue;
			}
			
			// Generate the string to seach in the loaded dataset
			int endIndex;
			if( (endIndex = strLine.indexOf(":")) != -1)	// cut the line to the ":" if presents
				strLine = strLine.substring( 0, endIndex);
			if( filePrefixToRemove != null && (!filePrefixToRemove.isEmpty()) && strLine.startsWith(filePrefixToRemove))	// remove the prefix if presents
				strLine = strLine.substring(filePrefixToRemove.length());
			if( !strLine.endsWith(ext))						// add the extension if not presents
				strLine += ext;
			
			// Partitions the dataset
			boolean added = false;
			for( int i = 0; i < dataset.size(); ++i) {
				if( dataset.get(i).getName().contains(strLine)) {
					partitioning.get(partitioning.size() - 1).add(dataset.get(i));
					added = true;
					break;
				}
			}
			if( !added) {
				br.close();
				throw new RuntimeException( "Error: " + strLine + " didn't found in the dataset's trajectories");
			}
		}
		in.close();
		br.close();
		
		return partitioning;
	}
	
	/**
	 * Load all the trajectory files with results in
	 * the path.
	 * The first column in the files must be the time.
	 * 
	 * @param nameIndexing name that identify the global node indexing
	 * @param dataset where add the new files (it must be instanced)
	 * @param path path from where load the trajectories
	 * @param extention of the file to load (* means each file)
	 * @param separator fields separator in the file
	 * @param classNodeName of the class column
	 * @param timeNodeName name of the time column
	 * @param trajectoryCounterName name of the column to use to cat consecutive trajectories (it can be null)
	 * @param validColumns set of the names of the column of the dataset to load (the actual data columns not the time column or other columns)
	 * @param resultsDataset dataset where add the loaded trajectories with classification results
	 * @return continuous time naive bayes model generated over the loaded dataset
	 * @throws IllegalArgumentException if case of illegal arguments
	 * @throws IOException in case of loading errors
	 */
	public static CTBNClassifier loadResultsDataset(String nameIndexing, String path, String ext, char separator, String classNodeName, String timeNodeName, String trajectoryCounterName, Set<String> validColumns, List<IClassificationResult<Double>> resultsDataset) throws IllegalArgumentException,IOException {
		
		List<ITrajectory<Double>> dataset = new ArrayList<ITrajectory<Double>>();
		CTBNClassifier nBModel = loadDataset(nameIndexing,  dataset, path, ext, separator, classNodeName, timeNodeName, trajectoryCounterName, validColumns, 1, 0);
		loadClassification(path, ext, separator, nBModel.getClassNode(), dataset, resultsDataset);
		
		return nBModel;
	}
	
	/**
	 * Load the results of the classification
	 * from files using the corresponding
	 * dataset of trajectories.
	 * 
	 * @param path path of the folder that contain the result dataset
	 * @param ext extension of the trajectories files
	 * @param separator fields separator in the file
	 * @param classNode class node
	 * @param dataset dataset of loaded trajectories
	 * @param resultsDataset dataset where insert the trajectories with the classification results
	 * @throws IOException in case of IO errors
	 */
	private static void loadClassification(
			String path, String ext, char separator, CTDiscreteNode classNode, List<ITrajectory<Double>> dataset, List<IClassificationResult<Double>> resultsDataset) throws IOException {
		
		// Get all the file in the directory
		File[] listOfFiles = (new File(path)).listFiles();
		ext = ext.toLowerCase();
		
		// For each file
		for(int f = 0; f < listOfFiles.length; ++f) {
			
			// Jumps all the file that have not the right extension
			if( (!ext.equals("*")) && (!listOfFiles[f].getName().toLowerCase().endsWith(ext)))
				continue;
			
			// Find the corresponding trajectory in the dataset
			ITrajectory<Double> trj = findTrajectory(dataset, listOfFiles[f].getName());
			if( trj == null)
				throw new IllegalArgumentException("Error: input dataset does not contain a trajectory with name \"" + listOfFiles[f].getName() + "\"");
			ClassificationResults<Double> res = new ClassificationResults<Double>(trj);
			
			// Load the file
			CSVReader reader = new CSVReader(new FileReader(listOfFiles[f].getAbsolutePath()), separator);
			// Read the file
		    String [] firstLine,nextLine;
		    if ((firstLine = reader.readNext()) == null)
		    	continue;
		    
		    // Generate the indexes of results column
		    Map<String, Integer> stateToIndex = new TreeMap<String,Integer>();
		    List<Integer> resultsColumns = new Vector<Integer>( classNode.getStatesNumber());
		    for( int s = 0; s < classNode.getStatesNumber(); ++s) {
		    	boolean found = false;
		    	for( int i = 0; i < firstLine.length && (!found); ++i)
		    		if( firstLine[i].equals(classNode.getStateName(s))) {
		    			if( stateToIndex.containsKey(firstLine[i])) {
		    				reader.close();
		    				throw new RuntimeException("Error: file " + listOfFiles[f].getAbsolutePath() + " contains two columns with " + firstLine[i] + " state name");
		    			}
		    			stateToIndex.put(firstLine[i], s);
		    			resultsColumns.add(i);
		    			found = true;
		    		}
		    	if( !found) {
		    		reader.close();
		    		throw new RuntimeException("Error: file " + listOfFiles[f].getAbsolutePath() + " does not contain the result for class " + classNode.getStateName(s));
		    	}
		    }
		    	
		    // For each line
		    int iTransition = 0;
	    	while ((nextLine = reader.readNext()) != null) {
	    		if( nextLine.length < 1)
	    			continue;
	    		
	    		// Get the probability distribution
	    		int iMax = 0;
	    		double[] p = new double[classNode.getStatesNumber()];
	    		for( int rC = 0; rC < resultsColumns.size(); ++rC) {
	    			p[rC] = Double.parseDouble(nextLine[resultsColumns.get(rC)]);
	    			if( p[rC] > p[iMax])
	    				iMax = rC;
	    		}	    			
	    		res.setProbability(iTransition, p, stateToIndex);
	    		
	    		// Set the classification with the class with the maximum probability
	    		res.setClassification(firstLine[resultsColumns.get(iMax)]);
	    		
	    		++iTransition;	// increase the line counter
			}
	    	reader.close();
	    	
	    	// Add the new trajectories results
	    	resultsDataset.add(res);
		}		
		
	}
	
	/**
	 * Find the trajectory with the target name
	 * in the input dataset.
	 * 
	 * @param dataset dataset where search the trajectory
	 * @param name target name
	 * @return trajectory
	 */
	private static ITrajectory<Double> findTrajectory(
			List<ITrajectory<Double>> dataset, String name) {

		for( int i = 0; i < dataset.size(); ++i)
			if( dataset.get(i).getName().equals( name))
				return dataset.get(i);
		
		return null;
	}

	
	/**
	 * Load all the trajectory files in the path.
	 * The first column in the files must be the time.
	 * 
	 * @param nameIndexing name that identify the global node indexing
	 * @param dataset where add the new files (it must be instanced)
	 * @param path path from where load the trajectories (it can be just a file)
	 * @param extention of the file to load (* means each file)
	 * @param separator fields separator in the file
	 * @param classNodeName name of the class column
	 * @param timeNodeName name of the time column
	 * @param trajectoryCounterName name of the column to use to cat consecutive trajectories (it can be null)
	 * @param validColumns set of the names of the column of the dataset to load (the actual data columns not the time column or other columns); if empty load all the columns
	 * @param timeFactor multiplication factor to the time in the dataset
	 * @param nClusters number of classes for the class node of the generated model (this parameter is used in case of clustering to specify the number of classes (clusters), if its value is less than 2 it is ignored)
	 * @return model generated from the dataset
	 * @throws RuntimeException in case of some error in the input files
	 * @throws IllegalArgumentException if case of illegal arguments
	 * @throws IOException in case of loading errors
	 */
	public static CTBNClassifier loadDataset(String nameIndexing, List<ITrajectory<Double>> dataset, String path, String ext, char separator, String classNodeName, String timeNodeName, String trajectoryCounterName, Set<String> validColumns, double timeFactor, int nClusters) throws IllegalArgumentException,IOException,RuntimeException {
		
		if( dataset == null)
			throw new IllegalArgumentException("Error: null dataset value. It must be instanced.");
		if( validColumns == null)
			throw new IllegalArgumentException("Error: null valid columns argument. It must be instanced.");
		
		Map<String,Set<String>> statesForNode = new TreeMap<String,Set<String>>();
		
		File[] listOfFiles;
		File input = new File(path);
		if( input.isDirectory()) {				// get all the file in the directory
			listOfFiles = (new File(path)).listFiles();
		} else if( input.isFile()) {
			listOfFiles = new File[1];			// select just the file
			listOfFiles[0] = input;
		} else {
			throw new IllegalArgumentException("Error: the input to load must be a file or a directory.");
		}
		ext = ext.toLowerCase();
		
		// For each file
		NodeIndexing nodeIndexing = null;
		for(int f = 0; f < listOfFiles.length; ++f) {
			
			// Jumps all the file that have not the right extension
			if( (!ext.equals("*")) && (!listOfFiles[f].getName().toLowerCase().endsWith(ext)))
				continue;
			
			// Load the file
			CSVReader reader = new CSVReader(new FileReader(listOfFiles[f].getAbsolutePath()), separator);
			// Read the file
		    String [] firstLine,nextLine;
		    List<Double> times = new Vector<Double>();
		    List<String[]> values = new Vector<String[]>();
		    // Index
		    int timeIdx = -1;
		    int trjCounterIdx = -1;
		    String counterState = null;
		    int trjCounter = 1;
		    
		    // Read the header and set the index of the state, time and, trajectories breaker column (if exist)
		    boolean setValidColumns = false;
		    if( validColumns.isEmpty()) {							// if empty, the valid columns are all the column except the class, the time and the trajectory separator columns
		    	setValidColumns = true;
		    }
		    if ((firstLine = reader.readNext()) == null)
		    	continue;
		    for(int i = 0; i < firstLine.length; ++i) {
		    	firstLine[i] = firstLine[i].trim();
		    	if( firstLine[i].equals(timeNodeName)) {													// time column
		    		if( timeIdx != -1) {
		    			reader.close();
		    			throw new RuntimeException("Error two or more time columns named " + timeNodeName);
		    		} else
		    			timeIdx = i;
		    	}else if( trajectoryCounterName != null && firstLine[i].equals(trajectoryCounterName)) {	// trajectory separator column
		    		if( trjCounterIdx != -1) {
		    			reader.close();
		    			throw new RuntimeException("Error two or more trajectory counter columns named " + trajectoryCounterName);
		    		} else
		    			trjCounterIdx = i;
		    	}else if( setValidColumns) {																// all the other columns
		    		validColumns.add(firstLine[i]);
		    	}
		    }
		    setValidColumns = false;
		    validColumns.add(classNodeName);																// the class must be alwasys in the valid columns
		    if( timeIdx == -1) {
		    	reader.close();
		    	throw new RuntimeException("Error time column named " + timeNodeName + " not found");
		    }
		    if( trajectoryCounterName != null && trjCounterIdx == -1) {
		    	reader.close();
		    	throw new RuntimeException("Error trajectory counter column named " + trajectoryCounterName + " not found");
		    }
		    if( nodeIndexing == null)
		    	nodeIndexing = NodeIndexing.getNodeIndexing(nameIndexing, firstLine, classNodeName, validColumns);
		    
		    // Read the file
	    	while ((nextLine = reader.readNext()) != null) {
	    		if( nextLine.length < 1)
	    			continue;
	    		if( nextLine.length != firstLine.length) {
	    			reader.close();
	    			throw new RuntimeException("Data row length does not correspond with header length");
	    		}

	    		// Trajectories separation
	    		if( trjCounterIdx != -1) {
	    			if( counterState == null) {
	    				//Start a new trajectory
	    				counterState = nextLine[trjCounterIdx];
	    			} else if( !nextLine[trjCounterIdx].equals( counterState)) {
	    				// Generate the finished trajectory
	    				CTTrajectory<Double> trj = new CTTrajectory<Double>(nodeIndexing, times, values);
	    		    	trj.setName(listOfFiles[f].getName() + "_" + trjCounter);
	    		    	dataset.add(trj);
	    		    	
	    				// Start the new trajectory
	    		    	times = new Vector<Double>();
	    				values = new Vector<String[]>();
	    				++trjCounter;
	    				counterState = nextLine[trjCounterIdx];
	    			}
	    		}
	    		
	    		// Read the row and generate its states
			    String[] rowValues = new String[nodeIndexing.getNodesNumber()];
			    for(int i  = 0; i < rowValues.length; ++i)
			    	rowValues[i] = null;
			    
			    Set<String> states;
			    for(int i = 0; i < firstLine.length; ++i) {
			    	if( !validColumns.contains(firstLine[i]))	// if it is not a column, ignore it
			    		continue;
			    	
			    	nextLine[i] = nextLine[i].trim();
			    	rowValues[ nodeIndexing.getIndex( firstLine[i])] = nextLine[i];
			    	if( statesForNode.containsKey(firstLine[i]))
			    		states = statesForNode.get(firstLine[i]);
			    	else
			    		states = new TreeSet<String>();
		    		states.add(nextLine[i]);
		    		statesForNode.put(firstLine[i], states);
			    }
			    values.add(rowValues);
			    times.add(Double.parseDouble(nextLine[timeIdx])*timeFactor);
			}
	    	reader.close();
	    	
	    	CTTrajectory<Double> trj = new CTTrajectory<Double>(nodeIndexing, times, values);
	    	if( trjCounterIdx == -1)
	    		trj.setName(listOfFiles[f].getName());
	    	else
	    		trj.setName(listOfFiles[f].getName() + "_" + trjCounter);
	    	dataset.add(trj);
		}
		
		// Set the number of clusters for the class node
		if( nClusters >= 2) {
			Set<String> clusters = new TreeSet<String>();
			for(int i = 1; i <= nClusters; ++i)
				clusters.add("" + i);
			statesForNode.put(classNodeName, clusters);
		}			

		return CTBNClassifierFactory.getNaiveBayesFromDataset(nodeIndexing, statesForNode);
		
	}
	
	/**
	 * Generate a new dataset that is a random
	 * permutation of the dataset in input.
	 * 
	 * @param dataset dataset to mix
	 * @return a random permutation of the input dataset
	 */
	static public List<ITrajectory<Double>> permuteDataset(List<ITrajectory<Double>> dataset) {
		
		ITrajectory<Double> trjTemp = null;
		for( int i = 0; i < dataset.size(); ++i) {
			int rIndex = (int) (Math.random() * dataset.size());
			trjTemp = dataset.get(rIndex);
			dataset.set(rIndex, dataset.get(i));
			dataset.set(i, trjTemp);
		}
		
		return dataset;		
	}
	
	/**
	 * Choose only a percentage of the trajectories
	 * in the dataset. These trajectory are cut of
	 * the same percentage length.
	 * 
	 * Ex: applying the function to 1000 trajectories
	 * long 10 seconds with 0.5 of percentage, the
	 * method will choose randomly 500 trajectories
	 * cutting their length to 5 seconds.
	 * 
	 * 
	 * @param dataset dataset to cut
	 * @param cutPercentage percentage of cutting to apply
	 * @return new dataset generated
	 * @throws IllegalArgumentException in case of illegal parameters
	 */
	static public List<ITrajectory<Double>> cutDataset(List<ITrajectory<Double>> dataset, double cutPercentage) throws IllegalArgumentException {
		
		if( dataset == null)
			throw new IllegalArgumentException("Error: null dataset");
		if( dataset.size() == 0)
			throw new IllegalArgumentException("Error: empty dataset");
		if( cutPercentage <= 0 || cutPercentage > 1)
			throw new IllegalArgumentException("Error: the cutting percentage must be in (0,1]");
		
		// Reduce the number of trajectories
		int nTrj = (int) Math.round( dataset.size() * cutPercentage);
		for(int i = dataset.size() - nTrj; i > 0; --i)
			dataset.remove( (int) (dataset.size() * Math.random()));
		
		// Cut the length of the trajectories
		for(int i  = 0; i < dataset.size(); ++i)
			dataset.set(i, CTTrajectory.cutTrajectory( (CTTrajectory<Double>)dataset.get(i), cutPercentage));
			
		return dataset;
	}
	
} public class GenericTestResults<TimeType extends Number & Comparable<TimeType>,NodeType extends INode,TM extends ICTClassifier<TimeType,NodeType>,PType extends IPerformances> implements ITestResults<TimeType,NodeType,TM,PType> {
	
	private String modelName;
	private TM trueModel;
	private List<ITrajectory<TimeType>> dataset;
	private PType performances;

	
	/**
	 * Base constructor.
	 * 
	 * @param modelName name of the tested model
	 * @param dataset dataset used for the test
	 * @param results result of the test
	 * @throws IllegalArgumentException if there is some error with the parameters
	 */
	public GenericTestResults(String modelName, List<ITrajectory<TimeType>> dataset, PType results) throws IllegalArgumentException{
		
		this( modelName, null, dataset, results);
	}
	

	/**
	 * Base constructor.
	 * 
	 * @param modelName name of the tested model
	 * @param trueModel model used to generate the dataset
	 * @param dataset dataset used for the test
	 * @param results result of the test
	 * @throws IllegalArgumentException if there is some error with the parameters
	 */
	public GenericTestResults(String modelName, TM trueModel, List<ITrajectory<TimeType>> dataset, PType results) throws IllegalArgumentException{
		
		if( modelName == null || modelName.equals(""))
			throw new IllegalArgumentException("Error: invilid model name");
		if( dataset == null)
			throw new IllegalArgumentException("Error: dataset can not be null");
		if( dataset.size() <= 0)
			throw new IllegalArgumentException("Error: empty dataset it is not allowed");
		if( results == null)
			throw new IllegalArgumentException("Erorr: null results are not allowed");
		
		this.modelName = modelName;
		this.trueModel = trueModel;
		this.dataset = dataset;
		this.performances = results;
		
	}
	

	/* (non-Javadoc)
	 * @see test.ITest#getModelName()
	 */
	@Override
	public String getModelName() {
		
		return this.modelName;
	}
	
	/* (non-Javadoc)
	 * @see test.ITest#getTrueModel()
	 */
	@Override
	public TM getTrueModel() {

		return this.trueModel;
	}
	
	/* (non-Javadoc)
	 * @see test.ITest#getDataset()
	 */
	@Override
	public List<ITrajectory<TimeType>> getDataset() {

		return this.dataset;
	}

	/* (non-Javadoc)
	 * @see test.ITest#getResults()
	 */
	@Override
	public PType getPerformances() {

		return this.performances;
	}


	/* (non-Javadoc)
	 * @see test.ITest#setTrueModel(CTBNToolkit.IModel)
	 */
	@Override
	public void setTrueModel(TM trueModel) {

		this.trueModel = trueModel;
	}
	
	
	/**
	 * Print the results of the test
	 * in a string.
	 * 
	 * @param classNodeName name of the class node
	 * @return string of test results
	 */
	@SuppressWarnings("unchecked")
	public String resultsToString(String classNodeName) {
		
		String str = "";
		
		if( IAggregatePerformances.class.isAssignableFrom(this.performances.getClass())) {
			IAggregatePerformances<TimeType,ISingleRunPerformances<TimeType>> aggPerformances = (IAggregatePerformances<TimeType,ISingleRunPerformances<TimeType>>) this.performances;
			Iterator<ISingleRunPerformances<TimeType>> singlePerformances = aggPerformances.getPerformances().iterator();
			for( int i = 1;singlePerformances.hasNext(); ++i) {
				str += "Test" + i + "\n";
				str += resultsToString(singlePerformances.next());
			}
		} else if( ISingleRunPerformances.class.isAssignableFrom(this.performances.getClass())) {
			str = resultsToString((ISingleRunPerformances<TimeType>) this.performances);
		} else
			str = "";
		
		return str;
	}
	
	/**
	 * Print the results of the test
	 * in a string.
	 * 
	 * @param performances to use to print results
	 * @param classNodeName name of the class node
	 * @return string of test results
	 */
	public static<TimeType extends Number & Comparable<TimeType>, NodeValueType> String resultsToString( ISingleRunPerformances<TimeType> performances) {

		String str = "";
		
		Iterator<IClassificationResult<TimeType>> results = performances.sortResultsTrajectoriesByNames().iterator();
		while( results.hasNext())
			str += results.next().resultToString() + "\n";
		
		return str;
	}


	/**
	 * Return the header for the metrics
	 * CSV file.
	 * 
	 * @param classStates set of classes states 
	 * @return header for the metrics CSV file
	 */
	public static String metricsHeader(Set<String> classStates) {
		
		int fS = 0; //(int)(classStates.size() / 2.0) - 1;	// separator first of the parameter name
		int sS = classStates.size() - fS - 1;			// separator after the parameter name
		
		String str = "Test;Model;Aggregation;ConfidenceLevel;DatasetDim;KFolds;" +
				";Accuracy;;" +
				"Error;" +
				addSeparator(fS) + "Precision;" + addSeparator(sS) + 
				addSeparator(fS) + "Recall;" + addSeparator(sS) +
				addSeparator(fS) + "F-Measure;" + addSeparator(sS) +
				addSeparator(fS) + "PR AUC;" + addSeparator(sS) +
				addSeparator(fS) + "Sensitivity;" + addSeparator(sS) +
				addSeparator(fS) + "Specificity;" + addSeparator(sS) +
				addSeparator(fS) + "TP-Rate;" + addSeparator(sS) +
				addSeparator(fS) + "FP-Rate;" + addSeparator(sS) +
				addSeparator(fS) + "ROC AUC;" + addSeparator(sS) +
				"Brier;Avg learning time;Var learning time;Avg inference time; Var inference time;\n" + 
				";;;;;;lower bound;;upper bound;;" +
				addClasses(classStates) +
				addClasses(classStates) +
				addClasses(classStates) +
				addClasses(classStates) +
				addClasses(classStates) +
				addClasses(classStates) +
				addClasses(classStates) +
				addClasses(classStates) +
				addClasses(classStates) +
				addSeparator(5);
		
		return str;		
	}
	
	/**
	 * Write the metrics in a ";"-separated
	 * string. 
	 * 
	 * @param classStates set of classes states
	 * @param confidenceLevel level of confidence to use
	 * @param datasetDim dataset dimension
	 * @param testName name of the test run
	 * @param kFolds number of fold in the cross validation
	 * @return metric in a CSV format
	 */
	@SuppressWarnings("unchecked")
	public String metricsToString( Set<String> classStates, String confidenceLevel, int datasetDim, String testName, int kFolds) {
		
		if(MicroMacroClassificationPerformances.class.isAssignableFrom(this.performances.getClass()))
			return GenericTestResults.metricsToString((MicroMacroClassificationPerformances<TimeType,IClassificationSingleRunPerformances<TimeType>>) performances, this.getModelName(), classStates, confidenceLevel, datasetDim, testName, kFolds);
		else if(MicroAvgAggregatePerformances.class.isAssignableFrom(this.performances.getClass()))
			return GenericTestResults.metricsToString((IClassificationPerformances<TimeType>) this.performances, this.getModelName(), classStates, confidenceLevel, datasetDim, "MicroAvg", testName, kFolds);
		else if(MacroAvgAggregatePerformances.class.isAssignableFrom(this.performances.getClass()))
			return GenericTestResults.metricsToString((IClassificationPerformances<TimeType>) this.performances, this.getModelName(), classStates, confidenceLevel, datasetDim, "MacroAvg", testName, kFolds);
		else if(IClassificationPerformances.class.isAssignableFrom(this.performances.getClass()))
			return GenericTestResults.metricsToString((IClassificationPerformances<TimeType>) this.performances, this.getModelName(), classStates, confidenceLevel, datasetDim, "", testName, kFolds);
		else
			throw new RuntimeException("Error: metricsToString method in " + this.getClass().getName() + " class is not defined for " + this.performances.getClass().getName() + " class");

	}
		
	/**
	 * Write the metrics in a ";"-separated
	 * string. 
	 * 
	 * @param performances performances where take metrics
	 * @param modelName model name
	 * @param classStates set of classes states
	 * @param confidenceLevel level of confidence to use
	 * @param datasetDim dataset dimension
	 * @param testName name of the test run
	 * @param kFolds number of fold in the cross validation
	 * @return metric in a CSV format
	 */
	public static<TimeType extends Number & Comparable<TimeType>> String metricsToString(MicroMacroClassificationPerformances<TimeType,?> performances, 
			String modelName, Set<String> classStates, String confidenceLevel, int datasetDim, String testName, int kFolds) {
		
		String str = metricsToString(performances.getMicroAveraging(), modelName, classStates, confidenceLevel, datasetDim, "MicroAvg",testName, kFolds) + "\n";

		performances.getMacroAveraging().setROCConfidenceInterval(confidenceLevel);
		performances.getMacroAveraging().setPrecisionRecallConfidenceInterval(confidenceLevel);
		str += metricsToString(performances.getMacroAveraging(), modelName, classStates, confidenceLevel, datasetDim, "MacroAvg",testName, kFolds);
		
		return str;
	}
	
	/**
	 * Write the metrics in a ";"-separated
	 * string.
	 * 
	 * @param performances performances where take metrics
	 * @param modelName model name
	 * @param classStates set of classes states
	 * @param confidenceLevel level of confidence to use
	 * @param datasetDim dataset dimension
	 * @param aggMode aggregation modality
	 * @param testName name of the test run
	 * @param kFolds number of fold in the cross validation
	 * @return return metric in a CSV format
	 */
	@SuppressWarnings("unchecked")
	public static<TimeType extends Number> String metricsToString(IClassificationPerformances<TimeType> performances, String modelName, Set<String> classStates, String confidenceLevel, int datasetDim, String aggMode, String testName, int kFolds) {
	
		Iterator<String> classesIter;
		String metricsStr;
		
		// Model name
		metricsStr = testName + ";" + modelName + ";" + aggMode + ";" + confidenceLevel + ";" + datasetDim + ";" + kFolds + ";";
		
		// Accuracy
		try {
			double[] acc = performances.getAccuracy(confidenceLevel);
			metricsStr += acc[0] + ";" + acc[1] + ";" + acc[2] + ";";
		}catch(Exception e) {
			metricsStr += "ERR;ERR;ERR;";
		}
		 
		// Error
		try {
			metricsStr += performances.getError() + ";";
		}catch(Exception e) {
			metricsStr += "ERR;";
		}
		
		// Precision
		classesIter = classStates.iterator();
		while( classesIter.hasNext()) {
			try {
				metricsStr += performances.getPrecision(classesIter.next()) + ";";
			}catch(Exception e) {
				metricsStr += "ERR;";
			}
		}
		
		// Recall
		classesIter = classStates.iterator();
		while( classesIter.hasNext()) {
			try {
				metricsStr += performances.getRecall(classesIter.next()) + ";";
			}catch(Exception e) {
				metricsStr += "ERR;";
			}
		}
		
		// FMeasure
		classesIter = classStates.iterator();
		while( classesIter.hasNext()) {
			try {
				metricsStr += performances.getFMeasure(classesIter.next()) + ";";
			}catch(Exception e) {
				metricsStr += "ERR;";
			}
		}
		
		// PR AUC
		classesIter = classStates.iterator();
		while( classesIter.hasNext()) {
			try {
				metricsStr += performances.getPrecisionRecallAUC(classesIter.next()) + ";";
			}catch(Exception e) {
				metricsStr += "ERR;";
			}
		}
		
		// Sensitivity
		classesIter = classStates.iterator();
		while( classesIter.hasNext()) {
			try {
				metricsStr += performances.getSensitivity(classesIter.next()) + ";";
			}catch(Exception e) {
				metricsStr += "ERR;";
			}
		}
		
		// Specificity
		classesIter = classStates.iterator();
		while( classesIter.hasNext()) {
			try {
				metricsStr += performances.getSpecificity(classesIter.next()) + ";";
			}catch(Exception e) {
				metricsStr += "ERR;";
			}
		}
		
		// TP Rate
		classesIter = classStates.iterator();
		while( classesIter.hasNext()) {
			try {
				metricsStr += performances.getTruePositiveRate(classesIter.next()) + ";";
			}catch(Exception e) {
				metricsStr += "ERR;";
			}
		}
		
		// FP Rate
		classesIter = classStates.iterator();
		while( classesIter.hasNext()) {
			try {
				metricsStr += performances.getFalsePositiveRate(classesIter.next()) + ";";
			}catch(Exception e) {
				metricsStr += "ERR;";
			}
		}
		
		// ROC AUC
		classesIter = classStates.iterator();
		while( classesIter.hasNext()) {
			try {
				metricsStr += performances.getROCAUC(classesIter.next()) + ";";
			}catch(Exception e) {
				metricsStr += "ERR;";
			}
		}
		
		// Brier
		try {
			metricsStr += performances.getBrier() + ";";
		}catch(Exception e) {
			metricsStr += "ERR;";
		}
		
		// Avg learning time
		try {
			metricsStr += performances.getLearningTime() + ";";
		}catch(Exception e) {
			metricsStr += "ERR;";
		}
		
		// Var learning time
		try {
			if(IAggregatePerformances.class.isAssignableFrom( performances.getClass()))
				metricsStr += ((IAggregatePerformances<TimeType,?>) performances).getVarianceLearningTime() + ";";
			else
				metricsStr += ";";
		}catch(Exception e) {
			metricsStr += "ERR;";
		}
		
		// Avg inference time
		try {
			metricsStr += performances.getAvgInferenceTime() + ";";
		}catch(Exception e) {
			metricsStr += "ERR;";
		}
		
		// Var inference time
		try {
			metricsStr += performances.getVarianceInferenceTime() + ";";
		}catch(Exception e) {
			metricsStr += "ERR;";
		}
		
		return metricsStr;
	}

	/**
	 * Write the metrics in a ";"-separated
	 * string. 
	 * 
	 * @param testName name of the test
	 * @param confidenceLevel level of confidence to use
	 * @param datasetDim dataset dimension
	 * @param kFolds number of fold in the cross validation
	 * @return metric in a CSV format
	 */
	@SuppressWarnings("unchecked")
	public String performancesSummary( String testName, String confidenceLevel, int datasetDim, int kFolds) {
		
		// Classification
		if(MicroMacroClassificationPerformances.class.isAssignableFrom(this.performances.getClass()))
			return GenericTestResults.performancesSummary((MicroMacroClassificationPerformances<TimeType,IClassificationSingleRunPerformances<TimeType>>) performances, testName, this.getModelName(), confidenceLevel, datasetDim, kFolds);
		else if(MicroAvgAggregatePerformances.class.isAssignableFrom(this.performances.getClass()))
			return GenericTestResults.performancesSummary((IClassificationPerformances<TimeType>) this.performances, testName, this.getModelName(), confidenceLevel, datasetDim, "MicroAvg", kFolds);
		else if(MacroAvgAggregatePerformances.class.isAssignableFrom(this.performances.getClass()))
			return GenericTestResults.performancesSummary((IClassificationPerformances<TimeType>) this.performances, testName, this.getModelName(), confidenceLevel, datasetDim, "MacroAvg", kFolds);
		else if(IClassificationPerformances.class.isAssignableFrom(this.performances.getClass()))
			return GenericTestResults.performancesSummary((IClassificationPerformances<TimeType>) this.performances, testName, this.getModelName(), confidenceLevel, datasetDim, "", kFolds);
		// Clustering
		else if(MicroMacroClusteringPerformances.class.isAssignableFrom(this.performances.getClass()))
			return GenericTestResults.performancesSummary((MicroMacroClusteringPerformances<TimeType, IExternalClusteringSingleRunPerformances<TimeType>>) this.performances, testName, this.getModelName(), confidenceLevel, datasetDim, kFolds);
		else if(MicroAvgExternalClusteringAggregatePerformances.class.isAssignableFrom(this.performances.getClass()))
			return GenericTestResults.performancesSummary((IExternalClusteringPerformances) this.performances, testName, this.getModelName(), confidenceLevel, datasetDim, "MicroAvg", kFolds);
		else if(MacroAvgExternalClusteringAggregatePerformances.class.isAssignableFrom(this.performances.getClass()))
			return GenericTestResults.performancesSummary((IExternalClusteringPerformances) this.performances, testName, this.getModelName(), confidenceLevel, datasetDim, "MacroAvg", kFolds);
		else if(IExternalClusteringPerformances.class.isAssignableFrom(this.performances.getClass()))
			return GenericTestResults.performancesSummary((IExternalClusteringPerformances) this.performances, testName, this.getModelName(), confidenceLevel, datasetDim, "", kFolds);
		else
			throw new RuntimeException("Error: metricsToString method in " + this.getClass().getName() + " class is not defined for " + this.performances.getClass().getName() + " class");

	}

	/**
	 * Performances summary in CSV format.
	 * 
	 * @param performances performances where take metrics
	 * @param testName name of the test
	 * @param modelName name of the model tested
	 * @param confidenceLevel level of confidence to use
	 * @param datasetDim dataset dimension
	 * @param kFolds number of fold in the cross validation
	 * @return summary of performances in CSV format
	 */
	private static<TimeType extends Number & Comparable<TimeType>> String performancesSummary(
			MicroMacroClusteringPerformances<TimeType, ?> performances,
			String testName, String modelName, String confidenceLevel,
			int datasetDim, int kFolds) {
		
		String str = performancesSummary(performances.getMicroAveraging(), testName, modelName, confidenceLevel, datasetDim, "MicroAvg", kFolds) + "\n";
		str += performancesSummary(performances.getMacroAveraging(), testName, modelName, confidenceLevel, datasetDim, "MacroAvg", kFolds);
		
		return str;
	}
	
	/**
	 * Performances summary in CSV format.
	 * 
	 * @param performances performances where take metrics
	 * @param testName name of the test
	 * @param modelName name of the model tested
	 * @param confidenceLevel level of confidence to use
	 * @param datasetDim dataset dimension
	 * @param aggMode aggregation modality (micro averaging = "MicroAvg", macro averaging = "MacroAvg", no aggregation (single run performances) = "")
	 * @param kFolds number of fold in the cross validation
	 * @return summary of performances in CSV format
	 */
	private static String performancesSummary(
			IExternalClusteringPerformances performances, String testName,
			String modelName, String confidenceLevel, int datasetDim,
			String aggMode, int kFolds) {
		
		StringBuilder strBuilder = new StringBuilder();
		
		if( kFolds > 1) {
			strBuilder.append( performancesSummary(performances, testName, modelName, confidenceLevel, datasetDim, aggMode, kFolds, "out of sample"));
			if( performances.getInSamplePerformances() != null)
				strBuilder.append( performancesSummary(performances.getInSamplePerformances(), testName, modelName, confidenceLevel, datasetDim, aggMode, kFolds, "in of sample"));
		} else
			strBuilder.append( performancesSummary(performances.getInSamplePerformances(), testName, modelName, confidenceLevel, datasetDim, aggMode, kFolds, "in of sample"));
		
		return strBuilder.toString();
	}
	
	/**
	 * Performances summary in CSV format.
	 * 
	 * @param performances performances where take metrics
	 * @param testName name of the test
	 * @param modelName name of the model tested
	 * @param confidenceLevel level of confidence to use
	 * @param datasetDim dataset dimension
	 * @param aggMode aggregation modality (micro averaging = "MicroAvg", macro averaging = "MacroAvg", no aggregation (single run performances) = "")
	 * @param kFolds number of fold in the cross validation
	 * @param sampleMode sampling modality ("in sample", "out of sample")
	 * @return summary of performances in CSV format
	 */
	private static String performancesSummary(
			IExternalClusteringPerformances performances, String testName,
			String modelName, String confidenceLevel, int datasetDim,
			String aggMode, int kFolds, String sampleMode) {
		
		StringBuilder strBuilder = new StringBuilder();
		
		// Model name
		strBuilder.append( testName + ";" + 
					modelName + ";" + 
					sampleMode + ";" +
					aggMode + ";" + 
					//confidenceLevel + ";" +
					datasetDim + ";" +
					kFolds + ";");
		
		// #classes
		try {
			strBuilder.append( performances.classesNumber() + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		// #clusters
		try {
			strBuilder.append( performances.clusterNumber() + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		// Rand statistic
		try {
			strBuilder.append( performances.getRandStatistic() + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		// Jaccard coefficient
		try {
			strBuilder.append( performances.getJaccardCoefficient() + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		// Folkes & Mallows index
		try {
			strBuilder.append( performances.getFolkesMallowsIndex() + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		// Avg learning time
		try {
			strBuilder.append( performances.getLearningTime() + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		// Var learning time
		try {
			if(IAggregatePerformances.class.isAssignableFrom( performances.getClass()))
				strBuilder.append( ((IAggregatePerformances<?,?>) performances).getVarianceLearningTime() + ";");
			else
				strBuilder.append( ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		// Avg inference time
		try {
			strBuilder.append( performances.getAvgInferenceTime() + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		// Var inference time
		try {
			strBuilder.append( performances.getVarianceInferenceTime() + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		return strBuilder.toString();
	}


	/**
	 * Performances summary in CSV format.
	 * 
	 * @param performances performances where take metrics
	 * @param testName name of the test
	 * @param modelName name of the model tested
	 * @param confidenceLevel level of confidence to use
	 * @param datasetDim dataset dimension
	 * @param kFolds number of fold in the cross validation
	 * @return summary of performances in CSV format
	 */
	public static<TimeType extends Number & Comparable<TimeType>> String performancesSummary(
			MicroMacroClassificationPerformances<TimeType,?> performances, 
			String testName, String modelName, String confidenceLevel, int datasetDim, int kFolds) {
		
		StringBuilder strBuilder = new StringBuilder();
		strBuilder.append( performancesSummary(performances.getMicroAveraging(), testName, modelName, confidenceLevel, datasetDim, "MicroAvg", kFolds) + "\n");
		strBuilder.append( performancesSummary(performances.getMacroAveraging(), testName, modelName, confidenceLevel, datasetDim, "MacroAvg", kFolds));
		
		return strBuilder.toString();
	}
	
	/**
	 * Performances summary in CSV format.
	 * 
	 * @param performances performances where take metrics
	 * @param testName name of the test
	 * @param modelName name of the model tested
	 * @param confidenceLevel level of confidence to use
	 * @param datasetDim dataset dimension
	 * @param aggMode aggregation modality
	 * @param kFolds number of fold in the cross validation
	 * @return summary of performances in CSV format
	 */
	@SuppressWarnings("unchecked")
	public static<TimeType extends Number> String performancesSummary(IClassificationPerformances<TimeType> performances, 
			String testName,  String modelName, String confidenceLevel, int datasetDim, String aggMode, int kFolds) {

		StringBuilder strBuilder = new StringBuilder();
		
		// Model name
		strBuilder.append( testName + ";" + 
					modelName + ";" + 
					aggMode + ";" + 
					confidenceLevel + ";" +
					datasetDim + ";" +
					kFolds + ";");
		
		// Accuracy
		try {
			double[] acc = performances.getAccuracy(confidenceLevel);
			strBuilder.append( acc[0] + ";" + acc[1] + ";" + acc[2] + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;ERR;ERR;");
		}
		
		// Error
		try {
			strBuilder.append( performances.getError() + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		// Brier
		try {
			strBuilder.append( performances.getBrier() + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		// Avg learning time
		try {
			strBuilder.append( performances.getLearningTime() + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		// Var learning time
		try {
			if(IAggregatePerformances.class.isAssignableFrom( performances.getClass()))
				strBuilder.append( ((IAggregatePerformances<TimeType,?>) performances).getVarianceLearningTime() + ";");
			else
				strBuilder.append( ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		// Avg inference time
		try {
			strBuilder.append( performances.getAvgInferenceTime() + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		// Var inference time
		try {
			strBuilder.append( performances.getVarianceInferenceTime() + ";");
		}catch(Exception e) {
			strBuilder.append( "ERR;");
		}
		
		return strBuilder.toString();
	}
	
	/**
	 * Return the header for the performance
	 * summary CSV.
	 * 
	 * @param clustering false to print the classification header, true to print clustering header
	 * @return header of performance summary CSV
	 */
	public static String performacesSummaryHeader(boolean clustering) {
		
		if( !clustering) {
			return "Test;" +
					"Model;" +
					"Aggregation;" +
					"ConfidenceLevel;" +
					"DatasetDim;" +
					"KFolds;" +
					";Accuracy;;" +
					"Error;" + 
					"Brier;" +
					"Avg learning time;" +
					"Var learning time;" +
					"Avg inference time; " +
					"Var inference time;\n" + 
					";;;;;;lower bound;;upper bound;;;;;;;";
		} else {
			return "Test;" +
					"Model;" +
					"Sampling;" +
					"Aggregation;" +
					//"ConfidenceLevel;" +
					"DatasetDim;" +
					"KFolds;" +
					"#classes;" +
					"#clusters;" +
					"Rand Stat;" +
					"Jaccard Coeff;" +
					"Folkes Mallows Idx;" +
					"Avg learning time;" +
					"Var learning time;" +
					"Avg inference time; " +
					"Var inference time;";
		}
		
	}
	
	
	/**
	 * Return a string with a sequence
	 * of N ";".
	 * 
	 * @param N number of separator in the sequence
	 * @return a string of N separators in sequence
	 */
	public static String addSeparator( int N) {
		
		String str = "";
		for( int i = 0; i < N; ++i)
			str += ";";
		
		return str;
	}
	
	/**
	 * Return a string with all the classes
	 * states written and separated by ";".
	 * 
	 * @param classStates set of classes states names
	 * @return string of the classes names separated by ";"
	 */
	public static String addClasses( Set<String> classStates) {
		
		String str = "";
		Iterator<String> iter = classStates.iterator();
		while( iter.hasNext())
			str += iter.next() + ";";
		
		return str;
	}
	
	/**
	 * Print the cumulative response curve.
	 * 
	 * @param performances macro-average performances
	 * @param classState state of the curve to print
	 * @param confidenceInterval confidence interval for the vertical averaging curve calculation
	 * @return cumulative response curve in CSV format
	 */
	public static<TimeType extends Number & Comparable<TimeType>> String printCumulativeResponse(
			MacroAvgAggregatePerformances<TimeType,?> performances, 
			String classState, String confidenceInterval) {
		
		performances.setLiftConfidenceInterval(confidenceInterval);
		try {
			return GenericTestResults.cumulativeRespHeader(true) + "\n" + GenericTestResults.curveToString(performances.getCumulativeResponse(classState));
		}catch(Exception e) {
			return e.toString();
		}
	}
	
	/**
	 * Print the cumulative response curve.
	 * 
	 * @param performances macro-average performances
	 * @param classState state of the curve to print
	 * @return cumulative response curve in CSV format
	 */
	public static<TimeType extends Number> String printCumulativeResponse(
			IClassificationPerformances<TimeType> performances, 
			String classState) {
		
		try {
			return GenericTestResults.cumulativeRespHeader(false) + "\n" + GenericTestResults.curveToString(performances.getCumulativeResponse(classState));
		}catch(Exception e) {
			return e.toString();
		}
	}
	
	/**
	 * Print the lift chart.
	 * 
	 * @param performances macro-average performances
	 * @param classState state of the curve to print
	 * @param confidenceInterval confidence interval for the vertical averaging curve calculation
	 * @return lift chart curve in CSV format
	 */
	public static<TimeType extends Number & Comparable<TimeType>> String printLiftChart(
			MacroAvgAggregatePerformances<TimeType,?>  performances, 
			String classState, String confidenceInterval) {
		
		performances.setLiftConfidenceInterval(confidenceInterval);
		try {
			return GenericTestResults.liftChartHeader(true) + "\n" + GenericTestResults.curveToString(performances.getLiftChart(classState));
		}catch(Exception e) {
			return e.toString();
		}
	}
	
	/**
	 * Print the lift chart.
	 * 
	 * @param performances macro-average performances
	 * @param classState state of the curve to print
	 * @return lift chart in CSV format
	 */
	public static<TimeType extends Number> String printLiftChart(
			IClassificationPerformances<TimeType> performances, 
			String classState) {
		
		try {
			return GenericTestResults.liftChartHeader(false) + "\n" + GenericTestResults.curveToString(performances.getLiftChart(classState));
		}catch(Exception e) {
			return e.toString();
		}
	}

	/**
	 * Print the ROC curve.
	 * 
	 * @param performances macro-average performances
	 * @param classState state of the ROC curve to print
	 * @param confidenceInterval confidence interval for the vertical averaging curve calculation
	 * @return ROC curve in CSV format
	 */
	public static<TimeType extends Number & Comparable<TimeType>> String printROC(
			MacroAvgAggregatePerformances<TimeType,?> performances, 
			String classState, String confidenceInterval) {
		
		performances.setROCConfidenceInterval(confidenceInterval);
		try {
			return GenericTestResults.ROCHeader(true) + "\n" + GenericTestResults.curveToString(performances.getROC(classState));
		}catch(Exception e) {
			return e.toString();
		}
	}
	
	/**
	 * Print the ROC curve.
	 * 
	 * @param performances micro-average performances
	 * @param classState state of the ROC curve to print
	 * @return ROC curve in CSV format
	 */
	public static<TimeType extends Number & Comparable<TimeType>> String printROC(
			MicroAvgAggregatePerformances<TimeType,?> performances, 
			String classState) {
		
		try {
			return GenericTestResults.ROCHeader(false) + "\n" + GenericTestResults.curveToString(performances.getROC(classState));
		}catch(Exception e) {
			return e.toString();
		}
	}
	
	/**
	 * Print the ROC curve.
	 * 
	 * @param performances single run performances
	 * @param classState state of the ROC curve to print
	 * @return ROC curve in CSV format
	 */
	public static<TimeType extends Number & Comparable<TimeType>> String printROC(
			IClassificationSingleRunPerformances<TimeType> performances, 
			String classState) {
		
		try {
			return GenericTestResults.ROCHeader(false) + "\n" + GenericTestResults.curveToString(performances.getROC(classState));
		}catch(Exception e) {
			return e.toString();
		}
	}
	
	/** 
	 * Print the header of the cumulative
	 * response curve.
	 * 
	 * @param intervalFlag if true print also the vertical interval header
	 * @return header of the cumulative response curve
	 */
	private static String cumulativeRespHeader(boolean intervalFlag) {
		
		if( intervalFlag)
			return "%-Population;%-Positive;%Pos-low;%Pos-up;";
		else
			return "%-Population;%-Positive;";
	}

	/** 
	 * Print the header of the lift chart.
	 * 
	 * @param intervalFlag if true print also the vertical interval header
	 * @return header of the lift chart
	 */
	private static String liftChartHeader(boolean intervalFlag) {
		
		if( intervalFlag)
			return "%-Population;Lift;Lift-low;Lift-up;";
		else
			return "%-Population;Lift;";
	}
	
	/**
	 * Print the header of the ROC
	 * curve.
	 * 
	 * @param intervalFalg if true print also the vertical interval header
	 * @return header of the ROC curve
	 */
	public static String ROCHeader(boolean intervalFlag) {
		
		if( intervalFlag)
			return "FP-Rate;TP-Rate;TPR-low;TPR-up;";
		else
			return "FP-Rate;TP-Rate;";
	}
	
	
	/**
	 * Print the precision-recall curve.
	 * 
	 * @param performances macro-average performances
	 * @param classState state of the ROC curve to print
	 * @param confidenceInterval confidence interval for the vertical averaging curve calculation
	 * @return precision-recall curve in CSV format
	 */
	public static<TimeType extends Number & Comparable<TimeType>> String printPRCurve(
			MacroAvgAggregatePerformances<TimeType,?> performances, 
			String classState, String confidenceInterval) {
		
		performances.setROCConfidenceInterval(confidenceInterval);
		try {
			return GenericTestResults.PRHeader(true) + "\n" + GenericTestResults.curveToString(performances.getPrecisionRecallCurve(classState));
		}catch(Exception e) {
			return e.toString();
		}
	}
	
	/**
	 * Print the precision-recall curve.
	 * 
	 * @param performances micro-average performances
	 * @param classState state of the ROC curve to print
	 * @return precision-recall curve in CSV format
	 */
	public static<TimeType extends Number & Comparable<TimeType>> String printPRCurve(
			MicroAvgAggregatePerformances<TimeType,?> performances, 
			String classState) {
		
		try{
			return GenericTestResults.PRHeader(false) + "\n" + GenericTestResults.curveToString(performances.getPrecisionRecallCurve(classState));
		}catch(Exception e) {
			return e.toString();
		}
	}
	
	/**
	 * Print the precision-recall curve.
	 * 
	 * @param performances single run performances
	 * @param classState state of the ROC curve to print
	 * @return precision-recall curve in CSV format
	 */
	public static<TimeType extends Number & Comparable<TimeType>> String printPRCurve(
			IClassificationSingleRunPerformances<TimeType> performances, 
			String classState) {
		
		try {
			return GenericTestResults.PRHeader(false) + "\n" + GenericTestResults.curveToString(performances.getPrecisionRecallCurve(classState));
		}catch(Exception e) {
			return e.toString();
		}
	}
	
	/**
	 * Print the header of the Precision-
	 * Recall curve.
	 * 
	 * @param intervalFalg if true print also the vertical interval header
	 * @return header of the Precision-Recall curve
	 */
	public static String PRHeader(boolean intervalFlag) {
		
		if( intervalFlag)
			return "Recall;Precision;Pr-low;Pr-up;";
		else
			return "Recall;Precision;";
	}
	
	
	/**
	 * Print the curve data in a
	 * CSV format.
	 * 
	 * @param curve curve to print
	 * @return string of the curve in CSV format.
	 */
	public static String curveToString(List<double[]> curve) {
		
		StringBuilder strBuilder = new StringBuilder(10000);
		
		for(int i = 0; i < curve.size(); ++i) {
			double[] data = curve.get(i);
			for( int j = 0; j < data.length; ++j)
				strBuilder.append( data[j] + ";");
			
			if( i != curve.size() - 1)
				strBuilder.append( "\n");
		}
		
		return strBuilder.toString();
	}
	
	
	/**
	 * Return a CSV ";" separated table that
	 * inform about the statistical differences
	 * in different methods accuracy.
	 * It is require the same number of tests
	 * over the same dataset partitions.
	 * 
	 * @param resultsList list of results
	 * @return table of comparisons between accuracy of different tests
	 */
	public static<TimeType extends Number & Comparable<TimeType>,NodeType extends INode,TM extends ICTClassifier<TimeType,NodeType>,PType extends IPerformances> String compareMethods( 
			List<GenericTestResults<TimeType,NodeType,TM,PType>> resultsList) {
	
		List<String> confidenceLevels = new Vector<String>(5);
		confidenceLevels.add("99%");
		confidenceLevels.add("95%");
		confidenceLevels.add("90%");
		confidenceLevels.add("80%");
		confidenceLevels.add("70%");
		
		return compareMethods( resultsList, confidenceLevels);
	}
	
	
	/**
	 * Return a CSV ";" separated table that
	 * inform about the statistical differences
	 * in different methods accuracy.
	 * It is require the same number of tests
	 * over the same dataset partitions.
	 * 
	 * @param resultsList list of results
	 * @param confidenceLevels list of confidence level to use in the table comparisons
	 * @return table of comparisons between accuracy of different tests
	 */
	@SuppressWarnings({ "rawtypes", "unchecked" })
	public static<TimeType extends Number & Comparable<TimeType>,NodeType extends INode,TM extends ICTClassifier<TimeType,NodeType>,PType extends IPerformances> String compareMethods( 
  			List<GenericTestResults<TimeType,NodeType,TM,PType>> resultsList, 
  			List<String> confidenceLevels) {
		
		if( resultsList == null)
			throw new IllegalArgumentException("Null result list argument");  		
  		if( resultsList.size() <= 1)
  			return "\n";
 
  		
  		StringBuilder strBuilder = new StringBuilder(1000);
  		
  		for( int iCL = 0; iCL < confidenceLevels.size(); ++iCL) {
  			
  			strBuilder.append( "Comparison test - ;" + confidenceLevels.get(iCL) + addSeparator(resultsList.size()) + "\n;");
  			
	  		for(int i = 0; i < resultsList.size(); ++i)
	  			strBuilder.append( resultsList.get(i).getModelName() + ";");
	  		
	  		for(int i = 0; i < resultsList.size(); ++i) {
	  			strBuilder.append( "\n" + resultsList.get(i).getModelName() + ";");
	  			for(int j = 0; j < resultsList.size(); ++j) {
	  				
	  				if( i == j)
	  					strBuilder.append( ";");
	  				else {
	  					if( MicroMacroClassificationPerformances.class.isAssignableFrom( resultsList.get(i).getPerformances().getClass()))
	  						strBuilder.append( 
	  								tTest( ((MicroMacroClassificationPerformances) resultsList.get(i).getPerformances()).getMacroAveraging(), 
	  									   ((MicroMacroClassificationPerformances) resultsList.get(j).getPerformances()).getMacroAveraging(), confidenceLevels.get(iCL)) + ";");
	  					else if(ClassificationStandardPerformances.class.isAssignableFrom( resultsList.get(i).getPerformances().getClass()))
	  						strBuilder.append( 
	  								tTest( (ClassificationStandardPerformances) resultsList.get(i).getPerformances(), 
	  									   (ClassificationStandardPerformances) resultsList.get(j).getPerformances(), confidenceLevels.get(iCL)) + ";");
	  					else
	  						strBuilder.append("ERR;");
	  				}
	  			}
	  		}
	  		
	  		strBuilder.append( "\n");
  		}
  		
  		return strBuilder.toString();
  	}

	
	/**
	 * Compare the accuracy of the performances
	 * of two tests.
	 * Return "LF" if the first is better then the
	 * second, "UP" if the second is better then
	 * the first, "0" if they are comparable.
	 * 
	 * @param performancesRow
	 * @param performancesColumn
	 * @param confidenceLevel
	 * @return LF if the row is statistically better then the column, UP in the opposite case, and 0 if they are comparable
	 */
	private static<TimeType extends Number & Comparable<TimeType>,NodeValueType,PType extends IClassificationSingleRunPerformances<TimeType>> String tTest(ClassificationStandardPerformances<TimeType> performancesRow,
			ClassificationStandardPerformances<TimeType> performancesColumn, String confidenceLevel) {

		try {
			double[] accRow = performancesRow.getAccuracy( confidenceLevel);
			double[] accCol = performancesColumn.getAccuracy( confidenceLevel);
			
			if( accRow[0] > accCol[2])
				return "LF";
			else if( accRow[2] < accCol[0] )
				return "UP";
			else
				return "0";
	
		}catch(Exception e) {
			return "ERR";
		}
	}
	

	/**
	 * Compare the accuracy of the performances
	 * of two tests.
	 * Return "LF" if the first is better then the
	 * second, "UP" if the second is better then
	 * the first, "0" if they are comparable.
	 * 
	 * @param performancesRow
	 * @param performancesColumn
	 * @param confidenceLevel
	 * @return LF if the row is statistically better then the column, UP in the opposite case, and 0 if they are comparable
	 */
	private static<TimeType extends Number & Comparable<TimeType>,NodeValueType,PType extends IClassificationSingleRunPerformances<TimeType>> String tTest(MacroAvgAggregatePerformances<TimeType,PType> performancesRow,
			MacroAvgAggregatePerformances<TimeType,PType> performancesColumn, String confidenceLevel) {

		try {
			double diff = performancesRow.getAccuracy() - performancesColumn.getAccuracy();
			
			int nTests = 0;
			double denominator = 0.0;
			Iterator<PType> iterRow = performancesRow.getPerformances().iterator();
			Iterator<PType> iterColumn = performancesColumn.getPerformances().iterator();
			while(iterRow.hasNext() && iterColumn.hasNext()) {
				double testDiff = iterRow.next().getAccuracy() - iterColumn.next().getAccuracy();
				
				denominator += Math.pow(diff - testDiff, 2);
				++nTests;
			}
			if(iterRow.hasNext() || iterColumn.hasNext())
				throw new IllegalArgumentException("Error: the two performances has a different number of tests");
			
			denominator = Math.sqrt( denominator / nTests);
			
			double res = diff/denominator;
			double z = StatisticalTables.tStudentTable(nTests - 1, confidenceLevel);
			if( res > z)
				return "LF";
			else if( res < -z )
				return "UP";
			else
				return "0";
	
		}catch(Exception e) {
			return "ERR";
		}
	}
  	
} public interface ITestFactory<TimeType extends Number & Comparable<TimeType>,NodeType extends INode,TM extends IModel<TimeType, NodeType>,PType extends IPerformances> {	
	/**
	 * Generate a new test, with an auto-generated dataset
	 * and return the results.
	 * 
	 * @param testName name of the test
	 * @param datasetDim dimension of the dataset
	 * @return result of the test
	 */
	public ITestResults<TimeType,NodeType,TM,PType> newTest(String testName, int datasetDim);
	
	/**
	 * Generate a new test and return the results.
	 * In the result the real model is unknown.
	 * 
	 * @param testName name of the test
	 * @param model to clone to allow the test learning
	 * @param dataset the dataset for the test
	 * @return result of the test
	 */
	public ITestResults<TimeType,NodeType,TM,PType> newTest(String testName, TM model, List<ITrajectory<TimeType>> dataset);
} public interface ITestResults<TimeType extends Number & Comparable<TimeType>,NodeType extends INode, TM extends IModel<TimeType, NodeType>,PType extends IPerformances> {

	
	/**
	 * Set the true model.
	 * 
	 * @param trueModel true model to set
	 */
	public void setTrueModel(TM trueModel);
	
	/**
	 * Return the model used to generate the dataset.
	 * 
	 * @return model used to generate the dataset
	 */
	public TM getTrueModel();
	
	/**
	 * Get the name of the model
	 * used in the test.
	 * 
	 * @return test name
	 */
	public String getModelName();
	
	/**
	 * Get the dataset used for the test.
	 * 
	 * @return the used dataset
	 */
	public List<ITrajectory<TimeType>> getDataset();
	
	/**
	 * Get the results of the test.
	 * 
	 * @return result of the test
	 */
	public PType getPerformances();
} public class ClusteringInSample<TimeType extends Number & Comparable<TimeType>,NodeType extends INode,TM extends ICTClassifier<TimeType,NodeType>,SingPType extends IExternalClusteringSingleRunPerformances<TimeType>>
		extends
		BaseValidationMethod<TimeType, NodeType, TM, SingPType>
		implements
		IValidationMethod<TimeType, NodeType, TM, SingPType> {

	private ISingleRunPerformancesFactory<TimeType, SingPType> singleRunerformancesFactory;
	
	
	/**
	 * Base constructor.
	 * 
	 * @param singleRunerformancesFactory factory to generate the single run performances
	 */
	public ClusteringInSample(ISingleRunPerformancesFactory<TimeType, SingPType> singleRunerformancesFactory) {
		
		if( singleRunerformancesFactory == null)
			throw new IllegalArgumentException("Error: null value for the performances factory parameter");
		
		this.singleRunerformancesFactory = singleRunerformancesFactory;
		
		super.setVerbose( false);
		super.setPrintSSFlag( false);
		super.setClusterInSampleFlag( true);
	}
	
	@Override
	public void setClusterInSampleFlag( boolean flag) { }
	
	/**
	 * Set the factory to generate the single
	 * run performances.
	 * 
	 * @param singleRunerformancesFactory factory to generate the single run performances
	 */
	public void setPerformancesFactory(ISingleRunPerformancesFactory<TimeType, SingPType> singleRunerformancesFactory) {
		
		if( singleRunerformancesFactory == null)
			throw new IllegalArgumentException("Error: null value for the performances factory parameter");
		
		this.singleRunerformancesFactory = singleRunerformancesFactory;
	}
	
	
	@SuppressWarnings("unchecked")
	public SingPType validate(
			TM mdl,
			ILearningAlgorithm<TimeType, NodeType> learnAlgo,
			List<ITrajectory<TimeType>> dataset) {

		// Initialization
		DiscreteNode classNode = mdl.getClassNode();
		Map<Integer,String> classIndexToValue = new TreeMap<Integer,String>();
		for(int i = 0; i < classNode.getStatesNumber(); ++i)
			classIndexToValue.put(i, classNode.getStateName(i));
		
		super.verbosePrint("\n\nIn sample test over all the " + dataset.size() + " trajectories\n");
		super.verbosePrint("\nDataset = Training set = Test set\n");
		
		long startTime = System.currentTimeMillis();
		ILearningResults classificationResult = learnAlgo.learn(mdl, dataset);
		double learningTime = (System.currentTimeMillis() - startTime) / 1000.0;
		
		if( !ClusteringResults.class.isAssignableFrom(classificationResult.getClass()))
			throw new RuntimeException("Error: learning algorithm must be a clustering algorithm that return an istance of \"ClusteringResults\" class");
		
		// Generate in sample performances
		SingPType inSamplePerformances =  this.singleRunerformancesFactory.newInstance(classIndexToValue);
		inSamplePerformances.addResults((( ClusteringResults<TimeType>)classificationResult).getClusterizedTrajectories() );
		
		// Sufficient statistics
		printSufficientStatistics(classificationResult, mdl);
		inSamplePerformances.setLearnedModel( mdl);
		inSamplePerformances.setLearningTime( learningTime);
		
		super.verbosePrint("Model learnt in " + learningTime + " seconds\n\n");
		
		return inSamplePerformances;
	}
	
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.tests.validators.IValidationMethod#validate(CTBNToolkit.IModel, CTBNToolkit.ILearningAlgorithm, CTBNToolkit.IClassifyAlgorithm, java.util.List)
	 */
	@Override
	public SingPType validate(
			TM mdl,
			ILearningAlgorithm<TimeType, NodeType> learnAlgo,
			IClassifyAlgorithm<TimeType, NodeType> infAlgo,
			List<ITrajectory<TimeType>> dataset) {

		return this.validate(mdl, learnAlgo, dataset);
	}

} public class CrossValidation<TimeType extends Number & Comparable<TimeType>,NodeType extends INode,TM extends ICTClassifier<TimeType,NodeType>, AggPType extends IAggregatePerformances<TimeType,SingPType>, SingPType extends ISingleRunPerformances<TimeType>>
		extends
		BaseValidationMethod<TimeType, NodeType, TM, AggPType>
		implements
		IValidationMethod<TimeType, NodeType, TM, AggPType> {

	private IAggregatePerformancesFactory<TimeType, AggPType, SingPType> aggregatePerformancesFactory;
	
	private int k;
	private boolean leaveOneOut;
	private boolean memorizeFolding;
	private List<List<ITrajectory<TimeType>>> foldedDataset;
	private int nTimeDatasetUse;
	
	
	/**
	 * Base constructor.
	 * k=10 fold cross validation without
	 * folding memorization.
	 * 
	 * @param aggregatePerformancesFactory factory of aggregate performances to use in the tests.
	 */
	public CrossValidation(IAggregatePerformancesFactory<TimeType, AggPType, SingPType> aggregatePerformancesFactory) {
		
		this(aggregatePerformancesFactory, 10, false);
	}
	
	/**
	 * Constructor.
	 * 
	 * @param aggregatePerformancesFactory factory of aggregate performances to use in the tests.
	 * @param k number of folds in the cross validation (if k <= 0, leave one out modality is enabled)
	 */
	public CrossValidation(IAggregatePerformancesFactory<TimeType, AggPType, SingPType> aggregatePerformancesFactory, int k) {
		
		this(aggregatePerformancesFactory, k, false);
	}
	
	/**
	 * Constructor.
	 * 
	 * @param aggregatePerformancesFactory factory of aggregate performances to use in the tests.
	 * @param k number of folds in the cross validation (if k <= 0, leave one out modality is enabled)
	 * @param memorizeFolding flag that if activate force in the validation to use the last stored dataset partitioning
	 */
	public CrossValidation(IAggregatePerformancesFactory<TimeType, AggPType, SingPType> aggregatePerformancesFactory, int k, boolean memorizeFolding) {
		
		if( aggregatePerformancesFactory == null)
			throw new IllegalArgumentException("Error: null aggregate performances factory argument");
		
		this.aggregatePerformancesFactory = aggregatePerformancesFactory;
		
		this.k = k;
		this.leaveOneOut = (k <= 0);
		this.memorizeFolding = memorizeFolding;
		this.foldedDataset = null;
		this.nTimeDatasetUse = 0;
		
		super.setVerbose( false);
		super.setPrintSSFlag( false);
		super.setClusterInSampleFlag( false);
	}
	
	
	/**
	 * Set the aggregate performances
	 * factory.
	 * 
	 * @param aggregatePerformancesFactory factory of aggregate performances to use in the tests.
	 * @exception IllegalArgumentException in case of illegal argument.
	 */
	public void setAggregatePerformancesFactory(IAggregatePerformancesFactory<TimeType, AggPType, SingPType> aggregatePerformancesFactory) throws IllegalArgumentException {
		
		if( aggregatePerformancesFactory == null)
			throw new IllegalArgumentException("Error: null aggregate performances factory argument");
		
		this.aggregatePerformancesFactory = aggregatePerformancesFactory;
	}
	
	/**
	 * Return the number of folds used
	 * in the test.
	 * 
	 * @return number of folds
	 */
	public int getKFolds() {
		
		if( this.foldedDataset != null)
			return this.foldedDataset.size();
		else
			return this.k;
	}
	
	/**
	 * Activate or disable the modality
	 * that force the validator to use a
	 * previously loaded (and folded) dataset.
	 * 
	 * @param memorizeFolding flag that if activate force in the validation to use the last stored dataset partitioning
	 */
	public void setMemorizeFolding(boolean memorizeFolding) {
		
		this.memorizeFolding = memorizeFolding;
	}
	
	/**
	 * Reset the last folded dataset in order
	 * to store a new one.
	 */
	public void resetFoldedDataset() {
		
		this.foldedDataset = null;
		super.verbosePrint("Folded dataset reseted after being used " + this.nTimeDatasetUse + " times!\n");
	}
	
	/**
	 * True if the last folded dataset
 	 * memorization is enabled. False
 	 * otherwise.
	 * 
	 * @return true if the last folded dataset memorization is enable, false otherwise.
	 */
	public boolean isFoldsMemorizationEnabled() {
		
		return this.memorizeFolding;
	}
	
	/**
	 * Set a folded dataset to use in
	 * the tests.
	 * 
	 * @param foldedDataset folded dataset to load
	 * @throws RuntimeException in case of illegal arguments or state of the class
	 */
	public void setFoldedDataset(List<List<ITrajectory<TimeType>>> foldedDataset) throws RuntimeException {
		
		if( !this.memorizeFolding)
			throw new RuntimeException("Not enabled use of pre-loaded folded dataset (memorizeFolding flag must be true)");
		if( foldedDataset == null)
			throw new IllegalArgumentException("Null foldedDataset argument");
		if( foldedDataset.isEmpty())
			throw new IllegalArgumentException("Empty foldedDataset in input");
		
		super.verbosePrint("Load a new folded dataset!\n");		
		this.foldedDataset = foldedDataset;
		this.nTimeDatasetUse = 0;
	}
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.tests.validators.IValidationMethod#validate(CTBNToolkit.IModel, CTBNToolkit.ILearningAlgorithm, CTBNToolkit.IClassifyAlgorithm, java.util.List)
	 */
	@SuppressWarnings({ "unchecked"})
	@Override
	public AggPType validate(TM mdl,
			ILearningAlgorithm<TimeType, NodeType> learnAlgo,
			IClassifyAlgorithm<TimeType, NodeType> infAlgo,
			List<ITrajectory<TimeType>> dataset) {

		// Initialization
		if( this.leaveOneOut)
			this.k = dataset.size();
		NodeIndexing nodeIndexing = mdl.getNodeIndexing();
		DiscreteNode classNode = mdl.getClassNode();
		Map<Integer,String> classIndexToValue = new TreeMap<Integer,String>();
		for(int i = 0; i < classNode.getStatesNumber(); ++i)
			classIndexToValue.put(i, classNode.getStateName(i));
		// Performance declaration
		AggPType aggOutOfSamplePerformances = this.aggregatePerformancesFactory.newAggregateInstance(classIndexToValue);
		AggPType aggInSamplePerformances = null;
		

		// K-fold division
		if( (!this.memorizeFolding) || this.foldedDataset == null) {
			super.verbosePrint("\n\nK-Folds Cross Validation schema using " + this.k + " folds (leav one out = " + this.leaveOneOut + ").\n");
			this.foldedDataset = this.makeKFolds(dataset, this.k);
		} else
			super.verbosePrint("\n\nK-Folds Cross Validation schema using " + this.foldedDataset.size() + " folds.\n");
		// Textual warning added because of the possible errors caused by accidental loading of pre-folded datasets
		if( this.memorizeFolding) {	
			this.nTimeDatasetUse++;
			super.verbosePrint("FOLDED DATASET USED FOR THE " + this.nTimeDatasetUse + " TIMES !!!\n");
		}

		
		// Cross Validation
		for ( int f = 0; f < foldedDataset.size(); ++f)
		{
			// Performances generation
			SingPType performances = this.aggregatePerformancesFactory.newSingleRunInstance(classIndexToValue);
			
			// Training set and test set generation
			List<ITrajectory<TimeType>> testSet  = this.foldedDataset.get( f);
			List<ITrajectory<TimeType>> trainSet = this.joinOtherFolds(this.foldedDataset, f);
			

			// Model learning
			super.verbosePrint( "\nCycle " + (f+1) + "\n");
			super.verbosePrint( "Learning: from " + trainSet.get(0).getName() + " To " + trainSet.get(trainSet.size()-1).getName() + "\n");
			super.verbosePrint( "          from " + testSet.get(0).getName() + " To " +  testSet.get(testSet.size()-1 ).getName() + " excluded\n");

			long startTime = System.currentTimeMillis();
			ILearningResults classificationResult = learnAlgo.learn( mdl, trainSet);
			double learningTime = (System.currentTimeMillis() - startTime) / 1000.0;
			// In sample performances
			if( super.getClusterInSampleFlag() && ClusteringResults.class.isAssignableFrom(classificationResult.getClass())) {
			
				if( aggInSamplePerformances == null)	// generate in sample aggregate container
					aggInSamplePerformances = this.aggregatePerformancesFactory.newAggregateInstance(classIndexToValue);
				
				// Create the in sample test performances
				SingPType inSampleSinglePerformance = this.aggregatePerformancesFactory.newSingleRunInstance(classIndexToValue);
				inSampleSinglePerformance.addResults((( ClusteringResults<TimeType>)classificationResult).getClusterizedTrajectories() );
				// Add the in sample test performances
				aggInSamplePerformances.addPerformances(inSampleSinglePerformance);
				inSampleSinglePerformance.calculateFinalResults(true);
			}
			// Sufficient statistics
			printSufficientStatistics(classificationResult, mdl);
			classificationResult = null;
			performances.setLearnedModel( (TM) mdl.clone());
			performances.setLearningTime( learningTime);
			
			super.verbosePrint("Model learnt in " + learningTime + " seconds\n\n");

			
			// Testing
			double prob;
			String classPrediction;
			String trueClass;
			super.verbosePrint( "Testing : from " +  testSet.get(0).getName() + " To " + testSet.get(testSet.size()-1).getName() + "\n");
			for( int iTrj = 0; iTrj < testSet.size(); ++iTrj)
			{				
				try
				{
					super.verbosePrint("Inference on " + testSet.get(iTrj).getName() + "\n");
					startTime = System.currentTimeMillis();
					IClassificationResult<TimeType> res = mdl.classify( infAlgo, testSet.get(iTrj));
					double inferenceTime =  (System.currentTimeMillis() - startTime) / 1000.0;
					
					classPrediction = res.getClassification();
					trueClass = res.getNodeValue(0, nodeIndexing.getClassIndex());
					if( infAlgo.probabilityFlag()) {
						prob = res.getProbability(classPrediction);
						super.verbosePrint( testSet.get( iTrj).getName() + ": True Class: " + trueClass + ", Predicted: " + classPrediction + ", Probability: " + prob + ", Inference time: " + inferenceTime + "\n");
					} else
						super.verbosePrint( testSet.get( iTrj).getName() + ": True Class: " + trueClass + ", Predicted: " + classPrediction + ", Inference time: " + inferenceTime + "\n");
					
					performances.addResult( res, inferenceTime);
				}
				catch(Exception e)
				{
					System.out.println(e.getMessage());
				}
			}
			
			aggOutOfSamplePerformances.addPerformances(performances);
		}
		
		if( super.getClusterInSampleFlag() && IExternalClusteringPerformances.class.isAssignableFrom( aggOutOfSamplePerformances.getClass()))
			(( IExternalClusteringPerformances) aggOutOfSamplePerformances).setInSamplePerformances(
					(IExternalClusteringPerformances) aggInSamplePerformances);

		return aggOutOfSamplePerformances;
	}

	/**
	 * Divide the dataset in folds.
	 * 
	 * @param dataset dataset to divide
	 * @param k number of folds
	 * @return k-folds
	 */
	private List<List<ITrajectory<TimeType>>> makeKFolds(List<ITrajectory<TimeType>> dataset, int k)
	{
		List<List<ITrajectory<TimeType>>> folds = new ArrayList<List<ITrajectory<TimeType>>>(k);

		int foldDim, surplus, index2;
		foldDim  = dataset.size() / k;
		surplus  = dataset.size() % k;
		for (int index1 = 0; index1 < dataset.size(); index1 = index2)
		{			
			index2 = index1 + foldDim;
			if( surplus != 0) {
				++index2;
				--surplus;
			}
			
			folds.add( dataset.subList(index1, index2));
		}
		
		return folds;
	}
	
	
	/**
	 * Merge all the folds in one list
	 * excluding the one indicated by
	 * the index.
	 * 
	 * @param folds list of folds
	 * @param index index of the fold to exclude
	 * @return merged list
	 */
	private ArrayList<ITrajectory<TimeType>> joinOtherFolds(List<List<ITrajectory<TimeType>>> folds, int index)
	{	
		ArrayList<ITrajectory<TimeType>> joinedList = new ArrayList<ITrajectory<TimeType>>(folds.get(0).size() * (folds.size()-1));

		for ( int i = 0; i < folds.size(); ++i)
			if (i != index)
				joinedList.addAll(folds.get( i));

		return joinedList;
	}

} public class HoldOut<TimeType extends Number & Comparable<TimeType>,NodeType extends INode,TM extends ICTClassifier<TimeType,NodeType>,SingPType extends ISingleRunPerformances<TimeType>>
		extends
		BaseValidationMethod<TimeType, NodeType, TM, SingPType>
		implements
		IValidationMethod<TimeType, NodeType, TM, SingPType> {

	private ISingleRunPerformancesFactory<TimeType, SingPType> singleRunerformancesFactory;
	private double cut;
	private List<ITrajectory<TimeType>> trainingSet, testSet;
	
	
	/**
	 * Base constructor.
	 * Fixed cut = 0.7: 70% training set,
	 * 30% test set.
	 * 
	 * @param singleRunerformancesFactory factory to generate the single run performances
	 */
	public HoldOut(ISingleRunPerformancesFactory<TimeType, SingPType> singleRunerformancesFactory) {
		
		this(singleRunerformancesFactory, 0.7);
	}
	
	/**
	 * Constructor.
	 * Allow to set test set and training set.
	 * If training set and test set are set, the
	 * dataset in the validation method call is
	 * ignored.
	 * 
	 * @param singleRunerformancesFactory factory to generate the single run performances
	 * @param trainingSet training set to use
	 * @param testSet test set to use
	 */
	public HoldOut(ISingleRunPerformancesFactory<TimeType, SingPType> singleRunerformancesFactory, List<ITrajectory<TimeType>> trainingSet, List<ITrajectory<TimeType>> testSet) {
		
		this(singleRunerformancesFactory, 0.7);
		
		this.setTrainingSet(trainingSet);
		this.setTestSet(testSet);
	}
	
	/**
	 * Constructor.
	 * 
	 * @param singleRunerformancesFactory factory to generate the single run performances
	 * @param cut cut value that fix the division between training set and test set (i.e. 0.7 => training set 70%, test set 30%)
	 */
	public HoldOut(ISingleRunPerformancesFactory<TimeType, SingPType> singleRunerformancesFactory, double cut) {
		
		if( singleRunerformancesFactory == null)
			throw new IllegalArgumentException("Error: null value for the performances factory parameter");
		
		this.trainingSet = null;
		this.testSet = null;
		this.singleRunerformancesFactory = singleRunerformancesFactory;
		this.cut = cut;
		
		super.setVerbose( false);
		super.setPrintSSFlag( false);
		super.setClusterInSampleFlag( false);
	}
	
	/**
	 * Set the factory to generate the single
	 * run performances.
	 * 
	 * @param singleRunerformancesFactory factory to generate the single run performances
	 */
	public void setPerformancesFactory(ISingleRunPerformancesFactory<TimeType, SingPType> singleRunerformancesFactory) {
		
		if( singleRunerformancesFactory == null)
			throw new IllegalArgumentException("Error: null value for the performances factory parameter");
		
		this.singleRunerformancesFactory = singleRunerformancesFactory;
	}
	
	/**
	 * Reset training set and test set.
	 */
	public void resetDatasets() {
		
		this.trainingSet = null;
		this.testSet = null;
	}
	
	/**
	 * Set the training set.
	 * 
	 * @param dataset dataset to set
	 */
	public void setTrainingSet( List<ITrajectory<TimeType>> dataset) {
		
		if( dataset == null)
			throw new IllegalArgumentException("Error: null dataset argument");
		if( dataset.size() < 1)
			throw new IllegalArgumentException("Error: too small dataset argument");
		
		this.trainingSet = dataset;
	}
	
	/**
	 * Set the test set.
	 * 
	 * @param dataset dataset to set
	 */
	public void setTestSet( List<ITrajectory<TimeType>> dataset) {
		
		if( dataset == null)
			throw new IllegalArgumentException("Error: null dataset argument");
		if( dataset.size() < 1)
			throw new IllegalArgumentException("Error: too small dataset argument");
		
		this.testSet = dataset;
	}
	
	
	
	/* (non-Javadoc)
	 * @see CTBNToolkit.tests.validators.IValidationMethod#validate(CTBNToolkit.IModel, CTBNToolkit.ILearningAlgorithm, CTBNToolkit.IClassifyAlgorithm, java.util.List)
	 */
	@Override
	public SingPType validate(
			TM mdl,
			ILearningAlgorithm<TimeType, NodeType> learnAlgo,
			IClassifyAlgorithm<TimeType, NodeType> infAlgo,
			List<ITrajectory<TimeType>> dataset) {

		SingPType results = null;
		
		if( this.trainingSet != null && this.testSet != null) {			// if predefined dataset were set
			results = this.validate(mdl, learnAlgo, infAlgo);
		} else {
			// Dataset cut												// if predefined dataset were not set
			int cutIndex = (int) Math.round(this.cut * dataset.size());
			
			super.verbosePrint("\n\nHold out schema (cut = " + this.cut + ")\n");
			super.verbosePrint("Training set uses the first " + cutIndex + " trajectories\n");
			super.verbosePrint("Test set uses the last " + (dataset.size()-cutIndex-1) + " trajectories\n\n");
			
			this.trainingSet = dataset.subList(0, cutIndex);
			this.testSet = dataset.subList(cutIndex, dataset.size());
			
			// Validation
			results = this.validate(mdl, learnAlgo, infAlgo);
			
			// Resetting of the state of the datasets
			this.resetDatasets();
		}
		
		return results;
	}

	
	/**
	 * Validate the algorithms using the training
	 * set and the test set previously defined.
	 * Learn the model in input using the specified
	 * learning algorithm. Then make inferences and
	 * returns the performances.
	 * 
	 * @param mdl model to learn
	 * @param learnAlgo learning algorithm
	 * @param infAlgo inference algorithm
	 * @return performances of the experimentation
	 */
	@SuppressWarnings("unchecked")
	public SingPType validate(
			TM mdl,
			ILearningAlgorithm<TimeType, NodeType> learnAlgo,
			IClassifyAlgorithm<TimeType, NodeType> infAlgo) {
		
		if( this.trainingSet == null)
			throw new RuntimeException("Error: training set not setted");
		if( this.testSet == null)
			throw new RuntimeException("Error: test set not setted");

		// Initialization
		NodeIndexing nodeIndexing = mdl.getNodeIndexing();
		DiscreteNode classNode = mdl.getClassNode();
		Map<Integer,String> classIndexToValue = new TreeMap<Integer,String>();
		for(int i = 0; i < classNode.getStatesNumber(); ++i)
			classIndexToValue.put(i, classNode.getStateName(i));
		// Performance declaration
		SingPType outOfSamplePerformances = this.singleRunerformancesFactory.newInstance(classIndexToValue);
		SingPType inSamplePerformances = null;		
		
		// Learning
		super.verbosePrint("Learning (training set of " + this.trainingSet.size() + " trajectories):\n");
		
		long startTime = System.currentTimeMillis();
		ILearningResults classificationResult = learnAlgo.learn(mdl, this.trainingSet);
		double learningTime = (System.currentTimeMillis() - startTime) / 1000.0;
		// In sample performances
		if( super.getClusterInSampleFlag() && ClusteringResults.class.isAssignableFrom(classificationResult.getClass())) {
		
			// Generate in sample performances
			inSamplePerformances = this.singleRunerformancesFactory.newInstance(classIndexToValue);
			inSamplePerformances.addResults((( ClusteringResults<TimeType>)classificationResult).getClusterizedTrajectories() );
			inSamplePerformances.calculateFinalResults(true);
		}
		// Sufficient statistics
		printSufficientStatistics(classificationResult, mdl);
		outOfSamplePerformances.setLearnedModel( mdl);
		outOfSamplePerformances.setLearningTime( learningTime);
		
		super.verbosePrint("Model learnt in " + learningTime + " seconds\n\n");
		
		
		// Testing
		double prob;
		String classPrediction;
		String trueClass;
		super.verbosePrint("Testing (test set of " + this.testSet.size() + "trajectories):\n");
		for (int iTrj = 0; iTrj < this.testSet.size(); ++iTrj)
		{
			try
			{
				super.verbosePrint("Inference on " + this.testSet.get(iTrj).getName() + " test set trajectory\n");
				startTime = System.currentTimeMillis();
				IClassificationResult<TimeType> res = mdl.classify( infAlgo, this.testSet.get( iTrj));
				double inferenceTime =  (System.currentTimeMillis() - startTime) / 1000.0;
				
				classPrediction = res.getClassification();
				trueClass = res.getNodeValue(0, nodeIndexing.getClassIndex());
				if( infAlgo.probabilityFlag()) {
					prob = res.getProbability(classPrediction);
					super.verbosePrint( this.testSet.get( iTrj).getName() + ": True Class: " + trueClass + ", Predicted: " + classPrediction + ", Probability: " + prob + ", Inference time: " + inferenceTime + "\n");
				} else
					super.verbosePrint( this.testSet.get( iTrj).getName() + ": True Class: " + trueClass + ", Predicted: " + classPrediction + ", Inference time: " + inferenceTime + "\n");
				
				outOfSamplePerformances.addResult( res, inferenceTime);
			}
			catch(Exception e)
			{
				System.out.println(e.getMessage());
			}
		}
		
		if( super.getClusterInSampleFlag() && IExternalClusteringPerformances.class.isAssignableFrom( outOfSamplePerformances.getClass()))
			(( IExternalClusteringPerformances) outOfSamplePerformances).setInSamplePerformances(
					(IExternalClusteringPerformances) inSamplePerformances);
		
		return outOfSamplePerformances;
	}
} public interface IValidationMethod<TimeType extends Number & Comparable<TimeType>,NodeType extends INode,TM extends IModel<TimeType, NodeType>,PType extends IPerformances> {
	
	/**
	 * Flag to activate or disable
	 * the verbose comments option.
	 * 
	 * @param verbose true to get a full comment, false to get just the essential.
	 */
	public void setVerbose(boolean verbose);
	
	/**
	 * Return the state of the
	 * verbose flag.
	 * 
	 * @return verbose flag state.
	 */
	public boolean getVerbose();
	
	/**
	 * Validate the algorithms.
	 * Learn the model in input using the
	 * specified learning algorithm. Then
	 * make inferences and returns the
	 * performances.
	 * Performances depends on the used
	 * validation method.  
	 * 
	 * @param mdl model to learn
	 * @param learnAlgo learning algorithm
	 * @param infAlgo inference algorithm
	 * @param dataset dataset to make the experimental campaign
	 * @return performances of the experimentation
	 */
	public PType validate(TM mdl,
			ILearningAlgorithm<TimeType, NodeType> learnAlgo,
			IClassifyAlgorithm<TimeType, NodeType> infAlgo,
			List<ITrajectory<TimeType>> dataset);

}
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Method;
import java.text.SimpleDateFormat;
import java.util.*;
import CTBNCToolkit.BinaryDecider;
import CTBNCToolkit.CTBNCClassifyAlgorithm;
import CTBNCToolkit.CTBNCLocalStructuralLearning;
import CTBNCToolkit.CTBNCParameterLLAlgorithm;
import CTBNCToolkit.CTBNClassifier;
import CTBNCToolkit.CTDiscreteNode;
import CTBNCToolkit.ICTClassifier;
import CTBNCToolkit.IClassifyDecider;
import CTBNCToolkit.ILearningAlgorithm;
import CTBNCToolkit.ITrajectory;
import CTBNCToolkit.MultipleCTBNC;
import CTBNCToolkit.MultipleCTBNCLearningAlgorithm;
import CTBNCToolkit.clustering.CTBNClusteringParametersLLAlgorithm;
import CTBNCToolkit.clustering.StandardStopCriterion;
import CTBNCToolkit.optimization.CLLHillClimbingFactory;
import CTBNCToolkit.optimization.CTBNCHillClimbingIndividual;
import CTBNCToolkit.optimization.ICTBNCHillClimbingFactory;
import CTBNCToolkit.optimization.LLHillClimbingFactory;
import CTBNCToolkit.performances.ClassificationStandardPerformances;
import CTBNCToolkit.performances.ClassificationStandardPerformancesFactory;
import CTBNCToolkit.performances.ClusteringExternalPerformances;
import CTBNCToolkit.performances.ClusteringExternalPerformancesFactory;
import CTBNCToolkit.performances.IPerformances;
import CTBNCToolkit.performances.MicroMacroClassificationPerformances;
import CTBNCToolkit.performances.MicroMacroClassificationPerformancesFactory;
import CTBNCToolkit.tests.CTBNCTestFactory;
import CTBNCToolkit.tests.GenericTestResults;
import CTBNCToolkit.tests.ITestFactory;
import CTBNCToolkit.tests.ITestResults;
import CTBNCToolkit.tests.validators.ClusteringInSample;
import CTBNCToolkit.tests.validators.CrossValidation;
import CTBNCToolkit.tests.validators.HoldOut;
import CTBNCToolkit.tests.validators.IValidationMethod;
	}
	char fileSeparator = ',';
	
	
	
	/**
	 * Set the column name of the class.
	 * Calling modifier: "--className".
	 *
	 * @param clmName name of the column
	 */
	public void setClassColumnName( LinkedList<String> clmName) {
	
		if( clmName.size() != 1) {
			System.err.println("Class column name parsing error. One and only one argument is required for --className modifier. For more information type --help.");
			System.exit(1);
		}
		
		this.classColumnName = clmName.get(0);
	}
	String classColumnName = "class"; 
										
	
	/**
	 * Set the column name of the time.
	 * Calling modifier: "--timeName".
	 *
	 * @param clmName name of the column
	 */
	public void setTimeColumnName( LinkedList<String> clmName) {
	
		if( clmName.size() != 1) {
			System.err.println("Time column name parsing error. One and only one argument is required for --timeName modifier. For more information type --help.");
			System.exit(1);
		}
		
		this.timeColumnName = clmName.get(0);
	}
	String timeColumnName = "t";
	
	
	/**
	 * Set the column name of the trajectory separator
	 * column.
	 * Calling modifier: "--trjSeparator".
	 *
	 * @param clmName name of the column
	 */
	public void setTrjSeparator( LinkedList<String> clmName) {
	
		if( clmName.size() != 1) {
			System.err.println("Trajectories separator column name parsing error. One and only one argument is required for --trjSeparator modifier. For more information type --help.");
			System.exit(1);
		}
		
		this.trjSeparatorColumnName = clmName.get(0);
	}
	String trjSeparatorColumnName = null;
	
	
	/**
	 * Set the names of the columns that represent
	 * a variable in the models.
	 * Calling modifier: "--validColumns".
	 *
	 * @param clmsNames names of the columns
	 */
	public void setValidColumns( LinkedList<String> clmsNames) {
	
		if( clmsNames.size() < 1) {
			System.err.println("Valid columns names parsing error. At least one argument is required for --validColumns modifier. For more information type --help.");
			System.exit(1);
		}
		
		for( int i = 0; i < clmsNames.size(); ++i) {
			
			this.validColumns.add( clmsNames.get(i));
		}
	}	
	Set<String> validColumns = new TreeSet<String>();;


	/**
	 * Set the time factor to use to scale the time
	 * of the trajectories.
	 * Calling modifier: "--timeFactor".
	 *
	 * @param tFactor time factor
	 */
	public void setTimeFactor( LinkedList<String> tFactor) {
	
		if( tFactor.size() != 1) {
			System.err.println("Time factor parsing error. One and only one argument is required for --timeFactor modifier. For more information type --help.");
			System.exit(1);
		}
		
		try {
			this.timeFactor = Double.parseDouble( tFactor.get(0));
		} catch( Exception e) {
			System.err.println("Time factor parsing error. Error in parsing double argument. For more information type --help.");
			System.exit(1);
		}
	}
	Double timeFactor = 1.0;
	
	
	/**
	 * Set the time threshold of the binary class decider.
	 * Calling modifier: "--bThreshold".
	 *
	 * @param threshold time factor
	 */
	public void setBinaryDecider( LinkedList<String> threshold) {
	
		if( threshold.size() != 1) {
			System.err.println("Binary class decision threshold parsing error. One and only one argument is required for --bThreshold modifier. For more information type --help.");
			System.exit(1);
		}
		
		try {
			this.classDecider = new BinaryDecider( Double.parseDouble( threshold.get(0)));