public class Configuration {
	public enum Sections {
		MACROSCOPY, MICROSCOPY, CYTOPATHOLOGY, LIQUID_CYTOPATHOLOGY, CONCLUSION, OTHERS;
	}

	public enum Criteria {
		ONE_REPORT_ONE_REGISTRY, MANY_REPORT_ONE_REGISTRY, MANY_REPORT_MANY_REGISTRY;
	}

	public enum Classifiers {
		NAIVE, BAYES, BAYES_NET, BERNOULLI, SVM;
	}

	public enum WeightingSchemes {
		TF, IDF, TFIDF;
	}

	public enum SmoothingTechniques {
		ADD_ONE, ADD_ALPHA, GOOD_TURING;
	}

	public enum Sources {
		DCE_REPORT, SIGH_REPORT, TOTLAUD_REPORT, ALL;
	}

	public enum Stemmers {
		// TODO test with PTStemmer V1 available at Maven repositories
		NONE, ORENGO, PORTER, SAVOY, COGROO;
	}

	public enum SentenceDetectors {
		NONE, OPENNLP, COGROO;
	}

	public enum Tokenizers {
		WORD, ALPHABETIC, OPENNLP, COGROO;
	}

	public enum Chunkers {
		NONE, COGROO;
	}

	public enum Targets {
		MORPHOLOGY, MORPHOLOGY_GROUP, TOPOGRAPHY, TOPOGRAPHY_CATEGORY, TOPOGRAPHY_GROUP;
	}

	public enum MetastasisStatus {
		ALL, NONM1;
	}

	private Set<Sections> sections;
	private Criteria criteria;
	private int minReports = 1;
	private Classifiers classifier;
	private WeightingSchemes weightScheme;
	private SmoothingTechniques smoothing;
	private double svmCostParameter = 1;
	private Sources source;
	private boolean stoplist;
	private int ngrams = 1;
	private SentenceDetectors sentenceDetector;
	private Tokenizers tokenizer;
	private Stemmers stemmer;
	private Chunkers chunker;
	private Targets target;
	private MetastasisStatus meta;
	private int patientYear = -1;

	public Configuration(Set<Sections> sections, Criteria criteria, Classifiers classifier,
			WeightingSchemes weightScheme, SmoothingTechniques smoothing, double svmCParameter, Sources source,
			boolean stoplist, int ngrams, SentenceDetectors sentenceDetector, Tokenizers tokenizer, Stemmers stemmer,
			Chunkers chunker, Targets target, int minReports, int year, MetastasisStatus meta) {
		super();
		this.sections = sections;
		this.criteria = criteria;
		if (criteria == Criteria.MANY_REPORT_ONE_REGISTRY)
			this.minReports = minReports;
		this.classifier = classifier;
		this.weightScheme = weightScheme;
		this.smoothing = smoothing;
		this.svmCostParameter = svmCParameter;
		this.source = source;
		this.stoplist = stoplist;
		this.ngrams = ngrams;
		this.sentenceDetector = sentenceDetector;
		this.tokenizer = tokenizer;
		this.stemmer = stemmer;
		this.chunker = chunker;
		this.target = target;
		this.patientYear = year;
		this.meta = meta;
	}

	public Set<Sections> getSections() {
		return sections;
	}

	public Criteria getCriteria() {
		return criteria;
	}

	public int getMinReports() {
		return minReports;
	}

	public Classifiers getClassifier() {
		return classifier;
	}

	public WeightingSchemes getWeightScheme() {
		return weightScheme;
	}

	public SmoothingTechniques getSmoothing() {
		return smoothing;
	}

	public double getSvmCostParameter() {
		return svmCostParameter;
	}

	public Sources getSource() {
		return source;
	}

	public boolean getStoplist() {
		return stoplist;
	}
	
	public int getNGrams() {
		return ngrams;
	}

	public SentenceDetectors getSentenceDetector() {
		return sentenceDetector;
	}

	public Tokenizers getTokenizer() {
		return tokenizer;
	}

	public Stemmers getStemmer() {
		return stemmer;
	}

	public Chunkers getChunker() {
		return chunker;
	}

	public Targets getTarget() {
		return target;
	}

	public MetastasisStatus getMetastasisStatus() {
		return meta;
	}

	public int getPatientYear() {
		return patientYear;
	}

	public String getInstanceDependentStringRepresentation() {
		return (Arrays.toString(sections.toArray()) + "-" + criteria + "-" + minReports + "-" + source + "-"
				+ sentenceDetector + "-" + tokenizer + "Tokenizer-" + stemmer + "Stemmer-" + chunker + "Chunker-"
				+ target + "-" + meta + "-" + patientYear).toLowerCase();
	}

	public String getStringRepresentation() {
		return (Arrays.toString(sections.toArray()) + "-" + criteria + "-" + minReports + "-" + classifier + "-"
				+ weightScheme + "-" + smoothing + "-" + svmCostParameter + "-" + Constants.ALPHA + "-" + source + "-"
				+ stoplist + "-" + ngrams + "-" + sentenceDetector + "-" + tokenizer + "-" + stemmer + "-" + chunker + "-" + target
				+ "-" + meta + "-" + patientYear).toLowerCase();
	}

} public class Constants {

	public static final int THREADS = 1;
	public static final int CONNECTIONS = THREADS * 2;

	// TODO change to something better, maybe setting automatically on the main method.
	public static final boolean RECREATE_DB = true;
	public static final int BATCH_SIZE = 100;

	// TODO move to a config file
	/** Root directory where reference tables are located. */
	public static final String REF_DIR = "src/ref/";

	/** Directory containing large data files. */
	public static final String DATA_DIR = "src/data/";

	// TODO defined in models.xml for CoGrOO implementation. May not be necessary.
	/** Directory containing models (e.g. OpenNLP models). */
	public static final String MODEL_DIR = "src/models/";

	/** Defines if a cache of instances is used or not. */
	public static final boolean CACHE = false;

	/** Indicates wheter to generate stats or not. */
	public static final boolean STATS = false;

	public static final float TEST_RATIO = 0.1f;
	// public static final int THRESHOLD = 10;

	// TODO move to a class smoothing or something
	public static final double ALPHA = 0.5d;

	public static Configuration CONFIG;

} public class DAOFactory {
	// TODO migrate to JPA Persistence:
	// http://docs.jboss.org/hibernate/stable/entitymanager/reference/en/html_single/
	private static final int CONNECTIONS = Constants.CONNECTIONS;

	private static final DAOFactory instance = new DAOFactory();

	public static DAOFactory getDAOFactory() {
		return instance;
	}

	private SessionFactory sessionFactory;
	private Session[] session = new Session[CONNECTIONS];

	private TopographyDAO[] topographyDAO = new TopographyDAO[CONNECTIONS];
	private TopographyCategoryDAO[] topographyCategoryDAO = new TopographyCategoryDAO[CONNECTIONS];
	private TopographyGroupDAO[] topographyGroupDAO = new TopographyGroupDAO[CONNECTIONS];

	private MorphologyDAO[] morphologyDAO = new MorphologyDAO[CONNECTIONS];
	private MorphologyGroupDAO[] morphologyGroupDAO = new MorphologyGroupDAO[CONNECTIONS];

	private DceDAO[] dceDAO = new DceDAO[CONNECTIONS];
	private SighDAO[] sighDAO = new SighDAO[CONNECTIONS];
	private RhcDAO[] rhcDAO = new RhcDAO[CONNECTIONS];
	private PatientDAO[] patientDAO = new PatientDAO[CONNECTIONS];

	private DAOFactory() {
		AnnotationConfiguration cfg = new AnnotationConfiguration();

		cfg.addAnnotatedClass(Topography.class);
		cfg.addAnnotatedClass(TopographyCategory.class);
		cfg.addAnnotatedClass(TopographyGroup.class);

		cfg.addAnnotatedClass(Morphology.class);
		cfg.addAnnotatedClass(MorphologyGroup.class);

		cfg.addAnnotatedClass(RHC.class);
		cfg.addAnnotatedClass(DCE.class);
		cfg.addAnnotatedClass(SighReport.class);
		cfg.addAnnotatedClass(Patient.class);
		cfg.configure();

		// TODO set fetch size according to Constants.BATCH_SIZE

		if (Constants.RECREATE_DB) {
			SchemaExport se = new SchemaExport(cfg);
			se.create(true, true);
		}

		this.sessionFactory = cfg.buildSessionFactory();

		for (int i = 0; i < CONNECTIONS; i++)
			session[i] = sessionFactory.openSession();

		for (int i = 0; i < CONNECTIONS; i++)
			topographyDAO[i] = new TopographyDAO(session[i]);
		for (int i = 0; i < CONNECTIONS; i++)
			topographyCategoryDAO[i] = new TopographyCategoryDAO(session[i]);
		for (int i = 0; i < CONNECTIONS; i++)
			topographyGroupDAO[i] = new TopographyGroupDAO(session[i]);

		for (int i = 0; i < CONNECTIONS; i++)
			morphologyDAO[i] = new MorphologyDAO(session[i]);
		for (int i = 0; i < CONNECTIONS; i++)
			morphologyGroupDAO[i] = new MorphologyGroupDAO(session[i]);

		for (int i = 0; i < CONNECTIONS; i++)
			dceDAO[i] = new DceDAO(session[i]);
		for (int i = 0; i < CONNECTIONS; i++)
			sighDAO[i] = new SighDAO(session[i]);
		for (int i = 0; i < CONNECTIONS; i++)
			rhcDAO[i] = new RhcDAO(session[i]);
		for (int i = 0; i < CONNECTIONS; i++)
			patientDAO[i] = new PatientDAO(session[i]);
	}

	int sess = 0;

	private int getNextSession() {
		sess++;
		if (sess >= CONNECTIONS)
			sess = 0;
		return sess;
	}

	public ClassifiableDAO getIcdClassDAO() {
		switch (Constants.CONFIG.getTarget()) {
		case MORPHOLOGY:
			return (ClassifiableDAO) getMorphologyDAO();
		case MORPHOLOGY_GROUP:
			return (ClassifiableDAO) getMorphologyGroupDAO();
		case TOPOGRAPHY:
			return (ClassifiableDAO) getTopographyDAO();
		case TOPOGRAPHY_CATEGORY:
			return (ClassifiableDAO) getTopographyCategoryDAO();
		case TOPOGRAPHY_GROUP:
			return (ClassifiableDAO) getTopographyGroupDAO();
		default:
			return null;
		}
	}

	public TopographyDAO getTopographyDAO() {
		return topographyDAO[getNextSession()];
	}

	public TopographyCategoryDAO getTopographyCategoryDAO() {
		return topographyCategoryDAO[getNextSession()];
	}

	public TopographyGroupDAO getTopographyGroupDAO() {
		return topographyGroupDAO[getNextSession()];
	}

	public MorphologyDAO getMorphologyDAO() {
		return morphologyDAO[getNextSession()];
	}

	public MorphologyGroupDAO getMorphologyGroupDAO() {
		return morphologyGroupDAO[getNextSession()];
	}

	public DceDAO getDceDAO() {
		return dceDAO[getNextSession()];
	}

	public SighDAO getSighDAO() {
		return sighDAO[getNextSession()];
	}

	public RhcDAO getRhcDAO() {
		return rhcDAO[getNextSession()];
	}

	public PatientDAO getPatientDAO() {
		return patientDAO[getNextSession()];
	}

	public void close() {
		for (Session s : session)
			s.close();
		sessionFactory.close();
	}

}
 public class Stats {

	private static final Logger LOG = Logger.getLogger(Stats.class);

	private int numClasses = 0;
	private int numDocs = 0;
	private int numPatients = 0;

	private int numSentences = 0;
	private int numTokens = 0;

	private int numUpperCaseChars = 0;
	private int numLowerCaseChars = 0;
	private int numNumbers = 0;
	private int numSymbols = 0;

	private Map<String, Integer> numPatientsPerClass = new HashMap<String, Integer>();
	private Map<String, Integer> numDocsPerClass = new HashMap<String, Integer>();

	public void setNumClasses(int numClasses) {
		this.numClasses = numClasses;
	}

	public void incNumDocs(int numDocs) {
		this.numDocs += numDocs;
	}

	public void setNumPatients(int numPatients) {
		this.numPatients = numPatients;
	}

	public void incNumSentences() {
		this.numSentences += 1;
	}

	public void incNumTokens() {
		this.numTokens += 1;
	}

	public void incNumPatientsPerClass(String clazz, int numPatients) {
		if (numPatientsPerClass.containsKey(clazz)) {
			Integer value = numPatientsPerClass.get(clazz);
			value += numPatients;
			numPatientsPerClass.put(clazz, value);
		} else {
			numPatientsPerClass.put(clazz, numPatients);
		}

	}

	public void incNumDocsPerClass(String clazz, int numDocs) {
		if (numDocsPerClass.containsKey(clazz)) {
			Integer value = numDocsPerClass.get(clazz);
			value += numDocs;
			numDocsPerClass.put(clazz, value);
		} else {
			numDocsPerClass.put(clazz, numDocs);
		}
	}

	public void countCharTypes(String text) {
		int lower = 0, upper = 0, symbol = 0, number = 0;
		for (int i = 0; i < text.length(); i++) {
			char c = text.charAt(i);
			if (Character.isLowerCase(c))
				lower++;
			else if (Character.isUpperCase(c))
				upper++;
			else if (Character.isDigit(c))
				number++;
			else if (!Character.isWhitespace(c)) {
				//LOG.trace("Symbol char found: " + c);
				symbol++;
			}
		}

		this.numLowerCaseChars += lower;
		this.numUpperCaseChars += upper;
		this.numSymbols += symbol;
		this.numNumbers += number;
	}

	public int getNumClasses() {
		return numClasses;
	}

	public int getNumDocs() {
		return numDocs;
	}

	public int getNumPatients() {
		return numPatients;
	}

	public int getNumSentences() {
		return numSentences;
	}

	public int getNumTokens() {
		return numTokens;
	}

	public int getNumChars() {
		return numLowerCaseChars + numUpperCaseChars + numSymbols + numNumbers;
	}
	
	public float getRatioCharSentence() {
		return getNumChars() / (float) getNumSentences();
	}
	
	public float getRatioTokenSentence() {
		return getNumTokens() / (float) getNumSentences();
	}

	public float getRatioUpperLower() {
		return numUpperCaseChars / (float)(numLowerCaseChars + numUpperCaseChars);
	}

	public float getRatioNumbersChars() {
		return numNumbers / (float)getNumChars();
	}

	public String getNumDocsPerClassAsTable() {
		StringBuilder sb = new StringBuilder();
		for (Map.Entry<String, Integer> entry : numDocsPerClass.entrySet()) {
			sb.append(entry.getKey());
			sb.append("\t");
			sb.append(entry.getValue());
			sb.append("\n");
		}
		return sb.toString();
	}

	public String getNumPatientsPerClassAsTable() {
		StringBuilder sb = new StringBuilder();
		for (Map.Entry<String, Integer> entry : numPatientsPerClass.entrySet()) {
			sb.append(entry.getKey());
			sb.append("\t");
			sb.append(entry.getValue());
			sb.append("\n");
		}
		return sb.toString();
	}

}
 public class AddAlphaNaiveBayesMultinomial extends NaiveBayesMultinomial {

	/**
	 * 
	 */
	private static final long serialVersionUID = -8331109311218425776L;

	protected double alpha = 1.0d;

	public void setAlpha(double alpha) {
		// TODO check != 0
		this.alpha = alpha;
	}

	/**
	 * Generates the classifier.
	 * 
	 * @param instances
	 *            set of instances serving as training data
	 * @throws Exception
	 *             if the classifier has not been generated successfully
	 */
	public void buildClassifier(Instances instances) throws Exception {
		// can classifier handle the data?
		getCapabilities().testWithFail(instances);

		// remove instances with missing class
		instances = new Instances(instances);
		instances.deleteWithMissingClass();

		m_headerInfo = new Instances(instances, 0);
		m_numClasses = instances.numClasses();
		m_numAttributes = instances.numAttributes();
		m_probOfWordGivenClass = new double[m_numClasses][];

		/*
		 * initialising the matrix of word counts NOTE: Add-alpha estimator
		 * introduced in case a word that does not appear for a class in the
		 * training set does so for the test set
		 */
		for (int c = 0; c < m_numClasses; c++) {
			m_probOfWordGivenClass[c] = new double[m_numAttributes];
			for (int att = 0; att < m_numAttributes; att++) {
				m_probOfWordGivenClass[c][att] = alpha;
			}
		}

		// enumerate through the instances
		Instance instance;
		int classIndex;
		double numOccurences;
		double[] docsPerClass = new double[m_numClasses];
		double[] wordsPerClass = new double[m_numClasses];

		java.util.Enumeration enumInsts = instances.enumerateInstances();
		while (enumInsts.hasMoreElements()) {
			instance = (Instance) enumInsts.nextElement();
			classIndex = (int) instance.value(instance.classIndex());
			docsPerClass[classIndex] += instance.weight();

			for (int a = 0; a < instance.numValues(); a++)
				if (instance.index(a) != instance.classIndex()) {
					if (!instance.isMissing(a)) {
						numOccurences = instance.valueSparse(a)
								* instance.weight();
						if (numOccurences < 0)
							throw new Exception(
									"Numeric attribute values must all be greater or equal to zero.");
						wordsPerClass[classIndex] += numOccurences;
						m_probOfWordGivenClass[classIndex][instance.index(a)] += numOccurences;
					}
				}
		}

		/*
		 * normalising probOfWordGivenClass values and saving each value as the
		 * log of each value
		 */
		// TODO why the decrement of 1 on m_numAttributes?
		for (int c = 0; c < m_numClasses; c++)
			for (int v = 0; v < m_numAttributes; v++)
				m_probOfWordGivenClass[c][v] = Math
						.log(m_probOfWordGivenClass[c][v]
								/ (wordsPerClass[c] + alpha
										* (m_numAttributes - 1)));

		/*
		 * calculating Pr(H) NOTE: Add-alpha estimator introduced in case a
		 * class does not get mentioned in the set of training instances
		 */
		final double numDocs = instances.sumOfWeights() + alpha * m_numClasses;
		m_probOfClass = new double[m_numClasses];
		for (int h = 0; h < m_numClasses; h++)
			m_probOfClass[h] = (double) (docsPerClass[h] + alpha) / numDocs;
	}
} public class GoodTuringNaiveBayesMultinomial extends NaiveBayesMultinomial {

	protected int m_threshold = 2;

	/**
	 * 
	 */
	private static final long serialVersionUID = -8331109311228425776L;

	public void setThreshold(int k) {
		this.m_threshold = k;
	}

	/**
	 * Generates the classifier.
	 * 
	 * @param instances
	 *            set of instances serving as training data
	 * @throws Exception
	 *             if the classifier has not been generated successfully
	 */
	public void buildClassifier(Instances instances) throws Exception {
		// can classifier handle the data?
		getCapabilities().testWithFail(instances);

		// remove instances with missing class
		instances = new Instances(instances);
		instances.deleteWithMissingClass();

		m_headerInfo = new Instances(instances, 0);
		m_numClasses = instances.numClasses();
		m_numAttributes = instances.numAttributes();
		m_probOfWordGivenClass = new double[m_numClasses][m_numAttributes];

		// enumerate through the instances
		Instance instance;
		int classIndex;
		double numOccurences;
		double[] docsPerClass = new double[m_numClasses];
		double[] wordsPerClass = new double[m_numClasses];

		java.util.Enumeration enumInsts = instances.enumerateInstances();
		while (enumInsts.hasMoreElements()) {
			instance = (Instance) enumInsts.nextElement();
			classIndex = (int) instance.value(instance.classIndex());
			docsPerClass[classIndex] += instance.weight();

			for (int a = 0; a < instance.numValues(); a++)
				if (instance.index(a) != instance.classIndex()) {
					if (!instance.isMissing(a)) {
						numOccurences = instance.valueSparse(a)
								* instance.weight();
						if (numOccurences < 0)
							throw new Exception(
									"Numeric attribute values must all be greater or equal to zero.");
						wordsPerClass[classIndex] += numOccurences;
						m_probOfWordGivenClass[classIndex][instance.index(a)] += numOccurences;
					}
				}
		}

		/*
		 * normalising probOfWordGivenClass values and saving each value as the
		 * log of each value
		 */
		double sumOfFrequencies;
		for (int c = 0; c < m_numClasses; c++) {
			m_probOfWordGivenClass[c] = adjustFrequencies(m_probOfWordGivenClass[c]);
			sumOfFrequencies = sumOfFrequencies(m_probOfWordGivenClass[c]);
			for (int v = 0; v < m_numAttributes; v++) {
				m_probOfWordGivenClass[c][v] = Math
						.log(m_probOfWordGivenClass[c][v] / sumOfFrequencies);
				if (m_probOfWordGivenClass[c][v] != m_probOfWordGivenClass[c][v]) {// NaN
					System.out.println(c + ":" + v + "=" + m_probOfWordGivenClass[c][v]);
				}
			}
		}

		/*
		 * calculating Pr(H) NOTE: Good-Turing estimator introduced in case a
		 * class does not get mentioned in the set of training instances
		 */
		docsPerClass = adjustFrequencies(docsPerClass);
		sumOfFrequencies = sumOfFrequencies(docsPerClass);
		m_probOfClass = new double[m_numClasses];
		for (int h = 0; h < m_numClasses; h++)
			m_probOfClass[h] = (double) (docsPerClass[h]) / sumOfFrequencies;

	}

	protected double[] adjustFrequencies(double[] frequency) {
		double[] frequencyOfFrequency = new double[m_threshold + 1];
		double[] adjustedFrequency = new double[frequency.length];
		for (int h = 0; h < frequency.length; h++)
			if (frequency[h] <= m_threshold)
				frequencyOfFrequency[(int) frequency[h]] += 1;
		for (int h = 0; h < frequency.length; h++) {
			int r = (int) frequency[h];
			adjustedFrequency[h] = (r < m_threshold) ? (frequencyOfFrequency[r + 1]
					* (r + 1) / frequencyOfFrequency[r])
					: frequency[h];
		}
		return adjustedFrequency;
	}

	protected double sumOfFrequencies(double[] frequency) {
		double numDocs = 0;
		for (int h = 0; h < m_numClasses; h++)
			numDocs += frequency[h];
		return numDocs != 0 ? numDocs : Double.MIN_VALUE;
	}

} public class NaiveClassifier extends weka.classifiers.Classifier {

	/**
	 * 
	 */
	private static final long serialVersionUID = 4729890351185372478L;

	private static final Logger LOG = Logger.getLogger(NaiveClassifier.class);

	/** number of class values */
	protected int m_numClasses;

	private int mostFrequentIndex;


	@Override
	public void buildClassifier(Instances data) throws Exception {
		m_numClasses = data.numClasses();

		Instance instance;
		int classIndex;
		double[] docsPerClass = new double[m_numClasses];

		java.util.Enumeration enumInsts = data.enumerateInstances();
		while (enumInsts.hasMoreElements()) {
			instance = (Instance) enumInsts.nextElement();
			classIndex = (int) instance.value(instance.classIndex());
			docsPerClass[classIndex] += instance.weight();
		}

		mostFrequentIndex = 0;
		for (int h = 0; h < m_numClasses; h++)
			if (docsPerClass[h] > docsPerClass[mostFrequentIndex])
				mostFrequentIndex = h;
	}

	@Override
	public double classifyInstance(Instance instance) throws Exception {
		return (double) mostFrequentIndex;
	}
} public class ExtendedEvaluation extends Evaluation {

	// TODO standard deviation?
	
	public ExtendedEvaluation(Instances data) throws Exception {
		super(data);
	}

	/**
	 * Calculate the F-Measure with respect to a particular class and a given
	 * beta.
	 * 
	 * @param classIndex
	 *            the index of the class to consider as "positive"
	 * @param beta
	 *            the beta-index
	 * @return the F-Measure
	 */
	public double fMeasure(int classIndex, int beta) {

		double precision = precision(classIndex);
		double recall = recall(classIndex);
		if ((precision + recall) == 0) {
			return 0;
		}
		return ((Math.pow(beta, 2) + 1) * precision * recall)
				/ (Math.pow(beta, 2) * precision + recall);
	}

	/**
	 * Calculates the weighted (by class size) F-Measure for a given beta.
	 * 
	 * @return the weighted F-Measure.
	 */
	public double weightedFMeasure(int beta) {
		double[] classCounts = new double[m_NumClasses];
		double classCountSum = 0;

		for (int i = 0; i < m_NumClasses; i++) {
			for (int j = 0; j < m_NumClasses; j++) {
				classCounts[i] += m_ConfusionMatrix[i][j];
			}
			classCountSum += classCounts[i];
		}

		double fMeasureTotal = 0;
		for (int i = 0; i < m_NumClasses; i++) {
			double temp = fMeasure(i, beta);
			fMeasureTotal += (temp * classCounts[i]);
		}

		return fMeasureTotal / classCountSum;
	}

	/**
	 * Calculates the macro averaged F-Measure for a given beta.
	 * 
	 * @return the macro averaged F-Measure.
	 */
	public double macroAveragedFMeasure(int beta) {
		double fMeasureTotal = 0;
		for (int i = 0; i < m_NumClasses; i++) {
			double temp = fMeasure(i, beta);
			fMeasureTotal += temp;
		}
		return fMeasureTotal / m_NumClasses;
	}
	
	/**
	 * Calculates the micro averaged F-Measure for a given beta.
	 * 
	 * @return the micro averaged F-Measure.
	 */
	public double microAveragedFMeasure(int beta) {
		return weightedFMeasure(beta);
	}

	/**
	 * Calculates the macro averaged precision. Macro averaging considers equal
	 * weight for all classes. The result therefore is closer to the least
	 * frequency class precision.
	 * 
	 * @return the macro averaged precision.
	 */
	public double macroAveragedPrecision() {
		double precisionTotal = 0;
		for (int i = 0; i < m_NumClasses; i++) {
			double temp = precision(i);
			precisionTotal += temp;
		}
		return precisionTotal / m_NumClasses;
	}
	
	/**
	 * Calculates the micro averaged recall.
	 * 
	 * @return the micro averaged recall.
	 */
	public double microAveragedPrecision() {
		return weightedPrecision();
	}

	/**
	 * Calculates the micro averaged recall.
	 * 
	 * @return the micro averaged recall.
	 */
	public double microAveragedRecall() {
		return weightedRecall();
	}

	/**
	 * Calculates the macro averaged recall.
	 * 
	 * @return the macro averaged recall.
	 */
	public double macroAveragedRecall() {
		return macroAveragedTruePositiveRate();
	}

	/**
	 * Calculates the macro averaged true positive rate.
	 * 
	 * @return the macro averaged true positive rate.
	 */
	public double macroAveragedTruePositiveRate() {
		double truePosTotal = 0;
		for (int i = 0; i < m_NumClasses; i++) {
			double temp = truePositiveRate(i);
			truePosTotal += temp;
		}

		return truePosTotal / m_NumClasses;
	}

	/**
	 * Calculates the accuracy.
	 * 
	 * @return the accuracy.
	 */
	public final double accuracy() {
		return m_Correct / m_WithClass;
	}

} @Entity
public class Patient {
	@Id
	@GeneratedValue
	private Long id;

	@Column(unique=true)
	@Index(name = "rgh_ndx")
	private Integer rgh;

	// bidirectional relationship
	@OneToMany(mappedBy="rgh")
	private List<RHC> rhc;

	// bidirectional relationship
	@OneToMany(mappedBy="rgh")
	private List<DCE> dce;
	
	// bidirectional relationship
	@OneToMany(mappedBy="rgh")
	private List<SighReport> sighReport;
	
	public Patient() {
	}

	public Patient(Integer rgh) {
		
		this.rgh = rgh;
	}

	public void addRHC(RHC rhc) {
		rhc.setRgh(this);
		if (this.rhc == null)
			this.rhc = new ArrayList<RHC>();
		this.rhc.add(rhc);
	}
	
	public void addDCE(DCE dce) {
		dce.setRgh(this);
		if (this.dce == null)
			this.dce = new ArrayList<DCE>();
		this.dce.add(dce);
	}
	
	public void addSighReport(SighReport report) {
		report.setRgh(this);
		if (this.sighReport == null)
			this.sighReport = new ArrayList<SighReport>();
		this.sighReport.add(report);
	}

	public Long getId() {
		return id;
	}

	public Integer getRgh() {
		return rgh;
	}

	public List<RHC> getRhc() {
		return rhc;
	}
	
	public List<RHC> getDistinctRhc() {
		List<RHC> ret = new ArrayList<RHC>();
		Set<String> control = new HashSet<String>();
		String code;
		for (RHC r : rhc) {
			code = r.getIcdClass().getCode();
			if (!control.contains(code)) {
				control.add(code);
				ret.add(r);
			}
		}
		return ret;
	}

	/**
	 * 
	 * @return
	 * @deprecated use getTexts instead.
	 */
	public List<DCE> getDce() {
		return dce;
	}
	
	/**
	 * 
	 * @return
	 * @deprecated use getTexts instead.
	 */
	public List<SighReport> getSighReport() {
		return sighReport;
	}
	
	public List getTexts() {
		switch (Constants.CONFIG.getSource()) {
		case SIGH_REPORT:
			return sighReport;
		case DCE_REPORT:
			return dce;
		case ALL:
			List<Report> all = new ArrayList<Report>(sighReport);
			all.addAll(dce);
			return all;
		case TOTLAUD_REPORT:
			// TODO
		default:
			return null;
		}
	}

} public class PatientIterator implements Iterator<Patient> {

	private static final Logger LOG = Logger.getLogger(PatientIterator.class);

	private List<Patient> patientList;
	private int currentPosition = -1;
	private int nextPosition = -1;

	public PatientIterator() {
		final PatientDAO patientDAO = DAOFactory.getDAOFactory()
				.getPatientDAO();
		this.patientList = patientDAO.list();
		this.nextPosition = findNextPosition();
	}

	private int findNextPosition() {
		Patient p;
		boolean accept;
		for (int i = currentPosition + 1; i < patientList.size(); i++) {
			// TODO define at global location
			// ICDC-5 we isolate 10% of patients for validation.
			if (i % 10 == 0)
				continue;
			p = patientList.get(i);
			
			accept = false;
			switch (Constants.CONFIG.getCriteria()) {
			case ONE_REPORT_ONE_REGISTRY:
				accept = p.getTexts() != null && p.getRhc() != null
						&& p.getTexts().size() == 1
						&& p.getDistinctRhc().size() == 1;
				if (accept && Constants.CONFIG.getPatientYear() != -1)
					accept &= p.getDistinctRhc().get(0).getAnoDiagnostico() == Constants.CONFIG
							.getPatientYear();
				if (accept
						&& Constants.CONFIG.getMetastasisStatus() == MetastasisStatus.NONM1)
					accept &= !p.getDistinctRhc().get(0).getMetastasis()
							.equals("1");
				break;
			case MANY_REPORT_ONE_REGISTRY:
				accept = p.getTexts() != null
						&& p.getRhc() != null
						&& p.getTexts().size() >= Constants.CONFIG
								.getMinReports()
						&& p.getDistinctRhc().size() == 1;
				if (accept && Constants.CONFIG.getPatientYear() != -1)
					accept &= p.getDistinctRhc().get(0).getAnoDiagnostico() == Constants.CONFIG
							.getPatientYear();
				if (accept
						&& Constants.CONFIG.getMetastasisStatus() == MetastasisStatus.NONM1)
					accept &= !p.getDistinctRhc().get(0).getMetastasis()
							.equals("1");
				break;
			case MANY_REPORT_MANY_REGISTRY:
				// FIXME Does not filter by year.
				// FIXME Does not filter out metastasis.
				accept = p.getTexts() != null && p.getRhc() != null
						&& p.getTexts().size() >= 1
						&& p.getDistinctRhc().size() >= 1;
				break;

			default:
				LOG.fatal("No criterion defined on Constants class.");
				break;
			}
			if (accept)
				return i;
		}

		return -1;
	}

	public boolean hasNext() {
		return this.nextPosition > this.currentPosition ? true : false;
	}

	public Patient next() {
		if (this.nextPosition <= this.currentPosition)
			return null;
		this.currentPosition = this.nextPosition;
		this.nextPosition = findNextPosition();

		Patient p = patientList.get(this.currentPosition);
		// Instance carrier = new Instance(p.getDce().get(0).getText(),
		// p.getRhc()
		// .get(0).getTopography().getCode(), p.getRgh().toString(), null);
		// return carrier;
		return p;

	}

	public void remove() {
		throw new IllegalStateException(
				"This Iterator<Instance> does not support remove().");

	}

} @Entity
public class RHC {
	@Id
	@GeneratedValue
	private Long id;

	// bidirectional relationship
	@ManyToOne
	@JoinColumn(name = "patient_fk")
	private Patient rgh;

	@ManyToOne(cascade = { CascadeType.PERSIST, CascadeType.MERGE })
	private Topography topography;

	@ManyToOne(cascade = { CascadeType.PERSIST, CascadeType.MERGE })
	private Morphology morphology;

	@Column
	private String estadioClinico;
	
	@Column
	private String t;

	@Column
	private String n;

	@Column
	private String m;

	@Column
	private String pt;

	@Column
	private String pn;

	@Column
	private String pm;
	
	@Column
	private Integer anoDiagnostico;
	
	@Transient
	private Classifiable icdClass;
	
	private static final Logger LOG = Logger.getLogger(RHC.class);

	public RHC() {
	}

	public void setRgh(Patient rgh) {
		this.rgh = rgh;
	}

	public RHC(Topography topography, Morphology morphology,
			String estadioClinico, String t, String n, String m, String pt, String pn, String pm, Integer anoDiagnostico) {
		super();
		this.topography = topography;
		this.morphology = morphology;
		this.estadioClinico = estadioClinico;
		this.t = t;
		this.n = n;
		this.m = m;
		this.pt = pt;
		this.pn = pn;
		this.pm = pm;
		this.anoDiagnostico = anoDiagnostico;
	}

	public Topography getTopography() {
		return topography;
	}

	public Morphology getMorphology() {
		return morphology;
	}
	
	
	public boolean isMetastasis() {
		// FIXME
		return false;
	}
	
	public Classifiable getIcdClass() {
		switch (Constants.CONFIG.getTarget()) {
		case MORPHOLOGY:
			return morphology;
		case MORPHOLOGY_GROUP:
			return morphology.getGroup();
		case TOPOGRAPHY:
			return topography;
		case TOPOGRAPHY_CATEGORY:
			return topography.getCategory();
		case TOPOGRAPHY_GROUP:
			return topography.getCategory().getGroup();			
		default:
			return null;
		}
	}
	
	public String getMetastasis() {
		return m;
	}

	public String getEstadioClinico() {
		return estadioClinico;
	}
	
	public Integer getAnoDiagnostico() {
		return anoDiagnostico;
	}

	/**
	 * 
	 * @param file
	 */
	public static void loadFromFile(File file) {
		List<String[]> myEntries = null;
		try {
			CSVReader reader = new CSVReader(new InputStreamReader(
					new FileInputStream(file), "UTF-8"));
			myEntries = reader.readAll();
			reader.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		DAOFactory factory = DAOFactory.getDAOFactory();
		
		/* ************************************************************************** */

		Iterator<String[]> iter = myEntries.iterator();
		iter.next();
		int size = myEntries.size();

		PatientDAO patientDao = factory.getPatientDAO();
		patientDao.beginTransaction();

		List<String[]> list = new ArrayList<String[]>();

		int i = 0;
		while (iter.hasNext()) {
			i++;
			if (i % Constants.BATCH_SIZE == 0) {
				LOG.debug(i + "/" + size + " (" + 100 * i / size
						+ "%)");
				patientDao.flushAndClear();
			}

			String[] entry = iter.next();

			Integer rgh = Integer.parseInt(entry[0]);

			Patient p = patientDao.locate(rgh);
			if (p == null) {
				p = new Patient(rgh);
				patientDao.save(p);
			}

			// TODO criar RHC aqui e salvar num Map<Integer,List<RHC>>. acelera no segundo loop.
			list.add(entry);

		}
		patientDao.commit();

		/* ************************************************************************** */

		RhcDAO rhcDao = factory.getRhcDAO();
		rhcDao.beginTransaction();

		TopographyDAO topoDao = factory.getTopographyDAO();
		MorphologyDAO morphoDao = factory.getMorphologyDAO();

		Iterator<String[]> listIter = list.iterator();

		i = 1;
		boolean error = false;
		while (listIter.hasNext()) {
			i++;
			if (i % Constants.BATCH_SIZE == 0) {
				LOG.debug(i + "/" + size + " (" + 100 * i
						/ size + "%)");
				rhcDao.flushAndClear();
			}

			String[] entry = listIter.next();

			error = false;
			
			////"rgh","topografia","morfologia","estadioclinico","pt","pn","pm"
			//rgh,topografia,morfologia,estadioclinico,t,n,m,pt,pn,pm,anodiag

			String code = new StringBuilder(entry[1].substring(1, 4)).insert(2,
					".").toString();
			Topography topography = topoDao.locate(code);
			if (topography == null) {
				LOG.error("Topography not found! Code: " + entry[1]
						+ " -> " + code);
				error = true;
			}

			code = new StringBuilder(entry[2]).insert(4, "/").toString();
			Morphology morphology = morphoDao.locate(code);
			if (morphology == null) {
				LOG.error("Morphology not found! Code: " + entry[2]
						+ " -> " + code);
				error = true;
			}

			Integer rgh = Integer.parseInt(entry[0]);

			Patient p = patientDao.locate(rgh);
			if (p == null) {
				LOG.error("Patient not found!");
				error = true;
			}
			
			Integer anoDiagnostico = Integer.parseInt(entry[10]);

			if (error != true) {
				RHC c = new RHC(topography, morphology, entry[3], entry[4],
						entry[5], entry[6], entry[7], entry[8], entry[9], anoDiagnostico);
				p.addRHC(c);
				rhcDao.save(c);
			}
		}
		rhcDao.commit();
	}
}
 public interface Report {
	public String getText();
	
	public String[] getTexts();
	
	/**
	 * 
	 * @return non-empty keys only
	 */
	public Map<Configuration.Sections, String> getZonedTexts();
} public interface Classifiable {
	public String getCode();
}
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.log4j.Logger;
import org.cogroo.analyzer.Analyzer;
import org.cogroo.analyzer.ComponentFactory;
import org.cogroo.text.Chunk;
import org.cogroo.text.Document;
import org.cogroo.text.Sentence;
import org.cogroo.text.Token;
import org.cogroo.text.impl.DocumentImpl;
import au.com.bytecode.opencsv.CSVWriter;
import br.usp.ime.icdc.Configuration;
import br.usp.ime.icdc.Constants;
import br.usp.ime.icdc.dao.DAOFactory;
import br.usp.ime.icdc.logic.evaluation.Stats;
import br.usp.ime.icdc.logic.weka.classifiers.AddAlphaNaiveBayesMultinomial;
import br.usp.ime.icdc.logic.weka.classifiers.GoodTuringNaiveBayesMultinomial;
import br.usp.ime.icdc.logic.weka.classifiers.NaiveClassifier;
import br.usp.ime.icdc.logic.weka.evaluation.ExtendedEvaluation;
import br.usp.ime.icdc.model.Patient;
import br.usp.ime.icdc.model.PatientIterator;
import br.usp.ime.icdc.model.RHC;
import br.usp.ime.icdc.model.Report;
import br.usp.ime.icdc.model.icd.Classifiable;
import weka.classifiers.Classifier;
import weka.classifiers.bayes.NaiveBayesMultinomial;
import weka.classifiers.functions.LibSVM;
import weka.classifiers.meta.FilteredClassifier;
import weka.core.Attribute;
import weka.core.FastVector;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.SelectedTag;
import weka.core.stemmers.PTStemmer;
import weka.core.tokenizers.AlphabeticTokenizer;
import weka.core.tokenizers.NGramTokenizer;
import weka.filters.Filter;
import weka.filters.MultiFilter;
import weka.filters.unsupervised.attribute.Remove;
import weka.filters.unsupervised.attribute.StringToWordVector;
package br.usp.ime.icdc.logic.classifier;




public class CipeClassifier {

	protected Instances dataset;
	protected Classifier model;

	// TODO remove (as its dependent on dataset used to eval).
	protected ExtendedEvaluation eval;

	protected Stats stats = new Stats();