public class Configuration {
	public enum Sections {
		MACROSCOPY, MICROSCOPY, CYTOPATHOLOGY, LIQUID_CYTOPATHOLOGY, CONCLUSION, OTHERS;
	}

	public enum Criteria {
		ONE_REPORT_ONE_REGISTRY, MANY_REPORT_ONE_REGISTRY, MANY_REPORT_MANY_REGISTRY;
	}

	public enum Classifiers {
		NAIVE, BAYES, BAYES_NET, BERNOULLI, SVM;
	}

	public enum WeightingSchemes {
		TF, IDF, TFIDF;
	}

	public enum SmoothingTechniques {
		ADD_ONE, ADD_ALPHA, GOOD_TURING;
	}

	public enum Sources {
		DCE_REPORT, SIGH_REPORT, TOTLAUD_REPORT, ALL;
	}

	public enum Stemmers {
		// TODO test with PTStemmer V1 available at Maven repositories
		NONE, ORENGO, PORTER, SAVOY, COGROO;
	}

	public enum SentenceDetectors {
		NONE, OPENNLP, COGROO;
	}

	public enum Tokenizers {
		WORD, ALPHABETIC, OPENNLP, COGROO;
	}

	public enum Chunkers {
		NONE, COGROO;
	}

	public enum Targets {
		MORPHOLOGY, MORPHOLOGY_GROUP, TOPOGRAPHY, TOPOGRAPHY_CATEGORY, TOPOGRAPHY_GROUP;
	}

	public enum MetastasisStatus {
		ALL, NONM1;
	}

	private Set<Sections> sections;
	private Criteria criteria;
	private int minReports = 1;
	private Classifiers classifier;
	private WeightingSchemes weightScheme;
	private SmoothingTechniques smoothing;
	private double svmCostParameter = 1;
	private Sources source;
	private boolean stoplist;
	private int ngrams = 1;
	private SentenceDetectors sentenceDetector;
	private Tokenizers tokenizer;
	private Stemmers stemmer;
	private Chunkers chunker;
	private Targets target;
	private MetastasisStatus meta;
	private int patientYear = -1;

	public Configuration(Set<Sections> sections, Criteria criteria, Classifiers classifier,
			WeightingSchemes weightScheme, SmoothingTechniques smoothing, double svmCParameter, Sources source,
			boolean stoplist, int ngrams, SentenceDetectors sentenceDetector, Tokenizers tokenizer, Stemmers stemmer,
			Chunkers chunker, Targets target, int minReports, int year, MetastasisStatus meta) {
		super();
		this.sections = sections;
		this.criteria = criteria;
		if (criteria == Criteria.MANY_REPORT_ONE_REGISTRY)
			this.minReports = minReports;
		this.classifier = classifier;
		this.weightScheme = weightScheme;
		this.smoothing = smoothing;
		this.svmCostParameter = svmCParameter;
		this.source = source;
		this.stoplist = stoplist;
		this.ngrams = ngrams;
		this.sentenceDetector = sentenceDetector;
		this.tokenizer = tokenizer;
		this.stemmer = stemmer;
		this.chunker = chunker;
		this.target = target;
		this.patientYear = year;
		this.meta = meta;
	}

	public Set<Sections> getSections() {
		return sections;
	}

	public Criteria getCriteria() {
		return criteria;
	}

	public int getMinReports() {
		return minReports;
	}

	public Classifiers getClassifier() {
		return classifier;
	}

	public WeightingSchemes getWeightScheme() {
		return weightScheme;
	}

	public SmoothingTechniques getSmoothing() {
		return smoothing;
	}

	public double getSvmCostParameter() {
		return svmCostParameter;
	}

	public Sources getSource() {
		return source;
	}

	public boolean getStoplist() {
		return stoplist;
	}
	
	public int getNGrams() {
		return ngrams;
	}

	public SentenceDetectors getSentenceDetector() {
		return sentenceDetector;
	}

	public Tokenizers getTokenizer() {
		return tokenizer;
	}

	public Stemmers getStemmer() {
		return stemmer;
	}

	public Chunkers getChunker() {
		return chunker;
	}

	public Targets getTarget() {
		return target;
	}

	public MetastasisStatus getMetastasisStatus() {
		return meta;
	}

	public int getPatientYear() {
		return patientYear;
	}

	public String getInstanceDependentStringRepresentation() {
		return (Arrays.toString(sections.toArray()) + "-" + criteria + "-" + minReports + "-" + source + "-"
				+ sentenceDetector + "-" + tokenizer + "Tokenizer-" + stemmer + "Stemmer-" + chunker + "Chunker-"
				+ target + "-" + meta + "-" + patientYear).toLowerCase();
	}

	public String getStringRepresentation() {
		return (Arrays.toString(sections.toArray()) + "-" + criteria + "-" + minReports + "-" + classifier + "-"
				+ weightScheme + "-" + smoothing + "-" + svmCostParameter + "-" + Constants.ALPHA + "-" + source + "-"
				+ stoplist + "-" + ngrams + "-" + sentenceDetector + "-" + tokenizer + "-" + stemmer + "-" + chunker + "-" + target
				+ "-" + meta + "-" + patientYear).toLowerCase();
	}

} public enum Chunkers {
	NONE, COGROO;
} public enum Classifiers {
	NAIVE, BAYES, BAYES_NET, BERNOULLI, SVM;
} public enum Criteria {
	ONE_REPORT_ONE_REGISTRY, MANY_REPORT_ONE_REGISTRY, MANY_REPORT_MANY_REGISTRY;
} public enum MetastasisStatus {
	ALL, NONM1;
} public enum Sections {
	MACROSCOPY, MICROSCOPY, CYTOPATHOLOGY, LIQUID_CYTOPATHOLOGY, CONCLUSION, OTHERS;
} public enum SentenceDetectors {
	NONE, OPENNLP, COGROO;
} public enum SmoothingTechniques {
	ADD_ONE, ADD_ALPHA, GOOD_TURING;
} public enum Sources {
	DCE_REPORT, SIGH_REPORT, TOTLAUD_REPORT, ALL;
} public enum Stemmers {
	// TODO test with PTStemmer V1 available at Maven repositories
	NONE, ORENGO, PORTER, SAVOY, COGROO;
} public enum Targets {
	MORPHOLOGY, MORPHOLOGY_GROUP, TOPOGRAPHY, TOPOGRAPHY_CATEGORY, TOPOGRAPHY_GROUP;
} public enum Tokenizers {
	WORD, ALPHABETIC, OPENNLP, COGROO;
} public enum WeightingSchemes {
	TF, IDF, TFIDF;
} public class Constants {

	public static final int THREADS = 1;
	public static final int CONNECTIONS = THREADS * 2;

	// TODO change to something better, maybe setting automatically on the main method.
	public static final boolean RECREATE_DB = true;
	public static final int BATCH_SIZE = 100;

	// TODO move to a config file
	/** Root directory where reference tables are located. */
	public static final String REF_DIR = "src/ref/";

	/** Directory containing large data files. */
	public static final String DATA_DIR = "src/data/";

	// TODO defined in models.xml for CoGrOO implementation. May not be necessary.
	/** Directory containing models (e.g. OpenNLP models). */
	public static final String MODEL_DIR = "src/models/";

	/** Defines if a cache of instances is used or not. */
	public static final boolean CACHE = false;

	/** Indicates wheter to generate stats or not. */
	public static final boolean STATS = false;

	public static final float TEST_RATIO = 0.1f;
	// public static final int THRESHOLD = 10;

	// TODO move to a class smoothing or something
	public static final double ALPHA = 0.5d;

	public static Configuration CONFIG;

} public class DAOFactory {
	// TODO migrate to JPA Persistence:
	// http://docs.jboss.org/hibernate/stable/entitymanager/reference/en/html_single/
	private static final int CONNECTIONS = Constants.CONNECTIONS;

	private static final DAOFactory instance = new DAOFactory();

	public static DAOFactory getDAOFactory() {
		return instance;
	}

	private SessionFactory sessionFactory;
	private Session[] session = new Session[CONNECTIONS];

	private TopographyDAO[] topographyDAO = new TopographyDAO[CONNECTIONS];
	private TopographyCategoryDAO[] topographyCategoryDAO = new TopographyCategoryDAO[CONNECTIONS];
	private TopographyGroupDAO[] topographyGroupDAO = new TopographyGroupDAO[CONNECTIONS];

	private MorphologyDAO[] morphologyDAO = new MorphologyDAO[CONNECTIONS];
	private MorphologyGroupDAO[] morphologyGroupDAO = new MorphologyGroupDAO[CONNECTIONS];

	private DceDAO[] dceDAO = new DceDAO[CONNECTIONS];
	private SighDAO[] sighDAO = new SighDAO[CONNECTIONS];
	private RhcDAO[] rhcDAO = new RhcDAO[CONNECTIONS];
	private PatientDAO[] patientDAO = new PatientDAO[CONNECTIONS];

	private DAOFactory() {
		AnnotationConfiguration cfg = new AnnotationConfiguration();

		cfg.addAnnotatedClass(Topography.class);
		cfg.addAnnotatedClass(TopographyCategory.class);
		cfg.addAnnotatedClass(TopographyGroup.class);

		cfg.addAnnotatedClass(Morphology.class);
		cfg.addAnnotatedClass(MorphologyGroup.class);

		cfg.addAnnotatedClass(RHC.class);
		cfg.addAnnotatedClass(DCE.class);
		cfg.addAnnotatedClass(SighReport.class);
		cfg.addAnnotatedClass(Patient.class);
		cfg.configure();

		// TODO set fetch size according to Constants.BATCH_SIZE

		if (Constants.RECREATE_DB) {
			SchemaExport se = new SchemaExport(cfg);
			se.create(true, true);
		}

		this.sessionFactory = cfg.buildSessionFactory();

		for (int i = 0; i < CONNECTIONS; i++)
			session[i] = sessionFactory.openSession();

		for (int i = 0; i < CONNECTIONS; i++)
			topographyDAO[i] = new TopographyDAO(session[i]);
		for (int i = 0; i < CONNECTIONS; i++)
			topographyCategoryDAO[i] = new TopographyCategoryDAO(session[i]);
		for (int i = 0; i < CONNECTIONS; i++)
			topographyGroupDAO[i] = new TopographyGroupDAO(session[i]);

		for (int i = 0; i < CONNECTIONS; i++)
			morphologyDAO[i] = new MorphologyDAO(session[i]);
		for (int i = 0; i < CONNECTIONS; i++)
			morphologyGroupDAO[i] = new MorphologyGroupDAO(session[i]);

		for (int i = 0; i < CONNECTIONS; i++)
			dceDAO[i] = new DceDAO(session[i]);
		for (int i = 0; i < CONNECTIONS; i++)
			sighDAO[i] = new SighDAO(session[i]);
		for (int i = 0; i < CONNECTIONS; i++)
			rhcDAO[i] = new RhcDAO(session[i]);
		for (int i = 0; i < CONNECTIONS; i++)
			patientDAO[i] = new PatientDAO(session[i]);
	}

	int sess = 0;

	private int getNextSession() {
		sess++;
		if (sess >= CONNECTIONS)
			sess = 0;
		return sess;
	}

	public ClassifiableDAO getIcdClassDAO() {
		switch (Constants.CONFIG.getTarget()) {
		case MORPHOLOGY:
			return (ClassifiableDAO) getMorphologyDAO();
		case MORPHOLOGY_GROUP:
			return (ClassifiableDAO) getMorphologyGroupDAO();
		case TOPOGRAPHY:
			return (ClassifiableDAO) getTopographyDAO();
		case TOPOGRAPHY_CATEGORY:
			return (ClassifiableDAO) getTopographyCategoryDAO();
		case TOPOGRAPHY_GROUP:
			return (ClassifiableDAO) getTopographyGroupDAO();
		default:
			return null;
		}
	}

	public TopographyDAO getTopographyDAO() {
		return topographyDAO[getNextSession()];
	}

	public TopographyCategoryDAO getTopographyCategoryDAO() {
		return topographyCategoryDAO[getNextSession()];
	}

	public TopographyGroupDAO getTopographyGroupDAO() {
		return topographyGroupDAO[getNextSession()];
	}

	public MorphologyDAO getMorphologyDAO() {
		return morphologyDAO[getNextSession()];
	}

	public MorphologyGroupDAO getMorphologyGroupDAO() {
		return morphologyGroupDAO[getNextSession()];
	}

	public DceDAO getDceDAO() {
		return dceDAO[getNextSession()];
	}

	public SighDAO getSighDAO() {
		return sighDAO[getNextSession()];
	}

	public RhcDAO getRhcDAO() {
		return rhcDAO[getNextSession()];
	}

	public PatientDAO getPatientDAO() {
		return patientDAO[getNextSession()];
	}

	public void close() {
		for (Session s : session)
			s.close();
		sessionFactory.close();
	}

}
 public class TopographyGroupDAO implements ClassifiableDAO {
	private Session session;

	public TopographyGroupDAO(Session session) {
		this.session = session;
	}

	public void save(TopographyGroup t) {
		this.session.save(t);
	}

	public void delete(TopographyGroup t) {
		this.session.delete(t);
	}

	public TopographyGroup load(Long id) {
		return (TopographyGroup) this.session.load(TopographyGroup.class, id);
	}

	public TopographyGroup locate(String category) {
		if (category.startsWith("C") || category.startsWith("c"))
			category = category.substring(1);
		Integer search = Integer.parseInt(category);

		return (TopographyGroup) this.session
				.createCriteria(TopographyGroup.class)
				.add(Restrictions.le("startCode", search))
				.add(Restrictions.ge("endCode", search)).uniqueResult();
	}
	
	public TopographyGroup locateCode(String code) {
		return (TopographyGroup) this.session
				.createCriteria(TopographyGroup.class)
				.add(Restrictions.eq("code", code)).uniqueResult();
	}

	public void update(TopographyGroup t) {
		this.session.update(t);
	}

	public void beginTransaction() {
		this.session.beginTransaction();
	}

	public void commit() {
		this.session.getTransaction().commit();
	}

	public void flushAndClear() {
		this.session.flush();
		this.session.clear();
	}
	
	@SuppressWarnings("unchecked")
	public List<Classifiable> list() {
		return this.session.createCriteria(TopographyGroup.class).list();
	}
}
 public class CipeClassifier {

	protected Instances dataset;
	protected Classifier model;

	// TODO remove (as its dependent on dataset used to eval).
	protected ExtendedEvaluation eval;

	protected Stats stats = new Stats();

	// TODO DB to ARFF converter to be used on GUI

	private static final Logger LOG = Logger.getLogger(CipeClassifier.class);

	public CipeClassifier() {
		parseClassifier();
		loadInstances();
		// buildClassifier();
	}

	public void setWordsToKeep(int wordsToKeep) {
		MultiFilter mf = (MultiFilter) ((FilteredClassifier) model).getFilter();
		Filter[] filters = mf.getFilters();
		for (Filter f : filters)
			if (f instanceof StringToWordVector)
				((StringToWordVector) f).setWordsToKeep(wordsToKeep);

		// TODO Precisa setar novamente o filter?
	}

	protected void parseClassifier() {
		String refDir = Constants.REF_DIR;

		LOG.debug("Config: " + Constants.CONFIG);
		switch (Constants.CONFIG.getClassifier()) {
		case NAIVE:
			model = new NaiveClassifier();
			break;
		case BAYES: {
			model = new FilteredClassifier();
			StringToWordVector f = new StringToWordVector();
			f.setAttributeIndices("first");
			f.setDoNotOperateOnPerClassBasis(true);
			f.setLowerCaseTokens(true);
			f.setMinTermFreq(1);
			f.setOutputWordCounts(true); // XXX This is Bayes!
			f = parseStemmer(f);
			// TODO qual a ordem em relação ao stemmer? stopwords devem ser
			// eliminadas ANTES!
			f.setStopwords(new File(refDir + "stopwords-sigh.txt")); // Stopwords!

			f = parseTokenizer(f);

			f.setUseStoplist(Constants.CONFIG.getStoplist()); // Stopwords!
			f.setWordsToKeep(5000);
			
			switch (Constants.CONFIG.getWeightScheme()) {
			case TF:
				f.setTFTransform(true);
				break;
			case IDF:
				f.setIDFTransform(true);
				break;
			case TFIDF:
				f.setTFTransform(true);
				f.setIDFTransform(true);
			default:
				break;
			}

			NaiveBayesMultinomial nbm = parseSmoothing();
			// nbm.setOptions(options);

			Remove remove = new Remove();
			remove.setAttributeIndices("1,2,4,5");

			MultiFilter mf = new MultiFilter();
			mf.setFilters(new Filter[] { remove, f });

			((FilteredClassifier) model).setFilter(mf);
			((FilteredClassifier) model).setClassifier(nbm);

			break;
		}
		case BERNOULLI: {
			model = new FilteredClassifier();
			StringToWordVector f = new StringToWordVector();
			f.setAttributeIndices("first");
			f.setDoNotOperateOnPerClassBasis(true);
			f.setLowerCaseTokens(true);
			f.setMinTermFreq(1);
			f.setOutputWordCounts(false); // XXX This is Bernoulli!
			f = parseStemmer(f);
			// TODO generate list again with 90% of all data.
			f.setStopwords(new File(refDir + "stopwords-sigh.txt")); // Stopwords!

			f = parseTokenizer(f);

			f.setUseStoplist(Constants.CONFIG.getStoplist()); // Stopwords!
			f.setWordsToKeep(5000);

			NaiveBayesMultinomial nbm = parseSmoothing();

			// nbm.setOptions(options);

			((FilteredClassifier) model).setFilter(f);
			((FilteredClassifier) model).setClassifier(nbm);
			break;
		}
		case SVM: {
			model = new FilteredClassifier();
			StringToWordVector f = new StringToWordVector();
			f.setAttributeIndices("first");
			f.setDoNotOperateOnPerClassBasis(true);
			f.setLowerCaseTokens(true);
			f.setMinTermFreq(1);
			f.setOutputWordCounts(true);
			f = parseStemmer(f);
			// TODO qual a ordem em relação ao stemmer? stopwords devem ser
			// eliminadas ANTES!
			f.setStopwords(new File(refDir + "stopwords-sigh.txt")); // Stopwords!

			f = parseTokenizer(f);

			f.setUseStoplist(Constants.CONFIG.getStoplist()); // Stopwords!
			f.setWordsToKeep(5000);
			
			switch (Constants.CONFIG.getWeightScheme()) {
			case TF:
				f.setTFTransform(true);
				break;
			case IDF:
				f.setIDFTransform(true);
				break;
			case TFIDF:
				f.setTFTransform(true);
				f.setIDFTransform(true);
			default:
				break;
			}

			LibSVM svm = new LibSVM();
			
			// -K <int>
			// Set type of kernel function (default: 2)
			// 0 = linear: u'*v
			// 1 = polynomial: (gamma*u'*v + coef0)^degree
			// 2 = radial basis function: exp(-gamma*|u-v|^2)
			// 3 = sigmoid: tanh(gamma*u'*v + coef0)
			// Linear kernel is commonly recommended for text classification
			svm.setKernelType(new SelectedTag(0, LibSVM.TAGS_KERNELTYPE));
			
			svm.setCost(Constants.CONFIG.getSvmCostParameter());

			Remove remove = new Remove();
			remove.setAttributeIndices("1,2,4,5");

			MultiFilter mf = new MultiFilter();
			mf.setFilters(new Filter[] { remove, f });

			((FilteredClassifier) model).setFilter(mf);
			((FilteredClassifier) model).setClassifier(svm);

			break;
		}
		default:
			throw new IllegalArgumentException(Constants.CONFIG.getClassifier() + " is not implemented.");
		}
	}

	private StringToWordVector parseTokenizer(StringToWordVector filter) {
		switch (Constants.CONFIG.getTokenizer()) {
		case ALPHABETIC:
			// Avoid. Does not support diacritics (ã, á, é, etc.)
			filter.setTokenizer(new AlphabeticTokenizer());
			break;
		case WORD:
			NGramTokenizer tokenizer = new NGramTokenizer();
			tokenizer.setNGramMaxSize(Constants.CONFIG.getNGrams());
			filter.setTokenizer(tokenizer);
			break;
		case OPENNLP:
			// TODO use WordTokenizer with a specific delimiter set via options
			// and printed by OpenNLP.
			break;
		case COGROO:
			// TODO use WordTokenizer with a specific delimiter set via options
			// and printed by CoGrOO.
			break;
		default:
			throw new IllegalArgumentException(Constants.CONFIG.getTokenizer() + " is not implemented.");
		}
		return filter;
	}

	private StringToWordVector parseStemmer(StringToWordVector filter) {
		switch (Constants.CONFIG.getStemmer()) {
		case NONE:
			break;
		case PORTER: {
			// Portuguese Snowball stemmer
			PTStemmer stemmer = new PTStemmer();
			stemmer.setStemmer(new SelectedTag(PTStemmer.STEMMER_PORTER, PTStemmer.TAGS_STEMMERS));
			filter.setStemmer(stemmer);
			break;
		}
		case ORENGO: {
			PTStemmer stemmer = new PTStemmer();
			stemmer.setStemmer(new SelectedTag(PTStemmer.STEMMER_ORENGO, PTStemmer.TAGS_STEMMERS));
			filter.setStemmer(stemmer);
			break;
		}
		// TODO Savoy requires PTStemmer V2, which is not currently implemented
		// on the Weka Wrapper.
		// We do have a prototype for it in V1 of the current project.
		// (see https://github.com/fracpete/ptstemmer-weka-package/issues/1)
		// case SAVOY: {
		// PTStemmer stemmer = new PTStemmer();
		// stemmer.setStemmer(new SelectedTag(PTStemmer.STEMMER_SAVOY,
		// PTStemmer.TAGS_STEMMERS));
		// filter.setStemmer(stemmer);
		// break;
		// }
		case COGROO:
			// CoGrOO is implemented during import phase.
			break;
		default:
			throw new IllegalArgumentException(Constants.CONFIG.getStemmer() + " is not implemented.");
		}
		return filter;
	}

	private NaiveBayesMultinomial parseSmoothing() {
		NaiveBayesMultinomial nbm;
		switch (Constants.CONFIG.getSmoothing()) {
		case ADD_ONE:
			nbm = new NaiveBayesMultinomial();
			break;
		case ADD_ALPHA:
			nbm = new AddAlphaNaiveBayesMultinomial();
			((AddAlphaNaiveBayesMultinomial) nbm).setAlpha(Constants.ALPHA);
			break;
		case GOOD_TURING:
			nbm = new GoodTuringNaiveBayesMultinomial();
			break;
		default:
			throw new IllegalArgumentException(Constants.CONFIG.getSmoothing() + " is not implemented.");
		}
		// NaiveBayes nbm = new NaiveBayes();
		return nbm;
	}

	public void buildClassifier() {
		LOG.debug("Building classifier...");
		try {
			model.buildClassifier(dataset);
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	public void saveToFile(File file) {
		LOG.debug("Saving to file...");
		// FIXME build and serialize model!
		FileOutputStream fos = null;
		ObjectOutputStream out = null;
		try {
			fos = new FileOutputStream(file);
			out = new ObjectOutputStream(fos);
			out.writeObject(dataset);
			out.close();
		} catch (IOException ex) {
			ex.printStackTrace();
		}
	}

	public void loadFromFile(File file) {
		FileInputStream fis = null;
		ObjectInputStream in = null;
		try {
			fis = new FileInputStream(file);
			in = new ObjectInputStream(fis);
			dataset = (Instances) in.readObject();
			in.close();
		} catch (IOException ex) {
			ex.printStackTrace();
		} catch (ClassNotFoundException ex) {
			ex.printStackTrace();
		}
	}

	private void loadInstances() {
		String filename = Constants.CONFIG.getInstanceDependentStringRepresentation() + ".bin";
		File file = new File(filename);
		try {
			LOG.debug(file.getCanonicalFile());
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		if (!Constants.CACHE || !file.exists()) {
			LOG.debug("Will import dataset.");
			// FIXME do not depend on patient iterator. maybe receive some
			// superclass for both iterator or file.
			PatientIterator iter = new PatientIterator();
			importDataset(iter);
			saveToFile(file);
			return;
		}
		LOG.debug("Will load from file.");
		loadFromFile(file);
	}

	/**
	 * 
	 * @param iter
	 * @return
	 */
	public Instances importDataset(Iterator iter) {
		final int numAttributes = 6;
		FastVector attributes = new FastVector(numAttributes);

		LOG.debug("Will generate classes.");
		FastVector nominalCids = new FastVector();
		List<Classifiable> icds = DAOFactory.getDAOFactory().getIcdClassDAO().list();
		Iterator<Classifiable> icdIter = icds.iterator();
		while (icdIter.hasNext()) {
			Classifiable c = icdIter.next();
			nominalCids.addElement(c.getCode());
		}
		stats.setNumClasses(icds.size());

		attributes.addElement(new Attribute("id"));
		attributes.addElement(new Attribute("year"));
		attributes.addElement(new Attribute("text", (FastVector) null));
		attributes.addElement(new Attribute("m", (FastVector) null));
		attributes.addElement(new Attribute("ec", (FastVector) null));
		attributes.addElement(new Attribute("code", nominalCids));

		Instances data = new Instances("CID", attributes, 1);

		// Detects if CoGrOO is needed and instantiates it.
		Analyzer cogroo = null;
		Document document = null;
		if (Constants.CONFIG.getSentenceDetector() == Configuration.SentenceDetectors.COGROO
				|| Constants.CONFIG.getTokenizer() == Configuration.Tokenizers.COGROO
				|| Constants.CONFIG.getStemmer() == Configuration.Stemmers.COGROO
				|| Constants.CONFIG.getChunker() == Configuration.Chunkers.COGROO) {
			ComponentFactory factory = null;
			try {
				factory = ComponentFactory.create(new FileInputStream("src/main/resources/models.xml"));
				cogroo = factory.createPipe();
				document = new DocumentImpl();
			} catch (FileNotFoundException e) {
				e.printStackTrace();
			}
		}

		// Map<String, String> map = new HashMap<String, String>();

		Patient p;
		List<Report> list;
		List<String> texts;
		StringBuilder sb;

		RHC rhc;

		String[] textsArray;
		String[] sentences;
		String[] tokens;
		String[] lemmas;
		String text, s, prefix;
		String code, m, ec;

		Integer id, year;

		Instance inst;

		// String[] sections;
		Map<Configuration.Sections, String> sections;

		int i = 0;
		final int total = 19560; // TODO remove hardcoded value
		long start = System.currentTimeMillis();
		while (iter.hasNext()) {
			i++;
			if (i % 100 == 0) {
				long now = System.currentTimeMillis();
				double milisecondsPerEntry = (now - start) / (double) i;
				int remain = (int) ((milisecondsPerEntry * (total - i)) / (60 * 1000));
				LOG.trace(i + "/" + total + " (" + 100 * i / total + "%) " + remain + " minutes remaining");
			}

			p = (Patient) iter.next();

			list = p.getTexts();
			int numDocs = list.size();
			stats.incNumDocs(numDocs);

			texts = new ArrayList<String>();
			for (int j = 0; j < numDocs; j++) {
				// sections = list.get(j).getTexts();
				sections = list.get(j).getZonedTexts();

				for (Map.Entry<Configuration.Sections, String> entry : sections.entrySet()) {
					// Zone selection
					if (Constants.CONFIG.getSections().contains(entry.getKey())) {
						texts.add(entry.getValue());
					}
				}
			}
			// We didn't find any valid text.
			if (texts.isEmpty()) {
				LOG.debug("Found empty texts array for RGH: " + p.getRgh());
				continue;
			}

			textsArray = new String[texts.size()];
			textsArray = texts.toArray(textsArray);

			sentences = textsArray;
			tokens = sentences;

			sb = new StringBuilder();
			for (int j = 0; j < tokens.length; j++) {
				sb.append(tokens[j]);
				sb.append(" ");
			}
			text = sb.toString();

			if (Constants.STATS)
				stats.countCharTypes(text);

			/* CoGrOO processing */
			if (cogroo != null) {

				// TODO change pipe to what we need. change dinamically
				// according to config??
				// TODO exclude opennlp sentdetect and tokenizer (?)
				document.setText(text);
				try {
					cogroo.analyze(document);
				} catch (Exception e) {
					System.out.println("Error while analyzing text: ");
					System.out.println(text);
					System.out.flush();
					e.printStackTrace();
					System.err.flush();
					continue;
				}

				sb = new StringBuilder();
				for (Sentence sentence : document.getSentences()) {
					stats.incNumSentences();

					switch (Constants.CONFIG.getChunker()) {
					case COGROO: {
						for (Chunk chunk : sentence.getChunks()) {
							prefix = "";
							for (Token token : chunk.getTokens()) {
								stats.incNumTokens();
								if (Constants.CONFIG.getStemmer() == Configuration.Stemmers.COGROO) {
									lemmas = token.getLemmas();
									s = lemmas.length > 0 ? lemmas[0] : token.getLexeme();
								} else
									s = token.getLexeme();
								sb.append(prefix);
								prefix = "_";
								sb.append(s);
							}
							sb.append(" ");
						}
						break;
					}
					case NONE: {
						for (Token token : sentence.getTokens()) {
							stats.incNumTokens();
							if (Constants.CONFIG.getStemmer() == Configuration.Stemmers.COGROO) {
								lemmas = token.getLemmas();
								s = lemmas.length > 0 ? lemmas[0] : token.getLexeme();
							} else
								s = token.getLexeme();
							sb.append(s);
							sb.append(" ");
						}
						break;
					}
					}
				}
				text = sb.toString();
			}

			// XXX We get the first one.
			rhc = p.getRhc().get(0);
			code = rhc.getIcdClass().getCode();

			if (Constants.STATS) {
				stats.incNumPatientsPerClass(code, 1);
				stats.incNumDocsPerClass(code, numDocs);
			}

			id = p.getRgh();

			year = rhc.getAnoDiagnostico();
			m = rhc.getMetastasis();
			ec = rhc.getEstadioClinico();

			// map.put(text, code);
			// }

			// Iterator<Map.Entry<String, String>> mapIter =
			// map.entrySet().iterator();
			// i = 0;
			// while (mapIter.hasNext()) {
			// i++;
			// if (i % 100 == 0)
			// LOG.trace("2nd phase: imported " + i + " instances.");
			// Map.Entry<String, String> entry = mapIter.next();

			// String text = entry.getKey();
			// String code = entry.getValue();

			inst = new Instance(numAttributes);
			inst.setValue((Attribute) attributes.elementAt(0), id);
			inst.setValue((Attribute) attributes.elementAt(1), year);
			inst.setValue((Attribute) attributes.elementAt(2), text);
			inst.setValue((Attribute) attributes.elementAt(3), m);
			inst.setValue((Attribute) attributes.elementAt(4), ec);
			inst.setValue((Attribute) attributes.elementAt(5), code);
			data.add(inst);
		}

		stats.setNumPatients(i);

		data.setClassIndex(numAttributes - 1);

		this.dataset = data;
		return data;
	}

	public void crossValidate() {
		try {
			LOG.debug("Will cross-validate.");
			eval = new ExtendedEvaluation(dataset);

			long start = System.currentTimeMillis();
			eval.crossValidateModel(model, dataset, 10, new Random(42));
			long end = System.currentTimeMillis();

			LOG.info("Cross-validate in " + ((end - start) / 1000) + " seconds.");
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	public void printClassifier() {
		String filename = "classifier-" + Constants.CONFIG.getStringRepresentation() + ".arff";
		printClassifier(new File(filename));
	}

	public void printClassifier(File file) {
		LOG.debug("Will build classifier.");
		try {
			model.buildClassifier(dataset);
		} catch (Exception e) {
			e.printStackTrace();
			return;
		}

		LOG.debug("Will write file");
		try {
			BufferedWriter bw = new BufferedWriter(new FileWriter(file));
			bw.write(model.toString());
			bw.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	// TODO receber corpora como parâmetros.
	// Permitir tanto ICDC-3 (treinamento = teste), como ICDC-5
	// (validação isolada em PatientIterator e carregada em importDataset()
	// separado).
	public void singleEvaluate() {
		try {
			int totalSize = dataset.numInstances();
			int testSize = (int) (totalSize * Constants.TEST_RATIO);
			int trainSize = totalSize - testSize;

			final Random rand = new Random(42);
			dataset.randomize(rand);

			Instances trainSet = new Instances(dataset, 0, trainSize);
			Instances testSet = new Instances(dataset, trainSize, testSize);

			// TODO same dataset for training and test to estimate maximum
			// (ICDC-3)
			model.buildClassifier(trainSet);
			eval = new ExtendedEvaluation(trainSet);
			eval.evaluateModel(model, testSet);
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	public void resultsAsCsv(File file, int threshold) {
		// TODO DRY singleEvaluate()
		int totalSize = dataset.numInstances();
		int testSize = (int) (totalSize * Constants.TEST_RATIO);
		int trainSize = totalSize - testSize;

		final Random rand = new Random(42);
		dataset.randomize(rand);

		Instances trainSet = new Instances(dataset, 0, trainSize);
		Instances testSet = new Instances(dataset, trainSize, testSize);

		try {
			model.buildClassifier(trainSet);

			Instances filteredTestSet = Filter.useFilter(testSet, ((FilteredClassifier) model).getFilter());

			Instance original = null;
			Instance filtered = null;

			CSVWriter writer = new CSVWriter(new FileWriter(file), ';');

			String[] header = { "raw", "filtered", "correct", "decided", "pos", "rgh", "year", "m", "ec" };
			writer.writeNext(header);

			for (int i = 0; i < testSet.numInstances(); i++) {
				original = testSet.instance(i);
				filtered = filteredTestSet.instance(i);

				double[] dist = model.distributionForInstance(original);
				double cls = model.classifyInstance(original);

				int pos = getCorrectPosition(dist, original.classValue());
				if (pos < threshold)
					continue;

				// inst.setValue((Attribute) attributes.elementAt(0), id);
				// inst.setValue((Attribute) attributes.elementAt(1), year);
				// inst.setValue((Attribute) attributes.elementAt(2), text);
				// inst.setValue((Attribute) attributes.elementAt(3), m);
				// inst.setValue((Attribute) attributes.elementAt(4), ec);
				// inst.setValue((Attribute) attributes.elementAt(5), code);

				String rawText = original.toString(2);
				String correctCode = original.toString(5);
				String filteredText = buildBagWords(filtered);
				String decidedCode = original.attribute(5).value((int) cls);
				String rgh = original.toString(0);
				String year = original.toString(1);
				String m = original.toString(3);
				String ec = original.toString(4);

				String[] entries = { rawText, filteredText, correctCode, decidedCode, String.valueOf(pos), rgh, year, m,
						ec };
				writer.writeNext(entries);
			}

			writer.close();

		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	/**
	 * @todo move to classifier??
	 * @todo junit
	 */
	private int getCorrectPosition(double[] dist, double correct) {
		int pos = 0;
		double min = dist[(int) correct];
		for (int i = 0; i < dist.length; i++) {
			if (dist[i] > min)
				pos++;
		}
		return pos;
	}

	/**
	 * @todo move to classifier??
	 * @todo junit
	 */
	private String buildBagWords(Instance instance) {
		StringBuilder sb = new StringBuilder();
		for (int i = 1; i < instance.numAttributes(); i++) {
			for (int j = 0; j < (int) instance.value(i); j++) {
				sb.append(instance.attribute(i).name());
				sb.append(" ");
			}
		}
		return sb.toString();
	}

	public String classify(String text) {
		// Try something like this:
		//
		// FilteredClassifier model = new FilteredClassifier();
		// model.setFilter(new StringToWordVector());
		// model.setClassifier(new NaiveBayes());
		// model.buildClassifier(trainInsts);
		// System.out.println(model);
		//
		// And then this for classifying your text:
		//
		// for (int i = 0; i < classifyInsts.numInstances(); i++) {
		// classifyInsts.instance(i).setClassMissing();
		// double cls = model.classifyInstance(classifyInsts.instance(i));
		// classifyInsts.instance(i).setClassValue(cls);
		// }
		// System.out.println(classifyInsts);

		final int numAttributes = 6;
		FastVector attributes = new FastVector(numAttributes);

		LOG.debug("Will generate classes.");
		FastVector nominalCids = new FastVector();
		List<Classifiable> icds = DAOFactory.getDAOFactory().getIcdClassDAO().list();
		Iterator<Classifiable> icdIter = icds.iterator();
		while (icdIter.hasNext()) {
			Classifiable c = icdIter.next();
			nominalCids.addElement(c.getCode());
		}

		attributes.addElement(new Attribute("id"));
		attributes.addElement(new Attribute("year"));
		attributes.addElement(new Attribute("text", (FastVector) null));
		attributes.addElement(new Attribute("m", (FastVector) null));
		attributes.addElement(new Attribute("ec", (FastVector) null));
		attributes.addElement(new Attribute("code", nominalCids));

		// XXX Weka BUG. attributes index is incorrectly initialized inside the
		// Instances constructor (reference updated).
		Instances classifyInsts = new Instances("CID", attributes, 1);

		Instance inst = new Instance(numAttributes);
		// inst.setValue((Attribute) attributes.elementAt(0), 1);
		// inst.setValue((Attribute) attributes.elementAt(1), 1);
		inst.setValue((Attribute) attributes.elementAt(2), text);
		// inst.setValue((Attribute) attributes.elementAt(3), 1);
		// inst.setValue((Attribute) attributes.elementAt(4), 1);
		// inst.setValue((Attribute) attributes.elementAt(5), "");

		// inst.setValue(2, text);

		classifyInsts.add(inst);

		classifyInsts.setClassIndex(numAttributes - 1);

		double cls = 0;
		try {
			// classifyInsts.instance(0).setClassMissing();
			cls = model.classifyInstance(classifyInsts.instance(0));
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		classifyInsts.instance(0).setClassValue(cls);
		String ret = classifyInsts.instance(0).classAttribute().value((int) cls);

		return ret;
	}

}
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.HashSet;
import java.util.Set;
import org.apache.log4j.Logger;
import br.usp.ime.icdc.Configuration;
import br.usp.ime.icdc.Configuration.Chunkers;
import br.usp.ime.icdc.Configuration.Classifiers;
import br.usp.ime.icdc.Configuration.Criteria;
import br.usp.ime.icdc.Configuration.MetastasisStatus;
import br.usp.ime.icdc.Configuration.Sections;
import br.usp.ime.icdc.Configuration.SentenceDetectors;
import br.usp.ime.icdc.Configuration.SmoothingTechniques;
import br.usp.ime.icdc.Configuration.Sources;
import br.usp.ime.icdc.Configuration.Stemmers;
import br.usp.ime.icdc.Configuration.Targets;
import br.usp.ime.icdc.Configuration.Tokenizers;
import br.usp.ime.icdc.Configuration.WeightingSchemes;
import br.usp.ime.icdc.Constants;
import br.usp.ime.icdc.dao.DAOFactory;
import br.usp.ime.icdc.dao.TopographyGroupDAO;
import br.usp.ime.icdc.logic.classifier.CipeClassifier;
package br.usp.ime.icdc.run;




public class ClassifierFacade {
	
	private static final Logger LOG = Logger.getLogger(CipeClassifier.class);

	public static void main(String[] args) throws Exception {
		Set<Sections> sections = new HashSet<Sections>();