public class Distiller {

    // all these fields will be injected via setter method
    /**
     * The first step of the actual pipeline: this annotator should decide in
     * which language the document is written.
     */
    private Annotator languageDetector;

    /**
     * The annotation pipelines. There should be one for language.
     */
    private Map<Locale, Pipeline> pipelines = new HashMap<>();

    /**
     * The language of the document.
     */
    private Locale documentLocale = null;

    /**
     * The Object that will contain the text all its annotations.
     */
    private Blackboard blackboard;
    
    /**
     * The verbose mode flag. If the flag is set to true, Distiller will
     * print information on the work he's doing on stdout. Default is false.
     */
    private boolean verbose = false;

    /**
     * Sets the language detector.
     *
     * @param languageDetector the language detector.
     */
    @Required
    public void setLanguageDetector(Annotator languageDetector) {
        this.languageDetector = languageDetector;
    }

    /**
     * Set the annotation pipelines.
     *
     * @param pipelines the annotation pipelines.
     */
    @Required
    public void setPipelines(HashMap<Locale, Pipeline> pipelines) {
        this.pipelines = pipelines;
    }

    /**
     * Adds a pipeline to the pipelines map.
     *
     * @param locale the language that the pipeline will process
     * @param pipeline the pipeline to add
     */
    public void addPipeline(Locale locale, Pipeline pipeline) {
        pipelines.put(locale, pipeline);
    }

    /**
     * Sets the locale in which the text extraction will be performed. The value
     * should be null for auto-detection of the locale of the IETF formatted
     * language tag if manual locale setting is desired. For example, passing
     * "en-US" will set the locale to English. An empty locale equals to the
     * "auto" parameter.
     *
     * @param locale the locale to use while processing the text.
     */
    public void setLocale(Locale locale) throws IllegalArgumentException {
        this.documentLocale = locale;
    }

    /**
     * Gets the blackboard.
     * 
     * @return the blackboard.
     */
    public Blackboard getBlackboard() {
        return blackboard;
    }

    /**
     * Sets the verbose mode of the Distiller.
     * 
     * @param verbose true to display information of the distillation process;
     * false for silent distillation.
     */
    public void setVerbose(boolean verbose) {
        this.verbose = verbose;
    }

    /**
     * Gets the verbose mode of the Distiller.
     * @return TRUE if the Distillation is set to VERBOSE mode.
     */
    public boolean  getVerbose() {
        return verbose;
    }
    
    
    /**
     * Perform the extraction of keyphrases of a specified string, and returns
     * the blackboard filled with document and annotations.
     *
     * @param text the text to distill.
     * @return the blackboard filled with the processed text
     */
    public Blackboard distillToBlackboard(String text) {

        blackboard = new Blackboard();
        blackboard.createDocument(text);

        if (documentLocale == null) // if no language has been set, automatically detect it.
        {
            if (languageDetector != null) {
                languageDetector.annotate(blackboard, blackboard.getStructure());
            } else // but if there's no language and no language detector, 
            // throw an exception.
            {
                throw new DistillerException(
                        "I can't decide the language of the document: no language is specified and no language detector is set.");
            }
        } else // set the pre-determined language
        {
            blackboard.getStructure().setLanguage(documentLocale);
        }

        Pipeline pipeline = pipelines.get(blackboard.getStructure().getLanguage());

        if (pipeline == null) {
            throw new DistillerException("No pipeline for the language "
                    + blackboard.getStructure().getLanguage().getLanguage());
        }

        for (Stage stage : pipeline.getStages()) {
            
            if (verbose) {
                System.out.println(String.format("Running %s...",
                        getStageName(stage)));
            }
            
            stage.run(blackboard);
        }
        
        if (verbose) {
            System.out.println("Extraction complete!");
            System.out.println();
        }

        return blackboard;
    }

    /**
     * Perform the extraction of keyphrases of a specified string, and returns a
     * developer-friendly object that allows quick access to the extracted
     * information.
     *
     * @param text the text to extract
     * @return the distilled output
     */
    public DistilledOutput distill(String text) {

        DistilledOutput output = new DistilledOutput();

        output.setOriginalText(text);

        distillToBlackboard(text);

        output.setDetectedLanguage(blackboard.getStructure().
                getLanguage().getLanguage());

        // Copy the grams, sorted by descending score
        output.initializeGrams(blackboard.getKeyphrases().size());
        
        Collection<Gram> grams = blackboard.getKeyphrases();
        Map<Keyphrase, Double> scoredGrams = new HashMap<>();

        for (Gram g : grams) {
            Keyphrase k = (Keyphrase)g;
            scoredGrams.put(k, k.getFeature(GenericEvaluatorAnnotator.SCORE));
        }

        List<Map.Entry<Keyphrase, Double>> sortedGrams
                = scoredGrams.entrySet().stream().sorted(
                        Collections.reverseOrder(Map.Entry.comparingByValue()))
                .collect(Collectors.toList());

        for (int i = 0; i < output.getGrams().length; i++) {
            DetectedGram gram = output.getGrams()[i];
            Keyphrase originalGram = sortedGrams.get(i).getKey();
            gram.setSurface(originalGram.getSurface());
            gram.setKeyphraseness(originalGram.getFeature(
                    it.uniud.ailab.dcore.annotation.annotators.GenericEvaluatorAnnotator.SCORE));

            UriAnnotation wikiAnn = (UriAnnotation) originalGram.getAnnotation(WIKIURI);
            if (wikiAnn != null) {
                gram.setConceptName(wikiAnn.getSurface());
                gram.setConceptPath(wikiAnn.getUri().toASCIIString());
            }
        }

        output.initializeRelatedConcepts(blackboard.getAnnotations(
                WikipediaInferenceAnnotator.RELATED).size());

        for (int i = 0; i < output.getRelatedConcepts().length; i++) {
            InferredConcept related = output.getRelatedConcepts()[i];
            InferenceAnnotation originalRelatedConcept
                    = (InferenceAnnotation) blackboard.getAnnotations(
                            WikipediaInferenceAnnotator.RELATED).get(i);

            related.setConcept(originalRelatedConcept.getConcept());
            related.setConceptPath(originalRelatedConcept.getUri().toASCIIString());
            related.setScore(originalRelatedConcept.getScore());
        }

        output.initializeHypernyms(blackboard.getAnnotations(
                WikipediaInferenceAnnotator.HYPERNYMS).size());

        for (int i = 0; i < output.getHypernyms().length; i++) {
            InferredConcept hypernym = output.getHypernyms()[i];
            InferenceAnnotation originalHypernym = (InferenceAnnotation) blackboard.getAnnotations(
                    WikipediaInferenceAnnotator.HYPERNYMS).get(i);

            hypernym.setConcept(originalHypernym.getConcept());
            hypernym.setConceptPath(originalHypernym.getUri().toASCIIString());
            hypernym.setScore(originalHypernym.getScore());
        }

        output.setExtractionCompleted(true);

        return output;
    }
} public class DistillerFactory {

    /**
     * Instantiates a Distiller object using the default XML configuration; if
     * it's not available, uses the safer (but less precise) code configuration,
     * which excludes TagMe and inference from the distillation process.
     *
     * @return a Distiller ready to work.
     */
    public static Distiller getDefault() {

        // While the code under this comment may be ugly, it works.
        //
        // It tries to load the default XML config. If the load fails, throws
        // the cause of the failure and immediately catches it.
        //
        // If the config file is not accessible (due to permission, 
        // non-existance, or whatever), the exception is caught and the default
        // code configuration runs.
        //
        // Otherwise, the exception is re-thrown, so that the developer can
        // handle the errors in the config file, which are the other most likely
        // cause of failure of configuration loading falure.
        try {
            return getDefaultXML();
        } catch (BeanDefinitionStoreException bsde) {
            try {
                throw bsde.getCause();
            } catch (IOException ioe) {
                // the configuration file does not exist or is not accessible:
                // load the fallback configuration
                System.out.println(
                        "Distiller config file not found: using fallback configuration");
                return getDefaultCode();
            } catch (Throwable te) {
                throw bsde;
            }
        }
    }

    /**
     * Instantiates a Distiller object using the default evaluation
     * configuration.
     *
     * @return a Distiller ready to work.
     */
    public static Distiller getDefaultEval() {

        try {
            ApplicationContext context = new ClassPathXmlApplicationContext("eval.xml");
            return (Distiller) context.getBean("distiller");
        } catch (BeanDefinitionStoreException bsde) {
            try {
                throw bsde.getCause();
            } catch (IOException ioe) {
                // the configuration file does not exist or is not accessible:
                throw new RuntimeException("FATAL: Impossible to load the default evaluation pipeline.", ioe);
            } catch (Throwable te) {
                throw bsde;
            }
        }
    }

    /**
     * Instantiates a Distiller object using the specified configuration and
     * returns it.
     *
     * @param path the path where the config file is located
     * @return a Distiller ready to work.
     */
    public static Distiller loadFromXML(File path) {
        // We add the file:// thing before because, for some ??? reason, 
        // the Spring Framework decided that all paths are relative paths.

        // So, we get the file, retrieve is absolute path, and then add the
        // file:// prefix to be sure that the Spring Frameworks treats 
        // all paths as absolute paths. 
        // This is less problematic, because the Java platform will handle the
        // File and produce its absolute path, even if it has been created
        // using a relative one.
        ApplicationContext context = new FileSystemXmlApplicationContext(
                "file://" + path.getAbsolutePath());
        return (Distiller) context.getBean("distiller");
    }

    /**
     * Instantiates a Distiller object using the default configuration and
     * returns it. Please note that you should create a config.xml file and copy
     * the content of default.xml inside it to get the framework to work.
     *
     * @return a Distiller ready to work.
     */
    public static Distiller getDefaultXML() {
        ApplicationContext context = new ClassPathXmlApplicationContext("config.xml");
        return (Distiller) context.getBean("distiller");
    }

    /**
     * Instantiates a Distiller object using a configuration packaged in the
     * Distiller JAR file and returns it.
     *
     * @param configPath the path of the pipeline
     * @return a Distiller ready to work.
     */
    public static Distiller loadFromPackagedXML(String configPath) {
        
        ApplicationContext context = new ClassPathXmlApplicationContext(configPath);
        return (Distiller) context.getBean("distiller");
    }

    public static Distiller getDefaultCode() {
        Distiller d = new Distiller();

        // set the language detector tool
        d.setLanguageDetector(new CybozuLanguageDetectorAnnotator());

        // build the pipeline
        Pipeline p = new Pipeline();
        // split the text
        p.addStage(new OpenNlpBootstrapperAnnotator());
        // add wikipedia tags to tokens

        //annotate tokens with stemming
        p.addStage(new PorterStemmerAnnotator());
        // Uncomment the lines below to use the TagMe service
        // TagMeTokenAnnotator tagme = new TagMeTokenAnnotator();        
        // tagme.setApiKey("INSERT KEY HERE");        
        // p.addStage(tagme);
        // generate ngrams
        p.addStage(new SimpleNGramGeneratorAnnotator());

        // remove stopwords
        p.addStage(new StopwordSimpleFilterAnnotator());

        // annotate ngrams
        p.addStage(new StatisticalAnnotator());

        // Uncomment to use TagMe
        // p.addStage(new TagMeGramAnnotator());
        // Uncomment to use the emotional intensity annotator.
        // This way you'll see how different annotators lead to different
        // keyphrases detection
        // p.addStage(new SyuzhetAnnotator());
        // evaluate ngram features        
        LinearEvaluatorAnnotator evaluator = new LinearEvaluatorAnnotator();
        evaluator.addWeight(StatisticalAnnotator.DEPTH, 0.15);
        evaluator.addWeight(StatisticalAnnotator.HEIGHT, 0.25);
        evaluator.addWeight(StatisticalAnnotator.LIFESPAN, 0.1);
        evaluator.addWeight(StatisticalAnnotator.FREQUENCY_SENTENCE, 0.1);
        evaluator.addWeight(GenericNGramGeneratorAnnotator.NOUNVALUE, 0.3);
        evaluator.addWeight(GenericWikipediaAnnotator.WIKIFLAG, 0.1);

        p.addStage(evaluator);

        // Uncomment the line below to infer concepts.
        // Watch out: the inference process sends lots of requests to Wikipedia, 
        // so it significantly slows down the process
        // p.addStage(new WikipediaInferenceAnnotator());
        // filter results
        p.addStage(new SkylineGramFilterAnnotator());

        // remove redundant grams
        //p.addStage(new GramMergerAnnotator());
        p.addStage(new GramPrinter());
        p.addStage(new SentencePrinter());

        d.addPipeline(Locale.ENGLISH, p);
        d.addPipeline(Locale.ITALIAN, p);

        return d;
    }

    public static Distiller getStanfordCode() {
        Distiller d = new Distiller();

        // set the language detector tool
        d.setLanguageDetector(new CybozuLanguageDetectorAnnotator());

        // build the pipeline
        Pipeline p = new Pipeline();
        // split the text
        p.addStage(new StanfordBootstrapperAnnotator());
        // add wikipedia tags to tokens

        //annotate tokens with stemming
        p.addStage(new PorterStemmerAnnotator());

        // Uncomment the lines below to use the TagMe service
        // TagMeTokenAnnotator tagme = new TagMeTokenAnnotator();        
        // tagme.setApiKey("INSERT KEY HERE");        
        // p.addStage(tagme);
        // generate ngrams
        p.addStage(new SimpleNGramGeneratorAnnotator());

//        // remove stopwords
        p.addStage(new StopwordSimpleFilterAnnotator());
//
//        // annotate ngrams
        p.addStage(new StatisticalAnnotator());
        p.addStage(new CoreferenceResolverAnnotator());
        p.addStage(new ChunkingNerAnnotator());
        // Uncomment to use TagMe
        // p.addStage(new TagMeGramAnnotator());
        // Uncomment to use the emotional intensity annotator.
        // This way you'll see how different annotators lead to different
        // keyphrases detection
        // p.addStage(new SyuzhetAnnotator());
        // evaluate ngram features        
        LinearEvaluatorAnnotator evaluator = new LinearEvaluatorAnnotator();
        evaluator.addWeight(StatisticalAnnotator.DEPTH, 0.15);
        evaluator.addWeight(StatisticalAnnotator.HEIGHT, 0.25);
        evaluator.addWeight(StatisticalAnnotator.LIFESPAN, 0.1);
        evaluator.addWeight(StatisticalAnnotator.FREQUENCY_SENTENCE, 0.1);
        evaluator.addWeight(GenericNGramGeneratorAnnotator.NOUNVALUE, 0.3);
        evaluator.addWeight(GenericWikipediaAnnotator.WIKIFLAG, 0.1);
        evaluator.addWeight(CoreferenceResolverAnnotator.NUMBER_OF_REFERENCE, 0.2);
        evaluator.addWeight(CoreferenceResolverAnnotator.IN_ANAPHORA, 0.2);
        evaluator.addWeight(ChunkingNerAnnotator.IS_NER, 0.2);

        p.addStage(evaluator);

        p.addStage(new GramPrinter());

        d.addPipeline(Locale.ENGLISH, p);

        return d;
    }

}
 public abstract class GenericDataset implements Comparator<String> {
    
    /**
     * The path where the evaluator will find the input documents and the
     * gold standard results.
     */
    protected final String datasetPath;
    
    /**
     * The training documents. Each document has an identifier and a content.
     */
    private Map<String, String> trainingDocuments;
    /**
     * The expected answers for the test documents. Each list of training answers
     * is paired to the identifier of the corresponding document.
     */
    private Map<String, String[]> trainingAnswers;
    
    /**
     * The test documents. Each document has an identifier and a content.
     */
    private Map<String, String> testDocuments;
    /**
     * The expected answers for the test documents. Each list of test answers
     * is paired to the identifier of the corresponding document.
     */
    private Map<String, String[]> testAnswers;
    
    /**
     * A value that indicates wheter if the documents have already been loaded
     * or not.
     */
    private boolean isLoaded;
    
    /**
     * An output-friendly string that identifies the dataset.
     */
    private final String identifier;
    
    /**
     * Create a concrete dataset that will contain the data contained in the 
     * specified path.
     * 
     * @param datasetPath The folder where the Dataset will look for the document. 
     * @param identifier An output-friendly string that identifies the dataset.
     */
    public GenericDataset(String datasetPath,String identifier) {
        this.datasetPath = datasetPath;
        this.isLoaded = false;
        this.identifier = identifier;
    }
    
    /**
     * Get an output-friendly string that identifies the dataset.
     * 
     * @return an output-friendly string that identifies the dataset.
     */
    public String getIdentifier() {
        return identifier;
    }
    
    /**
     * Compares a <b>candidate</b> item with a <b>dataset provided</b> item. Please 
     * note that the the object to test <b>must</b> be passed as first parameter,
     * while the object to test against <b>must</b> be passed as second 
     * parameter.
     * 
     * @param o1 the object to test, generated by the Distiller
     * @param o2 the reference object, provided by the training set.
     * @return 0 if o1 and o2 are equal, another number (indetermined) otherwise.
     */
    @Override
    public abstract int compare(String o1, String o2);
    
    /**
     * Loads the input documents and returns them.
     *
     * @return the input documents mapped with their identifier.
     */
    protected abstract Map<String, String> loadTestSet(); 

    /**
     * Loads the test set answers and returns them.
     *
     * @return the test set answers mapped with the identifier of the
     * document they belong to.
     */
    protected abstract Map<String, String[]> loadTestAnswers();
    
    /**
     * Loads the training documents and returns them.
     *
     * @return the training documents mapped with their identifier.
     */
    protected abstract Map<String, String> loadTrainingSet(); 

    /**
     * Loads the training set expected answers and returns them.
     *
     * @return the training set answers mapped with the identifier of the
     * document they belong to.
     */
    protected abstract Map<String, String[]> loadTrainingAnswers();

    /**
     * Get the test set documents for the dataset.
     * 
     * @return the test set for the dataset.
     */
    public Map<String, String> getTestSet() {
        return testDocuments;
    }

    /**
     * Get the test set results for the dataset.
     * 
     * @return the expected results on the test set of the dataset.
     */
    public Map<String, String[]> getTestAnswers() {
        return testAnswers;
    }
    
    /**
     * Get the training set documents for the dataset.
     * 
     * @return the training set for the dataset.
     */
    public Map<String, String> getTrainingSet() {
        return trainingDocuments;
    }

    /**
     * Get the training set results for the dataset.
     * 
     * @return the expected results on the training set of the dataset.
     */
    public Map<String, String[]> getTrainingAnswers() {
        return trainingAnswers;
    }
    
    /**
     * Gets the dataset status: true if the data has already been loaded,
     * false otherwise.
     * 
     * @return the dataset status.
     */
    public boolean isLoaded() {
        return isLoaded;
    }
    
    /**
     * Sets the dataset status.
     * 
     * @param isLoaded the dataset status.
     */
    private void setLoaded(boolean isLoaded) {
        this.isLoaded = isLoaded;
    }
    
    /**
     * Load the dataset. 
     */
    public void load() {
        trainingDocuments = loadTrainingSet();
        testDocuments = loadTestSet();
        trainingAnswers = loadTrainingAnswers();
        testAnswers = loadTestAnswers();
        
        setLoaded(true);
    }

    /**
     * Get the folder that contains the training set.
     *
     * @return the folder that contains the training set.
     */
    public abstract String getTrainingFolder();

    /**
     * Get the folder that contains the test set.
     *
     * @return the folder that contains the test set.
     */
    public abstract String getTestFolder();
    
} public class SemEval2010 extends GenericDataset {

    public SemEval2010(String goldStandardPath) {
        super(goldStandardPath, "SEMEVAL2010");
    }

    @Override
    public Map<String, String> loadTestSet() {

        Map<String, String> documents = new HashMap<>();

        try {
            File[] dir = new File(datasetPath + 
                    FileSystem.getSeparator() +
                    "test").listFiles();
            Arrays.sort(dir);

            for (File f : dir) {
                String document = String.join(
                        " ",
                        Files.readAllLines(
                                f.toPath(), StandardCharsets.UTF_8));

                String docName = f.getName() ;//.substring(0, f.getName().indexOf("."));

                documents.put(docName, document);
            }
        } catch (IOException ex) {
            Logger.getLogger(SemEval2010.class.getName()).log(Level.SEVERE, null, ex);
        }

        return documents;
    }

    @Override
    public Map<String, String[]> loadTestAnswers() {

        // default is a zero-sized output array
        Map<String, String[]> keyphrases = null;

        try {

            File gold = new File(datasetPath + 
                    FileSystem.getSeparator() +
                    "test_answer" +
                    FileSystem.getSeparator() +
                    "test.combined.stem.final");

            Map<String, String[]> buffer = new HashMap<>();

            // navigate through the lines
            try (Stream<String> lines = Files.lines(gold.toPath(), StandardCharsets.UTF_8)) {

                for (String line : (Iterable<String>) lines::iterator) {
                    List<String> documentKPs = new ArrayList<>();
                    // remove the name of the document
                    String kpLine = line.substring(line.indexOf(':') + 2);
                    for (String kp : kpLine.split(",")) {
                        documentKPs.add(kp);
                    }

                    buffer.put(line.substring(0, line.indexOf(':') - 1)
                            +".txt.final"
                            ,
                            documentKPs.toArray(new String[documentKPs.size()]));
                }
            }

            // copy the gold standard to the output array
            keyphrases = buffer;

        } catch (IOException ex) {
            Logger.getLogger(SemEval2010.class.getName()).log(Level.SEVERE, null, ex);
            throw new RuntimeException("Unable to load gold standard", ex);
        }

        return keyphrases;
    }

    @Override
    protected Map<String, String> loadTrainingSet() {
        Map<String, String> documents = new HashMap<>();

        try {
            File[] dir = new File(datasetPath + 
                    FileSystem.getSeparator() +
                    "train").listFiles();
            Arrays.sort(dir);

            for (File f : dir) {

                // skip files containing results
                if (!f.getName().startsWith("train")) {

                    String document = String.join(
                            " ",
                            Files.readAllLines(
                                    f.toPath(), StandardCharsets.UTF_8));

                    String docName = f.getName(); //.substring(0, f.getName().indexOf("."));

                    documents.put(docName, document);
                }
            }
        } catch (IOException ex) {
            Logger.getLogger(SemEval2010.class.getName()).log(Level.SEVERE, null, ex);
        }

        return documents;
    }

    @Override
    protected Map<String, String[]> loadTrainingAnswers() {

        Map<String, String[]> keyphrases = null;

        try {

            File gold = new File(datasetPath + 
                    FileSystem.getSeparator() +
                    "train_answer" +
                    FileSystem.getSeparator() +
                    "train.combined.stem.final");

            Map<String, String[]> buffer = new HashMap<>();

            // navigate through the lines
            try (Stream<String> lines = Files.lines(gold.toPath(), StandardCharsets.UTF_8)) {

                for (String line : (Iterable<String>) lines::iterator) {
                    List<String> documentKPs = new ArrayList<>();
                    // remove the name of the document
                    String kpLine = line.substring(line.indexOf(':') + 2);
                    for (String kp : kpLine.split(",")) {
                        documentKPs.add(kp);
                    }

                    buffer.put(line.substring(0, line.indexOf(':') - 1)
                            +".txt.final",
                            documentKPs.toArray(new String[documentKPs.size()]));
                }
            }

            // copy the gold standard to the output array
            keyphrases = buffer;

        } catch (IOException ex) {
            Logger.getLogger(SemEval2010.class.getName()).log(Level.SEVERE, null, ex);
            throw new RuntimeException("Unable to load gold standard", ex);
        }

        return keyphrases;
    }

    /**
     * Compares a <b>candidate</b> item with a <b>dataset provided</b> item. Please
     * note that the the object to test <b>must</b> be passed as first
     * parameter, while the object to test against <b>must</b> be passed as
     * second parameter.
     *
     * @param o1 the object to test, generated by the Distiller
     * @param o2 the reference object, provided by the training set.
     * @return 0 if o1 and o2 are equal, another number (indetermined)
     * otherwise.
     */
    @Override
    public int compare(String o1, String o2) {
        PorterStemmer stemmer = new PorterStemmer();
        String[] tokens = o1.split(" ");

        for (int i = 0; i < tokens.length; i++) {

            // this is necessary because SEMEVAL tokenizes in a different
            // way, using not only spaces but also hyphens to separate 
            // tokens 
            if (tokens[i].indexOf('-') < 0) {
                tokens[i] = stemmer.stem(tokens[i]);
            } else {
                String[] subtokens = tokens[i].split("-");
                for (int j = 0; j < subtokens.length; j++) {
                    subtokens[j] = stemmer.stem(subtokens[j]);
                }
                
                tokens[i] = String.join("-",subtokens);
            }
        }
        o1 = String.join(" ", tokens);

        boolean found = false;

        if (o2.indexOf('+') < 0) {
            found = o1.equals(o2);
        } else {
            String[] goldKPs = o2.split("\\+");
            for (int i = 0; i < goldKPs.length && !found; i++) {
                found = o1.equals(goldKPs[i]);
            }
        }

        return found ? 0 : 1;
    }

    /**
     * {@inheritDoc} 
     * @return {@inheritDoc} 
     */
    @Override
    public String getTrainingFolder() {
        return datasetPath + "/train";
    }

    /**
     * {@inheritDoc} 
     * @return {@inheritDoc} 
     */
    @Override
    public String getTestFolder() {
        return datasetPath + "/test";
    }

} public class KeyphraseEvaluatorAll extends Evaluator {

    private static boolean verbose = false;

    /**
     * An evaluator for the Keyphrase Extraction task.
     *
     * @param goldStandard the gold standard to evaluate.
     */
    public KeyphraseEvaluatorAll(GenericDataset goldStandard) {
        super(goldStandard);
    }

    /**
     * Evaluate the keyphrases using the given dataset and settings.
     *
     * @param pipeline the distiller instance to evaluate.
     * @return the metrics and their score.
     */
    @Override
    public Map<String, Double> evaluate(Distiller pipeline) {

        verbose = pipeline.getVerbose();

        if (!goldStandard.isLoaded()) {
            goldStandard.load();
        }

        int docIndex = 0;
        double precision = 0;
        double recall = 0;
        double fmeasure = 0;

        for (Map.Entry<String, String> documentEntry
                : goldStandard.getTestSet().entrySet()) {

            String document = documentEntry.getValue().replace("\\n", " ");

            System.out.println("Evaluating document " + ++docIndex
                    + " of " + goldStandard.getTestSet().size() + "...");

            System.out.println("Document identifier: " + documentEntry.getKey());
            System.out.println("Document's first 40 chars: "
                    + document.substring(0, 40) + "...");

            DistilledOutput output = pipeline.distill(document);

            String[] kps = new String[pipeline.getBlackboard().getKeyphrases().size()];
            for (int i = 0; i < kps.length; i++) {
                kps[i] = pipeline.getBlackboard()
                        .getKeyphrases().get(i).getSurface();
            }

            double docPrecision = computePrecision(
                    kps,
                    goldStandard.getTestAnswers().get(documentEntry.getKey()));
            double docRecall = computeRecall(
                    kps,
                    goldStandard.getTestAnswers().get(documentEntry.getKey()));
            double docFMeasure = computeFMeasure(docPrecision, docRecall);

            System.out.println("Precision   : " + docPrecision);
            System.out.println("Recall      : " + docRecall);
            System.out.println("FMeasure    : " + docFMeasure);

            precision = precision + docPrecision;
            recall = recall + docRecall;
            fmeasure = fmeasure + docFMeasure;
        }

        precision = precision / goldStandard.getTestSet().size();
        recall = recall / goldStandard.getTestSet().size();
        fmeasure = fmeasure / goldStandard.getTestSet().size();

        System.out.println();
        System.out.println("*** EVALUATION COMPLETE ***");
        System.out.println();
        System.out.println("Precision   : " + precision);
        System.out.println("Recall      : " + recall);
        System.out.println("F-Measure   : " + fmeasure);
        System.out.println();

        return null;
    }

    private double computePrecision(String[] kps, String[] goldKeyphrase) {
        double matches = 0;
        for (int i = 0; i < kps.length; i++) {
            for (int j = 0; j < goldKeyphrase.length; j++) {
                if (goldStandard.compare(
                        kps[i].toLowerCase(), goldKeyphrase[j]) == 0) {
                    matches++;
                }
            }
        }

        return matches / (kps.length * 1.0);
    }
 
    private double computeRecall(String[] kps, String[] goldKeyphrase) {
        double matches = 0;
        for (int j = 0; j < goldKeyphrase.length; j++) {

            boolean matched = false;

            for (int i = 0; i < kps.length && !matched; i++) {
                if (goldStandard.compare(
                        kps[i].toLowerCase(), goldKeyphrase[j]) == 0) {
                    matches++;
                    matched = true;
                }
            }

            if (!matched && verbose) {
                System.out.println(
                        "Non matched keyphrase: " + goldKeyphrase[j]);
            }
        }

        return matches / (goldKeyphrase.length * 1.0);
    }

    private double computeFMeasure(double precision, double recall) {

        return recall > 0 && precision > 0
                ? (2 * precision * recall) / (precision + recall)
                : 0;
    }

} public class KeyphraseTrainingSetGenerator extends TrainingSetGenerator {

    /**
     * A training set generator for the Keyphrase Extraction task.
     *
     * @param goldStandard the gold standard to evaluate against.
     */
    public KeyphraseTrainingSetGenerator(GenericDataset goldStandard) {
        super(goldStandard);
    }

    @Override
    public List<Pair<String, GenericSheetPrinter>> generateTrainingSet(Distiller pipeline) {
        if (!goldStandard.isLoaded()) {
            goldStandard.load();
        }

        List<Pair<String, GenericSheetPrinter>> outputFiles
                = doWork(pipeline,
                        goldStandard.getTrainingSet(),
                        goldStandard.getTrainingAnswers());

        return outputFiles;
    }

    @Override
    public List<Pair<String, GenericSheetPrinter>> generateTestSet(Distiller pipeline) {
        if (!goldStandard.isLoaded()) {
            goldStandard.load();
        }

        List<Pair<String, GenericSheetPrinter>> outputFiles
                = doWork(pipeline,
                        goldStandard.getTestSet(),
                        goldStandard.getTestAnswers());

        return outputFiles;
    }

    private List<Pair<String, GenericSheetPrinter>> doWork(Distiller pipeline,
            Map<String, String> workingSet,
            Map<String, String[]> workingAnswers) {

        int docIndex = 0;

        List<Pair<String, GenericSheetPrinter>> outputFiles = new ArrayList<>();
        for (Map.Entry<String, String> documentEntry
                : workingSet.entrySet()) {

            String document = documentEntry.getValue().replace("\\n", " ");

            System.out.println("Evaluating document " + ++docIndex
                    + " of " + workingSet.size() + "...");

            System.out.println("Document identifier: " + documentEntry.getKey());
            System.out.println("Document's first 40 chars: "
                    + document.substring(0, 40) + "...");

            String[] answers
                    = workingAnswers.
                    get(documentEntry.getKey());
            
            IOBlackboard.setCurrentDocument(
                    IOBlackboard.getDocumentsFolder() +
                    FileSystem.getSeparator() +
                    documentEntry.getKey());

            Blackboard b = pipeline.distillToBlackboard(document);

            Collection<Gram> candidates
                    = b.getKeyphrases();
            
            
            for (Gram gram : candidates) {
                Keyphrase candidate = (Keyphrase) gram;
                candidate.putFeature(goldStandard.getIdentifier(), 0);

                boolean found = false;
                for (int i = 0; !found && i < answers.length; i++) {
                    String answer = answers[i];
                    if (goldStandard.compare(gram.getSurface(), answer) == 0) {
                        candidate.putFeature(goldStandard.getIdentifier(), 1);
                        found = true;
                    }
                }
            }

            CsvPrinter printer = new CsvPrinter();
            printer.loadKeyphrases(b);
            printer.addToAll("DocID",documentEntry.getKey());
            outputFiles.add(new Pair<>(documentEntry.getKey(), printer));
        }
        return outputFiles;
    }

} public class CsvPrinter extends GenericSheetPrinter {

    /**
     * The delimiter used to separate values.
     */
    private final char delimiter;

    /**
     * Flag used to determine if the output file should contain the column
     * headers or not.
     */
    private final boolean printHeaders;

    /**
     * Default field delimiter.
     */
    public static final char DEFAULT_DELIMITER = ',';
    /**
     * Default print headers option.
     */
    public static final boolean DEFAULT_HEADERS = true;
    /**
     * Default allow duplicates options.
     */
    public static final boolean DEFAULT_DUPLICATES = false;

    /**
     * Instantiates the {@code CsvPrinter} with the default options.
     */
    public CsvPrinter() {
        this(DEFAULT_DELIMITER, DEFAULT_HEADERS,DEFAULT_DUPLICATES);
    }

    /**
     * Instantiates the {@code CsvPrinter} with the default header options.
     *
     * @param delimiter delimiter of the records in a row
     */
    public CsvPrinter(char delimiter) {
        this(delimiter, DEFAULT_HEADERS,DEFAULT_DUPLICATES);
    }

    /**
     * Instantiates the {@code CsvPrinter} with the specified options.
     *
     * @param delimiter delimiter of the records in a row
     * @param printHeaders true if the printer should write the headers of the
     * table; false otherwise.
     * @param allowDuplicates true if lines with the same ID are allowed;
     * false otherwise.
     */
    public CsvPrinter(char delimiter, boolean printHeaders,boolean allowDuplicates) {
        super(allowDuplicates);
        this.delimiter = delimiter;
        this.printHeaders = printHeaders;
    }

    /**
     * Write the annotations contained in the document n-grams in the specified
     * file.
     *
     * @param fileName the name of the file to write.
     * @param b the blackboard to analyze.
     */
    public void writeGrams(String fileName, Blackboard b) {

        this.loadKeyphrases(b);
        writeFile(fileName);
    }

    /**
     * Write the annotations contained in the document sentences in the
     * specified file.
     *
     * @param fileName the name of the file to write.
     * @param b the blackboard to analyze.
     */
    public void writeSentences(String fileName, Blackboard b) {
        this.loadSentences(b.getStructure());
        writeFile(fileName);
    }

    /**
     * Writes the annotations contained in the printer on the specified file.
     * 
     * @param fileName the path where to write.
     */
    @Override
    public void writeFile(String fileName) {

        try {
            CSVWriter writer
                    = new CSVWriter(new FileWriter(fileName),
                            delimiter, CSVWriter.DEFAULT_QUOTE_CHARACTER);

            writer.writeNext(getHeaders().
                    toArray(new String[getHeaders().size()]), false);

            // build the rows            
            for (Map<String, Either<String, Number>> row : this.getRows()) {

                String[] rowArray = new String[getHeaders().size()];

                for (int i = 0; i < getHeaders().size(); i++) {
                    String header = getHeaders().get(i);
                    Either<String, Number> cell = row.get(header);

                    if (cell == null) {
                        if (getHeaderTypes().get(i).isLeft()) {
                            rowArray[i] = "";
                        } else {
                            rowArray[i] = "0";
                        }
                    } else if (cell.isLeft()) { // the cell is a string
                        rowArray[i] = cell.getLeft();
                    } else { // the cell is a number
                        rowArray[i]
                                = // if there's no decimal part in the numeric
                                // value, avoid printing ".0"
                                cell.getRight().doubleValue()
                                == Math.floor(cell.getRight().doubleValue())
                                        ? String.format(
                                                Locale.US, "%d",
                                                cell.getRight().intValue())
                                        : String.format(
                                                Locale.US, "%f",
                                                cell.getRight().doubleValue());
                    }
                }

                writer.writeNext(rowArray, false);
            }

            writer.close();

        } catch (IOException ex) {
            Logger.getLogger(CsvPrinter.class.getName()).log(Level.SEVERE,
                    "Error while writing CSV file", ex);
        }
    }

} public abstract class GenericSheetPrinter {

    private List<String> headers;
    private List<Map<String, Either<String, Number>>> rows;
    private List<Either<String, Number>> headerTypes;
    private final boolean allowDuplicates;

    private final String ID_COLUMN = "ID";

    /**
     * Generates a generic sheet printer specifying whether it should allow
     * lines with the same ID or not.
     *
     * @param allowDuplicates true if lines with the same ID are allowed; false
     * otherwise.
     */
    protected GenericSheetPrinter(boolean allowDuplicates) {
        this.allowDuplicates = allowDuplicates;
        init();
    }

    /**
     * Writes the annotations contained in the printer on the specified file.
     *
     * @param fileName the path where to write.
     */
    public abstract void writeFile(String fileName);

    /**
     * Returns the headers detected by the printer.
     *
     * @return the headers of the table.
     */
    public List<String> getHeaders() {
        return headers;
    }

    /**
     * Return the types of the headers detected by the printer.
     *
     * @return the types of the headers of the table.
     */
    public List<Either<String, Number>> getHeaderTypes() {
        return headerTypes;
    }

    /**
     * Returns the rows detected by the printer.
     *
     * @return the rows of the table.
     */
    public List<Map<String, Either<String, Number>>> getRows() {
        return rows;
    }

    /**
     * Searches the table to check if contains a row with a certain identifier.
     *
     * @param id the identifier of the row to search
     * @return true if the table contains the row
     */
    public boolean containsRow(String id) {
        for (Map<String, Either<String, Number>> row : rows) {
            if (row.get(ID_COLUMN).getLeft().equals(id)) {
                return true;
            }
        }

        return false;
    }

    /**
     * Add all the values of another printer into the current printer.
     *
     * @param p the printer to merge into the current one.
     */
    public void addPrinter(GenericSheetPrinter p) {

        // merge headers
        for (int i = 0; i < p.getHeaders().size(); i++) {
            int h;
            if ((h = headers.indexOf(p.getHeaders().get(i))) >= 0) {
                if (!(headerTypes.get(h).isLeft()
                        == p.getHeaderTypes().get(i).isLeft())) {
                    throw new UnsupportedOperationException("Trying to merge "
                            + "header with the same name but different type: "
                            + p.getHeaders().get(i));
                }
            } else {
                headers.add(p.getHeaders().get(i));
                headerTypes.add(p.getHeaderTypes().get(i));
            }
        }

        // merge lines
        for (Map<String, Either<String, Number>> row : p.getRows()) {
            rows.add(row);
        }
    }

    /**
     * Stores a generic annotable object.
     *
     * @param a the Annotable to print.
     */
    private void addRow(Annotable annotable) {

        Map<String, Either<String, Number>> row = new HashMap<>();

        String rowId = annotable.getIdentifier();

        if (containsRow(rowId) && !allowDuplicates) {
            return;
        }

        row.put(ID_COLUMN, new Left<>(rowId));

        for (Annotation a : annotable.getAnnotations()) {

            // If it's a single-valued annotation, just add the value.
            // elsewhise, customize the headers with a counter.
            if (a.size() == 1) {
                // Check if the annotation is already tracked in the headers
                if (!headers.contains(a.getAnnotator())) {
                    headers.add(a.getAnnotator());
                    headerTypes.add(a.getValueAt(0));
                }

                row.put(a.getAnnotator(), a.getValueAt(0));

            } else {
                List<String> newHeaders = new ArrayList<>();
                List<Either<String, Number>> newHeaderTypes
                        = new ArrayList<>();

                for (int i = 0; i < a.size(); i++) {
                    newHeaders.add(a.getAnnotator() + "$" + i);
                    newHeaderTypes.add(a.getValueAt(i));
                }

                // Check if the annotation is already tracked in the headers 
                // else, add all the new headers
                if (!headers.contains(newHeaders.get(0))) {
                    headers.addAll(newHeaders);
                    headerTypes.addAll(newHeaderTypes);
                }

                for (int i = 0; i < newHeaders.size(); i++) {
                    row.put(newHeaders.get(i), a.getValueAt(i));
                }

            }
        }

        // add the row
        rows.add(row);

    }

    /**
     * Prints all the annotations of the sentences contained in the component.
     *
     * @param c the component to analyze.
     */
    public void loadSentences(DocumentComponent c) {

        init();

        if (!c.hasComponents()) {
            // c is a sentence, so print its annotations
            addRow(c);
        } else {
            for (Sentence s : DocumentUtils.getSentences(c)) {
                addRow(s);
            }
        }
    }

    /**
     * Loads all the keyphrases in the given blackboard to the sheet. 
     *
     * @param b the component to analyze.
     */
    public void loadKeyphrases(Blackboard b) {

        loadGrams(b,Keyphrase.KEYPHRASE);
    }

    /**
     * Loads all the n-grams in the given blackboard to the sheet.
     *
     * @param b the component to analyze.
     * @param gramIdentifier the identifier for the gram to print
     */
    public void loadGrams(Blackboard b, String gramIdentifier) {

        init();
        Collection<Gram> grams = b.getGramsByType(gramIdentifier);
        for (Gram g : grams) {
            addRow(g);
        }
    }

    /**
     * Loads all the tokens in the sheet.
     *
     * @param c the component to analyze.
     */
    public void loadTokens(DocumentComponent c) {

        init();

        List<Token> tokens;

        if (!c.hasComponents()) {
            tokens = ((Sentence) c).getTokens();
        } else {
            tokens = new ArrayList<>();
            for (Sentence s : DocumentUtils.getSentences(c)) {
                tokens.addAll(s.getTokens());
            }
        }

        for (Token t : tokens) {
            addRow(t);
        }
    }

    /**
     * Adds to all rows a field with the specified value.
     *
     * @param key the name of the column to add.
     * @param value the value of the field to add.
     */
    public void addToAll(String key, String value) {
        headers.add(key);
        headerTypes.add(new Left<>(key));
        for (Map<String, Either<String, Number>> row : rows) {
            row.put(key, new Left<>(value));
        }

    }

    /**
     * Clear the spreadsheet.
     */
    private void init() {
        headers = new ArrayList<>();
        rows = new ArrayList<>();
        headerTypes = new ArrayList<>();

        headers.add(ID_COLUMN);
        headerTypes.add(new Left<>(ID_COLUMN));
    }
} public final class IOBlackboard {

    // Where the output classes should write
    private static String outputPathPrefix;

    // Where to find all the documents (if a collection is used)
    private static String documentsFolder;
    
    // The path of the currently analyzed document
    private static String currentDocument;

    /**
     * Get the path of the folder that contains the documents to analyze (if the
     * Distiller is analyzing a collection)
     *
     * @return the path that contains the documents.
     */
    public static String getDocumentsFolder() {
        return documentsFolder;
    }

    /**
     * Set the path of the folder that contains the documents to analyze (if the
     * Distiller is analyzing a collection)
     *
     * @param documentsFolder the path that contains the documents.
     */
    public static void setDocumentsFolder(String documentsFolder) {
        IOBlackboard.documentsFolder = documentsFolder;
    }

    /**
     * Get the prefix of the output path where the module should write.
     *
     * @return the output path.
     */
    public static String getOutputPathPrefix() {
        return outputPathPrefix;
    }

    /**
     * Set the prefix of the output path where the module should write.
     *
     * @param outputPath the output path.
     */
    public static void setOutputPathPrefix(String outputPath) {
        outputPathPrefix = outputPath;
    }

    /**
     * Get the path of the currently analyzed document.
     * 
     * @return the path of the currently analyzed document.
     */
    public static String getCurrentDocument() {
        return currentDocument;
    }
    
    /**
     * Set the path of the currently analyzed document.
     * 
     * @param currentDocument the path of the currently analyzed document.
     */
    public static void setCurrentDocument(String currentDocument) {
        IOBlackboard.currentDocument = currentDocument;
    }
    
    
} public class FileSystem {

    /**
     * Get an {@link java.io.InputStreamReader} from the provided path, where
     * the path can point both to the underlying OS' file system or to a file
     * contained inside a JAR file.
     *
     * @param path the path which needs to be converted to an InputStream
     * @return the resulting {@link java.io.InputStreamReader}
     * @throws FileNotFoundException if the file does not exists
     */
    public static InputStreamReader getInputStreamReaderFromPath(String path)
            throws FileNotFoundException {
        InputStreamReader isr;

        // running from command-line and loading inside the JAR
        if (path.contains("!")) {
            isr = new InputStreamReader(
                    (new FileSystem()).getClass().getResourceAsStream(
                            path.substring(
                                    path.lastIndexOf("!") + 1)));
        } else {
            // normal operation
            isr = new FileReader(path);
        }

        return isr;
    }

    /**
     * Get an {@link java.io.InputStream} from the provided path, where the path
     * can point both to the underlying OS' file system or to a file contained
     * inside a JAR file.
     *
     * @param path the path which needs to be converted to an InputStream
     * @return the resulting {@link java.io.InputStream}
     * @throws FileNotFoundException if the file does not exists
     */
    public static InputStream getInputStreamFromPath(String path) throws FileNotFoundException {

        InputStream is;

        // running from command-line and loading inside the JAR
        if (path.contains("!")) {
            is = (new FileSystem()).getClass().getResourceAsStream(
                    path.substring(
                            path.lastIndexOf("!") + 1));
        } else {
            // normal operation
            is = new FileInputStream(path);
        }

        return is;
    }

    /**
     * Get the temporary directory path of the underlying file system. On
     * Windows, it should return the value of the %TMP% (or %TEMP%) environment
     * variable.
     *
     * @return the temporary directory path.
     * @see
     * <a href="https://blogs.msdn.microsoft.com/oldnewthing/20150417-00/?p=44213/">
     * Why are there both TMP and TEMP environment variables, and which one is
     * right?</a>
     */
    public static String getTmpPath() {
        return System.getProperty("java.io.tmpdir");
    }

    /**
     * Get the separator for the underlying OS.
     *
     * @return the path separator for the underlying OS.
     */
    public static String getSeparator() {
        return java.io.File.separator;
    }

    /**
     * Creates an empty directory in the specified path. If the directory
     * already exists, nothing is performed.
     * 
     * @param directoryPath the path where the new directory will be created
     */
    public static void createDirectoryIfNotExists(String directoryPath) {
        File newDir = new File(directoryPath);

        // if the directory does not exist, create it
        if (!newDir.exists()) {
            newDir.mkdir();
        }
    }
    
    /**
     * Get the path where the Distiller puts temporary files. NOTE: you should
     * check if the directory exists before trying to put something inside.
     * 
     * @return the path of the temporary folder of the Distiller.
     */
    public static String getDistillerTmpPath() {
        return getTmpPath().concat(getSeparator()).concat("distiller-tmp");
        
    }
} public class Pair<L,R> {
    
    /**
     * The left value.
     */
    private final L left;
    /**
     * The right value.
     */
    private final R right;
    
    /**
     * Constructs a pair.
     * 
     * @param l the left object.
     * @param r the right object.
     */
    public Pair(L l, R r) {
        this.left = l;
        this.right = r;
    }
    
    /**
     * Returns the left object.
     * 
     * @return the left object.
     */
    public L getLeft() {
        return left;
    }
    
    /**
     * Returns the right object.
     * 
     * @return the right object.
     */    
    public R getRight() {
        return right;
    }
}
import it.uniud.ailab.dcore.Distiller;
import it.uniud.ailab.dcore.DistillerFactory;
import it.uniud.ailab.dcore.eval.GenericDataset;
import it.uniud.ailab.dcore.eval.datasets.SemEval2010;
import it.uniud.ailab.dcore.eval.kp.KeyphraseEvaluatorAll;
import it.uniud.ailab.dcore.eval.training.KeyphraseTrainingSetGenerator;
import it.uniud.ailab.dcore.io.CsvPrinter;
import it.uniud.ailab.dcore.io.GenericSheetPrinter;
import it.uniud.ailab.dcore.io.IOBlackboard;
import it.uniud.ailab.dcore.utils.FileSystem;
import it.uniud.ailab.dcore.utils.Pair;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.List;
import java.util.Locale;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;

    /**
     * Displays the instructions.
     */
    private static void printHelp() {

        System.out.println("Distiller-CORE library - http://ailab.uniud.it");
        System.out.println();

        HelpFormatter formatter = new HelpFormatter();

        formatter.printHelp("dcore-"
                + Launcher.class.getPackage().getImplementationVersion()
                + ".jar", options);
    }

    /**
     * Decide what Distillation (single or directory) execute and run it.
     */
    private static void doWork() {

        switch (mode) {

            case EVALUATION:
                evaluate();
                break;
            case TRAINING_GENERATION:
                generateTrainingSet();
                break;
            default:
                try {
                    if (inputPath.isFile()) {
                        analyzeFile(inputPath);
                    } else {
                        analyzeDir(inputPath);
                    }
                } catch (IOException ioe) {
                    System.err.println(ioe.getLocalizedMessage());
                    System.err.println(ioe.toString());
                }
        }

    }

    /**
     * Performs the evaluation of a Distiller pipeline on the specified dataset,
     * deferring the work to the appropriate class.
     */
    private static void evaluate() {

        System.out.println("Launching evaluation...");

        if (!inputPath.isDirectory()) {
            System.err.println(
                    "You should set the folder containing the evaluation files as input.");
        }

        setupDistiller();

        GenericDataset kpDataset;

        switch (dataset) {
            case "semeval":
                kpDataset = new SemEval2010(inputPath.getAbsolutePath());
                break;
            default:
                kpDataset = null;
        }

        if (kpDataset == null) {
            throw new UnsupportedOperationException(
                    "Unknown dataset:" + dataset);
        }

        (new KeyphraseEvaluatorAll(kpDataset)).
                evaluate(distiller);

    }

    /**
     * Generates the training set of a Distiller pipeline on the specified
     * dataset, deferring the work to the appropriate class.
     */
    private static void generateTrainingSet() {

        System.out.println("Launching training set generation...");

        if (!inputPath.isDirectory()) {
            printError(
                    "You should set the folder containing the dataset files as input.");
            return;
        }

        if (outputPath == null || !outputPath.isDirectory()) {
            printError(
                    "You should set an output folder for the training set files.");
        }

        setupDistiller();

        GenericDataset kpDataset;

        switch (dataset) {
            case "semeval":
                kpDataset = new SemEval2010(inputPath.getAbsolutePath());
                break;
            default:
                kpDataset = null;
        }

        if (kpDataset == null) {
            throw new UnsupportedOperationException(
                    "Unknown dataset:" + dataset);
        }

        KeyphraseTrainingSetGenerator trainingGenerator
                = new KeyphraseTrainingSetGenerator(kpDataset);

        IOBlackboard.setDocumentsFolder(kpDataset.getTrainingFolder());

        List<Pair<String, GenericSheetPrinter>> trainingDocuments