public class Main {
    //This is has beeen reduced for production, the official commandline code has been moved to the ConsoleRegexTurtle project  
    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) throws TreeEvaluationException, IOException, Exception {
        if (args.length < 1) {
            printUsage();
            System.exit(0);
        }


        Configuration configuration = Configurator.configure(args[0]);
        //Configuration configuration = new Configuration();
           
        Logger.getLogger("").addHandler(new FileHandler(new File(configuration.getOutputFolder(), "log.xml").getCanonicalPath()));
        Results results = new Results(configuration);
        results.setMachineHardwareSpecifications(Utils.cpuInfo());

        ExecutionStrategy strategy = configuration.getStrategy();
        long startTime = System.currentTimeMillis();
        
        strategy.execute(configuration, new CoolTextualExecutionListener(args[0], configuration, results));
        
        if (configuration.getPostProcessor() != null) {
            startTime = System.currentTimeMillis() - startTime;
            configuration.getPostProcessor().elaborate(configuration, results, startTime);
        }

    }

    private static void printUsage() {
        System.out.println("Usage: java -jar \"Random_Regex_Turtle.jar\" configFileName [startGui]");
    }
} public class Configuration {

    private static final Logger LOG = Logger.getLogger(Configuration.class.getName());    
  
    /**
     * Initializes with default values, parameters and operators.
     */
    public Configuration() {
        this.evolutionParameters = new EvolutionParameters();
        this.evolutionParameters.setGenerations(1000);
        this.evolutionParameters.setPopulationSize(500);
        
        
        this.initialSeed = 0;
        this.jobId = 0;
        this.jobs = 4;
        this.objective = new PrecisionCharmaskLengthObjective() ;
        
        this.constants = Arrays.asList("\\d",
            "\\w",
            "\\.",":",",",";",
            "_","=","\"","'",
            "\\\\",
            "/",    
            "\\?","\\!",
            "\\}","\\{","\\(","\\)","\\[","\\]","<",">",
            "@","#"," "," ");
        this.ranges = new LinkedList<>();
        this.operators = new ArrayList<>(Arrays.asList("it.units.inginf.male.tree.operator.Group",
            "it.units.inginf.male.tree.operator.NonCapturingGroup",
            "it.units.inginf.male.tree.operator.ListMatch",
            "it.units.inginf.male.tree.operator.ListNotMatch",
            "it.units.inginf.male.tree.operator.MatchOneOrMore",
            "it.units.inginf.male.tree.operator.MatchZeroOrMore",
            "it.units.inginf.male.tree.operator.MatchZeroOrOne",
            "it.units.inginf.male.tree.operator.MatchMinMax"));
        //Add context wise operators (lookaround)
        this.operators.addAll(
                    Arrays.asList("it.units.inginf.male.tree.operator.PositiveLookbehind","it.units.inginf.male.tree.operator.NegativeLookbehind",
                            "it.units.inginf.male.tree.operator.PositiveLookahead", "it.units.inginf.male.tree.operator.NegativeLookahead"));
        
      
        this.initNodeFactory(); //initNodeFactory also instantiate the NodeFactory object, this decouples the terminalset between threads
        List<Leaf> terminalSet = this.nodeFactory.getTerminalSet();
        //Add default ranges
        terminalSet.add(new RegexRange("A-Z"));
        terminalSet.add(new RegexRange("a-z"));
        terminalSet.add(new RegexRange("A-Za-z"));
        
        this.evaluator = new CachedTreeEvaluator();
        this.evaluator.setup(Collections.EMPTY_MAP);
        
        this.outputFolderName = ".";
        
        this.strategyParameters = new HashMap<>();
        this.strategyParameters.put("runStrategy","it.units.inginf.male.strategy.impl.SeparateAndConquerStrategy");
        
        this.strategyParameters.put("runStrategy2","it.units.inginf.male.strategy.impl.DiversityElitarismStrategy");
        
        this.strategyParameters.put("objective2","it.units.inginf.male.objective.CharmaskMatchLengthObjective");
        
        
        this.strategyParameters.put("threads","2");
        this.strategy = new CombinedMultithreadStrategy(); //MultithreadStrategy();
        
        //TerminalSet and population builder setup is performed later
        this.terminalSetBuilderParameters = new HashMap<>();
        this.terminalSetBuilderParameters.put("tokenThreashold","80.0");
        this.terminalSetBuilder = new TokenizedContextTerminalSetBuilder();
        
        this.populationBuilderParameters = new HashMap<>();
        this.populationBuilderParameters.put("tokenThreashold","80.0");     
        this.populationBuilder = new TokenizedContextPopulationBuilder();
        
        this.postprocessorParameters = new HashMap<>();
        this.postprocessor = new BasicPostprocessor();
        this.postprocessor.setup(Collections.EMPTY_MAP);
         
        this.bestSelectorParameters = new HashMap<>();
        this.bestSelector = new BasicLearningBestSelector();
        this.bestSelector.setup(Collections.EMPTY_MAP);      
    }      
    
    /**
     * Updates dataset and datasetCotainer stats and structures, and initializes terminalSetBuilder and populationBuilder.
     * You should invoke this method when the original Dataset/DatasetContainer is modified.
     */
    public void setup(){ 
        this.datasetContainer.update();
        this.terminalSetBuilder.setup(this);
        this.populationBuilder.setup(this); 
    }

    public Configuration(Configuration cc) {
        this.evolutionParameters = cc.getEvolutionParameters();
        this.initialSeed = cc.getInitialSeed();
        this.jobId = cc.getJobId();
        this.jobs = cc.getJobs();
        this.objective = cc.getObjective();
        this.evaluator = cc.getEvaluator();
        this.outputFolder = cc.getOutputFolder();
        this.outputFolderName = cc.getOutputFolderName();
        this.strategy = cc.getStrategy();
        this.strategyParameters = new LinkedHashMap<>(cc.getStrategyParameters()); //Permits runtime modification of Configuration objects (do not affect other configurations); Used in combined strategy
        this.configName = cc.getConfigName();
        this.populationBuilder = cc.getPopulationBuilder();
        this.terminalSetBuilderParameters = cc.getTerminalSetBuilderParameters();
        this.terminalSetBuilder = cc.getTerminalSetBuilder();
        this.populationBuilderParameters = cc.getPopulationBuilderParameters();
        this.datasetContainer = cc.getDatasetContainer();
        this.postprocessor = cc.getPostProcessor();
        this.postprocessorParameters = cc.getPostprocessorParameters();
        //nodeFactory is not dublicated 
        this.bestSelector = cc.getBestSelector();
        this.bestSelectorParameters = cc.getBestSelectorParameters();
        this.constants = cc.constants;
        this.ranges = cc.ranges;
        this.operators = cc.operators;
        this.isFlagging = cc.isIsFlagging();
        this.initNodeFactory(); //initialized nodeFacory after introducing constants and operators
    }
    
    
    private EvolutionParameters evolutionParameters;
    private long initialSeed;
    private int jobs;
    private int jobId;
    private transient File outputFolder;
    private String outputFolderName;
    private transient Objective objective;
    private transient TreeEvaluator evaluator;
    private transient ExecutionStrategy strategy;    
    private Map<String, String> strategyParameters;  
    private String configName;
    private transient NodeFactory nodeFactory;
    private transient InitialPopulationBuilder populationBuilder;
    private Map<String, String> populationBuilderParameters;
    private Map<String, String> terminalSetBuilderParameters;
    private transient TerminalSetBuilder terminalSetBuilder;
    private DatasetContainer datasetContainer;
    private transient Postprocessor postprocessor;
    private Map<String, String> postprocessorParameters;
    private List<String> constants;
    private List<String> ranges;
    private List<String> operators;
    private transient BestSelector bestSelector;
    private Map<String, String> bestSelectorParameters;
    private boolean  isFlagging = false;

    public NodeFactory getNodeFactory() {
        return nodeFactory;
    }

    public void setNodeFactory(NodeFactory nodeFactory) {
        this.nodeFactory = nodeFactory;
    }
    
    public String getConfigName() {
        return configName;
    }

    public void setConfigName(String configName) {
        this.configName = configName;
    }

    public EvolutionParameters getEvolutionParameters() {
        return evolutionParameters;
    }

    public void setEvolutionParameters(EvolutionParameters evolutionParameters) {
        this.evolutionParameters = evolutionParameters;
    }

    public long getInitialSeed() {
        return initialSeed;
    }

    public void setInitialSeed(long initialSeed) {
        this.initialSeed = initialSeed;
    }

    public Map<String, String> getPostprocessorParameters() {
        return postprocessorParameters;
    }

    public void setPostprocessorParameters(Map<String, String> postprocessorParameters) {
        this.postprocessorParameters = postprocessorParameters;
    }

    public Map<String, String> getTerminalSetBuilderParameters() {
        return terminalSetBuilderParameters;
    }

    public void setTerminalSetBuilderParameters(Map<String, String> terminalSetBuilderParameters) {
        this.terminalSetBuilderParameters = terminalSetBuilderParameters;
    }

    public TerminalSetBuilder getTerminalSetBuilder() {
        return terminalSetBuilder;
    }

    public void setTerminalSetBuilder(TerminalSetBuilder terminalSetBuilder) {
        this.terminalSetBuilder = terminalSetBuilder;
    }

    public List<String> getConstants() {
        return constants;
    }

    public void setConstants(List<String> constants) {
        this.constants = constants;
    }

    public List<String> getRanges() {
        return ranges;
    }

    public void setRanges(List<String> ranges) {
        this.ranges = ranges;
    }

    public List<String> getOperators() {
        return operators;
    }

    public void setOperators(List<String> operators) {
        this.operators = operators;
    }

    public int getJobId() {
        return jobId;
    }

    public void setJobId(int jobId) {
        this.jobId = jobId;
    }

    /**
     * Returns a clone of the current objective, the strategies should get the objective once and cache the instance.
     * There should be and instance per strategy (and one instance per job).
     * Calling the objective a lot of times is going to instantiate a lot of instances. 
     * @return
     */
    public Objective getObjective() {
        return objective.cloneObjective();
    }

    public void setObjective(Objective objective) {
        this.objective = objective;
    }

    public TreeEvaluator getEvaluator() {
        return evaluator;
    }

    public void setEvaluator(TreeEvaluator evaluator) {
        this.evaluator = evaluator;
    }

    public File getOutputFolder() {
        return outputFolder;
    }

    public void setOutputFolder(File outputFolder) {
        this.outputFolder = outputFolder;
    }

    public int getJobs() {
        return jobs;
    }

    public void setJobs(int jobs) {
        this.jobs = jobs;
    }

    public ExecutionStrategy getStrategy() {
        return strategy;
    }

    public void setStrategy(ExecutionStrategy strategy) {
        this.strategy = strategy;
    }

    public Map<String, String> getStrategyParameters() {
        return strategyParameters;
    }

    public void setStrategyParameters(Map<String, String> strategyParameters) {
        this.strategyParameters = strategyParameters;
    }

    public InitialPopulationBuilder getPopulationBuilder() {
        return populationBuilder;
    }

    public void setPopulationBuilder(InitialPopulationBuilder populationBuilder) {
        this.populationBuilder = populationBuilder;
    }

    public Postprocessor getPostProcessor() {
        return postprocessor;
    }

    public void setPostProcessor(Postprocessor postprocessor) {
        this.postprocessor = postprocessor;
    }   
    
    public BestSelector getBestSelector() {
        return bestSelector;
    }

    public void setBestSelector(BestSelector bestSelector) {
        this.bestSelector = bestSelector;
        this.bestSelector.setup(Collections.EMPTY_MAP);
    }
    
    public Map<String, String> getBestSelectorParameters() {
        return bestSelectorParameters;
    }

    public boolean isIsFlagging() {
        return isFlagging;
    }

    public void setIsFlagging(boolean isFlagging) {
        this.isFlagging = isFlagging;
    }

    public void setBestSelectorParameters(Map<String, String> bestSelectorParameters) {
        this.bestSelectorParameters = bestSelectorParameters;
    }

    public String getOutputFolderName() {
        return outputFolderName;
    }

    public void setOutputFolderName(String outputFolderName) {
        this.outputFolderName = outputFolderName;
        this.outputFolder = new File(this.outputFolderName);
        checkOutputFolder(this.outputFolder);
    }
   
    private void checkOutputFolder(File outputFolder) throws ConfigurationException {
        if (outputFolder == null) {
            throw new IllegalArgumentException("The output folder must be set");
        }
        if (!outputFolder.isDirectory()) {
            if (!outputFolder.mkdirs()) {
                throw new ConfigurationException("Unable to create output folder \""+outputFolder+"\"");
            }
        }
    }   
    
    public DatasetContainer getDatasetContainer() {
        return datasetContainer;
    }

    /**
     * Sets the new datasetContainer.
     * you need to call the the setup command, in order to initialize the datasetContainer and 
     * in order to generate terminalSets and initial populations,
     * @param datasetContainer
     */
    public void setDatasetContainer(DatasetContainer datasetContainer) {
        this.datasetContainer = datasetContainer;
    }
    
    public Map<String, String> getPopulationBuilderParameters() {
        return populationBuilderParameters;
    }    
    
    public final void initNodeFactory() {
        NodeFactory factory = new NodeFactory();
        List<Leaf> terminals = factory.getTerminalSet();

        for (String c : constants) {            
            terminals.add(new Constant(c));
        }

        for (String s : ranges) {
            terminals.add(new RegexRange(s));
        }

        List<Node> functions = factory.getFunctionSet();
        for (String o : operators) {
            try {
                functions.add(buildOperatorInstance(o));
            } catch (ClassNotFoundException | IllegalAccessException | InstantiationException ex) {
                LOG.log(Level.SEVERE, "Unable to create required operator: " + o, ex);
                System.exit(1);
            }
        }
        this.nodeFactory = factory;
    }
    
    private Node buildOperatorInstance(String o) throws ClassNotFoundException, InstantiationException, IllegalAccessException {
        Class<? extends Node> operatorClass = Class.forName(o).asSubclass(Node.class);
        Node operator = operatorClass.newInstance();
        return operator;
    }
    
    /**
     * Changes the configured fitness with the java class objectiveClass
     * @param objectiveClass
     * @return
     */
    public void updateObjective(String objectiveClass) {
        try {
            Class<? extends Objective> operatorClass = Class.forName(objectiveClass).asSubclass(Objective.class);
            Objective operator = operatorClass.newInstance();
            this.objective = operator;
        } catch (ClassNotFoundException | InstantiationException | IllegalAccessException ex) {
            LOG.log(Level.SEVERE, "Unable to create required objective: " + objectiveClass, ex);
            System.exit(1);
        }
        //NO OP  
    }
} public class SimpleConfig {
    //Maximum unmatch_chars/match_chars ratio
    //and sets the maximum unmatch_chars/match_chars ratio; this value defines the margin size around the matches 
    transient private final double STRIPING_DEFAULT_MARGIN_SIZE = 10;
    public int numberThreads;
    public int numberOfJobs;
    public int generations;
    public int populationSize;
    public DataSet dataset;
    public boolean populateOptionalFields;
    public boolean isStriped = false;
    public boolean isFlagging = false;
    
    transient public String datasetName;
    transient public String outputFolder;

    /**
     * Percentange [0,100] of the number of the generations used for the Spared termination
     * criteria. 
     */
    public double termination = 20.0;
    public String comment;
    
    public Configuration buildConfiguration(){
        assert !(isFlagging&&isStriped);
        
        //
        Configuration configuration = new Configuration();
        configuration.setConfigName("Console config");
        configuration.getEvolutionParameters().setGenerations(generations);
        configuration.getEvolutionParameters().setPopulationSize(populationSize);
        configuration.setJobs(numberOfJobs);
        configuration.getStrategyParameters().put(MultithreadStrategy.THREADS_KEY, String.valueOf(numberThreads));
        
        int terminationGenerations = (int)(termination * configuration.getEvolutionParameters().getGenerations() / 100.0);
        if(termination==100.0){
            configuration.getStrategyParameters().put("terminationCriteria","false");  
        } else {
            configuration.getStrategyParameters().put("terminationCriteria","true");
        }
        configuration.getStrategyParameters().put("terminationCriteriaGenerations", String.valueOf(terminationGenerations));
        //Added terminationCriteria for the second strategy
        configuration.getStrategyParameters().put("terminationCriteria2","false");
        
        if(dataset == null){
            throw new IllegalArgumentException("You must define a dataset");
        }
        dataset.populateUnmatchesFromMatches();
        DatasetContainer datasetContainer = new DatasetContainer(dataset);
        datasetContainer.createDefaultRanges((int) configuration.getInitialSeed());
        //checks if striping is needed
        dataset.updateStats();
        if(isStriped){
            Logger.getLogger(this.getClass().getName()).info("Enabled striping.");
            datasetContainer.setDataSetsStriped(true);
            datasetContainer.setDatasetStripeMarginSize(STRIPING_DEFAULT_MARGIN_SIZE);
            datasetContainer.setProposedNormalDatasetInterval(100);//terminationGenerations+50);
        }
        configuration.setDatasetContainer(datasetContainer); //remind that after setting the DataSetContainer.. we need to update configuration in order to invoke datacontainer update methods
        
        //FLagging configuration
        //is an alternative configuration, experimental, that requires changes into the configuration defaults (extractor configuration)
        //Changes: bestSelector, fitness, terminalset builder configuration mod, population builders(?)
        configuration.setIsFlagging(isFlagging);
        if(this.isFlagging){
            configuration.setStrategy(new MultithreadStrategy());
            configuration.setBestSelector(new BasicFlaggingLearningBestSelector());
            configuration.setObjective(new FlaggingAccuracyPrecisionLengthObjective());
            configuration.setPopulationBuilder(new FlaggingNaivePopulationBuilder()); //disable context generation
            configuration.setTerminalSetBuilder(new FlaggingNgramsTerminalSetBuilder()); //disable context generation 
            //TODO change terminalSet to a more naive version?
            configuration.getTerminalSetBuilderParameters().put("discardWtokens", "false");//Takes significant chars too
            configuration.getStrategyParameters().put("isFlagging", "true"); //Enable strategy flagging
            //Remove lookarounds
            configuration.getOperators().removeAll(
                    Arrays.asList("it.units.inginf.male.tree.operator.PositiveLookbehind","it.units.inginf.male.tree.operator.NegativeLookbehind",
                            "it.units.inginf.male.tree.operator.PositiveLookahead", "it.units.inginf.male.tree.operator.NegativeLookahead"));
        }
        
        
        
        configuration.setup(); //initializes datasetcontainer, populationbuilder and terminalsetbuilder
        
        return configuration;
    }
} public class DataSet {

    public DataSet() {
    }

    public DataSet(String name) {
        this.name = name;
    }

    public DataSet(String name, String description, String regexTarget) {
        this.name = name;
        this.description = description;
        this.regexTarget = regexTarget;
    }
    
    public String name;
    public String description;
    public String regexTarget;
    public List<Example> examples = new LinkedList<>();
    
    private transient int numberMatches;
    private transient int numberUnmatches;
    private transient int numberMatchedChars;
    private transient int numberUnmatchedChars;
    private transient int numberUnAnnotatedChars;
    private transient int numberOfChars;
    
    private transient DataSet stripedDataset;
    
    private transient Map<Long, List<DataSet>> separateAndConquerLevels = new ConcurrentHashMap<>();
    //private transient DataSet datasetFocus = null;
    private final static Logger LOG = Logger.getLogger(DataSet.class.getName());  
    
    /**
     * Updates the dataset statistics, numberMatches, numberMatchesChars and so on
     */
    public void updateStats(){
        this.numberMatches = 0;
        this.numberUnmatches = 0;
        this.numberMatchedChars = 0;
        this.numberUnmatchedChars = 0;
        this.numberUnAnnotatedChars = 0;
        this.numberOfChars = 0;
    
        for (Example ex : this.examples) {
            this.numberMatches += ex.match.size();
            this.numberUnmatches += ex.unmatch.size();
            this.numberMatchedChars += ex.getNumberMatchedChars();
            this.numberUnmatchedChars += ex.getNumberUnmatchedChars();
            this.numberOfChars += ex.getNumberOfChars();
        }
        this.numberUnAnnotatedChars = this.numberOfChars - this.numberMatchedChars -this.numberUnmatchedChars;
    }

    public int getNumberMatches() {
        return numberMatches;
    }

    public int getNumberUnmatches() {
        return numberUnmatches;
    }

    public int getNumberMatchedChars() {
        return numberMatchedChars;
    }

    public int getNumberUnmatchedChars() {
        return numberUnmatchedChars;
    }

    public int getNumberUnannotatedChars() {
        return numberUnAnnotatedChars;
    }

    public int getNumberOfChars() {
        return numberOfChars;
    }
    
    public int getNumberExamples(){
        return this.examples.size();
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public String getDescription() {
        return description;
    }

    public void setDescription(String description) {
        this.description = description;
    }

    public String getRegexTarget() {
        return regexTarget;
    }

    public void setRegexTarget(String regexTarget) {
        this.regexTarget = regexTarget;
    }
    
    
    
    public String getStatsString(){
        StringBuilder stats = new StringBuilder();
        stats.append("DataSet ").append(this.name).append(" stats:\n")
        .append("number examples: ").append(this.getNumberExamples())
        .append("\noverall chars in dataset: ").append(this.getNumberOfChars())
        .append("\nnumber matches: ").append(this.getNumberMatches())
        .append("\ncharacters in matches: ").append(this.getNumberMatchedChars())
        .append("\nnumber unmatches: ").append(this.getNumberUnmatches())
        .append("\ncharacters in unmatches: ").append(this.getNumberUnmatchedChars())
        .append("\nunannotated chars: ").append(this.getNumberUnannotatedChars());
        return stats.toString();
    }
    
    public void randomize() {
        Random random = new Random();
        for (int i = 0; i < examples.size(); i++) {
            if((((i*100)/examples.size())%5)==0){
                System.out.format("randomize %d%%\n",((i*100)/examples.size()));
            }
            Example ex1;
            Example ex2;
            int destIndex = random.nextInt(examples.size());
            ex1 = examples.get(i);
            ex2 = examples.get(destIndex);
            examples.set(i, ex2);
            examples.set(destIndex, ex1);
        }
    }
    
    public void populateUnmatchesFromMatches(){
            for (Example ex : this.examples) {
                ex.populateUnmatchesFromMatches();
            }
        
    }
    
    /**
     * Populate examples with the temporary list of matched and unmatched strings (using current match and unmatch bounds)
     */
    public void populateAnnotatedStrings(){
            for(Example example : this.examples){
            example.populateAnnotatedStrings();
            }
    }

    public List<Example> getExamples() {
        return this.examples;
    }
    
    /**
     * Create a dataset which is a "view" of the current dataset.A subset of the dataset defined by ranges.
     * @param name
     * @param ranges
     * @return
     */
    public DataSet subDataset(String name, List<Range> ranges){
            // ranges are inclusive
            DataSet subDataset = new DataSet(name);
            for(Range range : ranges){
                for(int index = range.getStartIndex(); index <= range.getEndIndex(); index++){
                    subDataset.getExamples().add(this.getExamples().get(index));
                }           
            }
            return subDataset;
    }
    
    /**
     * 
     * @param marginSize is the how much the margin is bigger than match size. marginSize = 3 means the number of characters in margins is three times the match characters.
     * @return
     */
    public DataSet initStripedDatasetView(double marginSize){
        this.stripedDataset = new DataSet(this.name, this.description, this.regexTarget);
        for(Example example : this.examples){
            this.stripedDataset.getExamples().addAll(this.stripeExample(example, marginSize));
        }        
        return this.stripedDataset;
    }
    
    
    
    /**
     * creates a list of examples created from examples by splitting example in littler pieces.
     * The pieces are created from a window surrounding the example matches. 
     * When two or more windows overlaps, the windows merge together and a single multi-match example is created.
     * @param example
     * @param marginSize
     * @return
    */
        protected static List<Example> stripeExample(Example example, double marginSize){     
        List<Example> slicesExampleList = new LinkedList<>();    
        //create ranges covering the saved portions 
        List<Bounds> savedBounds = new LinkedList<>();
        for(Bounds match : example.getMatch()){
            //double -> int cast works like Math.floor(); Gives the same results of the older integer version
            int charMargin = (int) Math.max(((match.size() * marginSize) / 2.0),1.0);
            Bounds grownMatch = new Bounds(match.start - charMargin, match.end + charMargin);
            grownMatch.start = (grownMatch.start >= 0) ? grownMatch.start:0;
            grownMatch.end = (grownMatch.end <= example.getNumberOfChars()) ? grownMatch.end : example.getNumberOfChars();
            savedBounds.add(grownMatch);
        }
        //compact bounds, create a compact representation of saved portions
        //This bounds represents the slices we are going to cut the example to 
        savedBounds = Bounds.mergeBounds(savedBounds);
        
        //Create examples from slices
        for(Bounds slice : savedBounds){
            Example sliceExample = new Example();
            sliceExample.setString(example.getString().substring(slice.start, slice.end));
            
            //find owned matches
            for(Bounds match : example.getMatch()){
                if(match.start >= slice.end){
                    break; //only the internal for
                } else {
                    Bounds slicedMatch = match.windowView(slice);
                    if(slicedMatch != null){
                        sliceExample.getMatch().add(slicedMatch);
                    }
                }
            }
            //find owned unmatches
            for(Bounds unmatch : example.getUnmatch()){
                if(unmatch.start >= slice.end){
                    break; //only the internal for
                } else {
                    Bounds slicedUnmatch = unmatch.windowView(slice);
                    if(slicedUnmatch != null){
                        sliceExample.getUnmatch().add(slicedUnmatch);
            }
                }
            }
            
            sliceExample.populateAnnotatedStrings();
            slicesExampleList.add(sliceExample);
        }
        return slicesExampleList;
    }
    
    /**
     * Returns the last initialized striped version of this DataSet. In order to change the window size you have to call initStripedDatasetView() again.
     * @return
     */
    public DataSet getStripedDataset(){
//          if(this.stripedDataset == null){
//              Logger.getLogger(this.getClass().getName()).info("getStripedDataset returns, null dataset cause uninitialized striped dataset.");
//          }
          return this.stripedDataset;
    }
    
    /**
     * Returns the "Separate and Conquer" sub-dataset of the requested level.
     * Level 0 is the original dataset, level 1 is the dataset obtained from
     * the first reduction, level 2 is the dataset from the second reduction
     * and so on.
     * 
     * @param divideEtImperaLevel
     * @param jobId
     * @return
     */
    public DataSet getSeparateAndConquerDataSet(int divideEtImperaLevel, int jobId) {
         if(divideEtImperaLevel == 0){
             return this;
         }
         return getSeparateAndConquerLevels(jobId).get(divideEtImperaLevel - 1);
    }
    
    /**
     * Divide dataset, defaults to converting match to unmatches and text extraction problem.
     * @param individualRegex
     * @param jobId
     * @return
     */
    public boolean addSeparateAndConquerLevel(String individualRegex, int jobId){
        return this.addSeparateAndConquerLevel(individualRegex, jobId, true, false);
    }
    
    /**
     * From the last generated "Separate and conquer" sub-dataset, creates a new sub-dataset.
     * We evaluate the individualRegex on the dataset examples, matches that are 
     * correctly extracted are removed. A removed match is converted to unmatch or unannotated depending
     * on the convertToUnmatch value: True==unmatch
     * In striping mode, the base dataset and striped dataset are reduced separately.
     * There is no direct connection from --i.e the third level of the original dataset
     * and the third level of the striped dataset. The reduced dataset depends only 
     * by its parent.
     * NOTE: The levels are the sub-dataset, level 0 is the original dataset, level 1 is
     * the sub-dataset reduced from the original dataset, level 2 is a sub-dataset reduced
     * from the level 1 dataset and so on.
     * @param individualRegex
     * @param jobId
     * @param convertToUnmatch when true, the eliminated matches are converted into unmatches, otherwise unannotated area.
     * @param isFlagging
     * @return true, when the dataset has been modified by reduction
     */
    public boolean addSeparateAndConquerLevel(String individualRegex, int jobId, boolean convertToUnmatch, boolean isFlagging){
        boolean modified = false;
        DataSet oldDataset = this.getLastSeparateAndConquerDataSet(jobId);
        DataSet dataset = oldDataset.reduceSeparateAndConquerDataset(individualRegex, convertToUnmatch, isFlagging);
        dataset.updateStats();
        modified = (dataset.getNumberMatches() != oldDataset.getNumberMatches());
        this.getSeparateAndConquerLevels(jobId).add(dataset);
        if(this.getStripedDataset()!=null){
            modified = this.getStripedDataset().addSeparateAndConquerLevel(individualRegex,jobId, convertToUnmatch, isFlagging) || modified;       
        }
        return modified;
    }
    
   /**
     * Creates a DataSet (sub-dataset for "Separate and conquer") from this dataset instance.
     * We evaluate the individualRegex on the dataset examples, matches that are 
     * correctly extracted are removed. A removed match is converted to unmatch or unannotated depending
     * on the convertToUnmatch value: True==unmatch
     */
    private DataSet reduceSeparateAndConquerDataset(String individualRegex, boolean convertToUnmatch, boolean isFlagging ){
        //initialize pattern matcher
        Pattern pattern = Pattern.compile(individualRegex);
        Matcher individualRegexMatcher = pattern.matcher("");
    
        DataSet reducedDataset = new DataSet(this.name, "Reduction: "+individualRegex, this.regexTarget);
        for(Example example : this.examples){
            if(!isFlagging){
                reducedDataset.getExamples().add(this.reduceSeparateAndConquerExample(example, individualRegexMatcher, convertToUnmatch));
            } else {
                reducedDataset.getExamples().add(this.reduceSeparateAndConquerFlaggingExample(example, individualRegexMatcher));
            }
        }        
        return reducedDataset;
    }
    
    private boolean isTruePositiveFlaggingExample(Example example, Matcher individualRegexMatcher){
        try {
            Matcher m = individualRegexMatcher.reset(example.getString());
            return (m.find() && !example.match.isEmpty());
        } catch (StringIndexOutOfBoundsException ex) {
            return false;
            /**
             * Workaround: riferimento BUG: 6984178
             * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6984178
             * con i quantificatori greedy restituisce una eccezzione
             * invece che restituire un "false".
             */
        }
    }
    
    
    private Example reduceSeparateAndConquerFlaggingExample(Example example, Matcher individualRegexMatcher){
        //Negative or unannotated are left unchanged
        if(!isTruePositiveFlaggingExample(example, individualRegexMatcher)){
            return new Example(example);
        }
        Example unannotatedExample = new Example();
        unannotatedExample.setString(example.getString());
        return unannotatedExample;
    }
    
    private Example reduceSeparateAndConquerExample(Example example, Matcher individualRegexMatcher, boolean convertToUnmatch){
        return this.manipulateSeparateAndConquerExample(example, individualRegexMatcher, convertToUnmatch);
    }
    
    //creates a reduced ("Separate and conquer") Example instance from an example and a given Regex Instance 
    //ELIMINATED Feature: When doFocus is true, the method perform focus action instead of examples reduction (Focus action creates the complementary of the reduction in order to focus evolution).
    
    //When convertToUnmatch is true extracted matches are converted into unannotated. 
    private Example manipulateSeparateAndConquerExample(Example example, Matcher individualRegexMatcher, boolean convertToUnmatch){
        Example exampleClone = new Example(example);
        List<Bounds> extractions = new LinkedList<>();
        try {
            Matcher m = individualRegexMatcher.reset(example.getString());
            while (m.find()) {
                Bounds bounds = new Bounds(m.start(0), m.end(0));
                extractions.add(bounds);
            }
        } catch (StringIndexOutOfBoundsException ex) {
            /**
             * Workaround: riferimento BUG: 6984178
             * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6984178
             * con i quantificatori greedy restituisce una eccezzione
             * invece che restituire un "false".
             */
        }
        
        //remove extracted matches
        for (Iterator<Bounds> it = exampleClone.getMatch().iterator(); it.hasNext();) {
            Bounds match = it.next();
            for(Bounds extraction : extractions){
                //when doFocus is true, match is remove when equals
                if(match.equals(extraction)){
                    it.remove();
                    if(convertToUnmatch){
                        exampleClone.getUnmatch().add(match);
                    }
                    break;
                }
                //Optimization
                //extractions are in ascending order.. because find is going to extract in this order
                if(extraction.start > match.end){
                    break;
                }
            }
        }
        
        exampleClone.mergeUnmatchesBounds();
        exampleClone.populateAnnotatedStrings();
        return exampleClone;
    }
    
    public DataSet getLastSeparateAndConquerDataSet(int jobId){
        List<DataSet> datasetsList =  this.getSeparateAndConquerLevels(jobId);
        if(datasetsList.isEmpty()){
            return this;
        }
        return datasetsList.get(datasetsList.size() -1);
    }
    
    public int getNumberOfSeparateAndConquerLevels(int jobId){
        return this.getSeparateAndConquerLevels(jobId).size();
    }
    
    
    /**
     * Returns all the "Separate and conquer" sub-datasets lists, for all the current threads.
     * The map resolves a ThreadID to the thread sub-datasets (levels) list. 
     * @return
     */
    public Map<Long, List<DataSet>> getAllSeparateAndConquerLevels(){
        return this.separateAndConquerLevels;
    }
    
    /**
     * Returns the list of the sub-datasets created by "Separate and conquer", for the current thread.
     * The threads (aka active Jobs) have their own sub-datasets. This works right as
     * far as there is a *:1 relation between Jobs and Threads and we clean the generated
     * levels when a new Job is stated.
     * The levels are the sub-dataset, level 0 is the original dataset, level 1 is
     * the sub-dataset reduced from the original dataset, level 2 is a sub-dataset reduced
     * from the level 1 dataset and so on.
     * @param jobId
     * @return The list of sub datasets
     */
    public List<DataSet> getSeparateAndConquerLevels(long jobId){
        if (this.separateAndConquerLevels.containsKey(jobId)){
            return this.separateAndConquerLevels.get(jobId);
        } else {
            List<DataSet> newDatasetList = new LinkedList<>();
            this.separateAndConquerLevels.put(jobId, newDatasetList);
            return newDatasetList;
        }
    }
    
    /**
     * Resets the "Separate and conquer" for the current thread only.
     * Deletes all the generated (reduced) sub-datasets for the current thread.
     */
    public void resetSeparateAndConquer(long jobId){
        this.getSeparateAndConquerLevels(jobId).clear();
        if(this.getStripedDataset()!=null){
            this.getStripedDataset().getSeparateAndConquerLevels(jobId).clear();
        }
    }
    
    
    /**
     * Resets the "Separate and conquer", delete reduced sub-datasets, for all threads.
     */
    public void resetAllSeparateAndConquer(){
        this.getAllSeparateAndConquerLevels().clear();
    }
    
    
    public void removeSeparateAndConquerLevel(int jobID){
        List<DataSet> separateAndConquerLevelsForJob = this.getSeparateAndConquerLevels(jobID);
        if(!separateAndConquerLevelsForJob.isEmpty()){
            separateAndConquerLevelsForJob.remove(separateAndConquerLevelsForJob.size()-1);
        }
    }

   
    
    public static class Example {

        public Example() {
        }
        
        public Example(Example example) {
            this.string = example.string;
            this.match= new LinkedList<>(example.match);
            this.unmatch = new LinkedList<>(example.unmatch);
            if(example.matchedStrings!=null){
                this.matchedStrings = new LinkedList<>(example.matchedStrings);
            }
            if(example.unmatchedStrings!=null){
                this.unmatchedStrings = new LinkedList<>(example.unmatchedStrings);
        }
        }
        
        public String string;
        public List<Bounds> match = new LinkedList<>();
        public List<Bounds> unmatch = new LinkedList<>();
        transient protected List<String> matchedStrings = new LinkedList<>();
        transient protected List<String> unmatchedStrings = new LinkedList<>();

        public void addMatchBounds(int bs, int bf) {
            Bounds boundaries = new Bounds(bs, bf);
            match.add(boundaries);
        }
        
        public void addUnmatchBounds(int bs, int bf) {
            Bounds boundaries = new Bounds(bs, bf);
            unmatch.add(boundaries);
        }
        
        public int getNumberMatchedChars(){
            return getNumberCharsInsideIntervals(match);
        }
        
        public int getNumberUnmatchedChars(){
            return getNumberCharsInsideIntervals(unmatch);
        }
        
        public int getNumberOfChars(){
            return string.length();
        }
        
        private int getNumberCharsInsideIntervals(List<Bounds> textIntervals){
            int countChars = 0;
            for(Bounds interval : textIntervals){
                countChars += (interval.end - interval.start);
            }
            return countChars;
        }
        
        public void populateAnnotatedStrings(){
            this.matchedStrings.clear();
            for(Bounds bounds : this.match){
                this.matchedStrings.add(this.string.substring(bounds.start,bounds.end));
            }
            this.unmatchedStrings.clear();
            for(Bounds bounds : this.unmatch){
                this.unmatchedStrings.add(this.string.substring(bounds.start,bounds.end));
        }
        }

        
        public List<String> getMatchedStrings() {
            return matchedStrings;
        }
      
        public List<String> getUnmatchedStrings() {
            return unmatchedStrings;
        }
        
        public String getString() {
            return string;
        }

        public List<Bounds> getMatch() {
            return match;
        }

        public List<Bounds> getUnmatch() {
            return unmatch;
        }

        public void setString(String string) {
            this.string = string;
        }
       
        
        /**
         * Creates fully annotated examples based on the provided matches (bounds)
         * All chars are going to be annotated. 
         */
        public void populateUnmatchesFromMatches(){
            this.unmatch.clear();
            //generate unmatches
                int previousMatchFinalIndex = 0;
                for(DataSet.Bounds oneMatch : this.match){
                    if(oneMatch.start > previousMatchFinalIndex){
                        this.addUnmatchBounds(previousMatchFinalIndex, oneMatch.start);
                    }
                    previousMatchFinalIndex = oneMatch.end;
                }
                if(previousMatchFinalIndex < string.length()){
                    /*
                    the right value of the interval can be equal than the string.lenght
                    because the substrings are left-inclusive and right-exclusive
                    */
                    this.addUnmatchBounds(previousMatchFinalIndex, string.length());
                }
        }

        public int getNumberMatches() {
            return this.match.size();
        }
        
        /**
         * Returns the annotated strings, matches and unmatches
         * @return a list of all annotated strings
         */
        public List<String> getAnnotatedStrings(){
            List<String> annotatedStrings = new LinkedList<>();
            for(Bounds bounds : this.getMatch()){
                annotatedStrings.add(this.getString().substring(bounds.start, bounds.end));
            }
            for(Bounds bounds : this.getUnmatch()){
                annotatedStrings.add(this.getString().substring(bounds.start, bounds.end));
            }
            return annotatedStrings;
        }
        
        /**
         * This method generates all the annotated strings; the strings are in the same order
         * they appear into the example. This method has been created in order to mantain the
         * same behavior for the getAnnotatedStrings method (different order).
         * @return
         */
        public List<String> getOrderedAnnotatedStrings(){
            List<Bounds> boundsList = new LinkedList<>(this.getMatch());
            boundsList.addAll(this.getUnmatch());
            Collections.sort(boundsList);  
            List<String> annotatedStrings = new LinkedList<>();
            for(Bounds bounds : boundsList){
                annotatedStrings.add(this.getString().substring(bounds.start, bounds.end));
    }
            return annotatedStrings;
        }
    
         
        public void mergeUnmatchesBounds(){
            this.unmatch = Bounds.mergeBounds(this.unmatch);
        }
        
    }
    
    static public class Bounds implements Comparable<Bounds>{
        public Bounds(int start, int end) {
            this.start = start;
            this.end = end;
        }
        public int start;
        public int end;
        
        public int size(){
            return ( this.end - this.start );
        };

        @Override
        public int hashCode() {
            int hash = 3;
            hash = 97 * hash + this.start;
            hash = 97 * hash + this.end;
            return hash;
        }

        @Override
        public boolean equals(Object obj) {
            if (obj == null) {
                return false;
            }
            if (getClass() != obj.getClass()) {
                return false;
            }
            final Bounds other = (Bounds) obj;
            return ((this.end == other.end)&&(this.start == other.start));
        }
        
        /**
         * Convert a list of bounds in a more compact representation; this works with overlapping intervals too.
         * Remind that bounds define ranges with start inclusive and end inclusive index.
         * @param boundsList
         * @return
         */
        static public List<Bounds> mergeBounds(List<Bounds> boundsList){
            List<Bounds> newBoundsList = new LinkedList<>();
            if(boundsList.isEmpty()){
                return newBoundsList;
            }
            Collections.sort(boundsList);
            Bounds prevBounds = new Bounds(boundsList.get(0).start, boundsList.get(0).end);
            for (int i = 1; i < boundsList.size(); i++) {
                Bounds currentBounds = boundsList.get(i);
                if(currentBounds.start <= prevBounds.end){
                    //merge
                    prevBounds.end = Math.max(currentBounds.end,prevBounds.end);
                } else {
                    newBoundsList.add(prevBounds);
                    prevBounds = new Bounds(currentBounds.start, currentBounds.end);
                }
            }
            newBoundsList.add(prevBounds);
            return newBoundsList;
        }
        
        /**
         * Creates a new Bounds object representing this interval relative a rangeBounds
         * interval. --i.e: this [4,9] ; this.windowView([2,7]) == [2,7] 
         * When this Bounds object is out of the window, windowView returns null. 
         * @param rangeBounds
         * @return
         */
        public Bounds windowView(Bounds rangeBounds){
            Bounds newBounds = new Bounds(this.start-rangeBounds.start, this.end-rangeBounds.start);
            if((newBounds.start >= rangeBounds.size())||(newBounds.end<=0)){
                return null; //Out of window
            }
            newBounds.start = Math.max(newBounds.start,0);
            newBounds.end = Math.min(newBounds.end,rangeBounds.size());
            return newBounds;
        }
        
        @Override
        public int compareTo(Bounds o) {
           return (this.start - o.start);
        }
        
        public boolean isSubsetOf(Bounds bounds){
            return (this.start >= bounds.start) && (this.end <= bounds.end);
        }
        
        /**
         * When this Bounds objects overlaps the bounds argument (range intersection is not empty) returns true; otherwise returns false.
         * Zero length bounds are considered overlapping when they adhere each other --i.e: [3,5] and [5,5] are considered overlapping
         * [3,3] and [3,8] are overlapping ranges.
         * @param bounds The range to compare, for intersections, with the calling instance 
         * @return true when the ranges overlap
         */
        public boolean overlaps(Bounds bounds){
            return this.start == bounds.start || this.end == bounds.end || ((this.start < bounds.end) && (this.end > bounds.start));
        }
    
        
        //extracted ranges are always ordered collection. In case you have to sort the collection first.
        //AnnotatedRanges are sorted internally by the method
        /**
         * Counts the number of checkedRanges that overlaps with the zoneRanges. A Bounds object in checkedRanges who doesn't overlap
         * with zoneRanges are not counted.
         * @param ranges
         * @param zoneRanges
         * @return
         */
        static public int countRangesThatCollideZone(List<Bounds> ranges, List<Bounds> zoneRanges) {
            int overallEOAA = 0;
            Collections.sort(zoneRanges);  
             
            //This approach relies on the fact that both annotatedRanges and extracted ranges are ordered
            for (Bounds extractedBounds : ranges) {
                for (Bounds expectedBounds : zoneRanges) {
                    if(expectedBounds.start >= extractedBounds.end){
                        break;
                    } 
                    if(extractedBounds.overlaps(expectedBounds)){
                        overallEOAA++;
                        break;
                    }
                }
            }
            return overallEOAA;
        }

    }
    
    public Example getExample(int index){
        return examples.get(index);
    }
} public static class Example {

    public Example() {
    }
        
    public Example(Example example) {
        this.string = example.string;
        this.match= new LinkedList<>(example.match);
        this.unmatch = new LinkedList<>(example.unmatch);
        if(example.matchedStrings!=null){
            this.matchedStrings = new LinkedList<>(example.matchedStrings);
        }
        if(example.unmatchedStrings!=null){
            this.unmatchedStrings = new LinkedList<>(example.unmatchedStrings);
    }
    }
        
    public String string;
    public List<Bounds> match = new LinkedList<>();
    public List<Bounds> unmatch = new LinkedList<>();
    transient protected List<String> matchedStrings = new LinkedList<>();
    transient protected List<String> unmatchedStrings = new LinkedList<>();

    public void addMatchBounds(int bs, int bf) {
        Bounds boundaries = new Bounds(bs, bf);
        match.add(boundaries);
    }
        
    public void addUnmatchBounds(int bs, int bf) {
        Bounds boundaries = new Bounds(bs, bf);
        unmatch.add(boundaries);
    }
        
    public int getNumberMatchedChars(){
        return getNumberCharsInsideIntervals(match);
    }
        
    public int getNumberUnmatchedChars(){
        return getNumberCharsInsideIntervals(unmatch);
    }
        
    public int getNumberOfChars(){
        return string.length();
    }
        
    private int getNumberCharsInsideIntervals(List<Bounds> textIntervals){
        int countChars = 0;
        for(Bounds interval : textIntervals){
            countChars += (interval.end - interval.start);
        }
        return countChars;
    }
        
    public void populateAnnotatedStrings(){
        this.matchedStrings.clear();
        for(Bounds bounds : this.match){
            this.matchedStrings.add(this.string.substring(bounds.start,bounds.end));
        }
        this.unmatchedStrings.clear();
        for(Bounds bounds : this.unmatch){
            this.unmatchedStrings.add(this.string.substring(bounds.start,bounds.end));
    }
    }

        
    public List<String> getMatchedStrings() {
        return matchedStrings;
    }
      
    public List<String> getUnmatchedStrings() {
        return unmatchedStrings;
    }
        
    public String getString() {
        return string;
    }

    public List<Bounds> getMatch() {
        return match;
    }

    public List<Bounds> getUnmatch() {
        return unmatch;
    }

    public void setString(String string) {
        this.string = string;
    }
       
        
    /**
     * Creates fully annotated examples based on the provided matches (bounds)
     * All chars are going to be annotated. 
     */
    public void populateUnmatchesFromMatches(){
        this.unmatch.clear();
        //generate unmatches
            int previousMatchFinalIndex = 0;
            for(DataSet.Bounds oneMatch : this.match){
                if(oneMatch.start > previousMatchFinalIndex){
                    this.addUnmatchBounds(previousMatchFinalIndex, oneMatch.start);
                }
                previousMatchFinalIndex = oneMatch.end;
            }
            if(previousMatchFinalIndex < string.length()){
                /*
                the right value of the interval can be equal than the string.lenght
                because the substrings are left-inclusive and right-exclusive
                */
                this.addUnmatchBounds(previousMatchFinalIndex, string.length());
            }
    }

    public int getNumberMatches() {
        return this.match.size();
    }
        
    /**
     * Returns the annotated strings, matches and unmatches
     * @return a list of all annotated strings
     */
    public List<String> getAnnotatedStrings(){
        List<String> annotatedStrings = new LinkedList<>();
        for(Bounds bounds : this.getMatch()){
            annotatedStrings.add(this.getString().substring(bounds.start, bounds.end));
        }
        for(Bounds bounds : this.getUnmatch()){
            annotatedStrings.add(this.getString().substring(bounds.start, bounds.end));
        }
        return annotatedStrings;
    }
        
    /**
     * This method generates all the annotated strings; the strings are in the same order
     * they appear into the example. This method has been created in order to mantain the
     * same behavior for the getAnnotatedStrings method (different order).
     * @return
     */
    public List<String> getOrderedAnnotatedStrings(){
        List<Bounds> boundsList = new LinkedList<>(this.getMatch());
        boundsList.addAll(this.getUnmatch());
        Collections.sort(boundsList);  
        List<String> annotatedStrings = new LinkedList<>();
        for(Bounds bounds : boundsList){
            annotatedStrings.add(this.getString().substring(bounds.start, bounds.end));
}
        return annotatedStrings;
    }
    
         
    public void mergeUnmatchesBounds(){
        this.unmatch = Bounds.mergeBounds(this.unmatch);
    }
        
} public class FinalSolution extends Solution{

    public FinalSolution() {
    }

    public FinalSolution(Ranking individual) {
        super(individual);
        this.solutionJS = getDescriptionJavascript(individual.getTree());
    }
   
    private String getDescriptionJavascript(Node node){
        StringBuilder sb = new StringBuilder();
        node.describe(sb, new DescriptionContext(), Node.RegexFlavour.JS);
        return sb.toString();
    }

    /**
     * Create a FinalSolution for the Java regex (String) solution parameter.
     * Javascript version is not automatically generate, this needs initialization
     * with a Node individual.
     * @param solution
     */
    public FinalSolution(String solution) {
        super(solution);
    }

    public FinalSolution(String solution, double[] fitness) {
        super(solution, fitness);
    }

    private String solutionJS;
    private Map<String,Double> trainingPerformances = new HashMap<>();
    private Map<String,Double> validationPerformances = new HashMap<>();
    private Map<String,Double> learningPerformances = new HashMap<>();

    public Map<String, Double> getTrainingPerformances() {
        return trainingPerformances;
    }

    public void setTrainingPerformances(Map<String, Double> trainingPerformances) {
        this.trainingPerformances = trainingPerformances;
    }

    public Map<String, Double> getValidationPerformances() {
        return validationPerformances;
    }

    public void setValidationPerformances(Map<String, Double> validationPerformances) {
        this.validationPerformances = validationPerformances;
    }

    public Map<String, Double> getLearningPerformances() {
        return learningPerformances;
    }

    public void setLearningPerformances(Map<String, Double> learningPerformances) {
        this.learningPerformances = learningPerformances;
    }

    public String getSolutionJS() {
        return solutionJS;
    }
     
} public class Results {
    
    public Results(int numberOfJobs) {
        this.jobEvolutionTrace = new ArrayList<>(numberOfJobs);
        for (int i = 0; i < numberOfJobs; i++) {
            this.jobEvolutionTrace.add(new JobEvolutionTrace());
        }
    }
    
    public Results(Configuration configuration) {
        this(configuration.getJobs());
        this.configuration = configuration;
          
        this.datasetName = this.configuration.getDatasetContainer().getDataset().getName();
        this.methodDescription = this.configuration.getConfigName();
    }
    
    private String datasetName;
    private String methodDescription;
    private String comment;
    private Date experimentDate = new Date();
    transient private String machineHardwareSpecifications;
    private FinalSolution bestSolution;
    //extractions of best solution on the learning dataset
    private List<List<DataSet.Bounds>> bestExtractions;
    private List<List<String>> bestExtractionsStrings;
    //original dataset examples, these are optional properties
    private List<Example> examples;
    private List<BasicStats> bestExtractionsStats; //bestExtractionsStats stores for all examples tp and fp stats.
    
    private long overallExecutionTimeMillis;
    //Learning set stats, useful for Knoledge avaiable computation
    //These stats are about the learning dataset (all provided examples)
    private int numberMatches;
    private int numberUnmatches;  
    private int numberMatchedChars = 0;
    private int numberUnmatchedChars = 0;
    private int numberAnnotatedChars;
    //The characters of all Dataset
    private int numberAllChars;
    //training set informations
    private int numberTrainingMatches;
    private int numberTrainingUnmatches;  
    
    private long characterEvaluations = 0;
    transient private Configuration configuration;
    
    //This is set transient only in this developement code in order to not persist Jobs details in serialization.
    transient private List<JobEvolutionTrace> jobEvolutionTrace; 
    
    
    public JobEvolutionTrace getJobTrace(int jobID){
        return this.jobEvolutionTrace.get(jobID);
    }
    
    public String getDatasetName() {
        return datasetName;
    }

    public void setDatasetName(String datasetName) {
        this.datasetName = datasetName;
    }

    public String getMethodDescription() {
        return methodDescription;
    }

    public void setMethodDescription(String methodDescription) {
        this.methodDescription = methodDescription;
    }

    public Date getExperimentDate() {
        return experimentDate;
    }

    public void setExperimentDate(Date experimentDate) {
        this.experimentDate = experimentDate;
    }

    public Configuration getConfiguration() {
        return configuration;
    }

    public void setConfiguration(Configuration configuration) {
        this.configuration = configuration;
    }

    public String getMachineHardwareSpecifications() {
        return machineHardwareSpecifications;
    }

    public void setMachineHardwareSpecifications(String machineHardwareSpecifications) {
        this.machineHardwareSpecifications = machineHardwareSpecifications;
    }

    public List<JobEvolutionTrace> getJobEvolutionTraces() {
        return jobEvolutionTrace;
    }

    public void setJobEvolutionTraces(List<JobEvolutionTrace> jobEvolutionTraces) {
        this.jobEvolutionTrace = jobEvolutionTraces;
    }

    public FinalSolution getBestSolution() {
        return bestSolution;
    }

    public void setBestSolution(FinalSolution bestSolution) {
        this.bestSolution = bestSolution;
    }

    public long getOverallExecutionTimeMillis() {
        return overallExecutionTimeMillis;
    }

    public void setOverallExecutionTimeMillis(long overallExecutionTimeMillis) {
        this.overallExecutionTimeMillis = overallExecutionTimeMillis;
    }

    public int getNumberMatches() {
        return numberMatches;
    }

    public void setNumberMatches(int numberMatches) {
        this.numberMatches = numberMatches;
    }

    public int getNumberUnmatches() {
        return numberUnmatches;
    }

    public void setNumberUnmatches(int numberUnatches) {
        this.numberUnmatches = numberUnatches;
    }

    public long getCharacterEvaluations() {
        return characterEvaluations;
    }
    
    public void addCharachterEvaluated(long numChars){
        this.characterEvaluations += numChars;
    }   

    public int getNumberMatchedChars() {
        return numberMatchedChars;
    }

    public void setNumberMatchedChars(int numberMatchedChars) {
        this.numberMatchedChars = numberMatchedChars;
        this.numberAnnotatedChars = this.numberMatchedChars + this.numberUnmatchedChars;
    }

    public int getNumberUnmatchedChars() {
        return numberUnmatchedChars;
    }

    public void setNumberUnmatchedChars(int numberUnmatchedChars) {
        this.numberUnmatchedChars = numberUnmatchedChars;
        this.numberAnnotatedChars = this.numberMatchedChars + this.numberUnmatchedChars;
    }

    public int getNumberAllChars() {
        return numberAllChars;
    }

    public void setNumberAllChars(int numberAllChars) {
        this.numberAllChars = numberAllChars;
    }

    public int getNumberAnnotatedChars() {
        return numberAnnotatedChars;
    }

    public int getNumberTrainingMatches() {
        return numberTrainingMatches;
    }

    public void setNumberTrainingMatches(int numberTrainingMatches) {
        this.numberTrainingMatches = numberTrainingMatches;
    }

    public int getNumberTrainingUnmatches() {
        return numberTrainingUnmatches;
    }

    public void setNumberTrainingUnmatches(int numberTrainingUnmatches) {
        this.numberTrainingUnmatches = numberTrainingUnmatches;
    }

    public List<List<DataSet.Bounds>> getBestExtractions() {
        return bestExtractions;
    }

    public void setBestExtractions(List<List<DataSet.Bounds>> bestExtractions) {
        this.bestExtractions = bestExtractions;
    }

    public List<List<String>> getBestExtractionsStrings() {
        return bestExtractionsStrings;
    }

    public void setBestExtractionsStrings(List<List<String>> bestExtractionsStrings) {
        this.bestExtractionsStrings = bestExtractionsStrings;
    }

    public List<Example> getExamples() {
        return examples;
    }

    public void setExamples(List<Example> examples) {
        this.examples = examples;
    }

    public List<BasicStats> getBestExtractionsStats() {
        return bestExtractionsStats;
    }

    public void setBestExtractionsStats(List<BasicStats> bestStats) {
        this.bestExtractionsStats = bestStats;
    }

    public String getComment() {
        return comment;
    }

    public void setComment(String comment) {
        this.comment = comment;
    }
    
    
    public static Results load(String fileName) throws IOException{
        FileInputStream fis = new FileInputStream(new File(fileName));
        InputStreamReader isr = new InputStreamReader(fis);
        StringBuilder sb;
        try (BufferedReader bufferedReader = new BufferedReader(isr)) {
            sb = new StringBuilder();
            String line;
            while ((line = bufferedReader.readLine()) != null) {
                sb.append(line);
            }
        }
        String json = sb.toString();
        Gson gson = new Gson();
        Results result = gson.fromJson(json, Results.class);
        return result;
    }
    
} public class BasicPostprocessor implements Postprocessor {

    private static Logger LOG = Logger.getLogger(BasicPostprocessor.class.getName());
    private boolean populateOptionalFields = true;
    static public final String PARAMETER_NAME_POPULATE_OPTIONAL_FIELDS = "populateOptionalFields";
   
    @Override
    public void setup(Map<String, String> parameters) {
    }

    @Override
    public void elaborate(Configuration config, Results results, long timeTaken) {
        //The PostProcessor gets the parameters in real time
        Map<String, String> parameters = config.getPostprocessorParameters();
        if(parameters!=null){
            if(parameters.containsKey(PARAMETER_NAME_POPULATE_OPTIONAL_FIELDS)){
                this.populateOptionalFields = Boolean.valueOf(parameters.get(PARAMETER_NAME_POPULATE_OPTIONAL_FIELDS));
            }
        }
          
        //"Start evaluating results..."
        
        //crunches the results file and find out the best individual
        config.getBestSelector().elaborate(results);
        results.setOverallExecutionTimeMillis(timeTaken);
        //Populate optional fields
        if(populateOptionalFields){
            results.setExamples(config.getDatasetContainer().getDataset().getExamples());
        }
        try {
            //Populate extractions data and stats
            results.setBestExtractions(this.getEvaluations(results.getBestSolution().getSolution(), config, Context.EvaluationPhases.LEARNING));
            results.setBestExtractionsStrings(this.getEvaluationsStrings(results.getBestExtractions(),config.getDatasetContainer().getLearningDataset()));
            results.setBestExtractionsStats(this.getEvaluationStats(results.getBestExtractions(), config));
        } catch (TreeEvaluationException ex) {
            Logger.getLogger(BasicPostprocessor.class.getName()).log(Level.SEVERE, null, ex);
        }
        
        
        //Populate knowledge avaiable
        DataSet training = config.getDatasetContainer().getTrainingDataset();
        DataSet learning = config.getDatasetContainer().getLearningDataset();
        //training stats
        int numberTrainingMatches = training.getNumberMatches();
        int numberTrainingUnmatches = training.getNumberUnmatches();
        results.setNumberTrainingMatches(numberTrainingMatches);
        results.setNumberTrainingUnmatches(numberTrainingUnmatches);
        //Here is the KA from definitions
        int numberMatches = learning.getNumberMatches();
        int numberUnmatches = learning.getNumberUnmatches();
        int numberMatchedChars = learning.getNumberMatchedChars();
        int numberUnmatchedChars = learning.getNumberUnmatchedChars();
        int numberAllChars = config.getDatasetContainer().getDataset().getNumberOfChars();
        results.setNumberMatches(numberMatches);
        results.setNumberUnmatches(numberUnmatches);
        results.setNumberMatchedChars(numberMatchedChars);
        results.setNumberUnmatchedChars(numberUnmatchedChars);
        results.setNumberAllChars(numberAllChars);
        
    }
    
    private List<List<DataSet.Bounds>> getEvaluations(String solution, Configuration configuration, Context.EvaluationPhases phase) throws TreeEvaluationException{
        TreeEvaluator treeEvaluator = configuration.getEvaluator();
        Node bestIndividualReplica = new Constant(solution);
        return treeEvaluator.evaluate(bestIndividualReplica, new Context(phase, configuration));
    }
    
    private List<List<String>> getEvaluationsStrings(List<List<Bounds>> extractions, DataSet dataset){
        List<List<String>> evaluationsStrings = new LinkedList<>();
        Iterator<Example> it = dataset.getExamples().iterator();
        for (List<Bounds> extractionsOfExample : extractions) {
            Example example = it.next();
            List<String> extractionsOfExampleStrings = new LinkedList<>();
            for (Bounds bounds : extractionsOfExample) {
                extractionsOfExampleStrings.add(example.getString().substring(bounds.start,bounds.end));
            }
            evaluationsStrings.add(extractionsOfExampleStrings);
        }
        return evaluationsStrings;
    }

    
    //errors per example, on learning; 
    private List<BasicStats> getEvaluationStats(List<List<DataSet.Bounds>> evaluation, Configuration config) throws TreeEvaluationException{
        DataSet dataset = config.getDatasetContainer().getLearningDataset();
        List<BasicStats> statsPerExample = new LinkedList<>();
        for (int index = 0; index < dataset.getExamples().size(); index++) {
            List<DataSet.Bounds> extractionsList = evaluation.get(index);
            Set<DataSet.Bounds> extractionsSet = new HashSet<>(extractionsList);
            DataSet.Example example = dataset.getExample(index);
            extractionsSet.removeAll(example.getMatch()); //left only false extractions
            BasicStats exampleStats = new BasicStats();
            exampleStats.fn = -1; //unset, not interesting at the moment
            exampleStats.fp = extractionsSet.size();
            exampleStats.tp = extractionsList.size() - exampleStats.fp;
            exampleStats.tn = -1; //unset, not interesting at the moment
            statsPerExample.add(exampleStats);
        }
        return statsPerExample;
    }
} public class JsonPostProcessor extends BasicPostprocessor {

    @Override
    public void setup(Map<String, String> parameters) {
    }

    @Override
    public void elaborate(Configuration config, Results results, long timeTaken) {
        super.elaborate(config, results, timeTaken);
        //saving results
        System.out.println("Saving results...");
        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddhhmmss");
	String dateFormatted = sdf.format(new Date());
        String pathOfFile = config.getOutputFolder().getAbsolutePath() + File.separator + "results-"+dateFormatted+".json";
        saveToJson(results, pathOfFile);
        
        String time = String.format("%d h, %d m, %d s",
                TimeUnit.MILLISECONDS.toHours(timeTaken),
                TimeUnit.MILLISECONDS.toMinutes(timeTaken) - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(timeTaken)),
                TimeUnit.MILLISECONDS.toSeconds(timeTaken) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(timeTaken)));
        System.out.println("Time taken: "+time);
        
    }

    private void saveToJson(Results results, String pathOfFile) {
        Gson gson = new GsonBuilder().disableHtmlEscaping().setPrettyPrinting().registerTypeAdapter(Double.class, new DoubleTypeAdapter()).create();
        String json = gson.toJson(results);
        saveFile(json, pathOfFile);

    }

    private void saveFile(String text, String pathOfFile) {
        Writer writer;
        try {
            writer = new OutputStreamWriter(new FileOutputStream(pathOfFile), "utf-8");
            writer.write(text);
            writer.close();
        } catch (IOException ex) {
            Logger.getLogger(JsonPostProcessor.class.getName()).log(Level.SEVERE, "Cannot save:", ex);
        }

    }
   
} public class Utils {

    public static float[] calculateMeanFitness(List<Ranking> population) {
        float[] out = new float[population.get(0).getFitness().length];
        for (Ranking r : population) {
            double[] fitness = r.getFitness();
            for (int i = 0; i < out.length; i++) {
                out[i] += fitness[i];
            }
        }
        for (int i = 0; i < out.length; i++) {
            out[i] /= population.size();
        }
        return out;
    }

    public static boolean isAParetoDominateByB(double fitnessA[], double fitnessB[]) {
        boolean dominate = false;
        for (int i = 0; i < fitnessA.length; i++) {
            if (fitnessA[i] > fitnessB[i]) {
                dominate = true;
            } else if (fitnessA[i] < fitnessB[i]) {
                return false;
            }
        }
        return dominate;
    }

    public static List<Ranking> getFirstParetoFront(List<Ranking> tmp) {
        List<Ranking> front = new LinkedList<>();

        for (Ranking r1 : tmp) {
            boolean isDominate = false;
            for (Ranking r2 : tmp) {
                if (r1.equals(r2)) {
                    continue;
                }
                if (Utils.isAParetoDominateByB(r1.getFitness(), r2.getFitness())) {
                    isDominate = true;
                    break;
                }
            }
            if (!isDominate) {
                front.add(r1);
            }
        }
        return front;
    }

    public static String cpuInfo() throws IOException {
        if (!System.getProperty("os.name").toLowerCase().contains("linux")) {
            return "Unaviable";
        }
        FileInputStream fis = new FileInputStream(new File("/proc/cpuinfo"));
        InputStreamReader isr = new InputStreamReader(fis);
        BufferedReader bufferedReader = new BufferedReader(isr);
        String line;
        while ((line = bufferedReader.readLine()) != null) {
            if (line.matches("model name.*")) {
                bufferedReader.close();
                return line.replace("model name	: ", "");
            }
        }
        return "";
    }

    public static double diversity(List<Ranking> population) {
        Set<String> tmp = new HashSet<>();
        for (Ranking r : population) {
            tmp.add(r.getDescription());
        }
        return 100 * tmp.size() / (double) population.size();
    }

    //remove empty extractions 
    public static void removeEmptyExtractions(List<DataSet.Bounds> extractions) {
        for (Iterator<Bounds> it = extractions.iterator(); it.hasNext();) {
            Bounds bounds = it.next();
            if (bounds.size() == 0) {
                it.remove();
            }
        }
    }

    public static void saveFile(String text, String pathOfFile) {
        Writer writer;
        try {
            writer = new OutputStreamWriter(new FileOutputStream(pathOfFile), "utf-8");
            writer.write(text);
            writer.close();
        } catch (IOException ex) {
            Logger.getLogger(Utils.class.getName()).log(Level.SEVERE, "Cannot save:", ex);
        }
    }

    public static String humanReadableByteCount(long bytes, boolean si) {
        int unit = si ? 1000 : 1024;
        if (bytes < unit) {
            return bytes + " B";
        }
        int exp = (int) (Math.log(bytes) / Math.log(unit));
        String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i");
        return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
    }

    private static transient final Set<Character> quoteList = new TreeSet<>(
            Arrays.asList(new Character[]{'?', '+', '*', '.', '[', ']', '\\', '$', '(', ')', '^', '{', '|', '-', '&'}));

    /**
     * Returns a set with all n-grams; 1<n<4
     * @param word
     * @return
     */
    public static Set<String> subparts(String word) {
        return Utils.subparts(word,1,4);
    }

    /**
     * Returns a set with all n-grams; nMin<=n<=nMax
     * @param word
     * @return
     */
    public static Set<String> subparts(String word, int nMin, int nMax) {
        Set<String> subparts = new HashSet<String>();
        for (int n = nMin; n <= nMax; n++) {
            for (int i = 0; i < word.length(); i++) {
                StringBuilder builder = new StringBuilder();
                String w = word.substring(i, Math.min(i + n, word.length()));
                for (char c : w.toCharArray()) {
                    builder.append(escape(c));
                }
                subparts.add(builder.toString());
            }
        }
        return subparts;
    }
    
    public static String escape(char c) {
        if (quoteList.contains(c)) {
            return ("\\" + c);
        }
        return ("" + c);
    }
    
    public static String escape(String string){
        StringBuilder stringBuilder = new StringBuilder();
        char[] stringChars = string.toCharArray();
        for(char character : stringChars){
            stringBuilder.append(escape(character));
        }
        return stringBuilder.toString();
    }

    /**
     * Generates RegexRanges i.e. [a-z] from contiguous characters into the <code>charset</code> list.
     * Follows example where output is represented with regex strings:
     * When <code>charset</code> is {a,b,g,r,t,u,v,5}, the return value is {[a-b],[t-v]}.
     * @param charset the character list i.e. {a,b,g,r,t,u,v,5}
     * @return the contiguous character ranges i.e. {[a-b],[t-v]}
     */
    public static List<RegexRange> generateRegexRanges(Collection<Character> charset) {
         
        List<RegexRange> regexRangesList = new LinkedList<>();
        TreeSet<Character> orderedCharset = new TreeSet<>(charset);
        Character start = null;
        Character old = null;
        for (Character c : charset) {
            if (old == null) {
                //The first round
                old = orderedCharset.first();
                start = old;
                continue; //pick the next 
            }
            //when we have an "hole" or is the last character it checks if the previous range (old -start) is larger than 1; 
            //Ranges bigger than 1 char are saved
            if (((c - old) > 1 || Objects.equals(orderedCharset.last(), c))) {
                if ((old - start) > 1) {
                    regexRangesList.add(new RegexRange(escape(start) + "-" + escape(old)));
                }
                start = c;
            }
            old = c;
        }
        return regexRangesList;
    }
}
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import it.units.inginf.male.Main;
import it.units.inginf.male.configuration.Configuration;
import it.units.inginf.male.dto.SimpleConfig;
import it.units.inginf.male.inputs.DataSet;
import it.units.inginf.male.inputs.DataSet.Example;
import it.units.inginf.male.outputs.FinalSolution;
import it.units.inginf.male.outputs.Results;
import it.units.inginf.male.postprocessing.BasicPostprocessor;
import it.units.inginf.male.postprocessing.JsonPostProcessor;
import it.units.inginf.male.strategy.ExecutionStrategy;
import it.units.inginf.male.strategy.impl.CoolTextualExecutionListener;
import it.units.inginf.male.utils.Utils;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.logging.Level;
import java.util.logging.Logger;
/*
 * Copyright (C) 2015 Machine Learning Lab - University of Trieste, 
 * Italy (http://machinelearning.inginf.units.it/)  
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package it.units.inginf.male.console;


/**
 * Provides a commandline tool for the GP Engine, RandomRegexTurtle.
 *
 * @author MaleLabTs
 */
public class ConsoleRegexTurtle {

    private static String WARNING_MESSAGE = "\nWARNING\n"
            + "The quality of the solution depends on a number of factors, including size and syntactical properties of the learning information.\n"
            + "The algorithms embedded in this experimental prototype have always been tested with at least 25 matches over at least 2 examples.\n"
            + "It is very unlikely that a smaller number of matches allows obtaining a useful solution.\n";

    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) {
        SimpleConfig simpleConfiguration = new SimpleConfig();