public static final double[][][] ALLELE_FREQUENCIES = {
    {
        {Double.NaN, Double.NaN, Double.NaN},
        {0.5, 0.5},
        {0.0, 0.5, 0.5},
        {0.0, 0.0, 0.5, 0.5},
        {Double.NaN, Double.NaN, Double.NaN},
        {0.0, 1.0},
        {1.0, 0.0}
    },
    {
        {1.0, 0.0, 0.0},
        {0.5, 0.5},
        {0.0, 0.5, 0.5},
        {1.0, 0.0, 0.0, 0.0},
        {0.3333333333, 0.3333333333, 0.3333333333},
        {1.0, 0.0},
        {0.0, 1.0}
    },
    {
        {0.6, 0.0, 0.4},
        {0.5, 0.5},
        {0.0, 0.5, 0.5},
        {0.25, 0.25, 0.25, 0.25},
        {0.0, 0.5, 0.5},
        {0.0, 1.0},
        {1.0, 0.0}
    },
    {
        {Double.NaN, Double.NaN, Double.NaN},
        {1.0, 0.0},
        {Double.NaN, Double.NaN, Double.NaN},
        {0.0, 0.0, 1.0, 0.0},
        {0.3333333333, 0.3333333333, 0.3333333333},
        {0.0, 1.0},
        {1.0, 0.0}
    },
    {
        {0.3333333333, 0.3333333333, 0.3333333333},
        {0.5, 0.5},
        {0.0, 0.5, 0.5},
        {0.5, 0.0, 0.5, 0.0},
        {0.3333333333, 0.3333333333, 0.3333333333},
        {1.0, 0.0},
        {Double.NaN, Double.NaN}
    }
}; public static final String[][] ALLELE_NAMES = {
    {"mk1-1", "mk1-2", "mk1-3"},
    {"mk2-1", "mk2-2"},
    {null, "mk3-2", null},
    {"mk4-1", "mk4-2", "mk4-3", "mk4-4"},
    {null, "mk5-2", "mk5-3"},
    {"mk6-1", "mk6-2"},
    {"mk7-1", "mk7-2"}
}; public static final SimpleEntity[] HEADERS_NON_UNIQUE_NAMES; public static final String[] MARKER_NAMES = {
    "mk1",
    "mk2",
    "mk3",
    "mk4",
    "mk5",
    "mk6",
    "mk7"
}; public static final double[][] MODIFIED_ROGERS_DISTANCES = {
    {0, 0.626783170528009, 0.133630620956212, 0.267261241912424, 0.422577127364258},
    {0.626783170528009, 0, 0.611594325552174, 0.681385143869247, 0.288675134594813},
    {0.133630620956212, 0.611594325552174, 0, 0.318104505140176, 0.431221962511629},
    {0.267261241912424, 0.681385143869247, 0.318104505140176, 0, 0.462910049886276},
    {0.422577127364258, 0.288675134594813, 0.431221962511629, 0.462910049886276, 0}
}; public static final double[][] MODIFIED_ROGERS_DISTANCES_CEIL_MISSING = {
    {0, 0.823754471047914, 0.550973165019340, 0.707106781186548, 0.779193722473980},
    {0.823754471047914, 0, 0.611594325552174, 0.866025403784439, 0.475594865605671},
    {0.550973165019340, 0.611594325552174, 0, 0.622016689410149, 0.573419151938199},
    {0.707106781186548, 0.866025403784439, 0.622016689410149, 0, 0.801783725737273},
    {0.779193722473980, 0.475594865605671, 0.573419151938199, 0.801783725737273, 0}
}; public static final String NAME = "Test Dataset"; public static final double PRECISION = 1e-8; public class CoreHunterData extends DataPojo implements IntegerIdentifiedData {
    
    /**
     * 
     */
    private static final long serialVersionUID = 1L;
    private static final List<CoreHunterMeasure> NO_MEASURES = new LinkedList<>() ;
    private final FrequencyGenotypeData genotypicData; 
    private final FeatureData phenotypicData;
    private final DistanceMatrixData distancesData;
    private final List<CoreHunterObjectiveType> validObjectiveTypes;
    private final List<CoreHunterMeasure> validMeasures;

    /**
     * Initialize Core Hunter data consisting of genotypic data, phenotypic
     * traits and/or a precomputed distance matrix. At least one of these should
     * be defined (i.e. non <code>null</code>). Items should be ordered in the
     * same way across all datasets, which should all be of the same size n. If
     * headers are specified (item names and/or unique identifiers) in some or
     * all datasets these should also be consistent across datasets. If this is
     * not the case, an exception will be thrown.
     * <p>
     * Integer IDs as required by {@link IntegerIdentifiedData} are set to [0,
     * n-1]. The name of the dataset is set to "Core Hunter data".
     * 
     * @param genotypicData
     *            genotypic data (bi- or multiallelic)
     * @param phenotypicData
     *            phenotypic traits
     * @param distancesData
     *            precomputed distance matrix
     */
    public CoreHunterData(FrequencyGenotypeData genotypicData,
                          FeatureData phenotypicData, 
                          DistanceMatrixData distancesData) {
        super("Core Hunter data", mergeHeaders(genotypicData, phenotypicData, distancesData));
        // store data
        this.genotypicData = genotypicData;
        this.phenotypicData = phenotypicData;
        this.distancesData = distancesData;
        
        validObjectiveTypes = new LinkedList<>() ;
        validMeasures = new LinkedList<>() ;
        
        if (genotypicData != null || phenotypicData != null || distancesData != null) {
            validObjectiveTypes.add(CoreHunterObjectiveType.AV_ENTRY_TO_ENTRY) ;
            validObjectiveTypes.add(CoreHunterObjectiveType.AV_ENTRY_TO_NEAREST_ENTRY) ;
            validObjectiveTypes.add(CoreHunterObjectiveType.AV_ACCESSION_TO_NEAREST_ENTRY) ;
            
            if (genotypicData != null) {
                validObjectiveTypes.add(CoreHunterObjectiveType.COVERAGE) ;
                validObjectiveTypes.add(CoreHunterObjectiveType.HETEROZYGOUS_LOCI) ;
                validObjectiveTypes.add(CoreHunterObjectiveType.SHANNON_DIVERSITY) ;
                
                validMeasures.add(CoreHunterMeasure.MODIFIED_ROGERS) ;
                validMeasures.add(CoreHunterMeasure.CAVALLI_SFORZA_EDWARDS) ;
            }
            
            if (phenotypicData != null) {
                validMeasures.add(CoreHunterMeasure.GOWERS) ;
            }
            
            if (distancesData != null) {
                validMeasures.add(CoreHunterMeasure.PRECOMPUTED_DISTANCE) ;
            }
        }
        
    }

    /**
     * Initialize Core Hunter data consisting only of genotypic data.
     * <p>
     * Integer IDs as required by {@link IntegerIdentifiedData} are set to [0,
     * n-1]. The name of the dataset is set to "Core Hunter data".
     * 
     * @param genotypicData
     *            genotypic data (bi- or multiallelic)
     */
    public CoreHunterData(FrequencyGenotypeData genotypicData){
        this(genotypicData, null, null);
    }

    /**
     * Initialize Core Hunter data consisting only of phenotypicData data.
     * <p>
     * Integer IDs as required by {@link IntegerIdentifiedData} are set to [0,
     * n-1]. The name of the dataset is set to "Core Hunter data".
     * 
     * @param phenotypicData
     *            phenotypic traits
     */
    public CoreHunterData(FeatureData phenotypicData) {
        this(null, phenotypicData, null);
    }

    /**
     * Initialize Core Hunter data consisting only of distances data.
     * <p>
     * Integer IDs as required by {@link IntegerIdentifiedData} are set to [0,
     * n-1]. The name of the dataset is set to "Core Hunter data".
     * 
     * @param distancesData
     *            precomputed distance matrix
     */
    public CoreHunterData(DistanceMatrixData distancesData) {
        this(null, null, distancesData);
    }

    public FrequencyGenotypeData getGenotypicData() {
        return genotypicData;
    }

    public FeatureData getPhenotypicData() {
        return phenotypicData;
    }

    public DistanceMatrixData getDistancesData() {
        return distancesData;
    }
    
    public boolean hasGenotypes(){
        return genotypicData != null;
    }
    
    public boolean hasPhenotypes(){
        return phenotypicData != null;
    }
    
    public boolean hasDistances(){
        return distancesData != null;
    }

    private static int inferSize(FrequencyGenotypeData genotypicData,
            FeatureData phenotypicData, DistanceMatrixData distancesData){
        
        // check not all undefined
        if (genotypicData == null && phenotypicData == null && distancesData == null) {
            throw new IllegalArgumentException(
                    "At least one type of data (genotypic, phenotypic, distances) should be defined.");
        }
        // check same size
        int[] sizes = Arrays.asList(genotypicData, phenotypicData, distancesData).stream().filter(Objects::nonNull)
                .mapToInt(Data::getSize).distinct().toArray();
        boolean sameSize = (sizes.length == 1);

        if (!sameSize) {
            if (sizes.length == 2) {
                throw new IllegalArgumentException(
                        String.format("Provided datasets have different sizes. %d %d", sizes[0], sizes[1]));
            } else {
                if (sizes.length == 3) {
                    throw new IllegalArgumentException(String.format("Provided datasets have different sizes. %d %d %d",
                            sizes[0], sizes[1], sizes[2]));
                }
            }
        }

        return sizes[0];

    }

    // assumes that sizes have already been checked
    private static SimpleEntity[] mergeHeaders(FrequencyGenotypeData genotypicData,
            FeatureData phenotypicData, DistanceMatrixData distancesData){
        
        int size = inferSize(genotypicData, phenotypicData, distancesData) ;
        
        SimpleEntity[] headers = Arrays.asList(genotypicData, phenotypicData, distancesData)
                .stream()

                .filter(Objects::nonNull)
                // extract headers from each dataset
                .map(data -> {
                    SimpleEntity[] h = new SimpleEntity[data.getSize()];
                    for (int i = 0; i < h.length; i++) {
                        h[i] = data.getHeader(i);
                    }
                    return h;
                })
                // merge and check for inconsistencies
                .reduce((headers1, headers2) -> {
                    SimpleEntity[] h = new SimpleEntity[headers1.length];
                    for (int i = 0; i < h.length; i++) {
                        SimpleEntity h1 = headers1[i];
                        SimpleEntity h2 = headers2[i];
                        if (h1 == null) {
                            h[i] = h2;
                        } else if (h2 == null) {
                            h[i] = h1;
                        } else {
                            // both datasets have header for this item: check
                            // consistency
                            if (h1.equals(h2)) {
                                // equal means same id: check name
                                if (h1.getName() != null && h2.getName() != null
                                        && !h1.getName().equals(h2.getName())) {
                                    throw new IllegalArgumentException(String.format(
                                            "Headers do not match for item %d. "
                                                    + "Got same id %s but different names %s and %s.",
                                            i, h1.getUniqueIdentifier(), h1.getName(), h2.getName()));
                                }
                                // store (make sure to pick the one which has a
                                // name set, if not both)
                                if (h1.getName() != null) {
                                    h[i] = h1;
                                } else {
                                    h[i] = h2;
                                }
                            } else {
                                throw new IllegalArgumentException(
                                        String.format("Headers do not match for item %d. Got different ids %s and %s.",
                                                i, h1.getUniqueIdentifier(), h2.getUniqueIdentifier()));
                            }
                        }
                    }
                    return h;
                }).get();

        // create headers if not provided in any dataset
        if (Arrays.stream(headers).allMatch(Objects::isNull)) {
            headers = DataPojo.updateOrCreateHeaders(null, size);
        }

        return headers;
    }

    public final List<CoreHunterObjectiveType> getValidObjectiveTypes() {
        return validObjectiveTypes ;
    }  
    
    public final List<CoreHunterMeasure> getValidMeasures(CoreHunterObjectiveType objectiveType) {
        
        switch(objectiveType) {
            case AV_ACCESSION_TO_NEAREST_ENTRY:
            case AV_ENTRY_TO_ENTRY:
            case AV_ENTRY_TO_NEAREST_ENTRY:
                return validMeasures ;
            case COVERAGE:
            case HETEROZYGOUS_LOCI:
            case SHANNON_DIVERSITY:
            default:
                return NO_MEASURES ;
            
        }
    }
} public class SimpleFrequencyGenotypeData extends DataPojo implements FrequencyGenotypeData {

    private static final long serialVersionUID = 1L;

    private static final double SUM_TO_ONE_PRECISION = 0.01 + 1e-8;
    private static final String ID_HEADER = "X";
    private static final String NAMES_HEADER = "NAME";
    private static final String ALLELE_NAMES_HEADER = "ALLELE";
    private static final String IDENTIFIERS_HEADER = "ID";
    private static final String SELECTED_HEADER = "SELECTED";

    private final double[][][] alleleFrequencies;
    private final String[] markerNames; // null element means no marker name assigned
    private final String[][] alleleNames; // null element means no allele name assigned
    private final int totalNumberAlleles;

    /**
     * Create data with name "Allele frequency data". For details of the arguments see
     * {@link #SimpleFrequencyGenotypeData(String, SimpleEntity[], String[], String[][], double[][][])}.
     * 
     * @param itemHeaders item headers, specifying name and/or unique identifier
     * @param markerNames marker names
     * @param alleleNames allele names per marker
     * @param alleleFrequencies allele frequencies
     */
    public SimpleFrequencyGenotypeData(SimpleEntity[] itemHeaders, String[] markerNames,
                                       String[][] alleleNames, double[][][] alleleFrequencies) {
        this("Allele frequency data", itemHeaders, markerNames, alleleNames, alleleFrequencies);
    }

    /**
     * Create data with given dataset name, item headers, marker/allele names
     * and allele frequencies. The length of <code>alleleFrequencies</code>
     * denotes the number of items in the dataset. The length of
     * <code>alleleFrequencies[i]</code> should be the same for all
     * <code>i</code> and denotes the number of markers. Finally, the length of
     * <code>alleleFrequencies[i][m]</code> should also be the same for all
     * <code>i</code> and denotes the number of alleles of the <code>m</code>-th
     * marker. Allele counts may differ for different markers.
     * <p>
     * All frequencies should be positive and the values in
     * <code>alleleFrequencies[i][m]</code> should sum to one for all
     * <code>i</code> and <code>m</code>, with a precision of 0.01. Missing
     * values are encoded with {@link Double#NaN}.
     * If one or more allele frequencies are missing at a certain marker for a certain
     * individual, the remaining frequencies should sum to a value less than or equal to one.
     * <p>
     * Item headers are required and marker/allele names are optional. If marker
     * and/or allele names are given they need not be defined for all
     * markers/alleles nor unique. Each item should at least have a unique
     * identifier (names are optional). If not <code>null</code>, the length of
     * each header/name array should correspond to the dimensions of
     * <code>alleleFrequencies</code> (number of individuals, markers and
     * alleles per marker).
     * <p>
     * Violating any of the requirements will produce an exception.
     * <p>
     * Allele frequencies as well as assigned headers and names are copied into
     * internal data structures, i.e. no references are retained to any of the
     * arrays passed as arguments.
     * 
     * @param datasetName
     *            name of the dataset
     * @param itemHeaders
     *            item headers; its length should correspond to the number of
     *            individuals and each item should at least have a unique
     *            identifier (names are optional)
     * @param markerNames
     *            marker names, <code>null</code> if no marker names are
     *            assigned; if not <code>null</code> its length should
     *            correspond to the number of markers (can contain
     *            <code>null</code> values)
     * @param alleleNames
     *            allele names per marker, <code>null</code> if no allele names
     *            are assigned; if not <code>null</code> the length of
     *            <code>alleleNames</code> should correspond to the number of
     *            markers and the length of <code>alleleNames[m]</code> to the
     *            number of alleles of the m-th marker (can contain
     *            <code>null</code> values)
     * @param alleleFrequencies
     *            allele frequencies, may not be <code>null</code>; missing values
     *            are encoded with {@link Double#NaN}; dimensions indicate number
     *            of individuals, markers and alleles per marker
     */
    public SimpleFrequencyGenotypeData(String datasetName, SimpleEntity[] itemHeaders,
                                       String[] markerNames, String[][] alleleNames,
                                       double[][][] alleleFrequencies) {

        // pass dataset name and item headers to parent
        super(datasetName, itemHeaders);

        // check allele frequencies and infer number of individuals, markers and
        // alleles per marker
        int n = alleleFrequencies.length;
        int m = -1;
        int[] numberOfAllelesForMarker = null;
        // loop over individuals
        for (int i = 0; i < n; i++) {
            double[][] indFreqs = alleleFrequencies[i];
            if (indFreqs == null) {
                throw new IllegalArgumentException("Allele frequencies not defined for individual " + i);
            }
            if (m == -1) {
                m = indFreqs.length;
                numberOfAllelesForMarker = new int[m];
                Arrays.fill(numberOfAllelesForMarker, -1);
            } else if (indFreqs.length != m) {
                throw new IllegalArgumentException("All individuals should have same number of markers.");
            }
            // loop over markers
            for (int j = 0; j < m; j++) {
                double[] alleleFreqs = indFreqs[j];
                // check that frequencies are defined
                if (alleleFreqs == null) {
                    throw new IllegalArgumentException(
                        String.format("Allele frequencies not defined for individual %d at marker %d.", i, j)
                    );
                }
                // check or set number of alleles for current marker
                if (numberOfAllelesForMarker[j] == -1) {
                    numberOfAllelesForMarker[j] = alleleFreqs.length;
                } else if (alleleFreqs.length != numberOfAllelesForMarker[j]) {
                    throw new IllegalArgumentException(
                        "Number of alleles per marker should be consistent across all individuals."
                    );
                }
                // check positive
                if (Arrays.stream(alleleFreqs).anyMatch(f -> f < 0.0)) {
                    throw new IllegalArgumentException("All frequencies should be positive.");
                }
                // check sum <= one
                double sum = Arrays.stream(alleleFreqs)
                                   .filter(f -> !Double.isNaN(f))
                                   .sum();
                if (sum > 1.0 + SUM_TO_ONE_PRECISION) {
                    throw new IllegalArgumentException(
                        "Allele frequency sum per marker should not exceed one."
                    );
                }
                // if no missing values: should sum to one
                if (Arrays.stream(alleleFreqs).noneMatch(Double::isNaN)) {
                    if (1.0 - sum > SUM_TO_ONE_PRECISION) {
                        throw new IllegalArgumentException(
                            "Allele frequencies for marker should sum to one."
                        );
                    }
                    // normalize to avoid numerical imprecisions
                    for (int k = 0; k < alleleFreqs.length; k++) {
                        alleleFreqs[k] /= sum;
                    }
                }
            }
        }

        // set total number of alleles
        totalNumberAlleles = Arrays.stream(numberOfAllelesForMarker).sum();

        // copy allele frequencies
        this.alleleFrequencies = new double[n][m][];
        for (int i = 0; i < n; i++) {
            for (int j = 0; j < m; j++) {
                this.alleleFrequencies[i][j] = Arrays.copyOf(alleleFrequencies[i][j], numberOfAllelesForMarker[j]);
            }
        }

        // check and copy marker names
        if (markerNames == null) {
            this.markerNames = new String[m];
        } else {
            if (markerNames.length != m) {
                throw new IllegalArgumentException(
                    String.format(
                        "Incorrect number of marker names provided. Expected: %d, actual: %d.",
                        m, markerNames.length
                    )
                );
            }
            this.markerNames = Arrays.copyOf(markerNames, m);
        }

        // check and copy allele names
        this.alleleNames = new String[m][];
        if (alleleNames == null) {
            for (int j = 0; j < m; j++) {
                this.alleleNames[j] = new String[numberOfAllelesForMarker[j]];
            }
        } else {
            if (alleleNames.length != m) {
                throw new IllegalArgumentException(String.format(
                    "Incorrect number of marker-allele names provided. Expected: %d, actual: %d.",
                    m, alleleNames.length
                ));
            }
            for (int j = 0; j < m; j++) {
                if (alleleNames[j] == null) {
                    this.alleleNames[j] = new String[numberOfAllelesForMarker[j]];
                } else if (alleleNames[j].length != numberOfAllelesForMarker[j]) {
                    throw new IllegalArgumentException(String.format(
                        "Incorrect number of allele names provided for marker %d. Expected: %d, actual: %d.",
                        j, numberOfAllelesForMarker[j], alleleNames[j].length
                    ));
                } else {
                    this.alleleNames[j] = Arrays.copyOf(alleleNames[j], numberOfAllelesForMarker[j]);
                }
            }
        }

    }

    @Override
    public int getNumberOfMarkers() {
        return markerNames.length;
    }

    @Override
    public int getNumberOfAlleles(int markerIndex) {
        return alleleNames[markerIndex].length;
    }

    @Override
    public String getMarkerName(int markerIndex) {
        return markerNames[markerIndex];
    }

    @Override
    public int getTotalNumberOfAlleles() {
        return totalNumberAlleles;
    }

    @Override
    public String getAlleleName(int markerIndex, int alleleIndex) {
        return alleleNames[markerIndex][alleleIndex];
    }

    @Override
    public double getAlleleFrequency(int id, int markerIndex, int alleleIndex) {
        return alleleFrequencies[id][markerIndex][alleleIndex];
    }

    @Override
    public boolean hasMissingValues(int id, int markerIndex) {
        return Arrays.stream(alleleFrequencies[id][markerIndex]).anyMatch(Double::isNaN);
    }

    /**
     * Read genotype data from file. Only file types {@link FileType#TXT} and
     * {@link FileType#CSV} are allowed. Values are separated with a single tab
     * (txt) or comma (csv) character.
     * <p>
     * The file contains allele frequencies following the requirements as described in the constructor
     * {@link #SimpleFrequencyGenotypeData(String, SimpleEntity[], String[], String[][], double[][][])}.
     * Missing frequencies are encoding as empty cells. The file starts with a
     * compulsory header row from which marker names and allele counts are
     * inferred. All columns corresponding to the same marker occur
     * consecutively in the file and are named after that marker, optionally
     * including a suffix as described below. Marker names should be unique.
     * There is one compulsory header column "ID" containing unique item
     * identifiers. Optionally a second header column "NAME" can be included to
     * provide (not necessarily unique) item names in addition to the unique
     * identifiers. Finally, an optional second header row can be included to
     * define allele names per marker, identified with row header "ALLELE".
     * Allele names need not be unique and can be undefined for some alleles by
     * leaving the corresponding cells empty.
     * <p>
     * Leading and trailing whitespace is removed from names and
     * unique identifiers and they are unquoted if wrapped in single or double
     * quotes after whitespace removal. If it is intended to start or end a
     * name/identifier with whitespace this whitespace should be contained
     * within the quotes, as it will then not be removed. Also, column names may
     * optionally include an arbitrary suffix added to the marker name, starting
     * with a dash, underscore or dot character. The latter allows to use column
     * names such as "M1-1" and "M1-2", "M1.a" and "M1.b" or "M1_1" and "M1_2"
     * for a marker named "M1" with two columns. The column name prefix up to
     * before the last occurrence of any dash, underscore or dot character is
     * taken to be the marker name.
     * <p>
     * Trailing empty cells can be omitted from any row in the file.
     * <p>
     * The dataset name is set to the name of the file to which
     * <code>filePath</code> points.
     * 
     * @param filePath path to file that contains the data
     * @param fileType {@link FileType#TXT} or {@link FileType#CSV}
     * @return frequency genotype data read from the given file
     * @throws IOException if the file can not be read or is not correctly formatted
     */
    public static FrequencyGenotypeData readData(Path filePath, FileType fileType) throws IOException {

        // validate arguments
        if (filePath == null) {
            throw new IllegalArgumentException("File path not defined.");
        }

        if (!filePath.toFile().exists()) {
            throw new IOException("File does not exist : " + filePath + ".");
        }

        if (fileType == null) {
            throw new IllegalArgumentException("File type not defined.");
        }

        if (fileType != FileType.TXT && fileType != FileType.CSV) {
            throw new IllegalArgumentException(
                String.format("Only file types TXT and CSV are supported. Got: %s.", fileType)
            );
        }

        // read data from file

        try (RowReader reader = IOUtilities.createRowReader(
                filePath, fileType,
                TextFileRowReader.REMOVE_WHITE_SPACE,
                TextFileRowReader.ROWS_SAME_SIZE_AS_FIRST,
                TextFileRowReader.REMOVE_QUOTES
        )) {
            if (reader == null || !reader.ready()) {
                throw new IOException("Can not create reader for file " + filePath + ". File may be empty.");
            }

            if (!reader.hasNextRow()) {
                throw new IOException("File is empty.");
            }

            // 1: read marker names

            reader.nextRow();
            String[] markerNamesRow = reader.getRowCellsAsStringArray();
            // check presence of header columns
            if (markerNamesRow.length < 1 || !Objects.equals(markerNamesRow[0], IDENTIFIERS_HEADER)) {
                throw new IOException("Missing header row/column ID.");
            }
            boolean withNames = (markerNamesRow.length >= 2 && Objects.equals(markerNamesRow[1], NAMES_HEADER));
            int numHeaderCols = 1;
            if (withNames) {
                numHeaderCols++;
            }
            // infer and check number of data columns
            int numCols = markerNamesRow.length;
            int numDataCols = numCols - numHeaderCols;
            if (numDataCols == 0) {
                throw new IOException("No data columns.");
            }

            // extract marker column names
            markerNamesRow = Arrays.stream(markerNamesRow)
                                   .skip(numHeaderCols)
                                   .toArray(n -> new String[n]);

            // extract/check names and infer number of alleles per marker
            HashMap<String, Integer> markers;
            try {
                markers = inferMarkerNames(markerNamesRow);
            } catch (IllegalArgumentException ex) {
                throw new IOException(ex);
            }
            String[] markerNames = markers.keySet().toArray(new String[0]);
            Integer[] alleleCounts = markers.values().toArray(new Integer[0]);
            // infer number of markers
            int numMarkers = markers.size();

            // 2: read data rows (and allele names header if provided)

            String[][] alleleNames = null;
            List<String> itemNames = new ArrayList<>();
            List<String> itemIdentifiers = new ArrayList<>();
            List<double[][]> alleleFreqs = new ArrayList<>();
            int r = 1;
            while (reader.nextRow()) {

                // read row as strings
                String[] row = reader.getRowCellsAsStringArray();
                // check length
                if (row.length != numCols) {
                    throw new IOException(
                        String.format(
                            "Incorrect number of columns at row %d. Expected: %d, actual: %d.",
                            r, numCols, row.length
                        )
                    );
                }

                // check for allele names row
                if (Objects.equals(row[0], ALLELE_NAMES_HEADER)) {
                    // verify: second row
                    if (r != 1) {
                        throw new IOException("Allele names header should be the second row in the file.");
                    }
                    // extract allele names grouped per marker
                    int aglob = numHeaderCols;
                    alleleNames = new String[numMarkers][];
                    for (int m = 0; m < numMarkers; m++) {
                        alleleNames[m] = new String[alleleCounts[m]];
                        for(int a = 0; a < alleleNames[m].length; a++){
                            alleleNames[m][a] = row[aglob];
                            aglob++;
                        }
                    }
                } else {

                    // process data row

                    // extract unique item identifier
                    itemIdentifiers.add(row[0]);
                    // extract item name, if included
                    if(withNames){
                        itemNames.add(row[1]);
                    }

                    // group frequencies per marker
                    double[][] freqsPerMarker = new double[numMarkers][];
                    int fglob = numHeaderCols;
                    for (int m = 0; m < numMarkers; m++) {
                        freqsPerMarker[m] = new double[alleleCounts[m]];
                        for (int f = 0; f < freqsPerMarker[m].length; f++) {
                            double freq;
                            try {
                                freq = (row[fglob] == null ? Double.NaN : Double.parseDouble(row[fglob]));
                            } catch (NumberFormatException ex) {
                                // wrap in IO exception
                                throw new IOException(String.format(
                                        "Invalid frequency at row %d, column %d. Expected double value, got: \"%s\".",
                                        r, fglob, row[fglob]
                                    ), ex
                                );
                            }
                            freqsPerMarker[m][f] = freq;
                            fglob++;
                        }
                    }
                    // store frequencies
                    alleleFreqs.add(freqsPerMarker);

                }

                // next row
                r++;

            }
            int n = alleleFreqs.size();
            if (n == 0) {
                throw new IOException("No data rows.");
            }

            // combine names and identifiers in headers
            SimpleEntity[] headers = new SimpleEntity[n];
            for (int i = 0; i < n; i++) {
                String identifier = itemIdentifiers.get(i);
                String name = withNames ? itemNames.get(i) : itemIdentifiers.get(i);

                if (name != null) {
                    headers[i] = new SimpleEntityPojo(identifier, name);
                } else {
                    headers[i] = new SimpleEntityPojo(identifier);
                }
            }

            // convert collections to arrays
            double[][][] alleleFreqsArray = alleleFreqs.stream().toArray(k -> new double[k][][]);

            try {
                // create data
                return new SimpleFrequencyGenotypeData(
                        filePath.getFileName().toString(), headers, markerNames,
                        alleleNames, alleleFreqsArray
                );
            } catch (IllegalArgumentException ex) {
                // convert to IO exception
                throw new IOException(ex.getMessage());
            }

        }
    }

    /**
     * Infer marker names and number of columns per marker from column names.
     * Any suffix after the last dot, dash or underscore character is removed
     * from the column name to obtain the marker name.
     * 
     * @param columnNames column names
     * @return Linked hash map that maps the inferred marker names on the number
     *         of columns for that marker. The order of entries in the map
     *         corresponds to the original column order and is retained due to
     *         the behavior of a linked hash map.
     */
    public static LinkedHashMap<String, Integer> inferMarkerNames(String[] columnNames) {
        if (columnNames == null) {
            throw new IllegalArgumentException("Column names undefined.");
        }
        LinkedHashMap<String, Integer> markerNames = new LinkedHashMap<>();
        String curName = null;
        for (int c = 0; c < columnNames.length; c++) {
            String columnName = columnNames[c];

            if(columnName == null){
                throw new IllegalArgumentException("Missing column name for data column " + c + ".");
            }
            String markerName = inferMarkerName(columnName);
            if(markerName.equals("")){
                throw new IllegalArgumentException(String.format(
                        "Invalid marker name at data column %d (%s).", c, columnName
                ));

            }
            if (curName == null || !markerName.equals(curName)) {
                // first column for new marker
                if (markerNames.containsKey(markerName)) {
                    throw new IllegalArgumentException("Duplicate marker name: " + markerName + ". "
                        + "Columns corresponding to same marker should occur consecutively.");
                }
                markerNames.put(markerName, 1);
                curName = markerName;
            } else {
                // additional column for current marker
                markerNames.put(markerName, markerNames.get(markerName) + 1);
            }
        }
        return markerNames;
    }
    
    private static String inferMarkerName(String columnName) {
        int i = Stream.of('-', '_', '.').mapToInt(suf -> columnName.lastIndexOf(suf)).max().orElse(-1);
        String markerName = (i >= 0 ? columnName.substring(0, i) : columnName);
        return markerName;
    }

    @Override
    public void writeData(Path filePath, FileType fileType, SubsetSolution solution,
                          boolean includeSelected, boolean includeUnselected, boolean includeIndex)
                          throws IOException {

        // validate arguments
        if (filePath == null) {
            throw new IllegalArgumentException("File path not defined.");
        }

        if (filePath.toFile().exists()) {
            throw new IOException("File already exists: " + filePath + ".");
        }

        if (fileType == null) {
            throw new IllegalArgumentException("File type not defined.");
        }

        if (fileType != FileType.TXT && fileType != FileType.CSV) {
            throw new IllegalArgumentException(
                String.format("Only file types TXT and CSV are supported. Got: %s.", fileType)
            );
        }

        if (solution == null) {
            throw new NullPointerException("Solution must be defined");
        }

        if (!(solution.getAllIDs().equals(getIDs()))) {
            throw new IllegalArgumentException("Solution ids must match data.");
        }
        
        if(!includeSelected && !includeUnselected){
            throw new IllegalArgumentException(
                    "At least one of 'includeSelected' or 'includeUnselected' must be used."
            );
        }

        Files.createDirectories(filePath.getParent());

        // write data to file
        boolean markSelection = includeSelected && includeUnselected;
        try (RowWriter writer = IOUtilities.createRowWriter(
                filePath, fileType, TextFileRowWriter.ADD_QUOTES
        )) {

            if (writer == null || !writer.ready()) {
                throw new IOException("Can not create writer for file " + filePath + ".");
            }

            // write internal integer id column header
            if (includeIndex) {
                writer.writeCell(ID_HEADER);
                writer.newColumn();
            }
            
            // write string id and name column headers
            writer.writeCell(IDENTIFIERS_HEADER);
            writer.newColumn();
            writer.writeCell(NAMES_HEADER);
            
            // write selection column header if both selected and unselected are included
            if (markSelection) {
                writer.newColumn();
                writer.writeCell(SELECTED_HEADER);
            }

            // write marker names (column headers)
            for (int i = 0; i < markerNames.length; ++i) {
                for (int j = 0; j < alleleNames[i].length; ++j) {
                    writer.newColumn();
                    writer.writeCell(markerNames[i]);
                }
            }

            writer.newRow();

            // skip integer id column if included
            if (includeIndex) {
                writer.newColumn();
            }

            // write allele names row header
            writer.writeCell(ALLELE_NAMES_HEADER);

            writer.newColumn();
            
            // skip selection column if included
            if (markSelection) {
                writer.newColumn();
            }

            // write allele names
            for (int i = 0; i < alleleNames.length; ++i) {
                writer.newColumn();
                writer.writeRowCellsAsArray(alleleNames[i]);
            }

            // obtain sorted list of IDs included in output
            Set<Integer> includedIDs;
            if (markSelection) {
                includedIDs = getIDs();
            } else if (includeSelected) {
                includedIDs = solution.getSelectedIDs();
            } else if (includeUnselected) {
                includedIDs = solution.getUnselectedIDs();
            } else {
                throw new IllegalArgumentException(
                        "One of 'includeSelected' or 'includeUnselected' must be used."
                );
            }
            List<Integer> sortedIDs = new ArrayList<>(includedIDs);
            sortedIDs.sort(null);

            // write data rows
            Set<Integer> selected = solution.getSelectedIDs();
            for (int id : sortedIDs) {

                writer.newRow();

                // write integer ID if included
                if (includeIndex) {
                    writer.writeCell(id);
                    writer.newColumn();
                }

                // write string id and name
                SimpleEntity header = getHeader(id);
                writer.writeCell(header.getUniqueIdentifier());
                writer.newColumn();
                writer.writeCell(header.getName());

                // mark selection if needed
                if(markSelection){
                    writer.newColumn();
                    writer.writeCell(selected.contains(id));
                }
                
                // write allele frequencies
                for (int m = 0; m < alleleFrequencies[id].length; m++) {
                    for(int a = 0; a < alleleFrequencies[id][m].length; a++){
                        writer.newColumn();
                        double freq = alleleFrequencies[id][m][a];
                        writer.writeCell(Double.isNaN(freq) ? null : freq);
                    }
                }

            }

            writer.close();
        }

    }

} public enum MissingValuesPolicy {
    /**
     * Sets the contribution of variables (markers/features)
     * with missing values to zero.
     */
    FLOOR,
    /**
     * Sets the contribution of variables (markers/feature)
     * with missing values to the maximum attainable contribution.
     */
    CEIL;
} public class ModifiedRogersDistance extends AbstractDistanceMeasure {
    
    @Override
    public double computeDistance(int idX, int idY, CoreHunterData data) {
        
        if(idX == idY){
            return 0.0;
        }
        
        FrequencyGenotypeData genotypes = data.getGenotypicData();
        
        if(genotypes == null){
            throw new CoreHunterException("Genotypes are required for Modified Rogers distance.");
        }
        
        int numberOfMarkers = genotypes.getNumberOfMarkers();
        double sumSquareDiff = 0.0;
        
        for (int markerIndex = 0; markerIndex < numberOfMarkers; ++markerIndex) {
            
            if(genotypes.hasMissingValues(idX, markerIndex) || genotypes.hasMissingValues(idY, markerIndex)){
                // missing frequencies in at least one individual
                sumSquareDiff += missingValueContribution(2.0);
            } else {
                // frequencies available for both individuals
                int numberOfAlleles = genotypes.getNumberOfAlleles(markerIndex);
                for (int alleleIndex = 0; alleleIndex < numberOfAlleles; ++alleleIndex) {
                    double pxla = genotypes.getAlleleFrequency(idX, markerIndex, alleleIndex);
                    double pyla = genotypes.getAlleleFrequency(idY, markerIndex, alleleIndex);
                    sumSquareDiff += (pxla - pyla) * (pxla - pyla);
                }
            }
            
        }

        double distance = Math.sqrt(sumSquareDiff / (2*numberOfMarkers));

        return distance;

    }
    
    @Override
    public String toString(){
        return "Modified Rogers";
    }
    
}
import org.corehunter.data.CoreHunterData;
import org.corehunter.data.simple.SimpleFrequencyGenotypeData;
import org.corehunter.objectives.distance.measures.MissingValuesPolicy;
import org.corehunter.objectives.distance.measures.ModifiedRogersDistance;
import org.junit.Test;
import static org.corehunter.tests.TestData.ALLELE_FREQUENCIES;
import static org.corehunter.tests.TestData.ALLELE_NAMES;
import static org.corehunter.tests.TestData.HEADERS_NON_UNIQUE_NAMES;
import static org.corehunter.tests.TestData.MARKER_NAMES;
import static org.corehunter.tests.TestData.MODIFIED_ROGERS_DISTANCES;
import static org.corehunter.tests.TestData.MODIFIED_ROGERS_DISTANCES_CEIL_MISSING;
import static org.corehunter.tests.TestData.NAME;
import static org.corehunter.tests.TestData.PRECISION;
import static org.junit.Assert.assertEquals;
import java.util.Iterator;
/*--------------------------------------------------------------*/
/* Licensed to the Apache Software Foundation (ASF) under one   */
/* or more contributor license agreements.  See the NOTICE file */
/* distributed with this work for additional information        */
/* regarding copyright ownership.  The ASF licenses this file   */
/* to you under the Apache License, Version 2.0 (the            */
/* "License"); you may not use this file except in compliance   */
/* with the License.  You may obtain a copy of the License at   */
/*                                                              */
/*   http://www.apache.org/licenses/LICENSE-2.0                 */
/*                                                              */
/* Unless required by applicable law or agreed to in writing,   */
/* software distributed under the License is distributed on an  */
/* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       */
/* KIND, either express or implied.  See the License for the    */
/* specific language governing permissions and limitations      */
/* under the License.                                           */
/*--------------------------------------------------------------*/

package org.corehunter.tests.objectives.distance.measures;




/**
 * @author Guy Davenport, Herman De Beukelaer
 */
public class ModifiedRogersDistanceTest {

    @Test
    public void test() {
        
        SimpleFrequencyGenotypeData geno = new SimpleFrequencyGenotypeData(
                NAME, HEADERS_NON_UNIQUE_NAMES, MARKER_NAMES, ALLELE_NAMES, ALLELE_FREQUENCIES
        );
        CoreHunterData data = new CoreHunterData(geno);