public class BioNetDBConfiguration {

    private String logLevel;
    private String logFile;

    private DatabaseConfiguration database;
    private DownloadProperties download;

    protected static Logger logger = LoggerFactory.getLogger(BioNetDBConfiguration.class);

    public BioNetDBConfiguration() { }

    public BioNetDBConfiguration(DatabaseConfiguration databaseConfiguration) {
        database = databaseConfiguration;

    }

    /*
     * This method attempts to find and load the configuration from installation directory,
     * if not exists then loads JAR storage-configuration.yml.
     */
    @Deprecated
    public static BioNetDBConfiguration load() throws IOException {
        String appHome = System.getProperty("app.home", System.getenv("BIONETDB_HOME"));
        Path path = Paths.get(appHome + "/configuration.yml");
        if (appHome != null && Files.exists(path)) {
            logger.debug("Loading configuration from '{}'", appHome + "/configuration.yml");
            return BioNetDBConfiguration
                    .load(new FileInputStream(new File(appHome + "/configuration.yml")));
        } else {
            logger.debug("Loading configuration from '{}'",
                    BioNetDBConfiguration.class.getClassLoader()
                            .getResourceAsStream("configuration.yml")
                            .toString());
            return BioNetDBConfiguration
                    .load(BioNetDBConfiguration.class.getClassLoader().getResourceAsStream("configuration.yml"));
        }
    }

    public static BioNetDBConfiguration load(InputStream configurationInputStream) throws IOException {
        return load(configurationInputStream, "yaml");
    }

    public static BioNetDBConfiguration load(InputStream configurationInputStream, String format) throws IOException {
        BioNetDBConfiguration bioNetDBConfiguration;
        ObjectMapper objectMapper;
        switch (format) {
            case "json":
                objectMapper = new ObjectMapper();
                bioNetDBConfiguration = objectMapper.readValue(configurationInputStream, BioNetDBConfiguration.class);
                break;
            case "yml":
            case "yaml":
            default:
                objectMapper = new ObjectMapper(new YAMLFactory());
                bioNetDBConfiguration = objectMapper.readValue(configurationInputStream, BioNetDBConfiguration.class);
                break;
        }

        return bioNetDBConfiguration;
    }

    public void serialize(OutputStream configurationOututStream) throws IOException {
        ObjectMapper jsonMapper = new ObjectMapper(new YAMLFactory());
        jsonMapper.writerWithDefaultPrettyPrinter().writeValue(configurationOututStream, this);
    }

//    public DatabaseConfiguration findDatabase(String database) {
//        DatabaseConfiguration databaseConfiguration = null;
//        if (StringUtils.isNotEmpty(database)) {
//            for (DatabaseConfiguration databaseConf : databases) {
//                if (databaseConf.getId().equals(database)) {
//                    databaseConfiguration = databaseConf;
//                    break;
//                }
//            }
//        } else {
//            databaseConfiguration = this.databases.get(0);
//        }
//
//        return databaseConfiguration;
//    }
//
//    public void addDatabase(InputStream configurationInputStream) throws IOException {
//        addDatabase(configurationInputStream, "yaml");
//    }
//
//    public void addDatabase(InputStream configurationInputStream, String format) throws IOException {
//        DatabaseConfiguration databaseConfiguration;
//        ObjectMapper objectMapper;
//        switch (format) {
//            case "json":
//                objectMapper = new ObjectMapper();
//                databaseConfiguration = objectMapper.readValue(configurationInputStream, DatabaseConfiguration.class);
//                break;
//            case "yml":
//            case "yaml":
//            default:
//                objectMapper = new ObjectMapper(new YAMLFactory());
//                databaseConfiguration = objectMapper.readValue(configurationInputStream, DatabaseConfiguration.class);
//                break;
//        }
//
//        if (databaseConfiguration != null) {
//            this.databases.add(databaseConfiguration);
//        }
//    }

    @Override
    public String toString() {
        final StringBuilder sb = new StringBuilder("BioNetDBConfiguration{");
        sb.append("logLevel='").append(logLevel).append('\'');
        sb.append(", logFile='").append(logFile).append('\'');
        sb.append(", database=").append(database);
        sb.append(", download=").append(download);
        sb.append('}');
        return sb.toString();
    }

    public String getLogLevel() {
        return logLevel;
    }

    public BioNetDBConfiguration setLogLevel(String logLevel) {
        this.logLevel = logLevel;
        return this;
    }

    public String getLogFile() {
        return logFile;
    }

    public BioNetDBConfiguration setLogFile(String logFile) {
        this.logFile = logFile;
        return this;
    }

    public DatabaseConfiguration getDatabase() {
        return database;
    }

    public BioNetDBConfiguration setDatabase(DatabaseConfiguration database) {
        this.database = database;
        return this;
    }

    public DownloadProperties getDownload() {
        return download;
    }

    public BioNetDBConfiguration setDownload(DownloadProperties download) {
        this.download = download;
        return this;
    }

    public static Logger getLogger() {
        return logger;
    }

    public static void setLogger(Logger logger) {
        BioNetDBConfiguration.logger = logger;
    }
} public class BioNetDBException extends Exception {

    public BioNetDBException(String message) {
        super(message);
    }

    public BioNetDBException(String message, Throwable cause) {
        super(message, cause);
    }

    public BioNetDBException(Throwable cause) {
        super(cause);
    }

} public class NetworkPath extends Graph {
    private int startIndex;
    private int endIndex;

    public NetworkPath() {
        super();
        startIndex = 0;
        endIndex = 0;
    }

    @Override
    public String toString() {
        final StringBuilder sb = new StringBuilder("NetworkPath{");
        sb.append("startIndex=").append(startIndex);
        sb.append(", endIndex=").append(endIndex);
        sb.append(", nodes=").append(nodes);
        sb.append(", relations=").append(relations);
        sb.append('}');
        return sb.toString();
    }

    public int getStartIndex() {
        return startIndex;
    }

    public NetworkPath setStartIndex(int startIndex) {
        this.startIndex = startIndex;
        return this;
    }

    public int getEndIndex() {
        return endIndex;
    }

    public NetworkPath setEndIndex(int endIndex) {
        this.endIndex = endIndex;
        return this;
    }
} public class InterpretationAnalysis extends BioNetDBAnalysis {

    public InterpretationAnalysis(NetworkDBAdaptor networkDBAdaptor) {
        super(networkDBAdaptor);
    }

    public DataResult<Variant> proteinNetworkAnalysis(Pedigree pedigree, Disorder disorder, ClinicalProperty.ModeOfInheritance moi,
                                                      boolean complexOrReaction, Query query) throws BioNetDBException {
        ProteinNetworkInterpretationAnalysis analysis = new ProteinNetworkInterpretationAnalysis(networkDBAdaptor);
        return analysis.execute(pedigree, disorder, moi, complexOrReaction, query);
    }
} public class NetworkAnalysis extends BioNetDBAnalysis {

    public NetworkAnalysis(NetworkDBAdaptor networkDBAdaptor) {
        super(networkDBAdaptor);
    }
} public class VariantAnalysis extends BioNetDBAnalysis {

    public VariantAnalysis(NetworkDBAdaptor networkDBAdaptor) {
        super(networkDBAdaptor);
    }

    public DataResult<Variant> getDominantVariants(Pedigree pedigree, Disorder disorder, Query query)
            throws BioNetDBException, IOException {
        Map<String, List<String>> genotypes = ModeOfInheritance.dominant(pedigree, disorder, COMPLETE);
        putGenotypes(query, genotypes);

        return null;
//        return networkDBAdaptor.variantQuery(query, QueryOptions.empty());
    }

    public DataResult<Variant> getRecessiveVariants(Pedigree pedigree, Disorder disorder, Query query)
            throws BioNetDBException {
        Map<String, List<String>> genotypes = ModeOfInheritance.recessive(pedigree, disorder, COMPLETE);
        putGenotypes(query, genotypes);

        return null;
//        return networkDBAdaptor.variantQuery(query, QueryOptions.empty());
    }

    public DataResult<Variant> getXLinkedDominantVariants(Pedigree pedigree, Disorder disorder, Query query)
            throws BioNetDBException {
        Map<String, List<String>> genotypes = ModeOfInheritance.xLinked(pedigree, disorder, true, COMPLETE);
        query.put(VariantQueryParam.CHROMOSOME.key(), "X");
        putGenotypes(query, genotypes);

        return null;
//        return networkDBAdaptor.variantQuery(query, QueryOptions.empty());
    }

    public DataResult<Variant> getXLinkedRecessiveVariants(Pedigree pedigree, Disorder disorder, Query query)
            throws BioNetDBException {
        Map<String, List<String>> genotypes = ModeOfInheritance.xLinked(pedigree, disorder, false, COMPLETE);
        query.put(VariantQueryParam.CHROMOSOME.key(), "X");
        putGenotypes(query, genotypes);

        return null;
//        return networkDBAdaptor.variantQuery(query, QueryOptions.empty());
    }

    public DataResult<Variant> getYLinkedVariants(Pedigree pedigree, Disorder disorder, Query query) throws BioNetDBException {
        Map<String, List<String>> genotypes = ModeOfInheritance.yLinked(pedigree, disorder, COMPLETE);
        query.put(VariantQueryParam.CHROMOSOME.key(), "Y");
        putGenotypes(query, genotypes);

        return null;
//        return networkDBAdaptor.variantQuery(query, QueryOptions.empty());
    }

    public DataResult<Variant> getDeNovoVariants(Pedigree pedigree, Query query) throws BioNetDBException, IOException {
        Map<String, List<String>> genotypes = ModeOfInheritance.deNovo(pedigree);
        putGenotypes(query, genotypes);
        query.put(VariantQueryParam.INCLUDE_GENOTYPE.key(), true);

        // Get variants and genotypes
        return null;
//        DataResult<Variant> variantQueryResult = networkDBAdaptor.variantQuery(query, QueryOptions.empty());
//        List<Variant> variants = variantQueryResult.getResults();
//
//        int dbTime = 0;
//        if (CollectionUtils.isEmpty(variants)) {
//            return new DataResult<>(dbTime, new ArrayList<>(), 0, Collections.emptyList(), 0);
//        } else {
//            List<String> sampleNames = Arrays.asList(variants.get(0).getAnnotation().getAdditionalAttributes()
//                    .get("samples").getAttribute().get(NodeBuilder.SAMPLE).split(","));
//
//            // Filter deNovo variants
//            int probandSampleIdx = sampleNames.indexOf(pedigree.getProband().getId());
//            int motherSampleIdx = sampleNames.indexOf(pedigree.getProband().getMother().getId());
//            int fatherSampleIdx = sampleNames.indexOf(pedigree.getProband().getFather().getId());
//            List<Variant> deNovoVariants = ModeOfInheritance.deNovo(variants.iterator(), probandSampleIdx, motherSampleIdx,
//                    fatherSampleIdx);
//
//            return new DataResult<>(dbTime, Collections.emptyList(), deNovoVariants.size(), deNovoVariants, deNovoVariants.size());
//        }
    }

    public DataResult<Map<String, List<Variant>>> getCompoundHeterozygousVariants(Pedigree pedigree, Query query)
            throws BioNetDBException {
        Map<String, List<String>> genotypes = ModeOfInheritance.compoundHeterozygous(pedigree);
        putGenotypes(query, genotypes);

        query.put(VariantQueryParam.INCLUDE_GENOTYPE.key(), true);
        query.put(VariantQueryParam.INCLUDE_CONSEQUENCE_TYPE.key(), true);

        // Get variants and genotypes
        return null;
//        DataResult<Variant> variantQueryResult = networkDBAdaptor.variantQuery(query, QueryOptions.empty());
//        List<Variant> variants = variantQueryResult.getResults();
//
//        int dbTime = 0;
//        if (CollectionUtils.isEmpty(variants)) {
//            return new DataResult<>(dbTime, Collections.emptyList(), 0, Collections.emptyList(), 0);
//        } else {
//
//            List<String> sampleNames = Arrays.asList(variants.get(0).getAnnotation().getAdditionalAttributes().get("samples")
// .getAttribute()
//                    .get(NodeBuilder.SAMPLE).split(","));
//
//            // Filter deNovo variants
//            int probandSampleIdx = sampleNames.indexOf(pedigree.getProband().getId());
//            int motherSampleIdx = sampleNames.indexOf(pedigree.getProband().getMother().getId());
//            int fatherSampleIdx = sampleNames.indexOf(pedigree.getProband().getFather().getId());
//            Map<String, List<Variant>> chVariants = ModeOfInheritance.compoundHeterozygous(variants.iterator(), probandSampleIdx,
//                    motherSampleIdx, fatherSampleIdx);
//
//            return new DataResult<>(dbTime, Collections.emptyList(), chVariants.size(), Collections.singletonList(chVariants),
//                    chVariants.size());
//        }
    }

    //------------------------------------------------------------------
    //  P R I V A T E      M E T H O D S
    //------------------------------------------------------------------

    private void putGenotypes(Query query, Map<String, List<String>> genotypes) {
        // genotype format: sample:0/1,1/1,....
        List<String> gt = new ArrayList<>();
        for (String sample : genotypes.keySet()) {
            gt.add(sample + ":" + StringUtils.join(genotypes.get(sample), ","));
        }
        query.put(VariantQueryParam.GENOTYPE.key(), gt);
    }

    private List<Variant> queryNodes(String cypher) throws BioNetDBException {
        List<Variant> nodes = new ArrayList<>();

        NodeIterator nodeIterator = networkDBAdaptor.nodeIterator(cypher);
        while (nodeIterator.hasNext()) {
            nodes.add(NodeBuilder.newVariant(nodeIterator.next()));
        }

        return nodes;
    }
} public class TieringInterpretationAnalysis {

    private Driver driver;

    public TieringInterpretationAnalysis(Driver driver) {
        this.driver = driver;
    }

    /**
     * This method looks for all the variants of interest for a given pedigree. It should be aimed to a gene panel, and may be filtered
     * by chromosome too.
     *
     * @param listOfGenes      list of strings with the genes we want to check (may be null but shouldn't).
     * @param listOfChromosome list of strings with the genes we want to check (may be null).
     * @param individualsGT    list of strings with the genes we want to check (can't be null).
     * @return a QueryResult object containing the variants matching the specifications.
     */
    public QueryResult<Variant> getVariantsFromPedigree(List<String> listOfGenes, List<String> listOfChromosome,
                                                        Map<String, List<String>> individualsGT) {

        String queryString;
        List<String> matches = new ArrayList<>();
        List<String> individualsID = new ArrayList<>(individualsGT.keySet());
        String genesSubstring = getGenericSubstring(listOfGenes, "ref.id", true);
        String chromosomeSubstring = getGenericSubstring(listOfChromosome, "var.attr_chromosome", true);

        for (String individual : individualsID) {
            String genotypeSubstring = getGenericSubstring(individualsGT.get(individual), "vc.attr_GT", true);
            queryString = "MATCH (sam:SAMPLE)-[:SAMPLE__VARIANT_CALL]-(vc:VARIANT_SAMPLE_FORMAT)-[:VARIANT__VARIANT_CALL]-(var:VARIANT)"
                    + "-[:ANNOTATION___VARIANT___VARIANT_CONSEQUENCE_TYPE]-(:VARIANT_CONSEQUENCE_TYPE)-"
                    + "[:ANNOTATION___VARIANT_CONSEQUENCE_TYPE___TRANSCRIPT]-(:TRANSCRIPT)-"
                    + "[:HAS___GENE___TRANSCRIPT]-(:GENE)-[:XREF]-(ref:XREF)"
                    + " WHERE " + genesSubstring + chromosomeSubstring + genotypeSubstring + "sam.id='" + individual + "'";
            matches.add(queryString);
        }
        queryString = StringUtils.join(matches, " WITH DISTINCT var\n");
//        queryString = queryString + " RETURN DISTINCT var.name";


        queryString = queryString + " RETURN DISTINCT var.attr_chromosome AS " + NodeBuilder.CHROMOSOME
                + ", var.attr_start AS " + NodeBuilder.START
                + ", var.attr_end AS " + NodeBuilder.END
                + ", var.attr_reference AS " + NodeBuilder.REFERENCE
                + ", var.attr_alternate AS " + NodeBuilder.ALTERNATE
                + ", var.attr_type AS " + NodeBuilder.TYPE;
        System.out.println("queryString = " + queryString + "\n");

        return executeQuery(queryString);
    }

//  DE AQUÍ PARRIBA SON PA DOMINANT, RECESSIVE Y LINKED ///////////////////////////////////////////////////////
//  ///////////////////////////////////////////////////////////////////// DE AQUÍ PABAJO SON PAL CH Y EL DENOVO

    /**
     * This method looks for all the variants pertaining to a set of individuals. It should be aimed to a gene panel, and may be filtered
     * by chromosome too.
     *
     * @param listOfGenes      list of strings with the genes we want to check (may be null but shouldn't).
     * @param listOfChromosome list of strings with the genes we want to check (may be null).
     * @param listOfMembers    list of strings with the genes we want to check (can't be null).
     * @return a StatementResult object containing the variants matching the specifications.
     */
    public Neo4JVariantIterator variantsToIterator(List<String> listOfGenes, List<String> listOfChromosome,
                                                   List<Member> listOfMembers) {

        // It's likely that this action could be done more efficiently.
        List<String> stringListOfIndividuals = new LinkedList<>();
        for (Member member : listOfMembers) {
            stringListOfIndividuals.add(member.getId());
        }

        // Maybe we could change the filtering by chromosome to the genes instead of the variants. It's more efficient.
        // For that we should modify the method "getGenericSubstring". También habría que apañar la entrada por paneles o PH.
        String individualsSubstring = getGenericSubstring(stringListOfIndividuals, "sam.id", false);
        String genesSubstring = getGenericSubstring(listOfGenes, "ref.id", true);
        String chromosomeSubstring = getGenericSubstring(listOfChromosome, "var.attr_chromosome", true);
        String matchString = "MATCH (sam:SAMPLE)-[:SAMPLE__VARIANT_CALL]-(vc:VARIANT_SAMPLE_FORMAT)-[:VARIANT__VARIANT_CALL]-(var:VARIANT)"
                + "-[:ANNOTATION___VARIANT___VARIANT_CONSEQUENCE_TYPE]-(:VARIANT_CONSEQUENCE_TYPE)-"
                + "[:ANNOTATION___VARIANT_CONSEQUENCE_TYPE___TRANSCRIPT]-(:TRANSCRIPT)-"
                + "[:HAS___GENE___TRANSCRIPT]-(gene:GENE)-[:XREF]-(ref:XREF)"
                + " WHERE " + genesSubstring + chromosomeSubstring + individualsSubstring
                + " WITH var, collect(DISTINCT sam.id) AS sam_collection, collect(vc.attr_GT) AS gt_collection,"
                + " COUNT (DISTINCT sam.id) AS num_of_sam\n";
        String returnString = " RETURN DISTINCT var.attr_chromosome AS " + NodeBuilder.CHROMOSOME
                + ", var.attr_start AS " + NodeBuilder.START
                + ", var.attr_end AS " + NodeBuilder.END
                + ", var.attr_reference AS " + NodeBuilder.REFERENCE
                + ", var.attr_alternate AS " + NodeBuilder.ALTERNATE
                + ", var.attr_type AS " + NodeBuilder.TYPE
                + ", sam_collection, gt_collection[0..num_of_sam] AS gt_collection, num_of_sam";
        String queryString = matchString + returnString;
        System.out.println("queryString = " + queryString + "\n");

        Session session = this.driver.session();
        Result result = session.run(queryString);
        session.close();
        return new Neo4JVariantIterator(result);
    }

    /**
     * This method gets a list of variants from the database. It is only useful if we want different information than the one
     * given by the list we use as argument, otherwise neo4j result will be redundant
     *
     * @param listOfVariants the list of variants we want to get from neo4j database
     * @return a QueryResult object containing the variants in the list.
     */
    public QueryResult<Variant> getVariantsFromList(List<Variant> listOfVariants) {
        String startingString = "MATCH (var:VARIANT) WHERE (var.name='";
        String variantsString = StringUtils.join(listOfVariants, "' OR var.name='") + "')";
        String endingString = " RETURN DISTINCT var.attr_chromosome AS " + NodeBuilder.CHROMOSOME
                + ", var.attr_start AS " + NodeBuilder.START
                + ", var.attr_end AS " + NodeBuilder.END
                + ", var.attr_reference AS " + NodeBuilder.REFERENCE
                + ", var.attr_alternate AS " + NodeBuilder.ALTERNATE
                + ", var.attr_type AS " + NodeBuilder.TYPE;
        String queryString = startingString + variantsString + endingString;
        return executeQuery(queryString);
    }

///////////////////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////// THIS METHODS ARE AUXILIAR FOR THE ONES ABOVE ////////////////////////////////////

    /**
     * Builds the part of the cypher query aimed to act as a searching filter. We can fiter by the individual samples, their
     * genotype, the chromosome or the genes in which we want to look up.
     * <p>
     * [Mainly used for methods "getVariantsFromPedigree" and "getVariantsFromList"]
     *
     * @param stringList The list of elements that will compound the filter
     * @param calling    The index we want to use to call if from the database
     * @param isNotLast  A boolean that adds an "AND" operator at the end of the substring if needed
     * @return the substring with the filter ready to use for Neo4j
     */
    private static String getGenericSubstring(List<String> stringList, String calling, boolean isNotLast) {
        String substring = "";
        if (stringList.size() == 0) {
            return substring;
        } else {
            List<String> elements = new ArrayList<>();
            for (String element : stringList) {
                elements.add(substring + calling + "='" + element + "'");
            }
            substring = StringUtils.join(elements, " OR ");
            substring = "(" + substring + ")";
            if (isNotLast) {
                substring = substring + " AND ";
            }
            return substring;
        }
    }

    /**
     * This method calls Neo4j driver, executes the query and returns a list of variants inside a queryObject.
     * <p>
     * [Mainly used for methods "getVariantsFromPedigree" and "variantsToIterator"]
     *
     * @param queryString The cypher query we wish to execute in Neo4j database
     * @return QueryOption object containing a List of variants as a result of the query
     */
    private QueryResult<Variant> executeQuery(String queryString) {
        Session session = this.driver.session();
        Result result = session.run(queryString);
        session.close();
        List<Variant> variants = new ArrayList<>();
        while (result.hasNext()) {
            variants.add(new Neo4JVariantIterator(result).next());
        }
        QueryResult<Variant> variantsResult = new QueryResult<>("variants");
        variantsResult.setResult(variants);
        return variantsResult;
    }
} public interface NetworkDBAdaptor extends AutoCloseable {

    enum NetworkQueryParams implements QueryParam {
        NODE_TYPE("node.type", TEXT_ARRAY, ""),   // This is PHYSICAL_ENTITY, INTERACTION, XREF, ...
        NODE_UID("node.uid", STRING, ""),
        NODE_ID("node.id", TEXT_ARRAY, ""),

        SRC_NODE("src-node", TEXT_ARRAY, ""),
        DEST_NODE("dest-node", TEXT_ARRAY, ""),
        INTERM_NODE("interm-node", TEXT_ARRAY, ""),
        MAX_JUMPS("max-jumps", INTEGER, ""),
        OUTPUT("output", STRING, ""),

        PE_ID ("pe.id", TEXT_ARRAY, ""),
        PE_DESCRIPTION ("pe.description", TEXT_ARRAY, ""),
        REL_TYPE("rel.type", TEXT_ARRAY, ""),
        PE_ATTR_EXPR ("pe.attr.expr", TEXT_ARRAY, ""),  // example: "brain:t2>0.3;brain:t4<=0.3"
        PE_ONTOLOGY ("pe.ontology", TEXT_ARRAY, ""),  // example: "go:001234, go:002345"
        PE_CELLOCATION ("pe.cellularLocation", TEXT_ARRAY, ""), // example: "nucleoplasm,..."
        JUMPS("jumps", INTEGER, ""),
        SCRIPT ("script",  STRING, ""),

        ID("id", TEXT_ARRAY, ""),
        LABEL("label", TEXT_ARRAY, "");

        NetworkQueryParams(String key, Type type, String description) {
            this.key = key;
            this.type = type;
            this.description = description;
        }

        private final String key;
        private Type type;
        private String description;

        @Override public String key() {
            return key;
        }

        @Override public String description() {
            return description;
        }

        @Override public Type type() {
            return type;
        }
    }

    //-------------------------------------------------------------------------
    // I N D E X
    //-------------------------------------------------------------------------

    void index();
    void close();

    //-------------------------------------------------------------------------
    // I N S E R T     N E T W O R K S
    //-------------------------------------------------------------------------

    void insert(Network network, QueryOptions queryOptions) throws BioNetDBException;

    //-------------------------------------------------------------------------
    // A N N O T A T I O N     M E T H O D s
    //-------------------------------------------------------------------------

//    void annotateVariants(NodeQuery query, QueryOptions options, VariantClient variantClient) throws BioNetDBException, IOException;
//    void annotateVariants(List<String> variantIds, VariantClient variantClient) throws BioNetDBException, IOException;
//
//    void annotateGenes(NodeQuery query, QueryOptions options, GeneClient geneClient) throws BioNetDBException, IOException;
//    void annotateGenes(List<String> geneIds, GeneClient geneClient) throws BioNetDBException, IOException;
//
//    void annotateProteins(NodeQuery query, QueryOptions options, ProteinClient proteinClient) throws BioNetDBException, IOException;
//    void annotateProteins(List<String> proteinIds, ProteinClient proteinClient) throws BioNetDBException, IOException;

    //========================================================================
    // N E T W O R K
    //========================================================================

    //-------------------------------------------------------------------------
    // N O D E S
    //-------------------------------------------------------------------------

    NodeIterator nodeIterator(Query query, QueryOptions queryOptions);
    NodeIterator nodeIterator(String cypher);

    BioNetDBResult<Node> nodeQuery(Query query, QueryOptions queryOptions);
    BioNetDBResult<Node> nodeQuery(String cypher);

    BioNetDBResult<NodeStats> nodeStats(Query query);

    //-------------------------------------------------------------------------

    long addNode(Node node) throws BioNetDBException;
    void updateNode(Node node) throws BioNetDBException;
    void deleteNode(Node node) throws BioNetDBException;
    boolean existNode(Node node) throws BioNetDBException;

    void addRelation(Node origNode, Node destNode, Relation relation) throws BioNetDBException;
    void updateRelation(Node origNode, Node destNode, Relation relation) throws BioNetDBException;
    void deleteRelation(Node origNode, Node destNode, Relation relation) throws BioNetDBException;
    boolean existRelation(Node origNode, Node destNode, Relation relation) throws BioNetDBException;

    //-------------------------------------------------------------------------
    // T A B L E S / R O W S
    //   - a table is modeled as a list of rows, and
    //   - a row is modeled as a list of Object
    //-------------------------------------------------------------------------

//    RowIterator rowIterator(Query query, QueryOptions queryOptions) throws BioNetDBException;
//    RowIterator rowIterator(String cypher) throws BioNetDBException;
//
//    DataResult<List<Object>> rowQuery(Query query, QueryOptions queryOptions) throws BioNetDBException;
//    DataResult<List<Object>> rowQuery(String cypher) throws BioNetDBException;


    //-------------------------------------------------------------------------
    // N E T W O R K     P A T H S
    //   - a network path is modeled as a 'lineal' network, it contains
    //     the origin node and the destionation node
    //-------------------------------------------------------------------------

    NetworkPathIterator networkPathIterator(Query networkPathQuery, QueryOptions queryOptions) throws BioNetDBException;
    NetworkPathIterator networkPathIterator(String cypher);

    BioNetDBResult<NetworkPath> networkPathQuery(Query query, QueryOptions queryOptions) throws BioNetDBException;
    BioNetDBResult<NetworkPath> networkPathQuery(String cypher);

    //-------------------------------------------------------------------------
    // N E T W O R K S
    //-------------------------------------------------------------------------

    BioNetDBResult<NetworkStats> networkStats();

//    DataResult<Network> networkQuery(List<NodeQuery> nodeQueries, QueryOptions queryOptions) throws BioNetDBException;
//    DataResult<Network> networkQueryByPaths(List<NetworkPathQuery> pathQueries, QueryOptions queryOptions) throws BioNetDBException;
//    DataResult<Network> networkQuery(String cypher) throws BioNetDBException;

    //========================================================================
    // V A R I A N T
    //========================================================================

//    VariantIterator variantIterator(Query query, QueryOptions queryOptions) throws BioNetDBException;
//    VariantIterator variantIterator(String cypher) throws BioNetDBException;
//
//    DataResult<Variant> variantQuery(Query query, QueryOptions queryOptions) throws BioNetDBException;
//    DataResult<Variant> variantQuery(String cypher) throws BioNetDBException;

    //========================================================================
    // I N T E R P R E T A T I O N     A N A L Y S I S
    //========================================================================

//    DataResult<Variant> proteinNetworkInterpretationAnalysis(boolean complexOrReaction, Query query) throws BioNetDBException;
} public interface NetworkPathIterator extends Iterator<NetworkPath> {

//    @Override
//    public boolean hasNext() {
//        return hasNext();
//    }
//
//    @Override
//    public T next() {
//        return next();
//    }
} public interface RowIterator extends Iterator<List<Object>> {

//    @Override
//    public boolean hasNext() {
//        return hasNext();
//    }
//
//    @Override
//    public T next() {
//        return next();
//    }
} public class Neo4JNetworkDBAdaptor implements NetworkDBAdaptor {

    private Driver driver;
    private BioNetDBConfiguration configuration;

    public Neo4JNetworkDBAdaptor(BioNetDBConfiguration configuration) {
        this.configuration = configuration;

        // configuration class checks if database is null or empty
        DatabaseConfiguration databaseConfiguration = configuration.getDatabase();

        String databaseURI = databaseConfiguration.getHost() + ":" + databaseConfiguration.getPort();
        String user = databaseConfiguration.getUser();
        String password = databaseConfiguration.getPassword();

        driver = GraphDatabase.driver("bolt://" + databaseURI, AuthTokens.basic(user, password));
        Runtime.getRuntime().addShutdownHook(new Thread(() -> driver.close()));

//        // Add configuration node
//        if (!existConfigNode()) {
//            createConfigNode();
//        }
    }

    @Override
    public void close() {
        driver.close();
    }

    @Override
    public void index() {
        Session session = this.driver.session();
        if (session != null) {
            // Create indexes
            session.writeTransaction(tx -> {

                List<String> nodeTypes = new ArrayList<>(Arrays.asList(GENE.name(), ENSEMBL_GENE.name(), REFSEQ_GENE.name(),
                        TRANSCRIPT.name(), ENSEMBL_TRANSCRIPT.name(), REFSEQ_TRANSCRIPT.name(), PROTEIN.name(), VARIANT.name(),
                        SAMPLE.name(), INDIVIDUAL.name(), XREF.name(), DISEASE_PANEL.name()));

                for (String nodeType: nodeTypes) {
                    tx.run("CREATE INDEX IF NOT EXISTS FOR (n:" + nodeType + ") ON (n.uid)");
                    tx.run("CREATE INDEX IF NOT EXISTS FOR (n:" + nodeType + ") ON (n.id)");
                    tx.run("CREATE INDEX IF NOT EXISTS FOR (n:" + nodeType + ") ON (n.name)");
                }
                tx.run("CREATE INDEX IF NOT EXISTS FOR (n:" + Node.Label.CUSTOM.name() + ") ON (n.attr_type)");
                tx.commit();

                return 1;
            });

            // Wait for indexes been created
            session.writeTransaction(tx -> {
                tx.run("CALL db.awaitIndexes(3000000)");
                tx.commit();
                return 1;
            });
            session.close();
        }
    }

    public boolean isReady() {
        Transaction tx = null;
        try {
            tx = driver.session().beginTransaction();
            tx.run("MATCH (n:" + Node.Label.INTERNAL_CONNFIG + "{uid:0}) return n");
            tx.commit();
            return true;
        } catch (Exception e) {
            return false;
        } finally {
            if (tx != null) {
                tx.close();
            }
        }
    }

    //-------------------------------------------------------------------------
    // I N S E R T     N E T W O R K S
    //-------------------------------------------------------------------------

    /**
     * Insert an entire network into the Neo4J database.
     *
     * @param network      Object containing all the nodes and interactions
     * @param queryOptions Optional params
     */
    @Override
    public void insert(Network network, QueryOptions queryOptions) throws BioNetDBException {
        Session session = this.driver.session();

        // First, insert Neo4J nodes
        for (Node node : network.getNodes()) {
            session.writeTransaction(tx -> {
                addNode(node, tx);
                return 1;
            });
        }

        // Second, insert Neo4J relationships
        for (Relation relation : network.getRelations()) {
            session.writeTransaction(tx -> {
                addRelation(relation, tx);
                return 1;
            });
        }

        session.close();
    }

    //-------------------------------------------------------------------------
    // A N N O T A T I O N     M E T H O D s
    //-------------------------------------------------------------------------

//    public void annotateVariants(NodeQuery query, QueryOptions options, VariantClient variantClient) throws BioNetDBException,
//            IOException {
//        NodeIterator nodeIterator = nodeIterator(query, options);
//        List<String> variantIds = new ArrayList<>(1000);
//        while (nodeIterator.hasNext()) {
//            Node variantNode = nodeIterator.next();
//            variantIds.add(variantNode.getId());
//            if (variantIds.size() >= 1000) {
//                annotateVariants(variantIds, variantClient);
//                variantIds.clear();
//            }
//        }
//        if (CollectionUtils.isNotEmpty(variantIds)) {
//            annotateVariants(variantIds, variantClient);
//        }
//    }
//
//    public void annotateVariants(List<String> variantIds, VariantClient variantClient) throws BioNetDBException, IOException {
//        Neo4JVariantLoader variantLoader = new Neo4JVariantLoader(this);
//        CellBaseDataResponse<Variant> entryQueryResponse = variantClient.get(variantIds, QueryOptions.empty());
//        for (CellBaseDataResult<Variant> queryResult : entryQueryResponse.getResponses()) {
//            if (CollectionUtils.isNotEmpty(queryResult.getResults())) {
//                variantLoader.loadVariants(queryResult.getResults());
//            }
//        }
//    }

    //-------------------------------------------------------------------------

//    public void annotateGenes(NodeQuery query, QueryOptions options, GeneClient geneClient) throws BioNetDBException,
//            IOException {
//        NodeIterator nodeIterator = nodeIterator(query, options);
//        List<String> geneIds = new ArrayList<>(1000);
//        while (nodeIterator.hasNext()) {
//            Node geneNode = nodeIterator.next();
//            geneIds.add(geneNode.getId());
//            if (geneIds.size() >= 1000) {
//                annotateGenes(geneIds, geneClient);
//                geneIds.clear();
//            }
//        }
//        if (CollectionUtils.isNotEmpty(geneIds)) {
//            annotateGenes(geneIds, geneClient);
//        }
//    }

//    public void annotateGenes(List<String> geneIds, GeneClient geneClient) throws BioNetDBException, IOException {
//        Neo4JVariantLoader variantLoader = new Neo4JVariantLoader(this);
//        QueryOptions options = new QueryOptions("EXCLUDE", "transcripts.exons,transcripts.cDnaSequence");
//        CellBaseDataResponse<Gene> entryQueryResponse = geneClient.get(geneIds, options);
//        for (CellBaseDataResult<Gene> queryResult : entryQueryResponse.getResponses()) {
//            if (CollectionUtils.isNotEmpty(queryResult.getResults())) {
//                variantLoader.loadGenes(queryResult.getResults());
//            }
//        }
//    }

    //-------------------------------------------------------------------------

//    public void annotateProteins(NodeQuery query, QueryOptions options, ProteinClient proteinClient) throws BioNetDBException,
//            IOException {
//        NodeIterator nodeIterator = nodeIterator(query, options);
//        List<String> proteinIds = new ArrayList<>(1000);
//        while (nodeIterator.hasNext()) {
//            Node geneNode = nodeIterator.next();
//            proteinIds.add(geneNode.getId());
//            if (proteinIds.size() >= 1000) {
//                annotateProteins(proteinIds, proteinClient);
//                proteinIds.clear();
//            }
//        }
//        if (CollectionUtils.isNotEmpty(proteinIds)) {
//            annotateProteins(proteinIds, proteinClient);
//        }
//    }

//    public void annotateProteins(List<String> proteinIds, ProteinClient proteinClient) throws BioNetDBException, IOException {
//        Neo4JVariantLoader variantLoader = new Neo4JVariantLoader(this);
//        QueryOptions options = new QueryOptions("EXCLUDE", "transcripts.exons,transcripts.cDnaSequence");
//        CellBaseDataResponse<Entry> entryQueryResponse = proteinClient.get(proteinIds, options);
//        for (CellBaseDataResult<Entry> responses : entryQueryResponse.getResponses()) {
//            if (CollectionUtils.isNotEmpty(responses.getResults())) {
//                variantLoader.loadProteins(responses.getResults());
//            }
//        }
//    }

//    public void annotateProtein(ProteinClient proteinClient) throws BioNetDBException, IOException {
//        // First, get all proteins from the network
//        Query query = new Query();
//        String cypher = "MATCH path=(p:PROTEIN)-[xr:XREF]->(x:XREF) WHERE toLower(x.attr_source) = \"uniprot\" return path";
//        QueryResult<Network> networkResult = networkQuery(cypher);
//
//        if (ListUtils.isEmpty(networkResult.getResult())) {
//            System.out.println("Network not found!!");
//            return;
//        }
//        Network network = networkResult.getResult().get(0);
//
//        // Get proteins annotations from Cellbase...
//        // ... prepare list of protein id/names from xref/protein nodes
//        List<String> proteinIds = new ArrayList<>();
//        for (Node node: network.getNodes()) {
//            if (node.getLabels() == Node.Label.XREF) {
//                proteinIds.add(node.getId());
//            }
//        }
//
//        // ... finally, call Cellbase service
//        Map<String, Entry> proteinMap = new HashMap<>();
//        QueryResponse<Entry> entryQueryResponse = proteinClient.get(proteinIds, new QueryOptions(QueryOptions.EXCLUDE,
//                "reference,organism,comment,evidence,sequence"));
//        for (QueryResult<Entry> queryResult: entryQueryResponse.getResponse()) {
//            proteinMap.put(queryResult.getId(), queryResult.getResult().get(0));
//        }
//
//        for (String key: proteinMap.keySet()) {
//            System.out.println(key + " -> " + proteinMap.get(key));
//        }
//    }

    //-------------------------------------------------------------------------
    // N O D E     Q U E R I E S
    //-------------------------------------------------------------------------

    @Override
    public NodeIterator nodeIterator(Query query, QueryOptions queryOptions) {
        String cypher = Neo4JQueryParser.parseNodeQuery(query, queryOptions);
        return nodeIterator(cypher);
    }

    @Override
    public NodeIterator nodeIterator(String cypher) {
        Session session = this.driver.session();
        System.out.println("Cypher query: " + cypher);
        return new Neo4JNodeIterator(session.run(cypher));
    }

    @Override
    public BioNetDBResult<Node> nodeQuery(Query query, QueryOptions queryOptions) {
        String cypher = Neo4JQueryParser.parseNodeQuery(query, queryOptions);
        return nodeQuery(cypher);
    }

    @Override
    public BioNetDBResult<Node> nodeQuery(String cypher) {
        // Query for nodes using the node iterator
        StopWatch stopWatch = StopWatch.createStarted();
        NodeIterator nodeIterator = nodeIterator(cypher);
        List<Node> nodes = new ArrayList<>();
        while (nodeIterator.hasNext()) {
            nodes.add(nodeIterator.next());
        }
        int dbTime = (int) stopWatch.getTime(TimeUnit.MILLISECONDS);

        // Create QueryResult and return
        BioNetDBResult<Node> queryResult = new BioNetDBResult<>(dbTime, new ArrayList<>(), nodes.size(), nodes, nodes.size());
        return queryResult;
    }

    @Override
    public BioNetDBResult<NodeStats> nodeStats(Query query) {
        StopWatch stopWatch = StopWatch.createStarted();
        Session session = this.driver.session();

        StringBuilder where = new StringBuilder();
        List<String> filters = Neo4JQueryParser.getFilters("n", query);
        if (filters.size() > 0) {
            where.append(" where ").append(StringUtils.join(filters, " and"));
        }
        String cypher = "match (n)" + where.toString() + " with distinct labels(n) as label, count(labels(n)) as cnt return label, cnt";
        System.out.println("Cypher query: " + cypher);
        Result result = session.run(cypher);

        long total = 0;
        Map<String, Long> count = new HashMap<>();

        while (result.hasNext()) {
            Record record = result.next();
            List<Object> items = record.get(0).asList();
            long cnt = record.get(1).asLong();
            for (Object item: items) {
                String label = (String) item;
                if (!count.containsKey(label)) {
                    count.put(label, 0L);
                }
                count.put(label, count.get(label) + cnt);
            }
            total += cnt;
        }

        NodeStats stats = new NodeStats(total, count);
        int dbTime = (int) stopWatch.getTime(TimeUnit.MILLISECONDS);

        return new BioNetDBResult<>(dbTime, Collections.emptyList(), 1, Collections.singletonList(stats), 1);
    }

    //-------------------------------------------------------------------------

    @Override
    public long addNode(Node node) throws BioNetDBException {
        if (existNode(node)) {
            throw new BioNetDBException("Error adding node: it already exists");
        }

        long uid = getUidCounter();
        node.setUid(uid);

        Session session = driver.session();
        try (Transaction tx = session.beginTransaction()) {

            // Gather properties of the node to create a cypher string with them
            List<String> props = new ArrayList<>();
            props.add("n.uid=" + node.getUid());
            if (StringUtils.isNotEmpty(node.getId())) {
                props.add("n.id=\"" + cleanValue(node.getId()) + "\"");
            }
            if (StringUtils.isNotEmpty(node.getName())) {
                props.add("n.name=\"" + cleanValue(node.getName()) + "\"");
            }

            for (String key : node.getAttributes().keySet()) {
                if (StringUtils.isNumeric(node.getAttributes().getString(key))) {
                    props.add("n." + PREFIX_ATTRIBUTES + key + "=" + node.getAttributes().getString(key));
                } else {
                    props.add("n." + PREFIX_ATTRIBUTES + key + "=\"" + cleanValue(node.getAttributes().getString(key)) + "\"");
                }
            }
            //String propsJoined = "{" + String.join(",", props) + "}";

            // Create the desired node
            StringBuilder cypher = new StringBuilder("CREATE (n");
            if (CollectionUtils.isNotEmpty(node.getLabels())) {
                cypher.append(":").append(StringUtils.join(node.getLabels(), ":"));
            }
            cypher.append(")");
            if (CollectionUtils.isNotEmpty(props)) {
                cypher.append(" SET ").append(StringUtils.join(props, ","));
            }

            // Run cypher statement
            System.out.println(cypher.toString());
            //cypher.append(" RETURN ID(n) AS UID");
            tx.run(cypher.toString());
            tx.commit();
        }
        session.close();

        setUidCounter(uid + 1);

        return uid;
    }

    @Override
    public void updateNode(Node node) throws BioNetDBException {
        if (!existNode(node)) {
            throw new BioNetDBException("Error updating node: it does not exist");
        }

        Session session = driver.session();
        try (Transaction tx = session.beginTransaction()) {
            List<String> attrs = new ArrayList<>();
            for (String key : node.getAttributes().keySet()) {
                if (StringUtils.isNumeric(node.getAttributes().getString(key))) {
                    attrs.add("n." + PREFIX_ATTRIBUTES + key + "=" + node.getAttributes().getString(key));
                } else {
                    attrs.add("n." + PREFIX_ATTRIBUTES + key + "=\"" + cleanValue(node.getAttributes().getString(key)) + "\"");
                }
            }

            // Match the desired node
            StringBuilder cypher = new StringBuilder("MATCh (n");
            if (CollectionUtils.isNotEmpty(node.getLabels())) {
                cypher.append(":").append(StringUtils.join(node.getLabels(), ":"));
            }
            cypher.append(")");
            cypher.append(" WHERE n.id=\"").append(node.getId()).append("\"");
            if (CollectionUtils.isNotEmpty(attrs)) {
                cypher.append(" SET ").append(StringUtils.join(attrs, ","));
            }

            tx.run(cypher.toString());
            tx.commit();
        }
        session.close();
    }

    @Override
    public void deleteNode(Node node) throws BioNetDBException {
        checkNode(node);

        Query query = new Query("id", node.getId()).append("type", node.getLabels());
        BioNetDBResult<Node> result = nodeQuery(query, QueryOptions.empty());

        if (result.getNumResults() == 0) {
            throw new BioNetDBException("Error deleting node: it does not exist");
        }

        if (result.getNumResults() > 1) {
            throw new BioNetDBException("Error deleting node: multiple nodes found!");
        }

        Session session = driver.session();
        try (Transaction tx = session.beginTransaction()) {
            String cypher = "MATCH (n:" + node.getLabels() + "{id: '" + node.getId() + "'}) DETACH DELETE n";
            tx.run(cypher);
            tx.commit();
        }
        session.close();
    }

    @Override
    public boolean existNode(Node node) throws BioNetDBException {
        checkNode(node);

        Query query = new Query("id", node.getId()).append("label", node.getLabels().get(0).name());
        BioNetDBResult<Node> result = nodeQuery(query, QueryOptions.empty());

        return result.getNumResults() == 0 ? false : true;
    }

    //-------------------------------------------------------------------------

    @Override
    public void addRelation(Node origNode, Node destNode, Relation relation) throws BioNetDBException {
        checkRelationNodes(origNode, destNode);

        // TODO: check if relation exists

        List<String> props = new ArrayList<>();
        for (String key : relation.getAttributes().keySet()) {
            props.add(PREFIX_ATTRIBUTES + key + ":\"" + cleanValue(relation.getAttributes().getString(key)) + "\"");
        }
        String propsJoined = "{" + String.join(",", props) + "}";

        StringBuilder cypher = new StringBuilder();
        cypher.append("MATCH (o:").append(origNode.getLabels()).append("{id:\"")
                .append(origNode.getId()).append("\"}) MATCH (d:").append(destNode.getLabels()).append("{id:\"")
                .append(destNode.getId()).append("\"}) USING INDEX d:").append(destNode.getLabels())
                .append("(id) MERGE (o)-[r:").append(relation.getLabel());
        if (CollectionUtils.isNotEmpty(props)) {
            cypher.append(propsJoined);
        }
        cypher.append("]-(d)");

        System.out.println(cypher.toString());

        // Create the relationship
        Session session = driver.session();
        try (Transaction tx = session.beginTransaction()) {
            tx.run(cypher.toString());
            tx.commit();
        }
        session.close();
    }

    @Override
    public void updateRelation(Node origNode, Node destNode, Relation relation) throws BioNetDBException {
        checkRelationNodes(origNode, destNode);

        // TODO: check if relation exists

        List<String> props = new ArrayList<>();
        for (String key : relation.getAttributes().keySet()) {
            props.add("r." + PREFIX_ATTRIBUTES + key + "=\"" + cleanValue(relation.getAttributes().getString(key)) + "\"");
        }

        StringBuilder cypher = new StringBuilder();
        cypher.append("MATCH (o:").append(origNode.getLabels()).append("{id:\"")
                .append(origNode.getId()).append("\"})-[r:").append(relation.getLabel()).append("]-(d:").append(destNode.getLabels())
                .append("{id:\"").append(destNode.getId()).append("\"})");
        if (CollectionUtils.isNotEmpty(props)) {
            cypher.append(" SET ").append(StringUtils.join(props, ","));
        }

        System.out.println(cypher.toString());

        // Create the relationship
        Session session = driver.session();
        try (Transaction tx = session.beginTransaction()) {
            tx.run(cypher.toString());
            tx.commit();
        }
        session.close();
    }

    @Override
    public void deleteRelation(Node origNode, Node destNode, Relation relation) throws BioNetDBException {
        checkRelationNodes(origNode, destNode);

        // TODO: check if relation exists

        List<String> props = new ArrayList<>();
        for (String key : relation.getAttributes().keySet()) {
            props.add(PREFIX_ATTRIBUTES + key + ":\"" + cleanValue(relation.getAttributes().getString(key)) + "\"");
        }
        String propsJoined = "{" + String.join(",", props) + "}";

        StringBuilder cypher = new StringBuilder();
        cypher.append("MATCH (o:").append(origNode.getLabels()).append("{id:\"")
                .append(origNode.getId()).append("\"})-[r:").append(relation.getLabel()).append("]-(d:").append(destNode.getLabels())
                .append("{id:\"").append(destNode.getId()).append("\"}) DELETE r");

        System.out.println(cypher.toString());

        // Create the relationship
        Session session = driver.session();
        try (Transaction tx = session.beginTransaction()) {
            tx.run(cypher.toString());
            tx.commit();
        }
        session.close();
    }

    @Override
    public boolean existRelation(Node origNode, Node destNode, Relation relation) throws BioNetDBException {
        return false;
    }

    //-------------------------------------------------------------------------

    private void checkNode(Node node) throws BioNetDBException {
        if (StringUtils.isEmpty(node.getId())) {
            throw new BioNetDBException("Missing node ID");
        }

        if (node.getLabels() == null) {
            throw new BioNetDBException("Missing node type");
        }
    }

    private void checkRelationNodes(Node origNode, Node destNode) throws BioNetDBException {
        if (!existNode(origNode)) {
            throw new BioNetDBException("Origin node does not exist");
        }

        if (!existNode(destNode)) {
            throw new BioNetDBException("Destination node does not exist");
        }
    }

    //-------------------------------------------------------------------------
    // T A B L E     Q U E R I E S
    //-------------------------------------------------------------------------

//    @Override
//    public RowIterator rowIterator(Query query, QueryOptions queryOptions) throws BioNetDBException {
//        String cypher = Neo4JQueryParser.parseNodeQuery(query, queryOptions);
//        return rowIterator(cypher);
//    }
//
//    public RowIterator rowIterator(String cypher) throws BioNetDBException {
//        Session session = this.driver.session();
//        System.out.println("Cypher query: " + cypher);
//        return new Neo4JRowIterator(session.run(cypher));
//    }
//
//    @Override
//    public DataResult<List<Object>> rowQuery(Query query, QueryOptions queryOptions) throws BioNetDBException {
//        throw new UnsupportedOperationException("rowQuery not yet supported");
////        String cypher = Neo4JQueryParser.parseRowQuery(query, queryOptions);
////        return nodeQuery(cypher);
//    }
//
//    @Override
//    public DataResult<List<Object>> rowQuery(String cypher) throws BioNetDBException {
//        throw new UnsupportedOperationException("rowQuery not yet supported");
//    }

    //-------------------------------------------------------------------------
    // P A T H     Q U E R I E S
    //-------------------------------------------------------------------------

    @Override
    public NetworkPathIterator networkPathIterator(Query networkPathQuery, QueryOptions queryOptions) throws BioNetDBException {
        String cypher = Neo4JQueryParser.parseNetworkPathQuery(networkPathQuery, queryOptions);
        return networkPathIterator(cypher);
    }

    @Override
    public NetworkPathIterator networkPathIterator(String cypher) {
        Session session = this.driver.session();
        System.out.println("Cypher query: " + cypher);
        return new Neo4JNetworkPathIterator(session.run(cypher));
    }

    @Override
    public BioNetDBResult<NetworkPath> networkPathQuery(Query query, QueryOptions queryOptions) throws BioNetDBException {
        String cypher = Neo4JQueryParser.parseNetworkPathQuery(query, queryOptions);
        return networkPathQuery(cypher);
    }

    @Override
    public BioNetDBResult<NetworkPath> networkPathQuery(String cypher) {
        // Query for nodes using the node iterator
        StopWatch stopWatch = StopWatch.createStarted();
        NetworkPathIterator pathIterator = networkPathIterator(cypher);
        List<NetworkPath> networkPaths = new ArrayList<>();
        while (pathIterator.hasNext()) {
            networkPaths.add(pathIterator.next());
        }
        int dbTime = (int) stopWatch.getTime(TimeUnit.MILLISECONDS);

        // Create QueryResult and return
        BioNetDBResult<NetworkPath> queryResult = new BioNetDBResult<>(dbTime, new ArrayList<>(), networkPaths.size(), networkPaths,
                networkPaths.size());
        return queryResult;
    }

    //-------------------------------------------------------------------------
    // N E T W O R K     Q U E R I E S
    //-------------------------------------------------------------------------

    @Override
    public BioNetDBResult<NetworkStats> networkStats() {
        StopWatch stopWatch = StopWatch.createStarted();
        Session session = this.driver.session();

        String cypher = "call apoc.meta.stats yield stats";
        System.out.println("Cypher query: " + cypher);
        Result result = session.run(cypher);

//        long total = 0;
//        Map<String, Long> count = new HashMap<>();

        NetworkStats stats = new NetworkStats();

        while (result.hasNext()) {
            Record record = result.next();
            if (record.containsKey("stats")) {
                Map<String, Object> map = record.get("stats").asMap();

                stats.setNodeCount((Long) map.get("nodeCount"));
                stats.setRelationCount((Long) map.get("relCount"));

                stats.setNodeTypeCount((Long) map.get("labelCount"));
                stats.setRelationTypeCount((Long) map.get("relTypeCount"));
                stats.setAttributeTypeCount((Long) map.get("propertyKeyCount"));

                stats.setAggNodeTypes((Map<String, Long>) map.get("labels"));
                stats.setAggRelationTypes((Map<String, Long>) map.get("relTypes"));

                break;
            }
        }
        int dbTime = (int) stopWatch.getTime(TimeUnit.MILLISECONDS);

        return new BioNetDBResult<>(dbTime, Collections.emptyList(), 1, Collections.singletonList(stats), 1);
    }


//    @Override
//    public DataResult<Network> networkQuery(List<NodeQuery> nodeQueries, QueryOptions queryOptions)
//            throws BioNetDBException {
//        String cypher = Neo4JQueryParser.parseNodesForNetwork(nodeQueries, queryOptions);
//        return networkQuery(cypher);
//    }
//
//    @Override
//    public DataResult<Network> networkQueryByPaths(List<NetworkPathQuery> pathQueries, QueryOptions queryOptions)
//            throws BioNetDBException {
//        String cypher = Neo4JQueryParser.parsePathsForNetwork(pathQueries, queryOptions);
//        return networkQuery(cypher);
//    }
//
//    @Override
//    public DataResult<Network> networkQuery(String cypher) throws BioNetDBException {
//        // Open session
//        Session session = this.driver.session();
//
//        long startTime = System.currentTimeMillis();
//        Result run = session.run(cypher);
//        Network network = Neo4jConverter.toNetwork(run);
//        long stopTime = System.currentTimeMillis();
//
//        // Close session
//        session.close();
//
//        // Build the query result
//        int dbTime = (int) (stopTime - startTime) / 1000;
//        return new DataResult(dbTime, new ArrayList<>(), 1, Collections.singletonList(network), 1);
//    }
//
//    @Override
//    public VariantIterator variantIterator(Query query, QueryOptions queryOptions) throws BioNetDBException {
//        return null;
//    }

    //-------------------------------------------------------------------------
    // V A R I A N T
    //-------------------------------------------------------------------------

//    public VariantIterator variantIterator(String cypher) throws BioNetDBException {
//        Session session = this.driver.session();
//        System.out.println("Cypher query: " + cypher);
//        return new Neo4JVariantIterator(session.run(cypher));
//    }
//
//    @Override
//    public DataResult<Variant> variantQuery(Query query, QueryOptions queryOptions) throws BioNetDBException {
//        String cypher = Neo4JVariantQueryParser.parse(query, QueryOptions.empty());
//        return variantQuery(cypher);
//    }
//
//    @Override
//    public DataResult<Variant> variantQuery(String cypher) throws BioNetDBException {
//        if (org.apache.commons.lang.StringUtils.isEmpty(cypher)) {
//            throw new BioNetDBException("Missing cypher to query");
//        }
//
//        List<Variant> variants = new ArrayList<>();
//        VariantIterator variantIterator = variantIterator(cypher);
//        while (variantIterator.hasNext()) {
//            Variant variant = variantIterator.next();
//            if (variant != null) {
//                variants.add(variant);
//            }
//        }
//
//        int dbTime = 0;
//        return new DataResult(dbTime, new ArrayList<>(), variants.size(), variants, variants.size());
//    }

    //-------------------------------------------------------------------------
    // I N T E R P R  E T A T I O N     A N A L Y S I S
    //-------------------------------------------------------------------------

//    @Override
//    public DataResult<Variant> proteinNetworkInterpretationAnalysis(boolean complexOrReaction, Query query) throws BioNetDBException {
//        // Create cypher statement from query
//        String cypher = Neo4JVariantQueryParser.parseProteinNetworkInterpretation(query, QueryOptions.empty(), complexOrReaction);
//
//        // The next code is performed also by queryVariants method in MoIManager
//        List<Variant> variants = new ArrayList<>();
//
//        VariantIterator variantIterator = variantIterator(cypher);
//
//        while (variantIterator.hasNext()) {
//            variants.add(variantIterator.next());
//        }
//
//        int dbTime = 0;
//        return new DataResult(dbTime, new ArrayList<>(), variants.size(), variants, variants.size());
//    }

    //-------------------------------------------------------------------------
    //------------------------------------------------------------------------------------------------------------------

    public Result addNode(Node node, Transaction tx) {
        // Gather properties of the node to create a cypher string with them
        List<String> props = new ArrayList<>();
        props.add("n.uid=" + node.getUid());
        if (StringUtils.isNotEmpty(node.getId())) {
            props.add("n.id=\"" + cleanValue(node.getId()) + "\"");
        }
        if (StringUtils.isNotEmpty(node.getName())) {
            props.add("n.name=\"" + cleanValue(node.getName()) + "\"");
        }
//        if (StringUtils.isNotEmpty(node.getSource())) {
//            props.add("n.source=\"" + node.getSource() + "\"");
//        }
        if (node.getAttributes().containsKey("uidCounter")) {
            props.add("n." + PREFIX_ATTRIBUTES + "uidCounter=" + node.getAttributes().get("uidCounter"));
        }
        for (String key : node.getAttributes().keySet()) {
            if (StringUtils.isNumeric(node.getAttributes().getString(key))) {
                props.add("n." + PREFIX_ATTRIBUTES + key + "=" + node.getAttributes().getString(key));
            } else {
                props.add("n." + PREFIX_ATTRIBUTES + key + "=\"" + cleanValue(node.getAttributes().getString(key)) + "\"");
            }
        }
        //String propsJoined = "{" + String.join(",", props) + "}";

        // Create the desired node
        StringBuilder cypher = new StringBuilder("CREATE (n");
        if (CollectionUtils.isNotEmpty(node.getLabels())) {
            cypher.append(":").append(StringUtils.join(node.getLabels(), ":"));
        }
        cypher.append(")");
        if (CollectionUtils.isNotEmpty(props)) {
            cypher.append(" SET ").append(StringUtils.join(props, ","));
        }
        //cypher.append(" RETURN ID(n) AS UID");
        Result ret = tx.run(cypher.toString());
        //node.setUid(ret.peek().get("UID").asLong());
        return ret;
    }

    public void mergeNode(Node node, String byKey, Transaction tx) {
        Object value;
        switch (byKey) {
            case "id":
                value = node.getId();
                break;
            case "name":
                value = node.getName();
                break;
//            case "source":
//                value = node.getSource();
//                break;
            default:
                value = node.getAttributes().get(byKey);
                break;
        }
        StringBuilder cypher = new StringBuilder("MATCH (n");
        if (node.getLabels() != null) {
            cypher.append(":").append(node.getLabels());
        }
        cypher.append(") WHERE n.").append(byKey);
        if (value instanceof String) {
            cypher.append("=\"").append(cleanValue(value.toString())).append("\"");
        } else {
            cypher.append("=").append(value);
        }
        cypher.append(" RETURN n");

        Result ret = tx.run(cypher.toString());
        if (ret.hasNext()) {
            node.setUid(ret.next().get(0).asNode().get("uid").asLong());
        } else {
            node.setUid(node.getUid() + 1);
            addNode(node, tx);
        }
    }

    public void mergeNode(Node node, String byKey1, String byKey2, Transaction tx) {
        List<String> byKeys = new ArrayList<>(2);
        byKeys.add(byKey1);
        byKeys.add(byKey2);
        List<Object> values = new ArrayList<>(2);
        List<Boolean> prefix = new ArrayList<>(2);

        for (int i = 0; i < 2; i++) {
            switch (byKeys.get(i)) {
                case "id":
                    values.add(node.getId());
                    prefix.add(false);
                    break;
                case "name":
                    values.add(node.getName());
                    prefix.add(false);
                    break;
//                case "source":
//                    values.add(node.getSource());
//                    prefix.add(false);
//                    break;
                default:
                    values.add(node.getAttributes().get(byKeys.get(i)));
                    prefix.add(true);
                    break;
            }
        }
        StringBuilder cypher = new StringBuilder("MATCH (n");
        if (node.getLabels() != null) {
            cypher.append(":").append(node.getLabels());
        }
        cypher.append(") WHERE ");
        for (int i = 0; i < 2; i++) {
            cypher.append("n.");
            if (prefix.get(i)) {
                cypher.append(PREFIX_ATTRIBUTES + byKeys.get(i));
            } else {
                cypher.append(byKeys.get(i));
            }
            if (values.get(i) instanceof String) {
                cypher.append("=\"").append(cleanValue(values.get(i).toString())).append("\"");
            } else {
                cypher.append("=").append(values.get(i));
            }
            if (i == 0) {
                cypher.append(" AND ");
            }
        }
        cypher.append(" RETURN n");

        Result ret = tx.run(cypher.toString());
        if (ret.hasNext()) {
            node.setUid(ret.next().get(0).asNode().get("uid").asLong());
        } else {
            node.setUid(node.getUid() + 1);
            addNode(node, tx);
        }
    }

    public Result mergeRelation(Relation relation, Transaction tx) {
        return addRelation(relation, tx);
    }

    public Result addRelation(Relation relation, Transaction tx) {
        List<String> props = new ArrayList<>();
        //props.add("uid:" + relation.getUid());
        if (StringUtils.isNotEmpty(relation.getName())) {
            props.add("name:\"" + cleanValue(relation.getName()) + "\"");
        }
//        if (StringUtils.isNotEmpty(relation.getSource())) {
//            props.add("source:\"" + relation.getSource() + "\"");
//        }
        for (String key : relation.getAttributes().keySet()) {
            props.add(PREFIX_ATTRIBUTES + key + ":\"" + cleanValue(relation.getAttributes().getString(key)) + "\"");
        }
        String propsJoined = "{" + String.join(",", props) + "}";

        StringBuilder statementTemplate = new StringBuilder();
        statementTemplate.append("MATCH (o:").append(relation.getOrigLabel()).append("{uid:")
                .append(relation.getOrigUid()).append("}) MATCH (d:").append(relation.getDestLabel()).append("{uid:")
                .append(relation.getDestUid()).append("}) USING INDEX d:").append(relation.getDestLabel())
                .append("(uid) MERGE (o)-[r:").append(relation.getLabel());
        if (CollectionUtils.isNotEmpty(props)) {
            statementTemplate.append(propsJoined);
        }
        statementTemplate.append("]->(d)");

        //.append(" RETURN ID(r) AS UID");

        // Create the relationship
        Result ret = tx.run(statementTemplate.toString());
        //relation.setUid(ret.peek().get("UID").asLong());
        return ret;
    }

    //-------------------------------------------------------------------------
    // C O N F I G U R A T I O N     N O D E     M A N A G E M E N T
    //-------------------------------------------------------------------------

    private boolean existConfigNode() {
        Session session = this.driver.session();
        Result statementResult = session.run("match (n:" + Node.Label.INTERNAL_CONNFIG + "{uid:0}) return count(n) as count");
        session.close();

        return (statementResult.peek().get(0).asInt() == 1);
    }

    public long getUidCounter() {
        Session session = this.driver.session();
        StringBuilder cypher = new StringBuilder("match (n:").append(Node.Label.INTERNAL_CONNFIG).append("{uid:0}) return n.")
                .append(PREFIX_ATTRIBUTES).append("uidCounter");
        System.out.println(cypher.toString());

        // Run cypher statement
        Result result = session.run(cypher.toString());
        long uidCounter = result.peek().get(0).asLong();
        session.close();

        return uidCounter;
    }

    public void setUidCounter(long uidCounter) {
        // Build Cypher statement
        StringBuilder cypher = new StringBuilder();
        cypher.append("merge (n:" + Node.Label.INTERNAL_CONNFIG + "{uid:0}) set n.").append(PREFIX_ATTRIBUTES).append("uidCounter=")
                .append(uidCounter);
        System.out.println(cypher.toString());

        // Run cypher statement
        Session session = this.driver.session();
        session.run(cypher.toString());
        session.close();
    }

    private void createConfigNode() {
        // Create configuration node and set it uid = 0
        Session session = this.driver.session();
        Node node = new Node(0, "0", "config", Node.Label.INTERNAL_CONNFIG);
        node.addAttribute("uidCounter", 1);
        try (Transaction tx = session.beginTransaction()) {
            addNode(node, tx);
            //tx.success();
        }
        session.close();
    }

    private void updateConfigNode(ObjectMap attrs) {
        // Build Cypher statement
        StringBuilder cypher = new StringBuilder();
        cypher.append("merge (n{uid:0}) set ");
        List<String> sets = new ArrayList<>();
        for (String key : attrs.keySet()) {
            StringBuilder strSet = new StringBuilder("n.").append(PREFIX_ATTRIBUTES).append(key).append("=");
            if (attrs.get(key) instanceof String) {
                strSet.append("'").append(attrs.get(key)).append("'");
            } else {
                strSet.append(attrs.get(key));
            }
            sets.add(strSet.toString());
        }
        cypher.append(StringUtils.join(sets, ","));

        // Run cypher statement
        Session session = this.driver.session();
        try (Transaction tx = session.beginTransaction()) {
            tx.run(cypher.toString());
            //tx.success();
        }
        session.close();
    }

    //-------------------------------------------------------------------------
    // T E S T S
    //-------------------------------------------------------------------------

    public void loadTest() {
        Session session = this.driver.session();

        Long startUid = System.currentTimeMillis();
        List<Node> nodes = new ArrayList<>();
        for (int i = 0; i < 10000; i++) {
            Node node = new Node(i, "id_" + i, "name_" + i, Node.Label.CELLULAR_LOCATION);
            node.getAttributes().put("uid", node.getUid());
            node.getAttributes().put("id", node.getId());
            node.getAttributes().put("name", node.getName());
//            node.getAttributes().put("source", node.getSource());
            nodes.add(node);
        }

        long uid = startUid;
        System.out.println("Inserting nodes:");
        for (int j = 0; j < 10; j++) {
            long start = System.currentTimeMillis();
            try (Transaction tx = session.beginTransaction()) {
//                String query = "CREATE (n:PROTEIN{id:{id}, name:{name}}) RETURN ID(n) as ID";
                String cypher = "CREATE (n:CELLULAR_LOCATION) SET n = {map}";
                Map<String, Object> params = new HashMap<>();
                for (int i = 0; i < 1000; i++) {
                    Node node = nodes.get(1000 * j + i);
                    node.getAttributes().put("uid", uid++);
                    params.put("map", node.getAttributes());
                    Result ret = tx.run(cypher, params);
//                    node.setUid(ret.peek().get("ID").asLong());

//                    String cypher = "CREATE (n:PROTEIN)";
                    //String cypher = "CREATE (n:" + node.getLabels() + ") SET n.id = '" + node.getId() + "'";
                    // SET n.uid = " + (node.getUid() + (nodes.size() * j)) + ", n.name = '" + node.getName() + "', n.id = '"
                    // + node.getId() + "'";
//                    String cypher = "CREATE (n:" + node.getLabels() + ") SET n.uid = " + (node.getUid() + (nodes.size() * j))
// + ", n.name = '" + node.getName() + "', n.id = '" + node.getId() + "'";
//                    String cypher = "CREATE (n:PROTEIN{id:'" + node.getId() + "', name:'" + node.getName() + "'}) RETURN ID(n)";
//                    StatementResult ret = tx.run(cypher);

//                    StatementResult ret = tx.run(query, Values.parameters("id", node.getId(), "name", node.getName()));
//                    node.setUid(ret.peek().get("ID").asLong());

//            session.writeTransaction(tx -> {
//                String cypher = "CREATE (n:PROTEIN)"; // RETURN ID(n) as ID";
//                StatementResult ret = tx.run(cypher);
                    // long uid = ret.peek().get("ID").asLong();
//                //addNode(node, tx);
//                return 1;
//            });
                }
                //tx.success();
            }
//            System.out.println("Inserting " + nodes.size() + " nodes at " + (1000 * nodes.size() / (System.currentTimeMillis() - start))
// + " nodes/sec");
            System.out.println(1000 * 1000 / (System.currentTimeMillis() - start));
        }

        System.out.println("Update nodes:");
        Random r = new Random();
        for (int j = 0; j < 10; j++) {
            long start = System.currentTimeMillis();
            StringBuilder cypher = new StringBuilder();
            Map<String, Object> params = new HashMap<>();
            try (Transaction tx = session.beginTransaction()) {
                for (int i = 0; i < 1000; i++) {
                    Node node = nodes.get(r.nextInt(10000));

                    cypher.setLength(0);
                    cypher.append("merge (n:").append(Node.Label.CELLULAR_LOCATION);
                    cypher.append("{name:'").append(node.getName());
                    cypher.append("'})");
                    cypher.append(" set n.toto={toto}");
                    cypher.append(" return n.uid as uid");

                    Result ret = tx.run(cypher.toString(), Values.parameters("toto", "hello"));
                    //long retUid = ret.peek().get("uid").asLong();
                }
                //tx.success();
            }
            System.out.println(1000 * 1000 / (System.currentTimeMillis() - start));
        }

        System.out.println("Inserting relationships:");
        for (int j = 0; j < 10; j++) {
            long start = System.currentTimeMillis();
            StringBuilder cypher = new StringBuilder();
            try (Transaction tx = session.beginTransaction()) {
                for (long i = startUid; i < (startUid + 1000); i++) {
                    cypher.setLength(0);
                    long sourceUid = i + (j * 1000);
                    cypher.append("match (s:").append(Node.Label.CELLULAR_LOCATION);
                    cypher.append("), (d:").append(Node.Label.CELLULAR_LOCATION);
                    cypher.append(") where s.uid=").append(sourceUid).append(" AND d.uid=").append(sourceUid + 1000);

//                    cypher.append(" match (d:").append(Node.Label.CELLULAR_LOCATION).append(") where d.uid=").append(sourceUid + 1000);
//                    cypher.append("match (s:").append(Node.Label.CELLULAR_LOCATION).append(") where s.uid=").append(sourceUid);
//                    cypher.append(" match (d:").append(Node.Label.CELLULAR_LOCATION).append(") where d.uid=").append(sourceUid + 1000);
                    cypher.append(" create (s)-[r:").append(Node.Label.CELLULAR_LOCATION).append("{uid:").append(++uid).append("}]->(d)");
//                    cypher.append(" return r.uid");
                    Result ret = tx.run(cypher.toString());

//                String query = "CREATE (n:PROTEIN{id:{id}, name:{name}}) RETURN ID(n) as ID";
//                String cypher = "CREATE (n:CELLULAR_LOCATION) SET n = {map}";
//                Map<String, Object> params = new HashMap<>();
//                for (Node node: nodes) {
//                    node.getAttributes().put("uid", uid++);
//                    params.put("map", node.getAttributes());
//                    StatementResult ret = tx.run(cypher, params);
////                    node.setUid(ret.peek().get("ID").asLong());
//
////                    String cypher = "CREATE (n:PROTEIN)";
//                    //String cypher = "CREATE (n:" + node.getLabels() + ") SET n.id = '" + node.getId() + "'"; //SET n.uid = "
// + (node.getUid() + (nodes.size() * j)) + ", n.name = '" + node.getName() + "', n.id = '" + node.getId() + "'";
////                    String cypher = "CREATE (n:" + node.getLabels() + ") SET n.uid = " + (node.getUid() + (nodes.size() * j))
// + ", n.name = '" + node.getName() + "', n.id = '" + node.getId() + "'";
////                    String cypher = "CREATE (n:PROTEIN{id:'" + node.getId() + "', name:'" + node.getName() + "'}) RETURN ID(n)";
////                    StatementResult ret = tx.run(cypher);
//
////                    StatementResult ret = tx.run(query, Values.parameters("id", node.getId(), "name", node.getName()));
////                    node.setUid(ret.peek().get("ID").asLong());
//
////            session.writeTransaction(tx -> {
////                String cypher = "CREATE (n:PROTEIN)"; // RETURN ID(n) as ID";
////                StatementResult ret = tx.run(cypher);
//                    // long uid = ret.peek().get("ID").asLong();
////                //addNode(node, tx);
////                return 1;
////            });
                }
                //tx.success();
            }
//            System.out.println("Inserting " + nodes.size() + " nodes at " + (1000 * nodes.size() / (System.currentTimeMillis() - start))
// + " nodes/sec");
            System.out.println(1000 * 1000 / (System.currentTimeMillis() - start));
        }

        session.close();
    }

    //-------------------------------------------------------------------------
    // P R I V A T E     M E T H O D S
    //-------------------------------------------------------------------------

//    private String fixString(String in) {
//        return "\"" + in + "\"";
//    }

    private String cleanValue(String value) {
        return value.replace("\"", ",").replace("\\", "|");
    }

    public Driver getDriver() {
        return driver;
    }

    public Neo4JNetworkDBAdaptor setDriver(Driver driver) {
        this.driver = driver;
        return this;
    }
} public class NetworkQueryExecutor {

    private NetworkDBAdaptor networkDBAdaptor;

    public NetworkQueryExecutor(NetworkDBAdaptor networkDBAdaptor) {
        this.networkDBAdaptor = networkDBAdaptor;
    }

    public BioNetDBResult<NetworkStats> stats() {
        return networkDBAdaptor.networkStats();
    }
} public class NodeQueryExecutor {

    private static final int QUERY_MAX_RESULTS = 50000;

    private NetworkDBAdaptor networkDBAdaptor;

    public NodeQueryExecutor(NetworkDBAdaptor networkDBAdaptor) {
        this.networkDBAdaptor = networkDBAdaptor;
    }

    public BioNetDBResult<Node> getNode(long uid) throws BioNetDBException {
        Query query = new Query();
        query.put(NodeQueryParam.UID.key(), uid);
//        query.put(NodeQueryParam.OUTPUT.key(), "node");
        return query(query, QueryOptions.empty());
    }

    public BioNetDBResult<Node> getNode(String id) throws BioNetDBException {
        Query query = new Query();
        query.put(NodeQueryParam.ID.key(), id);
//        query.put(NodeQueryParam.OUTPUT.key(), "node");
        return query(query, QueryOptions.empty());
    }

    public BioNetDBResult<Node> query(Query query, QueryOptions queryOptions) throws BioNetDBException {
        NodeIterator nodeIterator = iterator(query, queryOptions);
        return getNodeQueryResult(nodeIterator);
    }

    public BioNetDBResult<Node> query(String cypher) throws BioNetDBException {
        NodeIterator nodeIterator = iterator(cypher);
        return getNodeQueryResult(nodeIterator);
    }

    public BioNetDBResult<NodeStats> stats() {
        return stats(new Query());
    }

    public BioNetDBResult<NodeStats> stats(Query query) {
        return networkDBAdaptor.nodeStats(query);
    }

    public NodeIterator iterator(Query query, QueryOptions queryOptions) throws BioNetDBException {
        return networkDBAdaptor.nodeIterator(query, queryOptions);
    }

    public NodeIterator iterator(String cypher) throws BioNetDBException {
        return networkDBAdaptor.nodeIterator(cypher);
    }

    //-------------------------------------------------------------------------

    public long add(Node node) throws BioNetDBException {
        return networkDBAdaptor.addNode(node);
    }

    public void update(Node node) throws BioNetDBException {
        networkDBAdaptor.updateNode(node);
    }

    public void delete(Node node) throws BioNetDBException {
        networkDBAdaptor.deleteNode(node);
    }

    //-------------------------------------------------------------------------

    public void link(Node origNode, Node destNode, Relation relation) throws BioNetDBException {
        networkDBAdaptor.addRelation(origNode, destNode, relation);
    }

    public void updateLink(Node origNode, Node destNode, Relation relation) throws BioNetDBException {
        networkDBAdaptor.updateRelation(origNode, destNode, relation);
    }

    public void unlink(Node origNode, Node destNode, Relation relation) throws BioNetDBException {
        networkDBAdaptor.deleteRelation(origNode, destNode, relation);
    }

    //-------------------------------------------------------------------------
    // P R I V A T E     M E T H O D S
    //-------------------------------------------------------------------------

    private BioNetDBResult<Node> getNodeQueryResult(NodeIterator nodeIterator) {
        List<Node> nodes = new ArrayList<>();

        long startTime = System.currentTimeMillis();
        while (nodeIterator.hasNext()) {
            if (nodes.size() >= this.QUERY_MAX_RESULTS) {
                break;
            }
            nodes.add(nodeIterator.next());
        }
        long stopTime = System.currentTimeMillis();

        int time = (int) (stopTime - startTime) / 1000;
        return new BioNetDBResult(time, Collections.emptyList(), nodes.size(), nodes, nodes.size());
    }
} public class PathQueryExecutor {

    private static final int QUERY_MAX_RESULTS = 50000;

    private NetworkDBAdaptor networkDBAdaptor;


    public PathQueryExecutor(NetworkDBAdaptor networkDBAdaptor) {
        this.networkDBAdaptor = networkDBAdaptor;
    }

    public BioNetDBResult<NetworkPath> query(Query query, QueryOptions queryOptions) throws BioNetDBException {
        NetworkPathIterator pathIterator = iterator(query, queryOptions);
        return getNetworkPathQueryResult(pathIterator);
    }

    public BioNetDBResult<NetworkPath> query(String cypher) throws BioNetDBException {
        NetworkPathIterator nodeIterator = iterator(cypher);
        return getNetworkPathQueryResult(nodeIterator);
    }

    public NetworkPathIterator iterator(Query query, QueryOptions queryOptions) throws BioNetDBException {
        return networkDBAdaptor.networkPathIterator(query, queryOptions);
    }

    public NetworkPathIterator iterator(String cypher) throws BioNetDBException {
        return networkDBAdaptor.networkPathIterator(cypher);
    }

    //-------------------------------------------------------------------------
    // P R I V A T E     M E T H O D S
    //-------------------------------------------------------------------------

    private BioNetDBResult<NetworkPath> getNetworkPathQueryResult(NetworkPathIterator pathIterator) {
        List<NetworkPath> networkPaths = new ArrayList<>();

        long startTime = System.currentTimeMillis();
        while (pathIterator.hasNext()) {
            if (networkPaths.size() >= this.QUERY_MAX_RESULTS) {
                break;
            }
            networkPaths.add(pathIterator.next());
        }
        long stopTime = System.currentTimeMillis();

        int time = (int) (stopTime - startTime) / 1000;
        return new BioNetDBResult(time, Collections.emptyList(), networkPaths.size(), networkPaths, networkPaths.size());
    }
} public class Builder {
    public static final Object ENSEMBL_GENE_FILENAME = "gene.json";
    public static final Object REFSEQ_GENE_FILENAME = "refseq.json";
    public static final Object GENE_DBNAME = "gene.rocksdb";

    public static final Object PANEL_FILENAME = "panels.json";

    public static final Object PROTEIN_FILENAME = "protein.json";
    public static final Object PROTEIN_DBNAME = "protein.rocksdb";

    public static final Object MIRNA_FILENAME = "mirna.csv";
    public static final Object MIRNA_DBNAME = "mirna.rocksdb";

    public static final Object NETWORK_FILENAME = "Homo_sapiens.owl";

    public static final Object CLINICAL_VARIANT_FILENAME = "clinical_variants.full.json";

    private List<String> additionalVariantFiles;
    private List<String> additionalNeworkFiles;

    private CsvInfo csv;
    private Path inputPath;
    private Path outputPath;
    private Map<String, Set<String>> filters;
    private ObjectMapper mapper;

    private List<String> sampleIds;

    protected static Logger logger;

    public Builder(Path inputPath, Path outputPath, Map<String, Set<String>> filters) {

        this.inputPath = inputPath;
        this.outputPath = outputPath;
        this.filters = filters;

        // Prepare CSV object
        csv = new CsvInfo(inputPath, outputPath);

        // Prepare jackson writer (object to string)
        mapper = new ObjectMapper();
        mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
        mapper.configure(MapperFeature.REQUIRE_SETTERS_FOR_GETTERS, true);

        this.logger = LoggerFactory.getLogger(this.getClass().toString());
    }

    public void build() throws IOException, NoSuchAlgorithmException {
        long start;

        // Check input files

        File ensemblGeneFile = new File(inputPath + "/" + ENSEMBL_GENE_FILENAME);
        if (!ensemblGeneFile.exists()) {
            ensemblGeneFile = new File(inputPath + "/" + ENSEMBL_GENE_FILENAME + ".gz");
        }

        File refSeqGeneFile = new File(inputPath + "/" + REFSEQ_GENE_FILENAME);
        if (!refSeqGeneFile.exists()) {
            refSeqGeneFile = new File(inputPath + "/" + REFSEQ_GENE_FILENAME + ".gz");
        }

        if (!ensemblGeneFile.exists() && !refSeqGeneFile.exists()) {
            throw new IOException("Missing Ensembl/RefSeq gene files");
        }

        File proteinFile = new File(inputPath + "/" + PROTEIN_FILENAME);
        if (!proteinFile.exists()) {
            proteinFile = new File(inputPath + "/" + PROTEIN_FILENAME + ".gz");
            FileUtils.checkFile(proteinFile.toPath());
        }

        File panelFile = new File(inputPath + "/" + PANEL_FILENAME);
        FileUtils.checkFile(panelFile.toPath());

        File networkFile = new File(inputPath + "/" + NETWORK_FILENAME);
        FileUtils.checkFile(networkFile.toPath());

        File clinicalVariantFile = new File(inputPath + "/" + CLINICAL_VARIANT_FILENAME);
        if (!clinicalVariantFile.exists()) {
            clinicalVariantFile = new File(inputPath + "/" + CLINICAL_VARIANT_FILENAME + ".gz");
            FileUtils.checkFile(clinicalVariantFile.toPath());
        }

        // Group the variant files before openning the CSV files
        List<File> variantFiles = new ArrayList<>();
        variantFiles.add(clinicalVariantFile);
        if (CollectionUtils.isNotEmpty(additionalVariantFiles)) {
            for (String additionalVariantFile : additionalVariantFiles) {
                File file = Paths.get(additionalVariantFile).toFile();
                if (file.exists()) {
                    variantFiles.add(file);
                } else {
                    System.out.println("Ignoring variant file " + additionalVariantFile + ": it does not exist.");
                }
            }
        }

        // Create and open CSV files
        csv.openCSVFiles(variantFiles);

        // Processing metadata files
        System.out.println("Processing metadata files...");
        logger.info("Processing metadata files...");
        start = System.currentTimeMillis();
        processMetadata(variantFiles);
        logger.info("Metadata processing done in {} s", (System.currentTimeMillis() - start) / 1000);
        System.out.println("Metadata processing done in " + (System.currentTimeMillis() - start) / 1000 + " s");

        // Processing proteins
        System.out.println("Processing proteins...");
        logger.info("Processing proteins...");
        start = System.currentTimeMillis();
        buildProteins(proteinFile.toPath());
        logger.info("Protein processing done in {} s", (System.currentTimeMillis() - start) / 1000);
        System.out.println("Protein processing done in " + (System.currentTimeMillis() - start) / 1000 + " s");

        // Processing genes
        if (ensemblGeneFile.exists()) {
            System.out.println("Processing Ensembl genes...");
            logger.info("Processing Ensembl genes...");
            start = System.currentTimeMillis();
            buildGenes(ensemblGeneFile.toPath());
            logger.info("Ensembl gene processing done in {} s", (System.currentTimeMillis() - start) / 1000);
            System.out.println("Ensembl gene processing done in " + (System.currentTimeMillis() - start) / 1000 + " s");
        }

        if (refSeqGeneFile.exists()) {
            System.out.println("Processing RefSeq genes...");
            logger.info("Processing RefSeq genes...");
            start = System.currentTimeMillis();
            buildGenes(refSeqGeneFile.toPath());
            logger.info("RefSeq gene processing done in {} s", (System.currentTimeMillis() - start) / 1000);
            System.out.println("RefSeq gene processing done in " + (System.currentTimeMillis() - start) / 1000 + " s");
        }

        // Disease panels support
        System.out.println("Processing disease panels...");
        logger.info("Processing disease panels...");
        start = System.currentTimeMillis();
        buildDiseasePanels(panelFile.toPath());
        logger.info("Disease panels processing done in {} s", (System.currentTimeMillis() - start) / 1000);
        System.out.println("Disease panels processing done in " + (System.currentTimeMillis() - start) / 1000 + " s");

        // Procesing BioPAX file
        System.out.println("Processing Reactome BioPax file...");
        logger.info("Processing Reactome BioPax file...");
        BioPAXProcessing biopaxProcessing = new BioPAXProcessing(this);
        Neo4jBioPaxBuilder bioPAXImporter = new Neo4jBioPaxBuilder(csv, filters, biopaxProcessing);
        start = System.currentTimeMillis();
        bioPAXImporter.build(networkFile.toPath());
        biopaxProcessing.post();
        System.out.println("Processing Reactome BioPax file done in " + ((System.currentTimeMillis() - start) / 1000) + " s");
        logger.info("Processing Reactome BioPax file done in {} s", (System.currentTimeMillis() - start) / 1000);

        // Processing additional variants
        if (CollectionUtils.isNotEmpty(additionalVariantFiles)) {
            for (String additionalVariantFile: additionalVariantFiles) {
                // Read sample IDs
                sampleIds = readSampleIds(additionalVariantFile);

                logger.info("Processing additional variant file {}...", additionalVariantFile);
                start = System.currentTimeMillis();
                buildVariants(Paths.get(additionalVariantFile));
                logger.info("Processing additional variant file done in {} s", (System.currentTimeMillis() - start) / 1000);
            }
        }

        // Processing clinical variants
        System.out.println("Processing clinical variants...");
        logger.info("Processing clinical variants...");
        start = System.currentTimeMillis();
        buildVariants(clinicalVariantFile.toPath());
        logger.info("Processing clinical variants done in {} s", (System.currentTimeMillis() - start) / 1000);
        System.out.println("Processing clinical variants done in " + (System.currentTimeMillis() - start) / 1000 + " s");

        // Processing additional networks
        if (CollectionUtils.isNotEmpty(additionalNeworkFiles)) {
            for (String additionalNeworkFile: additionalNeworkFiles) {
                System.out.println("Processing additional network file " + additionalNeworkFile + "...");
                logger.info("Processing additional network file {}...", additionalNeworkFile);
                start = System.currentTimeMillis();
                processAdditionalNetwork(additionalNeworkFile);
                logger.info("Processing additional network file done in {} s", (System.currentTimeMillis() - start) / 1000);
                System.out.println("Processing additional network file done in " + (System.currentTimeMillis() - start) / 1000 + " s");
            }
        }

        // Set internal config
        buildInternalConfigNode();
        logger.info("Configuration node created.");
        System.out.println("Configuration node created.");

        // Close CSV files
        csv.close();
        logger.info("CSV files closed.");
        System.out.println("CSV files closed.");
    }

    //-------------------------------------------------------------------------
    //  BioPAX importer callback object
    //-------------------------------------------------------------------------

    public class BioPAXProcessing implements Neo4jBioPaxBuilder.BioPAXProcessing {
        private Builder builder;

        private List<Node> dnaNodes;
        private List<Node> rnaNodes;


        public BioPAXProcessing(Builder builder) {
            this.builder = builder;
            dnaNodes = new ArrayList<>();
            rnaNodes = new ArrayList<>();
        }

        public void post() throws IOException {
            CsvInfo csv = builder.getCsvInfo();
            BufferedWriter bwNode, bwRel;

            // Post-process DNA nodes
            logger.info("Post-processing {} dna nodes", dnaNodes.size());
            for (Node node : dnaNodes) {
                // Write DNA node
                writeNodeLine(node);

                // Search for Ensembl and RefSeq genes to relation to
                Long geneUid;
                List<String> geneNames = getGeneNameFromDnaNode(node);
                for (String geneName : geneNames) {
                    // Check if it is an Ensembl ID
                    if (geneName.startsWith("ENSG00")) {
                        Gene gene = csv.getGene(geneName);
                        if (gene != null) {
                            geneName = gene.getName();
                        }
                    }

                    // Search ensembl gene
                    geneUid = csv.getLong(geneName, "ensembl");
                    if (geneUid != null) {
                        // Write dna-gene relation
                        writeRelationLine(IS___DNA___GENE.name(), node.getUid(), geneUid);
                    }

                    // Search refseq gene
                    geneUid = csv.getLong(geneName, "refseq");
                    if (geneUid != null) {
                        // Write dna-gene relation
                        writeRelationLine(IS___DNA___GENE.name(), node.getUid(), geneUid);
                    }
                }
            }

            // Post-process RNA nodes
            logger.info("Post-processing {} rna nodes", rnaNodes.size());
            for (Node node : rnaNodes) {
                // Write RNA node
                writeNodeLine(node);

                boolean isRelated = false;
                String xrefId = node.getAttributes().getString("xrefIds");
                if (StringUtils.isNotEmpty(xrefId)) {
                    if (xrefId.startsWith("MI")) {
                        // miRBase
                        Long uid = csv.getLong(xrefId, MIRNA.name());
                        System.out.println("RNA -> MIRNA: " + xrefId + ", " + uid);
                        if (uid != null) {
                            // Write relation rna - mirna
                            writeRelationLine(IS___RNA___MIRNA.name(), node.getUid(), uid);
                            isRelated = true;
                        }
                    } else if (xrefId.startsWith("ENST")) {
                        // Ensembl transcript
                        Long uid = csv.getLong(xrefId, TRANSCRIPT.name());
                        if (uid != null) {
                            // Write relation rna - mirna
                            writeRelationLine(IS___RNA___TRANSCRIPT.name(), node.getUid(), uid);
                            isRelated = true;
                        }
                    }
                }

                if (!isRelated && StringUtils.isNotEmpty(node.getName()) && node.getName().startsWith("miR")) {
                    String id = "hsa-" + node.getName().toLowerCase();
                    Long miRnaUid = csv.getLong(id, MIRNA.name());
                    if (miRnaUid != null) {
                        // Write relation rna - mirna
                        writeRelationLine(IS___RNA___MIRNA.name(), node.getUid(), miRnaUid);
                    }
                }
            }
        }

        @Override
        public void processNodes(List<Node> nodes) throws IOException {
            BufferedWriter bw;
            for (Node node : nodes) {
                if (CollectionUtils.isNotEmpty(node.getLabels())) {
                    bw = builder.getCsvInfo().getWriter(node.getLabels().get(0).name());

                    if (StringUtils.isNotEmpty(node.getName())) {
//                        if (node.getLabels().contains(PROTEIN)) {
//                            // Complete node proteins
//                            node = builder.completeProteinNode(node);
//                        } else
                        if (node.getLabels().contains(DNA)) {
                            // Save save gene nodes to process further, in the post-processing phase
                            dnaNodes.add(node);
                            continue;
                        } else if (node.getLabels().contains(RNA)) {
                            // Save save miRNA nodes to process further, in the post-processing phase
                            rnaNodes.add(node);
                            continue;
                        }
                    }

                    // Write node to CSV file
                    bw.write(builder.getCsvInfo().nodeLine(node));
                    bw.newLine();
                }
            }
        }

        @Override
        public void processRelations(List<Relation> relations) throws IOException {
            BufferedWriter bw;
            for (Relation relation : relations) {
                String id = relation.getLabel() + FILENAME_SEPARATOR + relation.getOrigLabel() + FILENAME_SEPARATOR
                        + relation.getDestLabel();
                bw = builder.getCsvInfo().getWriter(id);
                if (bw == null) {
                    logger.info("BioPAX relationship not yet supported {}", id);
                } else {
                    // Write relation to CSV file
                    bw.write(builder.getCsvInfo().relationLine(relation.getOrigUid(), relation.getDestUid()));
                    bw.newLine();
                }
            }
        }

        private List<String> getGeneNameFromDnaNode(Node dnaNode) {
            String dnaName = dnaNode.getName();
            String xrefId = dnaNode.getAttributes().getString("xrefIds");

            List<String> names = new ArrayList<>();

            if (StringUtils.isNotEmpty(dnaName)) {
                // Name
                String input = dnaName.replace("/", ",").replace("(", "").replace(")", "");
                if (!input.contains(",") && (!input.contains("genes") && !input.contains("Genes"))) {
                    // Only one gene
                    input = input.replace("gene", "").replace("Gene", "").trim();
                    if (!input.contains(" ")) {
                        names.add(input);
                    }
                } else {
                    // Multiple genes
                    input = input.replace("Genes", "").replace("genes", "").replace("Gene", "").replace("gene", "").replace(" ", "").trim();
                    String[] fields = input.split(",");
                    for (int i = 0; i < fields.length; i++) {
                        if (i > 0) {
                            if (StringUtils.isNumeric(fields[i])) {
                                names.add(fields[0].trim().substring(0, fields[0].length() - 1) + fields[i]);
                            } else {
                                if (fields[i].length() == 1) {
                                    names.add(fields[0].trim().substring(0, fields[0].length() - 1) + fields[i]);
                                } else {
                                    names.add(fields[i].trim());
                                }
                            }
                        } else {
                            names.add(fields[i].trim());
                        }
                    }
                }
            } else if (StringUtils.isNotEmpty(xrefId)) {
                names.add(xrefId);
            }

            return names;
        }
    }

    //-------------------------------------------------------------------------
    // P R I V A T E     M E T H O D S
    //-------------------------------------------------------------------------

    private void buildInternalConfigNode() throws IOException {
        Node node = new Node(0, null, null, INTERNAL_CONNFIG);
        node.addAttribute("uidCounter", csv.getUid());
        writeNodeLine(node);
    }

    private Long processGene(String geneId, String geneName) throws IOException {
        if (StringUtils.isEmpty(geneId)) {
            logger.info("Skip processing gene, (id, name) = (" + geneId + ", " + geneName + ")");
            return null;
        }
        Long geneUid = csv.getGeneUid(geneId);
        if (geneUid == null) {
            Node node;
            Gene gene = csv.getGene(geneId);
            if (gene != null) {
                // Create gene node
                node = createGeneNode(gene);
                // Save gene UID
                csv.saveGeneUid(geneId, node.getUid());
            } else {
                // Special case, gene not annotated!
                logger.info("Processing gene {}, {}, unknown!!", geneId, geneName);
                node = new Node(csv.getAndIncUid(), geneId, geneName, GENE);
                // Save gene UID
                csv.saveUnknownGeneUid(geneId, geneName, node.getUid());
            }

            // Write gene node into the CSV file
            BufferedWriter bw = csv.getWriter(GENE.name());
            bw.write(csv.nodeLine(node));
            bw.newLine();

            return node.getUid();
        } else {
            return geneUid;
        }
    }

    private Long processProtein(String proteinId, String proteinName) throws IOException {
        if (StringUtils.isEmpty(proteinId)) {
            return null;
        }
        Long proteinUid = csv.getProteinUid(proteinId);

        if (proteinUid == null) {
            Node node;
            Entry protein = csv.getProtein(proteinId);
            if (protein != null) {
                // Create protein node
                node = createProteinNode(protein);
                // Save protein UID
                csv.saveProteinUid(proteinId, node.getUid());
            } else {
                // Special case, protein not annotated!
                logger.info("Processing protein {}, {}, unknown!!", proteinId, proteinName);
                node = new Node(csv.getAndIncUid(), proteinId, proteinName, PROTEIN);
                // Save protein UID
                csv.saveUnknownProteinUid(proteinId, proteinName, node.getUid());
            }
            // Write protein node into the CSV file
            writeNodeLine(node);
            proteinUid = node.getUid();
        }

        return proteinUid;
    }

    private Long processTranscript(String transcriptId) {
        if (StringUtils.isEmpty(transcriptId)) {
            return null;
        }
        return csv.getLong(transcriptId, TRANSCRIPT.name());
    }

    private Node createGeneNode(Gene gene) throws IOException {
        return createGeneNode(gene, csv.getAndIncUid());
    }

    private Node createGeneNode(Gene gene, Long uid) throws IOException {
        Node n;

//        logger.info("----> creating gene node: " + gene.getId() + ", " + gene.getName());

        // Create gene node and save gene UID
        Node geneNode = NodeBuilder.newNode(uid, gene);

        // Model transcripts
        if (CollectionUtils.isNotEmpty(gene.getTranscripts())) {
            for (Transcript transcript: gene.getTranscripts()) {
                Long transcrUid = csv.getLong(transcript.getId(), TRANSCRIPT.name());
                if (transcrUid == null) {
                    // Create gene node and write the CSV file
                    Node transcrNode = createTranscriptNode(transcript);
                    transcrUid = transcrNode.getUid();
                    writeNodeLine(transcrNode);
                }

                // Write gene-transcript relation
                writeRelationLine(HAS___GENE___TRANSCRIPT.name(), uid, transcrUid);
            }
        }

        // Model miRNA gene and mature miRNA
        if (gene.getMirna() != null) {
            MiRnaGene miRna = gene.getMirna();
            Long miRnaUid = csv.getLong(miRna.getId(), MIRNA.name());
            if (miRnaUid == null) {
                // Create miRNA node and write the CSV file
                miRnaUid = csv.getAndIncUid();
                Node miRnaNode = NodeBuilder.newNode(miRnaUid, miRna);

                csv.putLong(miRna.getId(), MIRNA.name(), miRnaUid);
                if (StringUtils.isNotEmpty(miRna.getAccession())) {
                    csv.putLong(miRna.getAccession(), MIRNA.name(), miRnaUid);
                }
                writeNodeLine(miRnaNode);

                // Mature miRna
                if (CollectionUtils.isNotEmpty(miRna.getMatures())) {
                    for (MiRnaMature mature : miRna.getMatures()) {
                        Long matureUid = csv.getLong(mature.getId(), MIRNA_MATURE.name());
                        if (matureUid == null) {
                            matureUid = csv.getAndIncUid();
                            // Create mature miRNA node and write the CSV file
                            Node matureNode = NodeBuilder.newNode(matureUid, mature);
                            csv.putLong(mature.getId(), MIRNA_MATURE.name(), matureUid);
                            writeNodeLine(matureNode);
                        }
                        // Write mirna-mature relation
                        writeRelationLine(MATURE___MIRNA___MIRNA_MATURE.name(), miRnaUid, matureUid);
                    }
                }
            }
            // Write gene-mirna relation
            writeRelationLine(IS___GENE___MIRNA.name(), uid, miRnaUid);
        }

        // Model gene annotation: drugs, diseases,...
        if (gene.getAnnotation() != null) {
            // Model drugs
            if (CollectionUtils.isNotEmpty(gene.getAnnotation().getDrugs())) {
                for (GeneDrugInteraction drugInteraction : gene.getAnnotation().getDrugs()) {
                    // Gene drug interaction node
                    n = NodeBuilder.newNode(csv.getAndIncUid(), drugInteraction);
                    updateCSVFiles(uid, n, ANNOTATION___GENE___GENE_DRUG_INTERACTION.name());

                    // Drug node
                    Long drugUid = csv.getLong(drugInteraction.getDrugName(), DRUG.name());
                    if (drugUid == null) {
                        Node drugNode = new Node(csv.getAndIncUid(), drugInteraction.getDrugName(), drugInteraction.getChemblId(), DRUG);
                        drugNode.getLabels().add(PHYSICAL_ENTITY);
                        writeNodeLine(drugNode);

                        // Save drug and gene-drug UIDs
                        drugUid = drugNode.getUid();
                        csv.putLong(drugNode.getName(), DRUG.name(), drugUid);
                        csv.putLong(drugNode.getId(), DRUG.name(), drugUid);
                    }
                    writeRelationLine(ANNOTATION___DRUG___GENE_DRUG_INTERACTION.name(), drugUid, n.getUid());
                }
            }

            // Model gene trait association (diseases)
            if (CollectionUtils.isNotEmpty(gene.getAnnotation().getDiseases())) {
                Set<String> done = new HashSet<>();
                for (GeneTraitAssociation disease : gene.getAnnotation().getDiseases()) {
                    String diseaseId = disease.getId() + (disease.getHpo() != null ? "_" + disease.getHpo() : "");
                    // It is possible that the same trait association is several times in the list
                    if (!done.contains(diseaseId)) {
                        Long diseaseUid = csv.getLong(diseaseId, GENE_TRAIT_ASSOCIATION.name());
                        if (diseaseUid == null) {
                            n = NodeBuilder.newNode(csv.getAndIncUid(), disease);
                            updateCSVFiles(uid, n, ANNOTATION___GENE___GENE_TRAIT_ASSOCIATION.name());

                            csv.putLong(diseaseId, GENE_TRAIT_ASSOCIATION.name(), n.getUid());
                        } else {
                            // create gene-disease relation
                            writeRelationLine(ANNOTATION___GENE___GENE_TRAIT_ASSOCIATION.name(), uid, diseaseUid);
                        }
                        done.add(diseaseId);
                    }
                }
            }

            // Gene expression
            if (CollectionUtils.isNotEmpty(gene.getAnnotation().getExpression())) {
                for (Expression expression : gene.getAnnotation().getExpression()) {
                    n = NodeBuilder.newNode(csv.getAndIncUid(), expression);
                    // Write gene and expression relation
                    updateCSVFiles(uid, n, ANNOTATION___GENE___GENE_EXPRESSION.name());
                }
            }
        }

        // Xrefs
        Set<Xref> xrefSet = new HashSet<>();
        String dbName = "ensembl".equals(gene.getSource()) ? "Ensembl" : "RefSeq";
        xrefSet.add(new Xref(gene.getId(), dbName, dbName));
        xrefSet.add(new Xref(gene.getName(), dbName, dbName));
        if (CollectionUtils.isNotEmpty(gene.getTranscripts())) {
            for (Transcript transcript : gene.getTranscripts()) {
                if (CollectionUtils.isNotEmpty(transcript.getXrefs())) {
                    for (Xref xref : transcript.getXrefs()) {
                        xrefSet.add(xref);
                    }
                }
            }
        }
        Iterator<Xref> it = xrefSet.iterator();
        while (it.hasNext()) {
            Xref xref = it.next();
            Long xrefUid = csv.getLong(xref.getDbName() + "." + xref.getId(), XREF.name());
            if (xrefUid == null) {
                n = NodeBuilder.newNode(csv.getAndIncUid(), xref);
                writeNodeLine(n);

                xrefUid = n.getUid();
                csv.putLong(xref.getDbName() + "." + xref.getId(), XREF.name(), xrefUid);
            }
            writeRelationLine(ANNOTATION___GENE___XREF.name(), uid, xrefUid);
        }

        return geneNode;
    }

    private Node completeProteinNode(Node node) throws IOException {
        Entry protein = csv.getProtein(node.getName());
        if (protein == null) {
            return node;
        }

        // Create protein node and add the attributes from the previous node
        Node newNode = createProteinNode(protein, node.getUid());
        for (String key: node.getAttributes().keySet()) {
            newNode.addAttribute(key, node.getAttributes().get(key));
        }

        String geneId = csv.getGeneCache().getPrimaryId(node.getName());
        if (StringUtils.isNotEmpty(geneId)) {
            processGene(geneId, geneId);
        }

        return newNode;
    }

    private Node createProteinNode(Entry protein) throws IOException {
        return createProteinNode(protein, csv.getAndIncUid());
    }

    private Node createProteinNode(Entry protein, Long uid) throws IOException {
        Node n;

        // Create protein node and save protein UID
        Node proteinNode = NodeBuilder.newNode(uid, protein);

        // Model protein keywords
        if (CollectionUtils.isNotEmpty(protein.getKeyword())) {
            for (KeywordType keyword: protein.getKeyword()) {
                Long kwUid = csv.getLong(keyword.getId(), PROTEIN_KEYWORD.name());
                if (kwUid == null) {
                    n = new Node(csv.getAndIncUid(), keyword.getId(), keyword.getValue(), PROTEIN_KEYWORD);
                    updateCSVFiles(uid, n, ANNOTATION___PROTEIN___PROTEIN_KEYWORD.name());
                    csv.putLong(keyword.getId(), PROTEIN_KEYWORD.name(), n.getUid());
                } else {
                    // Create protein - protein keyword relation
                    writeRelationLine(ANNOTATION___PROTEIN___PROTEIN_KEYWORD.name(), uid, kwUid);
                }
            }
        }

        // Model protein features
        if (CollectionUtils.isNotEmpty(protein.getFeature())) {
            for (FeatureType feature: protein.getFeature()) {
                if (StringUtils.isNotEmpty(feature.getId())) {
                    Long featureUid = csv.getLong(feature.getId(), PROTEIN_FEATURE.name());
                    if (featureUid == null) {
                        n = NodeBuilder.newNode(csv.getAndIncUid(), feature);
                        updateCSVFiles(uid, n, ANNOTATION___PROTEIN___PROTEIN_FEATURE.name());
                        csv.putLong(feature.getId(), PROTEIN_FEATURE.name(), n.getUid());
                    } else {
                        // Create protein - protein feature relation
                        writeRelationLine(ANNOTATION___PROTEIN___PROTEIN_FEATURE.name(), uid, featureUid);
                    }
                } else {
                    n = NodeBuilder.newNode(csv.getAndIncUid(), feature);
                    updateCSVFiles(uid, n, ANNOTATION___PROTEIN___PROTEIN_FEATURE.name());
                }
            }
        }

        // Model Xrefs
        if (CollectionUtils.isNotEmpty(protein.getDbReference())) {
            Set<String> done = new HashSet<>();
            for (DbReferenceType dbRef: protein.getDbReference()) {
                if ("Ensembl".equals(dbRef.getType())) {
                    for (PropertyType propertyType : dbRef.getProperty()) {
                        if ("protein sequence ID".equals(propertyType.getType())) {
                            String xrefId = dbRef.getType() + "." + propertyType.getValue();
                            // In the list, one db reference can be multiple times
                            if (!done.contains(xrefId)) {
                                n = NodeBuilder.newNode(csv.getAndIncUid(), dbRef);
                                n.setId(propertyType.getValue());
                                updateCSVFiles(uid, n, ANNOTATION___PROTEIN___XREF.name());

                                done.add(xrefId);
                            }
                            break;
                        }
                    }
                }
                String xrefId = dbRef.getType() + "." + dbRef.getId();
                if (!done.contains(xrefId)) {
                    n = NodeBuilder.newNode(csv.getAndIncUid(), dbRef);
                    updateCSVFiles(uid, n, ANNOTATION___PROTEIN___XREF.name());

                    done.add(xrefId);
                }
            }
            if (CollectionUtils.isNotEmpty(protein.getAccession())) {
                for (String acc: protein.getAccession()) {
                    String xrefId = "UniProtKB." + acc;
                    if (!done.contains(xrefId)) {
                        n = new Node(csv.getAndIncUid(), acc, null, Node.Label.XREF);
                        n.addAttribute("dbName", "UniProtKB");
                        updateCSVFiles(uid, n, ANNOTATION___PROTEIN___XREF.name());

                        done.add(xrefId);
                    }
                }
            }
        }

        // Return node
        return proteinNode;
    }

    private Node createTranscriptNode(Transcript transcript) throws IOException {
        return createTranscriptNode(transcript, csv.getAndIncUid());
    }

    private Node createTranscriptNode(Transcript transcript, Long uid) throws IOException {
        // Create transcript node and save transcript UId
        Node node = NodeBuilder.newNode(uid, transcript);
        csv.putLong(transcript.getId(), TRANSCRIPT.name(), uid);
        if (StringUtils.isNotEmpty(transcript.getId()) && transcript.getId().contains(".")) {
            String[] split = transcript.getId().split("\\.");
            if (csv.getLong(split[0], TRANSCRIPT.name()) == null) {
                csv.putLong(split[0], TRANSCRIPT.name(), uid);
            }
        }

        Node n;

        // Get protein, remember that all proteins were created before genes/transcripts
        if (StringUtils.isNotEmpty(transcript.getProteinId())) {
            String dbName = "refseq".equals(transcript.getSource()) ? "RefSeq" : "Ensembl";
            String proteinId = csv.getProteinCache().getPrimaryId(dbName + "." + transcript.getProteinId());
            if (proteinId != null) {
                Long proteinUid = csv.getLong(proteinId, PROTEIN.name());
                if (proteinUid != null) {
                    // Write transcript-protein relation
                    writeRelationLine(IS___TRANSCRIPT___PROTEIN.name(), uid, proteinUid);
                }
            }
        }

        // Model exon
        if (CollectionUtils.isNotEmpty(transcript.getExons())) {
            for (Exon exon : transcript.getExons()) {
                Long exonUid = csv.getLong(exon.getId(), Node.Label.EXON.name());
                if (exonUid == null) {
                    n = createExonNode(exon, transcript.getSource());
                    exonUid = n.getUid();
                }
                // Add relation transcript-ontology to CSV file
                writeRelationLine(HAS___TRANSCRIPT___EXON.name(), uid, exonUid);
            }
        }

        // TFBS
        if (CollectionUtils.isNotEmpty(transcript.getTfbs())) {
            for (TranscriptTfbs tfbs: transcript.getTfbs()) {
                n = NodeBuilder.newNode(csv.getAndIncUid(), tfbs);
                updateCSVFiles(uid, n, ANNOTATION___TRANSCRIPT___TFBS.name());
            }
        }

        if (transcript.getAnnotation() != null) {
            // Model constraint
            if (CollectionUtils.isNotEmpty(transcript.getAnnotation().getConstraints())) {
                Set<String> done = new HashSet<>();
                for (Constraint constraint : transcript.getAnnotation().getConstraints()) {
                    String constraintId = constraint.getName() + "." + constraint.getSource() + "." + constraint.getValue();
                    if (!done.contains(constraintId)) {
                        Long constraintUid = checkConstraint(constraint);
                        writeRelationLine(ANNOTATION___TRANSCRIPT___TRANSCRIPT_CONSTRAINT_SCORE.name(), uid, constraintUid);
                        done.add(constraintId);
                    }
                }
            }

            // Model feature ontology term annotation
            if (CollectionUtils.isNotEmpty(transcript.getAnnotation().getOntologies())) {
                for (FeatureOntologyTermAnnotation ontology : transcript.getAnnotation().getOntologies()) {
                    // Create the feature ontology term annotation node and the relation with the transcript to CSV file
                    n = createFeatureOntologyTermAnnotationNode(ontology);
                    writeRelationLine(ANNOTATION___TRANSCRIPT___FEATURE_ONTOLOGY_TERM_ANNOTATION.name(), uid, n.getUid());

//                    Long ontologyUid = csv.getLong(ontology.getId(), Node.Label.FEATURE_ONTOLOGY_TERM_ANNOTATION.name());
//                    if (ontologyUid == null) {
//                        n = createFeatureOntologyTermAnnotationNode(ontology);
//                        ontologyUid = n.getUid();
//                    }
//                    // Add relation transcript-ontology to CSV file
//                    pwRel.println(uid + CsvInfo.SEPARATOR + ontologyUid);
                }
            }
        }

        // Model Xrefs
        if (CollectionUtils.isNotEmpty(transcript.getXrefs())) {
            Set<String> done = new HashSet<>();
            for (Xref xref: transcript.getXrefs()) {
                String xrefId = xref.getDbName() + "." + xref.getId();
                // In the list, one xref can be multiple times :(
                if (!done.contains(xrefId)) {
                    Long xrefUid = csv.getLong(xrefId, XREF.name());
                    if (xrefUid == null) {
                        n = NodeBuilder.newNode(csv.getAndIncUid(), xref);
                        writeNodeLine(n);
                        xrefUid = n.getUid();
                        csv.putLong(xrefId, XREF.name(), xrefUid);
                    }
                    writeRelationLine(ANNOTATION___TRANSCRIPT___XREF.name(), uid, xrefUid);
                    done.add(xrefId);
                }
            }
        }


        return node;
    }

    private Node createExonNode(Exon exon, String source) throws IOException {
        return createExonNode(exon, csv.getAndIncUid(), source);
    }

    private Node createExonNode(Exon exon, Long uid, String source) throws IOException {
        // Create exon node and save UId and wrrite CSV file
        Node node = NodeBuilder.newNode(uid, exon, source);
        writeNodeLine(node);
        csv.putLong(exon.getId(), Node.Label.EXON.name(), uid);

        return node;
    }

    private Node createFeatureOntologyTermAnnotationNode(FeatureOntologyTermAnnotation ontology) throws IOException {
        return createFeatureOntologyTermAnnotationNode(ontology, csv.getAndIncUid());
    }

    private Node createFeatureOntologyTermAnnotationNode(FeatureOntologyTermAnnotation ontology, Long uid) throws IOException {
        // Create ontology node and save UId and wrrite CSV file
        Node node = NodeBuilder.newNode(uid, ontology);
        writeNodeLine(node);
        csv.putLong(ontology.getId(), FEATURE_ONTOLOGY_TERM_ANNOTATION.name(), uid);

        if (CollectionUtils.isNotEmpty(ontology.getEvidence())) {
            for (AnnotationEvidence annotationEvidence : ontology.getEvidence()) {
                Node n = NodeBuilder.newNode(csv.getAndIncUid(), annotationEvidence);
                // Write evidence node and ontology-evidence relation
                updateCSVFiles(uid, n, HAS___FEATURE_ONTOLOGY_TERM_ANNOTATION___TRANSCRIPT_ANNOTATION_EVIDENCE.name());
            }
        }

        return node;
    }

    private Long processVariant(Variant variant) throws IOException, NoSuchAlgorithmException {
        Node variantNode = null;

        Long variantUid = csv.getLong(variant.toStringSimple(), VARIANT.name());
        if (variantUid == null) {
            variantNode = createVariantNode(variant);
            variantUid = variantNode.getUid();
            csv.putLong(variant.toStringSimple(), VARIANT.name(), variantUid);
        }

        return variantUid;
    }

    private Node createVariantNode(Variant variant) throws IOException, NoSuchAlgorithmException {
        return createVariantNode(variant, csv.getAndIncUid());
    }

    private Node createVariantNode(Variant variant, Long varUid) throws IOException, NoSuchAlgorithmException {
        Node varNode = NodeBuilder.newNode(varUid, variant);
        writeNodeLine(varNode);

        Node node;

        // Create study related nodes
        createStudyRelatedNodes(varUid, variant);

        //        // Processing clinical variants
//        logger.info("Processing clinical variants...");
//        start = System.currentTimeMillis();
//        buildVariants(clinicalVariantFile.toPath());
//        logger.info("Processing clinical variants done in {} s", (System.currentTimeMillis() - start) / 1000);

        // Structural variant
        if (variant.getSv() != null) {
            node = NodeBuilder.newNode(csv.getAndIncUid(), variant.getSv(), csv);
            updateCSVFiles(varUid, node, HAS___VARIANT___STRUCTURAL_VARIANT.name());
        }

        // Annotation management
        if (variant.getAnnotation() != null) {
            VariantAnnotation annotation = variant.getAnnotation();
            // Consequence types
            if (CollectionUtils.isNotEmpty(annotation.getConsequenceTypes())) {
                // Consequence type nodes
                for (ConsequenceType ct : annotation.getConsequenceTypes()) {
                    Node ctNode = NodeBuilder.newNode(csv.getAndIncUid(), ct);
                    updateCSVFiles(varUid, ctNode, ANNOTATION___VARIANT___VARIANT_CONSEQUENCE_TYPE.name());

                    // Gene
                    Long geneUid = processGene(ct.getEnsemblGeneId(), ct.getGeneName());
                    if (geneUid != null) {
                        // Relation: consequence type - gene
                        writeRelationLine(ANNOTATION___VARIANT_CONSEQUENCE_TYPE___GENE.name(), ctNode.getUid(), geneUid);
                    }

                    // Transcript
                    Long transcriptUid = processTranscript(ct.getEnsemblTranscriptId());
                    if (transcriptUid != null) {
                        // Relation: consequence type - transcript
                        writeRelationLine(ANNOTATION___VARIANT_CONSEQUENCE_TYPE___TRANSCRIPT.name(), ctNode.getUid(), transcriptUid);
                    } else {
                        if (geneUid != null) {
                            logger.info("Transcript UID is null for {} (gene {}), maybe version mismatch!!"
                                            + " Gene vs variant annotation!!!", ct.getEnsemblTranscriptId(),
                                    ct.getEnsemblGeneId());
                        }
                    }

                    // SO_TERM
                    if (CollectionUtils.isNotEmpty(ct.getSequenceOntologyTerms())) {
                        for (SequenceOntologyTerm so : ct.getSequenceOntologyTerms()) {
                            String soId = so.getAccession();
                            if (soId != null) {
                                Long soUid = csv.getLong(soId, SO_TERM.name());
                                if (soUid == null) {
                                    Node soNode = new Node(csv.getAndIncUid(), so.getAccession(), so.getName(), SO_TERM);
                                    updateCSVFiles(ctNode.getUid(), soNode, ANNOTATION___VARIANT_CONSEQUENCE_TYPE___SO_TERM
                                            .name());
                                    csv.putLong(soId, SO_TERM.name(), soNode.getUid());
                                } else {
                                    // Relation: consequence type - so
                                    writeRelationLine(ANNOTATION___VARIANT_CONSEQUENCE_TYPE___SO_TERM.name(), ctNode.getUid(), soUid);
                                }
                            }
                        }
                    }

                    // Protein variant annotation: substitution scores, keywords and features
                    if (ct.getProteinVariantAnnotation() != null) {
                        // Protein variant annotation node
                        Node pVANode = NodeBuilder.newNode(csv.getAndIncUid(), ct.getProteinVariantAnnotation());
                        updateCSVFiles(ctNode.getUid(), pVANode, ANNOTATION___VARIANT_CONSEQUENCE_TYPE___PROTEIN_VARIANT_ANNOTATION.name());

                        // Protein relationship management
                        Long protUid = processProtein(ct.getProteinVariantAnnotation().getUniprotAccession(),
                                ct.getProteinVariantAnnotation().getUniprotName());
                        if (protUid != null) {
                            // Relation: protein variant annotation - protein
                            writeRelationLine(ANNOTATION___PROTEIN_VARIANT_ANNOTATION___PROTEIN.name(), pVANode.getUid(), protUid);
                        }

                        // Protein substitution scores
                        if (CollectionUtils.isNotEmpty(ct.getProteinVariantAnnotation().getSubstitutionScores())) {
                            for (Score score : ct.getProteinVariantAnnotation().getSubstitutionScores()) {
                                Node scoreNode = NodeBuilder.newNode(csv.getAndIncUid(), score, PROTEIN_SUBSTITUTION_SCORE);
                                updateCSVFiles(pVANode.getUid(), scoreNode,
                                        ANNOTATION___PROTEIN_VARIANT_ANNOTATION___PROTEIN_SUBSTITUTION_SCORE.name());
                            }
                        }
                    }
                }
            }

            // HGVS
            if (CollectionUtils.isNotEmpty(annotation.getHgvs())) {
                for (String hgv : annotation.getHgvs()) {
                    // HGV node
                    node = new Node(csv.getAndIncUid(), hgv, hgv, HGV);
                    updateCSVFiles(varUid, node, ANNOTATION___VARIANT___HGV.name());
                }
            }

            // Variant population frequencies
            if (CollectionUtils.isNotEmpty(annotation.getPopulationFrequencies())) {
                for (PopulationFrequency popFreq : variant.getAnnotation().getPopulationFrequencies()) {
                    // Population frequency node
                    node = NodeBuilder.newNode(csv.getAndIncUid(), popFreq);
                    updateCSVFiles(varUid, node, ANNOTATION___VARIANT___VARIANT_POPULATION_FREQUENCY.name());
                }
            }

            // Variant conservation scores
            if (CollectionUtils.isNotEmpty(annotation.getConservation())) {
                for (Score score : annotation.getConservation()) {
                    // Conservation node
                    node = NodeBuilder.newNode(csv.getAndIncUid(), score, VARIANT_CONSERVATION_SCORE);
                    updateCSVFiles(varUid, node, ANNOTATION___VARIANT___VARIANT_CONSERVATION_SCORE.name());
                }
            }

            // Clinical evidence
            if (CollectionUtils.isNotEmpty(annotation.getTraitAssociation())) {
                for (EvidenceEntry evidence : annotation.getTraitAssociation()) {
                    // Clinical evidence node
                    node = NodeBuilder.newNode(csv.getAndIncUid(), evidence, CLINICAL_EVIDENCE, csv);
                    updateCSVFiles(varUid, node, ANNOTATION___VARIANT___CLINICAL_EVIDENCE.name());
                }
            }

            // Variant functional scores
            if (CollectionUtils.isNotEmpty(annotation.getFunctionalScore())) {
                for (Score score : annotation.getFunctionalScore()) {
                    // Functional score node
                    node = NodeBuilder.newNode(csv.getAndIncUid(), score, VARIANT_FUNCTIONAL_SCORE);
                    updateCSVFiles(varUid, node, ANNOTATION___VARIANT___VARIANT_FUNCTIONAL_SCORE.name());
                }
            }

            // Repeat
            if (CollectionUtils.isNotEmpty(annotation.getRepeat())) {
                for (Repeat repeat : annotation.getRepeat()) {
                    if (StringUtils.isNotEmpty(repeat.getId())) {
                        String repeatId = repeat.getId() + "." + repeat.getChromosome() + "." + repeat.getStart() + "." + repeat.getEnd();
                        Long repeatUid = csv.getLong(repeatId, REPEAT.name());
                        if (repeatUid == null) {
                            node = NodeBuilder.newNode(csv.getAndIncUid(), repeat);
                            writeNodeLine(node);
                            repeatUid = node.getUid();
                            csv.putLong(repeatId, REPEAT.name(), repeatUid);
                        }
                        writeRelationLine(ANNOTATION___VARIANT___REPEAT.name(), varUid, repeatUid);
                    }
                }
            }

            // Cytoband
            if (CollectionUtils.isNotEmpty(annotation.getCytoband())) {
                for (Cytoband cytoband : annotation.getCytoband()) {
                    if (StringUtils.isNotEmpty(cytoband.getName())) {
                        Long cytobandUid = csv.getLong(cytoband.getName(), CYTOBAND.name());
                        if (cytobandUid == null) {
                            node = NodeBuilder.newNode(csv.getAndIncUid(), cytoband);
                            writeNodeLine(node);
                            cytobandUid = node.getUid();
                            csv.putLong(cytoband.getName(), CYTOBAND.name(), cytobandUid);
                        }
                        writeRelationLine(ANNOTATION___VARIANT___CYTOBAND.name(), varUid, cytobandUid);
                    }
                }
            }

            // Variant drug interaction
            if (CollectionUtils.isNotEmpty(annotation.getDrugs())) {
                for (Drug variantDrugInteraction : annotation.getDrugs()) {
                    node = NodeBuilder.newNode(csv.getAndIncUid(), variantDrugInteraction);
                    updateCSVFiles(varUid, node, ANNOTATION___VARIANT___VARIANT_DRUG_INTERACTION.name());
                }
            }

            // Constraints
            if (CollectionUtils.isNotEmpty(annotation.getGeneConstraints())) {
                Set<String> done = new HashSet<>();
                for (Constraint constraint : annotation.getGeneConstraints()) {
                    String constraintId = constraint.getName() + "." + constraint.getSource() + "." + constraint.getValue();
                    if (!done.contains(constraintId)) {
                        Long constraintUid = checkConstraint(constraint);
                        writeRelationLine(ANNOTATION___VARIANT___TRANSCRIPT_CONSTRAINT_SCORE.name(), varUid, constraintUid);
                        done.add(constraintId);
                    }
                }
            }
        }

        return varNode;
    }

    private void createStudyRelatedNodes(Long varUid, Variant variant) throws IOException {
        if (CollectionUtils.isEmpty(variant.getStudies())) {
            return;
        }

        for (StudyEntry studyEntry : variant.getStudies()) {
            // Process file data
            if (CollectionUtils.isNotEmpty(studyEntry.getFiles())) {
                for (FileEntry fileEntry : studyEntry.getFiles()) {
                    Long fileUid = csv.getLong(fileEntry.getFileId(), VARIANT_FILE.name());
                    if (fileUid != null) {
                        Node fileDataNode = new Node(csv.getAndIncUid(), fileEntry.getFileId(), "", VARIANT_FILE_DATA);
                        if (MapUtils.isNotEmpty(fileEntry.getData())) {
                            for (Map.Entry<String, String> entry : fileEntry.getData().entrySet()) {
                                fileDataNode.addAttribute(entry.getKey(), entry.getValue());
                            }
                        }
                        // Write variant file data node and the relation with variant
                        updateCSVFiles(varUid, fileDataNode, DATA___VARIANT___VARIANT_FILE_DATA.name());

                        // and the relation with file
                        writeRelationLine(DATA___VARIANT_FILE___VARIANT_FILE_DATA.name(), fileUid, fileDataNode.getUid());
                    } else {
                        logger.info("File ID " + fileEntry.getFileId() + " for variant ID " + variant.toString() + ". Can't make relation: "
                                + " File - Variant File Data");
                    }
                }
            }

            // Process sample data
            if (CollectionUtils.isNotEmpty(studyEntry.getSamples())) {
                for (int sampleIndex = 0; sampleIndex < studyEntry.getSamples().size(); sampleIndex++) {
                    SampleEntry sampleEntry = studyEntry.getSamples().get(sampleIndex);
                    String sampleId = sampleIds.get(sampleIndex);
                    Long sampleUid = csv.getLong(sampleId, SAMPLE.name());
                    if (sampleUid != null) {
                        Node sampleDataNode = new Node(csv.getAndIncUid(), sampleId, "", VARIANT_SAMPLE_DATA);
                        if (CollectionUtils.isNotEmpty(sampleEntry.getData())) {
                            for (int i = 0; i < sampleEntry.getData().size(); i++) {
                                sampleDataNode.addAttribute(studyEntry.getSampleDataKeys().get(i), sampleEntry.getData().get(i));
                            }
                        }
                        // Write variant sample data node and the relation with variant
                        updateCSVFiles(varUid, sampleDataNode, DATA___VARIANT___VARIANT_SAMPLE_DATA.name());

                        // and the relation with sample
                        writeRelationLine(DATA___SAMPLE___VARIANT_SAMPLE_DATA.name(), sampleUid, sampleDataNode.getUid());

                        // and add relation variant file - sample
                        if (sampleEntry.getFileIndex() != null && CollectionUtils.isNotEmpty(studyEntry.getFiles())) {
                            if (sampleEntry.getFileIndex() < studyEntry.getFiles().size()) {
                                String fileId = studyEntry.getFiles().get(sampleEntry.getFileIndex()).getFileId();
                                Long fileUid = csv.getLong(fileId, VARIANT_FILE.name());
                                if (fileUid != null) {
                                    // First, check that this relation does not exist yet
                                    Long relUid = csv.getLong(fileId + "." + sampleId, HAS___VARIANT_FILE___SAMPLE.name());
                                    if (relUid == null) {
                                        // and the relation with sample
                                        writeRelationLine(HAS___VARIANT_FILE___SAMPLE.name(), fileUid, sampleUid);

                                        // Save this relation
                                        csv.putLong(fileId + "." + sampleId, HAS___VARIANT_FILE___SAMPLE.name(), 1);
                                    }
                                }
                            }
                        }

                    } else {
                        logger.info("Sample ID " + sampleEntry.getSampleId() + " for variant ID " + variant.toString() + ". Can't make "
                                + "relation: Sample - Variant Sample Data");
                    }
                }
            }
        }
    }


    private void processAdditionalNetwork(String additionalNeworkFilename) throws IOException {
        // Check file
        File addNetworkFile = Paths.get(additionalNeworkFilename).toFile();
        if (!addNetworkFile.exists()) {
            logger.info("Additional network file {} does not exist", additionalNeworkFilename);
            return;
        }

        ObjectMapper objectMapper = new ObjectMapper();
        Network network = objectMapper.readValue(addNetworkFile, Network.class);

        Map<Long, Long> nodeUidMap = new HashMap<>();

        // First, nodes
        if (CollectionUtils.isNotEmpty(network.getNodes())) {
            for (Node node: network.getNodes()) {
                if (CollectionUtils.isNotEmpty(node.getLabels())) {
                    Long uid = csv.getLong(node.getId(), node.getLabels().get(0).name());
                    if (uid == null) {
                        // Node does not exist in the !
                        nodeUidMap.put(node.getUid(), csv.getAndIncUid());
                        // Update UID and append node to the CSV file
                        node.setUid(nodeUidMap.get(node.getUid()));
                        writeNodeLine(node);
                    } else {
                        // Node already exists !!
                        nodeUidMap.put(node.getUid(), uid);
                    }
                }
            }
        }

        // Second, relations
        if (CollectionUtils.isNotEmpty(network.getRelations())) {
            for (Relation relation: network.getRelations()) {
                relation.setUid(csv.getAndIncUid());
                writeRelationLine(relation.getLabel().name(), nodeUidMap.get(relation.getOrigUid()), nodeUidMap.get(relation.getDestUid()));
            }
        }
    }

    private Long checkConstraint(Constraint constraint) throws IOException {
        Long constraintUid = csv.getLong(constraint.getName() + "." + constraint.getValue(), TRANSCRIPT_CONSTRAINT_SCORE.name());
        if (constraintUid == null) {
            Node node = NodeBuilder.newNode(csv.getAndIncUid(), constraint);
            writeNodeLine(node);
            constraintUid = node.getUid();
            csv.putLong(constraint.getName() + "." + constraint.getValue(), TRANSCRIPT_CONSTRAINT_SCORE.name(), constraintUid);
        }
        return constraintUid;
    }

    private void buildVariants(Path path) throws IOException, NoSuchAlgorithmException {
        // Reading file line by line, each line a JSON object
        BufferedReader reader = FileUtils.newBufferedReader(path);

        long counter = 0;
        ObjectMapper mapper = new ObjectMapper();
//        logger.info("Processing JSON file {}", file.getPath());
//        reader = FileUtils.newBufferedReader(file.toPath());
        String line = reader.readLine();
        while (line != null) {
            Variant variant = mapper.readValue(line, Variant.class);
            processVariant(variant);
            if (variant.getStrand().equals("-")) {
                System.out.println(variant.toStringSimple() + ", " + variant.getStrand());
            }

            // read next line
            line = reader.readLine();
            if (++counter % 5000 == 0) {
                logger.info("Parsing {} variants...", counter);
            }
        }
        reader.close();
        logger.info("Parsed {} variants from {}. Done!!!", counter, path);
    }


    private void writeNodeLine(Node node) throws IOException {
        BufferedWriter bw = csv.getWriter(node.getLabels().get(0).name());
        bw.write(csv.nodeLine(node));
        bw.newLine();
    }

    private void writeRelationLine(String filename, Long origUid, Long destUid) throws IOException {
        BufferedWriter bw = csv.getWriter(filename);
        bw.write(csv.relationLine(origUid, destUid));
        bw.newLine();
    }

    //-------------------------------------------------------------------------


    private void processMetadata(List<File> variantFiles) throws IOException {
        // For variant file info, variant sample format and sample nodes we have to read variant metadata files to know which attributes
        // are present
        if (CollectionUtils.isNotEmpty(variantFiles)) {
            for (File variantFile : variantFiles) {
                File metaFile = new File(variantFile.getAbsoluteFile() + ".meta.json");
                if (!metaFile.exists()) {
                    metaFile = new File(variantFile.getAbsoluteFile() + ".meta.json.gz");
                }
                if (!metaFile.exists()) {
                    continue;
                }

                // Read info, format and sample from metadata file
                ObjectMapper mapper = new ObjectMapper();
                VariantMetadata variantMetadata;
                try {
                    BufferedReader bufferedReader = FileUtils.newBufferedReader(metaFile.toPath());
                    String metadata = bufferedReader.readLine();
                    bufferedReader.close();
                    variantMetadata = mapper.readValue(metadata, VariantMetadata.class);
                } catch (IOException e) {
                    e.printStackTrace();
                    continue;
                }

                if (CollectionUtils.isNotEmpty(variantMetadata.getStudies())) {
                    // IMPORTANT: it considers only the first study
                    VariantStudyMetadata variantStudyMetadata = variantMetadata.getStudies().get(0);

                    // Create individual and family nodes and relations
                    for (Individual individual : variantStudyMetadata.getIndividuals()) {
                        Long individualUid = csv.getLong(individual.getId(), INDIVIDUAL.name());
                        if (individualUid == null) {
                            // First, check family
                            Long familyUid = null;
                            if (StringUtils.isNotEmpty(individual.getFamily())) {
                                familyUid = csv.getLong(individual.getFamily(), FAMILY.name());
                                if (familyUid == null) {
                                    Node familyNode = new Node(csv.getAndIncUid(), individual.getFamily(), individual.getFamily(), FAMILY);
                                    writeNodeLine(familyNode);
                                    familyUid = familyNode.getUid();
                                    csv.putLong(individual.getFamily(), FAMILY.name(), familyUid);
                                }
                            }

                            // Create individual node
                            Node individualNode = NodeBuilder.newNode(csv.getAndIncUid(), individual);
                            writeNodeLine(individualNode);
                            if (familyUid != null) {
                                writeRelationLine(HAS___FAMILY___INDIVIDUAL.name(), individualUid, familyUid);
                            }
                            individualUid = individualNode.getUid();

                            // Create sample nodes
                            if (CollectionUtils.isNotEmpty(individual.getSamples())) {
                                for (Sample sample : individual.getSamples()) {
                                    if (StringUtils.isNotEmpty(sample.getId())) {
                                        Node sampleNode = new Node(csv.getAndIncUid(), sample.getId(), sample.getId(), SAMPLE);
                                        if (MapUtils.isNotEmpty(sample.getAnnotations())) {
                                            for (Map.Entry<String, String> entry : sample.getAnnotations().entrySet()) {
                                                sampleNode.addAttribute(entry.getKey(), entry.getValue());
                                            }
                                        }
                                        writeNodeLine(sampleNode);
                                        writeRelationLine(HAS___INDIVIDUAL___SAMPLE.name(), individualUid, sampleNode.getUid());

                                        // Save sample UID
                                        csv.putLong(sample.getId(), SAMPLE.name(), sampleNode.getUid());
                                    }
                                }
                            }
                            csv.putLong(individual.getId(), INDIVIDUAL.name(), individualUid);
                        }
                    }

                    // Second loop, to fulfill mother-father relations
                    for (Individual individual : variantStudyMetadata.getIndividuals()) {
                        Long individualUid = csv.getLong(individual.getId(), INDIVIDUAL.name());
                        if (StringUtils.isNotEmpty(individual.getMother())) {
                            Long motherUid = csv.getLong(individual.getMother(), INDIVIDUAL.name());
                            if (motherUid != null) {
                                writeRelationLine(MOTHER_OF___INDIVIDUAL___INDIVIDUAL.name(), motherUid, individualUid);
                            }
                        }

                        if (StringUtils.isNotEmpty(individual.getFather())) {
                            Long fatherUid = csv.getLong(individual.getFather(), INDIVIDUAL.name());
                            if (fatherUid != null) {
                                writeRelationLine(FATHER_OF___INDIVIDUAL___INDIVIDUAL.name(), fatherUid, individualUid);
                            }
                        }
                    }

                    // File management
                    if (CollectionUtils.isNotEmpty(variantStudyMetadata.getFiles())) {
                        for (VariantFileMetadata fileMetadata : variantStudyMetadata.getFiles()) {
                            Long fileUid = csv.getLong(fileMetadata.getId(), VARIANT_FILE.name());
                            if (fileUid == null) {
                                Node fileNode = new Node(csv.getAndIncUid(), fileMetadata.getId(), fileMetadata.getPath(), VARIANT_FILE);
                                if (MapUtils.isNotEmpty(fileMetadata.getAttributes())) {
                                    for (Map.Entry<String, String> entry : fileMetadata.getAttributes().entrySet()) {
                                        fileNode.addAttribute(entry.getKey(), entry.getValue());
                                    }
                                }
                                writeNodeLine(fileNode);

                                // Save variant file UID
                                csv.putLong(fileMetadata.getId(), VARIANT_FILE.name(), fileNode.getUid());
                            }
                        }
                    }
                }
            }
        }
    }

    private void updateCSVFiles(long startUid, Node node, String relationType) throws IOException {
        // Update node CSV file
        writeNodeLine(node);

        // Update relation CSV file
        writeRelationLine(relationType, startUid, node.getUid());
    }

    public void buildGenes(Path genePath) throws IOException {
        GeneCache geneCache = csv.getGeneCache();
        BufferedReader reader = FileUtils.newBufferedReader(genePath);

        String jsonGene = reader.readLine();
        long geneCounter = 0;
        Set<String> idDone = new HashSet<>();
        while (jsonGene != null) {
            Gene gene = geneCache.getObjReader().readValue(jsonGene);
            String geneId = gene.getId();
            if (StringUtils.isNotEmpty(geneId)) {
                // RefSeq genes can have same ID but different names, we take the first one!!
                if (!idDone.contains(geneId)) {
                    idDone.add(geneId);
                    geneCounter++;
                    if (geneCounter % 5000 == 0) {
                        logger.info("Building {} genes...", geneCounter);
                    }
                    // Save gene
                    geneCache.saveObject(geneId, jsonGene);

                    // Save xrefs for that gene
                    geneCache.saveXref(geneId, geneId);
                    if (StringUtils.isNotEmpty(gene.getName())) {
                        geneCache.saveXref(gene.getSource() + "." + gene.getName(), geneId);
                    }

                    // Process gene to save to CSV file and save gene name, source and uid for post-processing DNA, disease panels...
                    Long geneUid = processGene(gene.getId(), gene.getName());
                    if (geneUid != null && StringUtils.isNotEmpty(gene.getName())) {
                        csv.putLong(gene.getName(), gene.getSource(), geneUid);
                    }
                }
            } else {
                String msg = "Skipping gene: missing gene ID from JSON file, gene name = " + gene.getName();
                System.out.println(msg);
                logger.info(msg);
            }

            // Next line
            jsonGene = reader.readLine();
        }

        // Second loop
        reader.close();
        reader = FileUtils.newBufferedReader(genePath);

        jsonGene = reader.readLine();
        while (jsonGene != null) {
            Gene gene = geneCache.getObjReader().readValue(jsonGene);
            if (gene.getAnnotation() != null && CollectionUtils.isNotEmpty(gene.getAnnotation().getMirnaTargets())) {
                Long geneUid = csv.getLong(gene.getId(), GENE.name());
                if (geneUid != null) {
                    for (MirnaTarget mirnaTarget : gene.getAnnotation().getMirnaTargets()) {
                        Long matureUid = csv.getLong(mirnaTarget.getSourceId(), MIRNA_MATURE.name());
                        if (matureUid != null) {
                            for (TargetGene target : mirnaTarget.getTargets()) {
                                Node targetNode = new Node(csv.getAndIncUid(), null, null, MIRNA_TARGET);
                                targetNode.addAttribute("experiment", target.getExperiment());
                                targetNode.addAttribute("evidence", target.getEvidence());
                                targetNode.addAttribute("pubmed", target.getPubmed());

                                // Write mirna target node into the CSV file
                                writeNodeLine(targetNode);

                                // Write mirna target - gene relation
                                writeRelationLine(ANNOTATION___GENE___MIRNA_TARGET.name(), geneUid, targetNode.getUid());

                                // And write mirna target - mirna mature relation
                                writeRelationLine(ANNOTATION___MIRNA_MATURE___MIRNA_TARGET.name(), matureUid, targetNode.getUid());
                            }
                        }
                    }
                }
            }

            // Next line
            jsonGene = reader.readLine();
        }

        logger.info("Building {} genes. Done.", geneCounter);

        reader.close();
    }

    public void buildProteins(Path proteinPath) throws IOException {
        ProteinCache proteinCache = csv.getProteinCache();
        BufferedReader reader = FileUtils.newBufferedReader(proteinPath);

        String jsonProtein = reader.readLine();
        long proteinCounter = 0;
        while (jsonProtein != null) {
            Entry protein = proteinCache.getObjReader().readValue(jsonProtein);
            if (CollectionUtils.isNotEmpty(protein.getAccession())) {
                proteinCounter++;
                if (proteinCounter % 5000 == 0) {
                    logger.info("Building {} proteins...", proteinCounter);
                }

                // Save protein in RocksDB
                String proteinAcc = protein.getAccession().get(0);
                proteinCache.saveObject(proteinAcc, jsonProtein);

                // Save protein xrefs
                proteinCache.saveXref(proteinAcc, proteinAcc);
                if (CollectionUtils.isNotEmpty(protein.getAccession())) {
                    for (String acc: protein.getAccession()) {
                        proteinCache.saveXref(acc, proteinAcc);
                    }
                }

                if (protein.getProtein() != null && protein.getProtein().getRecommendedName() != null
                        && CollectionUtils.isNotEmpty(protein.getProtein().getRecommendedName().getShortName())) {
                    for (EvidencedStringType shortName : protein.getProtein().getRecommendedName().getShortName()) {
                        if (StringUtils.isNotEmpty(shortName.getValue())) {
                            proteinCache.saveXref(shortName.getValue(), proteinAcc);
                        }
                    }
                }

                String proteinName = null;
                if (CollectionUtils.isNotEmpty(protein.getName())) {
                    proteinName = protein.getName().get(0);
                    for (String name: protein.getName()) {
                        proteinCache.saveXref(name, proteinAcc);
                    }
                }

                if (CollectionUtils.isNotEmpty(protein.getDbReference())) {
                    Set<String> done = new HashSet<>();
                    for (DbReferenceType dbRef: protein.getDbReference()) {
                        if ("Ensembl".equals(dbRef.getType())) {
                            for (PropertyType propertyType : dbRef.getProperty()) {
                                if ("protein sequence ID".equals(propertyType.getType())) {
                                    String xrefId = dbRef.getType() + "." + propertyType.getValue();
                                    // In the list, one db reference can be multiple times
                                    if (!done.contains(xrefId)) {
                                        proteinCache.saveXref(xrefId, proteinAcc);
                                        done.add(xrefId);
                                    }
                                    break;
                                }
                            }
                        }
                        String xrefId = dbRef.getType() + "." + dbRef.getId();
                        // In the list, one db reference can be multiple times
                        if (!done.contains(xrefId)) {
                            proteinCache.saveXref(xrefId, proteinAcc);
                            done.add(xrefId);
                        }
                    }
                }

                // And process protein to save to CSV file
                processProtein(proteinAcc, proteinName);
            } else {
                logger.info("Skipping building protein: missing protein accession from JSON file");
            }

            // Next line
            jsonProtein = reader.readLine();
        }
        logger.info("Building {} proteins. Done.", proteinCounter);

        reader.close();
    }

    public void buildDiseasePanels(Path panelPath) throws IOException {
        ObjectReader mapperReader = new ObjectMapper().reader(DiseasePanel.class);

        // Get CSV file writers
        BufferedReader reader = FileUtils.newBufferedReader(panelPath);

        String jsonDiseasePanel = reader.readLine();
        while (jsonDiseasePanel != null) {
            DiseasePanel diseasePanel = mapperReader.readValue(jsonDiseasePanel);

            // Create disease panel node and save CSV file
            Node diseasePanelNode = NodeBuilder.newNode(csv.getAndIncUid(), diseasePanel);
            writeNodeLine(diseasePanelNode);

            for (DiseasePanel.GenePanel panelGene : diseasePanel.getGenes()) {
                // Create panel gene node and save CSV file
                Node panelGeneNode = NodeBuilder.newNode(csv.getAndIncUid(), panelGene);
                writeNodeLine(panelGeneNode);

                writeRelationLine(HAS___DISEASE_PANEL___PANEL_GENE.name(), diseasePanelNode.getUid(), panelGeneNode.getUid());

                if (StringUtils.isEmpty(panelGene.getName())) {
                    String msg = "Missing gene name (" + panelGene.getId() + ") from disease panel " + diseasePanel.getId();
                    logger.warn(msg);
                    System.out.println(msg);
                } else {
                    Long ensemblGeneUid = csv.getLong(panelGene.getName(), "ensembl");
                    if (ensemblGeneUid == null) {
                        String msg = "Ensembl gene not found: gene " + panelGene.getName() + " (" + panelGene.getId()
                                + ") from disease panel " + diseasePanel.getId();
                        logger.warn(msg);
                        System.out.println(msg);
                    } else {
                        // Add relation to CSV file
                        writeRelationLine(ANNOTATION___GENE___PANEL_GENE.name(), ensemblGeneUid, panelGeneNode.getUid());
                    }

                    Long refSeqGeneUid = csv.getLong(panelGene.getName(), "refseq");
                    if (refSeqGeneUid == null) {
                        String msg = "RefSeq gene not found: gene " + panelGene.getName() + " (" + panelGene.getId()
                                + ") from disease panel " + diseasePanel.getId();
                        logger.warn(msg);
                        System.out.println(msg);
                    } else {
                        writeRelationLine(ANNOTATION___GENE___PANEL_GENE.name(), refSeqGeneUid, panelGeneNode.getUid());
                    }
                }
            }

            // Next panel
            jsonDiseasePanel = reader.readLine();
        }
    }

    private List<String> readSampleIds(String variantFile) {
        List<String> sampleIds = new ArrayList<>();

        File metaFile = new File(variantFile + ".meta.json");
        if (!metaFile.exists()) {
            metaFile = new File(variantFile + ".meta.json.gz");
        }
        if (!metaFile.exists()) {
            return sampleIds;
        }

        // Read info, format and sample from metadata file
        ObjectMapper mapper = new ObjectMapper();
        VariantMetadata variantMetadata;
        try {
            BufferedReader bufferedReader = FileUtils.newBufferedReader(metaFile.toPath());
            String metadata = bufferedReader.readLine();
            bufferedReader.close();
            variantMetadata = mapper.readValue(metadata, VariantMetadata.class);
        } catch (IOException e) {
            e.printStackTrace();
            return sampleIds;
        }

        if (CollectionUtils.isNotEmpty(variantMetadata.getStudies())) {
            // IMPORTANT: it considers only the first study
            VariantStudyMetadata variantStudyMetadata = variantMetadata.getStudies().get(0);

            // Get sample attributes
            for (Individual individual : variantStudyMetadata.getIndividuals()) {
                if (CollectionUtils.isNotEmpty(individual.getSamples())) {
                    for (Sample sample : individual.getSamples()) {
                        sampleIds.add(sample.getId());
                    }
                }
            }
        }

        return sampleIds;
    }

    public CsvInfo getCsvInfo() {
        return csv;
    }

    public List<String> getAdditionalVariantFiles() {
        return additionalVariantFiles;
    }

    public Builder setAdditionalVariantFiles(List<String> additionalVariantFiles) {
        this.additionalVariantFiles = additionalVariantFiles;
        return this;
    }

    public List<String> getAdditionalNeworkFiles() {
        return additionalNeworkFiles;
    }

    public Builder setAdditionalNeworkFiles(List<String> additionalNeworkFiles) {
        this.additionalNeworkFiles = additionalNeworkFiles;
        return this;
    }
} public class Downloader {

    private DownloadProperties downloadProperties;
    private Path outDir;

    private Logger logger;

    public Downloader() {
        this.logger = LoggerFactory.getLogger(this.getClass().toString());
    }

    public Downloader(DownloadProperties downloadProperties, Path outDir) {
        this.downloadProperties = downloadProperties;
        this.outDir = outDir;

        this.logger = LoggerFactory.getLogger(this.getClass().toString());
    }

    public void download() throws IOException {
        FileUtils.checkDirectory(outDir);

        // Ensembl gene
        if (downloadProperties.getEnsemblGene() != null) {
            try {
                downloadURL(new URL(downloadProperties.getEnsemblGene().getHost()), outDir);
            } catch (IOException e) {
                logger.error(e.getMessage());
            }
        }

        // RefSeq gene
        if (downloadProperties.getRefSeqGene() != null) {
            try {
                downloadURL(new URL(downloadProperties.getRefSeqGene().getHost()), outDir);
            } catch (IOException e) {
                logger.error(e.getMessage());
            }
        }

        // Protein
        if (downloadProperties.getProtein() != null) {
            try {
                downloadURL(new URL(downloadProperties.getProtein().getHost()), outDir);
            } catch (IOException e) {
                logger.error(e.getMessage());
            }
        }

        // Panel
        if (downloadProperties.getPanel() != null) {
            try {
                downloadURL(new URL(downloadProperties.getPanel().getHost()), outDir);
            } catch (IOException e) {
                logger.error(e.getMessage());
            }
        }

        // Clinical variant
        if (downloadProperties.getClinicalVariant() != null) {
            try {
                downloadURL(new URL(downloadProperties.getClinicalVariant().getHost()), outDir);
            } catch (IOException e) {
                logger.error(e.getMessage());
            }
        }

        // Reactome
        if (downloadProperties.getNetwork() != null) {
            try {
                downloadURL(new URL(downloadProperties.getNetwork().getHost()), outDir);
            } catch (IOException e) {
                logger.error(e.getMessage());
            }
        }

    }

    private void downloadURL(URL url, Path outDir) throws IOException {
        String filename = new File(url.getFile()).getName();
        if (outDir.resolve(filename).toFile().exists()) {
            logger.info("Skipping to downloadd " + filename + ". It already exists in " + outDir);
        } else {
            logger.info("Downloading " + url + "...");
            URLUtils.download(url, outDir);
            logger.info(url + " downloaded!");
        }
    }

    @Override
    public String toString() {
        final StringBuilder sb = new StringBuilder("Downloader{");
        sb.append("downloadProperties=").append(downloadProperties);
        sb.append(", outDir=").append(outDir);
        sb.append('}');
        return sb.toString();
    }

    public DownloadProperties getDownloadProperties() {
        return downloadProperties;
    }

    public Downloader setDownloadProperties(DownloadProperties downloadProperties) {
        this.downloadProperties = downloadProperties;
        return this;
    }

    public Path getOutDir() {
        return outDir;
    }

    public Downloader setOutDir(Path outDir) {
        this.outDir = outDir;
        return this;
    }
} public class Importer {

    private String database;
    private Path inputPath;

    private String neo4jHome;

    protected static Logger logger;

    public Importer() {
        this.logger = LoggerFactory.getLogger(this.getClass().toString());
    }

    public Importer(String database, Path inputPath) {
        this.database = database;
        this.inputPath = inputPath;

        this.logger = LoggerFactory.getLogger(this.getClass().toString());

        this.neo4jHome = System.getenv("NEO4J_HOME");
        if (neo4jHome == null) {
            logger.error("The environment variable NEO4J_HOME is not defined.");
        }
    }

    public void run() {
        // Check input directory
        if (!inputPath.toFile().isDirectory()) {
            logger.error("Input directory {} is invalid", inputPath);
            return;
        }

        String name;
        StringBuilder sb = new StringBuilder();
        Set<String> validNodes = new HashSet<>();

        sb.append(neo4jHome);
        sb.append("/bin/neo4j stop");
        Command command = new Command(sb.toString());
        command.run();
        logger.info("Executing: {}", sb);
        logger.info("Output: {}", command.getOutput());

        sb.setLength(0);
        sb.append(neo4jHome);
        sb.append("/bin/neo4j-admin import --database " + database + " --id-type INTEGER"
                + " --delimiter=\"" + StringEscapeUtils.escapeJava(CsvInfo.SEPARATOR) + "\""
                + " --array-delimiter=\"" + StringEscapeUtils.escapeJava(CsvInfo.ARRAY_SEPARATOR) + "\""
                + " --skip-duplicate-nodes");

        // Retrieving files from the input directory
        List<File> relationFiles = new ArrayList<>();
        for (File file : inputPath.toFile().listFiles()) {
            if (file.isFile()) {
                String filename = file.getName();
                if (filename.endsWith(".csv.gz")) {
                    if (filename.contains("__")) {
                        relationFiles.add(file);
                    } else {
                        if (!isEmptyCsvFile(file)) {
                            name = getNodeName(file);
                            validNodes.add(name);
                            sb.append(" --nodes=").append(file.getAbsolutePath());
                            //sb.append(" --nodes=").append(name).append("=").append(file.getAbsolutePath());
                        } else {
                            logger.info("Skipping node file {} to import: file empty", file.getAbsolutePath());
                        }
                    }
                } else {
                    logger.info("Skipping node file {} to import: no CSV format", file.getAbsolutePath());
                }
            }
        }

        // Now, relationhsip files
        for (File file : relationFiles) {
            System.out.println("checking file: " + file.getName());
            if (isValidRelationCsvFile(file, validNodes)) {
                name = getRelationName(file);
                sb.append(" --relationships=").append(name).append("=").append(file.getAbsolutePath());
            } else {
                logger.info("Skipping relationship file {} to import: invalid file", file.getAbsolutePath());
            }
        }

        System.out.println(sb.toString());

        command = new Command(sb.toString());
        command.run();
        logger.info("Executing: {}", sb);
        logger.info("Output: {}", command.getOutput());

        sb.setLength(0);
        sb.append(neo4jHome);
        sb.append("/bin/neo4j start");
        command = new Command(sb.toString());
        command.run();
        logger.info("Executing: {}", sb);
        logger.info("Output: {}", command.getOutput());
    }

    public boolean isRunning() {
        StringBuilder sb = new StringBuilder(neo4jHome);
        sb.append("/bin/neo4j status");
        Command command = new Command(sb.toString());
        command.run();
        System.out.println(command.getOutput());
        if (StringUtils.isNotEmpty(command.getOutput()) && command.getOutput().contains("is running")) {
            return true;
        }
        return false;
    }

    private boolean isEmptyCsvFile(File file) {
        boolean isEmpty = false;
        try {
            BufferedReader bufferedReader = FileUtils.newBufferedReader(file.toPath());
            int counter = 0;
            String line = bufferedReader.readLine();
            while (line != null) {
                counter++;
                if (counter > 1) {
                    break;
                }
                line = bufferedReader.readLine();
            }
            isEmpty = (counter < 2);
        } catch (IOException e) {
            logger.warn("Something wrong checking CSV file {}, skipping it!", file.toPath());
            isEmpty = false;
        }
        return isEmpty;
    }

    private boolean isValidRelationCsvFile(File file, Set<String> nodeNames) {
        boolean isValid = false;
        if (!isEmptyCsvFile(file)) {
            String name = file.getName();
            if (name.contains(CsvInfo.FILENAME_SEPARATOR)) {
                String[] fields = removeCsvExt(name).split(CsvInfo.FILENAME_SEPARATOR);
                isValid = nodeNames.contains(fields[1]) && nodeNames.contains(fields[2]);
            } else {
                String[] fields = removeCsvExt(name).split("__");
                isValid = nodeNames.contains(fields[0]) && nodeNames.contains(fields[1]);
            }
        }
        return isValid;
    }

    private String getNodeName(File file) {
        return removeCsvExt(file.getName());
    }

    private String getRelationName(File file) {
        String name = file.getName();
        if (name.contains(CsvInfo.FILENAME_SEPARATOR)) {
            return name.split(CsvInfo.FILENAME_SEPARATOR)[0];
        } else {
            return removeCsvExt(name);
        }
    }

    private String removeCsvExt(String filename) {
        String name = null;
        if (filename != null) {
            return filename.replace(".csv.gz", "");
        }
        return name;
    }
}
import htsjdk.variant.variantcontext.VariantContext;
import org.apache.commons.collections.CollectionUtils;
import org.opencb.biodata.models.variant.Variant;
import org.opencb.biodata.tools.variant.converters.avro.VariantContextToVariantConverter;
import org.opencb.bionetdb.core.config.BioNetDBConfiguration;
import org.opencb.bionetdb.core.exceptions.BioNetDBException;
import org.opencb.bionetdb.core.models.network.NetworkPath;
import org.opencb.bionetdb.lib.analysis.InterpretationAnalysis;
import org.opencb.bionetdb.lib.analysis.NetworkAnalysis;
import org.opencb.bionetdb.lib.analysis.VariantAnalysis;
import org.opencb.bionetdb.lib.analysis.interpretation.TieringInterpretationAnalysis;
import org.opencb.bionetdb.lib.api.NetworkDBAdaptor;
import org.opencb.bionetdb.lib.api.iterators.NetworkPathIterator;
import org.opencb.bionetdb.lib.api.iterators.RowIterator;
import org.opencb.bionetdb.lib.db.Neo4JNetworkDBAdaptor;
import org.opencb.bionetdb.lib.executors.NetworkQueryExecutor;
import org.opencb.bionetdb.lib.executors.NodeQueryExecutor;
import org.opencb.bionetdb.lib.executors.PathQueryExecutor;
import org.opencb.bionetdb.lib.utils.Builder;
import org.opencb.bionetdb.lib.utils.Downloader;
import org.opencb.bionetdb.lib.utils.Importer;
import org.opencb.commons.datastore.core.DataResult;
import org.opencb.commons.datastore.core.QueryResult;
import org.opencb.commons.utils.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Path;
import java.security.NoSuchAlgorithmException;
import java.util.*;
package org.opencb.bionetdb.lib;



/**
 * Created by joaquin on 1/29/18.
 */
public class BioNetDbManager {

    private BioNetDBConfiguration configuration;

    private NetworkDBAdaptor networkDBAdaptor;

    private Logger logger;

    private static final int QUERY_MAX_RESULTS = 50000;
    private TieringInterpretationAnalysis tieringInterpretationAnalysis;

    public BioNetDbManager(BioNetDBConfiguration configuration) throws BioNetDBException {