public class AtomDownloadJob extends MultipleFileDownloadJob {

    private static final String XPATH_LINKS =
        "//atom:entry[atom:id/text()=$VARIATION]/atom:link";

    private String url;
    private String dataset;
    private String variation;
    private File workingDir;

    public AtomDownloadJob(
        String url,
        String dataset,
        String variation,
        File workingDir,
        String user,
        String password,
        Log logger
    ) {
        super(user, password, logger);
        this.url = url;
        this.dataset = dataset;
        this.variation = variation;
        this.workingDir = workingDir;
    }

    private Document getDocument(String urlString)
        throws JobExecutionException {
        return getDocument(toURL(urlString));
    }

    private Document getDocument(URL docURL)
        throws JobExecutionException {

        CloseableHttpClient client = getClient(docURL);
        HttpGet httpget = getGetRequest(docURL);

        try {
            DocumentResponseHandler responseHandler
                = new DocumentResponseHandler(httpget);
            responseHandler.setNamespaceAware(true);

            Document document = client.execute(httpget, responseHandler);
            if (document == null) {
                throw new JobExecutionException(
                    I18n.format("atom.bad.xml", docURL.toString()));
            }
            return document;
        } catch (IOException ioe) {
            throw new JobExecutionException(
                I18n.format("atom.bad.download", docURL.toString()), ioe);
        } finally {
            HTTP.closeGraceful(client);
        }
    }

    private static final String DATASOURCE_XPATH
        = "/atom:feed/atom:entry[atom:id/text()=$CODE or"
        + " inspire_dls:spatial_dataset_identifier_code/text()=$CODE]"
        + "/atom:link[not(boolean(@type)) or"
        + " @type='application/atom+xml']/@href";

    private static final NamespaceContext NAMESPACE_CONTEXT =
        new NamespaceContextMap(
        "atom", "http://www.w3.org/2005/Atom",
        "inspire_dls", "http://inspire.ec.europa.eu/schemas/inspire_dls/1.0",
        "georss", "http://www.georss.org/georss");

    private String figureoutDatasource() throws JobExecutionException {

        Document doc = getDocument(this.url);
        HashMap<String, String> vars = new HashMap<>();
        vars.put("CODE", this.dataset);

        String ds = (String)XML.xpath(doc,
            DATASOURCE_XPATH, XPathConstants.STRING,
            NAMESPACE_CONTEXT, vars);

        if (ds == null || ds.isEmpty()) {
            throw new JobExecutionException(
                I18n.format("atom.dataset.not.found", this.dataset));
        }
        return ds;
    }

    @Override
    protected void download() throws JobExecutionException, InterruptedException {
        String dsURL = figureoutDatasource();
        URL root = absoluteURL(this.url, dsURL);
        Document ds = getDocument(root);
        HashMap<String, String> vars = new HashMap<>();
        vars.put("VARIATION", this.variation);
        NodeList nl = (NodeList)XML.xpath(
            ds, XPATH_LINKS, XPathConstants.NODESET,
            NAMESPACE_CONTEXT, vars);

        ArrayList<DLFile> files = new ArrayList<>(nl.getLength());

        AutoFileNames afn = new AutoFileNames(nl.getLength());

        for (int i = 0, n = nl.getLength(); i < n; i++) {
            Element link = (Element)nl.item(i);
            String href = link.getAttribute("href");
            if (href.isEmpty()) {
                continue;
            }
            URL dataURL = absoluteURL(root, href);
            String fileName;
            // Service call?
            if (dataURL.getQuery() != null) {
                String type = link.getAttribute("type");
                fileName = afn.nextFileName(type);
            } else { // Direct download.
                // XXX: Do more to prevent directory traversals?
                fileName = new File(dataURL.getPath())
                    .getName().replaceAll("\\.+", ".");

                if (fileName.isEmpty()) {
                    String type = link.getAttribute("type");
                    fileName = afn.nextFileName(type);
                }
            }
            File file = new File(this.workingDir, fileName);
            files.add(new DLFile(file, dataURL));
        }

        downloadFiles(files);
    }
} public class BroadcastJob implements Job {

    private String msg;

    public BroadcastJob(String msg) {
        this.msg = msg;
    }

    @Override
    public void run(Processor p) throws JobExecutionException {
        if (p == null) {
            throw new JobExecutionException();
        }
        p.broadcastMessage(msg);
    }
} public class CloseLogJob implements DeferredJob {

    private Log logger;

    public CloseLogJob(Log logger) {
        this.logger = logger;
    }

    @Override
    public void run(Processor p) {
        String msg = I18n.getMsg("gml.end.of.protocol");
        logger.log(msg);
        logger.close();
    }
} public class FileDownloadJob extends AbstractDownloadJob {

    private String urlString;
    private File file;
    private HttpEntity postParams;

    public FileDownloadJob(
            String urlString,
            File   file,
            String user,
            String password,
            Log    logger) {
        this(urlString, file, user, password, null, logger);
    }

    public FileDownloadJob(
            String urlString,
            File   file,
            String user,
            String password,
            HttpEntity postParams,
            Log    logger) {
        super(user, password, logger);
        this.urlString = urlString;
        this.file = file;
        this.postParams = postParams;
        this.listener.add(this);
    }

    @Override
    public void bytesCounted(long count) {
        broadcastMessage(I18n.format("file.download.bytes", count), false);
    }

    @Override
    public void download()
        throws JobExecutionException, InterruptedException {
        URL url = toURL(this.urlString);

        WrapInputStreamFactory wrapFactory
            = CountingInputStream.createWrapFactory(listener);

        CloseableHttpClient httpclient = getClient(url);

        String msg = I18n.format("download.file", url, this.file);
        broadcastMessage(msg);

        try {
            HttpRequestBase httpRequest;
            if (postParams == null) {
                httpRequest = getGetRequest(url);
            } else {
                HttpPost httppost = new HttpPost(url.toString());
                httppost.setHeader("Content-Type", "application/xml");
                httppost.setEntity(postParams);
                httpRequest = httppost;
            }
            FileResponseHandler responseHandler = new FileResponseHandler(this.file, wrapFactory, httpRequest);

            ExecutorService executorService = Executors.newSingleThreadExecutor();
            HttpDownloadExecutor downloadTask = new HttpDownloadExecutor(httpclient, httpRequest, responseHandler,
                this);
            Future<?> submit = executorService.submit(downloadTask);
            while (!submit.isDone()) {
                if (Thread.currentThread().isInterrupted()) {
                    httpRequest.abort();
                    throw new InterruptedException("Download interrupted.");
                }
            }
            if (downloadTask.isFailed()) {
                throw downloadTask.getJobExecutionException();
            }
        } finally {
            HTTP.closeGraceful(httpclient);
        }
        broadcastMessage(I18n.getMsg("file.download.finished"));
    }

    /**
     * @return the HttpEntity used a POST body.
     */
    public HttpEntity getPostParams() {
        return postParams;
    }

} public class FilesDownloadJob extends MultipleFileDownloadJob {

    private List<DLFile> files = new ArrayList<>();

    public FilesDownloadJob(String user, String password) {
        super(user, password);
    }

    /** Downloads an URL to a file.
     * @param file The file.
     * @param url The URL.
     */
    public void add(File file, URL url) {
        this.files.add(new DLFile(file, url));
    }

    /** Downloads an URL to a file using post.
     * @param file The file.
     * @param url The URL.
     * @param postParams Post params
     */
    public void add(File file, URL url, HttpEntity postParams) {
        this.files.add(new DLFile(file, url, postParams));
    }

    @Override
    protected void download() throws JobExecutionException, InterruptedException {
        downloadFiles(this.files);
    }
} public class GMLCheckJob implements Job {

    private static final long SCREENING_THESHOLD = 4096;

    private static final String ERROR_MESSAGE
        = "//*[local-name()='Exception']"
        + "/*[local-name()='ExceptionText']/text()";

    private List<File> files;

    private Log logger;

    private static final String[] EXCEPTION_INDICATORS = {
        "ExceptionReport"
    };

    public GMLCheckJob(Log logger) {
        this.logger = logger;
        this.files = new ArrayList<>();
    }

    public GMLCheckJob(File file, Log logger) {
        this.logger = logger;
        this.files = new ArrayList<>();
        this.files.add(file);
    }

    public GMLCheckJob(List<File> files, Log logger) {
        this.logger = logger;
        this.files = files;
    }

    /** Adds a GML file to check.
     * @param file The file to check.
     */
    public void add(File file) {
        this.files.add(file);
    }

    private void log(String msg) {
        if (this.logger != null) {
            this.logger.log(msg);
        }
    }

    private void checkForProblems(File file)
    throws JobExecutionException, SAXException,
            ParserConfigurationException, IOException {
        Document doc = XML.getDocument(file);
        if (doc == null) {
            String msg = I18n.format("gml.check.parsing.failed", file);
            log(msg);
            throw new JobExecutionException(msg);
        }
        String message = XML.xpathString(doc, ERROR_MESSAGE, null);
        if (message != null && !message.isEmpty()) {
            String msg = I18n.format("gml.check.wfs.problem", message);
            log(msg);
            throw new JobExecutionException(msg);
        }
    }

    @Override
    public void run(Processor p) throws JobExecutionException {

        for (File file: this.files) {
            p.broadcastMessage(I18n.format("gml.check.start", file));

            if (!file.isFile() || !file.canRead()) {
                String msg = I18n.format("gml.check.not.accessible", file);
                log(msg);
                throw new JobExecutionException(msg);
            }

            // If the document is large screen for
            // indicators of ExceptionReport first to avoid
            // building a large in memory DOM.
            if (file.length() > SCREENING_THESHOLD) {
                try {
                    if (XML.containsTags(file, EXCEPTION_INDICATORS) == null) {
                        // No indicators no cry ...
                        String msg = I18n.getMsg("gml.check.passed");
                        log(msg);
                        p.broadcastMessage(msg);
                        return;
                    }
                } catch (XMLStreamException | IOException e) {
                    String msg =
                        I18n.format("gml.check.processing.failed", file);
                    log(msg);
                    throw new JobExecutionException(msg);
                }
            }
            try {
                checkForProblems(file);
            } catch (SAXException
                    | ParserConfigurationException
                    | IOException e) {
                throw new JobExecutionException(e.getMessage());
            }
            String msg = I18n.getMsg("gml.check.passed");
            log(msg);
            p.broadcastMessage(msg);
        }
    }
} public class DownloadStepJob implements Job {

    private static final Logger LOG = LoggerFactory.getLogger(DownloadStepJob.class.getName());

    private List<Job> jobs;

    public DownloadStepJob() {
        jobs = new ArrayList<>();
    }

    /**
     * Add a job to this job list.
     * @param job The job to be added.
     */
    public void addJob(Job job) {
        jobs.add(job);
    }

    /**
     * Add all jobs to this job list.
     * @param other The jobs to add.
     */
    public void addJobs(List<Job> other) {
        jobs.addAll(other);
    }

    @Override
    public void run(Processor p) throws JobExecutionException, InterruptedException {
        LOG.info("Executing download step jobs");
        int i = 0;
        try {
            for (; i < jobs.size(); i++) {
                jobs.get(i).run(p);
                if (Thread.currentThread().isInterrupted()) {
                    throw new InterruptedException("Execution of the download job interrupted.");
                }
            }
        } finally {
            for (i++; i < jobs.size(); i++) {
                Job job = jobs.get(i);
                if (job instanceof DeferredJob) {
                    job.run(p);
                }
            }
        }
    }

    /**
     * Returns the List of Jobs.
     * @return list of Jobs
     */
    public List<Job> getJobList() {
        return this.jobs;
    }
} public class LogMetaJob implements Job {

    private Log          logger;
    private DownloadStep dls;

    public LogMetaJob(Log logger, DownloadStep dls) {
        this.logger = logger;
        this.dls    = dls;
    }

    @Override
    public void run(Processor p) throws JobExecutionException {
        if (this.logger == null) {
            return;
        }

        this.logger.log(I18n.format("meta.log.service", dls.getServiceType()));
        this.logger.log(I18n.format("meta.log.url", dls.getServiceURL()));
        this.logger.log(I18n.format("meta.log.dataset", dls.getDataset()));
        List<Parameter> parameters = dls.getParameters();
        if (parameters != null && !parameters.isEmpty()) {
            this.logger.log(I18n.getMsg("meta.log.parameters"));
            for (Parameter para: parameters) {
                this.logger.log(
                    "\t" + para.getKey() + ": " + para.getValue());
            }
        }
    }
} public class OpenLogJob implements Job {

    private Log logger;

    public OpenLogJob(Log logger) {
        this.logger = logger;
    }

    @Override
    public void run(Processor p) throws JobExecutionException {
        try {
            logger.open();
        } catch (IOException ioe) {
            throw new JobExecutionException(
                I18n.getMsg("processor.cannot.open.log"), ioe);
        }
    }
} public class UnzipJob implements Job {
    private static final int BUFFER_SIZE = 4096;
    private final FileTracker fileTracker;
    private final Log logger;

    /**
     * @param fileTracker never <code>null</code>
     * @param logger      never <code>null</code>
     */
    public UnzipJob(FileTracker fileTracker, Log logger) {
        this.fileTracker = fileTracker;
        this.logger = logger;
    }

    @Override
    public void run(Processor p) throws JobExecutionException {
        updateFileTracker(p);
        List<File> filesToUnzip = fileTracker.deltaGlob("*.{zip}");
        I18n.getMsg("unzip.start");
        unzip(p, filesToUnzip);
        I18n.getMsg("unzip.end");
    }

    private void unzip(Processor p, List<File> filesToUnzip)
        throws JobExecutionException {
        for (File fileToUnzip : filesToUnzip) {
            try {
                unzip(fileTracker.getDirectory(), fileToUnzip);
            } catch (IOException e) {
                String msg = "Could not unzip file " + fileToUnzip;
                JobExecutionException jee = new JobExecutionException(msg, e);
                logger.log(jee.getMessage());
                throw jee;
            }
        }
    }

    private void unzip(File directory, File fileToUnzip) throws IOException {
        try (FileInputStream in = new FileInputStream(fileToUnzip);
             ZipInputStream zipIn = new ZipInputStream(in)) {
            ZipEntry entry = zipIn.getNextEntry();
            while (entry != null) {
                String filePath = directory + File.separator + entry.getName();
                if (!entry.isDirectory()) {
                    extractFile(zipIn, filePath);
                } else {
                    File dir = new File(filePath);
                    dir.mkdir();
                }
                zipIn.closeEntry();
                entry = zipIn.getNextEntry();
            }
        }
    }

    private void extractFile(ZipInputStream zipIn, String filePath)
        throws IOException {
        try (FileOutputStream out = new FileOutputStream(filePath);
             BufferedOutputStream bos = new BufferedOutputStream(out)) {
            byte[] bytesIn = new byte[BUFFER_SIZE];
            int read = 0;
            while ((read = zipIn.read(bytesIn)) != -1) {
                bos.write(bytesIn, 0, read);
            }
        }
    }

    private void updateFileTracker(Processor p) throws JobExecutionException {
        this.fileTracker.push();
        if (!this.fileTracker.scan()) {
            String msg = I18n.format(
                "external.process.scan.dir.failed",
                this.fileTracker.getDirectory());
            JobExecutionException jee = new JobExecutionException(msg);
            logger.log(jee.getMessage());
            throw jee;
        }
    }
} public interface CountListener {

    /**
     * bytesCounted is called to report the number of bytes already copied.
     *
     * @param counter The number of bytes.
     */
    void bytesCounted(long counter);

} @XmlRootElement(name = "DownloadSchritt")
@XmlAccessorType(XmlAccessType.FIELD)
public class DownloadStep {

    @XmlElement(name = "ServiceTyp")
    private String serviceType;

    @XmlElement(name = "URL")
    private String serviceURL;

    @XmlElement(name = "DownloadPfad")
    private String path;

    @XmlElement(name = "Dataset")
    private String dataset;

    @XmlElementWrapper(name = "Verarbeitungskette")
    @XmlElement(name = "Verarbeitungsschritt")
    private List<ProcessingStep> processingSteps;

    @XmlElementWrapper(name = "Parameters")
    @XmlElement(name = "Parameter")
    private List<Parameter> parameters;

    public DownloadStep() {
    }

    public DownloadStep(String dataset,
                        List<Parameter> parameters,
                        String serviceType,
                        String serviceURL,
                        String path,
                        List<ProcessingStep> processingSteps) {
        this.dataset = dataset;
        this.parameters = parameters;
        this.serviceType = serviceType;
        this.serviceURL = serviceURL;
        this.path = path;
        this.processingSteps = processingSteps;
    }


    public DownloadStep(String dataset,
                        List<Parameter> parameters,
                        String serviceType,
                        String serviceURL,
                        String path) {
        this(dataset,
            parameters,
            serviceType,
            serviceURL,
            path,
            new ArrayList<ProcessingStep>());
    }

    /**
     * @param file The file to load data from.
     */
    public DownloadStep(File file) {
    }

    /**
     * @return the dataset
     */
    public String getDataset() {
        return dataset;
    }

    /**
     * @param dataset the dataset to set
     */
    public void setDataset(String dataset) {
        this.dataset = dataset;
    }

    public List<ProcessingStep> getProcessingSteps() {
        return this.processingSteps;
    }

    public void setProcessingSteps(List<ProcessingStep> processingSteps) {
        this.processingSteps = processingSteps;
    }

    /**
     * @return the parameters
     */
    public List<Parameter> getParameters() {
        return parameters;
    }

    /**
     * @param parameters the parameters to set
     */
    public void setParameters(List<Parameter> parameters) {
        this.parameters = parameters;
    }

    /**
     * @return the serviceType
     */
    public String getServiceType() {
        return serviceType;
    }

    /**
     * @param serviceType the serviceType to set
     */
    public void setServiceType(String serviceType) {
        this.serviceType = serviceType;
    }

    /**
     * @param serviceType the serviceType to set
     */
    public void setServiceType(ServiceType serviceType) {
        this.setServiceType(serviceType.toString());
    }

    /**
     * @return the serviceURL
     */
    public String getServiceURL() {
        return serviceURL;
    }

    /**
     * @param serviceURL the serviceURL to set
     */
    public void setServiceURL(String serviceURL) {
        this.serviceURL = serviceURL;
    }

    /**
     * @return the dataSet
     */
    public String getPath() {
        return path;
    }

    /**
     * @param path the path to set
     */
    public void setPath(String path) {
        this.path = path;
    }

    /**
     * Finds a value for a given parameter key.
     *
     * @param key The key.
     * @return The value if found else null.
     */
    public String findParameter(String key) {
        for (Parameter p : this.parameters) {
            if (p.getKey().equals(key)) {
                return p.getValue();
            }
        }
        return null;
    }

    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder("[DownloadStep:\n");
        sb.append("\tserviceType: \"").append(serviceType).append("\"\n");
        sb.append("\tserviceURL: \"").append(serviceURL).append("\"\n");
        sb.append("\tdataset: \"").append(dataset).append("\"\n");
        sb.append("\tpath: \"").append(path).append("\"\n");
        sb.append("\tparameters: ");
        for (int i = 0,
             n = parameters != null ? parameters.size() : 0; i < n; i++) {
            if (i > 0) {
                sb.append(", ");
            }
            sb.append(parameters.get(i));
        }

        sb.append("]\n");
        sb.append("\tprocessing steps:\n");
        for (int i = 0,
             n = processingSteps != null ? processingSteps.size() : 0;
             i < n; i++) {
            sb.append("\t\t");
            sb.append(processingSteps.get(i));
            sb.append('\n');
        }
        sb.append(']');
        return sb.toString();
    }

    /**
     * Save the Object.
     *
     * @param file File to save to
     * @throws IOException if space does not exist
     */
    public void write(File file) throws IOException {
        try (FileOutputStream fos = new FileOutputStream(file)) {
            JAXBContext context = JAXBContext.newInstance(DownloadStep.class);
            Marshaller m = context.createMarshaller();
            m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
            BufferedOutputStream bos = new BufferedOutputStream(fos);
            m.marshal(this, bos);
            bos.flush();
        } catch (JAXBException je) {
            throw new IOException("", je);
        }
    }


    /**
     * Loads DownloadStep from a file.
     *
     * @param file The file to load the DownloadStep from.
     * @return The restored DownloadStep.
     * @throws IOException Something went wrong.
     */
    public static DownloadStep read(File file) throws IOException {
        try (FileInputStream fis = new FileInputStream(file)) {
            return getStep(fis);
        } catch (JAXBException je) {
            throw new IOException("", je);
        }
    }

    private static DownloadStep getStep(InputStream is) throws JAXBException {
        JAXBContext context = JAXBContext.newInstance(DownloadStep.class);
        Unmarshaller um = context.createUnmarshaller();
        BufferedInputStream bis = new BufferedInputStream(is);
        return (DownloadStep) um.unmarshal(bis);
    }

    /**
     * Downloadstep from String.
     *
     * @param xmlString Configurationstring
     * @return unmarshalled DownloadStep
     * @throws IOException Just in case
     */
    public static DownloadStep read(String xmlString) throws IOException {
        try {
            InputStream stringStream = new ByteArrayInputStream(
                xmlString.getBytes());
            return getStep(stringStream);
        } catch (JAXBException je) {
            throw new IOException("", je);
        }
    }

} public class WFSMeta {

    private static final Logger LOG = LoggerFactory.getLogger(WFSMeta.class.getName());

    private static final String TAB_NEWLINE = "\t}\n";


    /** operation. */
    public static class Operation {
        /** name. */
        protected String name;
        /** get. */
        protected String get;
        /** post. */
        protected String post;
        /** output formats. */
        protected List<String> outputFormats;
        /** constraints. */
        protected List<Constraint> constraints;

        public String getGET() {
            return this.get;
        }

        public String getPOST() {
            return this.post;
        }

        public List<String> getOutputFormats() {
            return this.outputFormats;
        }

        public Operation() {
            this.outputFormats = new ArrayList<>();
            this.constraints = new ArrayList<>();
        }

        private String outputFormats() {
            return "[" + StringUtils.join(outputFormats, ", ") + "]";
        }

        private String constraints() {
            return "[" + StringUtils.join(constraints, ", ") + "]";
        }

        @Override
        public String toString() {
            String postString = (post != null)
                    ? " post: " + post + " " : "";
            return "operation: { name: " + name + " "
                + " get: " + get + " "
                + postString
                + " constraints: " + constraints() + " "
                + " output formats: " + outputFormats() + " }";
        }

        /**
         * Figure out if this service implements paging.
         * @return number of features per page.
         *         null if paging is not supported.
         */
        public Integer featuresPerPage() {
            return WFSMeta.featuresPerPage(this.constraints);
        }
    }


    /** feature. */
    public static class Feature {
        /** name. */
        protected String name;
        /** title. */
        protected String title;
        /** abstract. */
        protected String abstractDescription;
        /** default CRS. */
        protected String defaultCRS;
        /** other CRSs. */
        protected List<String> otherCRSs;
        /** outputFormats. */
        protected List<String> outputFormats;
        /** bbox. */
        protected ReferencedEnvelope bbox;

        public String getName() {
            return this.name;
        }

        public String getTitle() {
            return this.title;
        }

        public String getDefaultCRS() {
            return this.defaultCRS;
        }

        public List<String> getOtherCRSs() {
            return this.otherCRSs;
        }

        public ReferencedEnvelope getBBox() {
            return this.bbox;
        }

        public void setName(String name) {
            this.name = name;
        }

        public Feature() {
            this.otherCRSs = new ArrayList<>();
            this.outputFormats = new ArrayList<>();
        }

        private String otherCRSs() {
            return "[" + StringUtils.join(otherCRSs, ", ") + "]";
        }

        private String outputFormats() {
            return "[" + StringUtils.join(outputFormats, ", ") + "]";
        }

        @Override
        public String toString() {
            return "feature: { "
                + "name: " + name + " "
                + "title: " + title + " "
                + "abstract: " + abstractDescription + " "
                + "defaultCRS: " + defaultCRS + " "
                + "otherCRSs: " +  otherCRSs() + " "
                + "outputFormats: " +  outputFormats() + " "
                + "bbox: " + bbox + " }";
        }
    }

    /** stored Query. */
    public static class StoredQuery {
        /** id. */
        protected String id;
        /** title. */
        protected String title;
        /** abstract. */
        protected String abstractDescription;
        /** parameters. */
        protected List<Field> parameters;

        public StoredQuery() {
            parameters = new ArrayList<>();
        }

        public String getID() {
            return this.id;
        }

        public String getTitle() {
            return this.title;
        }

        public String getAbstractDescription() {
            return this.abstractDescription;
        }

        public List<Field> getParameters() {
            return this.parameters;
        }

        private String parameters() {
            return "[" + StringUtils.join(parameters, ", ") + "]";
        }

        @Override
        public String toString() {
            return "stored query: { "
                + "id: " + id + " "
                + "title: " + title + " "
                + "abstract: " + abstractDescription + " "
                + "parameters: " + parameters() + " }";
        }
    }

    /** Constraint. */
    public static class Constraint {
        /** name. */
        protected String name;
        /** value. */
        protected String value;
        /** allowed. */
        protected List<String> allowed;

        public Constraint() {
            allowed = new ArrayList<>();
        }

        @Override
        public String toString() {
            return "[ name: " + name
                + ", value: " + value
                + ", allowed: [" + StringUtils.join(allowed, ", ")
                + "]]";
        }
    }

    /** Version. */
    public static class Version implements Comparable<Version> {
        /** version. */
        private String versionString;
        /** parsed. */
        private int[] parsed;

        /** Version. */
        public Version() {
        }

        /** @param versionString The version. */
        public Version(String versionString) {
            this.versionString = versionString;
            parse();
        }

        /** Parses the version string to numbers. */
        public void parse() {
            try {
                String[] parts = versionString.split("\\.");
                int[] converted = new int[parts.length];
                for (int i = 0; i < converted.length; i++) {
                    converted[i] = Integer.parseInt(parts[i]);
                }
                this.parsed = converted;
            } catch (NumberFormatException nfe) {
                LOG.error(nfe.getMessage(), nfe);
            }
        }

        @Override
        public int compareTo(Version other) {
            if (other.parsed == null && this.parsed == null) {
                return this.versionString.compareTo(other.versionString);
            }
            if (this.parsed == null) {
                return +1;
            }
            if (other.parsed == null) {
                return -1;
            }
            int n = Math.min(this.parsed.length, other.parsed.length);
            for (int i = 0; i < n; i++) {
                int diff = this.parsed[i] - other.parsed[i];
                if (diff != 0) {
                    return diff;
                }
            }
            return 0;
        }

        @Override
        public boolean equals(Object obj) {
            return obj instanceof Version && compareTo((Version)obj) == 0;
        }

        private static final int THIRTYONE = 31;

        @Override
        public int hashCode() {
            int hash = 0;

            if (this.parsed != null) {
                hash = 1;
                for (int x: this.parsed) {
                    hash = THIRTYONE * hash + x;
                }
            }

            return hash + THIRTYONE * versionString.hashCode();
        }

        @Override
        public String toString() {
            return this.versionString;
        }
    }

    /** WFS 2.0.0. */
    public static final Version WFS2_0_0 = new Version("2.0.0");

    /** title. */
    protected String title;
    /** URL. */
    protected String url;
    /** abstract. */
    protected String abstractDescription;
    /** operations. */
    protected List<Operation> operations;
    /** constraints. */
    protected List<Constraint> constraints;
    /** features. */
    private final List<Feature> features;
    /** stored queries. */
    protected List<StoredQuery> storedQueries;
    /** output formats. */
    protected List<String> outputFormats;
    /** versions. */
    protected List<Version> versions;
    /** namespaces. */
    protected NamespaceContextMap namespaces;

    public String getTitle() {
        return this.title;
    }

    public String getURL() {
        return this.url;
    }

    public List<Feature> getFeatures() {
        return this.features;
    }

    public List<StoredQuery> getStoredQueries() {
        return this.storedQueries;
    }

    public List<String> getOutputFormats() {
        return this.outputFormats;
    }

    public NamespaceContextMap getNamespaces() {
        return this.namespaces;
    }

    /**
     * Adds a feature.
     *
     * @param feature to add
     */
    public void addFeature(Feature feature) {
        this.features.add(feature);
    }

    public WFSMeta() {
        operations = new ArrayList<>();
        constraints = new ArrayList<>();
        features = new ArrayList<>();
        storedQueries = new ArrayList<>();
        versions = new ArrayList<>();
        outputFormats = new ArrayList<>();
        namespaces = new NamespaceContextMap();
    }

    /**
     * Is operation supported?
     * @param name The name of the operation.
     * @return true if operation is supported false otherwise.
     */
    public boolean isOperationSupported(String name) {
        return findOperation(name) != null;
    }

    /**
     * Find an operation by name.
     * @param name The name of the operation.
     * @return The operation if found null otherwise.
     */
    public Operation findOperation(String name) {
        for (Operation op: this.operations) {
            if (op.name.equals(name)) {
                return op;
            }
        }
        return null;
    }

    /**
     * Return the highest supported version.
     * @param def The default. Used if there are no version informations.
     * @return The highest version.
     */
    public Version highestVersion(Version def) {
        return !versions.isEmpty()
            ? versions.get(versions.size() - 1)
            : def;
    }

    /**
     * Tells if this service is a SimpleWFS.
     * @return true if it is simple otherwise false.
     */
    public boolean isSimple() {
        for (Constraint c: constraints) {
            if (c.name.equals("QueryExpressions")) {
                boolean query = false;
                boolean storedQuery = false;
                for (String value: c.allowed) {
                    if (value.equals("wfs:Query")) {
                        query = true;
                    }
                    if (value.equals("wfs:StoredQuery")) {
                        storedQuery = true;
                    }
                }
                return !query && storedQuery;
            }
        }
        return false;
    }

    /**
     * Tells if this service is a BasicWFS.
     * @return true if it is basic otherwise false.
     */
    public boolean isBasic() {
        for (Constraint c: constraints) {
            if (c.name.equals("QueryExpressions")) {
                boolean query = false;
                boolean storedQuery = false;
                for (String value: c.allowed) {
                    if (value.equals("wfs:Query")) {
                        query = true;
                    }
                    if (value.equals("wfs:StoredQuery")) {
                        storedQuery = true;
                    }
                }
                return query && storedQuery;
            }
        }
        return false;
    }

    private static Integer featuresPerPage(List<Constraint> constraints) {
        for (Constraint constraint: constraints) {
            if (constraint.name != null
            && constraint.value != null
            && constraint.name.equals("CountDefault")) {
                try {
                    return Integer.valueOf(constraint.value);
                } catch (NumberFormatException nfe) {
                    // Ignore me.
                }
                return null;
            }
        }
        return null;
    }

    /**
     * Figure out if this service implements paging.
     * @return number of features per page.
     *         null if paging is not supported.
     */
    public Integer featuresPerPage() {
        return featuresPerPage(this.constraints);
    }

    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder("WFSMeta {\n");
        sb.append("\ttitle: ").append(this.title).append("\n");
        sb.append("\tabstract: ").append(this.abstractDescription)
          .append("\n");
        sb.append("\toperations: {\n");
        for (Operation op: operations) {
            sb.append("\t\t").append(op).append("\n");
        }
        sb.append(TAB_NEWLINE);
        sb.append("\tconstraints: {\n");
        for (Constraint c: constraints) {
            sb.append("\t\t").append(c).append("\n");
        }
        sb.append(TAB_NEWLINE);
        sb.append("\tfeatures: {\n");
        for (Feature f: features) {
            sb.append("\t\t").append(f).append("\n");
        }
        sb.append(TAB_NEWLINE);
        sb.append("\tstored queries: {\n");
        for (StoredQuery sq: storedQueries) {
            sb.append("\t\t").append(sq).append("\n");
        }
        sb.append(TAB_NEWLINE);
        sb.append("\tversions: {\n");
        for (Version version: versions) {
            sb.append("\t\t").append(version).append("\n");
        }
        sb.append(TAB_NEWLINE);
        sb.append("\toutput formats: {\n");
        for (String outputFormat: outputFormats) {
            sb.append("\t\t").append(outputFormat).append("\n");
        }
        sb.append(TAB_NEWLINE);
        sb.append("\tconstraints: {\n");
        for (Constraint constraint: constraints) {
            sb.append("\t\t").append(constraint).append("\n");
        }
        sb.append(TAB_NEWLINE);
        sb.append("}");
        return sb.toString();
    }
} public class WFSMetaExtractor {

    private static final CoordinateReferenceSystem WGS84;

    /** A RuntimeException wrapper around FactoryException. */
    public static final class RuntimeFactoryException
        extends RuntimeException {
        /**
         * Constructs a new RuntimeException wrapping a FactoryException.
         * @cause the cause.
         */
        public RuntimeFactoryException(Throwable cause) {
            super(cause);
        }
    }

    static {
        try {
            WGS84 = CRS.decode("EPSG:4326");
        } catch (FactoryException e) {
            throw new RuntimeFactoryException(e);
        }
    }

    private static final String OWS = "http://www.opengis.net/ows/1.1";
    private static final String WFS = "http://www.opengis.net/wfs/2.0";
    private static final String XLINK = "http://www.w3.org/1999/xlink";
    private static final String XSD = "http://www.w3.org/2001/XMLSchema";

    private static final String OWS_ALLOWED_VALUE_TEXT
        = "/ows:AllowedValues/ows:Value/text()";

    private static final String DESCRIBE_STORED_QUERIES
        = "DescribeStoredQueries";


    /** Common namespaces for WFS documents. */
    public static final NamespaceContext NAMESPACES =
        new NamespaceContextMap(
            "ows",   OWS,
            "wfs",   WFS,
            "xlink", XLINK,
            "xsd",   XSD);

    private static final String XPATH_TITLE
        = "//ows:ServiceIdentification/ows:Title/text()";

    private static final String XPATH_ABSTRACT
        = "//ows:ServiceIdentification/ows:Abstract/text()";

    private static final String XPATH_OPERATIONS
        = "//ows:OperationsMetadata/ows:Operation";

    private static final String XPATH_CONSTRAINTS_ALLOWED
        = "ows:AllowedValues/ows:Value/text()";

    private static final String XPATH_CONSTRAINTS
        = "//ows:OperationsMetadata/ows:Constraint";

    private static final String XPATH_FEATURETYPES
        = "//wfs:FeatureTypeList/wfs:FeatureType";

    private static final String XPATH_CONER_UPPER
            = "ows:WGS84BoundingBox/ows:UpperCorner";

    private static final String XPATH_CONER_LOWER
            = "ows:WGS84BoundingBox/ows:LowerCorner";

    private static final String XPATH_OPERATION_GET
        = "ows:DCP/ows:HTTP/ows:Get/@xlink:href";

    private static final String XPATH_OPERATION_POST
        = "ows:DCP/ows:HTTP/ows:Post/@xlink:href";

    private static final String XPATH_OPERATIONS_VERSIONS
        = "//ows:OperationsMetadata"
        + "/ows:Parameter[@name='version']"
        + OWS_ALLOWED_VALUE_TEXT;

    private static final String XPATH_OPERATION_OUT_FORMATS
        = "ows:Parameter[@name='outputFormat']"
        + OWS_ALLOWED_VALUE_TEXT;

    private static final String XPATH_OUT_FORMATS
        = "/wfs:WFS_Capabilities"
        + "/ows:OperationsMetadata/ows:Parameter[@name='outputFormat']"
        + OWS_ALLOWED_VALUE_TEXT;

    private static final String XPATH_FEATURE_OUT_FORMATS
        = "wfs:OutputFormats/wfs:Format/text()";

    private static final String XPATH_STORED_QUERIES
        = "//wfs:StoredQueryDescription";

    private String user;
    private String password;
    private String capURLString;

    public WFSMetaExtractor(String capURLString) {
        this.capURLString = capURLString;
    }

    public WFSMetaExtractor(
        String capURLString, String user, String password) {
        this(capURLString);
        this.user = user;
        this.password = password;
    }


    private static ReferencedEnvelope getBounds(
            Element feature, NamespaceContext nc) {

        String upperCorner = (String)
                XML.xpath(feature, XPATH_CONER_UPPER, XPathConstants.STRING,
                        nc);

        String lowerCorner = (String)
                XML.xpath(feature, XPATH_CONER_LOWER, XPathConstants.STRING,
                        nc);

        if (lowerCorner == null || upperCorner == null) {
            return null;
        }

        String[] upperCornerSplit = upperCorner.split("\\s+");
        String[] lowerCornerSplit = lowerCorner.split("\\s+");
        if (upperCornerSplit.length != lowerCornerSplit.length) {
            return null;
        }

        double minY = StringUtils.toDouble(lowerCornerSplit[0])[0];
        double minX = StringUtils.toDouble(lowerCornerSplit[1])[0];
        double maxY = StringUtils.toDouble(upperCornerSplit[0])[0];
        double maxX = StringUtils.toDouble(upperCornerSplit[1])[0];

        return new ReferencedEnvelope(minX, maxX, minY, maxY, WGS84);
    }

    /**
     * Extracts meta information from a given capabilities path.
     * @return The extracted meta data.
     * @throws IOException If something went wrong.
     * @throws URISyntaxException if the URL is wrong
     */
    public WFSMeta parse()
            throws IOException, URISyntaxException {
        WFSMeta meta = new WFSMeta();
        meta.url = capURLString;
        parseCapabilites(meta);
        parseDescribeStoredQueries(meta);
        return meta;
    }

    private Document getDocument(String url)
            throws IOException, URISyntaxException {
        Document doc = XML.getDocument(new URL(url), this.user, this.password);
        if (doc != null) {
            return doc;
        }
        throw new IOException("Cannot load document from " + url);
    }

    private static String url(
        String  url,
        String  request,
        WFSMeta meta
    ) {
        String post = "";
        int idx = url.lastIndexOf('?');
        if (idx >= 0) {
            post = url.substring(idx + 1);
            url = url.substring(0, idx);
        }
        url += "?request=" + request
            + "&service=wfs"
            + "&version="
            + StringUtils.urlEncode(
                meta.highestVersion(WFSMeta.WFS2_0_0).toString());
        if (post.length() > 0) {
            url += "&" + post;
        }
        return url;
    }

    private void parseDescribeStoredQueries(WFSMeta meta)
        throws IOException, URISyntaxException {

        WFSMeta.Operation op = meta.findOperation(DESCRIBE_STORED_QUERIES);
        if (op == null) {
            return;
        }

        String urlString = op.get != null
            ? url(op.get, DESCRIBE_STORED_QUERIES, meta)
            : capURLString.replace("GetCapabilities", DESCRIBE_STORED_QUERIES);

        Document dsqDoc = getDocument(urlString);
        meta.namespaces.join(dsqDoc);

        NodeList storedQueriesDesc = (NodeList)XML.xpath(
            dsqDoc, XPATH_STORED_QUERIES,
            XPathConstants.NODESET, NAMESPACES);

        for (int i = 0, n = storedQueriesDesc.getLength(); i < n; i++) {
            Element sqd = (Element)storedQueriesDesc.item(i);
            WFSMeta.StoredQuery sq = new WFSMeta.StoredQuery();
            sq.id = sqd.getAttribute("id");
            NodeList titles = sqd.getElementsByTagNameNS(WFS, "Title");
            if (titles.getLength() > 0) {
                sq.title = titles.item(0).getTextContent();
            }
            NodeList abstracts = sqd.getElementsByTagNameNS(WFS, "Abstract");
            if (abstracts.getLength() > 0) {
                sq.abstractDescription = abstracts.item(0).getTextContent();
            }
            NodeList parameters =
                sqd.getElementsByTagNameNS(WFS, "Parameter");
            for (int j = 0, m = parameters.getLength(); j < m; j++) {
                Element parameter = (Element)parameters.item(j);
                Field p = new Field(
                    parameter.getAttribute("name"),
                    parameter.getAttribute("type"));
                sq.parameters.add(p);
            }

            meta.storedQueries.add(sq);
        }
    }

    private void parseCapabilites(WFSMeta meta)
            throws IOException, URISyntaxException {

        Document capDoc = getDocument(this.capURLString);
        meta.namespaces.join(capDoc);

        meta.title = XML.xpathString(capDoc, XPATH_TITLE, NAMESPACES);
        meta.abstractDescription
            = XML.xpathString(capDoc, XPATH_ABSTRACT, NAMESPACES);

        NodeList versions = (NodeList)XML.xpath(
            capDoc, XPATH_OPERATIONS_VERSIONS,
            XPathConstants.NODESET, NAMESPACES);
        for (int i = 0, n = versions.getLength(); i < n; i++) {
            meta.versions.add(
                new WFSMeta.Version(versions.item(i).getTextContent()));
        }
        Collections.sort(meta.versions);

        NodeList outputFormats = (NodeList)XML.xpath(
            capDoc, XPATH_OUT_FORMATS,
            XPathConstants.NODESET, NAMESPACES);
        for (int i = 0, n = outputFormats.getLength(); i < n; i++) {
            meta.outputFormats.add(outputFormats.item(i).getTextContent());
        }

        NodeList nl = (NodeList)XML.xpath(
            capDoc, XPATH_OPERATIONS, XPathConstants.NODESET, NAMESPACES);
        for (int i = 0, n = nl.getLength(); i < n; i++) {
            WFSMeta.Operation operation = new WFSMeta.Operation();
            Element node = (Element)nl.item(i);
            operation.name = node.getAttribute("name");
            operation.get = XML.xpathString(
                node, XPATH_OPERATION_GET, NAMESPACES);
            String post = XML.xpathString(
                node, XPATH_OPERATION_POST, NAMESPACES);
            if (post != null) {
                operation.post = post;
            }

            NodeList outs = (NodeList)XML.xpath(
                node, XPATH_OPERATION_OUT_FORMATS,
                XPathConstants.NODESET, NAMESPACES);
            for (int j = 0, m = outs.getLength(); j < m; j++) {
                operation.outputFormats.add(outs.item(j).getTextContent());
            }

            NodeList constraints =
                node.getElementsByTagNameNS(OWS, "Constraint");

            for (int j = 0, m = constraints.getLength(); j < m; j++) {
                Element c = (Element)constraints.item(j);
                WFSMeta.Constraint constraint = new WFSMeta.Constraint();
                constraint.name = c.getAttribute("name");
                NodeList defVals = c.getElementsByTagNameNS(
                    OWS, "DefaultValue");
                if (defVals.getLength() > 0) {
                    constraint.value = defVals.item(0).getTextContent();
                }
                operation.constraints.add(constraint);
            }
            meta.operations.add(operation);
        }

        nl = (NodeList)XML.xpath(
            capDoc, XPATH_CONSTRAINTS,
            XPathConstants.NODESET, NAMESPACES);
        for (int i = 0, n = nl.getLength(); i < n; i++) {
            Element el = (Element)nl.item(i);
            WFSMeta.Constraint constraint = new WFSMeta.Constraint();
            constraint.name = el.getAttribute("name");
            NodeList defVals = el.getElementsByTagNameNS(OWS, "DefaultValue");
            if (defVals.getLength() > 0) {
                constraint.value = defVals.item(0).getTextContent();
            }
            NodeList allowed = (NodeList)XML.xpath(
                el, XPATH_CONSTRAINTS_ALLOWED,
                XPathConstants.NODESET, NAMESPACES);
            if (allowed != null) {
                for (int j = 0, m = allowed.getLength(); j < m; j++) {
                    constraint.allowed.add(allowed.item(j).getTextContent());
                }
            }
            meta.constraints.add(constraint);
        }

        nl = (NodeList)XML.xpath(
            capDoc, XPATH_FEATURETYPES, XPathConstants.NODESET, NAMESPACES);
        for (int i = 0, n = nl.getLength(); i < n; i++) {
            Element el = (Element)nl.item(i);

            WFSMeta.Feature feature = new WFSMeta.Feature();

            NodeList names = el.getElementsByTagNameNS(WFS, "Name");
            if (names.getLength() > 0) {
                feature.name = names.item(0).getTextContent();
            }

            NodeList titles = el.getElementsByTagNameNS(WFS, "Title");
            if (titles.getLength() > 0) {
                feature.title = titles.item(0).getTextContent();
            }

            NodeList abstracts = el.getElementsByTagNameNS(WFS, "Abstract");
            if (abstracts.getLength() > 0) {
                feature.abstractDescription
                    = abstracts.item(0).getTextContent();
            }

            NodeList defaultCRS = el.getElementsByTagNameNS(WFS, "DefaultCRS");
            if (defaultCRS.getLength() > 0) {
                feature.defaultCRS
                    = defaultCRS.item(0).getTextContent();
            }

            NodeList otherCRSs = el.getElementsByTagNameNS(WFS, "OtherCRS");
            for (int j = 0, m = otherCRSs.getLength(); j < m; j++) {
                feature.otherCRSs.add(otherCRSs.item(j).getTextContent());
            }

            NodeList outs = (NodeList)XML.xpath(
                el, XPATH_FEATURE_OUT_FORMATS,
                XPathConstants.NODESET, NAMESPACES);
            for (int j = 0, m = outs.getLength(); j < m; j++) {
                feature.outputFormats.add(outs.item(j).getTextContent());
            }

            feature.bbox = getBounds(el, NAMESPACES);

            meta.addFeature(feature);
        }
    }
} public final class Config {

    private Config() {
    }

    private static final Logger LOG = LoggerFactory.getLogger(Config.class.getName());

    /** Inner class to implicit synchronize the instance access. */
    private static final class Holder {
        private Holder() {
        }
        static final Config INSTANCE = new Config();
    }

    private boolean initialized;

    private Settings settings;

    private MIMETypes mimeTypes;

    private ProcessingConfiguration processingConfig;

    private ProxyConfiguration proxyConfig;

    private static final long FIVEHUNDRED = 500;

    /**
     * Access the instance of the configuration.
     * Use #load before to initialize it.
     * @return The configuration instance.
     */
    public static Config getInstance() {
        synchronized (Holder.INSTANCE) {
            try {
                while (!Holder.INSTANCE.initialized) {
                    Holder.INSTANCE.wait(FIVEHUNDRED);
                }
            } catch (InterruptedException ie) {
                Thread.currentThread().interrupt();
            }
            return Holder.INSTANCE;
        }
    }

    /**
     * @return The application Settings
     */
    public ApplicationSettings getApplicationSettings() {
        return settings.getApplicationSettings();
    }

    /**
     * @return the services
     */
    public ServiceSettings getServices() {
        return settings.getServiceSettings();
    }

    /**
     * @return the mimeTypes
     */
    public MIMETypes getMimeTypes() {
        return mimeTypes;
    }

    /**
     * @return the processingConfig
     */
    public ProcessingConfiguration getProcessingConfig() {
        return processingConfig;
    }

    /**
     * @return the proxy config
     */
    public ProxyConfiguration getProxyConfig() {
        return proxyConfig;
    }

    @Override
    public String toString() {
        return (proxyConfig == null ? "Proxy Settings: {}" : getProxyConfig())
            + ",\n " + getApplicationSettings()
            + ",\n " + getProcessingConfig()
            + ",\n " + getServices()
            + ",\n MIME-Types: " + getMimeTypes();
    }

    private static Settings loadSettings(File file)
        throws IOException {
        try {
            if (file != null && file.isFile() && file.canRead()) {
                return new Settings(file);
            }
            return new Settings();
        } catch (SAXException
                | ParserConfigurationException
                | IOException e) {
            LOG.error(e.getMessage(), Holder.INSTANCE);
            throwConfigFailureException(Settings.getName());
        }
        // Not reached.
        return null;
    }

    /**
     * Mark global config as unused.
     * @throws IOException when anythong goes wrong
     */
    private static void uninitialized()
        throws IOException {
        synchronized (Holder.INSTANCE) {
            LOG.info("No config directory given, starting with standard values...");
            Holder.INSTANCE.settings = loadSettings(null);
            Holder.INSTANCE.processingConfig =
                ProcessingConfiguration.loadDefault();
            Holder.INSTANCE.mimeTypes = MIMETypes.loadDefault();
            Holder.INSTANCE.initialized = true;
            Holder.INSTANCE.notifyAll();
        }
    }

    /**
     * Load configurations.
     * @param dirname The directory with the configuration files.
     * @throws IOException If something went wrong.
     */
    public static void initialize(String dirname) throws IOException {
        Controller.logToAppLog(I18n.format("dlc.start", Info.getVersion()));
        System.setProperty("java.util.logging.manager",
            "org.apache.logging.log4j.jul.LogManager");
        try {
            Logging.GEOTOOLS.setLoggerFactory(
                "org.geotools.util.logging.Log4JLoggerFactory");
        } catch (ClassNotFoundException e) {
            LOG.warn("Failed to initialize GeoTools logging subsystem: "
                + e.getLocalizedMessage());
        }
        if (dirname == null) {
            uninitialized();
        } else {
            synchronized (Holder.INSTANCE) {
                try {
                    load(dirname);
                } finally {
                    Holder.INSTANCE.initialized = true;
                    Holder.INSTANCE.notifyAll();
                }
            }
        }
        LOG.trace("System Properties:");
        System.getProperties().keySet().stream().map(
            k -> (String)k).sorted().forEach(k -> {
            LOG.trace(String.format("\t%s=%s", k,
                System.getProperty(k)));
        });
        Controller.logToAppLog(I18n.format("dlc.config", Config.getInstance()));
    }

    private static void load(String dirname) throws IOException {
        LOG.info("config directory: " + dirname);

        File dir = new File(dirname);

        if (!dir.isDirectory()) {
            throw new IOException("'" + dirname + "' is not a directory.");
        }

        File proxy = new File(dir, ProxyConfiguration.PROXY_CONFIG_FILE);
        if (proxy.isFile() && proxy.canRead()) {
            Holder.INSTANCE.proxyConfig = ProxyConfiguration.read(proxy);
            Holder.INSTANCE.proxyConfig.apply();
        } else {
            LOG.info("No Proxy config found, starting without proxy settings");
        }

        File services = new File(dir, Settings.SETTINGS_FILE);
        if (services.isFile() && services.canRead()) {
            Holder.INSTANCE.settings = loadSettings(services);
        } else {
            Holder.INSTANCE.settings = loadSettings(null);
            LOG.info("Settings config not found, using fallback...");
        }

        File procConfig = new File(
            dir, ProcessingConfiguration.PROCESSING_CONFIG_FILE);
        if (procConfig.isFile() && procConfig.canRead()) {
            Holder.INSTANCE.processingConfig =
                ProcessingConfiguration.read(procConfig);
        } else {
            Holder.INSTANCE.processingConfig =
                ProcessingConfiguration.loadDefault();
            LOG.info("Processing config not found, using fallback...");
        }

        if (Holder.INSTANCE.getProcessingConfig() == null) {
            throwConfigFailureException(ProcessingConfiguration.getName());
        }

        File mimeTypes = new File(dir, MIMETypes.MIME_TYPES_FILE);
        if (mimeTypes.isFile() && mimeTypes.canRead()) {
            Holder.INSTANCE.mimeTypes =
                MIMETypes.read(mimeTypes);
        } else {
            Holder.INSTANCE.mimeTypes = MIMETypes.loadDefault();
            LOG.info("MimeTypes config not found, using fallback...");
        }

        if (Holder.INSTANCE.getMimeTypes() == null) {
            throwConfigFailureException(MIMETypes.getName());
        }
    }

    private static void throwConfigFailureException(String configName)
        throws IOException {
        throw new IOException(
                "Failed on XML file for '"
                        + configName
                        + "'");
    }
} public class FileTracker {

    private static final Logger LOG = LoggerFactory.getLogger(FileTracker.class.getName());

    private File directory;
    private Map<File, Long> last;
    private Map<File, Long> current;

    /**
     * @param directory The directory to track.
     */
    public FileTracker(File directory) {
        this.directory = directory;
    }

    /**
     * Scans the directory.
     * @return true if success false otherwise.
     */
    public boolean scan() {
        File[] files = this.directory.listFiles();
        if (files == null) {
            LOG.error("Cannot read files from '{}'.", directory);
            return false;
        }

        this.current = new TreeMap<>();
        for (File file: files) {
            this.current.put(file, file.lastModified());
        }
        return true;
    }

    /**
     * Pushes the current scan for later comparison.
     */
    public void push() {
        this.last = this.current;
        this.current = null;
    }

    /**
     * Returns list of files new since last scan.
     * @return The list of new files.
     */
    private List<File> newFiles() {
        ArrayList<File> files = new ArrayList<>();
        for (Map.Entry<File, Long> entry: this.current.entrySet()) {
            Long time = this.last.get(entry.getKey());
            if (time == null || time < entry.getValue()) {
                files.add(entry.getKey());
            }
        }
        return files;
    }

    /**
     * @return the directory
     */
    public File getDirectory() {
        return directory;
    }

    private List<File> glob(Collection<File> all, String pattern) {
        ArrayList<File> files = new ArrayList<>(all.size());
        PathMatcher matcher =
            FileSystems.getDefault().getPathMatcher("glob:" + pattern);
        for (File file: all) {
            if (matcher.matches(new File(file.getName()).toPath())) {
                files.add(file);
            }
        }
        return files;
    }

    /**
     * Globs pattern in new files.
     * @param pattern The glob pattern.
     * @return The new files that match the pattern.
     */
    public List<File> deltaGlob(String pattern) {
        return glob(newFiles(), pattern);
    }

    /**
     * Globs pattern in all files.
     * @param pattern The glob pattern.
     * @return The files that match the pattern.
     */
    public List<File> globalGlob(String pattern) {
        return glob(current.keySet(), pattern);
    }

    /**
     * Retrieve file with pattern directly from file systems.
     * @param pattern The pattern to match, never <code>null</code>.
     * @return The files that match the pattern,
     * <code>null</code> if files could not read.
     */
    public List<File> retrieveFilesWithoutScan(String pattern) {
        File[] files = this.directory.listFiles();
        if (files == null) {
            LOG.error("Cannot read files from '{}'.", directory);
            return null;
        }

        List<File> candidates = new ArrayList<>();
        for (File file : files) {
            candidates.add(file);
        }
        return glob(candidates, pattern);
    }

} public class Log {

    private final SimpleDateFormat dateFormat
        = new SimpleDateFormat("yyyy-MM-dd-HH-mm:ss");

    private File file;
    private PrintWriter out;


    /**
     * @param file The file to log to.
     */
    public Log(File file) {
        this.file = file;
    }

    /**
     * Opens the underlaying log file for writing.
     * @throws IOException opening the underlaying file failed.
     */
    public synchronized void open() throws IOException {
        if (this.out != null) {
            this.out.close();
            this.out = null;
        }
        this.out = new PrintWriter(new FileWriter(this.file), true);
    }

    /**
     * @param msg The message to log.
     */
    public synchronized void log(String msg) {
        if (this.out != null) {
            Date now = new Date();
            out.format("%s: %s", dateFormat.format(now), msg);
            out.println();
        }
    }

    /**
     * Closes this logger.
     */
    public synchronized void close() {
        if (this.out != null) {
            this.out.close();
            this.out = null;
        }
    }
} public final class Misc {

    private static final String DF_PATTERN = "yyyyMMddHHmmss";

    /** Number of tries before giving up searching for collisions. */
    private static final int MAX_TRIES = 5000;

    private static final int TEN = 10;

    /** Common prefix. */
    public static final String PREFIX = "gdibydl-";

    private static final Logger LOG = LoggerFactory.getLogger(Misc.class.getName());

    private Misc() {
    }

    /**
     * Creates a directory in the parent directory. The name
     * is created from the common prefix and the current time.
     * To avoid collisions a integer is append in case.
     * @param parent The parent directory.
     * @return The new created directory or null if the creation
     * failed.
     */
    public static File createDir(File parent) {
        return createDir(parent, PREFIX);
    }

    /**
     * Creates a directory in the parent directory. The name
     * is created from the given prefix and the current time.
     * To avoid collisions an integer is appended in case.
     * @param parent The parent directory.
     * @param prefix The used prefix.
     * @return The new created directory or null if the creation
     * failed.
     */
    public static File createDir(File parent, String prefix) {

        Date now = new Date();

        SimpleDateFormat formatter = new SimpleDateFormat(DF_PATTERN);

        String dir = prefix + formatter.format(now);
        File path = new File(parent, dir);
        int count = 0;
        while (count < MAX_TRIES && path.exists()) {
            ++count;
            dir = prefix + formatter.format(now) + "-" + count;
            path = new File(parent, dir);
        }

        return count < MAX_TRIES && path.mkdirs() ? path : null;
    }

    /**
     * Creates a unique file name in the parent directory. The name
     * is created from the prefix, the extension and the current time.
     * To avoid collisions an integer is appended in case.
     * @param parent The parent directory.
     * @param prefix The used prefix.
     * @param ext The used extension.
     * @param tmpFiles Temporary files not in file system.
     * @return The unique or null if the creation failed.
     */
    public static File uniqueFile(
        File      parent,
        String    prefix,
        String    ext,
        Set<File> tmpFiles
    ) {
        Date now = new Date();

        SimpleDateFormat formatter = new SimpleDateFormat(DF_PATTERN);

        String f = prefix + formatter.format(now) + "." + ext;
        File path = new File(parent, f);
        int count = 0;
        while (count < MAX_TRIES
        && (path.exists() || (tmpFiles != null && tmpFiles.contains(path)))) {
            ++count;
            f = prefix + formatter.format(now) + "-" + count + "." + ext;
            path = new File(parent, f);
        }

        return count < MAX_TRIES ? path : null;
    }

    /**
     * returns file from resources.
     * @param path path or filename in rsources
     * @return returns the file
     */
    public static InputStream getResource(String path) {
        return Misc.class.getClassLoader().getResourceAsStream(path);
    }

    /**
     * Checks if a String is an Integer.
     * @param s string
     * @return true if integer; false if not
     */
    public static boolean isInteger(String s) {
        return isInteger(s, TEN);
    }

    private static boolean isInteger(String s, int radix) {
        if (s.isEmpty()) {
            return false;
        }
        for (int i = 0; i < s.length(); i++) {
            if (i == 0 && s.charAt(i) == '-') {
                if (s.length() == 1) {
                    return false;
                } else {
                    continue;
                }
            }
            if (Character.digit(s.charAt(i), radix) < 0) {
                return false;
            }
        }
        return true;
    }
    /**
     * Tries to start the external default browser.
     * @param url url
     */
    public static void startExternalBrowser(String url) {
        LOG.info("Starting external browser with address:" + url);
        try {
            new ProcessBuilder("x-www-browser", url).start();
        } catch (IOException e) {
            Runtime rt = Runtime.getRuntime();
            try {
                rt.exec("xdg-open " + url);
            } catch (IOException ex) {
                //When every standard fails, do the hard work
                String os = System.getProperty("os.name").toLowerCase();
                try {
                    if (os.contains("win")) {
                        rt.exec("rundll32 url.dll,FileProtocolHandler " + url);
                    } else if (os.contains("mac")) {
                        rt.exec("open " + url);
                    } else {
                        String[] browsers = {"epiphany",
                                "firefox",
                                "mozilla",
                                "konqueror",
                                "netscape",
                                "opera",
                                "links",
                                "lynx"};

                        StringBuilder cmd = new StringBuilder();
                        for (int i = 0; i < browsers.length; i++) {
                            cmd.append(i == 0 ? "" : " || ")
                               .append(browsers[i])
                               .append(" \"")
                               .append(url)
                               .append("\" ");
                        }
                        rt.exec(new String[] {"sh", "-c", cmd.toString()});
                    }
                } catch (IOException exc) {
                    WarningPopup wp = new WarningPopup();
                    wp.popup(I18n.format("gui.link-fail", url));
                    LOG.error("First Call"
                            + e.getMessage() + "\n\n\n" + "Second Call"
                            + ex.getMessage() + "\n\n\n" + "Third Call"
                            + exc.getMessage(),
                        exc);
                }
            }
        }
    }
} public class StringUtils {

    private static final Logger LOG = LoggerFactory.getLogger(StringUtils.class.getName());

    private static final int TEN = 10;

    private StringUtils() {
    }

    /**
     * URL-encodes a given string in UTF-8.
     *
     * @param s The string to encode.
     * @return The encodes string.
     */
    public static String urlEncode(String s) {
        return urlEncode(s, "UTF-8");
    }

    /**
     * URL-encodes a given string in a given encoding.
     *
     * @param s   The string to encode.
     * @param enc The encoding to use.
     * @return The encodes string.
     */
    private static String urlEncode(String s, String enc) {
        try {
            return URLEncoder.encode(s, enc);
        } catch (UnsupportedEncodingException e) {
            LOG.error("encoding problem", e);
        }
        return s;
    }

    /**
     * Searches a needle of a set of needles in a haystack.
     *
     * @param haystack The haystack.
     * @param needles  The needles.
     * @return true if the haystack contains one of the needles.
     * false otherwise.
     */
    public static boolean contains(String[] haystack, String[] needles) {
        return Stream.of(needles)
                .anyMatch(within(haystack));
    }

    private static Predicate<String> within(String[] haystack) {
        return needle -> Stream.of(haystack)
                .anyMatch(needle::equals);
    }

    /**
     * Split a command line.
     *
     * @param toProcess the command line to process.
     * @return the command line broken into strings.
     * An empty or null toProcess parameter results in a zero sized array.
     * @throws IllegalArgumentException Thrown if quotes are unbalanced.
     */
    public static String[] splitCommandLine(String toProcess)
            throws IllegalArgumentException {

        if (toProcess == null || toProcess.length() == 0) {
            //no command? no string
            return new String[0];
        }
        // parse with a simple finite state machine

        State state = State.NORMAL;
        StringTokenizer tok = new StringTokenizer(toProcess, "\"\' ", true);
        ArrayList<String> result = new ArrayList<>();
        StringBuilder current = new StringBuilder();
        boolean lastTokenHasBeenQuoted = false;

        while (tok.hasMoreTokens()) {
            String nextTok = tok.nextToken();
            switch (state) {
                case IN_QUOTE:
                    if ("\'".equals(nextTok)) {
                        lastTokenHasBeenQuoted = true;
                        state = State.NORMAL;
                    } else {
                        current.append(nextTok);
                    }
                    break;
                case IN_DOUBLE_QUOTE:
                    if ("\"".equals(nextTok)) {
                        lastTokenHasBeenQuoted = true;
                        state = State.NORMAL;
                    } else {
                        current.append(nextTok);
                    }
                    break;
                default:
                    if ("\'".equals(nextTok)) {
                        state = State.IN_QUOTE;
                    } else if ("\"".equals(nextTok)) {
                        state = State.IN_DOUBLE_QUOTE;
                    } else if (" ".equals(nextTok)) {
                        if (lastTokenHasBeenQuoted || current.length() != 0) {
                            result.add(current.toString());
                            current.setLength(0);
                        }
                    } else {
                        current.append(nextTok);
                    }
                    lastTokenHasBeenQuoted = false;
                    break;
            }
        }
        if (lastTokenHasBeenQuoted || current.length() != 0) {
            result.add(current.toString());
        }
        if (state == State.IN_QUOTE || state == State.IN_DOUBLE_QUOTE) {
            throw new IllegalArgumentException(
                    "unbalanced quotes in " + toProcess);
        }
        return result.toArray(new String[result.size()]);
    }

    /**
     * Splits a string by a pattern.
     *
     * @param s     The string to split.
     * @param delim The delimeter.
     * @param keep  Indicates if the delimeters should be kept.
     * @return The splitted string.
     */
    public static String[] split(String s, Pattern delim, boolean keep) {
        if (s == null) {
            s = "";
        }
        int lastMatch = 0;
        ArrayList<String> parts = new ArrayList<>();
        Matcher m = delim.matcher(s);
        while (m.find()) {
            String x = s.substring(lastMatch, m.start());
            if (!x.isEmpty()) {
                parts.add(x);
            }
            if (keep) {
                parts.add(m.group(0));
            }
            lastMatch = m.end();
        }
        String x = s.substring(lastMatch);
        if (!x.isEmpty()) {
            parts.add(x);
        }
        return parts.toArray(new String[parts.size()]);
    }

    /**
     * Joins a list of objects with a separator to a string.
     *
     * @param s   The list of objects.
     * @param sep The separator.
     * @return the joined string.
     */
    public static String join(List<?> s, String sep) {
        return s.stream()
                .map(Object::toString)
                .collect(Collectors.joining(sep));
    }

    /**
     * How many digit place does a number need?
     *
     * @param n The number.
     * @return the number of digits.
     */
    public static int places(int n) {
        int places = 1;
        if (n < 0) {
            places++;
            n = -n;
        }
        for (int value = TEN; n >= value; value *= TEN) {
            places++;
        }
        return places;
    }

    /**
     * Convert a whitespace separated list of double strings
     * to a double array.
     *
     * @param s the string with the double values.
     * @return the converted array. Returns an empty array
     * if the conversion fails.
     */
    public static double[] toDouble(String s) {
        try {
            String[] parts = s.split("[ \\t]+");
            double[] x = new double[parts.length];
            for (int i = 0; i < parts.length; i++) {
                x[i] = Double.parseDouble(parts[i]);
            }
            return x;
        } catch (NumberFormatException nfe) {
            return new double[0];
        }
    }

    /**
     * Splits a given string by a separator and ignores parts
     * that starts with a given prefixes (case insensitve).
     * Afterwards the parts are re-joined with the separator.
     *
     * @param s        The string.
     * @param sep      The separator.
     * @param prefixes The prefixes.
     * @return The treated string.
     */
    public static String ignorePartsWithPrefix(
            String s, String sep, String[] prefixes) {

        List<String> lcPrefixes = Stream.of(prefixes)
            .map(String::toLowerCase)
            .collect(Collectors.toList());

        List<String> use = Stream.of(s.split(Pattern.quote(sep)))
            .filter(str -> lcPrefixes.stream()
                .noneMatch(p -> str.toLowerCase().startsWith(p)))
            .collect(Collectors.toList());

        return join(use, sep);
    }

    /**
     * Checks if two strings are equal or both null.
     *
     * @param a The first string.
     * @param b The second strinh.
     * @return true if both strings are equal or both null.
     */
    public static boolean nullOrEquals(String a, String b) {
        return a == null && b == null || a != null && b != null && a.equals(b);
    }

    /**
     * States for the command line splitting.
     */
    private enum State {
        NORMAL,
        IN_QUOTE,
        IN_DOUBLE_QUOTE
    }
} public class XML {

    private static final Logger LOG = LoggerFactory.getLogger(XML.class.getName());

    private static final String UTF8 = "UTF-8";

    private XML() {
    }

    /**
     * Loads an XML document from a file.
     *
     * @param fileName the name of the XML file.
     * @return the loaded XML document of null if there was an error.
     * @throws SAXException if the xml is wrong
     * @throws ParserConfigurationException if the config is wrong
     * @throws IOException if something in io is wrong
     */
    public static Document getDocument(File fileName)
            throws SAXException, ParserConfigurationException, IOException {
        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
        DocumentBuilder builder = factory.newDocumentBuilder();
        return builder.parse(fileName);
    }

    /**
     * Loads an XML document from an input stream.
     *
     * @param input the input stream.
     * @return the loaded XML document of null if there was an error.
     * @throws SAXException if the xml is wrong
     * @throws ParserConfigurationException if the config is wrong
     * @throws IOException if something in io is wrong
     */
    public static Document getDocument(InputStream input)
            throws SAXException, ParserConfigurationException, IOException {
        return getDocument(input, null);
    }

    /**
     * Loads an XML document from an input stream.
     *
     * @param input          the input stream.
     * @param namespaceAware Load namespace aware?
     * @return the loaded XML document of null if there was an error.
     * @throws SAXException if the xml is wrong
     * @throws ParserConfigurationException if the config is wrong
     * @throws IOException if something in io is wrong
     */
    public static Document getDocument(
            InputStream input, Boolean namespaceAware
    ) throws SAXException, ParserConfigurationException, IOException {
        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
        if (namespaceAware != null) {
            factory.setNamespaceAware(namespaceAware);
        }
        DocumentBuilder builder = factory.newDocumentBuilder();
        return builder.parse(input);
    }

    /**
     * Gets an XML Document from a remote location.
     *
     * @param url the URL
     * @return and XML Document
     * @throws URISyntaxException if the url is wrong
     * @throws IOException if something in io is wrong
     */
    public static Document getDocument(URL url)
            throws URISyntaxException, IOException {
        return getDocument(url, null, null);
    }

    /**
     * Gets an XML Document from a remote location.
     *
     * @param url the URL
     * @param nameSpaceaware if namespace aware or not
     * @return and XML Document
     * @throws URISyntaxException if the url is wrong
     * @throws IOException if something in io is wrong
     */
    public static Document getDocument(URL url, boolean nameSpaceaware)
            throws URISyntaxException, IOException {
        return getDocument(url, null, null, nameSpaceaware);
    }
    /**
     * Gets an XML Document from a remote location.
     *
     * @param url            the URL
     * @param userName       the Username {NULL if none needed}
     * @param password       the Password {NULLL if none needed}
     * @param nameSpaceAware if namespace aware or not
     * @return an XML Document
     * @throws URISyntaxException if the url is wrong
     * @throws IOException if something in io is wrong
     */
    public static Document getDocument(
            URL url,
            String userName,
            String password,
            boolean nameSpaceAware)
            throws URISyntaxException, IOException {
        return getDocument(url, userName, password, nameSpaceAware, null);
    }
    /**
     * Gets an XML Document from a remote location.
     *
     * @param url            the URL
     * @param userName       the Username {NULL if none needed}
     * @param password       the Password {NULLL if none needed}
     * @param nameSpaceAware if namespace aware or not
     * @param postparams Post parameters, null if get shall be used
     * @return an XML Document
     * @throws URISyntaxException if the url is wrong
     * @throws IOException if something in io is wrong
     */
    public static Document getDocument(
            URL url,
            String userName,
            String password,
            boolean nameSpaceAware,
            HttpEntity postparams)
            throws URISyntaxException, IOException {
        try (
            CloseableHttpClient client =
                HTTP.getClient(url, userName, password);
        ) {
            if (postparams == null) {
                HttpGet request = HTTP.getGetRequest(url);
                DocumentResponseHandler handler
                        = new DocumentResponseHandler(request);
                handler.setNamespaceAware(nameSpaceAware);
                return client.execute(request, handler);
            } else {
                HttpPost postreq = new HttpPost(url.toString());
                postreq.setHeader("Content-Type", "application/xml");
                postreq.setEntity(postparams);
                DocumentResponseHandler handler
                        = new DocumentResponseHandler(postreq);
                handler.setNamespaceAware(nameSpaceAware);
                return client.execute(postreq, handler);
            }
        }
    }

    /**
     * Gets an XML Document from a remote location.
     *
     * @param url      the URL
     * @param userName the Username {NULL if none needed}
     * @param password the Password {NULLL if none needed}
     * @return an XML Document
     * @throws URISyntaxException if the url is wrong
     * @throws IOException if something in io is wrong
     */
    public static Document getDocument(
            URL url,
            String userName,
            String password)
            throws URISyntaxException, IOException {
        return getDocument(url, userName, password, true);
    }

    /**
     * Gets an XML Document from a remote location.
     *
     * @param url      the URL
     * @param userName the Username {NULL if none needed}
     * @param password the Password {NULLL if none needed}
     * @param postparams Post params if needed
     * @return an XML Document
     * @throws URISyntaxException if the url is wrong
     * @throws IOException if something in io is wrong
     */
    public static Document getDocument(
            URL url,
            String userName,
            String password,
            HttpEntity postparams)
            throws URISyntaxException, IOException {
        return getDocument(url, userName, password, true, postparams);
    }
    /**
     * Builds an XML Document from a String.
     *
     * @param xmlString The String
     * @return The build Document
     * @throws SAXException if the xml is wrong
     * @throws ParserConfigurationException if the config is wrong
     * @throws IOException if something in io is wrong
     */
    public static final Document getDocument(String xmlString)
            throws SAXException, ParserConfigurationException, IOException {
        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
        DocumentBuilder builder = factory.newDocumentBuilder();
        return builder.parse(xmlString);
    }

    /**
     * Getting a document by posting an xml to a url.
     * @param url the url
     * @param userName the username
     * @param password the password
     * @param postXML the xml to post
     * @param nameSpaceAware if namespace aware
     * @return the returned document
     * @throws URISyntaxException if the url is wrong
     * @throws IOException if something in io is wrong
     */
    public static final Document getDocument(
            URL url,
            String userName,
            String password,
            String postXML,
            boolean nameSpaceAware)
            throws URISyntaxException, IOException {
        try (
            CloseableHttpClient client =
                HTTP.getClient(url, userName, password);
        ) {
            HttpPost request = HTTP.getPostRequest(url);
            DocumentResponseHandler handler =
                    new DocumentResponseHandler(request);
            handler.setNamespaceAware(nameSpaceAware);
            InputStream inputStream =
                    new ByteArrayInputStream(postXML.getBytes());
            InputStreamEntity inputStreamEntity = new InputStreamEntity(
                    inputStream);
            request.setHeader("Content-type", "text/xml;charset=utf-8");
            request.setEntity(inputStreamEntity);
            return client.execute(request, handler);
        }
    }

    /**
     * Gets a Document by posting an XML to the given URL.
     * @param url the URL to post to
     * @param userName the username
     * @param password the password
     * @param postXML the XML that should be posted
     * @param nameSpaceAware if namespace aware or not
     * @return the build document
     * @throws URISyntaxException if the url is wrong
     * @throws TransformerException if the xml is wrong
     * @throws IOException if something in io is wrong
     */
    public static final Document getDocument(
            URL url,
            String userName,
            String password,
            Document postXML,
            boolean nameSpaceAware)
            throws URISyntaxException, IOException, TransformerException {

        Document doc = null;
        doc = getDocument(url, userName, password, documentToString(
                            postXML), nameSpaceAware);
        return doc;
    }

    /**
     * Creates a new XPath without a namespace context.
     *
     * @return the new XPath.
     */
    public static final XPath newXPath() {
        return newXPath(null, null);
    }

    /**
     * Creates a new XPath with a given namespace context.
     *
     * @param namespaceContext The namespace context to be used or null
     *                         if none should be used.
     * @param resolver         The name space resolver.
     * @return The new XPath
     */
    public static final XPath newXPath(
            NamespaceContext namespaceContext,
            XPathVariableResolver resolver
    ) {
        XPathFactory factory = XPathFactory.newInstance();
        XPath xpath = factory.newXPath();
        if (namespaceContext != null) {
            xpath.setNamespaceContext(namespaceContext);
        }

        if (resolver != null) {
            xpath.setXPathVariableResolver(resolver);
        }
        return xpath;
    }

    /**
     * Evaluates an XPath query on a given object and returns the result
     * as a given type. No namespace context is used.
     *
     * @param root      The object which is used as the root of the tree to
     *                  be searched in.
     * @param query     The XPath query
     * @param returnTyp The type of the result.
     * @return The result of type 'returnTyp' or null if something
     * went wrong during XPath evaluation.
     */
    public static final Object xpath(
            Object root,
            String query,
            QName returnTyp
    ) {
        return xpath(root, query, returnTyp, null);
    }

    /**
     * Evaluates an XPath query on a given object and returns the result
     * as a string. A given namespace context is used.
     *
     * @param root             The object which is used as the root of the tree
     *                         to be searched in.
     * @param query            The XPath query
     * @param namespaceContext The namespace context to be used or null
     *                         if none should be used.
     * @return The result of the query or null if something went wrong
     * during XPath evaluation.
     */
    public static final String xpathString(
            Object root, String query, NamespaceContext namespaceContext
    ) {
        return (String) xpath(
                root, query, XPathConstants.STRING, namespaceContext);
    }

    /**
     * Evaluates an XPath query on a given object and returns the result
     * as a given type. Optionally a namespace context is used.
     *
     * @param root             The object which is used as the root of the tree
     *                         to be searched in.
     * @param query            The XPath query
     * @param returnType       The type of the result.
     * @param namespaceContext The namespace context to be used or null
     *                         if none should be used.
     * @return The result of type 'returnTyp' or null if something
     * went wrong during XPath evaluation.
     */
    public static final Object xpath(
            Object root,
            String query,
            QName returnType,
            NamespaceContext namespaceContext
    ) {
        return xpath(root, query, returnType, namespaceContext, null);
    }

    /**
     * xpath evaluates ax XPath expression.
     *
     * @param root             The root object of the evaluation being applied
     *                         to.
     * @param query            The XPath query.
     * @param returnType       The type to be returned.
     * @param namespaceContext An optional namespace context.
     * @param variables        A map variables to be used during evaluation.
     * @return The result of the evaluation.
     */
    public static final Object xpath(
            Object root,
            String query,
            QName returnType,
            NamespaceContext namespaceContext,
            Map<String, String> variables
    ) {
        if (root == null) {
            return null;
        }

        XPathVariableResolver resolver = variables != null
                ? new MapXPathVariableResolver(variables)
                : null;

        try {
            XPath xpath = newXPath(namespaceContext, resolver);
            if (xpath != null) {
                return xpath.evaluate(query, root, returnType);
            }
        } catch (XPathExpressionException xpee) {
            LOG.error(xpee.getLocalizedMessage(), xpee);
        }

        return null;
    }

    /**
     * containsTags is a cheap way to detect if a XML file
     * contains some special tags. Useful for error detection.
     *
     * @param file The XML file to check.
     * @param tags the list of tags to check for.
     * @return The first tag that was found. Null if the tags are not in file.
     * @throws XMLStreamException if an exception happends while
     *                            processoing the XML file.
     * @throws IOException        if an I/O error happens.
     */
    public static final String containsTags(File file, String[] tags)
            throws XMLStreamException, IOException {

        XMLInputFactory xmlif = XMLInputFactory.newInstance();
        XMLStreamReader xmlr = xmlif.createXMLStreamReader(
                new BufferedInputStream(
                        new FileInputStream(file)));
        try {
            while (xmlr.hasNext()) {
                if (xmlr.next() == XMLEvent.START_ELEMENT) {
                    String needle = xmlr.getLocalName();
                    for (String tag : tags) {
                        if (needle.equals(tag)) {
                            return tag;
                        }
                    }
                }
            }
        } finally {
            xmlr.close();
        }
        return null;
    }

    /**
     * Prints a XML Document to a output.
     *
     * @param doc the document
     * @param out the outputstream i.e. System.out
     */
    public static void printDocument(Document doc, OutputStream out) {
        try {
            TransformerFactory tf = TransformerFactory.newInstance();
            Transformer transformer = tf.newTransformer();
            transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION,
                    "no");
            transformer.setOutputProperty(OutputKeys.METHOD, "xml");
            transformer.setOutputProperty(OutputKeys.ENCODING, UTF8);

            transformer.transform(new DOMSource(doc),
                    new StreamResult(new OutputStreamWriter(out, UTF8)));
        } catch (IOException
                | TransformerException e) {
            LOG.error(e.getMessage(), e);
        }
    }

    /**
     * Prints a Document to a String.
     * @param doc the XML document
     * @return the document as String
     * @throws TransformerException when something goes wrong
     */
    public static String documentToString(Document doc)
            throws TransformerException {
        TransformerFactory tf = TransformerFactory.newInstance();
        Transformer transformer = tf.newTransformer();
        transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
        StringWriter writer = new StringWriter();
        transformer.transform(new DOMSource(doc), new StreamResult(writer));
        return writer.getBuffer().toString().replaceAll("\n|\r", "");
    }

    /**
     * Wraps an XML writing into a ContentProducer for
     * lazy serialization.
     * @param doc the XML document to wrap.
     * @return The ContentProducer.
     * @throws IOException in case of errors
     */
    public static ContentProducer toContentProducer(final Document doc) {
        return out -> {
            try {
                TransformerFactory tf = TransformerFactory.newInstance();
                Transformer transformer = tf.newTransformer();
                transformer.setOutputProperty(OutputKeys.METHOD, "xml");
                transformer.setOutputProperty(OutputKeys.ENCODING, UTF8);
                transformer.setOutputProperty(OutputKeys.INDENT, "no");
                transformer.transform(
                    new DOMSource(doc), new StreamResult(out));
            } catch (TransformerException te) {
                throw new IOException(te);
            }
        };
    }
} public static Document create(
    DownloadStep dls,
    Set<String> usedVars,
    WFSMeta meta
) throws ConverterException {
    return create(dls, usedVars, meta, false, -1, -1, false);
}
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPathConstants;
import de.bayern.gdi.processor.job.AtomDownloadJob;
import de.bayern.gdi.processor.job.BroadcastJob;
import de.bayern.gdi.processor.job.CloseLogJob;
import de.bayern.gdi.processor.job.FileDownloadJob;
import de.bayern.gdi.processor.job.FilesDownloadJob;
import de.bayern.gdi.processor.job.GMLCheckJob;
import de.bayern.gdi.processor.job.DownloadStepJob;
import de.bayern.gdi.processor.job.LogMetaJob;
import de.bayern.gdi.processor.job.OpenLogJob;
import de.bayern.gdi.processor.job.UnzipJob;
import de.bayern.gdi.processor.listener.CountListener;
import org.apache.http.HttpEntity;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.entity.EntityTemplate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import de.bayern.gdi.model.DownloadStep;
import de.bayern.gdi.model.Parameter;
import de.bayern.gdi.services.WFSMeta;
import de.bayern.gdi.services.WFSMetaExtractor;
import de.bayern.gdi.config.Config;
import de.bayern.gdi.utils.FileTracker;
import de.bayern.gdi.utils.I18n;
import de.bayern.gdi.utils.Log;
import de.bayern.gdi.utils.Misc;
import de.bayern.gdi.utils.StringUtils;
import de.bayern.gdi.utils.XML;
import static de.bayern.gdi.processor.DownloadStepConverter.QueryType.DATASET;
import static de.bayern.gdi.processor.DownloadStepConverter.
    QueryType.SQLQUERY;
import static de.bayern.gdi.processor.DownloadStepConverter.
    QueryType.STOREDQUERY;
import static de.bayern.gdi.processor.WFSPostParamsBuilder.create;
        }
        return sb.toString();
    }

    private static final String[] IGNORE_WFS = {
        "service=",
        "request=",
        "acceptversions="
    };

    private static String vendorSpecific(String url) {
        int idx = url.lastIndexOf('?');
        if (idx < 0) {
            return "";
        }
        return StringUtils.ignorePartsWithPrefix(
            url.substring(idx + 1),
            "&",
            IGNORE_WFS);
    }

    private static String baseURL(String url) {
        int idx = url.indexOf('?');
        return idx >= 0 ? url.substring(0, idx) : url;

    }

    private static String capURL(String base) {
        return base
            + "?service=WFS&request=GetCapabilities&acceptversions=2.0.0";
    }

    private static String wfsURL(
        DownloadStep dls,
        Set<String>  usedVars,
        WFSMeta      meta
    ) {
        String url = dls.getServiceURL();
        String base = baseURL(url);
        String vendor = vendorSpecific(url);
        String getFeaturesURL = meta.findOperation(GETFEATURE).getGET();
        if (getFeaturesURL.startsWith("/")) {
            base += getFeaturesURL;
        } else {
            base = getFeaturesURL;
        }

        String version = StringUtils.urlEncode(
            meta.highestVersion(WFSMeta.WFS2_0_0).toString());

        String dataset = dls.getDataset();
        QueryType queryType = findQueryType(dls.getServiceType());

        StringBuilder sb = new StringBuilder();
        sb.append(base);
        if (!base.endsWith("?")) {
            sb.append("?");
        }
        sb.append("service=wfs&")
          .append("request=GetFeature&")
          .append("version=").append(version);

        if (queryType.equals(STOREDQUERY)) {
            sb.append("&STOREDQUERY_ID=")
              .append(StringUtils.urlEncode(dataset));
        } else {
            sb.append("&typeNames=")
              .append(StringUtils.urlEncode(dataset));
        }

        int idx = dataset.indexOf(':');
        if (idx >= 0) {
            String prefix = dataset.substring(0, idx);
            String ns = meta.getNamespaces().getNamespaceURI(prefix);
            sb.append("&namespaces=xmlns(")
                .append(StringUtils.urlEncode(prefix)).append(',')
                .append(StringUtils.urlEncode(ns)).append(')');
        }

        String parameters = encodeParameters(dls.getParameters(), usedVars);
        if (parameters.length() > 0) {
            sb.append('&').append(parameters);
        }

        if (!vendor.isEmpty()) {
            sb.append('&').append(vendor);
        }

        return sb.toString();
    }

    private void unpagedWFSDownload(
        DownloadStepJob jl,
        File         workingDir,
        Set<String>  usedVars,
        WFSMeta      meta
    ) throws ConverterException {
        WFSMeta.Operation getFeature = meta.findOperation(GETFEATURE);

        boolean usePost = getFeature.getPOST() != null;

        String url = usePost
            ? getFeature.getPOST()
            : wfsURL(dls, usedVars, meta);

        Document params = usePost
            ? create(dls, usedVars, meta)
            : null;

        LOG.info("WFS GetFeature KVP GET request: {}", url);

        String ext = extension();

        File gml = new File(workingDir, "download." + ext);
        LOG.info("Download to file '{}'", gml);
        logGetFeatureRequest(params);
        FileDownloadJob fdj = createFileDownloadJob(usePost, url, params, gml);
        jl.addJob(fdj);
        if (ext.equals("gml")) {
            jl.addJob(new GMLCheckJob(gml, logger));
            jl.addJob(new BroadcastJob(I18n.getMsg("file.download.success")));