public class ApplicationSettings {

    private static final Logger LOG = LoggerFactory.getLogger(ApplicationSettings.class.getName());

    private static final int DEFAULT_TIMEOUT = 10000;

    private static final int S_TO_MS = 1000;

    private final Document doc;

    private final Settings settings;

    private int requestTimeOutInMS = DEFAULT_TIMEOUT;

    private Credentials credentials;

    public ApplicationSettings(Document doc, Settings settings) throws IOException {
        this.doc = doc;
        this.settings = settings;
        parseNodeForElements("application");
    }

    /**
     * Returns the configured request timeout in ms.
     *
     * @return the request timeout in ms
     */
    public int getRequestTimeoutInMs() {
        return requestTimeOutInMS;
    }

    /**
     * Returns the configured credentials.
     *
     * @return the configured credentials, <code>null</code> if not configured
     */
    public Credentials getCredentials() {
        return this.credentials;
    }

    /**
     * Writes the passed credentials to the settings.xml.
     *
     * @param credentialsToPersist the credentials to store, if <code>null</code>
     *                             or username or password <code>null</code>
     *                             nothing is persisted.
     */
    public void persistCredentials(Credentials credentialsToPersist) {
        if (credentialsToPersist != null
            && credentialsToPersist.getUsername() != null
            && !credentialsToPersist.getUsername().isEmpty()
            && credentialsToPersist.getPassword() != null
            && !credentialsToPersist.getPassword().isEmpty()
            && !credentialsToPersist.equals(this.credentials)) {
            LOG.info("Saving credentials in settings.xml");
            modifyDocument(credentialsToPersist);
            settings.persistSettingsFile(doc);
        }
        this.credentials = credentialsToPersist;
    }

    @Override
    public String toString() {
        return "ApplicationSettings: ["
            + "requestTimeOutInMS: " + requestTimeOutInMS
            + ", credentials: " + credentials
            + "]";
    }

    /**
     * Parse application from settings.xml.
     */
    private void parseNodeForElements(String nodeName) throws IOException {
        NodeList nodes = doc.getElementsByTagName(nodeName);
        if (nodes.getLength() < 1) {
            throw new IOException("Node " + nodeName + " not found");
        }
        Node parent = nodes.item(0);

        NodeList childs = parent.getChildNodes();
        for (int i = 0; i < childs.getLength(); i++) {
            Node node = childs.item(i);
            if (node.getNodeType() == Node.ELEMENT_NODE) {
                if ("requestTimeout_s".equals(node.getNodeName())) {
                    parseRequestTimeout(node);
                } else if ("credentials".equals(node.getNodeName())) {
                    parseCredentials(node);
                }
            }
        }
    }

    private void parseRequestTimeout(Node node) {
        String ts = node.getTextContent();
        if (ts != null) {
            try {
                requestTimeOutInMS = S_TO_MS * Integer.parseInt(ts);
            } catch (NumberFormatException nfe) {
                LOG.error(nfe.getMessage());
            }
        }
    }

    private void parseCredentials(Node credentialsNode) {
        String username = null;
        String password = null;
        NodeList childs = credentialsNode.getChildNodes();
        for (int i = 0; i < childs.getLength(); i++) {
            Node node = childs.item(i);
            if ("username".equals(node.getNodeName())) {
                username = node.getTextContent();
            } else if ("password".equals(node.getNodeName())) {
                password = decrypt(node.getTextContent());
            }
        }
        if (username != null) {
            this.credentials = new Credentials(username, password);
        }
        LOG.debug("Found credentials for username " + username);
    }

    private void modifyDocument(Credentials newCredentials) {
        NodeList nodes = doc.getElementsByTagName("application");
        if (nodes.getLength() < 1) {
            LOG.warn("Credentials could not be stored. No application element found in settings.xml");
        }
        Node applicationNode = nodes.item(0);

        Node credentialsNode = parseNode(applicationNode, "credentials");
        if (credentialsNode == null) {
            credentialsNode = doc.createElement("credentials");
            applicationNode.appendChild(credentialsNode);
        }
        persistCredentials(credentialsNode, newCredentials);
    }

    private void persistCredentials(Node credentialsNode, Credentials newCredentials) {
        Node usernameNode = parseNode(credentialsNode, "username");
        Node passwordNode = parseNode(credentialsNode, "password");
        if (usernameNode == null) {
            usernameNode = doc.createElement("username");
            credentialsNode.appendChild(usernameNode);
        }
        if (passwordNode == null) {
            passwordNode = doc.createElement("password");
            credentialsNode.appendChild(passwordNode);
        }
        usernameNode.setTextContent(newCredentials.getUsername());
        passwordNode.setTextContent(encrypt(newCredentials.getPassword()));
    }

    private Node parseNode(Node parentNode, String nodeName) {
        NodeList childs = parentNode.getChildNodes();
        for (int i = 0; i < childs.getLength(); i++) {
            Node node = childs.item(i);
            if (node.getNodeType() == Node.ELEMENT_NODE) {
                if (nodeName.equals(node.getNodeName())) {
                    return node;
                }
            }
        }
        return null;
    }

    private String decrypt(String password) {
        if (password != null) {
            try {
                return CryptoUtils.decrypt(password);
            } catch (Exception e) {
                LOG.error("Password could not be decrypted", e);
            }
        }
        return password;
    }

    private String encrypt(String password) {
        if (password != null) {
            try {
                return CryptoUtils.encrypt(password);
            } catch (Exception e) {
                LOG.error("Password could not be encrypted", e);
            }
        }
        return password;
    }
} public final class Config {

    private Config() {
    }

    private static final Logger LOG = LoggerFactory.getLogger(Config.class.getName());

    /** Inner class to implicit synchronize the instance access. */
    private static final class Holder {
        private Holder() {
        }
        static final Config INSTANCE = new Config();
    }

    private boolean initialized;

    private Settings settings;

    private MIMETypes mimeTypes;

    private ProcessingConfiguration processingConfig;

    private ProxyConfiguration proxyConfig;

    private static final long FIVEHUNDRED = 500;

    /**
     * Access the instance of the configuration.
     * Use #load before to initialize it.
     * @return The configuration instance.
     */
    public static Config getInstance() {
        synchronized (Holder.INSTANCE) {
            try {
                while (!Holder.INSTANCE.initialized) {
                    Holder.INSTANCE.wait(FIVEHUNDRED);
                }
            } catch (InterruptedException ie) {
                Thread.currentThread().interrupt();
            }
            return Holder.INSTANCE;
        }
    }

    /**
     * @return The application Settings
     */
    public ApplicationSettings getApplicationSettings() {
        return settings.getApplicationSettings();
    }

    /**
     * @return the services
     */
    public ServiceSettings getServices() {
        return settings.getServiceSettings();
    }

    /**
     * @return the mimeTypes
     */
    public MIMETypes getMimeTypes() {
        return mimeTypes;
    }

    /**
     * @return the processingConfig
     */
    public ProcessingConfiguration getProcessingConfig() {
        return processingConfig;
    }

    /**
     * @return the proxy config
     */
    public ProxyConfiguration getProxyConfig() {
        return proxyConfig;
    }

    @Override
    public String toString() {
        return (proxyConfig == null ? "Proxy Settings: {}" : getProxyConfig())
            + ",\n " + getApplicationSettings()
            + ",\n " + getProcessingConfig()
            + ",\n " + getServices()
            + ",\n MIME-Types: " + getMimeTypes();
    }

    private static Settings loadSettings(File file)
        throws IOException {
        try {
            if (file != null && file.isFile() && file.canRead()) {
                return new Settings(file);
            }
            return new Settings();
        } catch (SAXException
                | ParserConfigurationException
                | IOException e) {
            LOG.error(e.getMessage(), Holder.INSTANCE);
            throwConfigFailureException(Settings.getName());
        }
        // Not reached.
        return null;
    }

    /**
     * Mark global config as unused.
     * @throws IOException when anythong goes wrong
     */
    private static void uninitialized()
        throws IOException {
        synchronized (Holder.INSTANCE) {
            LOG.info("No config directory given, starting with standard values...");
            Holder.INSTANCE.settings = loadSettings(null);
            Holder.INSTANCE.processingConfig =
                ProcessingConfiguration.loadDefault();
            Holder.INSTANCE.mimeTypes = MIMETypes.loadDefault();
            Holder.INSTANCE.initialized = true;
            Holder.INSTANCE.notifyAll();
        }
    }

    /**
     * Load configurations.
     * @param dirname The directory with the configuration files.
     * @throws IOException If something went wrong.
     */
    public static void initialize(String dirname) throws IOException {
        Controller.logToAppLog(I18n.format("dlc.start", Info.getVersion()));
        System.setProperty("java.util.logging.manager",
            "org.apache.logging.log4j.jul.LogManager");
        try {
            Logging.GEOTOOLS.setLoggerFactory(
                "org.geotools.util.logging.Log4JLoggerFactory");
        } catch (ClassNotFoundException e) {
            LOG.warn("Failed to initialize GeoTools logging subsystem: "
                + e.getLocalizedMessage());
        }
        if (dirname == null) {
            uninitialized();
        } else {
            synchronized (Holder.INSTANCE) {
                try {
                    load(dirname);
                } finally {
                    Holder.INSTANCE.initialized = true;
                    Holder.INSTANCE.notifyAll();
                }
            }
        }
        LOG.trace("System Properties:");
        System.getProperties().keySet().stream().map(
            k -> (String)k).sorted().forEach(k -> {
            LOG.trace(String.format("\t%s=%s", k,
                System.getProperty(k)));
        });
        Controller.logToAppLog(I18n.format("dlc.config", Config.getInstance()));
    }

    private static void load(String dirname) throws IOException {
        LOG.info("config directory: " + dirname);

        File dir = new File(dirname);

        if (!dir.isDirectory()) {
            throw new IOException("'" + dirname + "' is not a directory.");
        }

        File proxy = new File(dir, ProxyConfiguration.PROXY_CONFIG_FILE);
        if (proxy.isFile() && proxy.canRead()) {
            Holder.INSTANCE.proxyConfig = ProxyConfiguration.read(proxy);
            Holder.INSTANCE.proxyConfig.apply();
        } else {
            LOG.info("No Proxy config found, starting without proxy settings");
        }

        File services = new File(dir, Settings.SETTINGS_FILE);
        if (services.isFile() && services.canRead()) {
            Holder.INSTANCE.settings = loadSettings(services);
        } else {
            Holder.INSTANCE.settings = loadSettings(null);
            LOG.info("Settings config not found, using fallback...");
        }

        File procConfig = new File(
            dir, ProcessingConfiguration.PROCESSING_CONFIG_FILE);
        if (procConfig.isFile() && procConfig.canRead()) {
            Holder.INSTANCE.processingConfig =
                ProcessingConfiguration.read(procConfig);
        } else {
            Holder.INSTANCE.processingConfig =
                ProcessingConfiguration.loadDefault();
            LOG.info("Processing config not found, using fallback...");
        }

        if (Holder.INSTANCE.getProcessingConfig() == null) {
            throwConfigFailureException(ProcessingConfiguration.getName());
        }

        File mimeTypes = new File(dir, MIMETypes.MIME_TYPES_FILE);
        if (mimeTypes.isFile() && mimeTypes.canRead()) {
            Holder.INSTANCE.mimeTypes =
                MIMETypes.read(mimeTypes);
        } else {
            Holder.INSTANCE.mimeTypes = MIMETypes.loadDefault();
            LOG.info("MimeTypes config not found, using fallback...");
        }

        if (Holder.INSTANCE.getMimeTypes() == null) {
            throwConfigFailureException(MIMETypes.getName());
        }
    }

    private static void throwConfigFailureException(String configName)
        throws IOException {
        throw new IOException(
                "Failed on XML file for '"
                        + configName
                        + "'");
    }
} public class Credentials {

    private String username;

    private String password;

    /**
     * Instantiate with username and password.
     *
     * @param username never <code>null</code>
     * @param password never <code>null</code>
     */
    public Credentials(String username, String password) {
        this.username = username;
        this.password = password;
    }

    /**
     * @return the name of the user, never <code>null</code>
     */
    public String getUsername() {
        return username;
    }

    /**
     * @return the password of the user, never <code>null</code>
     */
    public String getPassword() {
        return password;
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) {
            return true;
        }
        if (o == null || getClass() != o.getClass()) {
            return false;
        }
        Credentials that = (Credentials) o;
        return Objects.equals(username, that.username) && Objects.equals(password, that.password);
    }

    @Override
    public int hashCode() {
        return Objects.hash(username, password);
    }

    @Override
    public String toString() {
        return "["
            + "username='" + username + '\''
            + ", password='" + "***" + '\''
            + ']';
    }
} @XmlRootElement(name = "DownloadSchritt")
@XmlAccessorType(XmlAccessType.FIELD)
public class DownloadStep {

    @XmlElement(name = "ServiceTyp")
    private String serviceType;

    @XmlElement(name = "URL")
    private String serviceURL;

    @XmlElement(name = "DownloadPfad")
    private String path;

    @XmlElement(name = "Dataset")
    private String dataset;

    @XmlElementWrapper(name = "Verarbeitungskette")
    @XmlElement(name = "Verarbeitungsschritt")
    private List<ProcessingStep> processingSteps;

    @XmlElementWrapper(name = "Parameters")
    @XmlElement(name = "Parameter")
    private List<Parameter> parameters;

    public DownloadStep() {
    }

    public DownloadStep(String dataset,
                        List<Parameter> parameters,
                        String serviceType,
                        String serviceURL,
                        String path,
                        List<ProcessingStep> processingSteps) {
        this.dataset = dataset;
        this.parameters = parameters;
        this.serviceType = serviceType;
        this.serviceURL = serviceURL;
        this.path = path;
        this.processingSteps = processingSteps;
    }


    public DownloadStep(String dataset,
                        List<Parameter> parameters,
                        String serviceType,
                        String serviceURL,
                        String path) {
        this(dataset,
            parameters,
            serviceType,
            serviceURL,
            path,
            new ArrayList<ProcessingStep>());
    }

    /**
     * @param file The file to load data from.
     */
    public DownloadStep(File file) {
    }

    /**
     * @return the dataset
     */
    public String getDataset() {
        return dataset;
    }

    /**
     * @param dataset the dataset to set
     */
    public void setDataset(String dataset) {
        this.dataset = dataset;
    }

    public List<ProcessingStep> getProcessingSteps() {
        return this.processingSteps;
    }

    public void setProcessingSteps(List<ProcessingStep> processingSteps) {
        this.processingSteps = processingSteps;
    }

    /**
     * @return the parameters
     */
    public List<Parameter> getParameters() {
        return parameters;
    }

    /**
     * @param parameters the parameters to set
     */
    public void setParameters(List<Parameter> parameters) {
        this.parameters = parameters;
    }

    /**
     * @return the serviceType
     */
    public String getServiceType() {
        return serviceType;
    }

    /**
     * @param serviceType the serviceType to set
     */
    public void setServiceType(String serviceType) {
        this.serviceType = serviceType;
    }

    /**
     * @param serviceType the serviceType to set
     */
    public void setServiceType(ServiceType serviceType) {
        this.setServiceType(serviceType.toString());
    }

    /**
     * @return the serviceURL
     */
    public String getServiceURL() {
        return serviceURL;
    }

    /**
     * @param serviceURL the serviceURL to set
     */
    public void setServiceURL(String serviceURL) {
        this.serviceURL = serviceURL;
    }

    /**
     * @return the dataSet
     */
    public String getPath() {
        return path;
    }

    /**
     * @param path the path to set
     */
    public void setPath(String path) {
        this.path = path;
    }

    /**
     * Finds a value for a given parameter key.
     *
     * @param key The key.
     * @return The value if found else null.
     */
    public String findParameter(String key) {
        for (Parameter p : this.parameters) {
            if (p.getKey().equals(key)) {
                return p.getValue();
            }
        }
        return null;
    }

    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder("[DownloadStep:\n");
        sb.append("\tserviceType: \"").append(serviceType).append("\"\n");
        sb.append("\tserviceURL: \"").append(serviceURL).append("\"\n");
        sb.append("\tdataset: \"").append(dataset).append("\"\n");
        sb.append("\tpath: \"").append(path).append("\"\n");
        sb.append("\tparameters: ");
        for (int i = 0,
             n = parameters != null ? parameters.size() : 0; i < n; i++) {
            if (i > 0) {
                sb.append(", ");
            }
            sb.append(parameters.get(i));
        }

        sb.append("]\n");
        sb.append("\tprocessing steps:\n");
        for (int i = 0,
             n = processingSteps != null ? processingSteps.size() : 0;
             i < n; i++) {
            sb.append("\t\t");
            sb.append(processingSteps.get(i));
            sb.append('\n');
        }
        sb.append(']');
        return sb.toString();
    }

    /**
     * Save the Object.
     *
     * @param file File to save to
     * @throws IOException if space does not exist
     */
    public void write(File file) throws IOException {
        try (FileOutputStream fos = new FileOutputStream(file)) {
            JAXBContext context = JAXBContext.newInstance(DownloadStep.class);
            Marshaller m = context.createMarshaller();
            m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
            BufferedOutputStream bos = new BufferedOutputStream(fos);
            m.marshal(this, bos);
            bos.flush();
        } catch (JAXBException je) {
            throw new IOException("", je);
        }
    }


    /**
     * Loads DownloadStep from a file.
     *
     * @param file The file to load the DownloadStep from.
     * @return The restored DownloadStep.
     * @throws IOException Something went wrong.
     */
    public static DownloadStep read(File file) throws IOException {
        try (FileInputStream fis = new FileInputStream(file)) {
            return getStep(fis);
        } catch (JAXBException je) {
            throw new IOException("", je);
        }
    }

    private static DownloadStep getStep(InputStream is) throws JAXBException {
        JAXBContext context = JAXBContext.newInstance(DownloadStep.class);
        Unmarshaller um = context.createUnmarshaller();
        BufferedInputStream bis = new BufferedInputStream(is);
        return (DownloadStep) um.unmarshal(bis);
    }

    /**
     * Downloadstep from String.
     *
     * @param xmlString Configurationstring
     * @return unmarshalled DownloadStep
     * @throws IOException Just in case
     */
    public static DownloadStep read(String xmlString) throws IOException {
        try {
            InputStream stringStream = new ByteArrayInputStream(
                xmlString.getBytes());
            return getStep(stringStream);
        } catch (JAXBException je) {
            throw new IOException("", je);
        }
    }

} public class ConverterException extends Exception {

    public ConverterException() {
    }

    public ConverterException(String msg) {
        super(msg);
    }

    public ConverterException(String msg, Throwable cause) {
        super(msg, cause);
    }
} public class DownloadStepConverter {

    private static final Logger LOG = LoggerFactory.getLogger(DownloadStepConverter.class.getName());

    private static final String OUTPUTFORMAT_PARAM = "outputformat";

    private static final String APPLICATION_X_SHAPEFILE
        = "application/x-shapefile";

    private static final String APPLICATION_X_GMZ = "application/x-gmz";

    /**
     * Type of a Query.
     */
    public enum QueryType {
        /**
         * StoredQuery.
         */
        STOREDQUERY,
        /**
         * Dataset.
         */
        DATASET,
        /**
         * SQL Query.
         */
        SQLQUERY }

    private static final String GETFEATURE = "GetFeature";

    private String user;

    private String password;

    private DownloadStep dls;

    private Log logger;

    private List<CountListener> listener = new ArrayList<>();

    public DownloadStepConverter() {
    }

    /**
     * Creates a converter with given user name and password.
     * @param user The user name.
     * @param password The password.
     */
    public DownloadStepConverter(String user, String password) {
        this.user = user;
        this.password = password;
    }

    /**
     * Adds a CountListener.
     * @param listenerToAdd listener to add, never <code>null</code>
     */
    public void addListener(CountListener listenerToAdd) {
        this.listener.add(listenerToAdd);
    }

    /**
     * Converts a DownloadStep into a sequence of jobs for the processor.
     * @param downloadSteps DownloadStep the configuration to be converted.
     * @return A job list for the download processor.
     * @throws ConverterException If the conversion went wrong.
     */
    public DownloadStepJob convert(DownloadStep downloadSteps)
        throws ConverterException {

        this.dls = downloadSteps;

        ProcessingStepConverter psc = new ProcessingStepConverter();

        File path = new File(dls.getPath());

        if (!path.exists()) {
            if (!path.mkdirs()) {
                throw new ConverterException(
                    I18n.format("dls.converter.cant.create.dir", path));
            }
        } else if (!path.isDirectory()) {
            throw new ConverterException(
                I18n.format("dls.converter.not.dir", path));
        }

        File dlLog = Misc.uniqueFile(path, "download-", "log", null);
        if (dlLog == null) {
            throw new ConverterException(
                I18n.getMsg("dls.converter.no.unique.filename"));
        }

        this.logger = new Log(dlLog);

        OpenLogJob olj = new OpenLogJob(this.logger);
        CloseLogJob clj = new CloseLogJob(this.logger);

        LogMetaJob lmj = new LogMetaJob(this.logger, dls);

        FileTracker fileTracker = new FileTracker(path);
        if (!fileTracker.scan()) {
            throw new ConverterException(
                I18n.getMsg("dls.converter.init.scan.failed"));
        }

        psc.convert(dls, fileTracker, logger);

        DownloadStepJob jl = new DownloadStepJob();

        jl.addJob(olj);
        jl.addJob(lmj);

        if (dls.getServiceType().equals("ATOM")) {
            createAtomDownload(jl, path);
        } else {
            createWFSDownload(jl, path, psc.getUsedVars());
        }

        if (psc.hasJobs() && isZippedOutputformat(downloadSteps)) {
            UnzipJob unzipJob = new UnzipJob(fileTracker, this.logger);
            jl.addJob(unzipJob);
        }

        jl.addJobs(psc.getJobs());

        jl.addJob(clj);

        return jl;
    }


    /**
     * Finds WFS service type for given dataset.
     * @param type The dataset type.
     * @return The query type.
     */
    protected static QueryType findQueryType(String type) {

        switch (type) {
            case "ATOM":
            case "WFS2_SIMPLE":
                return STOREDQUERY;
            case "WFS2_SQL":
                return SQLQUERY;
            default:
                return DATASET;
        }
    }

    private static String encodeParameters(
        List<Parameter> parameters,
        Set<String>     usedVars
    ) {
        StringBuilder sb = new StringBuilder();
        for (Parameter p: parameters) {
            if (p.getValue() == null || p.getValue().isEmpty()
            || usedVars.contains(p.getKey())) {
                continue;
            }
            if (sb.length() > 0) {
                sb.append('&');
            }
            sb.append(StringUtils.urlEncode(p.getKey()))
              .append('=')
              .append(StringUtils.urlEncode(p.getValue()));
        }
        return sb.toString();
    }

    private static final String[] IGNORE_WFS = {
        "service=",
        "request=",
        "acceptversions="
    };

    private static String vendorSpecific(String url) {
        int idx = url.lastIndexOf('?');
        if (idx < 0) {
            return "";
        }
        return StringUtils.ignorePartsWithPrefix(
            url.substring(idx + 1),
            "&",
            IGNORE_WFS);
    }

    private static String baseURL(String url) {
        int idx = url.indexOf('?');
        return idx >= 0 ? url.substring(0, idx) : url;

    }

    private static String capURL(String base) {
        return base
            + "?service=WFS&request=GetCapabilities&acceptversions=2.0.0";
    }

    private static String wfsURL(
        DownloadStep dls,
        Set<String>  usedVars,
        WFSMeta      meta
    ) {
        String url = dls.getServiceURL();
        String base = baseURL(url);
        String vendor = vendorSpecific(url);
        String getFeaturesURL = meta.findOperation(GETFEATURE).getGET();
        if (getFeaturesURL.startsWith("/")) {
            base += getFeaturesURL;
        } else {
            base = getFeaturesURL;
        }

        String version = StringUtils.urlEncode(
            meta.highestVersion(WFSMeta.WFS2_0_0).toString());

        String dataset = dls.getDataset();
        QueryType queryType = findQueryType(dls.getServiceType());

        StringBuilder sb = new StringBuilder();
        sb.append(base);
        if (!base.endsWith("?")) {
            sb.append("?");
        }
        sb.append("service=wfs&")
          .append("request=GetFeature&")
          .append("version=").append(version);

        if (queryType.equals(STOREDQUERY)) {
            sb.append("&STOREDQUERY_ID=")
              .append(StringUtils.urlEncode(dataset));
        } else {
            sb.append("&typeNames=")
              .append(StringUtils.urlEncode(dataset));
        }

        int idx = dataset.indexOf(':');
        if (idx >= 0) {
            String prefix = dataset.substring(0, idx);
            String ns = meta.getNamespaces().getNamespaceURI(prefix);
            sb.append("&namespaces=xmlns(")
                .append(StringUtils.urlEncode(prefix)).append(',')
                .append(StringUtils.urlEncode(ns)).append(')');
        }

        String parameters = encodeParameters(dls.getParameters(), usedVars);
        if (parameters.length() > 0) {
            sb.append('&').append(parameters);
        }

        if (!vendor.isEmpty()) {
            sb.append('&').append(vendor);
        }

        return sb.toString();
    }

    private void unpagedWFSDownload(
        DownloadStepJob jl,
        File         workingDir,
        Set<String>  usedVars,
        WFSMeta      meta
    ) throws ConverterException {
        WFSMeta.Operation getFeature = meta.findOperation(GETFEATURE);

        boolean usePost = getFeature.getPOST() != null;

        String url = usePost
            ? getFeature.getPOST()
            : wfsURL(dls, usedVars, meta);

        Document params = usePost
            ? create(dls, usedVars, meta)
            : null;

        LOG.info("WFS GetFeature KVP GET request: {}", url);

        String ext = extension();

        File gml = new File(workingDir, "download." + ext);
        LOG.info("Download to file '{}'", gml);
        logGetFeatureRequest(params);
        FileDownloadJob fdj = createFileDownloadJob(usePost, url, params, gml);
        jl.addJob(fdj);
        if (ext.equals("gml")) {
            jl.addJob(new GMLCheckJob(gml, logger));
            jl.addJob(new BroadcastJob(I18n.getMsg("file.download.success")));
        }
    }

    private FileDownloadJob createFileDownloadJob(boolean usePost, String url, Document params, File gml) {
        final FileDownloadJob fdj;
        if (usePost) {
            HttpEntity ent = new EntityTemplate(
                XML.toContentProducer(params));

            fdj = new FileDownloadJob(
                url, gml,
                this.user, this.password,
                ent,
                this.logger);
        } else {
            fdj = new FileDownloadJob(
                url, gml,
                this.user, this.password,
                this.logger);
        }
        listener.forEach(l -> fdj.addListener(l));
        return fdj;
    }

    private static URL newURL(String url) throws ConverterException {
        try {
            return new URL(url);
        } catch (MalformedURLException mfe) {
            throw new ConverterException(mfe.getMessage(), mfe);
        }
    }

    private static final String XPATH_SERVICE_EXCEPTION
        = "//ows:ExceptionReport/ows:Exception/ows:ExceptionText";

    private static void checkServiceException(Document doc)
        throws ConverterException {

        String exceptionText = (String)XML.xpath(
            doc, XPATH_SERVICE_EXCEPTION,
            XPathConstants.STRING,
            WFSMetaExtractor.NAMESPACES);

        if (exceptionText != null && !exceptionText.isEmpty()) {
            throw new ConverterException(
                I18n.format(
                    "dls.converter.wfs.exception",
                    exceptionText));
        }
    }

    private static final String[] OUTPUTFORMAT = new String[] {
        "outputformat="
    };

    private static String hitsURL(String wfsURL) {
        // outputformat parameters irritates wfs servers
        // when doing hits requests.
        int idx = wfsURL.lastIndexOf('?');
        if (idx >= 0) {
            String prefix = wfsURL.substring(0, idx + 1);
            String rest = wfsURL.substring(idx + 1);
            wfsURL = prefix + StringUtils.ignorePartsWithPrefix(
                rest, "&", OUTPUTFORMAT);
        }
        return wfsURL + "&resultType=hits";
    }

    // This is an XPATH expression and not an URI.
    @java.lang.SuppressWarnings("squid:S1075")
    private static final String XPATH_NUMBER_MATCHED
        = "/wfs:FeatureCollection/@numberMatched";

    private int numFeatures(String wfsURL, Document postparams)
        throws ConverterException {

        URL url = postparams == null
            ? newURL(hitsURL(wfsURL))
            : newURL(wfsURL);

        Document hitsDoc = null;
        try {
            HttpEntity ent = postparams == null
                ? null
                : new EntityTemplate(XML.toContentProducer(postparams));

            hitsDoc = XML.getDocument(url, user, password, ent);
        } catch (SocketTimeoutException | ConnectTimeoutException te) {
            throw new ConverterException(
                I18n.format(
                    "file.download.failed_reason",
                    I18n.getMsg("file.download.failed.timeout")),
                te);
        } catch (URISyntaxException | IOException e) {
            throw new ConverterException(e.getMessage());
        }

        if (hitsDoc == null) {
            throw new ConverterException(
                I18n.getMsg("dls.converter.no.hits.doc"));
        }

        checkServiceException(hitsDoc);
        String numberMatchedString = (String)XML.xpath(
            hitsDoc, XPATH_NUMBER_MATCHED,
            XPathConstants.STRING,
            WFSMetaExtractor.NAMESPACES);

        if (numberMatchedString == null || numberMatchedString.isEmpty()) {
            throw new ConverterException(
                I18n.getMsg("file.download.no.number"));
        }
        try {
            return Integer.parseInt(numberMatchedString);
        } catch (NumberFormatException nfe) {
            throw new ConverterException(nfe.getLocalizedMessage(), nfe);
        }
    }

    private URL pagedFeatureURL(String wfsURL, int ofs, int count, boolean wfs2)
    throws ConverterException {
        StringBuilder sb = new StringBuilder(wfsURL)
            .append("&startIndex=").append(ofs)
            .append(wfs2 ? "&count=" : "&maxFeatures=").append(count);
        String wfsGetFeatureRequest = sb.toString();
        LOG.info("WFS GetFeature GET KVP request: {}", wfsGetFeatureRequest);
        return newURL(wfsGetFeatureRequest);
    }

    private String extension() {
        String mimeType = dls.findParameter("outputformat");
        return  Config.getInstance().getMimeTypes()
            .findExtension(mimeType, "gml");
    }

    private void createWFSDownload(
        DownloadStepJob jl,
        File         workingDir,
        Set<String>  usedVars
    ) throws ConverterException {

        String url = dls.getServiceURL();
        String base = baseURL(url);
        String cap = capURL(base);
        String vendor = vendorSpecific(url);

        if (!vendor.isEmpty()) {
            cap += "&" + vendor;
        }

        WFSMetaExtractor extractor =
            new WFSMetaExtractor(cap, this.user, this.password);

        WFSMeta meta;
        try {
            meta = extractor.parse();
        } catch (URISyntaxException | IOException ioe) {
            throw new ConverterException(
                I18n.getMsg("dls.converter.no.meta.data"), ioe);
        }

        WFSMeta.Operation getFeatureOp = meta.findOperation(GETFEATURE);
        if (getFeatureOp == null) {
            throw new ConverterException(
                I18n.getMsg("dls.converter.getfeature.unsupported"));
        }

        Integer fpp = getFeatureOp.featuresPerPage();
        if (fpp == null) { // Fall back to global default.
            fpp = meta.featuresPerPage();
        }

        if (fpp == null) {
            unpagedWFSDownload(jl, workingDir, usedVars, meta);
            return;
        }

        boolean usePost = getFeatureOp.getPOST() != null;
        String wfsURL;
        Document params = null;
        int numFeatures;

        if (usePost) {
            wfsURL = getFeatureOp.getPOST();
            params = create(dls, usedVars, meta, true);
            logGetFeatureRequest(params);
            numFeatures = numFeatures(wfsURL, params);
        } else {
            wfsURL = wfsURL(dls, usedVars, meta);
            LOG.info("WFS GetFeature GET KVP request: {}", wfsURL);
            numFeatures = numFeatures(wfsURL, null);
        }

        // Page size greater than number features -> Normal download.
        if (numFeatures < fpp) {
            unpagedWFSDownload(jl, workingDir, usedVars, meta);
            return;
        }

        LOG.info("Total number of features: {}", numFeatures);

        FilesDownloadJob fdj = new FilesDownloadJob(this.user, this.password);
        listener.forEach(l -> fdj.addListener(l));
        GMLCheckJob gcj = new GMLCheckJob(logger);

        String ext = extension();

        boolean isGML = ext.equals("gml");

        int numFiles = Math.max(1, numFeatures / fpp);
        // 1000 file -> 000, 000, ..., 999
        int places = StringUtils.places(Math.max(0, numFiles - 1));
        String format = "%0" + places + "d-%d." + ext;

        boolean wfs2 =
            meta.highestVersion(WFSMeta.WFS2_0_0)
                .compareTo(WFSMeta.WFS2_0_0) >= 0;

        for (int ofs = 0, i = 0; ofs < numFeatures; ofs += fpp, i++) {
            String filename = String.format(format, i, ofs);
            File file = new File(workingDir, filename);
            LOG.info("Download to file: {}", file);
            if (!usePost) {
                fdj.add(file, pagedFeatureURL(wfsURL, ofs, fpp, wfs2));
            } else {
                URL wfs = newURL(wfsURL);
                Document pagedParams = create(
                    dls, usedVars, meta, ofs, fpp, wfs2);
                logGetFeatureRequest(pagedParams);
                fdj.add(file, wfs,
                    new EntityTemplate(XML.toContentProducer(pagedParams)));
            }
            if (isGML) {
                gcj.add(file);
            }
        }

        jl.addJob(fdj);
        jl.addJob(gcj);
        jl.addJob(new BroadcastJob(I18n.getMsg("file.download.success")));
    }


    private static void check(String s, String name)
    throws ConverterException {
        if (s == null || s.isEmpty()) {
            throw new ConverterException(
                I18n.format("dls.converter.missing", name));
        }
    }

    private void createAtomDownload(
        DownloadStepJob jl,
        File workingDir
    ) throws ConverterException {

        String dataset = dls.getDataset();
        String url = dls.getServiceURL();
        String variation = dls.findParameter("VARIATION");

        check(url, "service url");
        check(dataset, "dataset");
        check(variation, "VARIATION");

        AtomDownloadJob job = new AtomDownloadJob(
            url,
            dataset,
            variation,
            workingDir,
            this.user, this.password,
            this.logger);
        listener.forEach(l -> job.addListener(l));
        jl.addJob(job);
    }

    private void logGetFeatureRequest(Document params) {
        try {
            String xmlAsString = XML.documentToString(params);
            LOG.info("WFS GetFeature POST XML request: {}", xmlAsString);
        } catch (TransformerException e) {
            // nothing to do
            LOG.info("WFS GetFeature Request cannot be logged: "
                + e.getLocalizedMessage());
        }
    }

    private boolean isZippedOutputformat(DownloadStep downloadStep) {
        String outputformat = downloadStep.findParameter(OUTPUTFORMAT_PARAM);
        return APPLICATION_X_SHAPEFILE.equals(outputformat)
            || APPLICATION_X_GMZ.equals(outputformat);
    }

} public class Processor implements Runnable {

    private static final Logger LOG = LoggerFactory.getLogger(Processor.class.getName());

    private Deque<DownloadStepJob> jobs = new ArrayDeque<>();;

    private List<ProcessorListener> listeners = new CopyOnWriteArrayList<>();

    public Processor(DownloadStepJob jobToExecute) {
        jobs.add(jobToExecute);
    }

    public Processor(List<DownloadStepJob> jobsToExecute) {
        jobs.addAll(jobsToExecute);
    }

    /**
     * Adds listeners to the list of listeners.
     *
     * @param listenersToAdd The listeners to add.
     * @return the processor instance, never <code>null</code>
     */
    public Processor withListeners(ProcessorListener... listenersToAdd) {
        Arrays.stream(listenersToAdd).forEach(listenerToAdd -> {
                if (!listeners.contains(listenerToAdd)) {
                    listeners.add(listenerToAdd);
                }
            }
        );
        return this;
    }

    /** Broadcasts a message to all listeners.
     * @param message The message to broadcast.
     */
    public void broadcastMessage(String message) {
        ProcessorEvent pe = new ProcessorEvent(this, message);
        for (ProcessorListener pl: listeners) {
            pl.receivedMessage(pe);
        }
    }

    @Override
    public void run() {
        while (!this.jobs.isEmpty()) {
            try {
                DownloadStepJob job = this.jobs.poll();
                job.run(this);
                downloadStepJobFinished();
            } catch (JobExecutionException jee) {
                LOG.error(jee.getMessage(), jee);
                downloadStepJobFailed(jee);
            } catch (InterruptedException e) {
                broadcastMessage(I18n.getMsg("status.download.cancelled"));
            }
        }
    }

    private void downloadStepJobFailed(JobExecutionException jee) {
        ProcessorEvent pe = new ProcessorEvent(this, jee);
        for (ProcessorListener pl: listeners) {
            pl.processingFailed(pe);
        }
    }

    private void downloadStepJobFinished() {
        for (ProcessorListener pl: listeners) {
            pl.processingFinished();
        }
    }
} public interface ProcessorListener extends EventListener {

    /**
     * Called if a message was sent during job execution.
     * @param pe The event. pe.getMessage() returns the
     * message.
     */
    void receivedMessage(ProcessorEvent pe);

    /**
     * Called if the processing of a DownloadJob failed.
     * @param pe The event. pe.getException() returns the
     * thrown exception.
     */
    void processingFailed(ProcessorEvent pe);

    /**
     * Called if the processing of a DownloadJob finished.
     */
    void processingFinished();
} public class DownloadStepJob implements Job {

    private static final Logger LOG = LoggerFactory.getLogger(DownloadStepJob.class.getName());

    private List<Job> jobs;

    public DownloadStepJob() {
        jobs = new ArrayList<>();
    }

    /**
     * Add a job to this job list.
     * @param job The job to be added.
     */
    public void addJob(Job job) {
        jobs.add(job);
    }

    /**
     * Add all jobs to this job list.
     * @param other The jobs to add.
     */
    public void addJobs(List<Job> other) {
        jobs.addAll(other);
    }

    @Override
    public void run(Processor p) throws JobExecutionException, InterruptedException {
        LOG.info("Executing download step jobs");
        int i = 0;
        try {
            for (; i < jobs.size(); i++) {
                jobs.get(i).run(p);
                if (Thread.currentThread().isInterrupted()) {
                    throw new InterruptedException("Execution of the download job interrupted.");
                }
            }
        } finally {
            for (i++; i < jobs.size(); i++) {
                Job job = jobs.get(i);
                if (job instanceof DeferredJob) {
                    job.run(p);
                }
            }
        }
    }

    /**
     * Returns the List of Jobs.
     * @return list of Jobs
     */
    public List<Job> getJobList() {
        return this.jobs;
    }
} public class DocumentResponseHandler implements ResponseHandler<Document> {


    private WrapInputStreamFactory wrapFactory;
    private Boolean namespaceAware;
    private HttpRequestBase request;

    private static Unauthorized unauthorized;

    private static final Logger LOG = LoggerFactory.getLogger(DocumentResponseHandler.class.getName());


    /** Set the Unauthorized handler.
     * @param setUnauthorized The handler to set.
     */
    public static synchronized void setUnauthorized(
        Unauthorized setUnauthorized) {
        unauthorized = setUnauthorized;

    }

    private static synchronized void callUnauthorized() {
        if (unauthorized != null) {
            unauthorized.unauthorized();
        }
    }

    /**
     * Constructor.
     *
     * @param request Request
     */
    public DocumentResponseHandler(HttpRequestBase request) {
        this.request = request;
    }

    /**
     * Constructor.
     * @param wrapFactory the factory
     */
    public DocumentResponseHandler(WrapInputStreamFactory wrapFactory) {
        this.wrapFactory = wrapFactory;
    }

    /**
     * Set the XML parse to be namespace aware or not.
     * @param namespaceAware Should the parse be namespace aware?
     */
    public void setNamespaceAware(Boolean namespaceAware) {
        this.namespaceAware = namespaceAware;
    }

    private InputStream wrap(InputStream in) {
        return this.wrapFactory != null
            ? this.wrapFactory.wrap(in)
            : in;
    }

    @Override
    public Document handleResponse(HttpResponse response)
        throws IOException {
        int status = response.getStatusLine().getStatusCode();
        String reason = response.getStatusLine().getReasonPhrase();
        Controller.logToAppLog(status + " " + reason + " "
                + request.toString());

        if (status < HttpStatus.SC_OK
            || status >= HttpStatus.SC_MULTIPLE_CHOICES) {
            if (status == HttpStatus.SC_UNAUTHORIZED) {
                callUnauthorized();
            }
            throw new ClientProtocolException("Unexpected response status: "
                    + status);
        }
        HttpEntity entity = response.getEntity();
        try {
            return entity == null
                    ? null
                    : XML.getDocument(wrap(entity.getContent()), namespaceAware);
        } catch (org.xml.sax.SAXException
                | ParserConfigurationException e) {
            LOG.error(e.toString(), entity);
            return null;
        }
    }
} public class FileResponseHandler implements ResponseHandler<Boolean> {



    private static Unauthorized unauthorized;

    /** Set the Unauthorized handler.
     * @param setUnauthorized The handler to set.
     */
    public static synchronized void setUnauthorized(
            Unauthorized setUnauthorized) {
        unauthorized = setUnauthorized;

    }

    private static synchronized void callUnauthorized() {
        if (unauthorized != null) {
            unauthorized.unauthorized();
        }
    }

    private static final Logger LOG = LoggerFactory.getLogger(FileResponseHandler.class.getName());

    private static final int BUF_SIZE = 4096;

    private File file;

    private HttpUriRequest request;

    private WrapInputStreamFactory wrapFactory;

    public FileResponseHandler() {
    }

    public FileResponseHandler(File file, HttpUriRequest request) {
        this(file, null, request);
    }

    public FileResponseHandler(File file, WrapInputStreamFactory wrapFactory,
            HttpUriRequest request) {
        this.file = file;
        this.wrapFactory = wrapFactory;
        this.request = request;
    }

    private InputStream wrap(InputStream in) {
        return this.wrapFactory != null
            ? this.wrapFactory.wrap(in)
            : in;
    }

    @Override
    public Boolean handleResponse(HttpResponse response)
        throws IOException {
        int status = response.getStatusLine().getStatusCode();
        String reason = response.getStatusLine().getReasonPhrase();
        Controller.logToAppLog(status + " " + reason + " "
                + request.toString());
        if (status < HttpStatus.SC_OK
            || status >= HttpStatus.SC_MULTIPLE_CHOICES) {
            if (status == HttpStatus.SC_UNAUTHORIZED) {
                callUnauthorized();
            }
            throw new ClientProtocolException("Unexpected response status: "
                    + status);
        }
        HttpEntity entity = response.getEntity();
        if (entity == null) {
            return Boolean.FALSE;
        }
        try {
            try (FileOutputStream out = new FileOutputStream(this.file)) {
                byte[] buf = new byte[BUF_SIZE];
                int r;
                InputStream ins = wrap(entity.getContent());
                while ((r = ins.read(buf)) >= 0) {
                    out.write(buf, 0, r);
                }
                out.flush();
            }
        } catch (IOException ioe) {
            // Clean up debris if there was an error.
            // XXX: Maybe keep it to recover download?
            deleteGraceful();
            throw ioe;
        }
        return Boolean.TRUE;
    }

    private void deleteGraceful() {
        try {
            Files.delete(this.file.toPath());
        } catch (IOException ioe) {
            LOG.warn("delete failed: {}", ioe.getMessage());
        }
    }
} public interface Unauthorized {
    /** Called if an auth failed. */
    void unauthorized();
} public final class UnauthorizedLog implements Unauthorized {

    private static final Logger LOG = LoggerFactory.getLogger(UnauthorizedLog.class.getName());

    /**
     * Constructor.
     */
    public UnauthorizedLog() {
        // Do nothing.
    }

    /**
     * writes to the log, when user and pw is wrong.
     */
    @Override
    public void unauthorized() {
        LOG.error(I18n.getMsg("gui.wrong.user.and.pw"), this);
    }
}
import de.bayern.gdi.config.ApplicationSettings;
import de.bayern.gdi.config.Config;
import de.bayern.gdi.config.Credentials;
import de.bayern.gdi.model.DownloadStep;
import de.bayern.gdi.processor.ConverterException;
import de.bayern.gdi.processor.DownloadStepConverter;
import de.bayern.gdi.processor.JobExecutionException;
import de.bayern.gdi.processor.Processor;
import de.bayern.gdi.processor.ProcessorEvent;
import de.bayern.gdi.processor.ProcessorListener;
import de.bayern.gdi.processor.job.DownloadStepJob;
import de.bayern.gdi.utils.DocumentResponseHandler;
import de.bayern.gdi.utils.FileResponseHandler;
import de.bayern.gdi.utils.Unauthorized;
import de.bayern.gdi.utils.UnauthorizedLog;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/*
 * DownloadClient Geodateninfrastruktur Bayern
 *
 * (c) 2016 GSt. GDI-BY (gdi.bayern.de)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package de.bayern.gdi;



/**
 * The command line tool.
 */
public class Headless implements ProcessorListener {

    private static final Logger LOG = LoggerFactory.getLogger(Headless.class);

    /**
     * @param downloadFiles The command line arguments.
     * @param credentials   Optional credentials
     * @return Non zero if the operation fails.
     */
    public int runHeadless(String[] downloadFiles, Credentials credentials) {