public abstract class AbstractSearch<T extends Object> {
    protected URI url;
    protected Polygon2D aoi;
    protected double cloudFilter;
    protected String sensingStart;
    protected String sensingEnd;
    protected int relativeOrbit;
    protected Set<String> tiles;
    protected T productType;
    protected AbstractSearch<T> additionalProvider;
    protected List<NameValuePair> params;
    protected UsernamePasswordCredentials credentials;
    protected String filter;
    protected boolean allPages;
    protected int offset;
    protected int pageSize;

    public AbstractSearch(String url) throws URISyntaxException {
        this.url = new URI(url);
        this.cloudFilter = Double.MAX_VALUE;
    }

    public AbstractSearch<T> auth(String user, String pwd) {
        this.credentials = new UsernamePasswordCredentials(user, pwd);
        return this;
    }

    public void copyFiltersFrom(AbstractSearch<T> anotherSearch) {
        this.aoi = anotherSearch.aoi;
        this.cloudFilter = anotherSearch.cloudFilter;
        this.sensingStart = anotherSearch.sensingStart;
        this.sensingEnd = anotherSearch.sensingEnd;
        this.relativeOrbit = anotherSearch.relativeOrbit;
        this.tiles = anotherSearch.tiles;
        this.productType = anotherSearch.productType;
        this.params = anotherSearch.params;
        this.filter = anotherSearch.filter;
    }

    public void setPageSize(int value) { this.pageSize = value; }

    public void setOffset(int value) { this.offset = value; }

    public abstract AbstractSearch<T> limit(int value);

    public abstract AbstractSearch<T> start(int value);

    public AbstractSearch<T> filter(List<ProductDescriptor> productNames) { return this; }

    public void setSensingStart(String sensingStart) {
        this.sensingStart = sensingStart;
    }

    public void setSensingEnd(String sensingEnd) {
        this.sensingEnd = sensingEnd;
    }

    public void setAreaOfInterest(Polygon2D polygon) { this.aoi = polygon; }

    public void setClouds(double clouds) {
        this.cloudFilter = clouds;
    }

    public void setOrbit(int orbit) { this.relativeOrbit = orbit; }

    public void setTiles(Set<String> tiles) { this.tiles = tiles; }

    public Set<String> getTiles() { return this.tiles; }

    public void setProductType(T type) { this.productType = type; }

    public T getProductType() { return this.productType; }

    public void setAdditionalProvider(AbstractSearch<T> provider) {
        this.additionalProvider = provider;
    }

    public void setRetrieveAllPages(boolean value) { this.allPages = value; }

    public List<ProductDescriptor> execute() throws Exception {
        List<ProductDescriptor> products = new ArrayList<>();
        if (this.allPages) {
            final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd");
            final Calendar calendar = Calendar.getInstance();
            calendar.set(2016, Calendar.DECEMBER, 5);
            List<ProductDescriptor> subList;
            do {
                start(this.offset);
                subList = executeImpl();
                if (subList != null) {
                    products.addAll(subList.stream()
                                .filter(p -> {
                                    try {
                                        return  dateFormat.parse(p.getSensingDate()).after(calendar.getTime()) &&
                                                tiles.stream().anyMatch(t -> p.getName().contains(t));
                                    } catch (ParseException e) {
                                        return false;
                                    }
                                }).collect(Collectors.toList()));
                    products.addAll(subList.stream().filter(p -> {
                        try {
                            return dateFormat.parse(p.getSensingDate()).before(calendar.getTime());
                        } catch (ParseException e) {
                            return false;
                        }
                    }).collect(Collectors.toList()));
                    this.offset += this.pageSize;
                    start(this.offset);
                    limit(this.pageSize);
                }
            } while (subList != null && !subList.isEmpty());
        } else {
            products.addAll(executeImpl());
        }
        if (hasAdditionalProvider()) {
            this.additionalProvider.start(0);
            this.additionalProvider.setPageSize(this.pageSize);
            products.addAll(this.additionalProvider.execute());
        }
        return products;
    }

    protected abstract List<ProductDescriptor> executeImpl() throws Exception;

    protected boolean hasAdditionalProvider() { return this.additionalProvider != null; }
} public enum DownloadMode {
    /**
     * Product will be downloaded from the remote site and the corresponding local product,
     * if exists, it will be overwritten
     */
    OVERWRITE,
    /**
     * Product will be downloaded from the remote site and, if a corresponding local product exists,
     * the download will be resumed from the current length of the local product
     */
    RESUME,
    /**
     * The product will be copied from a local (or shared) folder into the output folder.
     * No remote download will be performed.
     * This works only in conjunction with the --input command line parameter.
     */
    COPY,
    /**
     * Only a symlink to the product file system location, into the output folder, will be created.
     * No remote download will be performed.
     * This works only in conjunction with the --input command line parameter.
     */
    SYMLINK,
    /**
     * Only symlinks to the product filtered tiles file system location, into the output folder, will be created.
     * No remote download will be performed.
     * This works only in conjunction with the --input command line parameter.
     */
    FILTERED_SYMLINK
} public abstract class ProductDescriptor {
    protected String name;
    protected String id;
    protected double cloudsPercentage;
    protected String sensingDate;
    protected String version;

    public ProductDescriptor() {}

    public ProductDescriptor(String name) {
        setName(name);
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        if (!verifyProductName(name)) {
            throw new IllegalArgumentException(String.format("The product name %s doesn't match the expected pattern", name));
        }
        this.name = name;
    }

    public String getId() {
        return id;
    }

    public void setId(String id) {
        this.id = id;
    }

    public String getVersion() { return version; }

    public void setVersion(String version) { this.version = version; }

    public double getCloudsPercentage() {
        return cloudsPercentage;
    }

    public void setCloudsPercentage(double cloudsPercentage) {
        this.cloudsPercentage = cloudsPercentage;
    }

    public String getSensingDate() { return sensingDate; }

    public void setSensingDate(String date) { this.sensingDate = date; }

    public abstract String getProductRelativePath();

    @Override
    public String toString() {
        return this.name;
    }

    protected abstract boolean verifyProductName(String name);

    protected String[] getTokens(Pattern pattern, String input, Map<Integer, String> replacements) {
        String[] tokens = null;
        Matcher matcher = pattern.matcher(input);
        if (matcher.matches()) {
            int count = matcher.groupCount();
            tokens = new String[count];
            for (int i = 0; i < tokens.length; i++) {
                if (replacements != null && replacements.containsKey(i)) {
                    tokens[i] = replacements.get(i);
                } else {
                    tokens[i] = matcher.group(i + 1);
                }
            }
        } else {
            throw new RuntimeException("Name doesn't match the specifications");
        }
        return tokens;
    }
} public enum SensorType {
    /**
     * Sentinel-2
     */
    S2,
    /**
     * Landsat 8
     */
    L8
} public abstract class TileMap {
    protected final Map<String, Rectangle2D> tiles;

    protected TileMap() {
        tiles = new TreeMap<>();
    }

    public void read(InputStream inputStream) throws IOException {
        try (Scanner scanner = new Scanner(inputStream)) {
            String line, tile;
            while (scanner.hasNextLine()) {
                line = scanner.nextLine();
                if (scanner.ioException() != null) {
                    throw scanner.ioException();
                }
                tile = line.substring(0, line.indexOf(" "));
                line = line.replaceAll(tile, "").trim();
                //line = line.substring(0, line.length() - 1);
                String[] tokens = line.split(",");
                Rectangle2D.Double rectangle = new Rectangle2D.Double(
                        Double.parseDouble(tokens[0].substring(2)),
                        Double.parseDouble(tokens[1].substring(2)),
                        Double.parseDouble(tokens[2].substring(2)),
                        Double.parseDouble(tokens[3].substring(2)));
                tiles.put(tile, rectangle);
            }
        } finally {
            if (inputStream != null)
                inputStream.close();
        }
    }

    public void write(Path file) throws IOException {
        BufferedWriter bufferedWriter = null;
        try {
            bufferedWriter = Files.newBufferedWriter(file, StandardOpenOption.CREATE);
            StringBuilder line = new StringBuilder();
            for (Map.Entry<String, Rectangle2D> entry : tiles.entrySet()) {
                line.append(entry.getKey()).append(" ");
                Rectangle2D rectangle = entry.getValue();
                line.append("x=").append(rectangle.getX()).append(",");
                line.append("y=").append(rectangle.getY()).append(",");
                line.append("w=").append(rectangle.getWidth()).append(",");
                line.append("h=").append(rectangle.getHeight()).append("\n");
                bufferedWriter.write(line.toString());
                bufferedWriter.flush();
                line.setLength(0);
            }
        } finally {
            if (bufferedWriter != null)
                bufferedWriter.close();
        }
    }

    public void fromKmlFile(String file) throws IOException {
        Path kmlFile = Paths.get(file);
        if (Files.exists(kmlFile)) {
            fromKml(Files.newBufferedReader(kmlFile));
        }
    }

    public abstract void fromKml(BufferedReader bufferedReader) throws IOException;

    public List<String> getTileNames() {
        return this.tiles.keySet().stream().collect(Collectors.toList());
    }

    /**
     * Returns the number of tiles contained in this map
     */
    public int getCount() {
        return tiles.size();
    }

    /**
     * Computes the bounding box for the given list of tile identifiers
     * @param tileCodes     List of tile identifiers
     */
    public Rectangle2D boundingBox(Set<String> tileCodes) {
        if (tileCodes == null) {
            return null;
        }
        Rectangle2D accumulator = null;
        for (String code : tileCodes) {
            Rectangle2D rectangle2D = tiles.get(code);
            if (rectangle2D != null) {
                if (accumulator == null) {
                    accumulator = rectangle2D;
                } else {
                    accumulator = accumulator.createUnion(rectangle2D);
                }
            }
        }
        return accumulator;
    }

    /**
     * Computes the list of tiles that intersect the given area of interest (rectangle).
     *
     * @param ulx   The upper left corner longitude (in degrees)
     * @param uly   The upper left corner latitude (in degrees)
     * @param lrx   The lower right corner longitude (in degrees)
     * @param lry   The lower right corner latitude (in degrees)
     */
    public Set<String> intersectingTiles(double ulx, double uly, double lrx, double lry) {
        return intersectingTiles(new Rectangle2D.Double(ulx, uly, ulx - lrx, uly - lry));
    }
    /**
     * Computes the list of tiles that intersect the given area of interest (rectangle).
     *
     * @param aoi   The area of interest bounding box
     */
    public Set<String> intersectingTiles(Rectangle2D aoi) {
        Set<String> tileCodes = new HashSet<>();
        tileCodes.addAll(
                tiles.entrySet().stream()
                        .filter(entry -> entry.getValue().intersects(aoi))
                        .map(Map.Entry::getKey)
                        .collect(Collectors.toSet()));
        return tileCodes;
    }

    protected Rectangle2D boundingBox(Rectangle2D...rectangles) {
        if (rectangles == null) {
            return null;
        }
        if (rectangles.length == 1) {
            return rectangles[0];
        } else {
            Rectangle2D accumulator = rectangles[0];
            for (int i = 1; i < rectangles.length; i++) {
                accumulator.add(rectangles[i]);
            }
            return accumulator;
        }
    }

} public enum CollectionCategory {
    RT,
    T1,
    T2
} public class LandsatAWSSearch extends AbstractSearch<CollectionCategory> {

    public LandsatAWSSearch(String url) throws URISyntaxException {
        super(url);
    }

    @Override
    public AbstractSearch<CollectionCategory> limit(int value) {
        this.pageSize = value;
        return this;
    }

    @Override
    public AbstractSearch<CollectionCategory> start(int value) {
        this.offset = value;
        return this;
    }

    @Override
    protected List<ProductDescriptor> executeImpl() throws Exception {
        Map<String, ProductDescriptor> results = new LinkedHashMap<>();
        Set<String> tiles = this.tiles != null && this.tiles.size() > 0 ?
                this.tiles :
                this.aoi != null ?
                        LandsatTilesMap.getInstance().intersectingTiles(this.aoi.getBounds2D()) :
                        new HashSet<>();
        final DateTimeFormatter fileDateFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd");
        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
        LocalDate todayDate = LocalDate.now();
        if (this.sensingStart == null || this.sensingStart.isEmpty()) {
            this.sensingStart = todayDate.minusDays(30).format(fileDateFormat);
        }
        if (this.sensingEnd == null || this.sensingEnd.isEmpty()) {
            this.sensingEnd = dateFormat.format(fileDateFormat);
        }
        //http://sentinel-s2-l1c.s3.amazonaws.com/?delimiter=/&prefix=c1/L8/
        Calendar startDate = Calendar.getInstance();
        startDate.setTime(dateFormat.parse(this.sensingStart));
        Calendar endDate = Calendar.getInstance();
        endDate.setTime(dateFormat.parse(this.sensingEnd));
        final String baseUrl = this.url.toString();
        final boolean isPreCollection = baseUrl.contains("prefix=c1");
        for (String tile : tiles) {
            String path = tile.substring(0, 3);
            String row = tile.substring(3, 6);
            String tileUrl = baseUrl + path + ProductDownloader.URL_SEPARATOR + row + ProductDownloader.URL_SEPARATOR;
            Result productResult = ResultParser.parse(NetUtils.getResponseAsString(tileUrl));
            if (productResult.getCommonPrefixes() != null) {
                Set<String> names = productResult.getCommonPrefixes().stream()
                        .map(p -> p.replace(productResult.getPrefix(), "").replace(productResult.getDelimiter(), ""))
                        .collect(Collectors.toSet());
                for (String name : names) {
                    if (!isPreCollection || (this.productType != null && name.endsWith(this.productType.toString()))) {
                        LandsatProductDescriptor temporaryDescriptor = new LandsatProductDescriptor(name);
                        Calendar productDate = temporaryDescriptor.getAcquisitionDate();
                        if (startDate.before(productDate) && endDate.after(productDate)) {
                            String jsonTile = tileUrl + name + ProductDownloader.URL_SEPARATOR + name + "_MTL.json";
                            jsonTile = jsonTile.replace(Constants.L8_SEARCH_URL_SUFFIX, "");
                            double clouds = getTileCloudPercentage(jsonTile);
                            if (clouds > this.cloudFilter) {
                                productDate.add(Calendar.MONTH, -1);
                                Logger.getRootLogger().warn(
                                        String.format("Tile %s from %s has %.2f %% clouds",
                                                      tile, dateFormat.format(productDate.getTime()), clouds));
                            } else {
                                ProductDescriptor descriptor = parseProductJson(jsonTile);
                                results.put(descriptor.getName(), descriptor);
                            }
                        }
                    }
                }
            }
        }
        Logger.getRootLogger().info("Query returned %s products", results.size());
        return new ArrayList<>(results.values());
    }

    private ProductDescriptor parseProductJson(String jsonUrl) throws IOException, URISyntaxException {
        JsonReader reader = null;
        ProductDescriptor descriptor = null;
        try (InputStream inputStream = new URI(jsonUrl).toURL().openStream()) {
            reader = Json.createReader(inputStream);
            JsonObject obj = reader.readObject()
                    .getJsonObject("L1_METADATA_FILE")
                    .getJsonObject("METADATA_FILE_INFO");
            if (obj.containsKey("LANDSAT_PRODUCT_ID")) {
                descriptor = new LandsatProductDescriptor(obj.getString("LANDSAT_PRODUCT_ID"));
            } else {
                descriptor = new LandsatProductDescriptor(obj.getString("LANDSAT_SCENE_ID"));
            }
            descriptor.setId(obj.getString("LANDSAT_SCENE_ID"));
        } finally {
            if (reader != null) {
                reader.close();
            }
        }
        return descriptor;
    }

    private double getTileCloudPercentage(String jsonUrl) throws IOException, URISyntaxException {
        JsonReader reader = null;
        try (InputStream inputStream = new URI(jsonUrl).toURL().openStream()) {
            reader = Json.createReader(inputStream);
            JsonObject obj = reader.readObject();
            return obj.getJsonObject("L1_METADATA_FILE")
                    .getJsonObject("IMAGE_ATTRIBUTES")
                    .getJsonNumber("CLOUD_COVER").doubleValue();
        } finally {
            if (reader != null) {
                reader.close();
            }
        }
    }
} public enum LandsatCollection {
    PRE,
    C1;
} public class LandsatProductDescriptor extends ProductDescriptor {
    private static final Pattern preCollectionNamePattern = Pattern.compile("L\\w[1-8](\\d{3})(\\d{3})(\\d{4})(\\d{3})\\w{3}\\d{2}");
    private static final Pattern collection1NamePattern = Pattern.compile("L\\w\\d{2}_L[1-2]\\w{2}_(\\d{3})(\\d{3})_(\\d{4})(\\d{2})(\\d{2})_\\d{8}_\\d{2}_(\\w{2})");
    private boolean oldFormat;
    private String[] nameTokens;
    private String row;
    private String path;
    private CollectionCategory productType;

    public LandsatProductDescriptor() {
    }

    public LandsatProductDescriptor(String name) {
        super(name);
        this.version = this.oldFormat ? Constants.L8_PRECOLL : Constants.L8_COLL;
        this.nameTokens = this.oldFormat ? getTokens(preCollectionNamePattern, name, null)
                : getTokens(collection1NamePattern, name, null);
    }

    @Override
    public String getVersion() {
        if (this.version == null) {
            this.version = this.oldFormat ? Constants.L8_PRECOLL : Constants.L8_COLL;
        }
        return this.version;
    }

    @Override
    public String getSensingDate() {
        return new SimpleDateFormat("yyyyMMdd").format(getAcquisitionDate().getTime());
    }

    public CollectionCategory getProductType() {
        if (this.productType == null) {
            this.productType = Enum.valueOf(CollectionCategory.class, nameTokens[5]);
        }
        return this.productType;
    }

    String getRow() {
        if (this.row == null && this.nameTokens != null) {
            this.row = nameTokens[1];
        }
        return this.row;
    }

    void setRow(String row) {
        this.row = row;
    }

    public String getPath() {
        if (this.path == null && this.nameTokens != null) {
            this.path = nameTokens[0];
        }
        return this.path;
    }

    public void setPath(String path) {
        this.path = path;
    }

    @Override
    public String getProductRelativePath() {
        StringBuilder buffer = new StringBuilder();
        if (!this.oldFormat) {
            buffer.append("c1").append(ProductDownloader.URL_SEPARATOR);
        }
        buffer.append("L8").append(ProductDownloader.URL_SEPARATOR);
        buffer.append(getPath()).append(ProductDownloader.URL_SEPARATOR);
        buffer.append(getRow()).append(ProductDownloader.URL_SEPARATOR);
        buffer.append(this.name).append(ProductDownloader.URL_SEPARATOR);
        return buffer.toString();
    }

    Calendar getAcquisitionDate() {
        Calendar calendar = Calendar.getInstance();
        LocalDate localDate = null;
        Matcher matcher;
        if (this.oldFormat) {
            matcher = preCollectionNamePattern.matcher(name);
            //noinspection ResultOfMethodCallIgnored
            matcher.matches();
            localDate = Year.of(Integer.parseInt(matcher.group(3))).atDay(Integer.parseInt(matcher.group(4)));
        } else {
            matcher = collection1NamePattern.matcher(name);
            //noinspection ResultOfMethodCallIgnored
            matcher.matches();
            localDate = LocalDate.parse(matcher.group(3)
                                                + "-" + matcher.group(4)
                                                + "-" + matcher.group(5));
        }
        calendar.setTime(Date.from(localDate.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant()));
        return calendar;
    }

    @Override
    protected boolean verifyProductName(String name) {
        this.oldFormat = preCollectionNamePattern.matcher(name).matches();
        return this.oldFormat || collection1NamePattern.matcher(name).matches();
    }
} public class LandsatProductDownloader extends ProductDownloader<LandsatProductDescriptor> {
    private static final Set<String> bandFiles = new LinkedHashSet<String>() {{
        add("_B1.TIF");
        add("_B2.TIF");
        add("_B3.TIF");
        add("_B4.TIF");
        add("_B5.TIF");
        add("_B6.TIF");
        add("_B7.TIF");
        add("_B8.TIF");
        add("_B9.TIF");
        add("_B10.TIF");
        add("_B11.TIF");
        add("_BQA.TIF");
    }};

    public LandsatProductDownloader(String targetFolder, Properties properties) {
        super(targetFolder, properties, null);

        baseUrl = props.getProperty("l8.aws.products.url", "http://landsat-pds.s3.amazonaws.com/");
        if (!baseUrl.endsWith("/"))
            baseUrl += "/";
    }

    @Override
    protected String getMetadataUrl(LandsatProductDescriptor descriptor) {
        return getProductUrl(descriptor) + descriptor.getName() + "_MTL.txt";
    }

    @Override
    protected Path download(LandsatProductDescriptor product) throws IOException {
        String url;
        String productName = product.getName();
        Path rootPath = Utilities.ensureExists(Paths.get(destination, productName));
        productLogger = new Logger.ScopeLogger(rootPath.resolve("download.log").toString());
        url = getMetadataUrl(product);
        Path metadataFile = rootPath.resolve(productName + "_MTL.txt");
        currentStep = "Metadata";
        getLogger().debug("Downloading metadata file %s", metadataFile);
        metadataFile = downloadFile(url, metadataFile);
        if (metadataFile != null && Files.exists(metadataFile)) {
            for (String suffix : bandFiles) {
                String bandName = suffix.substring(1, suffix.indexOf("."));
                if (this.bands == null || this.bands.contains(bandName)) {
                    String bandFileName = productName + suffix;
                    currentStep = "Band " + bandFileName;
                    try {
                        String bandFileUrl = getProductUrl(product) + bandFileName;
                        Path path = rootPath.resolve(bandFileName);
                        getLogger().debug("Downloading band raster %s from %s", path, bandFileUrl);
                        downloadFile(bandFileUrl, path);
                    } catch (IOException ex) {
                        getLogger().warn("Download for %s failed [%s]", bandFileName, ex.getMessage());
                    }
                }
            }
            if (Constants.L8_COLL.equals(product.getVersion())) {
                String fileName = productName + "_ANG.txt";
                try {
                    String fileUrl = getProductUrl(product) + fileName;
                    Path path = rootPath.resolve(fileName);
                    getLogger().debug("Downloading band raster %s from %s", path, fileUrl);
                    downloadFile(fileUrl, path);
                } catch (IOException ex) {
                    getLogger().warn("Download for %s failed [%s]", fileName, ex.getMessage());
                }
            }
        } else {
            getLogger().warn("Either the product %s was not found or the metadata file could not be downloaded", productName);
            rootPath = null;
        }
        if (shouldCompress && rootPath != null) {
            getLogger().debug("Compressing product %s", product);
            Zipper.compress(rootPath, rootPath.getFileName().toString(), shouldDeleteAfterCompression);
        }
        return rootPath;
    }

    @Override
    protected String getProductUrl(LandsatProductDescriptor descriptor) {
        return baseUrl + descriptor.getProductRelativePath();
    }

    @Override
    protected boolean isIntendedFor(LandsatProductDescriptor product) {
        return true;
    }
} public class LandsatTilesMap extends TileMap {
    private static final LandsatTilesMap instance;

    static {
        instance = new LandsatTilesMap();
    }

    public static LandsatTilesMap getInstance() {
        return instance;
    }

    private LandsatTilesMap() { super(); }

    @Override
    public void fromKml(BufferedReader bufferedReader) throws IOException {
        try {
            String line;
            String path = null, row = null;
            boolean inElement = false;
            while ((line = bufferedReader.readLine()) != null) {
                if (line.contains("<Placemark>")) {
                    inElement = true;
                } else {
                    if (inElement && line.contains("name=\"PATH\"")) {
                        int i = line.indexOf("name=\"PATH\"");
                        path = line.substring(i + 12, line.indexOf("</"));
                        path = ("000" + path).substring(path.length());
                    }
                    if (inElement && line.contains("name=\"ROW\"")) {
                        int i = line.indexOf("name=\"ROW\"");
                        row = line.substring(i + 11, line.indexOf("</"));
                        row = ("000" + row).substring(row.length());
                    }
                    if (inElement && !line.trim().startsWith("<")) {
                        String[] tokens = line.trim().split(" ");
                        Polygon2D polygon = new Polygon2D();
                        for (String point : tokens) {
                            String[] coords = point.split(",");
                            polygon.append(Double.parseDouble(coords[0]), Double.parseDouble(coords[1]));
                        }
                        tiles.put(path + row, polygon.getBounds2D());
                        inElement = false;
                    }
                }
            }
        } finally {
            if (bufferedReader != null)
                bufferedReader.close();
        }
    }
} public enum ProductStore {
    /*
     * Amazon public bucket
     */
    AWS,
    /*
     * Copernicus Science Hub
     */
    SCIHUB,
    /*
     * File system
     */
    LOCAL
} public enum ProductType {
    S2MSI1C,
    S2MSI2Ap,
    S2MSI2A
} public class S2L1CProductDescriptor extends SentinelProductDescriptor {

    private static final Pattern ProductV13 = Pattern.compile("(S2[A-B])_(OPER)_(PRD)_(MSIL1C)_(PDMC)_(\\d{8}T\\d{6})_(R\\d{3})_(V\\d{8}T\\d{6})_(\\d{8}T\\d{6})(?:.SAFE)?");
    private static final Pattern ProductV14 = Pattern.compile("(S2[A-B])_(MSIL1C)_(\\d{8}T\\d{6})_(N\\d{4})_(R\\d{3})_(T\\d{2}\\w{3})_(\\d{8}T\\d{6})(?:.SAFE)?");
    private static final Pattern TileV13 = Pattern.compile("(S2[A-B])_(OPER)_(MSI)_(L1C)_(TL)_(\\w{3})__(\\d{8}T\\d{6})_(A\\d{6})_(T\\d{2}\\w{3})_(N\\d{2}.\\d{2})");

    private boolean oldFormat;

    public S2L1CProductDescriptor() {
        super();
    }

    public S2L1CProductDescriptor(String name) {
        super(name);
        this.version = this.oldFormat ? Constants.PSD_13 : Constants.PSD_14;
    }

    @Override
    public PlatformType getPlatform() {
        String[] tokens = this.oldFormat ?
                getTokens(ProductV13, this.name, null) :
                getTokens(ProductV14, this.name, null);
        return Enum.valueOf(PlatformType.class, tokens[0]);
    }

    @Override
    public String getSensingDate() {
        String[] tokens = getTokens(this.oldFormat ? ProductV13 : ProductV14, this.name, null);
        return this.oldFormat ? tokens[7].substring(1, 9) : tokens[2].substring(0, 8);
    }

    @Override
    public String getVersion() {
        if (this.version == null) {
            this.version = this.oldFormat ? Constants.PSD_13 : Constants.PSD_14;;
        }
        return this.version;
    }

    @Override
    public String getProductRelativePath() {
        String year, day, month;
        if (this.oldFormat) {
            String[] tokens = getTokens(ProductV13, this.name, null);
            String dateToken = tokens[7];
            year = dateToken.substring(1, 5);
            month = String.valueOf(Integer.parseInt(dateToken.substring(5, 7)));
            day = String.valueOf(Integer.parseInt(dateToken.substring(7, 9)));
        } else {
            String[] tokens = getTokens(ProductV14, this.name, null);
            String dateToken = tokens[2];
            year = dateToken.substring(0, 4);
            month = String.valueOf(Integer.parseInt(dateToken.substring(4, 6)));
            day = String.valueOf(Integer.parseInt(dateToken.substring(6, 8)));
        }
        return year + ProductDownloader.URL_SEPARATOR + month + ProductDownloader.URL_SEPARATOR + day + ProductDownloader.URL_SEPARATOR + this.name + ProductDownloader.URL_SEPARATOR;
    }

    @Override
    String getTileIdentifier() {
        String tileId = null;
        if (!this.oldFormat) {
            tileId = getTokens(ProductV14, this.name, null)[5];
            tileId = tileId.substring(1);
        }
        return tileId;
    }

    @Override
    String getMetadataFileName() {
        String metaName;
        String[] tokens;
        String prodName = this.name.endsWith(".SAFE") ? this.name.substring(0, this.name.length() - 5) : this.name;
        if (this.oldFormat) {
            tokens = getTokens(ProductV13, prodName, new HashMap<Integer, String>() {{
                put(2, "MTD"); put(3, "SAFL1C");
            }});
            metaName = String.join("_", tokens) + ".xml";
        } else {
            metaName = "MTD_MSIL1C.xml";
        }
        return metaName;
    }

    @Override
    String getDatastripMetadataFileName(String datastripIdentifier) {
        String name;
        if (this.oldFormat) {
            name = datastripIdentifier.substring(0, datastripIdentifier.lastIndexOf("_")) + ".xml";
        } else {
            name = "MTD_DS.xml";
        }
        return name;
    }

    @Override
    String getDatastripFolder(String datastripIdentifier) {
        String folder;
        if (this.oldFormat) {
            folder = datastripIdentifier;
        } else {
            folder = datastripIdentifier.substring(17, 57);
        }
        return folder;
    }

    @Override
    String getGranuleFolder(String datastripIdentifier, String granuleIdentifier) {
        String folder;
        if (this.oldFormat) {
            folder = granuleIdentifier;
        } else {
            folder = granuleIdentifier.substring(13, 16) + "_" +
                        granuleIdentifier.substring(49, 55) + "_" +
                        granuleIdentifier.substring(41, 48) + "_" +
                        datastripIdentifier.substring(42, 57);
        }
        return folder;
    }

    @Override
    String getGranuleMetadataFileName(String granuleIdentifier) {
        String metaName;
        if (this.oldFormat) {
            String tokens[] = getTokens(TileV13, granuleIdentifier, new HashMap<Integer, String>() {{
                put(2, "MTD"); put(9, "");
            }});
            metaName = String.join("_", Arrays.copyOfRange(tokens, 0, 6)) + "__" + String.join("_", Arrays.copyOfRange(tokens, 6, 9)) + ".xml";
        } else {
            metaName = "MTD_TL.xml";
        }
        return metaName;
    }

    @Override
    String getBandFileName(String granuleIdentifier, String band) {
        String fileName;
        String[] tokens;
        String prodName = this.name.endsWith(".SAFE") ? this.name.substring(0, this.name.length() - 5) : this.name;
        if (this.oldFormat) {
            tokens = getTokens(TileV13, granuleIdentifier, null);
            fileName = String.join("_", Arrays.copyOfRange(tokens, 0, 6)) + "__" + String.join("_", Arrays.copyOfRange(tokens, 6, 9)) + "_" + band;
        } else {
            tokens = getTokens(ProductV14, prodName, null);
            fileName = tokens[5] + "_" + tokens[2] + "_" + band;
        }
        return fileName;
    }

    @Override
    String getEcmWftFileName(String granuleIdentifier) {
        String fileName;
        String prodName = this.name.endsWith(".SAFE") ? this.name.substring(0, this.name.length() - 5) : this.name;
        if (this.oldFormat) {
            String[] granuleTokens = getTokens(TileV13, granuleIdentifier, null);
            String[] productTokens = getTokens(ProductV13, prodName, null);
            SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss");
            String dateStart = "", dateEnd = "";
            try {
                Date date1 = dateFormat.parse(productTokens[7].substring(1, productTokens[7].length()));
                Calendar cal1 = Calendar.getInstance();
                cal1.setTime(date1);;
                cal1.set(cal1.get(Calendar.YEAR), cal1.get(Calendar.MONTH), cal1.get(Calendar.DAY_OF_MONTH),
                        cal1.get(Calendar.HOUR_OF_DAY), 0, 0);
                dateStart = dateFormat.format(cal1.getTime());
                cal1.add(Calendar.HOUR_OF_DAY, 12);
                dateEnd = dateFormat.format(cal1.getTime());
            } catch (ParseException e) {
                Logger.getRootLogger().error(e.getMessage());
            }
            fileName = String.join("_", productTokens[0], productTokens[1], "AUX", "ECMWFT", granuleTokens[5], "", granuleTokens[6], dateStart, dateEnd);
        } else {
            fileName = "AUX_ECMWFT";
        }
        return fileName;
    }

    @Override
    protected boolean verifyProductName(String name) {
        this.oldFormat = ProductV13.matcher(name).matches();
        return this.oldFormat || ProductV14.matcher(name).matches();
    }
} public class Sentinel2PreOpsDownloader extends SentinelProductDownloader {

    public Sentinel2PreOpsDownloader(ProductStore source, String targetFolder, Properties properties, NetUtils netUtils) {
        super(source, targetFolder, properties, netUtils);

        zipsUrl = props.getProperty(Constants.PROPERTY_NAME_AWS_SEARCH_URL, Constants.PROPERTY_DEFAULT_AWS_SEARCH_URL);
        if (!zipsUrl.endsWith("/"))
            zipsUrl += "/";
        zipsUrl += "zips/";
        baseUrl = props.getProperty(Constants.PROPERTY_NAME_AWS_TILES_URL, Constants.PROPERTY_DEFAULT_AWS_TILES_URL);
        if (!baseUrl.endsWith("/"))
            baseUrl += "/";
        productsUrl = baseUrl + "products/";
        ODataPath odp = new ODataPath();
        String scihubUrl = props.getProperty(Constants.PROPERTY_NAME_SCIHUB_PREOPS_PRODUCTS_URL,
                                             Constants.PROPERTY_DEFAULT_SCIHUB_PREOPS_PRODUCTS_URL);
        odataProductPath = odp.root(scihubUrl + "/Products('${UUID}')").node("${PRODUCT_NAME}.SAFE").path();
        odataArchivePath = odp.root(scihubUrl + "/Products('${UUID}')").value();
        odp.root(odataProductPath).node(Constants.FOLDER_GRANULE).node("${tile}");
        odataTilePath = odp.path();
        odataMetadataPath = odp.root(odataProductPath).node(Constants.ODATA_XML_PLACEHOLDER).value();
    }

    @Override
    protected boolean isIntendedFor(SentinelProductDescriptor product) {
        return PlatformType.S2B.equals(product.getPlatform());
    }
} public class SentinelProductDownloader extends ProductDownloader<SentinelProductDescriptor> {
    private static final Set<String> l1cBandFiles = new LinkedHashSet<String>() {{
        add("B01.jp2");
        add("B02.jp2");
        add("B03.jp2");
        add("B04.jp2");
        add("B05.jp2");
        add("B06.jp2");
        add("B07.jp2");
        add("B08.jp2");
        add("B8A.jp2");
        add("B09.jp2");
        add("B10.jp2");
        add("B11.jp2");
        add("B12.jp2");
    }};
    private static final Map<String, Set<String>> l2aBandFiles = new HashMap<String, Set<String>>() {{
        Set<String> files = new LinkedHashSet<>();
        put("R10m", files);
        files.add("AOT_10m.jp2");
        files.add("B02_10m.jp2");
        files.add("B03_10m.jp2");
        files.add("B04_10m.jp2");
        files.add("B08_10m.jp2");
        files.add("TCI_10m.jp2");
        files.add("WVP_10m.jp2");
        files = new LinkedHashSet<>();
        put("R20m", files);
        files.add("AOT_20m.jp2");
        files.add("B02_20m.jp2");
        files.add("B03_20m.jp2");
        files.add("B04_20m.jp2");
        files.add("B05_20m.jp2");
        files.add("B06_20m.jp2");
        files.add("B07_20m.jp2");
        files.add("B11_20m.jp2");
        files.add("B12_20m.jp2");
        files.add("B8A_20m.jp2");
        files.add("SCL_20m.jp2");
        files.add("TCI_20m.jp2");
        files.add("VIS_20m.jp2");
        files.add("WVP_20m.jp2");
        files = new LinkedHashSet<>();
        put("R60m", files);
        files.add("AOT_60m.jp2");
        files.add("B01_60m.jp2");
        files.add("B02_60m.jp2");
        files.add("B03_60m.jp2");
        files.add("B04_60m.jp2");
        files.add("B05_60m.jp2");
        files.add("B06_60m.jp2");
        files.add("B07_60m.jp2");
        files.add("B09_60m.jp2");
        files.add("B11_60m.jp2");
        files.add("B12_60m.jp2");
        files.add("B8A_60m.jp2");
        files.add("SCL_60m.jp2");
        files.add("TCI_60m.jp2");
        files.add("WVP_60m.jp2");
    }};
    private static final Set<String> l2aMasks = new LinkedHashSet<String>() {{
        add("CLD_20m.jp2");
        add("CLD_60m.jp2");
        add("PVI.jp2");
        add("SNW_20m.jp2");
        add("SNW_60m.jp2");
    }};

    String productsUrl;
    String baseUrl;
    String zipsUrl;
    String odataProductPath;
    String odataArchivePath;
    String odataTilePath;
    String odataMetadataPath;

    Set<String> filteredTiles;
    boolean shouldFilterTiles;
    Pattern tileIdPattern;
    FillAnglesMethod fillMissingAnglesMethod;

    Logger.ScopeLogger productLogger;

    public SentinelProductDownloader(ProductStore source, String targetFolder, Properties properties, NetUtils netUtils) {
        super(targetFolder, properties, netUtils);
        this.store = source;

        zipsUrl = props.getProperty(Constants.PROPERTY_NAME_AWS_SEARCH_URL, Constants.PROPERTY_DEFAULT_AWS_SEARCH_URL);
        if (!zipsUrl.endsWith("/"))
            zipsUrl += "/";
        zipsUrl += "zips/";
        baseUrl = props.getProperty(Constants.PROPERTY_NAME_AWS_TILES_URL, Constants.PROPERTY_DEFAULT_AWS_TILES_URL);
        if (!baseUrl.endsWith("/"))
            baseUrl += "/";
        productsUrl = baseUrl + "products/";
        ODataPath odp = new ODataPath();
        String scihubUrl;
        scihubUrl = props.getProperty(Constants.PROPERTY_NAME_SCIHUB_PRODUCTS_URL,
                                      Constants.PROPERTY_DEFAULT_SCIHUB_PRODUCTS_URL);
        if (source.equals(ProductStore.SCIHUB) && !this.netUtils.isAvailable(scihubUrl)) {
            System.err.println(scihubUrl + " is not available!");
            scihubUrl = props.getProperty(Constants.PROPERTY_NAME_SCIHUB_BACKUP_SEARCH_URL,
                                          Constants.PROPERTY_DEFAULT_SCIHUB_BACKUP_SEARCH_URL);
        }
        odataProductPath = odp.root(scihubUrl + "/Products('${UUID}')").node("${PRODUCT_NAME}.SAFE").path();
        odataArchivePath = odp.root(scihubUrl + "/Products('${UUID}')").value();
        odp.root(odataProductPath).node(Constants.FOLDER_GRANULE).node("${tile}");
        odataTilePath = odp.path();
        odataMetadataPath = odp.root(odataProductPath).node(Constants.ODATA_XML_PLACEHOLDER).value();
        fillMissingAnglesMethod = FillAnglesMethod.NONE;
    }

    public void setFilteredTiles(Set<String> tiles, boolean unpacked) {
        this.filteredTiles = tiles;
        if (shouldFilterTiles = (tiles != null && tiles.size() > 0) || unpacked) {
            StringBuilder text = new StringBuilder();
            text.append("(?:.+)(");
            if (tiles != null) {
                int idx = 1, n = tiles.size();
                for (String tile : tiles) {
                    text.append(tile);
                    if (idx++ < n)
                        text.append("|");
                }
                text.append(")(?:.+)");
                tileIdPattern = Pattern.compile(text.toString());
            }
        }
    }

    public void setFillMissingAnglesMethod(FillAnglesMethod value) {
        this.fillMissingAnglesMethod = value;
    }

    @Override
    public void setBandList(String[] bands) {
        if (bands != null) {
            this.bands = new HashSet<>();
            for (String band : bands) {
                this.bands.add(band);
                if (band.substring(1).length() == 1) {
                    this.bands.add(band.substring(0, 1) + String.format("%02d", Integer.parseInt(band.substring(1))));
                }
            }
        }
    }

    public void copyOptionsFrom(SentinelProductDownloader anotherDownloader) {
        this.downloadMode = anotherDownloader.downloadMode;
        this.props = anotherDownloader.props;
        this.destination = anotherDownloader.destination;
        this.shouldCompress = anotherDownloader.shouldCompress;
        this.shouldDeleteAfterCompression = anotherDownloader.shouldDeleteAfterCompression;
        this.productLogger = anotherDownloader.productLogger;
        this.batchProgressListener = anotherDownloader.batchProgressListener;
        this.fileProgressListener = anotherDownloader.fileProgressListener;
        this.bands = anotherDownloader.bands;
        this.filteredTiles = anotherDownloader.filteredTiles;
        this.shouldFilterTiles = anotherDownloader.shouldFilterTiles;
        this.tileIdPattern = anotherDownloader.tileIdPattern;
        this.fillMissingAnglesMethod = anotherDownloader.fillMissingAnglesMethod;
        this.store = anotherDownloader.store;
    }

    @Override
    protected Path download(SentinelProductDescriptor product) throws IOException {
        String tileId = product.getTileIdentifier();
        if (tileId != null && this.filteredTiles != null && this.filteredTiles.size() > 0 && !this.filteredTiles.contains(tileId)) {
            getLogger().warn("(" + currentProduct + ") The product %s did not contain any tiles from the tile list", product.getName());
            return null;
        }
        resetCounter();
        Path result = null;
        switch (store) {
            case AWS:
                result = downloadFromAWS(product);
                break;
            case SCIHUB:
            default:
                String uuid = product.getId();
                if (uuid == null || uuid.isEmpty()) {
                    getLogger().warn("Product identifier for %s missing from record", product.getName());
                } else {
                    result = downloadFromSciHub(product);
                }
                break;
        }
        if (result != null) {
            getLogger().info("(" + currentProduct + ") Average download speed: %.2f kB/s", getAverageSpeed());
        }
        return result;
    }

    @Override
    protected Path link(SentinelProductDescriptor product) throws IOException {
        String tileId = product.getTileIdentifier();
        if (tileId != null && this.filteredTiles != null && !this.filteredTiles.contains(tileId)) {
            getLogger().warn("(" + currentProduct + ") The product %s did not contain any tiles from the tile list", product.getName());
            return null;
        }
        return linkProduct(product);
    }

    @Override
    protected String getMetadataUrl(SentinelProductDescriptor descriptor) {
        String url = null;
        switch (store) {
            case AWS:
                url = getProductUrl(descriptor) + "metadata.xml";
                break;
            case SCIHUB:
                String metadateFileName = descriptor.getMetadataFileName();
                url = odataMetadataPath.replace(Constants.ODATA_UUID, descriptor.getId())
                                  .replace(Constants.ODATA_PRODUCT_NAME, descriptor.getName())
                                  .replace(Constants.ODATA_XML_PLACEHOLDER, metadateFileName);
                break;
        }
        return url;
    }

    @Override
    protected boolean isIntendedFor(SentinelProductDescriptor product) {
        final PlatformType platform = product.getPlatform();
        return PlatformType.S2A.equals(platform) || PlatformType.S2B.equals(platform);
    }

    @Override
    protected Path findProductPath(Path root, SentinelProductDescriptor product) {
        String name = product.getName();
        if (!name.endsWith(".SAFE")) {
            product.setName(name + ".SAFE");
        }
        return super.findProductPath(root, product);
    }

    private Path downloadFromSciHub(SentinelProductDescriptor productDescriptor) throws IOException {
        Path rootPath = null;
        String url;
        Utilities.ensureExists(Paths.get(destination));
        String productName = productDescriptor.getName();
        boolean isL1C = productDescriptor instanceof S2L1CProductDescriptor;
        if (Constants.PSD_13.equals(productDescriptor.getVersion()) && !shouldFilterTiles) {
            currentStep = "Archive";
            url = odataArchivePath.replace(Constants.ODATA_UUID, productDescriptor.getId());
            rootPath = Paths.get(destination, productName + ".zip");
            rootPath = downloadFile(url, rootPath, this.netUtils.getAuthToken());
        }
        if (rootPath == null || !Files.exists(rootPath)) {
            rootPath = Utilities.ensureExists(Paths.get(destination, productName + ".SAFE"));
            url = getMetadataUrl(productDescriptor);
            Path metadataFile = rootPath.resolve(productDescriptor.getMetadataFileName());
            currentStep = "Metadata";
            downloadFile(url, metadataFile, this.netUtils.getAuthToken());
            if (Files.exists(metadataFile)) {
                List<String> allLines = Files.readAllLines(metadataFile);
                List<String> metaTileNames = Utilities.filter(allLines, "<Granule" + (Constants.PSD_13.equals(productDescriptor.getVersion()) ? "s" : " "));
                boolean hasTiles = updateMedatata(metadataFile, allLines) != null;
                if (!shouldFilterTiles || hasTiles) {
                    Path tilesFolder = Utilities.ensureExists(rootPath.resolve(Constants.FOLDER_GRANULE));
                    Utilities.ensureExists(rootPath.resolve(Constants.FOLDER_AUXDATA));
                    Path dataStripFolder = Utilities.ensureExists(rootPath.resolve(Constants.FOLDER_DATASTRIP));
                    Map<String, String> tileNames = new HashMap<>();
                    String dataStripId = null;
                    String skippedTiles = "";
                    for (String tileName : metaTileNames) {
                        String tileId;
                        if (isL1C) {
                            tileId = tileName.substring(0, tileName.lastIndexOf(NAME_SEPARATOR));
                            tileId = tileId.substring(tileId.lastIndexOf(NAME_SEPARATOR) + 2);
                        } else {
                            int idx = tileName.lastIndexOf(NAME_SEPARATOR + "T");
                            tileId = tileName.substring(idx + 2, idx + 7);
                        }
                        if (filteredTiles.size() == 0 || filteredTiles.contains(tileId)) {
                            String granuleId = Utilities.getAttributeValue(tileName, Constants.XML_ATTR_GRANULE_ID);
                            if (dataStripId == null) {
                                dataStripId = Utilities.getAttributeValue(tileName, Constants.XML_ATTR_DATASTRIP_ID);
                            }
                            String granule = productDescriptor.getGranuleFolder(dataStripId, granuleId);
                            tileNames.put(granuleId, odataTilePath.replace(Constants.ODATA_UUID, productDescriptor.getId())
                                    .replace(Constants.ODATA_PRODUCT_NAME, productName)
                                    .replace("${tile}", granule));
                        } else {
                            skippedTiles += tileId + " ";
                        }
                    }
                    if (skippedTiles.trim().length() > 0) {
                        getLogger().debug("Skipped tiles: %s", skippedTiles);
                    }
                    String count = String.valueOf(tileNames.size());
                    int tileCounter = 1;
                    ODataPath pathBuilder = new ODataPath();
                    for (Map.Entry<String, String> entry : tileNames.entrySet()) {
                        long start = System.currentTimeMillis();
                        currentStep = "Tile " + String.valueOf(tileCounter++) + "/" + count;
                        String tileUrl = entry.getValue();
                        String granuleId = entry.getKey();
                        String tileName = productDescriptor.getGranuleFolder(dataStripId, granuleId);
                        Path tileFolder = Utilities.ensureExists(tilesFolder.resolve(tileName));
                        Path auxData = Utilities.ensureExists(tileFolder.resolve(Constants.FOLDER_AUXDATA));
                        Path imgData = Utilities.ensureExists(tileFolder.resolve(Constants.FOLDER_IMG_DATA));
                        Path qiData = Utilities.ensureExists(tileFolder.resolve(Constants.FOLDER_QI_DATA));
                        String metadataName = productDescriptor.getGranuleMetadataFileName(granuleId);
                        Path tileMetaFile = downloadFile(pathBuilder.root(tileUrl).node(metadataName).value(),
                                                         tileFolder.resolve(metadataName),
                                                         this.netUtils.getAuthToken());
                        if (tileMetaFile != null) {
                            if (Files.exists(tileMetaFile)) {
                                List<String> tileMetadataLines = MetadataRepairer.parse(tileMetaFile, this.fillMissingAnglesMethod);
                                if (isL1C) {
                                    for (String bandFileName : l1cBandFiles) {
                                        if (this.bands == null || this.bands.contains(bandFileName.substring(0, bandFileName.indexOf(".")))) {
                                            downloadFile(pathBuilder.root(tileUrl)
                                                                 .node(Constants.FOLDER_IMG_DATA)
                                                                 .node(productDescriptor.getBandFileName(granuleId, bandFileName))
                                                                 .value(),
                                                         imgData.resolve(productDescriptor.getBandFileName(granuleId, bandFileName)),
                                                         this.netUtils.getAuthToken());
                                        } else {
                                            getLogger().debug("Band %s skipped", bandFileName.substring(0, bandFileName.indexOf(".")));
                                        }
                                    }
                                } else {
                                    for (Map.Entry<String, Set<String>> resEntry : l2aBandFiles.entrySet()) {
                                        Path imgDataRes = Utilities.ensureExists(imgData.resolve(resEntry.getKey()));
                                        for (String bandFileName : resEntry.getValue()) {
                                            if (this.bands == null || this.bands.contains(bandFileName.substring(0, bandFileName.indexOf(".")))) {
                                                downloadFile(pathBuilder.root(tileUrl)
                                                                     .node(Constants.FOLDER_IMG_DATA)
                                                                     .node(resEntry.getKey())
                                                                     .node(productDescriptor.getBandFileName(granuleId, bandFileName))
                                                                     .value(),
                                                             imgDataRes.resolve(productDescriptor.getBandFileName(granuleId, bandFileName)),
                                                             this.netUtils.getAuthToken());
                                            } else {
                                                getLogger().debug("Band %s skipped", bandFileName.substring(0, bandFileName.indexOf(".")));
                                            }
                                        }
                                    }
                                }
                                List<String> lines = Utilities.filter(tileMetadataLines, "<MASK_FILENAME");
                                for (String line : lines) {
                                    line = line.trim();
                                    int firstTagCloseIdx = line.indexOf(">") + 1;
                                    int secondTagBeginIdx = line.indexOf("<", firstTagCloseIdx);
                                    String maskFileName = line.substring(firstTagCloseIdx, secondTagBeginIdx);
                                    maskFileName = maskFileName.substring(maskFileName.lastIndexOf(URL_SEPARATOR) + 1);
                                    final String mfn = maskFileName;
                                    if (this.bands == null || this.bands.stream().anyMatch(mfn::contains)) {
                                        downloadFile(pathBuilder.root(tileUrl)
                                                             .node(Constants.FOLDER_QI_DATA)
                                                             .node(maskFileName)
                                                             .value(),
                                                     qiData.resolve(maskFileName),
                                                     this.netUtils.getAuthToken());
                                    } else {
                                        getLogger().debug("Mask %s skipped", mfn);
                                    }
                                }
                                if (!isL1C) {
                                    for (String maskFileName : l2aMasks) {
                                        downloadFile(pathBuilder.root(tileUrl)
                                                             .node(Constants.FOLDER_QI_DATA)
                                                             .node(productDescriptor.getBandFileName(granuleId, maskFileName))
                                                             .value(),
                                                     qiData.resolve(productDescriptor.getBandFileName(granuleId, maskFileName)),
                                                     this.netUtils.getAuthToken());
                                    }
                                }
                                getLogger().info("Tile download completed in %s", Utilities.formatTime(System.currentTimeMillis() - start));
                            } else {
                                getLogger().error("File %s was not downloaded", tileMetaFile.getFileName());
                            }
                        }
                    }
                    if (dataStripId != null) {
                        String dataStripPath = pathBuilder.root(odataProductPath.replace(Constants.ODATA_UUID, productDescriptor.getId())
                                                                                .replace(Constants.ODATA_PRODUCT_NAME, productName))
                                                          .node(Constants.FOLDER_DATASTRIP).node(productDescriptor.getDatastripFolder(dataStripId))
                                                          .node(productDescriptor.getDatastripMetadataFileName(dataStripId))
                                                          .value();
                        Path dataStrip = Utilities.ensureExists(dataStripFolder.resolve(productDescriptor.getDatastripFolder(dataStripId)));
                        String dataStripFile = productDescriptor.getDatastripMetadataFileName(dataStripId);
                        downloadFile(dataStripPath, dataStrip.resolve(dataStripFile), this.netUtils.getAuthToken());
                    }
                } else {
                    Files.deleteIfExists(metadataFile);
                    //Files.deleteIfExists(rootPath);
                    rootPath = null;
                    getLogger().warn("(" + currentProduct + ") The product %s did not contain any tiles from the tile list", productName);
                }
            } else {
                getLogger().warn("(" + currentProduct + ") The product %s was not found in %s data bucket", productName, store);
                rootPath = null;
            }
        }
        if (rootPath != null && Files.exists(rootPath) && shouldCompress && !rootPath.endsWith(".zip")) {
            getLogger().debug("Compressing product %s", productDescriptor);
            Zipper.compress(rootPath, rootPath.getFileName().toString(), shouldDeleteAfterCompression);
        }
        return rootPath;
    }

    private Path downloadFromAWS(SentinelProductDescriptor productDescriptor) throws IOException {
        S2L1CProductDescriptor product = (S2L1CProductDescriptor) productDescriptor;
        Path rootPath = null;
        String url;
        String productName = product.getName();
        if (!shouldFilterTiles) {
            currentStep = "Archive";
            url = zipsUrl + productName + ".zip";
            rootPath = Paths.get(destination, product + ".zip");
            productLogger = new Logger.ScopeLogger(rootPath.getParent().resolve("download.log").toString());
            rootPath = downloadFile(url, rootPath);
        }
        if (rootPath == null || !Files.exists(rootPath)) {
            // let's try to assemble the product
            rootPath = Utilities.ensureExists(Paths.get(destination, productName + ".SAFE"));
            productLogger = new Logger.ScopeLogger(rootPath.resolve("download.log").toString());
            String baseProductUrl = getProductUrl(product);
            url = baseProductUrl + "metadata.xml";
            Path metadataFile = rootPath.resolve(product.getMetadataFileName()); //rootPath.resolve(productName.replace("PRD_MSIL1C", "MTD_SAFL1C") + ".xml");
            currentStep = "Metadata";
            getLogger().debug("Downloading metadata file %s", metadataFile);
            metadataFile = downloadFile(url, metadataFile);
            if (metadataFile != null && Files.exists(metadataFile)) {
                Path inspireFile = metadataFile.resolveSibling("INSPIRE.xml");
                Path manifestFile = metadataFile.resolveSibling("manifest.safe");
                Path previewFile = metadataFile.resolveSibling("preview.png");
                List<String> allLines = Files.readAllLines(metadataFile);
                List<String> metaTileNames = Utilities.filter(allLines, "<Granule" + (Constants.PSD_13.equals(product.getVersion()) ? "s" : " "));
                // maybe there's an old product with incorrect metadata
                if (metaTileNames.isEmpty() && Constants.PSD_13.equals(product.getVersion())) {
                    metaTileNames = Utilities.filter(allLines, "<Granule ");
                }
                boolean hasTiles = updateMedatata(metadataFile, allLines) != null;
                if (hasTiles) {
                    downloadFile(baseProductUrl + "inspire.xml", inspireFile);
                    downloadFile(baseProductUrl + "manifest.safe", manifestFile);
                    if (Constants.PSD_13.equals(productDescriptor.getVersion()))
                        downloadFile(baseProductUrl + "preview.png", previewFile);

                    // rep_info folder and contents
                    Path repFolder = Utilities.ensureExists(rootPath.resolve("rep_info"));
                    Path schemaFile = repFolder.resolve("S2_User_Product_Level-1C_Metadata.xsd");
                    copyFromResources(String.format("S2_User_Product_Level-1C_Metadata%s.xsd", product.getVersion()), schemaFile);
                    // HTML folder and contents
                    Path htmlFolder = Utilities.ensureExists(rootPath.resolve("HTML"));
                    copyFromResources("banner_1.png", htmlFolder);
                    copyFromResources("banner_2.png", htmlFolder);
                    copyFromResources("banner_3.png", htmlFolder);
                    copyFromResources("star_bg.jpg", htmlFolder);
                    copyFromResources("UserProduct_index.html", htmlFolder);
                    copyFromResources("UserProduct_index.xsl", htmlFolder);

                    Path tilesFolder = Utilities.ensureExists(rootPath.resolve(Constants.FOLDER_GRANULE));
                    Utilities.ensureExists(rootPath.resolve(Constants.FOLDER_AUXDATA));
                    Path dataStripFolder = Utilities.ensureExists(rootPath.resolve(Constants.FOLDER_DATASTRIP));
                    String productJsonUrl = baseProductUrl + "productInfo.json";
                    HttpURLConnection connection = null;
                    InputStream inputStream = null;
                    JsonReader reader = null;
                    try {
                        getLogger().debug("Downloading json product descriptor %s", productJsonUrl);
                        connection = NetUtils.openConnection(productJsonUrl);
                        inputStream = connection.getInputStream();
                        reader = Json.createReader(inputStream);
                        getLogger().debug("Parsing json descriptor %s", productJsonUrl);
                        JsonObject obj = reader.readObject();
                        final Map<String, String> tileNames = getTileNames(obj, metaTileNames, product.getVersion());
                        String dataStripId = null;
                        String count = String.valueOf(tileNames.size());
                        int tileCounter = 1;
                        for (Map.Entry<String, String> entry : tileNames.entrySet()) {
                            currentStep = "Tile " + String.valueOf(tileCounter++) + "/" + count;
                            String tileUrl = entry.getValue();
                            String tileName = entry.getKey();
                            Path tileFolder = Utilities.ensureExists(tilesFolder.resolve(tileName));
                            Path auxData = Utilities.ensureExists(tileFolder.resolve(Constants.FOLDER_AUXDATA));
                            Path imgData = Utilities.ensureExists(tileFolder.resolve(Constants.FOLDER_IMG_DATA));
                            Path qiData = Utilities.ensureExists(tileFolder.resolve(Constants.FOLDER_QI_DATA));
                            String metadataName = product.getGranuleMetadataFileName(tileName);
                            getLogger().debug("Downloading tile metadata %s", tileFolder.resolve(metadataName));
                            Path tileMetaFile = downloadFile(tileUrl + "/metadata.xml", tileFolder.resolve(metadataName));
                            List<String> tileMetadataLines = MetadataRepairer.parse(tileMetaFile, this.fillMissingAnglesMethod);
                            for (String bandFileName : l1cBandFiles) {
                                if (this.bands == null || this.bands.contains(bandFileName.substring(0, bandFileName.indexOf(".")))) {
                                    try {
                                        String bandFileUrl = tileUrl + URL_SEPARATOR + bandFileName;
                                        Path path = imgData.resolve(product.getBandFileName(tileName, bandFileName));
                                        getLogger().debug("Downloading band raster %s from %s", path, bandFileName);
                                        downloadFile(bandFileUrl, path);
                                    } catch (IOException ex) {
                                        getLogger().warn("Download for %s failed [%s]", bandFileName, ex.getMessage());
                                    }
                                } else {
                                    getLogger().debug("Band %s skipped", bandFileName.substring(0, bandFileName.indexOf(".")));
                                }
                            }
                            List<String> lines = Utilities.filter(tileMetadataLines, "<MASK_FILENAME");
                            for (String line : lines) {
                                line = line.trim();
                                int firstTagCloseIdx = line.indexOf(">") + 1;
                                int secondTagBeginIdx = line.indexOf("<", firstTagCloseIdx);
                                String maskFileName = line.substring(firstTagCloseIdx, secondTagBeginIdx);
                                if (this.bands == null || this.bands.stream().anyMatch(maskFileName::contains)) {
                                    String remoteName;
                                    Path path;
                                    if (Constants.PSD_13.equals(product.getVersion())) {
                                        String[] tokens = maskFileName.split(NAME_SEPARATOR);
                                        remoteName = tokens[2] + NAME_SEPARATOR + tokens[3] + NAME_SEPARATOR + tokens[9] + ".gml";
                                        path = qiData.resolve(maskFileName);
                                    } else {
                                        remoteName = maskFileName.substring(maskFileName.lastIndexOf(URL_SEPARATOR) + 1);
                                        path = rootPath.resolve(maskFileName);
                                    }

                                    try {
                                        String fileUrl = tileUrl + "/qi/" + remoteName;
                                        getLogger().debug("Downloading file %s from %s", path, fileUrl);
                                        downloadFile(fileUrl, path);
                                    } catch (IOException ex) {
                                        getLogger().warn("Download for %s failed [%s]", path, ex.getMessage());
                                    }
                                } else {
                                    getLogger().debug("Mask %s skipped", maskFileName);
                                }
                            }
                            getLogger().debug("Trying to download %s", tileUrl + "/auxiliary/ECMWFT");
                            downloadFile(tileUrl + "/auxiliary/ECMWFT", auxData.resolve(product.getEcmWftFileName(tileName))); //auxData.resolve(refName.replace(tilePrefix, auxPrefix)));
                            if (dataStripId == null) {
                                String tileJson = tileUrl + "/tileInfo.json";
                                HttpURLConnection tileConnection = null;
                                InputStream is = null;
                                JsonReader tiReader = null;
                                try {
                                    getLogger().debug("Downloading json tile descriptor %s", tileJson);
                                    tileConnection = NetUtils.openConnection(tileJson);
                                    is = tileConnection.getInputStream();
                                    tiReader = Json.createReader(is);
                                    getLogger().debug("Parsing json tile descriptor %s", tileJson);
                                    JsonObject tileObj = tiReader.readObject();
                                    dataStripId = tileObj.getJsonObject("datastrip").getString("id");
                                    String dataStripPath = tileObj.getJsonObject("datastrip").getString("path") + "/metadata.xml";
                                    Path dataStrip = Utilities.ensureExists(dataStripFolder.resolve(product.getDatastripFolder(dataStripId)));
                                    String dataStripFile = product.getDatastripMetadataFileName(dataStripId);
                                    Utilities.ensureExists(dataStrip.resolve(Constants.FOLDER_QI_DATA));
                                    getLogger().debug("Downloading %s", baseUrl + dataStripPath);
                                    downloadFile(baseUrl + dataStripPath, dataStrip.resolve(dataStripFile));
                                } finally {
                                    if (tiReader != null) tiReader.close();
                                    if (is != null) is.close();
                                    if (tileConnection != null) tileConnection.disconnect();
                                }
                            }
                        }
                    } finally {
                        if (reader != null) reader.close();
                        if (inputStream != null) inputStream.close();
                        if (connection != null) connection.disconnect();
                    }
                } else {
                    Files.deleteIfExists(metadataFile);
                    rootPath = null;
                    getLogger().warn("The product %s did not contain any tiles from the tile list", productName);
                }
            } else {
                getLogger().warn("Either the product %s was not found in %s data bucket or the metadata file could not be downloaded", productName, store);
                rootPath = null;
            }
        }
        if (rootPath != null && Files.exists(rootPath) && shouldCompress) {
            getLogger().debug("Compressing product %s", product);
            Zipper.compress(rootPath, rootPath.getFileName().toString(), shouldDeleteAfterCompression);
        }
        return rootPath;
    }

    private Path linkProduct(SentinelProductDescriptor productDescriptor) throws IOException {
        S2L1CProductDescriptor product = (S2L1CProductDescriptor) productDescriptor;
        String productName = product.getName();
        final Path productRepositoryPath = Paths.get(super.baseUrl);
        if (!shouldFilterTiles) {
            return link(productDescriptor, productRepositoryPath, Paths.get(destination));
        }
        Path destinationPath = Utilities.ensureExists(Paths.get(destination, productName + ".SAFE"));
        Path productSourcePath = findProductPath(productRepositoryPath, product);
        if (productSourcePath == null) {
            getLogger().warn("%s not found locally", productName);
            return null;
        }
        productLogger = new Logger.ScopeLogger(destinationPath.resolve("download.log").toString());
        Path metadataFile = destinationPath.resolve(product.getMetadataFileName());
        currentStep = "Metadata";
        getLogger().debug("Copying metadata file %s", metadataFile);
        copyFile(productSourcePath.resolve(metadataFile.getFileName()), metadataFile);
        if (Files.exists(metadataFile)) {
            List<String> allLines = Files.readAllLines(metadataFile);
            Set<String> tileNames = updateMedatata(metadataFile, allLines);
            if (tileNames != null) {
                List<Path> folders = Utilities.listFolders(productSourcePath);
                final Path destPath = destinationPath;
                folders.stream()
                        .filter(folder -> !folder.toString().contains("GRANULE") ||
                                          "GRANULE".equals(folder.getName(folder.getNameCount() - 1).toString()) ||
                                          tileNames.stream().anyMatch(tn -> folder.toString().contains(tn)))
                        .forEach(folder -> {
                                try {
                                    Utilities.ensureExists(destPath.resolve(productSourcePath.relativize(folder)));
                                    Utilities.listFiles(folder)
                                            .forEach(file -> {
                                                try {
                                                    linkFile(file,
                                                             destPath.resolve(productSourcePath.relativize(file)));
                                                } catch (IOException e) {
                                                    getLogger().warn(e.getMessage());
                                                }
                                            });
                                } catch (IOException e) {
                                    e.printStackTrace();
                                }
                        });
            } else {
                Files.deleteIfExists(metadataFile);
                getLogger().warn("The product %s did not contain any tiles from the tile list", productName);
                destinationPath = null;
            }
        } else {
            getLogger().warn("Either the product %s was not found in %s data bucket or the metadata file could not be downloaded", productName, store);
            destinationPath = null;
        }
        return destinationPath;
    }

    @Override
    protected String getProductUrl(SentinelProductDescriptor descriptor) {
        return productsUrl + descriptor.getProductRelativePath();
    }

    private String tilePathToTileId(String tilePath) {
        String[] tokens = tilePath.split(URL_SEPARATOR);
        return String.format("%02d%s%s",
                Integer.parseInt(tokens[1]),
                tokens[2],
                tokens[3]);
    }

    private Map<String, String> getTileNames(JsonObject productInfo, List<String> metaTileNames, String psdVersion) {
        Map<String, String> ret = new HashMap<>();
        String dataTakeId = productInfo.getString("datatakeIdentifier");
        String[] dataTakeTokens = dataTakeId.split(NAME_SEPARATOR);
        JsonArray tiles = productInfo.getJsonArray("tiles");
        String skippedTiles = "";
        for (JsonObject result : tiles.getValuesAs(JsonObject.class)) {
            String tilePath = result.getString("path");
            String tileId = tilePathToTileId(tilePath);
            if (!shouldFilterTiles || (filteredTiles.size() == 0 || filteredTiles.contains(tileId))) {
                tileId = "T" + tileId;
                String tileName = Utilities.find(metaTileNames, tileId, psdVersion);
                /*if (tileName == null) {
                    tileName = tilePrefix + dataTakeTokens[1] + "_A" + dataTakeTokens[2] + NAME_SEPARATOR + tileId + NAME_SEPARATOR + dataTakeTokens[3];
                }*/
                ret.put(tileName, baseUrl + tilePath);
            } else {
                skippedTiles += tileId + " ";
            }
        }
        if (skippedTiles.length() > 0) {
            getLogger().debug("Skipped tiles: %s", skippedTiles);
        }
        return ret;
    }

    private Set<String> updateMedatata(Path metaFile, List<String> originalLines) throws IOException {
        Set<String> extractedTileNames = null;
        if (shouldFilterTiles) {
            int tileCount = 0;
            List<String> lines = new ArrayList<>();
            for (int i = 0; i < originalLines.size(); i++) {
                String line = originalLines.get(i);
                boolean canProceed = line.contains("<Granule_List>") | line.contains("<Granule");
                if (canProceed) {
                    final String nextLine = originalLines.get(i + 1);
                    if (nextLine.contains("<Granules")) {
                        final Matcher matcher = tileIdPattern.matcher(nextLine);
                        if (matcher.matches()) {
                            if (extractedTileNames == null) {
                                extractedTileNames = new HashSet<>();
                            }
                            extractedTileNames.add(matcher.group(1));
                            lines.addAll(originalLines.subList(i, i + 17));
                            tileCount++;
                        }
                        i += 16;
                    } else if (line.contains("<Granule ")) {
                        final Matcher matcher = tileIdPattern.matcher(line);
                        if (matcher.matches()) {
                            if (extractedTileNames == null) {
                                extractedTileNames = new HashSet<>();
                            }
                            extractedTileNames.add(matcher.group(1));
                            lines.addAll(originalLines.subList(i, i + 16));
                            tileCount++;
                        }
                        i += 15;
                    } else {
                        lines.add(line);
                    }
                } else {
                    lines.add(line);
                }
            }
            if (tileCount > 0) {
                Files.write(metaFile, lines, StandardCharsets.UTF_8);
            }
        }
        return extractedTileNames;
    }

    private void copyFromResources(String fileName, Path file) throws IOException {
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(getClass().getResourceAsStream(fileName)))) {
            String line;
            StringBuilder builder = new StringBuilder();
            while ((line = reader.readLine()) != null) {
                builder.append(line).append("\n");
            }
            if (Files.isDirectory(file)) {
                Utilities.ensurePermissions(Files.write(file.resolve(fileName), builder.toString().getBytes()));
            } else {
                Utilities.ensurePermissions(Files.write(file, builder.toString().getBytes()));
            }
        }
    }

    class ODataPath {
        private StringBuilder buffer;

        ODataPath() {
            buffer = new StringBuilder();
        }

        ODataPath root(String path) {
            buffer.setLength(0);
            buffer.append(path);
            return this;
        }

        ODataPath node(String nodeName) {
            buffer.append("/Nodes('").append(nodeName).append("')");
            return this;
        }

        String path() {
            return buffer.toString();
        }

        String value() {
            return buffer.toString() + "/$value";
        }
    }
} public class SentinelTilesMap extends TileMap {

    private static final SentinelTilesMap instance;

    static {
        instance = new SentinelTilesMap();
    }

    public static SentinelTilesMap getInstance() {
        return instance;
    }

    private SentinelTilesMap() { super(); }

    @Override
    public void fromKml(BufferedReader bufferedReader) throws IOException {
        try {
            String line;
            String tileCode = null;
            boolean inElement = false;
            while ((line = bufferedReader.readLine()) != null) {
                if (line.contains("<Placemark>")) {
                    inElement = true;
                } else {
                    if (inElement && line.contains("<name>")) {
                        int i = line.indexOf("<name>");
                        tileCode = line.substring(i + 6, i + 11);
                    }
                    if (inElement && !line.trim().startsWith("<")) {
                        String[] tokens = line.trim().split(" ");
                        Polygon2D polygon = new Polygon2D();
                        for (String point : tokens) {
                            String[] coords = point.split(",");
                            polygon.append(Double.parseDouble(coords[0]), Double.parseDouble(coords[1]));
                        }
                        tiles.put(tileCode, polygon.getBounds2D());
                        inElement = false;
                    }
                }
            }
        } finally {
            if (bufferedReader != null)
                bufferedReader.close();
        }
    }
} public class AmazonSearch extends AbstractSearch<ProductType> {

    public AmazonSearch(String url) throws URISyntaxException {
        super(url);
    }

    @Override
    public AbstractSearch<ProductType> limit(int value) {
        this.pageSize = value;
        return this;
    }

    @Override
    public AbstractSearch<ProductType> start(int value) {
        this.offset = value;
        return this;
    }

    @Override
    protected List<ProductDescriptor> executeImpl() throws Exception {
        Map<String, ProductDescriptor> results = new LinkedHashMap<>();
        Set<String> tiles = this.tiles != null && this.tiles.size() > 0 ?
                this.tiles :
                this.aoi != null ?
                        SentinelTilesMap.getInstance().intersectingTiles(this.aoi.getBounds2D()) :
                        new HashSet<>();
        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
        if (this.sensingStart == null || this.sensingStart.isEmpty()) {
            this.sensingStart = "2015-06-27";
        }
        if (this.sensingEnd == null || this.sensingEnd.isEmpty()) {
            this.sensingEnd = dateFormat.format(new Date(System.currentTimeMillis()));
        }
        //http://sentinel-s2-l1c.s3.amazonaws.com/?delimiter=/&prefix=tiles/15/R/TM/
        Calendar startDate = Calendar.getInstance();
        startDate.setTime(dateFormat.parse(this.sensingStart));
        Calendar endDate = Calendar.getInstance();
        endDate.setTime(dateFormat.parse(this.sensingEnd));
        int yearStart = startDate.get(Calendar.YEAR);
        int monthStart = startDate.get(Calendar.MONTH) + 1;
        int dayStart = startDate.get(Calendar.DAY_OF_MONTH);
        int yearEnd = endDate.get(Calendar.YEAR);
        int monthEnd = endDate.get(Calendar.MONTH) + 1;
        int dayEnd = endDate.get(Calendar.DAY_OF_MONTH);
        for (String tile : tiles) {
            String utmCode = Integer.toString(Integer.parseInt(tile.substring(0, 2)));
            String latBand = tile.substring(2, 3);
            String square = tile.substring(3, 5);
            String tileUrl = this.url.toString() + utmCode + "/" + latBand + "/" + square + "/";
            for (int year = yearStart; year <= yearEnd; year++) {
                String yearUrl = tileUrl + String.valueOf(year) + "/";
                Result yearResult = ResultParser.parse(NetUtils.getResponseAsString(yearUrl));
                if (yearResult.getCommonPrefixes() != null) {
                    Set<Integer> months = yearResult.getCommonPrefixes().stream()
                            .map(p -> {
                                String tmp = p.replace(yearResult.getPrefix(), "");
                                return Integer.parseInt(tmp.substring(0, tmp.indexOf(yearResult.getDelimiter())));
                            }).collect(Collectors.toSet());
                    int monthS = year == yearStart ? monthStart : 1;
                    int monthE = year == yearEnd ? monthEnd : 12;
                    for (int month = monthS; month <= monthE; month++) {
                        if (months.contains(month)) {
                            String monthUrl = yearUrl + String.valueOf(month) + "/";
                            Result monthResult = ResultParser.parse(NetUtils.getResponseAsString(monthUrl));
                            if (monthResult.getCommonPrefixes() != null) {
                                Set<Integer> days = monthResult.getCommonPrefixes().stream()
                                        .map(p -> {
                                            String tmp = p.replace(monthResult.getPrefix(), "");
                                            return Integer.parseInt(tmp.substring(0, tmp.indexOf(monthResult.getDelimiter())));
                                        }).collect(Collectors.toSet());
                                int dayS = month == monthS ? dayStart : 1;
                                Calendar calendar = new Calendar.Builder().setDate(year, month + 1, 1).build();
                                calendar.add(Calendar.DAY_OF_MONTH, -1);
                                int dayE = month == monthE ? dayEnd : calendar.get(Calendar.DAY_OF_MONTH);
                                for (int day = dayS; day <= dayE; day++) {
                                    if (days.contains(day)) {
                                        String dayUrl = monthUrl + String.valueOf(day) + "/";
                                        Result dayResult = ResultParser.parse(NetUtils.getResponseAsString(dayUrl));
                                        if (dayResult.getCommonPrefixes() != null) {
                                            Set<Integer> sequences = dayResult.getCommonPrefixes().stream()
                                                    .map(p -> {
                                                        String tmp = p.replace(dayResult.getPrefix(), "");
                                                        return Integer.parseInt(tmp.substring(0, tmp.indexOf(dayResult.getDelimiter())));
                                                    }).collect(Collectors.toSet());
                                            for (int sequence : sequences) {
                                                String jsonTile = dayUrl + String.valueOf(sequence) + "/tileInfo.json";
                                                jsonTile = jsonTile.replace("?delimiter=/&prefix=", "");
                                                double clouds = getTileCloudPercentage(jsonTile);
                                                if (clouds > this.cloudFilter) {
                                                    Calendar instance = new Calendar.Builder().setDate(year, month - 1, day).build();
                                                    Logger.getRootLogger().warn(
                                                            String.format("Tile %s from %s has %.2f %% clouds",
                                                                    tile, dateFormat.format(instance.getTime()), clouds));
                                                } else {
                                                    String jsonProduct = dayUrl + String.valueOf(sequence) + "/productInfo.json";
                                                    jsonProduct = jsonProduct.replace("?delimiter=/&prefix=", "");
                                                    ProductDescriptor descriptor = parseProductJson(jsonProduct);
                                                    if (this.relativeOrbit == 0 ||
                                                            descriptor.getName().contains("_R" + String.format("%03d", this.relativeOrbit))) {
                                                        results.put(descriptor.getName(), descriptor);
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
        Logger.getRootLogger().info("Query returned %s products", results.size());
        return new ArrayList<>(results.values());
    }

    private ProductDescriptor parseProductJson(String jsonUrl) throws IOException, URISyntaxException {
        JsonReader reader = null;
        ProductDescriptor descriptor = null;
        try (InputStream inputStream = new URI(jsonUrl).toURL().openStream()) {
            reader = Json.createReader(inputStream);
            JsonObject obj = reader.readObject();
            descriptor = new S2L1CProductDescriptor();
            descriptor.setName(obj.getString("name"));
            descriptor.setId(obj.getString("id"));
        } finally {
            if (reader != null) {
                reader.close();
            }
        }
        return descriptor;
    }

    private double getTileCloudPercentage(String jsonUrl) throws IOException, URISyntaxException {
        JsonReader reader = null;
        try (InputStream inputStream = new URI(jsonUrl).toURL().openStream()) {
            reader = Json.createReader(inputStream);
            JsonObject obj = reader.readObject();
            return obj.getJsonNumber("cloudyPixelPercentage").doubleValue();
        } finally {
            if (reader != null) {
                reader.close();
            }
        }
    }
} public enum FillAnglesMethod {
    /*
     * Nothing to do
     */
    NONE,
    /*
     * Missing angle value replaced with NaN
     */
    NAN,
    /*
     * Missing angle value interpolated from existing values
     */
    INTERPOLATE
} public class ProductInspector {

    private Path root;
    private FillAnglesMethod method;
    private Set<String> products;

    public ProductInspector(String path, FillAnglesMethod fillAnglesMethod, Set<String> products) throws FileNotFoundException{
        this.root = Paths.get(path);
        if (!Files.exists(this.root)) {
            throw new FileNotFoundException(path);
        }
        this.products = products;
        this.method = fillAnglesMethod;
    }

    public void traverse() throws IOException {
        Files.walkFileTree(root, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE,
                new FileVisitor<Path>() {
                    @Override
                    public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
                        if (dir.equals(root)) {
                            return FileVisitResult.CONTINUE;
                        }
                        if (!acceptPath(dir)) {
                            return FileVisitResult.SKIP_SUBTREE;
                        }
                        String dirName = dir.getFileName().toString();
                        if ("AUX_DATA".equals(dirName) || "DATASTRIP".equals(dirName)) {
                            return FileVisitResult.SKIP_SUBTREE;
                        }
                        return FileVisitResult.CONTINUE;
                    }

                    @Override
                    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                        String fileName = file.getFileName().toString();
                        if (fileName.startsWith("S2A_OPER_MTD_L1C_TL_") && fileName.endsWith(".xml")) {
                            if (!Files.exists(file.resolveSibling(fileName.replace(".xml", ".orig")))) {
                                Path previous = file.resolveSibling(fileName + ".bkp");
                                List<String> originalLines;
                                if (Files.exists(previous)) {
                                    originalLines = MetadataRepairer.parse(previous, method);
                                    Files.move(Paths.get(previous.toAbsolutePath().toString() + ".bkp"), Paths.get(file.toAbsolutePath().toString().replace(".xml", ".orig")));
                                    Files.delete(previous);
                                    Files.move(file, previous);
                                } else {
                                    originalLines = MetadataRepairer.parse(file, method);
                                }
                                Files.write(file, originalLines, StandardCharsets.UTF_8);
                            } else {
                                Logger.getRootLogger().info(String.format("%s already processed", fileName));
                            }
                            return FileVisitResult.SKIP_SIBLINGS;
                        }
                        return FileVisitResult.CONTINUE;
                    }

                    @Override
                    public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
                        return FileVisitResult.CONTINUE;
                    }

                    @Override
                    public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
                        return FileVisitResult.CONTINUE;
                    }
                });
    }

    private boolean acceptPath(Path path) {
        boolean result = true;
        if (products != null) {
            String fullPath = path.toAbsolutePath().toString();
            boolean found = false;
            for (String product : products) {
                if (fullPath.contains(product)) {
                    found = true;
                    break;
                }
            }
            result = found;
        }
        return result;
    }
} public class PreOpsSciHubSearch extends AbstractSearch<ProductType> {

    public PreOpsSciHubSearch(String url, ProductType type) throws URISyntaxException {
        super(url);
        this.productType = type;
        this.filter = "platformName:Sentinel-2";
        if (this.productType != null) {
            this.filter = "(" + this.filter + " AND producttype:" + this.productType.toString() + ")";
        }
        this.params = new ArrayList<>();
    }

    public PreOpsSciHubSearch filter(String key, String value) {
        if (key != null && !key.isEmpty() && value != null && !value.isEmpty()) {
            this.filter += " AND " + key + ":" + value;
        }
        return this;
    }

    @Override
    public PreOpsSciHubSearch filter(List<ProductDescriptor> products) {
        if (products != null) {
            StringBuilder list = new StringBuilder("(");
            boolean more = products.size() > 1;
            for (ProductDescriptor product : products) {
                list.append(product.getName());
                if (more) {
                    list.append(" OR ");
                }
            }
            if (more) {
                list = new StringBuilder(list.substring(0, list.length() - 4) + ")");
            }
            this.filter += " AND " + list;
        }
        return this;
    }

    public PreOpsSciHubSearch limit(int number) {
        if (number > 0) {
            params.add(new BasicNameValuePair("rows", String.valueOf(number)));
        }
        return this;
    }

    public PreOpsSciHubSearch start(int start) {
        if (start >= 0) {
            params.add(new BasicNameValuePair("start",String.valueOf(start)));
        }
        return this;
    }

    @Override
    public void setProductType(ProductType type) {
        super.setProductType(type);
        if (this.productType != null) {
            int idx = this.filter.indexOf("productType");
            if (idx > 0) {
                int idx2 = this.filter.indexOf(")", idx);
                this.filter = this.filter.substring(0, idx)
                        + "productType:" + this.productType.toString() + this.filter.substring(idx2);
            } else {
                idx = this.filter.indexOf("platformName");
                int idx2 = this.filter.indexOf("2", idx);
                this.filter = this.filter.substring(0, idx) + "(platformName:Sentinel-2 AND productType:" +
                        this.productType.toString() + ")" + this.filter.substring(idx2 + 1);
            }
        }
    }

    private String getQuery() {
        Optional<NameValuePair> valuePair = params.stream().filter(p -> p.getName().equals("q")).findFirst();
        valuePair.ifPresent(nameValuePair -> params.remove(nameValuePair));
        valuePair = params.stream().filter(p -> p.getName().equals("start")).findFirst();
        String offset = "0";
        if (valuePair.isPresent()) {
            NameValuePair nameValuePair = valuePair.get();
            offset = nameValuePair.getValue();
            params.remove(nameValuePair);
        }
        valuePair = params.stream().filter(p -> p.getName().equals("rows")).findFirst();
        String limit = "10";
        if (valuePair.isPresent()) {
            NameValuePair nameValuePair = valuePair.get();
            limit = nameValuePair.getValue();
            params.remove(nameValuePair);
        }
        params.add(new BasicNameValuePair("filter", filter));
        params.add(new BasicNameValuePair("offset", offset));
        params.add(new BasicNameValuePair("limit", limit));
        return this.url.toString() + "?" + URLEncodedUtils.format(params, "UTF-8").replace("+", "%20");
    }

    protected List<ProductDescriptor> executeImpl() throws IOException {
        List<ProductDescriptor> results = new ArrayList<>();
        if (this.aoi.getNumPoints() > 0) {
            filter("footprint", "\"Intersects(" + (this.aoi.getNumPoints() < 200 ? this.aoi.toWKT() : this.aoi.toWKTBounds()) + ")\"");
        }
        String queryUrl = getQuery();
        Logger.getRootLogger().info(queryUrl);
        try (CloseableHttpResponse response = NetUtils.openConnection(queryUrl, credentials)) {
            switch (response.getStatusLine().getStatusCode()) {
                case 200:
                    ObjectMapper mapper = new ObjectMapper();
                    Product[] products = mapper.readValue(EntityUtils.toString(response.getEntity()), Product[].class);
                    results.addAll(Arrays.stream(products).filter(
                            p -> p.getIndexes().stream().anyMatch(
                                    i -> i.getChildren().stream().anyMatch(
                                            c -> "Cloud cover percentage".equals(c.getName()) &&
                                                    (cloudFilter == 0 || Double.parseDouble(c.getValue()) <= cloudFilter))))
                            .map(p-> ((this.productType == null) || ProductType.S2MSI1C.equals(this.productType)) ?
                                    new S2L1CProductDescriptor() {{
                                        setName(p.getIdentifier());
                                        setId(p.getUuid());
                                        setCloudsPercentage(
                                                p.getIndexes().stream().mapToDouble(
                                                        i -> Double.parseDouble(i.getChildren().stream().filter(
                                                                c -> "Cloud cover percentage".equals(c.getName())
                                                        ).findFirst().get().getValue())
                                                ).findFirst().getAsDouble()
                                        );
                                    }}
                                    : new S2L2AProductDescriptor() {
                            }).collect(Collectors.toList()));
                    break;
                case 401:
                    Logger.getRootLogger().info("The supplied credentials are invalid!");
                    break;
                default:
                    Logger.getRootLogger().info("The request was not successful. Reason: %s", response.getStatusLine().getReasonPhrase());
                    break;
            }
        }
        Logger.getRootLogger().info("Query returned %s Sentinel-2B products", results.size());
        return results;
    }

} public class SciHubSearch extends AbstractSearch<ProductType> {

    public SciHubSearch(String url, ProductType type) throws URISyntaxException {
        super(url);
        this.productType = type;
        this.filter = "platformName:Sentinel-2";
        if (this.productType != null) {
            this.filter = "(" + this.filter + " AND producttype:" + this.productType.toString() + ")";
        }
        this.params = new ArrayList<>();
    }

    public SciHubSearch filter(String key, String value) {
        if (key != null && !key.isEmpty() && value != null && !value.isEmpty()) {
            this.filter += " AND " + key + ":" + value;
        }
        return this;
    }

    @Override
    public SciHubSearch filter(List<ProductDescriptor> products) {
        if (products != null && products.size() > 0) {
            String list = "(";
            boolean more = products.size() > 1;
            for (ProductDescriptor product : products) {
                list += product.getName();
                if (more) {
                    list += " OR ";
                }
            }
            if (more) {
                list = list.substring(0, list.length() - 4);
            }
            list += ")";
            this.filter += " AND " + list;
        }
        return this;
    }

    @Override
    public SciHubSearch limit(int number) {
        if (number > 0) {
            Optional<NameValuePair> pair = params.stream().filter(p -> p.getName().equals("rows")).findFirst();
            pair.ifPresent(nameValuePair -> params.remove(nameValuePair));
            params.add(new BasicNameValuePair("rows", String.valueOf(number)));
            setPageSize(number);
        }
        return this;
    }

    @Override
    public SciHubSearch start(int start) {
        if (start >= 0) {
            Optional<NameValuePair> pair = params.stream().filter(p -> p.getName().equals("start")).findFirst();
            pair.ifPresent(nameValuePair -> params.remove(nameValuePair));
            params.add(new BasicNameValuePair("start",String.valueOf(start)));
            setOffset(start);
        }
        return this;
    }

    @Override
    public void setProductType(ProductType type) {
        super.setProductType(type);
        if (this.productType != null) {
            int idx = this.filter.indexOf("productType");
            if (idx > 0) {
                int idx2 = this.filter.indexOf(")", idx);
                this.filter = this.filter.substring(0, idx)
                        + "productType:" + this.productType.toString() + this.filter.substring(idx2);
            } else {
                idx = this.filter.indexOf("platformName");
                int idx2 = this.filter.indexOf("2", idx);
                this.filter = this.filter.substring(0, idx) + "(platformName:Sentinel-2 AND productType:" +
                        this.productType.toString() + ")" + this.filter.substring(idx2 + 1);
            }
        }
    }

    private String getQuery() {
        Optional<NameValuePair> pair = params.stream().filter(p -> p.getName().equals("q")).findFirst();
        pair.ifPresent(nameValuePair -> params.remove(nameValuePair));
        params.add(new BasicNameValuePair("q", filter));
        return this.url.toString() + "?" + URLEncodedUtils.format(params, "UTF-8").replace("+", "%20");
    }

    protected List<ProductDescriptor> executeImpl() throws IOException {
        List<ProductDescriptor> results = new ArrayList<>();
        if (this.aoi.getNumPoints() > 0) {
            if (!this.filter.contains("footprint")) {
                filter("footprint", "\"Intersects(" + (this.aoi.getNumPoints() < 200 ? this.aoi.toWKT() : this.aoi.toWKTBounds()) + ")\"");
            }
        }
        String queryUrl = getQuery();
        Logger.getRootLogger().info(queryUrl);
        try (CloseableHttpResponse response = NetUtils.openConnection(queryUrl, credentials)) {
            switch (response.getStatusLine().getStatusCode()) {
                case 200:
                    String[] strings = EntityUtils.toString(response.getEntity()).split("\n");
                    ProductDescriptor currentProduct = null;
                    double currentClouds;
                    for (String string : strings) {
                        if (string.contains("<entry>")) {
                            currentProduct = (this.productType == null || ProductType.S2MSI1C.equals(this.productType)) ?
                                    new S2L1CProductDescriptor() : new S2L2AProductDescriptor();
                        } else if (string.contains("</entry>")) {
                            if (currentProduct != null) {
                                double cloudsPercentage = currentProduct.getCloudsPercentage();
                                if (cloudFilter == 0 || cloudsPercentage <= cloudFilter) {
                                    results.add(currentProduct);
                                } else {
                                    Logger.getRootLogger().debug("%s skipped [clouds: %s]", currentProduct, cloudsPercentage);
                                }
                            }
                        } else if (string.contains("<title>")) {
                            if (currentProduct != null) {
                                currentProduct.setName(string.replace("<title>", "").replace("</title>", ""));
                            }
                        } else if (string.contains("cloudcoverpercentage")) {
                            currentClouds = Double.parseDouble(string.replace("<double name=\"cloudcoverpercentage\">", "").replace("</double>", ""));
                            if (currentProduct != null) {
                                currentProduct.setCloudsPercentage(currentClouds);
                            }
                        } else if (string.contains("<id>")) {
                            if (currentProduct != null) {
                                currentProduct.setId(string.replace("<id>", "").replace("</id>", ""));
                            }
                        }
                    }
                    break;
                case 401:
                    Logger.getRootLogger().info("The supplied credentials are invalid!");
                    break;
                default:
                    Logger.getRootLogger().info("The request was not successful. Reason: %s", response.getStatusLine().getReasonPhrase());
                    break;
            }
        }
        Logger.getRootLogger().info("Query returned %s Sentinel-2A products", results.size());
        return results;
    }

} public class Constants {
    public static final String PARAM_AREA = "a";
    public static final String PARAM_TILE_LIST = "t";
    public static final String PARAM_PRODUCT_LIST = "p";
    public static final String PARAM_BAND_LIST = "b";
    public static final String PARAM_PRODUCT_UUID_LIST = "uuid";
    public static final String PARAM_OUT_FOLDER = "o";
    public static final String PARAM_INPUT_FOLDER = "i";
    public static final String PARAM_AREA_FILE = "af";
    public static final String PARAM_TILE_SHAPE_FILE = "ts";
    public static final String PARAM_TILE_LIST_FILE = "tf";
    public static final String PARAM_PRODUCT_LIST_FILE = "pf";
    public static final String PARAM_CLOUD_PERCENTAGE = "cp";
    public static final String PARAM_START_DATE = "start";
    public static final String PARAM_END_DATE = "end";
    public static final String PARAM_RESULTS_LIMIT = "l";
    public static final String PARAM_DOWNLOAD_MODE = "m";
    public static final String PARAM_FLAG_COMPRESS = "z";
    public static final String PARAM_FLAG_DELETE = "d";
    public static final String PARAM_FLAG_PREOPS = "pre";
    public static final String PARAM_DOWNLOAD_STORE = "s";
    public static final String PARAM_FLAG_UNPACKED = "u";
    public static final String PARAM_FLAG_SEARCH_AWS = "aws";
    public static final String PARAM_USER = "user";
    public static final String PARAM_PASSWORD = "pwd";
    public static final String PARAM_SENSOR = "sen";
    public static final String PARAM_L8_COLLECTION = "l8col";
    public static final String PARAM_S2_PRODUCT_TYPE = "s2t";
    public static final String PARAM_L8_PRODUCT_TYPE = "l8t";
    public static final double DEFAULT_CLOUD_PERCENTAGE = 100.0;
    public static final String DEFAULT_START_DATE = "NOW-7DAY";
    public static final String PATTERN_START_DATE = "NOW-%sDAY";
    public static final String DEFAULT_END_DATE = "NOW";
    public static final int DEFAULT_RESULTS_LIMIT = 10;
    public static final String PROPERTY_NAME_SEARCH_URL = "scihub.search.url";
    public static final String PROPERTY_NAME_SEARCH_PREOPS_URL = "preops.scihub.search.url";
    public static final String PROPERTY_NAME_SCIHUB_PRODUCTS_URL = "scihub.product.url";
    public static final String PROPERTY_NAME_SCIHUB_PREOPS_PRODUCTS_URL = "preops.scihub.product.url";
    public static final String PROPERTY_NAME_AWS_SEARCH_URL = "s2.aws.search.url";
    public static final String PROPERTY_NAME_LANDSAT_SEARCH_URL = "l8.aws.pre.search.url";
    public static final String PROPERTY_NAME_LANDSAT_AWS_SEARCH_URL = "l8.aws.search.url";
    public static final String PROPERTY_NAME_DEFAULT_LANDSAT_SEARCH_URL = "https://landsat-pds.s3.amazonaws.com/?delimiter=/&prefix=c1/L8/";
    public static final String PROPERTY_DEFAULT_SEARCH_URL = "https://scihub.copernicus.eu/apihub/search";
    public static final String PROPERTY_DEFAULT_SEARCH_PREOPS_URL = "https://scihub.copernicus.eu/s2b/api/stub/products";
    public static final String PROPERTY_DEFAULT_AWS_SEARCH_URL = "http://sentinel-products-l1c.s3.amazonaws.com/?delimiter=/&prefix=tiles/";
    public static final String PROPERTY_NAME_SEARCH_URL_SECONDARY = "scihub.search.backup.url";
    public static final String PROPERTY_DEFAULT_SEARCH_URL_SECONDARY = "https://scihub.copernicus.eu/dhus/search";
    public static final String SEARCH_PARAM_INTERVAL = "beginPosition";
    public static final String PARAM_RELATIVE_ORBIT = "ro";

    public static final String LEVEL_1 = "      ";
    public static final String LEVEL_2 = "        ";
    public static final String LEVEL_3 = "          ";
    public static final String LEVEL_4 = "            ";
    public static final String PARAM_FILL_ANGLES = "ma";
    public static final String PARAM_VERBOSE = "v";
    public static final String PARAM_SEARCH_ONLY = "q";
    public static final String SEARCH_PARAM_RELATIVE_ORBIT_NUMBER = "relativeOrbitNumber";
    public static final String PARAM_PROXY_TYPE = "ptype";
    public static final String PARAM_PROXY_HOST = "phost";
    public static final String PARAM_PROXY_PORT = "pport";
    public static final String PARAM_PROXY_USER = "puser";
    public static final String PARAM_PROXY_PASSWORD = "ppwd";
    public static final String PARAM_GUI = "gui";
    public static final String PSD_13 = "13";
    public static final String PSD_14 = "14";
    public static final String L8_PRECOLL = "preColl";
    public static final String L8_COLL = "coll";
    public static final String L8_SEARCH_URL_SUFFIX = "?delimiter=/&prefix=";
    public static final String XML_ATTR_GRANULE_ID = "granuleIdentifier";
    public static final String XML_ATTR_DATASTRIP_ID = "datastripIdentifier";
    public static final String ODATA_UUID = "${UUID}";
    public static final String ODATA_PRODUCT_NAME = "${PRODUCT_NAME}";
    public static final String ODATA_XML_PLACEHOLDER = "${xmlname}";
    public static final String FOLDER_GRANULE = "GRANULE";
    public static final String FOLDER_AUXDATA = "AUX_DATA";
    public static final String FOLDER_DATASTRIP = "DATASTRIP";
    public static final String FOLDER_IMG_DATA = "IMG_DATA";
    public static final String FOLDER_QI_DATA = "QI_DATA";
    public static final String PROPERTY_DEFAULT_SCIHUB_PRODUCTS_URL = "https://scihub.copernicus.eu/apihub/odata/v1";
    public static final String PROPERTY_NAME_SCIHUB_BACKUP_SEARCH_URL = "scihub.product.backup.url";
    public static final String PROPERTY_DEFAULT_SCIHUB_BACKUP_SEARCH_URL = "https://scihub.copernicus.eu/dhus/odata/v1";

    public static final String PROPERTY_NAME_AWS_TILES_URL = "s2.aws.tiles.url";
    public static final String PROPERTY_DEFAULT_AWS_TILES_URL = "http://sentinel-products-l1c.s3-website.eu-central-1.amazonaws.com";
    public static final String PROPERTY_DEFAULT_SCIHUB_PREOPS_PRODUCTS_URL = "https://scihub.copernicus.eu/s2b/odata/v1";
} public class Logger {

    public interface CustomLogger {
        void debug(String message, Object...args);
        void info(String message, Object...args);
        void warn(String message, Object...args);
        void error(String message, Object...args);
    }

    private static final java.util.logging.Logger logger;
    private static String rootLogFile;
    private static CustomLogger rootLogger = new CustomLogger() {
        @Override
        public void debug(String message, Object... args) { fine(message, args); }

        @Override
        public void info(String message, Object... args) {
            information(message, args);
        }

        @Override
        public void warn(String message, Object... args) {
            warning(message, args);
        }

        @Override
        public void error(String message, Object... args) {
            err(message, args);
        }
    };

    static {
        logger = java.util.logging.Logger.getLogger(java.util.logging.Logger.GLOBAL_LOGGER_NAME);
        java.util.logging.Logger rootLogger = java.util.logging.Logger.getLogger("");
        Handler[] handlers = rootLogger.getHandlers();
        if (handlers[0] instanceof ConsoleHandler) {
            handlers[0].setFormatter(new LogFormatter());
        }
        logger.setLevel(Level.ALL);
    }

    public static void initialize(String masterLogFile, boolean verbose) throws IOException {
        synchronized (logger) {
            if (rootLogFile == null) {
                rootLogFile = masterLogFile;
                registerHandler(rootLogFile, verbose);
            }
        }
    }

    public static CustomLogger getRootLogger() {
        return rootLogger;
    }

    private static Handler registerHandler(String logFile, boolean verbose) throws IOException {
        Handler fileHandler = new FileHandler(logFile, true);
        fileHandler.setFormatter(new LogFormatter());
        logger.addHandler(fileHandler);
        Level level = verbose ? Level.ALL : Level.INFO;
        logger.setLevel(level);
        Arrays.stream(java.util.logging.Logger.getLogger("").getHandlers())
                .forEach(h -> h.setLevel(level));
        return fileHandler;
    }

    public static void registerHandler(Handler handler) {
        handler.setFormatter(new LogFormatter());
        handler.setLevel(logger.getLevel());
        logger.addHandler(handler);
    }

    private static void fine(String message, Object...args) {
        if (args != null && args.length > 0) {
            message = String.format(message, args);
        }
        logger.fine(message);
    }

    private static void information(String message, Object...args) {
        if (args != null && args.length > 0) {
            message = String.format(message, args);
        }
        logger.info(message);
    }

    private static void warning(String message, Object...args) {
        if (args != null && args.length > 0) {
            message = String.format(message, args);
        }
        logger.warning(message);
    }

    private static void err(String message, Object...args) {
        if (args != null && args.length > 0) {
            message = String.format(message, args);
        }
        logger.severe(message);
    }

    public static class ScopeLogger implements CustomLogger, AutoCloseable {

        private Handler fileHandler;

        public ScopeLogger(String logFile) throws IOException {
            //fileHandler = registerHandler(logFile);
            fileHandler = new FileHandler(logFile);
            fileHandler.setFormatter(new LogFormatter());
        }

        @Override
        public void debug(String message, Object...args) {
            if (args != null && args.length > 0) {
                message = String.format(message, args);
            }
            fileHandler.publish(new LogRecord(Level.FINE, message));
        }

        @Override
        public void info(String message, Object...args) {
            if (args != null && args.length > 0) {
                message = String.format(message, args);
            }
            fileHandler.publish(new LogRecord(Level.INFO, message));
        }

        @Override
        public void warn(String message, Object...args) {
            if (args != null && args.length > 0) {
                message = String.format(message, args);
            }
            fileHandler.publish(new LogRecord(Level.WARNING, message));
        }

        @Override
        public void error(String message, Object...args) {
            if (args != null && args.length > 0) {
                message = String.format(message, args);
            }
            fileHandler.publish(new LogRecord(Level.SEVERE, message));
        }

        @Override
        public void close() throws IOException {
            if (fileHandler != null) {
                fileHandler.close();
                logger.removeHandler(fileHandler);
            }
        }
    }

} public class NetUtils {

    private String authToken;
    private static Proxy javaNetProxy;
    private static HttpHost apacheHttpProxy;
    private static CredentialsProvider proxyCredentials;
    private static int timeout = 30000;

    public void setAuthToken(String value) {
        authToken = value;
    }

    public String getAuthToken() {
        return authToken;
    }

    public static void setProxy(String type, final String host, final int port, final String user, final String pwd) {
        if (type != null && host != null) {
            Proxy.Type proxyType = Enum.valueOf(Proxy.Type.class, type.toUpperCase());
            javaNetProxy = new Proxy(proxyType, new InetSocketAddress(host, port));
            Authenticator.setDefault(new Authenticator() {
                @Override
                protected PasswordAuthentication getPasswordAuthentication() {
                    return new PasswordAuthentication(user, pwd.toCharArray());
                }
            });
            if (user != null && pwd != null) {
                proxyCredentials = new BasicCredentialsProvider();
                proxyCredentials.setCredentials(new AuthScope(host, port), new UsernamePasswordCredentials(user, pwd));
            }
            apacheHttpProxy = new HttpHost(host, port, proxyType.name());
        }
    }

    public static void setTimeout(int newTimeout) {
        timeout = newTimeout;
    }

    public boolean isAvailable(String url) {
        boolean status;
        try {
            Logger.getRootLogger().debug("Verifying url: %s", url);
            HttpURLConnection connection = openConnection(url, authToken);
            connection.setRequestMethod("GET");
            connection.connect();
            final int responseCode = connection.getResponseCode();
            status = (200 == responseCode || 400 == responseCode || 401 == responseCode);
            Logger.getRootLogger().debug("Url status: %s [code %s]", url, responseCode);
        } catch (Exception e) {
            Logger.getRootLogger().debug("Verification failed: %s", e.getMessage());
            status = false;
        }
        return status;
    }

    public static HttpURLConnection openConnection(String url) {
        return openConnection(url, (String) null);
    }

    public static HttpURLConnection openConnection(String url, String authToken) {
        HttpURLConnection connection = null;
        try {
            URL urlObj = new URL(url);
            if (javaNetProxy == null) {
                connection = (HttpURLConnection) urlObj.openConnection();
                Logger.getRootLogger().debug("Proxyless connection to %s opened", url);
            } else {
                connection = (HttpURLConnection) urlObj.openConnection(javaNetProxy);
                Logger.getRootLogger().debug("Proxy connection to %s opened", url);
            }
            if (authToken != null) {
                connection.setRequestProperty("Authorization", authToken);
            }
            connection.setConnectTimeout(timeout);
            connection.setReadTimeout(timeout);
        } catch (IOException e) {
            Logger.getRootLogger().debug("Could not open connection to %s [%s]", url, e.getMessage());
        }
        if (connection != null) {
            StringBuilder builder = new StringBuilder();
            Map<String, List<String>> requestProperties = connection.getRequestProperties();
            for (Map.Entry<String, List<String>> entry : requestProperties.entrySet()) {
                builder.append(entry.getKey()).append("=");
                for (String value : entry.getValue()) {
                    builder.append(value).append(",");
                }
                builder.append(";");
            }
            if (builder.length() > 0) {
                Logger.getRootLogger().debug("Request details: %s", builder.toString());
            }
        }
        return connection;
    }

    public static CloseableHttpResponse openConnection(String url, Credentials credentials) {
        CloseableHttpResponse response = null;
        try {
            CloseableHttpClient httpClient;
            URI uri = new URI(url);
            CredentialsProvider credentialsProvider = null;
            if (credentials != null) {
                credentialsProvider = proxyCredentials != null ? proxyCredentials : new BasicCredentialsProvider();
                credentialsProvider.setCredentials(new AuthScope(uri.getHost(), uri.getPort()), credentials);
            }
            if (credentialsProvider != null) {
                httpClient = HttpClients.custom()
                                        .setDefaultCredentialsProvider(credentialsProvider)
                                        .build();
            } else {
                httpClient = HttpClients.custom().build();
            }
            HttpGet get = new HttpGet(uri);
            if (apacheHttpProxy != null) {
                RequestConfig config = RequestConfig.custom().setProxy(apacheHttpProxy).build();
                get.setConfig(config);
            }
            //Logger.getRootLogger().debug("HTTP GET %s", url);
            RequestConfig config = get.getConfig();
            if (config != null) {
                Logger.getRootLogger().debug("Details: %s", config.toString());
            }
            response = httpClient.execute(get);
            Logger.getRootLogger().debug("HTTP GET %s returned %s", url, response.getStatusLine().getStatusCode());
        } catch (URISyntaxException | IOException e) {
            Logger.getRootLogger().debug("Could not create connection to %s : %s", url, e.getMessage());
        }
        return response;
    }

    public static String getResponseAsString(String url) throws IOException {
        String result = null;
        try (CloseableHttpResponse yearResponse = NetUtils.openConnection(url, (Credentials) null)) {
            switch (yearResponse.getStatusLine().getStatusCode()) {
                case 200:
                    result = EntityUtils.toString(yearResponse.getEntity());
                    break;
                case 401:
                    Logger.getRootLogger().info("The supplied credentials are invalid!");
                    break;
                default:
                    Logger.getRootLogger().info("The request was not successful. Reason: %s", yearResponse.getStatusLine().getReasonPhrase());
                    break;
            }
        }
        return result;
    }
} public class Polygon2D {
    private static final Pattern polyPattern = Pattern.compile("POLYGON\\(\\(.*\\)\\)");
    private static final Pattern coordPattern = Pattern.compile("((?:-?(?:\\d+\\.\\d+)) (?:-?(?:\\d+\\.\\d+)))");

    private Path2D.Double polygon;
    private int numPoints;

    /**
     * Creates a polygon from a well-known text.
     * For now, only single POLYGONs are supported.
     * If multiple polygons are encountered, only the first one is taken into account.
     *
     * @param wkt   The text to parse.
     * @return      A closed polygon.
     */
    public static Polygon2D fromWKT(String wkt) {
        Polygon2D polygon = new Polygon2D();
        Matcher matcher = polyPattern.matcher(wkt);
        if (matcher.matches()) {
            String polyText = matcher.toMatchResult().group(0);
            if (polyText != null) {
                Matcher coordMatcher = coordPattern.matcher(polyText);
                while (coordMatcher.find()) {
                    //for (int i = 0; i < coordMatcher.groupCount(); i++) {
                        String[] coords = coordMatcher.group().split(" ");
                        polygon.append(Double.parseDouble(coords[0]), Double.parseDouble(coords[1]));
                    //}
                }
            }
        } else {
            // maybe we have only a list of coordinates, without being wrapped in a POLYGON((..))
            Matcher coordMatcher = coordPattern.matcher(wkt);
            while (coordMatcher.find()) {
                //for (int i = 0; i < coordMatcher.groupCount(); i++) {
                String[] coords = coordMatcher.group().split(" ");
                polygon.append(Double.parseDouble(coords[0]), Double.parseDouble(coords[1]));
                //}
            }
        }
        return polygon;
    }

    public Polygon2D() {
    }

    /**
     * Adds a point to the current polygon.
     * If this is not the first point, then it also adds a line between the previous point and the new one.
     *
     * @param x     The x coordinate
     * @param y     The y coordinate
     */
    public void append(double x, double y) {
        if (polygon == null) {
            polygon = new Path2D.Double();
            polygon.moveTo(x, y);
        } else {
            polygon.lineTo(x, y);
        }
        numPoints++;
    }

    /**
     * Adds a list of points to the current polygon.
     * The list items are pairs of coordinates.
     *
     * @param points    The points to be added
     */
    public void append(List<double[]> points) {
        if (points != null) {
            for (double[] pair : points) {
                if (pair != null && pair.length == 2) {
                    append(pair[0], pair[1]);
                }
            }
        }
    }

    /**
     * Returns the number of points of the current polygon.
     *
     */
    public int getNumPoints() {
        return numPoints;
    }

    /**
     * Produces a WKT representation of this polygon.
     */
    public String toWKT() {
        StringBuilder buffer = new StringBuilder();
        buffer.append("POLYGON((");
        PathIterator pathIterator = polygon.getPathIterator(null);
        while (!pathIterator.isDone()) {
            double[] segment = new double[6];
            pathIterator.currentSegment(segment);
            buffer.append(String.valueOf(segment[0])).append(" ").append(String.valueOf(segment[1])).append(",");
            pathIterator.next();
        }
        buffer.setLength(buffer.length() - 1);
        buffer.append("))");
        return buffer.toString();
    }

    public Rectangle2D getBounds2D() {
        return polygon.getBounds2D();
    }

    public String toWKTBounds() {
        Rectangle2D bounds2D = polygon.getBounds2D();
        return  "POLYGON((" +
                bounds2D.getMinX() + " " + bounds2D.getMinY() + "," +
                bounds2D.getMaxX() + " " + bounds2D.getMinY() + "," +
                bounds2D.getMaxX() + " " + bounds2D.getMaxY() + "," +
                bounds2D.getMinX() + " " + bounds2D.getMaxY() + "," +
                bounds2D.getMinX() + " " + bounds2D.getMinY() + "))";
    }

} public class ReturnCode {
    public static final int OK = 0;
    public static final int EMPTY_PRODUCT = -1;
    public static final int DOWNLOAD_ERROR = -2;
    public static final int CONNECTION_ERROR = -3;
    public static final int ERROR = -4;
} public class Utilities {

    private static Boolean supportsPosix;

    public static String join(Iterable collection, String separator) {
        String result = "";
        if (collection != null) {
            boolean hasElements = false;
            for (Object aCollection : collection) {
                hasElements = true;
                result += (aCollection != null ? aCollection.toString() : "null") + separator;
            }
            if (hasElements) {
                result = result.substring(0, result.length() - separator.length());
            }
        }
        return result;
    }

    public static List<String> filter(List<String> input, String filter) {
        List<String> result = new ArrayList<>();
        if (input != null) {
            for (String item : input) {
                if (item.contains(filter)) {
                    result.add(item);
                }
            }
        }
        return result;
    }

    public static String find(List<String> input, String filter, String psdVersion) {
        String value = null;
        String granuleIdentifier;
        switch (psdVersion) {
            case Constants.PSD_13 :
                for (String line : input) {
                    granuleIdentifier = getAttributeValue(line, "granuleIdentifier");
                    if (granuleIdentifier.contains(filter)) {
                        value = granuleIdentifier;
                        break;
                    }
                }
                break;
            case Constants.PSD_14:
                String datastripIdentifier;
                for (String line : input) {
                    granuleIdentifier = getAttributeValue(line, "granuleIdentifier");
                    if (granuleIdentifier.contains(filter)) {
                        datastripIdentifier = getAttributeValue(line, "datastripIdentifier");
                        value = granuleIdentifier.substring(13, 16) + "_" +
                                granuleIdentifier.substring(49, 55) + "_" +
                                granuleIdentifier.substring(41, 48) + "_" +
                                datastripIdentifier.substring(42, 57);
                        break;
                    }
                }
                break;
        }
        return value;
    }

    public static String formatTime(long millis) {
        return String.format("%02dh:%02dm:%02ds",
                TimeUnit.MILLISECONDS.toHours(millis),
                TimeUnit.MILLISECONDS.toMinutes(millis) % TimeUnit.HOURS.toMinutes(1),
                TimeUnit.MILLISECONDS.toSeconds(millis) % TimeUnit.MINUTES.toSeconds(1));
    }

    public static String getAttributeValue(String xmlLine, String name) {
        String value = null;
        int idx = xmlLine.indexOf(name);
        if (idx > 0) {
            int start = idx + name.length() + 2;
            value = xmlLine.substring(start, xmlLine.indexOf("\"", start));
        }
        return value;
    }

    public static Path ensureExists(Path folder) throws IOException {
        if (folder != null && !Files.exists(folder)) {
            if (isPosixFileSystem()) {
                Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwxr-xr-x");
                FileAttribute<Set<PosixFilePermission>> attrs = PosixFilePermissions.asFileAttribute(perms);
                folder = Files.createDirectories(folder, attrs);
            } else {
                folder = Files.createDirectories(folder);
            }

        }
        return folder;
    }

    public static Path ensurePermissions(Path file) {
        try {
            if (file != null && Files.exists(file)) {
                if (isPosixFileSystem()) {
                    Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwxr-xr-x");
                    file = Files.setPosixFilePermissions(file, perms);
                }
            }
        } catch (IOException ex) {
            Logger.getRootLogger().warn("Cannot set permissions for %s", file.toString());
        }
        return file;
    }

    public static List<Path> listFolders(Path root) throws IOException {
        final List<Path> folders = new ArrayList<>();
        Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
                folders.add(dir);
                return FileVisitResult.CONTINUE;
            }
        });
        return folders;
    }

    public static List<Path> listFiles(Path folder) throws IOException {
        final File[] files = folder.toFile().listFiles();
        return files != null ?
                Arrays.stream(files)
                      .filter(File::isFile)
                      .map(File::toPath)
                      .collect(Collectors.toList()) :
                null;
    }

    private static boolean isPosixFileSystem() {
        if (supportsPosix == null) {
            supportsPosix = Boolean.FALSE;
            FileSystem fileSystem = FileSystems.getDefault();
            Iterable<FileStore> fileStores = fileSystem.getFileStores();
            for (FileStore fs : fileStores) {
                supportsPosix = fs.supportsFileAttributeView(PosixFileAttributeView.class);
                if (supportsPosix) {
                    break;
                }
            }
        }
        return supportsPosix;
    }
}
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.MissingOptionException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import ro.cs.products.base.AbstractSearch;
import ro.cs.products.base.DownloadMode;
import ro.cs.products.base.ProductDescriptor;
import ro.cs.products.base.SensorType;
import ro.cs.products.base.TileMap;
import ro.cs.products.landsat.CollectionCategory;
import ro.cs.products.landsat.LandsatAWSSearch;
import ro.cs.products.landsat.LandsatCollection;
import ro.cs.products.landsat.LandsatProductDescriptor;
import ro.cs.products.landsat.LandsatProductDownloader;
import ro.cs.products.landsat.LandsatTilesMap;
import ro.cs.products.sentinel2.ProductStore;
import ro.cs.products.sentinel2.ProductType;
import ro.cs.products.sentinel2.S2L1CProductDescriptor;
import ro.cs.products.sentinel2.Sentinel2PreOpsDownloader;
import ro.cs.products.sentinel2.SentinelProductDownloader;
import ro.cs.products.sentinel2.SentinelTilesMap;
import ro.cs.products.sentinel2.amazon.AmazonSearch;
import ro.cs.products.sentinel2.angles.FillAnglesMethod;
import ro.cs.products.sentinel2.angles.ProductInspector;
import ro.cs.products.sentinel2.scihub.PreOpsSciHubSearch;
import ro.cs.products.sentinel2.scihub.SciHubSearch;
import ro.cs.products.util.Constants;
import ro.cs.products.util.Logger;
import ro.cs.products.util.NetUtils;
import ro.cs.products.util.Polygon2D;
import ro.cs.products.util.ReturnCode;
import ro.cs.products.util.Utilities;
import java.awt.geom.Rectangle2D;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Calendar;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package ro.cs.products;



/**
 * Executor execution class.
 *
 * @author Cosmin Cara
 */
public class Executor {

    private static Options options;
    private static Properties props;
    private static String version;
    private static BatchProgressListener batchProgressListener;
    private static ProgressListener fileProgressListener;

    static {
        options = new Options();
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(Executor.class.getResourceAsStream("parameters.def")))) {
            HashMap<String, List<String[]>> groups = new HashMap<>();
            reader.lines()
                    .filter(l -> {
                          String trimmed = l.trim();
                          return !trimmed.startsWith("#") &&
                                  !(trimmed.startsWith("\r") || trimmed.startsWith("\n"));
                    })
                    .forEach(s -> {
                        String[] tokens = s.split(";");
                        if (tokens.length == 8) {
                            String key = tokens[0].trim();
                            if (!groups.containsKey(key)) {
                                groups.put(key, new ArrayList<>());
                            }
                            groups.get(key).add(tokens);
                        }
                    });
            for (String key : groups.keySet()) {
                if ("n/a".equals(key)) {
                    groups.get(key).forEach(values -> options.addOption(buildOption(values)));
                } else {
                    OptionGroup group = new OptionGroup();
                    groups.get(key).forEach(values -> group.addOption(buildOption(values)));
                    options.addOptionGroup(group);
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
            System.exit(Integer.MAX_VALUE);
        }
        props = new Properties();
        try {
            props.load(Executor.class.getResourceAsStream("download.properties"));
            version = props.getProperty("version");
        } catch (IOException ignored) {
        }
    }

    public static void main(String[] args) throws Exception {
        if (args.length == 0) {
            HelpFormatter formatter = new HelpFormatter();
            formatter.printHelp("ProductDownload-" + version, options);
            System.exit(0);
        }
        CommandLineParser parser = new DefaultParser();
        CommandLine commandLine = parser.parse(options, args);
        System.exit(execute(commandLine));
    }

    public static int execute(String[] args) throws Exception {
        CommandLineParser parser = new DefaultParser();
        CommandLine commandLine = parser.parse(options, args);
        return execute(commandLine);
    }

    private static Option buildOption(String[] values) {
        Option.Builder optionBuilder = Option.builder(values[1].trim())
                .longOpt(values[2].trim())
                .argName(values[4].trim())
                .desc(values[7].trim())
                .required(!Boolean.parseBoolean(values[6].trim()));
        String cardinality = values[3].trim();
        switch (cardinality) {
            case "1":
                optionBuilder.hasArg();
                break;
            case "n":
                optionBuilder.hasArgs().valueSeparator(values[5].trim().charAt(1));
                break;
            default:
                break;
        }
        return optionBuilder.build();
    }

    private static int execute(CommandLine commandLine) throws Exception {
        int retCode = ReturnCode.OK;
        String logFile = props.getProperty("master.log.file");
        String folder;
        boolean debugMode = getArgValue(commandLine, Constants.PARAM_VERBOSE, Boolean.class, false);
        boolean searchMode = getArgValue(commandLine, Constants.PARAM_SEARCH_ONLY, Boolean.class, false);
        Logger.CustomLogger logger;
        SensorType sensorType = getArgValue(commandLine, Constants.PARAM_SENSOR, SensorType.class, SensorType.S2);