// Path: halvade_upload_tool/src/be/ugent/intec/halvade/uploader/input/FileReaderFactory.java
// public class FileReaderFactory extends BaseFileReader implements Runnable {
//     protected ArrayList<BaseFileReader> readers;
//     protected BaseFileReader currentReader = null;
//     protected static FileReaderFactory allReaders = null;
//     protected int threads;
//     protected boolean fromHDFS = false;
// 
//     public FileReaderFactory(int threads, boolean fromHDFS) {
//         super(false);
//         readers = new ArrayList<>();
//         this.threads = threads;
//         this.fromHDFS = fromHDFS;
//     }
//     
//     public static FileReaderFactory getInstance(int threads, boolean fromHDFS) {
//         if(allReaders == null) {
//             allReaders = new FileReaderFactory(threads, fromHDFS);
//         }
//         return allReaders;
//     }
//     
//     public static BaseFileReader createFileReader(boolean fromHDFS, String fileA, String fileB, boolean interleaved) throws IOException {
//         if (fileB == null) {
//             return new SingleFastQReader(fromHDFS, fileA, interleaved);
//         } else {
//             return new PairedFastQReader(fromHDFS, fileA, fileB);
//         }     
//     }
//     
//     public void addReader(String fileA, String fileB, boolean interleaved) throws IOException {
//         readers.add(createFileReader(fromHDFS, fileA, fileB, interleaved));
//     }
//     
//     public void addReader(BaseFileReader reader) {
//         readers.add(reader);
//     }
// 
//     
//     public ReadBlock retrieveBlock() {
//         try {
//             ReadBlock block = null;
//             while((check || blocks.size() > 0) && block == null) {
//                 block = blocks.poll(1000, TimeUnit.MILLISECONDS);
//             }
//             return block;
//         } catch (InterruptedException ex) {
//             Logger.EXCEPTION(ex);
//             return null;
//         }
//     }
//     
//     
//     @Override
//     protected int addNextRead(ReadBlock block) throws IOException {
//         return currentReader.addNextRead(block);
//     }
//     
//     protected synchronized boolean getNextReader() {
//         if(currentReader == null) {
//             if(readers.size() > 0) {
//                 currentReader = readers.remove(readers.size() - 1);
//                 Logger.DEBUG("Reader: " + currentReader);
//                 return true;
//             } else {
//                 Logger.DEBUG("Processed all readers");
//                 return false;
//             }
//         } else return true;
//     }
// 
// 
//     protected boolean check = true;
//     protected ArrayBlockingQueue<ReadBlock> blocks;
//     protected int READ_BLOCK_CAPACITY_PER_THREAD = 10;
//     
//     @Override
//     public void run() {
//         blocks = new ArrayBlockingQueue<>(READ_BLOCK_CAPACITY_PER_THREAD*threads);
//         if(currentReader == null) {
//             if(!getNextReader()) check = false;
//         }
//         try {
//             while(check) {
//                 ReadBlock block = new ReadBlock();
//                 boolean hasReads = super.getNextBlock(block);
//                 if (!hasReads) {
//                     currentReader = null;
//                     if(!getNextReader())
//                         check = false;
//                 } else {
//                     blocks.put(block);
//                 }
//             }
//         } catch (InterruptedException ex) {
//             Logger.EXCEPTION(ex);
//         }
//     }
// }


import be.ugent.intec.halvade.uploader.input.FileReaderFactory;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.commons.cli.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
            formatter.printHelp( "hadoop jar HalvadeUploaderWithLibs.jar -1 <MANIFEST> -O <OUT> [options]", options );
        } catch (Throwable ex) {
            Logger.THROWABLE(ex);
        }
        return 0;
    }
    
    private int processFiles() throws IOException, InterruptedException, URISyntaxException, Throwable {    
        Timer timer = new Timer();
        timer.start();
        
        AWSUploader upl = null;
        FileSystem fs = null;
        // write to s3?
        boolean useAWS = false;
        if(outputDir.startsWith("s3")) {
            useAWS = true;
            String existingBucketName = outputDir.replace("s3://","").split("/")[0];
            outputDir = outputDir.replace("s3://" + existingBucketName + "/", "");
            upl = new AWSUploader(existingBucketName, SSE, profile);
        } else {
            Configuration conf = getConf();
            fs = FileSystem.get(new URI(outputDir), conf);
            Path outpath = new Path(outputDir);
            if (fs.exists(outpath) && !fs.getFileStatus(outpath).isDirectory()) {
                Logger.DEBUG("please provide an output directory");
                return 1;
            }
        }
        
// Path: halvade_upload_tool/src/be/ugent/intec/halvade/uploader/input/FileReaderFactory.java
// public class FileReaderFactory extends BaseFileReader implements Runnable {
//     protected ArrayList<BaseFileReader> readers;
//     protected BaseFileReader currentReader = null;
//     protected static FileReaderFactory allReaders = null;
//     protected int threads;
//     protected boolean fromHDFS = false;
// 
//     public FileReaderFactory(int threads, boolean fromHDFS) {
//         super(false);
//         readers = new ArrayList<>();
//         this.threads = threads;
//         this.fromHDFS = fromHDFS;
//     }
//     
//     public static FileReaderFactory getInstance(int threads, boolean fromHDFS) {
//         if(allReaders == null) {
//             allReaders = new FileReaderFactory(threads, fromHDFS);
//         }
//         return allReaders;
//     }
//     
//     public static BaseFileReader createFileReader(boolean fromHDFS, String fileA, String fileB, boolean interleaved) throws IOException {
//         if (fileB == null) {
//             return new SingleFastQReader(fromHDFS, fileA, interleaved);
//         } else {
//             return new PairedFastQReader(fromHDFS, fileA, fileB);
//         }     
//     }
//     
//     public void addReader(String fileA, String fileB, boolean interleaved) throws IOException {
//         readers.add(createFileReader(fromHDFS, fileA, fileB, interleaved));
//     }
//     
//     public void addReader(BaseFileReader reader) {
//         readers.add(reader);
//     }
// 
//     
//     public ReadBlock retrieveBlock() {
//         try {
//             ReadBlock block = null;
//             while((check || blocks.size() > 0) && block == null) {
//                 block = blocks.poll(1000, TimeUnit.MILLISECONDS);
//             }
//             return block;
//         } catch (InterruptedException ex) {
//             Logger.EXCEPTION(ex);
//             return null;
//         }
//     }
//     
//     
//     @Override
//     protected int addNextRead(ReadBlock block) throws IOException {
//         return currentReader.addNextRead(block);
//     }
//     
//     protected synchronized boolean getNextReader() {
//         if(currentReader == null) {
//             if(readers.size() > 0) {
//                 currentReader = readers.remove(readers.size() - 1);
//                 Logger.DEBUG("Reader: " + currentReader);
//                 return true;
//             } else {
//                 Logger.DEBUG("Processed all readers");
//                 return false;
//             }
//         } else return true;
//     }
// 
// 
//     protected boolean check = true;
//     protected ArrayBlockingQueue<ReadBlock> blocks;
//     protected int READ_BLOCK_CAPACITY_PER_THREAD = 10;
//     
//     @Override
//     public void run() {
//         blocks = new ArrayBlockingQueue<>(READ_BLOCK_CAPACITY_PER_THREAD*threads);
//         if(currentReader == null) {
//             if(!getNextReader()) check = false;
//         }
//         try {
//             while(check) {
//                 ReadBlock block = new ReadBlock();
//                 boolean hasReads = super.getNextBlock(block);
//                 if (!hasReads) {
//                     currentReader = null;
//                     if(!getNextReader())
//                         check = false;
//                 } else {
//                     blocks.put(block);
//                 }
//             }
//         } catch (InterruptedException ex) {
//             Logger.EXCEPTION(ex);
//         }
//     }
// }


// Path: halvade_upload_tool/src/be/ugent/intec/halvade/uploader/HalvadeUploader.java
import be.ugent.intec.halvade.uploader.input.FileReaderFactory;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.commons.cli.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

            formatter.printHelp( "hadoop jar HalvadeUploaderWithLibs.jar -1 <MANIFEST> -O <OUT> [options]", options );
        } catch (Throwable ex) {
            Logger.THROWABLE(ex);
        }
        return 0;
    }
    
    private int processFiles() throws IOException, InterruptedException, URISyntaxException, Throwable {    
        Timer timer = new Timer();
        timer.start();
        
        AWSUploader upl = null;
        FileSystem fs = null;
        // write to s3?
        boolean useAWS = false;
        if(outputDir.startsWith("s3")) {
            useAWS = true;
            String existingBucketName = outputDir.replace("s3://","").split("/")[0];
            outputDir = outputDir.replace("s3://" + existingBucketName + "/", "");
            upl = new AWSUploader(existingBucketName, SSE, profile);
        } else {
            Configuration conf = getConf();
            fs = FileSystem.get(new URI(outputDir), conf);
            Path outpath = new Path(outputDir);
            if (fs.exists(outpath) && !fs.getFileStatus(outpath).isDirectory()) {
                Logger.DEBUG("please provide an output directory");
                return 1;
            }
        }
        
        FileReaderFactory factory = FileReaderFactory.getInstance(mthreads, fromHDFS);