// Path: src/org/ohdsi/utilities/RandomUtilities.java
// public class RandomUtilities {
// 	public static <T> List<T> sampleWithoutReplacement(List<T> items, int sampleSize) {
// 		if (sampleSize > items.size()) {
// 			return items;
// 		}
// 		Random random = new Random();
// 		List<T> sample = new ArrayList<T>();
// 		for (int i = 0; i < sampleSize; i++)
// 			sample.add(items.remove(random.nextInt(items.size())));
// 		return sample;
// 	}
// 	
// 	public static int[] sampleWithoutReplacement(int minValue, int maxValue, int sampleSize) {
// 		if (sampleSize > (maxValue - minValue + 1)) {
// 			int[] sample = new int[maxValue - minValue + 1];
// 			for (int i = minValue; i <= maxValue; i++)
// 				sample[i - minValue] = i;
// 			return sample;
// 		} else {
// 			Random random = new Random();
// 			int[] sample = new int[sampleSize];
// 			for (int i = 0; i < sampleSize; i++) {
// 				int number = random.nextInt(maxValue - minValue + 1);
// 				while (contains(sample, number))
// 					number = random.nextInt(maxValue - minValue + 1);
// 				sample[i] = number;
// 			}
// 			return sample;
// 		}
// 		
// 	}
// 	
// 	private static boolean contains(int[] sample, int number) {
// 		for (int i : sample)
// 			if (i == number)
// 				return true;
// 		return false;
// 	}
// 	
// }
// 
// Path: src/org/ohdsi/utilities/concurrency/BatchProcessingThread.java
// public abstract class BatchProcessingThread extends Thread {
// 	private final Semaphore newInputSemaphore = new Semaphore();
// 	private final Semaphore newOutputSemaphore = new Semaphore();
// 	private boolean terminated = false;
// 
// 	public BatchProcessingThread(){
// 		super();
// 		this.start();
// 
// 	}
// 	
// 	public void run(){
// 		while (true){
// 			newInputSemaphore.release();
// 			if (terminated)
// 				break;
// 			process();
// 			newOutputSemaphore.take();
// 		}
// 	}
// 
// 	protected abstract void process();
// 
// 	public void proceed(){ //This method will be run in the other thread!
// 		newInputSemaphore.take();
// 	}
// 
// 	public void waitUntilFinished(){ //Runs in other thread!
// 		newOutputSemaphore.release();
// 	}
// 
// 	public synchronized void terminate(){//Runs in other thread!
// 		terminated = true;
// 		newInputSemaphore.take();
// 	}
// 
// 
// }


import org.ohdsi.utilities.RandomUtilities;
import org.ohdsi.utilities.concurrency.BatchProcessingThread;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.zip.GZIPInputStream;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

/*******************************************************************************
 * Copyright 2014 Observational Health Data Sciences and Informatics
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ******************************************************************************/
package org.ohdsi.medlineXmlToDatabase;




/**
 * Iterates over all xml.gz files in a specified folder, decompressing and parsing them in a separate thread.
 * 
 * @author Schuemie
 * 
 */
public class XMLFileIterator implements Iterator<Document> {
	
	private Iterator<File>				fileIterator;
	private DecompressAndParseThread	decompressAndParseThread	= new DecompressAndParseThread();
	private boolean						hasNext						= true;
	
	/**
	 * @param folder
	 *            Specifies the absolute path to the folder containing the xml files
	 */
	public XMLFileIterator(String folder) {
		this(folder, Integer.MAX_VALUE);
	}
	
	/**
	 * 
	 * @param folder
	 *            Specifies the absolute path to the folder containing the xml files
	 * @param sampleSize
	 *            Specifies the maximum number of files that is randomly sampled
	 */
	public XMLFileIterator(String folder, int sampleSize) {
		List<File> files = new ArrayList<File>();
		for (File file : new File(folder).listFiles())
			if (file.getAbsolutePath().endsWith("xml.gz"))
				files.add(file);

// Path: src/org/ohdsi/utilities/RandomUtilities.java
// public class RandomUtilities {
// 	public static <T> List<T> sampleWithoutReplacement(List<T> items, int sampleSize) {
// 		if (sampleSize > items.size()) {
// 			return items;
// 		}
// 		Random random = new Random();
// 		List<T> sample = new ArrayList<T>();
// 		for (int i = 0; i < sampleSize; i++)
// 			sample.add(items.remove(random.nextInt(items.size())));
// 		return sample;
// 	}
// 	
// 	public static int[] sampleWithoutReplacement(int minValue, int maxValue, int sampleSize) {
// 		if (sampleSize > (maxValue - minValue + 1)) {
// 			int[] sample = new int[maxValue - minValue + 1];
// 			for (int i = minValue; i <= maxValue; i++)
// 				sample[i - minValue] = i;
// 			return sample;
// 		} else {
// 			Random random = new Random();
// 			int[] sample = new int[sampleSize];
// 			for (int i = 0; i < sampleSize; i++) {
// 				int number = random.nextInt(maxValue - minValue + 1);
// 				while (contains(sample, number))
// 					number = random.nextInt(maxValue - minValue + 1);
// 				sample[i] = number;
// 			}
// 			return sample;
// 		}
// 		
// 	}
// 	
// 	private static boolean contains(int[] sample, int number) {
// 		for (int i : sample)
// 			if (i == number)
// 				return true;
// 		return false;
// 	}
// 	
// }
// 
// Path: src/org/ohdsi/utilities/concurrency/BatchProcessingThread.java
// public abstract class BatchProcessingThread extends Thread {
// 	private final Semaphore newInputSemaphore = new Semaphore();
// 	private final Semaphore newOutputSemaphore = new Semaphore();
// 	private boolean terminated = false;
// 
// 	public BatchProcessingThread(){
// 		super();
// 		this.start();
// 
// 	}
// 	
// 	public void run(){
// 		while (true){
// 			newInputSemaphore.release();
// 			if (terminated)
// 				break;
// 			process();
// 			newOutputSemaphore.take();
// 		}
// 	}
// 
// 	protected abstract void process();
// 
// 	public void proceed(){ //This method will be run in the other thread!
// 		newInputSemaphore.take();
// 	}
// 
// 	public void waitUntilFinished(){ //Runs in other thread!
// 		newOutputSemaphore.release();
// 	}
// 
// 	public synchronized void terminate(){//Runs in other thread!
// 		terminated = true;
// 		newInputSemaphore.take();
// 	}
// 
// 
// }


// Path: src/org/ohdsi/medlineXmlToDatabase/XMLFileIterator.java
import org.ohdsi.utilities.RandomUtilities;
import org.ohdsi.utilities.concurrency.BatchProcessingThread;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.zip.GZIPInputStream;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

/*******************************************************************************
 * Copyright 2014 Observational Health Data Sciences and Informatics
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ******************************************************************************/
package org.ohdsi.medlineXmlToDatabase;




/**
 * Iterates over all xml.gz files in a specified folder, decompressing and parsing them in a separate thread.
 * 
 * @author Schuemie
 * 
 */
public class XMLFileIterator implements Iterator<Document> {
	
	private Iterator<File>				fileIterator;
	private DecompressAndParseThread	decompressAndParseThread	= new DecompressAndParseThread();
	private boolean						hasNext						= true;
	
	/**
	 * @param folder
	 *            Specifies the absolute path to the folder containing the xml files
	 */
	public XMLFileIterator(String folder) {
		this(folder, Integer.MAX_VALUE);
	}
	
	/**
	 * 
	 * @param folder
	 *            Specifies the absolute path to the folder containing the xml files
	 * @param sampleSize
	 *            Specifies the maximum number of files that is randomly sampled
	 */
	public XMLFileIterator(String folder, int sampleSize) {
		List<File> files = new ArrayList<File>();
		for (File file : new File(folder).listFiles())
			if (file.getAbsolutePath().endsWith("xml.gz"))
				files.add(file);

		files = RandomUtilities.sampleWithoutReplacement(files, sampleSize);
