public class DateFormat {
	private static Logger logger = Logger.getLogger(DataMiner.class.getName());
	String name;
	String pattern;
	String dateFormat;
	String regExp;
	String id;

	public String generate() {
		String generatedUniqueId = UUID.randomUUID().toString();
		logger.info("unique ID: " + generatedUniqueId);
		return generatedUniqueId;
	}

	public DateFormat(String _name, String _pattern, String _dateFormat) {
		name = _name;
		pattern = _pattern;
		dateFormat = _dateFormat;
		id = generate();
	}

	public String getId() {
		if (id == null) {
			id = generate();
		}
		return id;
	}

	public String getName() {
		return name;
	}

	public void setName(String name) {
		this.name = name;
	}

	public String getPattern() {
		return pattern;
	}

	public void setPattern(String pattern) {
		this.pattern = pattern;
	}

	public DateFormat clone (){
		String _name=name.toString();
		String _pattern =pattern.toString();
		String _dateFormat= dateFormat.toString();
		
		return new DateFormat(_name, _pattern, _dateFormat);
	}
	
	public String getDateFormat() {
		return dateFormat;
	}

	public void setDateFormat(String dateFormat) {
		this.dateFormat = dateFormat;
	}

	public void update(String _name, String _pattern, String _dateFormat) {
		name = _name;
		pattern = _pattern;
		dateFormat = _dateFormat;
		if (id == null) {
			id = generate();
		}
	}
} public class ExtendedTimeSeries {

	TimeSeries timeSeries;
	RecordingItem recordingItem;
	int[] stat={0,0};

	public ExtendedTimeSeries(TimeSeries ts, int[] stats) {
		stat = stats;
		timeSeries = ts;
		// TODO Auto-generated constructor stub
	}

	public ExtendedTimeSeries(RecordingItem recItem2, Class<FixedMillisecond> class1) {
		timeSeries = new TimeSeries(recItem2.getName(), class1);
		recordingItem=recItem2;
		// TODO Auto-generated constructor stub
	}

	public TimeSeries getTimeSeries() {
		return timeSeries;
	}

	public void setTimeSeries(TimeSeries timeSeries) {
		this.timeSeries = timeSeries;
	}

	public int[] getStat() {
		return stat;
	}

	public void setStat(int[] stat) {
		this.stat = stat;
	}
	
	public RecordingItem getRecordingItem() {
		return recordingItem;
	}

} public class Preferences {
	private static Logger logger = Logger.getLogger(Preferences.class.getName());

	static PreferenceData prefData;
	
	public Preferences() {
		logger.info("Preferences()");
		//PreferenceData prefData= (PreferenceData) Persister.open(new File("Preference.xml"));
		//getPreferences();
	}
	public static void load() {
		prefData= (PreferenceData) Persister.open(new File("Preference.xml"));
		if (prefData ==null || prefData.preferences==null) {
			prefData= new PreferenceData();
			logger.debug(prefData+";"+prefData.preferences);
			Persister.save(new File("Preference.xml"), (PreferenceData) prefData);
			logger.info("Created Preference.xml");
		}
		getPreferences();
	}
	
	public static HashMap<String, String> getPreferences() {
		if (prefData.preferences == null) {
			prefData.preferences = new HashMap<String, String>();
			logger.info("new prefData.preferences");
		}
		if (!prefData.preferences.containsKey("timings")) {
			prefData.preferences.put("timings", "false");
		}
		if (!prefData.preferences.containsKey("matches")) {
			prefData.preferences.put("matches", "true");
		}
		if (!prefData.preferences.containsKey("stats")) {
			logger.info("stats set to true");
			prefData.preferences.put("stats", "true");
		}
		if (!prefData.preferences.containsKey("chartSize")) {
			prefData.preferences.put("chartSize", "350");
		}
		if (!prefData.preferences.containsKey("lastPath")) {
			prefData.preferences.put("lastPath", ".");
		}
		if (!prefData.preferences.containsKey("ThreadPool_SourceGroup")) {
			prefData.preferences.put("ThreadPool_SourceGroup", "4");
		}
		if (!prefData.preferences.containsKey("ThreadPool_MineProcessor")) {
			prefData.preferences.put("ThreadPool_MineProcessor", "8");
		}
		if (!prefData.preferences.containsKey("ThreadPool_ProcessorByGroup")) {
			prefData.preferences.put("ThreadPool_ProcessorByGroup", "2");
		}
		if (!prefData.preferences.containsKey("editorCommand")) {
			prefData.preferences.put("editorCommand", "gvim -R +$line \"$file\"");
		}
		if (!prefData.preferences.containsKey("gatherstats")) {
			prefData.preferences.put("gatherstats", "true");
		}
		if (!prefData.preferences.containsKey("gatherevents")) {
			prefData.preferences.put("gatherevents", "true");
		}
		if (!prefData.preferences.containsKey("gatherreports")) {
			prefData.preferences.put("gatherreports", "true");
		}
		if (!prefData.preferences.containsKey("eventsAsDots")) {
			prefData.preferences.put("eventsAsDots", "true");
		}
		if (!prefData.preferences.containsKey("MiningFileChunk")) {
			prefData.preferences.put("MiningFileChunk", "1000");
		}		
		if (!prefData.preferences.containsKey("ForceSourceDateFormat")) {
			prefData.preferences.put("ForceSourceDateFormat", "false");
		}				
		
		
		
		return prefData.preferences;
	}
	
	public static void setPreference(String key, String value) {
		prefData.preferences.put(key, value);
		logger.debug(prefData.preferences);
		Persister.save(new File("Preference.xml"), (PreferenceData) prefData);
	}

	public static String getPreference(String key) {
		return getPreferences().get(key);
	}
	
	public static boolean isStats() {
		return Boolean.parseBoolean(getPreference("stats"));
	}

	public void setStats(boolean stats) {
		setPreference("stats", Boolean.toString(stats));
	}

	public static boolean isTimings() {
		return Boolean.parseBoolean(getPreference("timings"));
	}

	public void setTimings(boolean timings) {
		setPreference("timings", Boolean.toString(timings));
	}
	public static boolean isMatches() {
		return Boolean.parseBoolean(getPreference("matches"));
	}
	
	public void setMatches(boolean matches) {
		setPreference("matches", Boolean.toString(matches));
	}
	
	public static boolean getBooleanPreference(String key){
		// TBD : implement a cache 
		return Boolean.parseBoolean(getPreferences().get(key));
	}

	public static void persist() {
		logger.debug(prefData);
		Persister.save(new File("Preference.xml"), (PreferenceData) prefData);
		// TODO Auto-generated method stub
		
	}

} public class Repository {
	private static Logger logger = Logger.getLogger(DataMiner.class.getName());
	private ArrayList<Recording> recordings;
	private ArrayList<Source> sources;
	private String baseSourcePath;
	private ArrayList<DateFormat> dates;
	private boolean recursiveMode;
	private boolean onlyMatches;
	private boolean stats; // unused
	private boolean timings; // unused
	private HashMap<String, String> preferences;

	public Repository() {
		preferences = new HashMap<String, String>();
		preferences.put("timings", "false");
		preferences.put("stats", "true");
		preferences.put("chartSize", "350");
		preferences.put("ThreadPool_Group", "4");
		preferences.put("ThreadPool_File", "8");
		preferences.put("editorCommand", "gvim -R +$line $file");
		preferences.put("gatherstats", "true");
		preferences.put("gatherevents", "true");
		preferences.put("gatherreports", "true");
		recordings = new ArrayList<Recording>();
		dates = new ArrayList<DateFormat>();
		sources = new ArrayList<Source>();
		recursiveMode = false;
		// logger.info("repository ArrayList initialized");
	}

	public boolean isRecursiveMode() {
		return recursiveMode;
	}

	public void setPreference(String key, String value) {
		preferences.put(key, value);
	}

	public String getPreference(String key) {
		return getPreferences().get(key);
	}

	public void setRecursiveMode(boolean recursiveMode) {
		// logger.info("recursive mode is :"+recursiveMode);
		this.recursiveMode = recursiveMode;
	}

	public ArrayList<Source> getSources() {
		return sources;
	}

	public Source getSource(String name) {
		Iterator sourceIterator = sources.iterator();
		int cnt = 0;
		while (sourceIterator.hasNext()) {
			Source src = (Source) sourceIterator.next();
			if (src.getSourceName() == name) {
				return src;
			}
		}
		return (Source) null;
	}

	public void setSources(ArrayList<Source> sources) {
		this.sources = sources;
	}

	public void addSource(Source s) {
		if (sources == null) {
			sources = new ArrayList<Source>();
		}
		sources.add(s);
	}

	public void deleteSource(int id) {
		sources.remove(id);
	}

	public Source getSource(int id) {
		return sources.get(id);
	}

	public void updateSource(int id, String txtName, String txtRegularExp, Boolean active) {
		sources.get(id).setSourceName(txtName);
		sources.get(id).setSourcePattern(txtRegularExp);
		sources.get(id).setActive(active);
	}

	public ArrayList<Recording> getRecordings() {
		// logger.info(xstream.toXML(recordings));
		return recordings;
	}

	public ArrayList<Recording> getRecordings(Class _class, boolean onlyActive) {

		ArrayList<Recording> statRecordingArrayList = new ArrayList<Recording>();
		Iterator recordingIterator = recordings.iterator();
		int cnt = 0;
		while (recordingIterator.hasNext()) {
			Recording r = (Recording) recordingIterator.next();
			if (r.getClass().equals(_class)) { 
				if (onlyActive){
					if (r.getIsActive())
					statRecordingArrayList.add(r);	
				} else {
				statRecordingArrayList.add(r);
				}
			}
		}

		return (ArrayList<Recording>) statRecordingArrayList;
	}


	public ArrayList getReportRecordings(MineResultSet mineResultSet1, boolean b) {
		ArrayList<Recording> temp = new ArrayList<Recording>();
		ArrayList<Recording> returned= new ArrayList<Recording>();		
		Iterator<Map<Recording, Map<List<Object>, Long>>> it1 = mineResultSet1.getOccurenceReport().values().iterator();
		while (it1.hasNext()){
			Iterator test = (Iterator) it1.next().keySet().iterator();
			while (test.hasNext())
			{
			temp.add((Recording) test.next());
			}
		}
		
		Iterator<Map<Recording, Map<List<Object>, Double>>> it2 =mineResultSet1.getSumReport().values().iterator();
		while (it2.hasNext()){
			Iterator test = (Iterator) it2.next().keySet().iterator();
			while (test.hasNext())
			{
			temp.add((Recording) test.next());
			}
		}
		
		Iterator<Map<Recording, SortedMap<Double, List<Object>>>> it3 =mineResultSet1.getTop100Report().values().iterator();
		while (it3.hasNext()){
			Iterator test = (Iterator) it3.next().keySet().iterator();
			while (test.hasNext())
			{
			temp.add((Recording) test.next());
			}
		}
		
		ArrayList<Recording> aL = getRecordings(ReportRecording.class, b);
		Iterator ite = aL.iterator();
		while (ite.hasNext()){
			Recording rec= (Recording) ite.next();
			if (temp.contains(rec))	{
			returned.add(rec);	
			}
		}
		return returned;
	}
	
	
	public Recording getRecording(Class _class, int id, boolean onlyActive) {
		return getRecordings(_class,onlyActive).get(id);
	}

	public HashMap<String, String> getPreferences() {
		if (preferences == null) {
			preferences = new HashMap<String, String>();
			logger.info("new preferences");
		}
		if (!preferences.containsKey("timings")) {
			preferences.put("timings", "false");
		}

		if (!preferences.containsKey("stats")) {
			logger.info("stats set to true");
			preferences.put("stats", "true");
		}

		if (!preferences.containsKey("chartSize")) {
			preferences.put("chartSize", "350");
		}

		if (!preferences.containsKey("ThreadPool_Group")) {
			preferences.put("ThreadPool_Group", "4");
		}
		if (!preferences.containsKey("ThreadPool_File")) {
			preferences.put("ThreadPool_File", "8");
		}
		if (!preferences.containsKey("editorCommand")) {
			preferences.put("editorCommand", "gvim -R +$line $file");
		}
		
		return preferences;
	}

	public void addRecording(Recording r) {
		recordings.add(r);
		// logger.info(xstream.toXML(recordings));<
	}

	public int getRecordingCount() {
		return recordings.size();
	}

	public void deleteRecording(int id) {
		recordings.remove(id);
		ArrayList<Source> sources= this.getSources();
		Iterator<Source> ite=sources.iterator();
		while (ite.hasNext()){
			Source src=(Source) ite.next();
			src.removeActiveRecording(recordings.get(id));
		}
	}

	public Recording getRecording(int id) {
		return recordings.get(id);
	}

	public Recording getRecording(String _id) {
		Iterator recordingIterator = recordings.iterator();
		Recording recReturn = null; 
		int cnt = 0;
		while (recordingIterator.hasNext()) {
			Recording r = (Recording) recordingIterator.next();
			if (r.getId() == _id) {
				recReturn = r;
			}
		}
		return recReturn;
	}

	public void duplicateRecording(int id) {
		Recording newRecording = recordings.get(id).duplicate();
		this.addRecording(newRecording);
		ArrayList<Source> sources= this.getSources();
		Iterator<Source> ite=sources.iterator();
		if (!MetadataRecording.class.isInstance(recordings.get(id))){
		while (ite.hasNext()){
			Source src=(Source) ite.next();
			if (src.isActiveRecordingOnSource(recordings.get(id))){
				src.toggleActiveRecording(newRecording);
			}
		}}
	}

	public void update(Repository repo) {
		recordings = repo.getRecordings();
	}

	public void save(File file) {

		FileOutputStream fos = null;
		try {
			String xml = new XStream(new StaxDriver()).toXML(recordings);
			fos = new FileOutputStream(file);
			// fos.write("<?xml version=\"1.0\"?>".getBytes("UTF-8"));
			byte[] bytes = xml.getBytes("UTF-8");
			fos.write(bytes);

		} catch (Exception e) {
			System.err.println("Error in XML Write: " + e.getMessage());
		} finally {
			if (fos != null) {
				try {
					fos.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}

	public void open(File file) {
		// XStream xstream = new XStream(new StaxDriver());
		FileOutputStream fos = null;
		try {
			recordings = (ArrayList<Recording>) new XStream(new StaxDriver()).fromXML(file);
			/*
			 * fos = new FileOutputStream(file);
			 * 
			 * byte[] bytes = xml.getBytes("UTF-8"); fos.write(bytes);
			 */
			logger.info(new XStream(new StaxDriver()).toXML(recordings));
		} catch (Exception e) {
			System.err.println("Error in XML Write: " + e.getMessage());
		} finally {
			if (fos != null) {
				try {
					fos.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}

	public void setRecordings(ArrayList recordings) {
		this.recordings = recordings;
	}

	public void save() {

	}

	public String getBaseSourcePath() {
		return baseSourcePath;
	}

	public void setBaseSourcePath(String baseSourcePath) {
		this.baseSourcePath = baseSourcePath;
	}

	public ArrayList<DateFormat> getDates() {
		return dates;
	}

	public void setDates(ArrayList<DateFormat> hm) {
		dates = hm;
	}

	public void addDateFormat(DateFormat df) {
		if (dates == null) {
			dates = new ArrayList<DateFormat>();
		}
		dates.add(df);
		// logger.info(xstream.toXML(recordings));
	}

	public void deleteDateFormat(int id) {
		dates.remove(id);
	}

	public DateFormat getDateFormat(int id) {
		return dates.get(id);
	}

	public DateFormat getDateFormat(String id) {
		Iterator dateFormatIterator = dates.iterator();
		int cnt = 0;
		while (dateFormatIterator.hasNext()) {
			DateFormat df = (DateFormat) dateFormatIterator.next();
			if (df.getId() == id) {
				return df;
			}
		}
		return (DateFormat) null;
	}

	public void setOnlyMatches(boolean selected) {
		onlyMatches = selected;

	}

	public boolean isOnlyMatches() {
		return onlyMatches;
	}

	public boolean isStats() {
		return Boolean.parseBoolean(getPreference("stats"));
	}

	public void setStats(boolean stats) {
		setPreference("stats", Boolean.toString(stats));
	}

	public boolean isTimings() {
		return Boolean.parseBoolean(getPreference("timings"));
	}

	public void setTimings(boolean timings) {
		setPreference("timings", Boolean.toString(timings));
	}

} public class Source implements Comparable{
	private static Logger logger = Logger.getLogger(Source.class.getName());
	private String sourceName;
	private String sourcePattern;
	private DateFormat dateFormat;
	private Boolean active;
	private MetadataRecording activeMetadataRecording;
	private ArrayList<String> selectedRecordingIds;

	public Source(String name, String pattern, DateFormat _dateFormat, Boolean _active) {
		sourceName = name;
		sourcePattern = pattern;
		dateFormat = _dateFormat;
		setActive(_active);
		selectedRecordingIds = new ArrayList<String>();
		logger.info("Source loaded");

	}

	public String getSourcePattern() {
		return sourcePattern;
	}

	public void setSourcePattern(String sourcePattern) {
		this.sourcePattern = sourcePattern;
	}

	public void setActiveMetadata(MetadataRecording _activeMetadataRecording){
		activeMetadataRecording=_activeMetadataRecording;
	}

	public MetadataRecording getActiveMetadata(){
		return activeMetadataRecording;
	}
	
	public void toggleActiveRecording(Recording recording) {
		// logger.info("toggleActiveRecording of "+recording.getName());
		if (selectedRecordingIds.contains(recording.getId())) {
			selectedRecordingIds.remove(recording.getId());
			if (logger.isEnabledFor(Level.TRACE))
				logger.trace("toggleActiveRecording removed " + recording.getName() + " now size " + selectedRecordingIds.size());
		} else {
			selectedRecordingIds.add(recording.getId());
			if (logger.isEnabledFor(Level.TRACE))
				logger.trace("toggleActiveRecording added " + recording.getName() + " now size " + selectedRecordingIds.size());
		}

	}

	public void removeActiveRecording(Recording recording) {
		// logger.info("toggleActiveRecording of "+recording.getName());
		if (selectedRecordingIds.contains(recording.getId())) {
			selectedRecordingIds.remove(recording.getId());
			if (logger.isEnabledFor(Level.TRACE))
				logger.trace("toggleActiveRecording removed " + recording.getName() + " now size " + selectedRecordingIds.size());
		} 
	}
	
	
	
	public Boolean isActiveRecordingOnSource(Recording recording) {
		if (MetadataRecording.class.isInstance(recording)){
				if (getActiveMetadata()!=null && getActiveMetadata()==recording){
					return true;
				}
				else return false;
		}
		// logger.info("***isActiveRecordingOnSource of "+recording.getName());
		Boolean b = false;
		Iterator it = selectedRecordingIds.iterator();
		String recordingId = recording.getId();
		while (it.hasNext()) {
			// String temp=(String)it.next();
			if ((String) it.next() == recordingId) {
				// logger.info("***isActiveRecordingOnSource of "+recording.getName()
				// + " with id: "+ temp);
				b = true;
			}
		}
		return b;
	}

	public String getSourceName() {
		return sourceName;
	}

	public void setSourceName(String sourceName) {
		this.sourceName = sourceName;
	}

	public Boolean getActive() {
		return active;
	}

	public void setActive(Boolean active) {
		this.active = active;
	}

	@Override
	public int compareTo(Object o) {
		String local = getSourceName();
		String remote = ((Source)o).getSourceName();
		//return remote.compareTo(local);
		return local.compareTo(remote);

	}

	public DateFormat getDateFormat() {
		return dateFormat;
	}

	public void setDateFormat(DateFormat dateFormat) {
		this.dateFormat = dateFormat;
	}
} public class FileLine {
	int fileId;
	int lineNumber;

	public FileLine() {
		// TODO Auto-generated constructor stub
	}

	public FileLine(int _fileId, int _lineNumber) {
		lineNumber = _lineNumber;
		fileId = _fileId;
	}

	public int getFileId() {
		return fileId;
	}

	public int getLineNumber() {
		return lineNumber;
	}
	public String toString(){
		return ""+fileId+":"+lineNumber;
	}
} public class FileMineResult {
	private static Logger logger = Logger.getLogger(FileMineResult.class.getName());
	public Map<String, ExtendedTimeSeries> statGroupTimeSeries;
	public Map<String, ExtendedTimeSeries> eventGroupTimeSeries;
	public Map<String,Map<Date, FileLine>> fileLineDateMap;
	public Map<String, long[]> matchingStats; // 0-> sum of time for success
												// matching of given recording ;
												// 1-> sum of time for failed
												// matching ; 2-> count of match
												// attempts, 3->count of success
												// attempts
	private Date startDate;
	private Date endDate;
	private FileRecord file;

	public Date getStartDate() {
		return startDate;
	}

	public Date getEndDate() {
		return endDate;
	}

	public FileMineResult(FileRecord fileRecord, Map<String, ExtendedTimeSeries> _statGroupTimeSeries, Map<String, ExtendedTimeSeries> _eventGroupTimeSeries,
			Map<String, long[]> _matchingStats, Map<String, Map<Date, FileLine>> _fileLineDateMap, Date startDate2, Date endDate2) {
		file = fileRecord;
		startDate = startDate2;
		endDate = endDate2;
		statGroupTimeSeries = _statGroupTimeSeries;
		eventGroupTimeSeries = _eventGroupTimeSeries;
		matchingStats = _matchingStats;
		fileLineDateMap=_fileLineDateMap;
		if (logger.isTraceEnabled()) {
			logger.trace(file.getFile().getName()+_fileLineDateMap.toString());
			logger.trace("start date: " + startDate2 + " end date: " + endDate2);
		}

	}

	public FileRecord getFile() {
		return file;
	}
} public class FileRecord {
	private static Logger logger = Logger.getLogger(FileRecord.class.getName());
int id;
File file;

	public FileRecord() {
		// TODO Auto-generated constructor stub
	}

	public FileRecord(int _id, File _file) {
		id=_id;
		file=_file;
	}

	public int getId() {
		return id;
	}

	public String getCompletePath() {
		try {
			return file.getCanonicalPath();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			return "OUCH";
		}
	}
	
	public String toString(){
		try {
			return file.getCanonicalPath()+" "+id;
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return null;
				}
	public File getFile() {
	return file;
}
	
} public class MineData {
	public final List<FileMineResult> fileMineResultArray =   Collections.synchronizedList( new ArrayList<FileMineResult>());
	int blocks=0;
	
	public MineData() {
		// TODO Auto-generated constructor stub
	}
	public void clear(){
		fileMineResultArray.clear();
	}
	public void setBlocks(int blocks) {
		this.blocks = blocks;
	}
	public int getBlocks() {
		return blocks;
	}
} public class MineItem {
	BlockingQueue<ReportItem> mineQueue;
	int offset;
	FileRecord fileRecord;
	ArrayList<String[]> dataBlock;
	Map<Recording, String> recMatch1;
	Repository repo;
	Source source;
	boolean stats; 
	boolean timings; 
	boolean matches; 
	private static Logger logger = Logger.getLogger(MineItem.class.getName());
	
	public MineItem(BlockingQueue<ReportItem> _mineQueue, int _offset, FileRecord _fileRecord, ArrayList<String[]> _dataBlock, Map<Recording, String> _recMatch1, Repository _repo,
			Source _source, boolean _stats, boolean _timings, boolean _matches) {
		// TODO Auto-generated constructor stub
		mineQueue =_mineQueue;
		offset =_offset;
		fileRecord = _fileRecord;
		dataBlock = _dataBlock;
		recMatch1 = _recMatch1;
		repo = _repo;
		source = _source;
		stats = _stats; 
		timings = _timings; 
		matches = _matches; 
	}

	public Source getSource() {
		return source;
	}

	public BlockingQueue<ReportItem> getMineQueue() {
		return mineQueue;
	}

	public int getOffset() {
		return offset;
	}

	public FileRecord getFileRecord() {
		return fileRecord;
	}

	public ArrayList<String[]> getDataBlock() {
		return dataBlock;
	}

	public Map<Recording, String> getRecMatch1() {
		return recMatch1;
	}

	public Repository getRepo() {
		return repo;
	}

	public boolean isStats() {
		return stats;
	}

	public boolean isTimings() {
		return timings;
	}

	public boolean isMatches() {
		return matches;
	}
	
} public class ReportItem {
	String line;
	ReportRecording recording;
	Matcher matcher;
	Source source;

	public ReportItem(Source _source, String _line, Matcher _matcher, ReportRecording _recording) {
		// TODO Auto-generated constructor stub
		line = _line;
		recording = _recording;
		matcher = _matcher;
		source = _source;
	}

	public String getLine() {
		return line;
	}

	public ReportRecording getRecording() {
		return recording;
	}

	public Matcher getMatcher() {
		return matcher;
	}

	public Source getSource() {
		return source;
	}

} public class EventRecording extends Recording {
	private static Logger logger = Logger.getLogger(EventRecording.class.getName());
	private String dateFormat;

	public EventRecording(String _name, String _regexp, String _exampleLine, String _dateFormat, Boolean _isActive, Boolean _useSourceDateFormat, boolean _caseSensitive, ArrayList<RecordingItem> _recordingItem) {
		setName(_name);
		setRegexp(_regexp);
		setExampleLine(_exampleLine);
		setIsActive(_isActive);
		setUseSourceDateFormat(_useSourceDateFormat);
		dateFormat = _dateFormat;
		recordingItem = _recordingItem;
		setCaseSensitive(_caseSensitive);
		super.id = generate();
		logger.info("New EventRecording name: " + _name + ", regexp: " + _regexp + ", id: " + super.id);
		if (recordingItem != null)
			logger.info("New EventRecording with recordingItem ArrayList: " + recordingItem.toString());
	}

	public String getType() {
		return "Event";
	}

	public String getDateFormat() {
		return dateFormat;
	}

	public void setDateFormat(String dateFormat) {
		this.dateFormat = dateFormat;
	}

	public void setRecordingItem(ArrayList<RecordingItem> recordingItem) {
		this.recordingItem = recordingItem;
	}

	public void update(String txtName, String txtRegularExp, String exampleLine, String _dateFormat, Boolean _isActive, Boolean _useSourceDateFormat, Boolean _caseSensitive, ArrayList<RecordingItem> _recordingItem) {
		setName(txtName);
		setRegexp(txtRegularExp);
		setExampleLine(exampleLine);
		setIsActive(_isActive);
		setCaseSensitive(_caseSensitive);
		setUseSourceDateFormat(_useSourceDateFormat);
		dateFormat = _dateFormat;
		recordingItem = _recordingItem;
		logger.debug("RIArrayLit size: "+ recordingItem.size());
		if (this.id == null) {
			super.id = generate();
		}
	}

	public Recording duplicate() {

		ArrayList<RecordingItem> _recordingItem = null;
		// might need put back .toString() to those?? *** TBR
		String _name = getName().toString();
		String _regexp = getRegexp().toString();
		String _exampleLine = getExampleLine().toString();
		String _dateFormat = dateFormat.toString();

		Boolean _isActive = getIsActive().booleanValue();
		Boolean _caseSensitive = isCaseSensitive();
		Boolean _useSourceDateFormat=getUseSourceDateFormat();
		if (recordingItem != null) {
			_recordingItem = (ArrayList<RecordingItem>) recordingItem.clone();
		}
		EventRecording eR=new EventRecording(_name, _regexp, _exampleLine, _dateFormat, _isActive, _useSourceDateFormat, _caseSensitive, _recordingItem);
		eR.setDateFormatID(this.getDateFormatID());
		return eR;
	}
} public abstract class Recording {
	private static Logger logger = Logger.getLogger(DataMiner.class.getName());
	private String name;
	private String regexp;
	protected ArrayList recordingItem;
	private String FastDateFormat;
	private String exampleLine;
	private Boolean isActive;
	protected String id;
	private Boolean caseSensitive;
	private Boolean useSourceDateFormat;
	private String dateFormatID;

	public String getId() {
		return id;
	}

	public String generate() {
		String generatedUniqueId = UUID.randomUUID().toString();
		logger.info("unique ID: " + generatedUniqueId);
		return generatedUniqueId;
	}

	public ArrayList getRecordingItem() {
		// logger.info("***********"+recordingItem.toString()+recordingItem.size());
		// Thread.currentThread().dumpStack();
		return recordingItem;
	}

	public String getName() {
		return name;
	}

	public void setName(String name) {
		this.name = name;
	}

	public String getRegexp() {
		return regexp;
	}

	public void setRegexp(String regexp) {
		this.regexp = regexp;
	}

	public String getExampleLine() {
		return exampleLine;
	}

	public void setExampleLine(String exampleLine) {
		this.exampleLine = exampleLine;
	}

	public abstract String getType();

	public Boolean getIsActive() {
		return isActive;
	}

	public boolean isCaseSensitive() {
		if (caseSensitive!=null){
			return caseSensitive;
		}else {
			return true;
		}
	}

	public void setCaseSensitive(boolean caseSensitive) {
		this.caseSensitive = caseSensitive;
	}

	public void setIsActive(Boolean isActive) {
		this.isActive = isActive;
	}

	abstract public Recording duplicate();

	public String getDateFormatID() {
		return dateFormatID;
	}

	public void setDateFormatID(String dateFormatID) {
		this.dateFormatID = dateFormatID;
	}

	public Boolean getUseSourceDateFormat() {
		if (useSourceDateFormat==null){
			return false;
		} else{
		return useSourceDateFormat;}
	}

	public void setUseSourceDateFormat(Boolean useSourceDateFormat) {
		this.useSourceDateFormat = useSourceDateFormat;
	}

} public class RecordingItem {
	private String processingType;
	protected String name;
	protected String before;
	protected String after;
	protected String type;
	protected String inside;
	protected String value;
	protected boolean selected;
	protected boolean show;

	public String getBefore() {
		return before;
	}

	public void setBefore(String before) {
		this.before = before;
	}

	public String getAfter() {
		return after;
	}

	public void setAfter(String after) {
		this.after = after;
	}

	public String getProcessingType() {
		return processingType;
	}

	public void setProcessingType(String processingType) {
		this.processingType = processingType;
	}

	public RecordingItem(String name, String before, String type, String processingType, String insideRegex, String after, Boolean isSelected, Boolean show, String value) {
		this.name = name;
		this.before = before;
		this.after = after;
		this.value = value;
		this.type = type;
		this.selected = isSelected;
		this.processingType = processingType;
		this.inside=insideRegex;
		this.show=show;
		// super(name, before, type, after, isSelected, value);
		// TODO Auto-generated constructor stub
	}

	public RecordingItem(String name, String before, String type, String insideRegex, String after, Boolean isSelected, Boolean show, String value) {
		this.name = name;
		this.before = before;
		this.after = after;
		this.value = value;
		this.type = type;
		this.selected = isSelected;
		this.inside=insideRegex;
		this.show=show;
	}

	public String getName() {
		return name;
	}

	public void setName(String name) {
		this.name = name;
	}

	public String getType() {
		return type;
	}

	public void setType(String type) {
		this.type = type;
	}

	public String getValue() {
		return value;
	}

	public void setValue(String value) {
		this.value = value;
	}

	public boolean isSelected() {
		return selected;
	}

	public void setSelected(boolean selected) {
		this.selected = selected;
	}

	public boolean isShow() {
		return show;
	}

	public void setShow(boolean show) {
		this.show = show;
	}

	public String getInside() {
		return inside;
	}

	public void setInside(String _inside) {
		this.inside = _inside;
	}
	
	

} public class ReportRecording extends Recording {
	private static Logger logger = Logger.getLogger(ReportRecording.class.getName());
	private String dateFormat;
	private String subType;
	private int top100Index = -1;

	public ReportRecording(String _name, String _regexp, String _exampleLine, String _dateFormat, Boolean _isActive,Boolean _useSourceDateFormat, ArrayList<RecordingItem> _recordingItem,
			String subType1, boolean _caseSensitive) {
		setName(_name);
		setRegexp(_regexp);
		setExampleLine(_exampleLine);
		setIsActive(_isActive);
		setCaseSensitive(_caseSensitive);
		setUseSourceDateFormat(_useSourceDateFormat);
		dateFormat = _dateFormat;
		recordingItem = _recordingItem;
		subType = subType1;
		super.id = generate();
		computeTop100RecordID();
		logger.info("New ReportRecording name: " + _name + ", regexp: " + _regexp + ", id: " + super.id);
		if (recordingItem != null)
			logger.info("New ReportRecording with recordingItem ArrayList: " + recordingItem.toString());
	}

	public String getType() {
		return "Report";
	}

	public void setSubType(String subType1) {
		this.subType = subType1;
	}

	public String getSubType() {
		return subType;
	}

	private void computeTop100RecordID() {
		if (recordingItem != null) {
			Iterator<RecordingItem> rIIte = recordingItem.iterator();
			top100Index = -1;
			int count = 0;
			while (rIIte.hasNext() && top100Index == -1) {
				if (rIIte.next().getProcessingType().equals("top100")) {
					top100Index = count;
				}
				count++;
			}
		}
	}

	public int getTop100RecordID() {
		if (top100Index == -1) {
			computeTop100RecordID();
			;
			return top100Index;
		} else {
			return top100Index;
		}

	}

	public String getDateFormat() {
		return dateFormat;
	}

	public void setDateFormat(String dateFormat) {
		this.dateFormat = dateFormat;
	}

	public void setRecordingItem(ArrayList<RecordingItem> recordingItem) {
		this.recordingItem = recordingItem;
	}

	public void update(String txtName, String txtRegularExp, String exampleLine, String _dateFormat, Boolean _isActive, Boolean _useSourceDateFormat,
			ArrayList<RecordingItem> _recordingItem, String subType1, boolean _caseSensitive) {
		setName(txtName);
		setRegexp(txtRegularExp);
		setExampleLine(exampleLine);
		setIsActive(_isActive);
		setUseSourceDateFormat(_useSourceDateFormat);
		setCaseSensitive(_caseSensitive);
		dateFormat = _dateFormat;
		subType = subType1;
		recordingItem = _recordingItem;
		if (this.id == null) {
			super.id = generate();
		}
		computeTop100RecordID();

	}

	public Recording duplicate() {

		ArrayList<RecordingItem> _recordingItem = null;
		// might need put back .toString() to those?? *** TBR
		String _name = getName().toString();
		String _regexp = getRegexp().toString();
		String _exampleLine = getExampleLine().toString();
		String _dateFormat = dateFormat.toString();
		String _subType = subType.toString();

		Boolean _isActive = getIsActive().booleanValue();
		Boolean _caseSensitive = isCaseSensitive();
		Boolean _useSourceDateFormat=getUseSourceDateFormat();
		if (recordingItem != null) {
			_recordingItem = (ArrayList<RecordingItem>) recordingItem.clone();
		}
		ReportRecording eR = new ReportRecording(_name, _regexp, _exampleLine, _dateFormat, _isActive,_useSourceDateFormat, _recordingItem, _subType, _caseSensitive);
		eR.setDateFormatID(this.getDateFormatID());
		return eR;
	}
} public class StatRecording extends Recording {
	private static Logger logger = Logger.getLogger(StatRecording.class.getName());
//	private ArrayList recordingItem;
	private String dateFormat;

	public String getType() {
		return "Stat";
	}

	public StatRecording(String _name, String _regexp, String _exampleLine, String _dateFormat, Boolean _isActive, Boolean _useSourceDateFormat, Boolean _caseSensitive, ArrayList _recordingItem) {
		setName(_name);
		setRegexp(_regexp);
		setExampleLine(_exampleLine);
		setIsActive(_isActive);
		setUseSourceDateFormat(_useSourceDateFormat);
		setCaseSensitive(_caseSensitive);
		dateFormat = _dateFormat;
		recordingItem = _recordingItem;
		super.id = generate();
		// logger.info("New EventRecording name: "+_name + ", regexp: "+ _regexp
		// + ", id: "+ super.id +recordingItem.toString());
	}

	public String getDateFormat() {
		return dateFormat;
	}

	public void setDelimitator(String delimitator) {
	//	this.dateFormat = dateFormat;
	}

	public ArrayList getRecordingItem() {
		// logger.info("***********"+recordingItem.toString()+recordingItem.size());
		// Thread.currentThread().dumpStack();
		return recordingItem;
	}

	public void setRecordingItem(ArrayList recordingItem) {
		this.recordingItem = recordingItem;
	}

	public void update(String txtName, String txtRegularExp, String exampleLine, String _dateFormat, Boolean _isActive, Boolean _useSourceDateFormat, Boolean _caseSensitive, ArrayList _recordingItem) {
		setName(txtName);
		setRegexp(txtRegularExp);
		setExampleLine(exampleLine);
		setIsActive(_isActive);
		setCaseSensitive(_caseSensitive);
		setUseSourceDateFormat(_useSourceDateFormat);
		dateFormat = _dateFormat;
		recordingItem = _recordingItem;
		if (this.id == null) {
			super.id = generate();
		}
	}

	public Recording duplicate() {

		ArrayList _recordingItem = null;
		// might need put back .toString() to those?? *** TBR
		String _name = getName().toString();
		String _regexp = getRegexp().toString();
		String _exampleLine = getExampleLine().toString();
		String _dateFormat = dateFormat.toString();

		Boolean _isActive = getIsActive().booleanValue();
		Boolean _caseSensitive = isCaseSensitive();
		Boolean _useSourceDateFormat=getUseSourceDateFormat();
		if (recordingItem != null) {
			_recordingItem = (ArrayList) recordingItem.clone();
		}
		StatRecording eR=new StatRecording(_name, _regexp, _exampleLine, _dateFormat, _isActive,_useSourceDateFormat, _caseSensitive, _recordingItem);
		eR.setDateFormatID(this.getDateFormatID());
		return eR;
		
	}
} public class ClassCache {
//	private static Logger logger = Logger.getLogger(DataMiner.class.getName());
	private static Map<Object, Class> classMap = new HashMap<Object, Class>();

public Class getClass(Object rec) {
	if (!classMap.containsKey(rec)) {
		classMap.put(rec, rec.getClass());
	}
	return classMap.get(rec);
}

} public class PatternCache {
//	private static Logger logger = Logger.getLogger(DataMiner.class.getName());
	private Map<String, Pattern> pattern = new HashMap<String, Pattern>();
	private Map<String, Matcher> matcher = new HashMap<String, Matcher>();

	public Pattern getPattern(String regexp, boolean caseSensitive) {
		Pattern temp = pattern.get(regexp+Boolean.toString(caseSensitive));
		if (temp!=null){
			return temp;
		} else{
			if (caseSensitive){
				pattern.put(regexp+caseSensitive, Pattern.compile(regexp));
			}else{
				pattern.put(regexp+caseSensitive, Pattern.compile(regexp, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE));
			}			
		}
		return pattern.get(regexp+caseSensitive);
	}
	
	public Matcher getMatcher(String regexp, boolean caseSensitive,String str) {
		Matcher tempMatcher=matcher.get(regexp+caseSensitive);
		if (tempMatcher!=null){
			return tempMatcher.reset(str);
		} else{
			if (caseSensitive){
				matcher.put(regexp+caseSensitive, getPattern(regexp,caseSensitive).matcher(str));
			}else{
				matcher.put(regexp+caseSensitive, getPattern(regexp,caseSensitive).matcher(str));	
			}
		}
		return matcher.get(regexp+caseSensitive);
	}
	
	public Matcher getNewMatcher(String regexp, boolean caseSensitive,String str) {
		return getPattern(regexp,caseSensitive).matcher(str);
	}
	
} public class ThreadLocalDateFormatMap {

	
	protected static ThreadLocalDateFormatMap INSTANCE = new ThreadLocalDateFormatMap();

	public static ThreadLocalDateFormatMap getInstance() {
	return INSTANCE;
	}

	protected ThreadLocal<Map> localDateFormatMap = new ThreadLocal<Map>() {
		
	protected Map initialValue() {
	return new HashMap();
	}

	};
	public DateFormat createSimpleDateFormat(String pattern) {
	DateFormat result = new SimpleDateFormat(pattern);
	putDateFormat(pattern, result);
	return result;
	}

	public DateFormat putDateFormat(String pattern, DateFormat format) {
	return (DateFormat) localDateFormatMap.get().put(pattern, format);
	}

	public DateFormat getDateFormat(String pattern) {
	DateFormat format = (DateFormat) localDateFormatMap.get().get(pattern);
	if (format == null) {
	format = createSimpleDateFormat(pattern);
	}
	return format;
	}
}
import java.lang.management.ManagementFactory;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.BlockingQueue;
import java.util.regex.Matcher;
import org.apache.log4j.Logger;
import org.jfree.data.time.FixedMillisecond;
import org.jfree.data.time.TimeSeriesDataItem;
import logdruid.data.DateFormat;
import logdruid.data.ExtendedTimeSeries;
import logdruid.data.Preferences;
import logdruid.data.Repository;
import logdruid.data.Source;
import logdruid.data.mine.FileLine;
import logdruid.data.mine.FileMineResult;
import logdruid.data.mine.FileRecord;
import logdruid.data.mine.MineData;
import logdruid.data.mine.MineItem;
import logdruid.data.mine.ReportItem;
import logdruid.data.record.EventRecording;
import logdruid.data.record.Recording;
import logdruid.data.record.RecordingItem;
import logdruid.data.record.ReportRecording;
import logdruid.data.record.StatRecording;
import logdruid.util.ClassCache;
import logdruid.util.PatternCache;
import logdruid.util.ThreadLocalDateFormatMap;
package logdruid.engine;




public class MineProcessor implements Runnable {
	private final BlockingQueue<MineItem> queue;
	private final  BlockingQueue<ReportItem> reportQueue;
	private MineData mineData;
	private static Logger logger = Logger.getLogger(MineProcessor.class.getName());
	private int count2 = 0;
	private final static MineItem POISON = new MineItem(null, 0, null, null, null, null, null, false, false, false); 

	public MineProcessor(BlockingQueue<MineItem> queue1, BlockingQueue<ReportItem> _reportQueue, MineData mineData1) {
		queue = queue1;
		mineData = mineData1;
		reportQueue = _reportQueue;
	}
	public void shutdown(){
	    try {
	        queue.put(POISON);
	    } catch (InterruptedException e) {
	        // TODO Auto-generated catch block
	        e.printStackTrace();
	    }
	}
	public void run() {
		while (true) {
			try {
				MineItem mi=queue.take();
	            if (mi == POISON) {
	            	queue.put(mi); // keep in the queue so all workers stop
	                break;
	            }
				consume(mi);

			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}

	private void consume(MineItem mineItem) {
		count2++;
		if (count2 >= 10000) {
			logger.debug("consumed 10k");
			count2 = 0;
		}
		
		int offset = mineItem.getOffset();
		FileRecord fileRecord = mineItem.getFileRecord();
		ArrayList<String[]> dataBlock = mineItem.getDataBlock();
		Map<Recording, String> recMatch1 = mineItem.getRecMatch1();

		Repository repo = mineItem.getRepo();
		Source source = mineItem.getSource();
		boolean stats = mineItem.isStats();
		boolean timings = mineItem.isTimings();
		boolean matches = mineItem.isMatches();

		ExtendedTimeSeries ts = null;