public class DbManager {

    protected Session session = null;    
    protected Logger log = null;    
    protected DbManager()

            throws DbException { 

        this.log = Logger.getLogger(this.getClass());

        try {
            // This step will read hibernate.cfg.xml             
            //    and prepare hibernate for use
            SessionFactory sessionFactory = new Configuration().configure().buildSessionFactory();            
            session = sessionFactory.openSession();
        }
        catch (HibernateException he) {                    
            he.printStackTrace();
            throw new DbException("Failed to initialize DbManager", he);
        }              
    }


    public static DbManager createDbManager(TypedProperties props) 
            throws DbException { 

        DbManager dbm = null;

        if (props.isNoDb() || props.isMailDryRun() ||  props.isMailDryRun2()) {
            dbm = new ReadOnlyDbManager();            
        }    
        else {
            dbm = new DbManager();
        }
        return dbm;
    }


    protected Object loadLongObject (Class<?> clazz,  long objId) 
            throws DbException { 

        Object obj = session.get(clazz, objId);

        if (obj == null) {
            String msg = ("Invalid object reference: " + objId + ".");
            log.error(msg);
            throw new DbException(msg);
        }
        return obj;
    }

    protected Object loadIntegerObject (Class<?> clazz,  int objId) 
            throws DbException { 

        Object obj = session.get(clazz, objId);

        if (obj == null) {
            String msg = ("Invalid object reference: " + objId + ".");
            log.error(msg);
            throw new DbException(msg);
        }
        return obj;
    }


    public Object genericLoadObject(String className, String[] searchFields, Object[] values)
            throws DbException { 

        if (searchFields.length != values.length) {
            throw new DbException("Number of search fields and number of values are not maching");
        }
        if (searchFields.length < 1) {
            throw new DbException("Search fields are empty");
        }

        Object obj = null;

        String hql = "from " + className + " where ";                

        try {

            for (int i=0; i<searchFields.length; i++) {            
                hql = hql + ( i == 0 ? searchFields[i] + "= ?" :  " AND " +  searchFields[i]  + "= ?");
            }
            Query query = session.createQuery(hql);

            for (int i=0; i<searchFields.length; i++) {

                String typeName = values[i].getClass().getSimpleName();

                Method method = null;
                if (typeName.equals("Integer")) {
                    query.setInteger(i, ((Integer)values[i]).intValue());
                }
                else if (typeName.equals("Long")) {
                    query.setLong(i, ((Long)values[i]).longValue());
                }
                else {
                    method = query.getClass().getMethod("set" + typeName, Integer.TYPE, values[i].getClass());
                    Object[] args = {i, values[i]};
                    method.invoke(query, args);
                }                
            }                       
            obj = query.uniqueResult();

        } catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in genericLoadObject", e);
        }

        return obj;
    }



    public Object genericLoadObject(String className, String searchField, Object value)
            throws DbException { 

        Object obj = null;

        Query query = session.createQuery("from " + className + " where " + searchField + "= ?");

        String typeName = value.getClass().getSimpleName();

        try {

            Method method = null;
            if (typeName.equals("Integer")) {
                query.setInteger(0, ((Integer)value).intValue());
            }
            else if (typeName.equals("Long")) {
                query.setLong(0, ((Long)value).longValue());
            } 
            else {
                method = query.getClass().getMethod("set" + typeName, Integer.TYPE, value.getClass());
                Object[] args = {0, value};
                method.invoke(query, args);
            }            

            obj = query.uniqueResult();

        } catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in genericLoadObject", e);
        }

        return obj;

    }

    public void addPriority(Priority prio) 
            throws DbException { 

        try {                
            session.saveOrUpdate(prio);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in addPriority", e);
        }
    }


    public void addEntryType(EntryType entryType) 
            throws DbException { 
        try {                    
            session.saveOrUpdate(entryType);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in addEntryType", e);
        }
    }


    public void addLogEntry(LogEntry logEntry) 
            throws DbException { 
        try {                
            session.save(logEntry);
            session.flush();
            session.evict(logEntry);
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in addLogEntry", e);
        }
    }


    public void updateLogEntry(LogEntry logEntry) 
            throws DbException { 
        try {                
            session.update(logEntry);
            session.flush();
            session.evict(logEntry);
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in updateLogEntry", e);
        }
    }


    @SuppressWarnings("unchecked")
    public List<LogEntry> listLogEntries(long logJobId, int startIndex,
            int noOfRecords) 
                    throws DbException { 
        try {
            Query query = session.createQuery("from LogEntry where Job.Id = ? order by Id asc");
            query.setLong(0, logJobId);
            query.setMaxResults(noOfRecords);
            query.setFirstResult(startIndex);
            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in listLogEntries", e);
        }
    }


    @SuppressWarnings("unchecked")
    public List<LogEntry> listLogEntries(long logJobId, boolean usePrimaryOrg, int startIndex,
            int noOfRecords) 
                    throws DbException { 

        try {
            Query query = null;

            if (usePrimaryOrg) {
                query = session.createQuery("from LogEntry where Job.Id = ? order by Organization, Id asc");
            }
            else {
                query = session.createQuery("from LogEntry where Job.Id = ? order by Organization2, Id asc");
            }
            query.setLong(0, logJobId);
            query.setMaxResults(noOfRecords);
            query.setFirstResult(startIndex);

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in listLogEntries", e);
        }
    }


    /**
     * Returns high priority entries for specified job. Both primary and 
     * secondary organization is checked (org_id and org_id2).
     * 
     * Note, this method contains MySQL specific query parameters.
     * Must be checked if another database is used.
     * 
     * @param prio log entries linked to an organization with a priority equals
     *     or above this value will be included in the result. 
     * 
     * @return High priority entries, sorted by prio.
     */

    @SuppressWarnings({ "deprecation" })
    public List<LogEntry> listLogEntries(long logJobId, int prio, int startIndex, int noOfRecords) throws DbException {

        try {            
            ArrayList<LogEntry> entries = new ArrayList<LogEntry>();

            String sql = "select distinct l.id from log_entry l, organization o, prio p where l.job_id = ? and (l.org_id = o.id or l.org_id2 = o.id) and o.prio_id = p.id and p.prio >= ? order by prio desc limit ?, ?;";    
            PreparedStatement ps = session.connection().prepareStatement(sql);
            ps.setLong(1, logJobId);
            ps.setInt(2, prio);
            ps.setInt(3, startIndex);
            ps.setInt(4, startIndex + noOfRecords);

            boolean result = ps.execute();
            if (result) {
                while (ps.getResultSet().next()) {                        
                    long id = ps.getResultSet().getLong("id");
                    LogEntry entry = (LogEntry)this.loadLongObject(LogEntry.class, id);
                    entries.add(entry);                        
                }                             
            }
            ps.close();     

            return entries;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in listLogEntries", e);
        }                               
    }


    /**
     * Returns the log entry with the latest "created" timestamp, where 
     * ipAddress is equal to specified IP. Null is returned if not found.
     * Only ipAddress is checked, not ipAddress2 or ipRange.
     * 
     * @param ip ip-address for the log entry.
     * @param jobId log entries for this job will be ignored. 
     */
    @SuppressWarnings("deprecation")
    public LogEntry getLastSeenLogEntry(long ip, long jobId) throws DbException {

        LogEntry entry = null;

        try {
            String sql = "select id from log_entry where ip_address = ? and job_id != ? order by created desc";    
            PreparedStatement ps = session.connection().prepareStatement(sql);
            ps.setLong(1, ip);            
            ps.setLong(2, jobId);
            boolean result = ps.execute();
            if (result) {
                if (ps.getResultSet().first()) {
                    long id = ps.getResultSet().getLong("id");
                    entry = (LogEntry)this.loadLongObject(LogEntry.class, id);
                }                
            }
            ps.close();     
        }        
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getLastSeenLogEntry", e);
        }
        return entry;
    }


    /**
     * Returns the mail job with the latest "started" timestamp, which have
     * mailed specified IP. Null is returned if not found.
     * Only ipAddress is checked, not ipAddress2 or ipRange.
     */
    @SuppressWarnings("deprecation")
    public MailJob getLastSeenMailJob(long ip) throws DbException {

        MailJob mailJob = null;

        try {
            String sql = "select mj.id from mail_job mj, mail_job_log_entry_mapping mjlem, log_entry le " + 
                    "where mjlem.log_entry_id = le.id and le.ip_address = ? and mjlem.mail_job_id = mj.id order by mj.started desc";

            PreparedStatement ps = session.connection().prepareStatement(sql);
            ps.setLong(1, ip);                   
            boolean result = ps.execute();
            if (result) {
                if (ps.getResultSet().first()) {
                    long id = ps.getResultSet().getLong("id");
                    mailJob = (MailJob)this.loadLongObject(MailJob.class, id);
                }                
            }
            ps.close();     
        }       
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getLastSeenMailJob", e);
        }
        return mailJob;
    }


    public void addOrganization(Organization org, String modifiedBy) 
            throws DbException { 
        try {
            java.util.Date date = new java.util.Date();
            long time = SqlUtil.convertTimestamp(date);        
            org.setLastModified(time);
            org.setModifiedBy(modifiedBy);
            if (org.getCreated() == null) {
                org.setCreated(time);            
            }      
            session.save(org);
            session.flush();
        } 

        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " not possible to add organiation", e);
        }
    }



    public void updateOrganization(Organization org, String modifiedBy) 
            throws DbException { 
        try {
            java.util.Date date = new java.util.Date();
            long time = SqlUtil.convertTimestamp(date);        
            org.setLastModified(time);
            org.setModifiedBy(modifiedBy);
            session.update(org);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in updateOrganization", e);
        }
    }


    @SuppressWarnings("unchecked")
    public List<Organization> searchOrganizations(String name, int startIndex,
            int noOfRecords) 
                    throws DbException { 
        try {
            Query query = session.createQuery("from Organization where name like ?");
            query.setString(0, "%" + name +"%");

            query.setMaxResults(noOfRecords);
            query.setFirstResult(startIndex);

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrganizations", e);
        }
    }


    public Organization searchOrganization(String name) 
            throws DbException { 
        try {
            Query query = session.createQuery("from Organization where name = ?");
            query.setString(0, name);

            Organization org = (Organization)query.uniqueResult();

            return org;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrganization", e);
        }

    }


    @SuppressWarnings("unchecked")
    public List<Organization> listOrganizations(int startIndex, int noOfRecords) 
            throws DbException { 

        try {
            Query query = session.createQuery("from Organization order by Name asc");        
            query.setMaxResults(noOfRecords);
            query.setFirstResult(startIndex);                

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in listOrganizations", e);
        }

    }


    public Organization getOrganization(int orgId) 
            throws DbException { 

        try {
            Organization org = (Organization)loadIntegerObject(Organization.class, orgId);

            return org;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getOrganization", e);
        }

    }


    public void addMailJob(MailJob mailJob) 
            throws DbException { 

        try {
            mailJob.setStarted(SqlUtil.convertTimestamp(new Date()));
            session.saveOrUpdate(mailJob);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in addMailJob", e);
        }
    }


    /*        
        The updateMailJob is deprecated due to a problem when persisting the MailJob object after
        a change in the LogEntry collection. The reason could be due to some missconfiguration in the
        Hibernate object mapping for MailJob or a bug in Hibernate. 
        Use finishMailJob to persist the MailJob once it has been completed.      
     */
    @Deprecated
    public void updateMailJob(MailJob mailJob) 
            throws DbException { 

        try {
            session.update(mailJob);        
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in updateMailJob", e);
        }
    }


    public void finishMailJob(MailJob mailJob, String errorMsg) 
            throws DbException { 

        try {
            mailJob.setErrorMsg(errorMsg);
            mailJob.setFinished(SqlUtil.convertTimestamp(new Date()));
            session.update(mailJob);            
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in finishMailJob", e);
        }
    }


    public long deleteMailJob(MailJob mailJob) 
            throws DbException { 

        try {
            Query query = session.createSQLQuery(
                    "SELECT COUNT(*) FROM mail_job_log_entry_mapping WHERE mail_job_id = ?");
            query.setLong(0, mailJob.getId());
            long noLogEntriesInJob = ((BigInteger)query.list().get(0)).longValue();

            session.delete(mailJob);
            session.flush();

            long entriesLeft = ((BigInteger)query.list().get(0)).longValue();

            return noLogEntriesInJob - entriesLeft;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in deleteMailJob", e);
        }
    }


    // Search MailJob by time frame
    @SuppressWarnings("unchecked")
    public List<MailJob> searchMailJobs(Date startTime,
            Date endTime, int startIndex, int noOfRecords) 
                    throws DbException { 
        try {
            Query query = session.createQuery("from MailJob where started >= ? and started <= ? order by Id asc");
            query.setLong(0, SqlUtil.convertTimestamp(startTime));
            query.setLong(1, SqlUtil.convertTimestamp(endTime));
            query.setMaxResults(noOfRecords);
            query.setFirstResult(startIndex);

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchMailJobs", e);
        }
    }


    // Search MailJob by Job entity
    @SuppressWarnings("unchecked")
    public List<MailJob> searchMailJobs(Job job) 
            throws DbException { 

        try {
            Query query = session.createQuery("from MailJob where Job = ?");
            query.setEntity(0, job);

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchMailJobs", e);
        }
    }

    
    // Search MailJob by log entry
    public MailJob searchMailJob(LogEntry entry)
        throws DbException {
        
        try {
            Query query = session.createSQLQuery("select mail_job_id from mail_job_log_entry_mapping where log_entry_id = ?");
            query.setLong(0, entry.getId());
          
            Object result = query.uniqueResult();
            MailJob mailJob = null;
            if (result != null) {
                Long mailJobId = Long.valueOf(result.toString());
                mailJob = (MailJob)this.loadLongObject(MailJob.class, mailJobId.longValue());
            }    
            return mailJob;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchMailJob", e);
        }
    }


    public void addJobType(JobType jobType) 
            throws DbException { 

        try {                
            session.saveOrUpdate(jobType);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in addJobType", e);
        }
    }

    public void addLogJob(Job job) 
            throws DbException {

        try {
            session.save(job);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in addLogJob", e);
        }
    }

    public boolean existsFileHash(String hash)
            throws DbException {

        try {
            Query query = session.createQuery("from Job where FileHash = ?");
            query.setString(0, hash);

            return query.list().size() > 0;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in existsFileHash", e);
        }
    }

    @SuppressWarnings("unchecked")
    public List<Job> searchLogJobs(Date startTime, Date endTime, int startIndex,
            int noOfRecords) 
                    throws DbException { 

        try {

            Query query = session.createQuery("from Job where started >= ? and started <= ? order by Id asc");
            query.setLong(0, SqlUtil.convertTimestamp(startTime));
            query.setLong(1, SqlUtil.convertTimestamp(endTime));
            query.setMaxResults(noOfRecords);
            query.setFirstResult(startIndex);

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchLogJobs", e);
        }

    }                           


    public Job searchLogJob(String name)
            throws DbException { 

        try {
            Query query = session.createQuery("from Job where name = ?");
            query.setString(0, name);

            return (Job)query.uniqueResult();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchLogJob", e);
        }

    } 


    public void finishLogJob(Job job, String errorMsg) 
            throws DbException { 

        try {
            job.setFinished(SqlUtil.convertTimestamp(new Date()));
            job.setErrorMsg(errorMsg);
            session.update(job);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in finishLogJob", e);
        }

    }


    @SuppressWarnings("deprecation")
    public long deleteLogJob(Job job) 
            throws DbException {

        long jobId = job.getId();        


        // Use JDBC connection to delete additional_item, free_text & original_log_entry
        try {

            // additional_item
            String sqlDelete = "delete from additional_item where log_entry_id in (select id from log_entry where job_id = ?)";       
            PreparedStatement ps = session.connection().prepareStatement(sqlDelete);
            ps.setLong(1, jobId);
            ps.execute();
            ps.close();        

            // free_text
            sqlDelete = "delete from free_text where log_entry_id in (select id from log_entry where job_id = ?)";
            ps = session.connection().prepareStatement(sqlDelete);
            ps.setLong(1, jobId);
            ps.execute();
            ps.close();

            // original_log_entry
            sqlDelete = "delete from original_log_entry where id in (select id from log_entry where job_id = ?)";
            ps = session.connection().prepareStatement(sqlDelete);
            ps.setLong(1, jobId);
            ps.execute();
            ps.close();

        }
        catch(SQLException sqle) {
            throw handleException("SQL exception in deleteLogJob", sqle);
        }


        try {
            // Delete LogEntry using hql
            String hqlDelete = "delete LogEntry where Job.Id = ?";
            long  numberOfDeletedLogEntries = session.createQuery( hqlDelete ).setLong(0, job.getId()).executeUpdate();

            session.evict(job);
            Job realJob = (Job)loadLongObject(job.getClass(), jobId);

            // Delete Job
            session.delete(realJob);        
            session.flush();


            return numberOfDeletedLogEntries;
        }

        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in deleteLogJob", e);
        }

    }


    public JobType searchJobType(String name) 
            throws DbException { 

        try {
            Query query = session.createQuery("from JobType where name = ?");
            query.setString(0, name);

            return (JobType)query.uniqueResult();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchJobType", e);
        }

    }


    public EntryType searchEntryType(String name) 
            throws DbException { 

        try {

            Query query = session.createQuery("from EntryType where name = ?");
            query.setString(0, name);

            return (EntryType)query.uniqueResult();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchEntryType", e);
        }

    }


    public Priority getPriority(int prio)
            throws DbException { 
        try {
            Query query = session.createQuery("from Priority where prio = ?");
            query.setInteger(0, prio);

            return (Priority)query.uniqueResult();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getPriority", e);
        }

    }


    @SuppressWarnings("unchecked")
    public List<Priority> getAllPriorities()
            throws DbException { 

        try {
            Query query = session.createQuery("from Priority order by Prio desc");


            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getAllPriorities", e);
        }

    }


    @SuppressWarnings("unchecked")
    public List<IpRange> getAllIpRanges(boolean includeDisabledOrgs)
            throws DbException { 

        try {

            Query query = null;
            if (includeDisabledOrgs) {
                query = session.createQuery("from IpRange");
            }
            else {
                query = session.createQuery("from IpRange where OrganizationId in (from Organization where Enabled = ?)");
                query.setBoolean(0, true);
            }

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getAllIpRanges", e);
        }

    }


    @SuppressWarnings("unchecked")
    public List<ASNumber> getAllASNumbers(boolean includeDisabledOrgs)
            throws DbException { 

        try {

            Query query = null;
            if (includeDisabledOrgs) {
                query = session.createQuery("from ASNumber");
            }
            else {
                query = session.createQuery("from ASNumber where OrganizationId in (from Organization where Enabled = ?)");
                query.setBoolean(0, true);
            }

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getAllASNumbers", e);
        }

    }


    @SuppressWarnings("unchecked")
    public List<DomainName> getAllDomainNames(boolean includeDisabledOrgs)
            throws DbException { 

        try {

            Query query = null;
            if (includeDisabledOrgs) {
                query = session.createQuery("from DomainName");
            }
            else {
                query = session.createQuery("from DomainName where OrganizationId in (from Organization where Enabled = ?)");
                query.setBoolean(0, true);
            }
            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getAllDomainNames", e);
        }

    }

    // Gets the organization with the given domain name
    public Organization getOrgByDomainName(String domainName)
            throws DbException { 

        try {

            Organization org = null;
            // Ugly to step search...
            Query query = session.createQuery("from DomainName where Name = ?");
            query.setString(0, domainName);

            DomainName dName = (DomainName) query.uniqueResult();

            if (dName != null) {
                org = (Organization)loadIntegerObject(Organization.class, dName.getOrganizationId());               
            }
            return org;

        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrgByDomainName", e);
        }

    }

    // Searches for organizations with domain names like the input name
    public List<Organization> searchOrgForDomainName(String domainName)
            throws DbException { 

        try {                       
            //Query query = session.createQuery("OrganizationId from DomainName where Name like '%?%'");
            //query.setString(0, domainName);

            String sql = "SELECT DISTINCT org_id FROM domain_name WHERE domain_name LIKE ? ORDER BY org_id ASC";            
            Query query = session.createSQLQuery(sql);
            query.setString(0, "%" + domainName + "%");
            @SuppressWarnings("unchecked")
            List<Object> orgIDs = query.list();
            List<Organization> orgs = new ArrayList<Organization>();    

            Iterator<Object> itr = orgIDs.iterator();           
            while (itr.hasNext()) { 
                Organization org = getOrganization(((Integer)itr.next()).intValue());
                if (org != null) {
                    orgs.add(org);
                }               
            }           
            return orgs;            
            
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrgForDomainName", e);
        }        
    }
    
    
    public List<Organization> searchOrganizationByPrioNumber(Integer prioNumber) 
            throws DbException { 
        
        try { 
            String sql = "SELECT DISTINCT o.id FROM organization o, prio p WHERE o.prio_id = p.id and p.prio = ? ORDER BY id ASC";
            Query query = session.createSQLQuery(sql);
            query.setInteger(0, prioNumber);
            
            @SuppressWarnings("unchecked")
            List<Object> orgIDs = query.list();
            List<Organization> orgs = new ArrayList<Organization>();    

            Iterator<Object> itr = orgIDs.iterator();           
            while (itr.hasNext()) { 
                Organization org = getOrganization(((Integer)itr.next()).intValue());
                if (org != null) {
                    orgs.add(org);
                }               
            }           
            return orgs;            
            
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrgByPrioNumber", e);
        }  
    }

    public Organization searchOrganizationByASNumber(int asn)
            throws DbException { 

        try {

            Organization org = null;
            // Ugly two step search...
            Query query = session.createQuery("from ASNumber where Number = ?");
            query.setInteger(0, asn);

            ASNumber asnum = (ASNumber) query.uniqueResult();

            if (asnum != null) {
                org = (Organization)loadIntegerObject(Organization.class, asnum.getOrganizationId());               
            }
            return org;

        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrgByASNumber", e);
        }

    }


    @SuppressWarnings("unchecked")
    public List<Organization> searchOrganizationsByIPrange(long start, long end) 
            throws DbException { 
        try {
            String sql = "SELECT DISTINCT org_id FROM ip_range WHERE (end_address >= ? AND end_address <= ?) OR (start_address <=  ? AND end_address >= ?) OR  (start_address >= ? AND start_address <= ?) OR (start_address >= ? AND end_address <= ?) ORDER BY org_id ASC";

            Query query = session.createSQLQuery(sql);
            query.setLong(0, start);   
            query.setLong(1, end);
            query.setLong(2, start);
            query.setLong(3, end);
            query.setLong(4, start);
            query.setLong(5, end);
            query.setLong(6, start);
            query.setLong(7, end);


            List<Object> orgIDs = query.list();
            List<Organization> orgs = new ArrayList<Organization>();    

            Iterator<Object> itr = orgIDs.iterator();           
            while (itr.hasNext()) { 
                Organization org = getOrganization(((Integer)itr.next()).intValue());
                if (org != null) {
                    orgs.add(org);
                }               
            }           
            return orgs;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrgsByIPrange", e);
        }           
    }   


    @SuppressWarnings("unchecked")
    public List<LogEntry> searchLogEntriesByIPrange(long start, long end) 
            throws DbException { 
        try {
            //String sql = "SELECT id FROM log_entry where (ip_address >= ? AND ip_address <= ?) OR (ip_address2 >= ? AND ip_address2 <= ?)";
            Query query = session.createQuery("from LogEntry where (IpAddress >= ? AND IpAddress <= ?) OR (IpAddress2 >= ? AND IpAddress2 <= ?)");

            query.setLong(0, start);   
            query.setLong(1, end);
            query.setLong(2, start);
            query.setLong(3, end);


            /*List<Object> entryIDs = query.list();
                List<LogEntry> entries = new ArrayList<LogEntry>(); 

                Iterator<Object> itr = entryIDs.iterator();         
                while (itr.hasNext()) {                                 
                    LogEntry entry = (LogEntry)loadLongObject(LogEntry.class, ((Long)itr.next()).longValue());                                  

                    if (entry != null) {
                        entries.add(entry);
                    }               
                }           */
            return query.list();
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchLogEntriesByIPrange", e);
        }           
    }   

    @SuppressWarnings("unchecked")
    public List<LogEntry> searchLogEntriesByOrgId(int orgId)            
            throws DbException { 
        try {
            Query query = session.createQuery("from LogEntry where Organization = ?");
            query.setInteger(0, orgId);

            return query.list();
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchLogEntriesByOrgId", e);
        }       
    }

    @SuppressWarnings("unchecked")
    public List<LogEntry> searchLogEntriesByASNumber(long asn) 
            throws DbException { 
        try {

            Query query = session.createQuery("from LogEntry where Asn = ? or Asn2 = ?");           
            query.setLong(0, asn);   
            query.setLong(1, asn);

            return query.list();
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchLogEntriesByASNumber", e);
        }           
    }   


    @SuppressWarnings("unchecked")
    public List<Organization> searchOrganizationByEmailAddress(String address) 
            throws DbException {

        List<Organization> orgs = new ArrayList<Organization>();

        try {
            Query query = session.createQuery("from Contact where EmailAddress like ?");
            query.setString(0, "%" + address +"%");
            List<Contact> contacts = query.list();
            for (Contact contact: contacts) {   
                Organization org = (Organization)loadIntegerObject(Organization.class, contact.getOrganizationId());                
                if (orgs.contains(org) == false) {              
                    orgs.add(org);
                }
            }       
            return orgs;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrganizationByEmailAddress", e);
        }

    }


    public long getNumberOfLogEntries()
            throws DbException { 
        try {

            Query query = session.createSQLQuery(
                    "SELECT COUNT(*) FROM log_entry");

            return ((BigInteger)query.list().get(0)).longValue();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfLogEntries", e);
        }

    }


    public boolean existsMailForIp(long ip, Organization organization, long periodInSecs, boolean usePrimaryOrg)
            throws DbException { 

        try {        
            String orgIdColName = "org_id";
            String ipAddrColName = "ip_address";
            if (usePrimaryOrg != true) {
                orgIdColName = "org_id2";
                ipAddrColName = "ip_address2";
            }

            String select = "SELECT COUNT(*) FROM log_entry le, mail_job_log_entry_mapping map, mail_job mj where mj.started >= ? AND mj.finished <= ? AND mj.id = map.mail_job_id AND map.log_entry_id = le.id AND le." + orgIdColName + " = ? AND le." + ipAddrColName + " = ?";                  
            Query query = session.createSQLQuery(select); 

            long endTime = SqlUtil.convertTimestamp(new Date());        
            long startTime = endTime - periodInSecs;

            query.setLong(0, startTime);
            query.setLong(1, endTime);    
            query.setInteger(2, organization.getId());
            query.setLong(3, ip);

            return ((BigInteger)query.list().get(0)).intValue() > 0 ? true:false;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in existsMailForIp", e);
        }

    }


    public DbStatisticsData fetchStatistics(Date startDate, Date endDate)
            throws DbException { 
        try {

            DbStatisticsData dbStats = new DbStatisticsData(startDate, endDate);

            dbStats.setNoOfSuccessfulJobs(getNumberOfSuccessfulLogJobs(startDate, endDate));
            dbStats.setNoOfFailedJobs(getNumberOfFailedLogJobs(startDate, endDate));
            dbStats.setNoOfProcessedLines(getNumberOfProcessedLines(startDate, endDate));
            dbStats.setNoOfLogEntries(getNumberOfLogEntries(startDate, endDate));
            dbStats.setNoOfMatchedLogEntries(getNumberOfMatchedLogEntries(startDate, endDate));
            dbStats.setNoOfSentEmails(getNumberOfSentEmails(startDate, endDate));        
            dbStats.setNoOfNewOrganizations(getNumberOfNewOrganizations(startDate, endDate));
            dbStats.setNoOfModifiedOrganizations(getNumberOfModifiedOrganizations(startDate, endDate));

            return dbStats;

        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in fetchStatistics", e);
        }

    }

    /**
     * Fetches number of processed lines for successful jobs per job type.
     * 
     * @return Map with result, where key is job type and value is no. 
     *     of processed lines.
     */
    @SuppressWarnings("unchecked")
    public Map<String, Long> fetchNoOfProcessedLinesPerEntryType(Date startDate, Date endDate) 
            throws DbException { 
        try {

            Map<String, Long> result = new HashMap<String, Long>();

            String select = "SELECT et.name, sum(j.processed_lines) FROM job j, job_type jt, entry_type et WHERE job_type_id = jt.id AND jt.entry_type_id = et.id AND finished >= ? AND finished <= ? group by et.name;";

            Query query = session.createSQLQuery(select);        
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            List<Object[]> resultSet = query.list();

            if (resultSet != null) {
                for (Object[] obj: resultSet) {                
                    result.put(new String(obj[0].toString()), new Long(((java.math.BigDecimal)obj[1]).longValue()));                
                }
            }    

            return result;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in fetchNoOfProcessedLinesPerJobType", e);
        }
    }


    /**
     * Fetches number of log entries for successful jobs per job type.
     * 
     * @param onlyMatchedEntries If true, count only log entries where organization and 
     *     organization2 is not null. If false, count all log entries. 
     * 
     * @return Map with result, where key is job type and value is no. 
     *     of processed lines.
     */
    @SuppressWarnings("unchecked")
    public Map<String, Long> fetchNoOfLogEntriesPerEntryType(Date startDate, Date endDate, boolean onlyMatchedEntries) 
            throws DbException { 

        try {

            Map<String, Long> result = new HashMap<String, Long>();

            String select = "SELECT et.name, count(le.id) FROM log_entry le, job j, job_type jt, entry_type et WHERE le.job_id = j.id AND j.job_type_id = jt.id AND jt.entry_type_id = et.id AND j.finished >= ? AND j.finished <= ?";

            if (onlyMatchedEntries) {
                select = select + " AND (le.org_id IS NOT null OR le.org_id2 IS NOT null)";
            }
            select = select + " GROUP BY jt.entry_type_id";

            Query query = session.createSQLQuery(select);        
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            List<Object[]> resultSet = query.list();

            if (resultSet != null) {
                for (Object[] obj: resultSet) {                
                    result.put(new String(obj[0].toString()), new Long(((BigInteger)obj[1]).longValue()));                
                }
            }    

            return result;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in fetchNoOfLogEntriesPerJobType", e);
        }

    }


    /**
     * Fetches log entries for use in GeolocationXmlReportGenerator. Hibernate 
     * is not used due to memory constraints and complexity of the query.  
     * 
     * @param jobTypeKillList list if job types to exclude (the db-field "job_type.name").
     */
    @SuppressWarnings("deprecation")
    public List<LogEntry> fetchLogEntriesForGeolocation(Date startDate, Date endDate, List<String> jobTypeKillList) throws DbException {
        List<LogEntry> result = new ArrayList<LogEntry>(2048);
        PreparedStatement ps = null;
        try {
            // -- build query
            String queryStr = 
                    "select log_entry.id, log_timestamp, log_entry.created, ip_address, ip_range_start, port, hostname, asn, prio.name, additional_item.name, additional_item.value " +
                            "from (log_entry, job, job_type) " +
                            "left join additional_item on log_entry.id = additional_item.log_entry_id " +
                            "left join organization on log_entry.org_id = organization.id " +
                            "left join prio on organization.prio_id = prio.id " + 
                            "where " +
                            "log_entry.log_timestamp >= ? and log_entry.log_timestamp <= ? and " +
                            "log_entry.job_id = job.id and " +
                            "job.job_type_id = job_type.id and " +
                            "job_type.name not in (@inList@) " + 
                            "order by log_entry.log_timestamp, log_entry.id asc;";
            if ((jobTypeKillList == null) || jobTypeKillList.isEmpty()) {
                jobTypeKillList = new ArrayList<String>();
                jobTypeKillList.add("");
            }
            String inList = StringUtil.toSingleQuotedString(jobTypeKillList);
            queryStr = StringUtil.replace(queryStr, "@inList@", inList);

            // -- execute query
            log.debug("Executing query: " + queryStr);
            long t1 = System.currentTimeMillis();
            ps = session.connection().prepareStatement(queryStr);
            ps.setLong(1, SqlUtil.convertTimestamp(startDate));
            ps.setLong(2, SqlUtil.convertTimestamp(endDate));
            boolean successful = ps.execute();
            String durationStr = DateUtil.formatDuration(System.currentTimeMillis() - t1);
            log.debug("Query executed. Time: " + durationStr);

            // -- read result set; only one LogEntry per ip-address.
            final String dbIdKey = "dbId";
            final String timesSeenKey = "timesSeen";
            final String lastSeenKey = "lastSeen";
            final String prioNameKey = "prioName";
            final String infectionKey = "infection";
            if (successful) {
                long id = 0L;
                Map<Long, LogEntry> ipAddressMap = new HashMap<Long, LogEntry>();
                ResultSet resultSet = ps.getResultSet(); 
                while (resultSet.next()) {
                    Long ipAddress = resultSet.getLong(4);
                    if (ipAddress == null) {
                        ipAddress = resultSet.getLong(5);
                        if (ipAddress == null) {
                            continue;
                        }
                    }
                    LogEntry logEntry = ipAddressMap.get(ipAddress);
                    if (logEntry != null) {
                        int timesSeen = Integer.parseInt(logEntry.getAdditionalItems().get(timesSeenKey)); 
                        logEntry.getAdditionalItems().put(timesSeenKey, Integer.toString(++timesSeen));
                        long lastSeen = Long.parseLong(logEntry.getAdditionalItems().get(lastSeenKey));
                        long maxLastSeen = Math.max(lastSeen, resultSet.getLong(2));
                        logEntry.getAdditionalItems().put(lastSeenKey, Long.toString(maxLastSeen));
                    } 
                    else {
                        logEntry = new LogEntry(id++);
                        ipAddressMap.put(ipAddress, logEntry);
                        result.add(logEntry);
                        if (logEntry.getAdditionalItems() == null) {
                            logEntry.setAdditionalItems(new HashMap<String, String>());
                        }
                        logEntry.setIpAddress(ipAddress);
                        // logTimestamp == firstSeen
                        logEntry.setLogTimestamp(resultSet.getLong(2));
                        logEntry.setCreated(resultSet.getLong(3));
                        logEntry.setPort(resultSet.getInt(6));
                        logEntry.setHostname(resultSet.getString(7));
                        logEntry.setAsn(resultSet.getLong(8));
                        logEntry.getAdditionalItems().put(dbIdKey, Long.toString(resultSet.getLong(1))); 
                        logEntry.getAdditionalItems().put(timesSeenKey, "1");
                        logEntry.getAdditionalItems().put(lastSeenKey, logEntry.getLogTimestamp().toString());
                        logEntry.getAdditionalItems().put(prioNameKey, StringUtil.getNotNull(resultSet.getString(9)));
                    }
                    // add infection if missing
                    String infection = logEntry.getAdditionalItems().get(infectionKey);
                    if (StringUtil.isNullOrEmpty(infection)) {
                        infection = null;
                        String name = resultSet.getString(10);
                        if ((name != null) && name.equals(infectionKey)) {
                            infection = resultSet.getString(11);
                            if ((infection != null) && (infection.equals("irc") || infection.equalsIgnoreCase("sinkhole") || infection.equalsIgnoreCase("other"))) {
                                infection = null;
                            }
                        }
                        logEntry.getAdditionalItems().put(infectionKey, StringUtil.getNotNull(infection));
                    }
                }
            }
        } catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in fetchLogEntriesForGeolocation", e);
        } finally {
            try { if (ps != null) ps.close(); } catch (SQLException ignored) { }
        }

        return result;
    }


    /**
     * Fetches log entries for specified organization for use in OrganizationReportGenerator.
     * 
     * @param jobTypes list of job types (db-field "job_type.name") for log entries to be included in the result.
     */
    @SuppressWarnings("deprecation")
    public List<LogEntry> fetchLogEntriesForOrganizationReport(int orgId, Date startDate, Date endDate, List<String> jobTypes) 
            throws DbException {
        List<LogEntry> entries = new ArrayList<LogEntry>(2048);
        PreparedStatement ps = null;
        if ((jobTypes == null) || jobTypes.isEmpty()) {
            return entries;
        }

        try {
            String inList = StringUtil.toSingleQuotedString(jobTypes);
            String sql = 
                    "SELECT le.id from log_entry le, job j, job_type jt " +
                            "WHERE j.finished >= ? AND j.finished <= ? AND j.job_type_id = jt.id AND jt.name in (" + inList + ") AND le.job_id = j.id AND le.org_id = ?;";

            log.debug("Executing query: " + sql);
            ps = session.connection().prepareStatement(sql);            
            ps.setLong(1, SqlUtil.convertTimestamp(startDate));
            ps.setLong(2, SqlUtil.convertTimestamp(endDate));
            ps.setInt(3, orgId);
            boolean successful = ps.execute();

            if (successful) {
                ResultSet resultSet = ps.getResultSet(); 
                while (resultSet.next()) {
                    long entryID = resultSet.getLong(1);
                    LogEntry entry = (LogEntry)this.loadLongObject(LogEntry.class, entryID);
                    entries.add(entry);
                }
            }
        } catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in fetchLogEntriesForOrganizationReport", e);
        } finally {
            try { if (ps != null) ps.close(); } catch (SQLException ignored) { }
        }

        return entries;
    }


    public long getNumberOfSuccessfulLogJobs(Date startDate, Date endDate) 
            throws DbException {
        try {
            /** No. of successful log jobs (excluding mail jobs). */
            String sql = "SELECT COUNT(*) FROM job WHERE finished >= ? AND finished <= ? AND error_msg IS NULL";
            Query query = session.createSQLQuery(sql);

            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return (query.list() != null) ? ((BigInteger)query.list().get(0)).longValue() : 0;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfSuccessfulLogJobs", e);
        }

    }


    public long getNumberOfFailedLogJobs(Date startDate, Date endDate) 
            throws DbException {
        try {    
            /** No. of failed log jobs (excluding mail jobs). */
            String sql = "SELECT COUNT(*) FROM job WHERE started >= ? AND started <= ? AND finished is null";
            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return (query.list() != null) ? ((BigInteger)query.list().get(0)).longValue() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfFailedLogJobs", e);
        }   
    }


    public long getNumberOfProcessedLines(Date startDate, Date endDate) 
            throws DbException { 
        try {
            /** No. of processed lines (after possible split or merge). */
            String sql = "SELECT SUM(processed_lines) FROM job WHERE finished >= ? AND finished <= ?";
            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return ((query.list() != null) && (query.list().get(0) != null)) ? ((java.math.BigDecimal)query.list().get(0)).longValue() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfProcessedLines", e);
        }
    }


    public long getNumberOfLogEntries(Date startDate, Date endDate) 
            throws DbException { 
        try {

            /** No. of processed lines (after possible split or merge). */
            String sql = "SELECT COUNT(*) FROM job j, log_entry l WHERE j.finished >= ? AND j.finished <= ? AND j.id = l.job_id" ;
            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return (query.list() != null) ? ((BigInteger)query.list().get(0)).longValue() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfLogEntries", e);
        }
    }


    public long getNumberOfMatchedLogEntries(Date startDate, Date endDate) 
            throws DbException { 
        try {

            /** No. of log entries that have been matched to an organisation. */
            String sql = "SELECT COUNT(*) FROM job j, log_entry l WHERE j.finished >= ? AND j.finished <= ? AND j.id = l.job_id AND (org_id IS NOT NULL or org_id2 IS NOT NULL)" ;
            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return (query.list() != null) ? ((BigInteger)query.list().get(0)).longValue() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfMatchedLogEntries", e);
        }
    }


    public long getNumberOfSentEmails(Date startDate, Date endDate) 
            throws DbException { 
        try {

            /** No. of sent emails in all mail jobs. */
            String sql = "SELECT DISTINCT le.org_id FROM log_entry le, mail_job mj, mail_job_log_entry_mapping mjle WHERE le.id = mjle.log_entry_id AND mj.id = mjle.mail_job_id AND mj.started >= ? AND mj.finished <= ?;";

            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            List<?> rs = query.list();

            return rs != null ? (long)rs.size() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfSentEmails", e);
        }
    }


    public long getNumberOfNewOrganizations(Date startDate, Date endDate) 
            throws DbException { 
        try {
            /** No. of created organizations. */        
            String sql = "SELECT COUNT(*) FROM organization WHERE created >= ? AND created <= ?";
            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return (query.list() != null) ? ((BigInteger)query.list().get(0)).longValue() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfNewOrganizations", e);
        }
    }


    public long getNumberOfModifiedOrganizations(Date startDate, Date endDate) 
            throws DbException { 
        try {    
            /** No. of modified organizations. */        
            String sql = "SELECT COUNT(*) FROM organization WHERE last_modified >= ? AND last_modified <= ? AND created <  last_modified";
            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return (query.list() != null) ? ((BigInteger)query.list().get(0)).longValue() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfModifiedOrganizations", e);
        }   
    }


    public boolean isRangeOverlapping(long start, long end) 
            throws DbException { 
        try {
            String sql = "SELECT COUNT(*) FROM ip_range WHERE (end_address >= ? AND end_address <= ?) OR (start_address <=  ? AND end_address >= ?) OR  (start_address >= ? AND start_address <= ?) OR (start_address >= ? AND end_address <= ?)";

            Query query = session.createSQLQuery(sql);
            query.setLong(0, start);   
            query.setLong(1, end);
            query.setLong(2, start);
            query.setLong(3, end);
            query.setLong(4, start);
            query.setLong(5, end);
            query.setLong(6, start);
            query.setLong(7, end);

            long result = ((BigInteger)query.list().get(0)).longValue();

            return query.list() != null && result > 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in isRangeOverlapping", e);
        }           
    }


    @SuppressWarnings("unchecked")
    public List<BigInteger> isRangeOverlappingOrgId(long start, long end)
            throws DbException {

        try {
            String sql = "SELECT ORG_ID FROM ip_range WHERE (end_address >= ? AND end_address <= ?) OR (start_address <=  ? AND end_address >= ?) OR  (start_address >= ? AND start_address <= ?) OR (start_address >= ? AND end_address <= ?)";

            Query query = session.createSQLQuery(sql);
            query.setLong(0, start);
            query.setLong(1, end);
            query.setLong(2, start);
            query.setLong(3, end);
            query.setLong(4, start);
            query.setLong(5, end);
            query.setLong(6, start);
            query.setLong(7, end);

            return query.list();

        } catch (Exception e) {
            throw handleException(e.getClass().getSimpleName()
                    + " exception in isRangeOverlappingOrgId", e);
        }
    }


    @SuppressWarnings("unchecked")
    public List<Contact> searchContactsByOrgIDsAndRoles(String[] orgIDs, String roles) 
            throws DbException {

        /*
         * Searches for contacts by orgId (if present) and roles (if present).
         * If no criterias are given, all contacts are returned.
         */
        ArrayList<Contact> contacts = new ArrayList<Contact>();
        Query query = null;


        if (orgIDs != null && orgIDs.length > 0) {
            for (String orgID : orgIDs) {           
                if (roles != null && roles.isEmpty() == false) {
                    // Search by orgId and roles
                    query = session.createQuery("from Contact where OrganizationId = ? and Role in (:roles)");              
                    query.setParameterList("roles", roles.split(","));

                    //System.out.println("1" + query.getQueryString() + " roles: " + roles);
                }   
                else {
                    // Search by orgId only
                    query = session.createQuery("from Contact where OrganizationId = ?");       
                    //System.out.println("2" + query.getQueryString());

                }
                query.setInteger(0, Integer.parseInt(orgID));
                //System.out.println("3" + query.getQueryString());
                //System.out.println("Found " + query.list().size() + " number of contacts.");
                if (query.list().isEmpty() == false) {              
                    contacts.addAll((List<Contact>)query.list());
                }
            }
        }
        else {

            if (roles != null && roles.isEmpty() == false) {    
                // Search by roles only
                query = session.createQuery("from Contact where Role in (:roles)");             
                query.setParameterList("roles", roles.split(","));
            }
            else {
                // Search all contacts
                query = session.createQuery("from Contact");
            }

            contacts.addAll((List<Contact>)query.list());
        }
        return contacts;

    }


    @SuppressWarnings("unchecked")
    public List<Contact>  searcContactsByPrioAndRoles(String prioHigh, String prioLow, String searchRoles)
            throws DbException {

        ArrayList<Contact> contacts = new ArrayList<Contact>();

        Query query = null;
        query = session.createQuery("from Organization where Priority.Prio <= ? and Priority.Prio >= ?");
        query.setString(0, prioHigh);
        query.setString(1, prioLow);

        if (query.list() != null) {         
            List<Organization> orgs = query.list();         
            Iterator<Organization> itr = orgs.iterator();           
            while (itr.hasNext()) {                     
                Set<Contact> orgContacts = ((Organization)itr.next()).getContacts();

                if (orgContacts != null && orgContacts.isEmpty() == false) {
                    if (searchRoles != null && searchRoles.isEmpty() == false) {
                        for (Contact contact : orgContacts) {
                            if (contact != null && contact.getRole() != null && searchRoles.contains(contact.getRole())) {

                                contacts.add(contact);                              
                            }
                        }
                    }
                    else {
                        contacts.addAll(orgContacts);
                    }
                }                               
            }                       
        }       

        return contacts;


    }

    @SuppressWarnings("unchecked")
    public List<String> getAllContactRoleNames() 
            throws DbException {
        try {
            String sql = "SELECT DISTINCT role FROM contact where role is not null ORDER BY role";
            Query query = session.createSQLQuery(sql);

            return (List<String>)query.list();

        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getAllContactRoleNames", e);
        }

    }


    public void genericDelete(Object obj) 
            throws DbException {
        try {
            session.delete(obj);
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in genericDelete", e);
        }
    }


    public void flushSession()
            throws DbException {
        try {    
            session.flush();
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in flushSession", e);
        }
    }    

    public void close() 
            throws DbException {

        try {
            session.close();
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in close", e);
        }
    }

    private DbException handleException (String msg, Exception e) {

        return new DbException(msg, e);

    }

} public class OrganizationMatcherDecorator implements IDecorator {
    private static final Logger log = Logger.getLogger(OrganizationMatcherDecorator.class);

    // -- cache handling
    private static final long CACHE_TTL = 30*60*1000L;
    private static final Object cacheMutex = new Object();
    private static Map<Long, ASNumber> asnMapCached;
    private static Map<String, DomainName> domainNameMapCached;
    private static IntervalList ipIntervalsCached;
    private static long lastCacheFetch;
    
    private JobContext jobContext;
    private Map<Long, ASNumber> asnMap;
    private Map<String, DomainName> domainNameMap;
    private IntervalList ipIntervals;
    private boolean matchIpAddress;
    private boolean matchHostname;
    private boolean matchAsn;
    private boolean checkHighPriorityOrganization;
    
    private long noOfOrganizationsFound;
    private long noOfOrganizationsFound2;
    private long noOfHighPriorityOrganizationsFound;
    private long noOfHighPriorityOrganizationsFound2;

    private List<Organization> highPriorityOrganizations;
    private int highPriorityThreshold;
    
    
    public OrganizationMatcherDecorator() {
        // empty
    }

    
    @Override
    public void init(JobContext jobContext) throws MegatronException {
        this.jobContext = jobContext;
        init();
    }


    @Override
    public void execute(LogEntry logEntry) throws MegatronException {
        int orgId = 0;
        boolean highPriorityEntryFound = false;
        DbManager dbManager = jobContext.getDbManager();
        
        orgId = findOrganizationId(logEntry.getIpAddress(), logEntry.getHostname(), getAsn(true, logEntry));
        if (orgId == -1) {
            orgId = findOrganizationId(logEntry.getIpRangeStart(), logEntry.getIpRangeEnd()); 
        }
        if (orgId != -1) {           
            Organization organization = dbManager.getOrganization(orgId);
            logEntry.setOrganization(organization);
            ++noOfOrganizationsFound;
            if (isHighPriorityOrganization(organization)) {
                addHighPriorityOrganization(organization);
                ++noOfHighPriorityOrganizationsFound;
                highPriorityEntryFound = true;
            }
        }

        orgId = findOrganizationId(logEntry.getIpAddress2(), logEntry.getHostname2(), getAsn(false, logEntry));
        if (orgId != -1) {
            Organization organization2 = dbManager.getOrganization(orgId);
            logEntry.setOrganization2(organization2);
            ++noOfOrganizationsFound2;
            if (isHighPriorityOrganization(organization2)) {
                addHighPriorityOrganization(organization2);
                ++noOfHighPriorityOrganizationsFound2;
                highPriorityEntryFound = true;
            }
        }
        
        if (highPriorityEntryFound) {
            jobContext.incNoOfHighPriorityEntries(1);
        }
    }

    
    @Override
    public void close() throws MegatronException {
        long noOfTotalOrganizationsFound = noOfOrganizationsFound + noOfOrganizationsFound2;
        log.info("No. of organizations found: " + noOfTotalOrganizationsFound + " (" + noOfOrganizationsFound + "+" + noOfOrganizationsFound2 + ")."); 

        long noOfTotalHighPriosFound = noOfHighPriorityOrganizationsFound + noOfHighPriorityOrganizationsFound2;
        log.info("No. of high priority organizations found: " + noOfTotalHighPriosFound + " (" + noOfHighPriorityOrganizationsFound + "+" + noOfHighPriorityOrganizationsFound2 + ")."); 
    }

    
    public List<Organization> getHighPriorityOrganizations() {
        ObjectStringSorter prioritySorter = new ObjectStringSorter() {
                @Override
                protected String getObjectString(Object obj) {
                    if (obj == null) {
                        return null;
                    }
                    Integer prio = ((Organization)obj).getPriority().getPrio();
                    return (prio != null) ? prio.toString() : null;
                }
            };
        Collections.sort(highPriorityOrganizations, prioritySorter);
        Collections.reverse(highPriorityOrganizations);
        return highPriorityOrganizations;
    }

    
    public void setMatchIpAddress(boolean matchIpAddress) {
        this.matchIpAddress = matchIpAddress;
    }


    public void setMatchHostname(boolean matchHostname) {
        this.matchHostname = matchHostname;
    }


    public void setMatchAsn(boolean matchAsn) {
        this.matchAsn = matchAsn;
    }


    public void setCheckHighPriorityOrganization(boolean checkHighPriorityOrganization) {
        this.checkHighPriorityOrganization = checkHighPriorityOrganization;
    }


    private static void initCache(DbManager dbManager) throws DbException {
        long t1 = System.currentTimeMillis();
        if (lastCacheFetch + CACHE_TTL < t1) {
            boolean success = false;
            try {
                // asnMap
                asnMapCached = new HashMap<Long, ASNumber>();
                List<ASNumber> asNumbers = dbManager.getAllASNumbers(false);
                for (Iterator<ASNumber> iterator = asNumbers.iterator(); iterator.hasNext(); ) {
                    ASNumber asNumber = iterator.next();
                    asnMapCached.put(asNumber.getNumber(), asNumber);
                }
                log.info("All ASNs read from db. Size: " + asnMapCached.size());
                
                // domainNameMap
                domainNameMapCached = new HashMap<String, DomainName>();
                List<DomainName> domainNames = dbManager.getAllDomainNames(false);
                for (Iterator<DomainName> iterator = domainNames.iterator(); iterator.hasNext(); ) {
                    DomainName domainName = iterator.next();
                    domainNameMapCached.put(domainName.getName().toLowerCase(), domainName);
                }
                log.info("All domain names read from db. Size: " + domainNameMapCached.size());
                
                // ipIntervals
                ipIntervalsCached = new IntervalList();
                List<IpRange> ipRanges = dbManager.getAllIpRanges(false);
                for (Iterator<IpRange> iterator = ipRanges.iterator(); iterator.hasNext(); ) {
                    IpInterval ipInterval = new IpInterval(iterator.next());
                    ipIntervalsCached.add(ipInterval);
                }
                log.info("All IP ranges read from db. Size: " + ipRanges.size());
                
                if (log.isDebugEnabled()) {
                    String durationStr = DateUtil.formatDuration(System.currentTimeMillis() - t1);
                    log.debug("Time to read contact info from db: " + durationStr);
                }
                
                lastCacheFetch = System.currentTimeMillis();
                success = true;
            } finally {
                if (!success) {
                    lastCacheFetch = 0L;
                }
            }
        }
    }

    
    private Map<Long, ASNumber> createAsnMap() throws DbException {
        // Note: initCache will be locked by cacheMutex. A new list must be 
        // returned; asnMapCached may be in use when it's re-initialized.
        synchronized (cacheMutex) {
            initCache(jobContext.getDbManager());
            return new HashMap<Long, ASNumber>(asnMapCached);
        }
    }

    
    private Map<String, DomainName> createDomainNameMap() throws DbException {
        synchronized (cacheMutex) {
            initCache(jobContext.getDbManager());
            return new HashMap<String, DomainName>(domainNameMapCached);
        }
    }
    
    
    private IntervalList createIpIntervals() throws DbException {
        synchronized (cacheMutex) {
            initCache(jobContext.getDbManager());
            return new IntervalList(ipIntervalsCached);
        }
    }
    
    
    private void init() throws DbException {
        asnMap = createAsnMap();
        domainNameMap = createDomainNameMap();
        ipIntervals = createIpIntervals();

        highPriorityOrganizations = new ArrayList<Organization>();
        highPriorityThreshold = jobContext.getProps().getInt(AppProperties.HIGH_PRIORITY_THRESHOLD_KEY, 50);
        
        matchIpAddress = jobContext.getProps().getBoolean(AppProperties.DECORATOR_MATCH_IP_ADDRESS_KEY, true);
        matchHostname = jobContext.getProps().getBoolean(AppProperties.DECORATOR_MATCH_HOSTNAME_KEY, true);
        matchAsn = jobContext.getProps().getBoolean(AppProperties.DECORATOR_MATCH_ASN_KEY, true);
        checkHighPriorityOrganization = true;
    }

    
    private Long getAsn(boolean usePrimaryOrg, LogEntry logEntry) {
        Long result = null;
        Long asn = usePrimaryOrg ? logEntry.getAsn() : logEntry.getAsn2();
        String key = usePrimaryOrg ? AsnGeoIpDecorator.AS_NUMBER : AsnGeoIpDecorator.AS_NUMBER2;
        if ((asn != null) && (asn.longValue() != 0L)) {
            result = asn;
        } else {
            Map<String, String> additionalItems = logEntry.getAdditionalItems();
            if (additionalItems != null) {
                String longStr = additionalItems.get(key);
                if (longStr != null) {
                    try {
                        result = new Long(longStr);
                    } catch (NumberFormatException e) {
                        log.error("Cannot parse ASN: " + longStr, e);
                    }
                }
            }
        }
        return result;
    }

    
    private int findOrganizationId(Long ipAddress, String hostname, Long asn) {
        if (matchIpAddress && (ipAddress != null) && (ipAddress.longValue() != 0L)) {
            IpInterval ipInterval = (IpInterval)ipIntervals.findFirstInterval(ipAddress.longValue());
            if (ipInterval != null) {
                log.debug("Organization#" + ipInterval.getOrganizationId() + " found by IP: " + ipAddress);
                return ipInterval.getOrganizationId();
            }
        }
        
        String originalHostname = hostname;
        while (matchHostname && (hostname != null)) {
            DomainName domainName = domainNameMap.get(hostname.toLowerCase());
            if (domainName != null) {
                log.debug("Organization#" + domainName.getOrganizationId() + " found by hostname: " + originalHostname);
                return domainName.getOrganizationId();
            }
            hostname = extractPrefixLabel(hostname);
        }

        if (matchAsn && (asn != null) && (asn.longValue() != 0L)) {
            ASNumber asNumber = asnMap.get(asn.longValue());
            if (asNumber != null) {
                log.debug("Organization#" + asNumber.getOrganizationId() + " found by ASN: " + asn);
                return asNumber.getOrganizationId();
            }
        }

        return -1;
    }

    
    private int findOrganizationId(Long ipRangeStart, Long ipRangeEnd) {
        if (matchIpAddress && (ipRangeStart != null) && (ipRangeStart.longValue() != 0L) && (ipRangeEnd != null) && (ipRangeEnd.longValue() != 0L)) {
            Interval ipRange = new Interval(ipRangeStart.longValue(), ipRangeEnd.longValue());
            IpInterval ipInterval = (IpInterval)ipIntervals.findFirstInterval(ipRange);
            if (ipInterval != null) {
                log.debug("Organization#" + ipInterval.getOrganizationId() + " found by IP-range: " + ipRange + " overlaps with " + ipInterval);
                return ipInterval.getOrganizationId();
            }
        }
        return -1;
    }
    
    
    private boolean isHighPriorityOrganization(Organization organization) {
        if (!checkHighPriorityOrganization) {
            return false;
        }

        Integer priority = organization.getPriority().getPrio();
        boolean result = false;
        if (priority != null) {
            result = priority.intValue() >= highPriorityThreshold;
        } else {
            log.warn("Priority is missing for organization: " + organization.getName());
        }
        return result;
    }

    
    private void addHighPriorityOrganization(Organization organization) {
        if (!highPriorityOrganizations.contains(organization)) {
            highPriorityOrganizations.add(organization);
        }
    }
    
    
    /**
     * Extracts first label from specified hostname, e.g. "www.foobar.se" --> "foobar.se".
     * 
     * @return hostname - prefix label, or null if extraction not possible.
     */
    private String extractPrefixLabel(String hostname) {
        String[] headTail = StringUtil.splitHeadTail(hostname, ".", false);
        return ((headTail == null) || (headTail[1].indexOf('.') == -1)) ? null : headTail[1];
    }

    
    private static class IpInterval extends Interval {
        private int organizationId;
        
        
        public IpInterval(IpRange ipRange) {
            super(ipRange.getStartAddress(), ipRange.getEndAddress());
            organizationId = ipRange.getOrganizationId();
        }

        
        public int getOrganizationId() {
            return organizationId;
        }

    }

} public class Job extends BaseJob {
	private static final long serialVersionUID = 1L;

/*[CONSTRUCTOR MARKER BEGIN]*/
	public Job () {
		super();
	}

	/**
	 * Constructor for primary key
	 */
	public Job (java.lang.Long id) {
		super(id);
	}

	/**
	 * Constructor for required fields
	 */
	public Job (
		java.lang.Long id,
		java.lang.String name,
		java.lang.String filename,
		java.lang.String fileHash,
		java.lang.Long fileSize,
		java.lang.Long started) {

		super (
			id,
			name,
			filename,
			fileHash,
			fileSize,
			started);
	}

/*[CONSTRUCTOR MARKER END]*/


} public class LogEntry extends BaseLogEntry {
	private static final long serialVersionUID = 1L;
	
	
/*[CONSTRUCTOR MARKER BEGIN]*/
	public LogEntry () {
		super();
	}

	/**
	 * Constructor for primary key
	 */
	public LogEntry (java.lang.Long id) {
		super(id);
	}

	/**
	 * Constructor for required fields
	 */
	public LogEntry (
		java.lang.Long id,
		java.lang.Long created,
		java.lang.Long logTimestamp) {

		super (
			id,
			created,
			logTimestamp);
	}

/*[CONSTRUCTOR MARKER END]*/
	
	@Override
    public int compareTo(Object obj) {

        int result = 0;
        
        if (this.getId() == 0 || ((LogEntry)obj).getId() == 0) {
            // Assert that only objects that have been persited are used. 
            throw new java.lang.AssertionError("LogEntry ID is undefined (0)");
        }
        		
		if (this.getId() == ((LogEntry)obj).getId()) {		
			result = 0;
		}
		else if (this.getId() < ((LogEntry)obj).getId()) {
			result = -1;
		}
		else {
			return 1;
		}
		return result;
	}
	
} public class Organization extends BaseOrganization {
    private static final long serialVersionUID = 1L;

    /*[CONSTRUCTOR MARKER BEGIN]*/
    public Organization () {
        super();
    }

    /**
     * Constructor for primary key
     */
    public Organization (java.lang.Integer id) {
        super(id);
    }

    /**
     * Constructor for required fields
     */
    public Organization (
            java.lang.Integer id,
            java.lang.String name,
            boolean enabled,
            java.lang.Long created,
            java.lang.Long lastModified,
            java.lang.String modifiedBy,
            boolean autoUpdateMatchFields) {

        super (
                id,
                name,
                enabled,
                created,
                lastModified,
                modifiedBy,            
                autoUpdateMatchFields);
    }

    /*[CONSTRUCTOR MARKER END]*/


    // Overloaded method from the base class that adds organization id to the Contact.
    @Override
    public void addToContacts (se.sitic.megatron.entity.Contact contact) {
        if (null == getContacts()) setContacts(new java.util.TreeSet<se.sitic.megatron.entity.Contact>());

        contact.setOrganizationId(this.getId());        
        super.addToContacts(contact);        
    }    
    // Overloaded method from the base class that adds organization id to the IP-range.
    @Override
    public void addToIpRanges (se.sitic.megatron.entity.IpRange ipRange) {
        if (null == getIpRanges()) setIpRanges(new java.util.TreeSet<se.sitic.megatron.entity.IpRange>());

        ipRange.setOrganizationId(this.getId());        
        super.addToIpRanges(ipRange);        
    }
    // Overloaded method from the base class that adds organization id to the DomainName.
    @Override
    public void addToDomainNames (se.sitic.megatron.entity.DomainName domainName) {
        if (null == getDomainNames()) setDomainNames(new java.util.TreeSet<se.sitic.megatron.entity.DomainName>());

        domainName.setOrganizationId(this.getId());
        super.addToDomainNames(domainName);        
    }
    // Overloaded method from the base class that adds organization id to the ASNumber.
    @Override
    public void addToASNumbers (se.sitic.megatron.entity.ASNumber asNumber) {
        if (null == getASNumbers()) setASNumbers(new java.util.TreeSet<se.sitic.megatron.entity.ASNumber>());

        asNumber.setOrganizationId(this.getId());
        super.addToASNumbers(asNumber);
    }
    
    // Returning all enabled email addresses
    public String getEmailAddresses(){        
        return this.getEmailAddresses(null, true);
    }

    // This method returns all email addresses for the organization that matches the given email type and 
    // the enabledOnly flag. If the email type is null or empty, all addresses matching the enabledOnly
    // flag are returned.

    public String getEmailAddresses(String emailType, boolean enabledOnly){

        java.util.Set<se.sitic.megatron.entity.Contact> allContacts = getContacts();
        java.util.Set<String> addresses = new java.util.TreeSet<String>(); 

        for (se.sitic.megatron.entity.Contact contact : allContacts) {
            if (contact.isEnabled() == true || enabledOnly == false) {
                if (emailType == null) {
                    addresses.add(contact.getEmailAddress());
                }
                else if (contact.getEmailType().toLowerCase().equals(emailType.toLowerCase())) {                    
                    addresses.add(contact.getEmailAddress());    
                }                
            }
        }
        if (addresses.isEmpty()) {
            return "";
        }
        else {
            String emails = java.util.Arrays.toString(addresses.toArray());        
            return emails.substring(1, emails.length()-1);
        }
    }
} public interface IFileProcessor {

    public void init(JobContext jobContext) throws MegatronException;

    
    /**
     * Processes the specified file, and returns the result file.
     */
    public File execute(File inputFile) throws MegatronException;

    
    public void close(boolean jobSuccessful) throws MegatronException;
    
} public class LogEntryFilterManager {
    private static final Logger log = Logger.getLogger(LogEntryFilterManager.class);
    
    private JobContext jobContext;
    private List<ILogEntryFilter> filters;

    
    public LogEntryFilterManager(JobContext jobContext) {
        this.jobContext = jobContext;
    }    

    
    public void init(String propKey) throws MegatronException {
        filters = createLogEntryFilters(propKey);
    }

    
    public void init(String[] classNames) throws MegatronException {
        filters = createLogEntryFilters(classNames);
    }

    
    public boolean executeFilters(LogEntry logEntry) throws MegatronException {
        if (filters.isEmpty()) {
            return true;
        }
        
        boolean result = true;
        for (Iterator<ILogEntryFilter> iterator = filters.iterator(); result && iterator.hasNext(); ) {
            result &= iterator.next().accept(logEntry);
        }
        return result;
    }

    
    public void closeFilters() {
        if (filters == null) {
            return;
        }
        for (Iterator<ILogEntryFilter> iterator = filters.iterator(); iterator.hasNext(); ) {
            try {
                iterator.next().close();
            } catch (Exception e) {
                log.error("Cannot close log entry filter.", e);
            }
        }
    }

    
    private List<ILogEntryFilter> createLogEntryFilters(String propKey) throws MegatronException {
        TypedProperties props = jobContext.getProps();
        String[] classNames = props.getStringList(propKey, new String[0]);
        return createLogEntryFilters(classNames);
    }

    
    private List<ILogEntryFilter> createLogEntryFilters(String[] classNames) throws MegatronException {
        List<ILogEntryFilter> result = new ArrayList<ILogEntryFilter>();
        if (classNames.length > 0) {
            log.debug("Using log entry filters: " + Arrays.asList(classNames));
        }
        for (int i = 0; i < classNames.length; i++) {
            String className = classNames[i];
            if (className.trim().length() == 0) {
                continue;
            }
            try {
                Class<?> clazz = Class.forName(className);
                ILogEntryFilter filter = (ILogEntryFilter)clazz.newInstance();
                filter.init(jobContext);
                result.add(filter);
            } catch (MegatronException e) {
                String msg = "Cannot initialize filter class: " + className;
                throw new MegatronException(msg, e);
            } catch (Exception e) {
                // ClassNotFoundException, InstantiationException, IllegalAccessException
                String msg = "Cannot instantiate filter class: " + className;
                throw new MegatronException(msg, e);
            }
        }
        return result;
    }
    
} public class MailSender {
    private static final Logger log = Logger.getLogger(MailSender.class);

    private static final String JAVA_MAIL_SMTP_HOST_KEY = "mail.smtp.host";

    private String[] smtpHosts;
    private int smtpHostIndex;
    private boolean debug;
    private boolean htmlMail;
    private boolean encrypt;
    private boolean sign;

    private String fromAddress;
    private String toAddresses;
    private String bccAddresses;
    private String ccAddresses;
    private String replyToAddresses;
    private String subject;
    private String body;
    private List<MailAttachment> attachments;


    /**
     * Constructor.
     */
    public MailSender() {
        // empty
    }


    /**
     * Sends mail. All fields are optional and may be read from properties, e.g from-address, to-addresses, subject, and body.
     */
    public void send(TypedProperties props) throws MailException {
        // Init properties
        this.smtpHosts = props.getStringList(AppProperties.MAIL_SMTP_HOST_KEY, null);
        this.debug = props.getBoolean(AppProperties.MAIL_DEBUG_KEY, false);
        this.htmlMail = props.getBoolean(AppProperties.MAIL_HTML_MAIL_KEY, true);
        this.sign = props.getBoolean(AppProperties.MAIL_ENCRYPTION_ENCRYPT_KEY, false);
        this.encrypt = props.getBoolean(AppProperties.MAIL_ENCRYPTION_SIGN_KEY, false);

        // -- Check pre-condition
        if (getSmtpHost() == null) {
            throw new MailException("SMTP host not specified.");
        }

        String from = (fromAddress != null) ? fromAddress : props.getString(AppProperties.MAIL_FROM_ADDRESS_KEY, "megatron@example.org");
        String to = (toAddresses != null) ? toAddresses : props.getString(AppProperties.MAIL_TO_ADDRESSES_KEY, null);
        String bcc = (bccAddresses != null) ? bccAddresses : props.getString(AppProperties.MAIL_BCC_ADDRESSES_KEY, null);
        String cc = (ccAddresses != null) ? ccAddresses : props.getString(AppProperties.MAIL_CC_ADDRESSES_KEY, null);
        String replyTo = (replyToAddresses != null) ? replyToAddresses : props.getString(AppProperties.MAIL_REPLY_TO_ADDRESSES_KEY, null);

        // -- Logging
        if (log.isDebugEnabled()) {
            log.debug("Sending mail to: " + (toAddresses != null ? to : to + " [default]"));
            log.debug("  CC-address: " + (ccAddresses != null ? cc : cc + " [default]"));
            log.debug("  BCC-address: " + (bccAddresses != null ? bcc : bcc + " [default]"));
            log.debug("  From-address: " + (fromAddress != null ? from : from + " [default]"));
            log.debug("  ReplyTo-address: " + (replyToAddresses != null ? replyTo : replyTo + " [default]"));
            log.debug("  Subject: " + (subject != null ? subject : "[default]"));
            log.debug("  Body size: " + ((body != null) ? body.length() : 0));
            log.debug("  Attachments: " + ((attachments != null) ? attachments.size() : 0));
        }

        // -- Set properties
        Properties sessionProps = new Properties();
        sessionProps.put(JAVA_MAIL_SMTP_HOST_KEY, getSmtpHost());
        Session session = Session.getDefaultInstance(sessionProps, null);
        session.setDebug(debug);

        try {
            // -- Create message
            MimeMessage message = new MimeMessage(session);
            message.setSentDate(new Date());
            
            // -- Set addresses
            message.setFrom(new InternetAddress(from));
            if (!StringUtil.isNullOrEmpty(replyTo)) {
                message.setReplyTo(parseAddresses(replyTo));
            }
            message.setRecipients(Message.RecipientType.TO, parseAddresses(to));
            if (bcc != null) {
                message.setRecipients(Message.RecipientType.BCC, parseAddresses(bcc));
            }
            if (cc != null) {
                message.setRecipients(Message.RecipientType.CC, parseAddresses(cc));
            }
            
            // -- Set subject and content
            message.setSubject(StringUtil.getNotNull(subject, "[Empty Subject]"));
            String mimeSubType = htmlMail ? "html" : "plain";
            if ((attachments == null) || (attachments.size() == 0)) {
                message.setText(StringUtil.getNotNull(body, "[Empty Body]"), Constants.UTF8, mimeSubType);
            } else {
                MimeBodyPart messageBodyPart = new MimeBodyPart();
                messageBodyPart.setText(StringUtil.getNotNull(body, "[Empty Body]"), Constants.UTF8, mimeSubType);

                // body
                Multipart multipart = new MimeMultipart();
                multipart.addBodyPart(messageBodyPart);

                // attachments
                for (Iterator<MailAttachment> iterator = attachments.iterator(); iterator.hasNext(); ) {
                    MailAttachment attachment = iterator.next();
                    try {
                        messageBodyPart = new MimeBodyPart();
                        DataSource dataSource = new ByteArrayDataSource(attachment.getContent(), attachment.getMimeType());
                        messageBodyPart.setDataHandler(new DataHandler(dataSource));
                        messageBodyPart.setFileName(attachment.getAttachmentName());
                        multipart.addBodyPart(messageBodyPart);
                    } catch (IOException e) {
                        String msg = "Cannot read content for attachment: " + attachment.getAttachmentName();
                        log.error(msg, e);
                        throw new MailException(msg, e);
                    }
                }

                // add to message
                message.setContent(multipart);

            }

            if (encrypt || sign) {
                throw new UnsupportedOperationException("Mail signing and encryption not implemented.");

                // TODO Add support for mail signing and encryption
//                try {
//                    String encryptionType = props.getString(AppProperties.MAIL_ENCRYPTION_TYPE, "PGP");
//                    MailEncryptor encryptor = new MailEncryptor (encryptionType);
//
//                    if (encrypt) {
//                        System.out.println("Encrypt");
//                        message = encryptor.encrypt(session, message);
//                    }
//                    if (sign) {
//                        System.out.println("Sign");
//                        message = encryptor.sign(session, message);
//                    }
//                }
//            catch (EncryptionException e) {
//                    String msg = "Cannot sign or encrypt the mail: ";
//                    log.error(msg, e);
//                    throw new MailException(msg, e);
//                }
            }

            // -- Send mail (try all SMTP hosts in list if mail cannot be delivered)
            boolean mailSent = false;
            while (!mailSent) {
                try {
                    Transport.send(message);
                    mailSent = true;
                } catch (MessagingException e) {
                    String msg = "Cannot send mail using SMTP host '" + getSmtpHost() + "'.";
                    log.error(msg, e);
                    ++this.smtpHostIndex;
                    if (getSmtpHost() == null) {
                        log.info("SMTP host list exhausted. Giving up; mail not delivered.", e);
                        smtpHostIndex = 0;
                        throw e;
                    }
                    sessionProps.put(JAVA_MAIL_SMTP_HOST_KEY, getSmtpHost());
                    log.info("Trying to send mail with new SMTP host: " + getSmtpHost());
                }
            }
            log.debug("Mail sent.");
        } catch (AddressException e) {
            String msg = "Cannot handle to-, from-, or replyTo-address. Mail not sent. To-address: " + toAddresses + "; CC-address: " + ccAddresses + 
                "; BCC-address: " + bccAddresses +"; From-address: " + fromAddress + "; ReplyTo-address: " + replyToAddresses;
            log.error(msg, e);
            throw new MailException(msg, e);
        } catch (MessagingException e) {
            String msg = "Cannot send Mail; general error.";
            log.error(msg, e);
            throw new MailException(msg, e);
        }
    }


    /**
     * Clears instance variables.
     */
    public void clear() {
        this.fromAddress = null;
        this.toAddresses = null;
        this.ccAddresses = null;
        this.bccAddresses = null;
        this.replyToAddresses = null;
        this.subject = null;
        this.body = null;
        this.attachments = null;
    }


    public void addAttachment(MailAttachment attachment) {
        if (attachments == null) {
            attachments = new ArrayList<MailAttachment>();
        }
        attachments.add(attachment);
    }


    public void setFromAddress(String fromAddress) {
        this.fromAddress = fromAddress;
    }


    public void setToAddresses(String toAddresses) {
        this.toAddresses = toAddresses;
    }

    
    public void setCcAddresses(String ccAddresses) {
        this.ccAddresses = ccAddresses;
     }
    
    
    public void setBccAddresses(String bccAddresses) {
        this.bccAddresses = bccAddresses;
    }


    public void setReplyToAddresses(String replyToAddresses) {
        this.replyToAddresses = replyToAddresses;
    }


    public void setSubject(String subject) {
        this.subject = subject;
    }


    public void setBody(String body) {
        this.body = body;
    }


    public void setAttachments(List<MailAttachment> attachments) {
        this.attachments = attachments;
    }


    @SuppressWarnings("null")
    private InternetAddress[] parseAddresses(String addressesStr) throws AddressException {
        if (addressesStr == null) {
            throw new AddressException("Address is not assigned.");
        }

        String[] addresses = addressesStr.trim().split(";|,");
        if ((addresses == null) || (addresses.length == 0)) {
            new AddressException("Address is missing.");
        }

        List<InternetAddress> resultList = new ArrayList<InternetAddress>(addresses.length);
        for (int i = 0; i < addresses.length; i++) {
            String address = addresses[i].trim();
            if (address.length() > 0) {
                resultList.add(new InternetAddress(address));
            }
        }

        if (resultList.size() == 0) {
            new AddressException("Address is missing; result list is empty.");
        }

        return resultList.toArray(new InternetAddress[resultList.size()]);
    }


    private String getSmtpHost() {
        if ((smtpHosts == null) || (smtpHostIndex >= smtpHosts.length)) {
            return null;
        }

        return smtpHosts[smtpHostIndex];
    }

} public class RegExpParser implements IParser {
    // private static final Logger log = Logger.getLogger(RegExpParser.class);
    
    private TypedProperties props;
    private JobContext jobContext;

    private List<String> variables;
    private Matcher matcher;
    private boolean trimValue;
    private String removeEnclosingCharsFromValue;
    private AttributeValueRewriter rewriter;


    public RegExpParser() {
        // empty
    }

    
    @Override
    public void init(JobContext jobContext) throws InvalidExpressionException, MegatronException {
        this.jobContext = jobContext;
        this.props = jobContext.getProps();
        
        String lineRegExp = props.getString(AppProperties.PARSER_LINE_REG_EXP_KEY, "");
        if (lineRegExp.length() == 0) {
            String msg = "Line expression is missing (parser.lineRegExp).";
            throw new InvalidExpressionException(msg);
        }
        LineExpression expression = new LineExpression(props, lineRegExp);
        variables = expression.extractVariables();
        matcher = expression.createRegExp();
        trimValue = props.getBoolean(AppProperties.PARSER_TRIM_VALUE_KEY, false);
        removeEnclosingCharsFromValue = props.getString(AppProperties.PARSER_REMOVE_ENCLOSING_CHARS_FROM_VALUE_KEY, null);
        String[] rewriterArray = props.getStringList(AppProperties.PARSER_REWRITERS_KEY, null);
        rewriter = AttributeValueRewriter.createAttributeValueRewriter(rewriterArray);
    }

    
    @Override
    public LogEntry parse(String logLine) throws ParseException {
        LogEntry result = null;

        matcher.reset(logLine);
        Map<String, String> parsedLogEntry = new HashMap<String, String>();
        if (matcher.find()) {
            if (matcher.groupCount() != variables.size()) {
                String msg = "Cannot parse line at " + jobContext.getLineNo() + "; groups in reg-exp does not match (" + variables.size() + "!=" + 
                    matcher.groupCount() + "). Line: '" + logLine + "'.";
                throw new ParseException(msg);
            }
            
            for (int i = 0; i < variables.size(); i++) {
                String value = matcher.group(i+1);
                if (value == null) {
                    String msg = "Cannot parse line at " + jobContext.getLineNo() + "; group " + i + " is unmatched. Line: '" + logLine + "'.";
                    throw new ParseException(msg);
                }
                if (trimValue) {
                    value = value.trim();
                }
                // empty value is treated as NULL
                if (value.length() != 0) {
                    if ((removeEnclosingCharsFromValue != null) && (removeEnclosingCharsFromValue.length() != 0)) {
                        value = StringUtil.removeEnclosingChars(value, removeEnclosingCharsFromValue);
                    }
                    parsedLogEntry.put(variables.get(i).substring(1), value);
                } else if (variables.get(i).startsWith(LogEntryMapper.VARIABLE_PREFIX + LogEntryMapper.FREE_TEXT_PREFIX)) {
                    // empty value is stored for free text (otherwise list order cannot be preserved) 
                    parsedLogEntry.put(variables.get(i).substring(1), value);
                }
            }
            
            if (rewriter != null) {
                rewriter.rewrite(parsedLogEntry);
            }
        } else {
            String msg = "Cannot parse line at " + jobContext.getLineNo() + "; line is unmatched. Line: '" + logLine + "'.";
            throw new ParseException(msg);
        }
        
        LogEntryMapper mapper = new LogEntryMapper(props, parsedLogEntry);
        try {
            result = mapper.createLogEntry();
        } catch (ConversionException e) {
            String msg = "Cannot parse line at " + jobContext.getLineNo() + "; " + e.getMessage() + " Line: '" + logLine + "'.";
            throw new ParseException(msg, e);
        }

        return result;
    }


    @Override
    public void close() throws MegatronException {
        // empty
    }
    
} public abstract class Constants {

    /** Line break in files etc. */
    public static final String LINE_BREAK = "\n";

    /** UTF-8 character-set in Java core API. */
    public static final String UTF8 = "UTF-8";

    /** ISO-8859 character-set in Java core API. */
    public static final String ISO8859 = "ISO-8859-1";

    /** MIME-type for plain text. */
    public static final String MIME_TEXT_PLAIN = "text/plain";
    
    /** Comments in config files starts with this string. */
    public static final String CONFIG_COMMENT_PREFIX = "#";

    /** Hash algoritm to use. */
    public static final String DIGEST_ALGORITHM = "md5";

    /** Job type name to use when name is missing in the job_type table. */
    public static final String DEFAULT_JOB_TYPE = "default";
    
    // Values for the property filter.countryCodeFilter.organizationToFilter
    // and filter.asnFilter.organizationToFilter
    public static final String ORGANIZATION_PRIMARY = "primary";
    public static final String ORGANIZATION_SECONDARY = "secondary";
    public static final String ORGANIZATION_BOTH = "both";

    // Additional format strings for parser.timestampFormat.
    public static final String TIME_STAMP_FORMAT_EPOCH_IN_SEC = "epochInSec";
    public static final String TIME_STAMP_FORMAT_EPOCH_IN_MS = "epochInMs";
    public static final String TIME_STAMP_FORMAT_WINDOWS_EPOCH = "windowsEpoch";
    
} public abstract class SqlUtil {
    // UNUSED: private static final Logger log = Logger.getLogger(SqlUtil.class);

    
    /**
     * Converts specified timestamp in seconds to a Date.
     */
    public static Date convertTimestamp(long timestampInSec) {
        return new Date(timestampInSec * 1000L);
    }

    
    /**
     * Converts specified timestamp in milliseconds to seconds.
     */
    public static long convertTimestampToSec(long timestampInMs) {
        return Math.round(timestampInMs / 1000d);  
    }

    
    /**
     * Converts specified timestamp to seconds.
     */
    public static long convertTimestamp(Date timestamp) {
        return Math.round(timestamp.getTime() / 1000d);  
    }

} public abstract class StringUtil {


    /**
     * Replaces a substring in a string with another string. Replaces all
     * occurrences.
     * <p>
     * Example: replace("name=@name@", "@name@", "hubba") --> "name=hubba".
     *
     * @throws NullPointerException if the from- or to-argument is null.
     */
    public static String replace(String str, String from, String to) {
        return replaceInternal(str, from, to, false);
    }

    
    /**
     * As replace, but replace one the first occurrence.  
     */
    public static String replaceFirst(String str, String from, String to) {
        return replaceInternal(str, from, to, true);
    }


    /**
     * Removes prefix from specified string.<p>
     * Example: removePrefix("foobar.config", "foobar.") --> "config"
     */
    public static String removePrefix(String str, String prefix) {
        if ((str == null) || (prefix == null)) {
            return str;
        }

        String result = null;
        int hitIndex = str.indexOf(prefix);
        if (hitIndex == 0) {
            if (prefix.length() < str.length()) {
                result = str.substring(prefix.length());
            } else {
                result = "";
            }
        } else {
            result = str;
        }

        return result;
    }


    /**
     * Removes suffix from specified string.<p>
     * Example: removeSuffix("foobar.config", ".config") --> "foobar"
     */
    public static String removeSuffix(String str, String suffix) {
        if ((str == null) || (suffix == null)) {
            return str;
        }

        String result = str;
        int hitIndex = str.lastIndexOf(suffix);
        if ((hitIndex != -1) && ((hitIndex + suffix.length()) == str.length())) {
            result = result.substring(0, hitIndex);
        }
        return result;
    }


    /**
     * Splits specified string in two parts at delimiter position.
     *
     * @param str string to split.
     * @param delim delimiter that marks split between head and tail.
     * @param reverse is search for delimiter reverse (search starts at end of string)?
     * @return null if one argument is null, otherwise a String-array of length
     *         two in following format: { head, tail }. If delimiter is not
     *         found, then tail is empty if isReverse is false or
     *         head is empty if isReverse is true. Delimiter is not
     *         included.
     */
    public static String[] splitHeadTail(String str, String delim, boolean reverse) {
        // check param
        if ((str == null) || (delim == null)) {
            return null;
        }

        String[] result = new String[2];
        int hitPos = reverse ? str.lastIndexOf(delim) : str.indexOf(delim);
        if (hitPos != -1) {
            result[0] = str.substring(0, hitPos);
            int tailPos = hitPos + delim.length();
            if (tailPos < str.length()) {
                result[1] = str.substring(tailPos);
            } else {
                result[1] = "";
            }
        } else {
            if (reverse) {
                result[0] = "";
                result[1] = str;
            } else {
                result[0] = str;
                result[1] = "";
            }
        }

        return result;
    }


    /**
     * Returns true if specified string is null or empty.
     */
    public static boolean isNullOrEmpty(String str) {
        return str == null || str.length() == 0;
    }


    /**
     * Returns specified string if not null, otherwise an empty string.
     */
    public static String getNotNull(String str) {
        return (str != null) ? str : "";
    }


    /**
     * Returns specified string if not null, otherwise specified default value.
     */
    public static String getNotNull(String str, String defaultStr) {
        return (str != null) ? str : defaultStr;
    }


    /**
     * Converts specifed byte-array to hex-string.
     */
    public static String encode(byte[] buf) {
        StringBuffer result = new StringBuffer(2 * buf.length);

        for (int i = 0; i < buf.length; i++) {
            int byteAsInt = buf[i] & 0xFF;
            if (byteAsInt < 16) {
                result.append("0");
            }
            result.append(Integer.toString(byteAsInt, 16).toLowerCase());
        }

        return result.toString();
    }


    /**
     * Converts specified hex-string to byte-array.
     */
    public static byte[] decodeHexString(String hexStr) throws NumberFormatException {
        if (hexStr == null) {
            return null;
        }
        // is every digit of size two?
        if ((hexStr.length() % 2) != 0) {
            throw new NumberFormatException("Bad hex-string. Length of string must be even.");
        }

        byte[] result = new byte[hexStr.length() / 2];
        for (int i = 0; i < hexStr.length(); i += 2) {
            String hexDigit = hexStr.substring(i, i + 2);
            int byteValue = Integer.parseInt(hexDigit, 16);
            result[i / 2] = (byte)(byteValue & 0xFF);
        }

        return result;
    }


    /**
     * Encodes specifed text using named entities, e.g "<" converts
     * to "&amp;lt;".
     * <p>
     * The result is safe to include in XML or HTML.
     */
    public static String encodeCharacterEntities(String str) {
        if (str == null) {
            return null;
        }

        StringBuilder result = new StringBuilder(str.length() + 50);
        for (int i = 0; i < str.length(); i++) {
            char chr = str.charAt(i);
            switch (chr) {
                case '<':
                    result.append("&lt;");
                    break;
                case '>':
                    result.append("&gt;");
                    break;
                case '&':
                    result.append("&amp;");
                    break;
                case '"':
                    result.append("&quot;");
                    break;
                default:
                    result.append(chr);
            }
        }
        return result.toString();
    }

    
    /**
     * Removes line beaks from specified string.
     */
    public static String removeLineBreaks(String str, String replaceWith) {
        String result = str;
        result = replace(result, "\n", replaceWith);
        result = replace(result, "\r", replaceWith);
        return result;
    }


    /**
     * Removes trailing white spaces (including line breaks) from 
     * specified string.
     */
    public static String removeTrailingSpaces(String str) {
        if ((str == null) || (str.length() == 0)) {
            return str;
        }

        int endIndex = 0;
        for (int i = (str.length()-1); i >= 0; i--) {
            if (!Character.isWhitespace(str.charAt(i))) {
                endIndex = i + 1;
                break;
            }
        }
        
        return str.substring(0, endIndex);
    }
    
    
    /**
     * Removes enclosing characters from string.
     * Example: str="'foobar'", enclosingChars="'" will return "foobar". 
     */
    public static String removeEnclosingChars(String str, String enclosingChars) {
        if ((str == null) || !(str.startsWith(enclosingChars) && str.endsWith(enclosingChars))) {
            return str;
        }

        return removeSuffix(removePrefix(str, enclosingChars), enclosingChars); 
    }

    
    /**
     * Truncates specified string if longer than max length. "..." is added
     * as suffix to mark that the string have been truncated.
     */
    public static String truncateString(String str, int maxLength) {
        if ((str == null) || (str.length() <= maxLength)) {
            return str;
        }
        final String suffix = "...";
        if (maxLength < suffix.length()) {
            return suffix;
        }
        
        StringBuilder result = new StringBuilder(maxLength);
        int endIndex = maxLength - suffix.length();
        result.append(str.substring(0, endIndex)).append(suffix);

        return result.toString();
    }

    
    /**
     * Returns a string containing strToCopy duplicated
     * noOfCopies times.
     */
    public static String nCopyString(String strToCopy, int noOfCopies) {
        // check param
        if (strToCopy == null) {
            return null;
        }

        StringBuilder result = new StringBuilder(noOfCopies * strToCopy.length());
        for (int i = 0; i < noOfCopies; i++) {
            result.append(strToCopy);
        }

        return result.toString();
    }

    
    /**
     * Pads specified string in head with spaces.
     */
    public static String leftPad(String str, int len) {
        return pad(str, len, " ", true);
    }


    /**
     * Pads specified string in tail with spaces.
     */
    public static String rightPad(String str, int len) {
        return pad(str, len, " ", false);
    }


    /**
     * Pads specified string in head with pad-string.
     */
    public static String leftPad(String str, int len, String padString) {
        return pad(str, len, padString, true);
    }


    /**
     * Pads specified string in tail with pad-string.
     */
    public static String rightPad(String str, int len, String padString) {
        return pad(str, len, padString, false);
    }

    
    /**
     * Returns specified files as a string list. Returns never null.
     */
    public static String toString(List<File> files, boolean shortName) {
        if (files == null) {
            return "[null]";
        }

        StringBuilder result = new StringBuilder(256);
        for (Iterator<File> iterator = files.iterator(); iterator.hasNext(); ) {
            File file = iterator.next();
            if (result.length() > 0) {
                result.append(", ");
            }

            String filename = shortName ? file.getName() : file.getAbsolutePath();
            result.append(filename);
        }

        return result.toString();
    }

    
    /**
     * Returns specified list as a string where every element is quoted and
     * comma separated.
     */
    public static String toQuotedString(List<?> list) {
        return toQuotedString(list, "\"");
    }

    
    /**
     * As toQuotedString, but use a single quote instead of a double quote.
     */
    public static String toSingleQuotedString(List<?> list) {
        return toQuotedString(list, "\'");
    }

    
    /**
     * Returns specified list as a string where every element is quoted with
     * specified character and comma separated.
     */
    private static String toQuotedString(List<?> list, String quoteChar) {
        if ((list == null) || (list.size() == 0)) {
            return "";
        }
        
        StringBuffer result = new StringBuffer(256);
        for (Iterator<?> iterator = list.iterator(); iterator.hasNext(); ) {
            Object obj = iterator.next();
            String str = (obj != null) ? obj.toString() : "";
            if (result.length() > 0) {
                result.append(",");
            }
            result.append(quoteChar).append(str).append(quoteChar);
        }

        return result.toString();
    }

    
    /**
     * Pads specified string with pad-string.
     */
    private static String pad(String str, int len, String padString, boolean leftPad) {
        if ((str == null) || (str.length() >= len) || (padString == null)) {
            return str;
        }

        int n = (len - str.length()) / padString.length();
        String paddedString = nCopyString(padString, n);
        StringBuilder result = new StringBuilder(len);
        if (leftPad) {
            result.append(paddedString).append(str);
        } else {
            result.append(str).append(paddedString);
        }

        return result.toString();
    }
    
    
    @SuppressWarnings("null")
    private static String replaceInternal(String str, String from, String to, boolean replaceFirst) {
        if (str == null) {
            return null;
        }
        if ((from == null) || (to == null)) {
            throw new NullPointerException("from- or to-argument is null.");
        }

        StringBuilder result = null;
        int fromlength = 0;
        int index = 0;
        boolean quit = false;
        while (!quit) {
            int hitIndex = str.indexOf(from, index);
            if (hitIndex < 0) {
                break;
            }
            if (result == null) {
                result = new StringBuilder(str.length() + 40);
                fromlength = from.length();
            }
            result.append(str.substring(index, hitIndex));
            result.append(to);
            index = hitIndex + fromlength;
            quit = replaceFirst;
        }
        if (index == 0) {
            return str;
        }
        result.append(str.substring(index));
        return result.toString();
    }


}
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import org.apache.log4j.Logger;
import se.sitic.megatron.db.DbManager;
import se.sitic.megatron.decorator.DecoratorManager;
import se.sitic.megatron.decorator.OrganizationMatcherDecorator;
import se.sitic.megatron.entity.Job;
import se.sitic.megatron.entity.JobType;
import se.sitic.megatron.entity.LogEntry;
import se.sitic.megatron.entity.Organization;
import se.sitic.megatron.entity.OriginalLogEntry;
import se.sitic.megatron.fileprocessor.IFileProcessor;
import se.sitic.megatron.filter.ILineFilter;
import se.sitic.megatron.filter.LogEntryFilterManager;
import se.sitic.megatron.lineprocessor.ILineProcessor;
import se.sitic.megatron.mail.MailSender;
import se.sitic.megatron.parser.IParser;
import se.sitic.megatron.parser.RegExpParser;
import se.sitic.megatron.rss.JobRssFile;
import se.sitic.megatron.util.Constants;
import se.sitic.megatron.util.DateUtil;
import se.sitic.megatron.util.FileUtil;
import se.sitic.megatron.util.SqlUtil;
import se.sitic.megatron.util.StringUtil;
                    log.info(msg);
                } else {
                    String msg = "Invalid property value for " + AppProperties.FILE_ALREADY_PROCESSED_ACTION_KEY + ": " + action;
                    throw new MegatronException(msg);
                }
            }
            
            // -- process job
            dbManager.addLogJob(job);
            
            // -- process file
            if ((fileProcessors != null) && (fileProcessors.size() > 0)){
                for (Iterator<IFileProcessor> iterator = fileProcessors.iterator(); iterator.hasNext(); ) {
                    IFileProcessor fileProcessor = iterator.next();
                    log.debug("Executing file processor: " + fileProcessor.getClass().getName());
                    file = fileProcessor.execute(file);    
                    log.debug("File processor finished. New input filename: " + file.getAbsolutePath());
                }
                // re-calc no. of lines for new file
                long noOfLines = countNoOfLines(file);
                log.info("File processor executed. No. of lines before and after: " + jobContext.getNoOfLines() + " -- " + noOfLines);
                jobContext.setNoOfLines(noOfLines);
            }
            
            // -- process line by line
            BufferedReader in = null;
            // init attributes for printProgress 
            lastProgressPrintTime = System.currentTimeMillis(); 
            lastProgressPrintLineNo = 0L;
            try {
                String charSet = props.getString(AppProperties.INPUT_CHAR_SET_KEY, Constants.UTF8);
                in = new BufferedReader(new InputStreamReader(new FileInputStream(file), charSet));
                String line = null;
                while ((line = in.readLine()) != null) {
                    processLine(line);
                }
            } catch (IOException e) {
                String msg = "Cannot read file: " + file.getAbsolutePath();
                throw new MegatronException(msg, e);
            } finally {
                try { if (in != null) in.close(); } catch (Exception ignored) {}
            }
            
            // -- finishing job
            closeFileProcessors(fileProcessors, true);
            fileProcessors = null;
            
            job.setProcessedLines(jobContext.getLineNoAfterProcessor());
            dbManager.finishLogJob(job, null);
            
            if (fileExporter != null) {
                fileExporter.writeFooter(job);
            }
            
            // -- write finished message to log and console
            writeFinishedMessage();
            if (fileExporter != null) {
                // close() will flush file; file size will now be reported correct 
                fileExporter.close();
                fileExporter.writeFinishedMessageToLog();
            }
        
            // -- send email if high priority organizations exists
            OrganizationMatcherDecorator organizationMatcher = preStorageDecoratorManager.getOrganizationMatcher();
            if (!props.isNoDb() && (organizationMatcher != null)) {
                List<Organization> organizations = organizationMatcher.getHighPriorityOrganizations();
                if (organizations.size() > 0) {
                    sendNotificationEmail(organizations);    
                } else {
                    log.info("No notification email sent; no high priority organizations found in job.");
                }
            }
        } catch (Exception e) {
            String jobName = (job != null) ? job.getName() : "[job name is null]";
            String msg = "Job (" + jobName + ") failed: " + e.getMessage();
            // stack-trace is logged elsewhere
            log.error(msg);
            MegatronException newException = (e instanceof MegatronException) ? (MegatronException)e : new MegatronException(msg, e);
            msg = "Job failed: " + e.getMessage();
            if ((job != null) && (job.getId() != null) && (job.getId() > 0L)) {
                try {
                    if (jobContext != null) {
                        job.setProcessedLines(jobContext.getLineNoAfterProcessor());
                    }
                    dbManager.finishLogJob(job, msg);
                } catch (MegatronException e2) {
                    String msg2 = "Cannot write error message to job; finishLogJob failed.";
                    log.error(msg2, e2);
                }
            }
            throw newException;
        } finally {
            closeFileProcessors(fileProcessors, false);
            closeLineFilters(preLineProcessorFilters);
            closeLineFilters(preParserFilters);
            if (preDecoratorFilterManager != null) preDecoratorFilterManager.closeFilters();
            if (preStorageFilterManager != null) preStorageFilterManager.closeFilters();
            if (preExportFilterManager != null) preExportFilterManager.closeFilters();
            if (preStorageDecoratorManager != null) preStorageDecoratorManager.closeDecorators();
            if (preExportDecoratorManager != null) preExportDecoratorManager.closeDecorators();
            try { if (parser != null) parser.close(); } catch (Exception ignored) {}
            try { if (lineProcessor != null) lineProcessor.close(); } catch (Exception ignored) {}
            try { if (fileExporter != null) fileExporter.close(); } catch (Exception ignored) {}
            try { dbManager.close(); } catch (Exception ignored) {}
        }
    }

    
    private JobContext createJobContext(File file) throws MegatronException {
        long startedTimestamp = System.currentTimeMillis();
        String jobName = getJobName(startedTimestamp);
        String fileHash = null;
        log.debug("Calculating hash for file: " + file.getAbsolutePath());
        try {
            fileHash = FileUtil.hashFile(file);
        } catch (IOException e) {
            String msg = "Cannot calculate hash for file: " + file.getAbsolutePath();
            throw new MegatronException(msg, e);
        }
        
        Job job = new Job(0L, jobName, file.getName(), fileHash, file.length(), SqlUtil.convertTimestampToSec(startedTimestamp));