public class AppProperties {
    // -- Keys in properties file --

    // Implicit
    public static final String JOB_TYPE_NAME_KEY = "implicit.jobTypeName";

    // General
    public static final String LOG4J_FILE_KEY = "general.log4jConfigFile";
    public static final String LOG_DIR_KEY = "general.logDir";
    public static final String JOB_TYPE_CONFIG_DIR_KEY = "general.jobTypeConfigDir";
    public static final String SLURP_DIR_KEY = "general.slurpDir";
    public static final String OUTPUT_DIR_KEY = "general.outputDir";
    public static final String TMP_DIR_KEY = "general.tmpDir";
    public static final String INPUT_CHAR_SET_KEY = "general.inputCharSet";
    // Deprecated: geoIp.databaseFile (use defaultValuegeoIp.countryDatabaseFile instead)
    public static final String GEO_IP_DATABASE_FILE_KEY = "geoIp.databaseFile";
    public static final String GEO_IP_COUNTRY_DATABASE_FILE_KEY = "geoIp.countryDatabaseFile";
    public static final String GEO_IP_ASN_DATABASE_FILE_KEY = "geoIp.asnDatabaseFile";
    public static final String GEO_IP_CITY_DATABASE_FILE_KEY = "geoIp.cityDatabaseFile";
    public static final String GEO_IP_USE_CITY_DATABASE_FOR_COUNTRY_LOOKUPS_KEY = "geoIp.useCityDatabaseForCountryLookups";
    public static final String FILENAME_MAPPER_LIST_KEY = "general.filenameMapperList";
    public static final String USE_DNS_JAVA_KEY = "general.useDnsJava";
    public static final String WHOIS_SERVER_KEY = "general.whoisServer";
    public static final String HIGH_PRIORITY_THRESHOLD_KEY = "general.highPriorityNotification.threshold";
    public static final String TIMESTAMP_WARNING_MAX_AGE_KEY = "general.timestampWarning.maxAge";
    public static final String PRINT_PROGRESS_INTERVAL_KEY = "general.printProgressInterval";
    public static final String FILE_ALREADY_PROCESSED_ACTION_KEY = "general.fileAlreadyProcessedAction";
    
    // dnsjava
    public static final String DNS_JAVA_USE_DNS_JAVA_KEY = "dnsJava.useDnsJava";
    public static final String DNS_JAVA_USE_SIMPLE_RESOLVER_KEY = "dnsJava.useSimpleResolver";
    public static final String DNS_JAVA_DNS_SERVERS_KEY = "dnsJava.dnsServers";
    public static final String DNS_JAVA_TIME_OUT_KEY = "dnsJava.timeOut";
    
    // Database
    public static final String DB_USERNAME_KEY = "db.username";
    public static final String DB_PASSWORD_KEY = "db.password";
    public static final String DB_SERVER_KEY = "db.server";
    public static final String DB_PORT_KEY = "db.port";
    public static final String DB_NAME_KEY = "db.name";
    public static final String JDBC_URL_KEY = "db.jdbc.url";
    public static final String JDBC_DRIVER_CLASS_KEY = "db.jdbc.driverClassName";

    // BGP
    public static final String BGP_IMPORT_FILE_KEY = "bgp.importFile";
    public static final String BGP_HARD_CODED_PREFIXES_KEY = "bgp.hardCodedPrefixes";
    
    // Export
    public static final String EXPORT_TEMPLATE_DIR_KEY = "export.templateDir";
    public static final String EXPORT_HEADER_FILE_KEY = "export.headerFile";
    public static final String EXPORT_ROW_FILE_KEY = "export.rowFile";
    public static final String EXPORT_FOOTER_FILE_KEY = "export.footerFile";
    public static final String EXPORT_CHAR_SET_KEY = "export.charSet";
    public static final String EXPORT_TIMESTAMP_FORMAT_KEY = "export.timestampFormat";
    public static final String EXPORT_REWRITERS_KEY = "export.rewriters";
    public static final String EXPORT_JOB_TYPE_NAME_MAPPER_KEY = "export.jobTypeNameMapper";    

    // Mail
    public static final String MAIL_SMTP_HOST_KEY = "mail.smtpHost";
    public static final String MAIL_DEBUG_KEY = "mail.debug";
    public static final String MAIL_FROM_ADDRESS_KEY = "mail.fromAddress";
    public static final String MAIL_TO_ADDRESSES_KEY = "mail.toAddresses";
    public static final String MAIL_CC_ADDRESSES_KEY = "mail.ccAddresses";
    public static final String MAIL_BCC_ADDRESSES_KEY = "mail.bccAddresses";
    public static final String MAIL_ARCHIVE_BCC_ADDRESSES_KEY = "mail.archiveBccAddresses";
    public static final String MAIL_SUMMARY_TO_ADDRESSES_KEY = "mail.mailJobSummary.toAddresses";
    public static final String MAIL_NOTIFICATION_TO_ADDRESSES_KEY = "mail.highPriorityNotification.toAddresses";
    public static final String MAIL_REPLY_TO_ADDRESSES_KEY = "mail.replyToAddresses";
    public static final String MAIL_HTML_MAIL_KEY = "mail.htmlMail";
    public static final String MAIL_IP_QUARANTINE_PERIOD_KEY = "mail.ipQuarantinePeriod";
    
    public static final String MAIL_SUBJECT_TEMPLATE_KEY = "mail.subjectTemplate";
    public static final String MAIL_JOB_SUMMARY_SUBJECT_TEMPLATE_KEY = "mail.mailJobSummary.subjectTemplate";
    public static final String MAIL_TEMPLATE_DIR_KEY = "mail.templateDir";
    public static final String MAIL_DEFAULT_LANGUAGE_CODE_KEY = "mail.defaultLanguageCode";
    public static final String MAIL_HEADER_FILE_KEY = "mail.headerFile";
    public static final String MAIL_ROW_FILE_KEY = "mail.rowFile";
    public static final String MAIL_FOOTER_FILE_KEY = "mail.footerFile";
    public static final String MAIL_ATTACHMENT_HEADER_FILE_KEY = "mail.attachmentHeaderFile";
    public static final String MAIL_ATTACHMENT_ROW_FILE_KEY = "mail.attachmentRowFile";
    public static final String MAIL_ATTACHMENT_FOOTER_FILE_KEY = "mail.attachmentFooterFile";    
    public static final String MAIL_ATTACHMENT_NAME_KEY = "mail.attachmentName";
    public static final String MAIL_TIMESTAMP_FORMAT_KEY = "mail.timestampFormat";
    public static final String MAIL_RAISE_ERROR_FOR_DEBUG_TEMPLATE_KEY = "mail.raiseErrorForDebugTemplate";
    
    // Mail Encryption
    public static final String MAIL_ENCRYPTION_TYPE_KEY = "mail.encryptionType";
    public static final String MAIL_ENCRYPTION_KEYPATH_KEY = "mail.encryptionKeyPath";
    public static final String MAIL_ENCRYPTION_SMIME_PW_KEY = "mail.enryptionSMIMEpw";
    public static final String MAIL_ENCRYPTION_PGP_PW_KEY = "mail.enryptionPGPpw";
    public static final String MAIL_ENCRYPTION_ENCRYPT_KEY = "mail.encryptMail";
    public static final String MAIL_ENCRYPTION_SIGN_KEY = "mail.signMail";

    // Filters
    public static final String FILTER_PRE_LINE_PROCESSOR_KEY = "filter.preLineProcessor.classNames";
    public static final String FILTER_PRE_PARSER_KEY = "filter.preParser.classNames";
    public static final String FILTER_PRE_DECORATOR_KEY = "filter.preDecorator.classNames";
    public static final String FILTER_PRE_STORAGE_KEY = "filter.preStorage.classNames";
    public static final String FILTER_PRE_EXPORT_KEY = "filter.preExport.classNames";
    public static final String FILTER_PRE_MAIL_KEY = "filter.preMail.classNames";
    public static final String FILTER_EXCLUDE_REG_EXP_KEY = "filter.regExpLineFilter.excludeRegExp";
    public static final String FILTER_INCLUDE_REG_EXP_KEY = "filter.regExpLineFilter.includeRegExp";
    public static final String FILTER_LINE_NUMBER_EXCLUDE_INTERVALS_KEY = "filter.lineNumberFilter.excludeIntervals";
    public static final String FILTER_LINE_NUMBER_INCLUDE_INTERVALS_KEY = "filter.lineNumberFilter.includeIntervals";
    public static final String FILTER_PRIORITY_INCLUDE_INTERVALS_KEY = "filter.priorityFilter.includeIntervals";
    public static final String FILTER_EXCLUDE_COUNTRY_CODES_KEY = "filter.countryCodeFilter.excludeCountryCodes";
    public static final String FILTER_INCLUDE_COUNTRY_CODES_KEY = "filter.countryCodeFilter.includeCountryCodes";
    public static final String FILTER_COUNTRY_CODE_ORGANIZATION_KEY = "filter.countryCodeFilter.organizationToFilter";
    public static final String FILTER_EXCLUDE_AS_NUMBERS_KEY = "filter.asnFilter.excludeAsNumbers";
    public static final String FILTER_INCLUDE_AS_NUMBERS_KEY = "filter.asnFilter.includeAsNumbers";
    public static final String FILTER_ASN_ORGANIZATION_KEY = "filter.asnFilter.organizationToFilter";
    public static final String FILTER_ATTRIBUTE_NAME_KEY = "filter.attributeFilter.attributeName";
    public static final String FILTER_ATTRIBUTE_EXCLUDE_REG_EXP_KEY = "filter.attributeFilter.excludeRegExp";
    public static final String FILTER_ATTRIBUTE_INCLUDE_REG_EXP_KEY = "filter.attributeFilter.includeRegExp";
    public static final String FILTER_OCCURRENCE_ATTRIBUTE_NAMES_KEY = "filter.occurrenceFilter.attributeNames"; 
    public static final String FILTER_OCCURRENCE_EXCLUDE_INTERVALS_KEY = "filter.occurrenceFilter.excludeIntervals";
    public static final String FILTER_OCCURRENCE_INCLUDE_INTERVALS_KEY = "filter.occurrenceFilter.includeIntervals";
    public static final String FILTER_OCCURRENCE_FILE_SORTED_KEY = "filter.occurrenceFilter.fileSorted";
    public static final String FILTER_MATCH_IP_ADDRESS_KEY = "filter.organizationFilter.matchIpAddress";
    public static final String FILTER_MATCH_HOSTNAME_KEY = "filter.organizationFilter.matchHostname";
    public static final String FILTER_MATCH_ASN_KEY = "filter.organizationFilter.matchAsn";
    
    // File Processor
    public static final String FILE_PROCESSOR_CLASS_NAME_KEY = "fileProcessor.className";
    public static final String FILE_PROCESSOR_CLASS_NAMES_KEY = "fileProcessor.classNames";
    public static final String FILE_PROCESSOR_DELETE_TMP_FILES_KEY = "fileProcessor.deleteTmpFiles";
    public static final String FILE_PROCESSOR_OS_COMMAND_KEY = "fileProcessor.osCommandProcessor.command";
    public static final String FILE_PROCESSOR_DIFF_COMMAND_KEY = "fileProcessor.diffProcessor.command";
    public static final String FILE_PROCESSOR_DIFF_OLD_FILES_DIR_KEY = "fileProcessor.diffProcessor.oldFilesDir";
    public static final String FILE_PROCESSOR_DIFF_NO_OF_BACKUPS_TO_KEEP_KEY = "fileProcessor.diffProcessor.noOfBackupsToKeep";
    public static final String FILE_PROCESSOR_XML_TO_ROW_START_ELEMENT_KEY = "fileProcessor.xmlToRowFileProcessor.startElement";
    public static final String FILE_PROCESSOR_XML_TO_ROW_ELEMENTS_TO_SAVE_KEY = "fileProcessor.xmlToRowFileProcessor.elementsToSave";
    public static final String FILE_PROCESSOR_XML_TO_ROW_OUTPUT_SEPARATOR_KEY = "fileProcessor.xmlToRowFileProcessor.outputSeparator";
    // Deprecated: fileProcessor.xmlToRowFileProcessor.deleteOutputFile (use fileProcessor.deleteTmpFiles instead)
    public static final String FILE_PROCESSOR_XML_TO_ROW_DELETE_OUTPUT_FILE_KEY = "fileProcessor.xmlToRowFileProcessor.deleteOutputFile";
    public static final String FILE_PROCESSOR_DNS_NO_OF_THREADS_KEY = "fileProcessor.multithreadedDnsProcessor.noOfThreads";
    public static final String FILE_PROCESSOR_DNS_REVERSE_DNS_LOOKUP_KEY = "fileProcessor.multithreadedDnsProcessor.reverseDnsLookup";
    public static final String FILE_PROCESSOR_DNS_REG_EXP_IP_KEY = "fileProcessor.multithreadedDnsProcessor.regExpIp";
    public static final String FILE_PROCESSOR_DNS_REG_EXP_HOSTNAME_KEY = "fileProcessor.multithreadedDnsProcessor.regExpHostname";
    
    // Line Processor
    public static final String LINE_PROCESSOR_CLASS_NAME_KEY = "lineProcessor.className";
    public static final String LINE_MERGER_START_REG_EXP_KEY = "lineProcessor.merger.startRegExp";
    public static final String LINE_MERGER_END_REG_EXP_KEY = "lineProcessor.merger.endRegExp";
    public static final String LINE_MERGER_RESTART_IF_START_FOUND_KEY = "lineProcessor.merger.restartIfStartFound";
    public static final String LINE_MERGER_SEPARATOR_KEY = "lineProcessor.merger.separator";
    public static final String LINE_SPLITTER_SEPARATOR_REG_EXP_KEY = "lineProcessor.splitter.separatorRegExp";
    public static final String LINE_SPLITTER_ITEM_REG_EXP_KEY = "lineProcessor.splitter.itemRegExp";
    public static final String LINE_SPLITTER_APPEND_ORIGINAL_LOG_ROW_KEY = "lineProcessor.splitter.appendOriginalLogRow";
    
    // Decorators
    public static final String DECORATOR_CLASS_NAMES_KEY = "decorator.classNames";
    public static final String DECORATOR_PRE_EXPORT_CLASS_NAMES_KEY = "decorator.preExport.classNames";
    public static final String DECORATOR_PRE_MAIL_CLASS_NAMES_KEY = "decorator.preMail.classNames";
    public static final String DECORATOR_COMBINED_DECORATOR_CLASS_NAMES_KEY = "decorator.combinedDecorator.classNames";
    public static final String DECORATOR_USE_ORGANIZATION_MATCHER_KEY = "decorator.useOrganizationMatcher";
    public static final String DECORATOR_MATCH_IP_ADDRESS_KEY = "decorator.organizationMatcher.matchIpAddress";
    public static final String DECORATOR_MATCH_HOSTNAME_KEY = "decorator.organizationMatcher.matchHostname";
    public static final String DECORATOR_MATCH_ASN_KEY = "decorator.organizationMatcher.matchAsn";
    public static final String DECORATOR_COUNTRY_CODES_TO_ADD_KEY = "decorator.countryCodeFromHostnameDecorator.countryCodesToAdd";
    public static final String DECORATOR_USE_ASN_IN_LOG_ENTRY_KEY = "decorator.asnGeoIpDecorator.useAsnInLogEntry";
    public static final String DECORATOR_ADD_AS_NAME_KEY = "decorator.asnGeoIpDecorator.addAsName";
    public static final String DECORATOR_URL_TO_HOSTNAME_USE_PRIMARY_ORG_KEY = "decorator.urlToHostnameDecorator.usePrimaryOrg";
    public static final String DECORATOR_GEOLOCATION_FIELDS_TO_ADD_KEY = "decorator.geolocationDecorator.fieldsToAdd";
    
    // Parser
    public static final String PARSER_CLASS_NAME_KEY = "parser.className";
    public static final String PARSER_PARSE_ERROR_THRESHOLD_KEY = "parser.parseErrorThreshold";
    public static final String PARSER_MAX_NO_OF_PARSE_ERRORS = "parser.maxNoOfParseErrors";
    public static final String PARSER_TIME_STAMP_FORMAT_KEY = "parser.timestampFormat";
    public static final String PARSER_ADD_CURRENT_DATE_TO_TIMESTAMP_KEY = "parser.addCurrentDateToTimestamp";
    public static final String PARSER_DEFAULT_TIME_ZONE_KEY = "parser.defaultTimeZone";
    public static final String PARSER_CHECK_UNUSED_VARIABLES_KEY = "parser.checkUnusedVariables";
    public static final String PARSER_LINE_REG_EXP_KEY = "parser.lineRegExp";
    public static final String PARSER_ITEM_PREFIX = "parser.item.";
    public static final String PARSER_ADDITIONAL_ITEM_PREFIX = "parser.item.additionalItem.";
    public static final String PARSER_FREE_TEXT_KEY = "parser.item.freeText";
    public static final String PARSER_TRIM_VALUE_KEY = "parser.trimValue";
    public static final String PARSER_REMOVE_ENCLOSING_CHARS_FROM_VALUE_KEY = "parser.removeEnclosingCharsFromValue";
    public static final String PARSER_REWRITERS_KEY = "parser.rewriters";
    public static final String PARSER_REMOVE_TRAILING_SPACES_KEY = "parser.removeTrailingSpaces";
    public static final String PARSER_EXPAND_IP_RANGE_WITH_ZERO_OCTETS_KEY = "parser.expandIpRangeWithZeroOctets";
    
    // RSS
    public static final String RSS_FACTORY_CLASS_NAME_KEY = "rss.factoryClassName";
    public static final String RSS_FORMAT_KEY = "rss.format";
    // Job RSS
    public static final String RSS_JOB_ENABLED_KEY = "rss.job.enabled";
    public static final String RSS_JOB_FILE_KEY = "rss.job.file";
    public static final String RSS_JOB_CONTENT_TITLE_KEY = "rss.job.content.title";
    public static final String RSS_JOB_CONTENT_LINK_KEY = "rss.job.content.link";
    public static final String RSS_JOB_CONTENT_DESCRIPTION_KEY = "rss.job.content.description";
    public static final String RSS_JOB_CONTENT_AUTHOR_KEY = "rss.job.content.author";
    public static final String RSS_JOB_CONTENT_COPYRIGHT_KEY = "rss.job.content.copyright";
    public static final String RSS_JOB_MAX_NO_OF_ITEMS_KEY = "rss.job.maxNoOfItems";
    public static final String RSS_JOB_ITEM_EXPIRE_TIME_IN_MINUTES_KEY = "rss.job.itemExpireTimeInMinutes";
    // Stats RSS
    public static final String RSS_STATS_FORMAT_KEY = "rss.stats.format";
    public static final String RSS_STATS_FILE_KEY = "rss.stats.file";
    public static final String RSS_STATS_CONTENT_TITLE_KEY = "rss.stats.content.title";
    public static final String RSS_STATS_CONTENT_LINK_KEY = "rss.stats.content.link";
    public static final String RSS_STATS_CONTENT_DESCRIPTION_KEY = "rss.stats.content.description";
    public static final String RSS_STATS_CONTENT_AUTHOR_KEY = "rss.stats.content.author";
    public static final String RSS_STATS_CONTENT_COPYRIGHT_KEY = "rss.stats.content.copyright";
    public static final String RSS_STATS_MAX_NO_OF_ITEMS_KEY = "rss.stats.maxNoOfItems";
    public static final String RSS_STATS_ITEM_EXPIRE_TIME_IN_MINUTES_KEY = "rss.stats.itemExpireTimeInMinutes";

    // Report
    // Deprecated: flash.outputDir (use report.outputDir instead)
    public static final String FLASH_NO_OF_WEEKS_KEY = "flash.noOfWeeks";
    // Deprecated: flash.noOfWeeks (use report.statistics.noOfWeeks instead)
    public static final String FLASH_OUTPUT_DIR_KEY = "flash.outputDir";
    public static final String REPORT_CLASS_NAMES_KEY = "report.classNames";
    public static final String REPORT_OUTPUT_DIR_KEY = "report.outputDir";
    public static final String REPORT_TEMPLATE_DIR_KEY = "report.templateDir";
    public static final String REPORT_STATISTICS_NO_OF_WEEKS_KEY = "report.statistics.noOfWeeks";
    public static final String REPORT_GEOLOCATION_NO_OF_WEEKS_KEY = "report.geolocation.noOfWeeks";
    public static final String REPORT_GEOLOCATION_GENERATE_INTERNAL_REPORT_KEY = "report.geolocation.generateInternalReport";
    public static final String REPORT_GEOLOCATION_NO_OF_ENTRIES_IN_CITY_REPORT_KEY = "report.geolocation.noOfEntriesInCityReport";
    public static final String REPORT_GEOLOCATION_JOB_TYPE_KILL_LIST_KEY = "report.geolocation.jobTypeKillList";
    public static final String REPORT_GEOLOCATION_ORGANIZATION_TYPE_KILL_LIST_KEY = "report.geolocation.organizationTypeKillList";
    public static final String REPORT_GEOLOCATION_ORGANIZATION_TYPE_NAME_MAPPER_KEY = "report.geolocation.organizationTypeNameMapper";
    public static final String REPORT_ORGANIZATION_NO_OF_HOURS_KEY = "report.organization.noOfHours";
    public static final String REPORT_ORGANIZATION_JOB_TYPES_KEY = "report.organization.jobTypes";
    public static final String REPORT_ORGANIZATION_RECIPIENTS_KEY = "report.organization.recipients";
        
    // Import
    public static final String CONTACTS_IMPORT_FILE_KEY = "import.dataFile";
    
    // UI (management command line user interface)
    public static final String UI_DEFAULT_CC_KEY = "ui.organizationHandler.defaultCountryCode";
    public static final String UI_DEFAULT_LC_KEY = "ui.organizationHandler.defaultLanguageCode";
    public static final String UI_OUTPUT_DIR_KEY = "ui.organizationHandler.outputDir";
    public static final String UI_VALID_ROLES_KEY = "ui.organizationHandler.validRoles";
    public static final String UI_TIMESTAMP_FORMAT_KEY = "ui.organizationHandler.timestampFormat";
    
    // TicketHandler
    public static final String TICKET_HANDLER_CLASS = "ticketHandler.className";
    public static final String TICKET_HANDLER_CREATE_CHILD = "ticketHandler.createChild";
    public static final String TICKET_HANDLER_VALUE_KEYS = "ticketHandler.valueKeys";
    public static final String TICKET_HANDLER_SENDS_MAIL = "ticketHandler.sendsMail";
    public static final String TICKET_HANDLER_QUEUE_NAME = "ticketHandler.queueName";
    public static final String TICKET_HANDLER_USER = "ticketHandler.user";
    public static final String TICKET_HANDLER_PASSWORD = "ticketHandler.password";
    public static final String TICKET_HANDLER_TICKET_OWNER = "ticketHandler.ticketOwner";
    public static final String TICKET_HANDLER_URL = "ticketHandler.url";
    public static final String TICKET_HANDLER_RESOLVE_AFTER_SEND = "ticketHandler.resolveAfterSend";
    public static final String TICKET_HANDLER_RESOLVED_STATUS = "ticketHandler.resolvedStatus";
    public static final String TICKET_HANDLER_RESOLVE_SLEEP_TIME = "ticketHandler.resolveSleepTime";
    public static final String TICKET_HANDLER_TO_ADDRESS = "ticketHandler.toAddress";
    public static final String TICKET_HANDLER_FROM_ADDRESS = "ticketHandler.fromAddress";
    public static final String TICKET_HANDLER_ARCHIVE_ADDRESS = "ticketHandler.archiveAddress";
    
    

    /** Filename for Megatron global properties. */
    public static final String GLOBALS_PROPS_FILE = "/etc/megatron/megatron-globals.properties";

    /** Key in system properties for global properties file. */
    public static final String MEGATRON_CONFIG_FILE_KEY = "megatron.configfile";

    // Cannot use logging becuse it's not yet initialized.
    // private static final Logger log = Logger.getLogger(GlobalProperties.class);

    private static final String TRUE = "true";

    private static AppProperties singleton;

    private Map<String, String> globalProps;
    private TypedProperties globalTypedProps;
    private List<File> jobTypeFiles;
    private List<String> jobTypeNames;
    private List<String> inputFiles;


    /**
     * Returns singleton.
     */
    public static synchronized AppProperties getInstance() {
        if (singleton == null) {
            singleton = new AppProperties();
        }
        return singleton;
    }


    /**
     * Constructor. Private due to singleton.
     */
    private AppProperties() {
        // empty
    }


    /**
     * Loads properties file and parses specified command line arguments.
     */
    public void init(String[] args) throws MegatronException, CommandLineParseException {
        // -- Hard-coded properties
        // Use US as default locale. Dates with names, e.g. 04/Jul/2009, may otherwise be incorrect.
        Locale.setDefault(Locale.US);
        
        // Use UTC as default time-zone. All time-stamps in db are in UTC-time.
        TimeZone.setDefault(TimeZone.getTimeZone("UTC"));

        // -- Read global properties and parse command line
        this.globalProps = loadGlobalProperties();
        parseCommandLine(args);
        this.globalTypedProps = new TypedProperties(globalProps, null);

        // -- init jobTypeFiles and jobTypeNames
        this.jobTypeFiles = listJobTypeFiles();
        this.jobTypeNames = new ArrayList<String>();
        for (Iterator<File> iterator = jobTypeFiles.iterator(); iterator.hasNext(); ) {
            String filename = iterator.next().getName();
            this.jobTypeNames.add(StringUtil.removeSuffix(filename, ".properties"));
        }
    }


    /**
     * Returns global properties.<p>
     * Note: This method should be used sparsely. Use TypedProperties-object created by
     * createTypedPropertiesForCli or createTypedPropertiesForWeb instead.
     */
    public TypedProperties getGlobalProperties() {
        return globalTypedProps;
    }

    
    /**
     * Maps specified filename (without path) to a job-type.
     * 
     * @return found job-type, or null if not found.
     */
    public String mapFilenameToJobType(String filename, boolean ignoreCliArgument) {
        String result = null;
        Logger log = Logger.getLogger(getClass());

        if (!ignoreCliArgument) { 
            result = globalTypedProps.getString(TypedProperties.CLI_JOB_TYPE_KEY, null);
            if (result != null) {
                log.debug("Using job-type specified in CLI-argument: " + result);
                return result;
            }
        }
    
        List<NameValuePair> nvList = globalTypedProps.getNameValuePairList(FILENAME_MAPPER_LIST_KEY, null);
        if (nvList != null) {
            for (Iterator<NameValuePair> iterator = nvList.iterator(); (result == null) && iterator.hasNext(); ) {
                NameValuePair nv = iterator.next();
                try {
                    if (filename.matches(nv.getName())) {
                        log.debug("Job-type found: " + nv.getValue() + ". Filename '" + filename + "' matches reg-exp '" + nv.getName() + "'.");
                        result = nv.getValue();
                    }
                } catch(PatternSyntaxException e) {
                    log.error("Invalid reg-exp in filename mapper: " + nv.getName());
                }
            }
        } else {
            log.error("Filename mapper not defined in config. Property name: " + FILENAME_MAPPER_LIST_KEY);
        }
        
        return result;
    }

    
    /**
     * Creates properties for specified job-type, which includes CLI-arguments and global properties.
     * Use this method in the CLI application.
     */
    public TypedProperties createTypedPropertiesForCli(String jobType) throws MegatronException {
        try {
            Map<String, String> jobTypeProps = loadJobTypeProperties(jobType);
            return new TypedProperties(jobTypeProps, Collections.singletonList(globalProps));
        } catch (MegatronException e) {
            throw new MegatronException("Invalid job-type name: " + jobType, e);
        }        
    }

    
    /**
     * Creates properties for specified job-type, which includes specifed CLI-arguments and global properties.
     * CLI-properties are used in the web application, but assigned for each run.
     * Use this method in the web application.
     * 
     * @param jobType job-type name, e.g. "shadowserver-ddos".
     * @param cliArgs CLI-arguments. Key: CLI-consts, e.g. CLI_NO_DB. Value: argument value, e.g. "true". 
     */
    public TypedProperties createTypedPropertiesForWeb(String jobType, Map<String, String> cliArgs) throws MegatronException {
        try {
            Map<String, String> jobTypeProps = loadJobTypeProperties(jobType);
            for (Iterator<String> iterator = cliArgs.keySet().iterator(); iterator.hasNext(); ) {
                String key = iterator.next();
                String value = cliArgs.get(key);
                jobTypeProps.put(key, value);
            }
            return new TypedProperties(jobTypeProps, Collections.singletonList(globalProps));
        } catch (MegatronException e) {
            throw new MegatronException("Invalid job-type name: " + jobType, e);
        }        
    }


    /**
     * Returns input filenames specified at the command line.
     */
    public List<String> getInputFiles() {
        return this.inputFiles;
    }

    
    public List<File> getJobTypeFiles() {
        return this.jobTypeFiles;
    }


    public List<String> getJobTypeNames() {
        return this.jobTypeNames;
    }


    public String getJobTypeConfigDir() {
        return globalTypedProps.getString(JOB_TYPE_CONFIG_DIR_KEY, null);
    }
    
    
    public String getJdbcUrl() {
        // Format: jdbc:mysql://{db.server}:{db.port}/{db.name}
        String result = globalTypedProps.getString(JDBC_URL_KEY, "jdbc:mysql://{db.server}:{db.port}/{db.name}");
        String dbServer = globalTypedProps.getString(DB_SERVER_KEY, "127.0.0.1");
        String dbPort = globalTypedProps.getString(DB_PORT_KEY, "3306");
        String dbName = globalTypedProps.getString(DB_NAME_KEY, "ossec");

        result = StringUtil.replace(result, "{db.server}", dbServer);
        result = StringUtil.replace(result, "{db.port}", dbPort);
        result = StringUtil.replace(result, "{db.name}", dbName);

        return result;
    }

    
    /**
     * Loads global properties.
     *
     * @throws MegatronException if global properties cannot be read.
     */
    private Map<String, String> loadGlobalProperties() throws MegatronException {
        String filename = System.getProperty(MEGATRON_CONFIG_FILE_KEY);
        filename = (filename != null) ? filename : GLOBALS_PROPS_FILE;
        return loadPropertiesFile(filename);
    }


    /**
     * Loads job-type properties.
     *
     * @throws IOException if properties cannot be read.
     */
    private Map<String, String> loadJobTypeProperties(String jobTypeName) throws MegatronException {
        String filename = FileUtil.concatPath(getJobTypeConfigDir(), jobTypeName + ".properties");
        Map<String, String> result = loadPropertiesFile(filename); 
        result.put(JOB_TYPE_NAME_KEY, jobTypeName);
        return result;
    }


    /**
     * Loads specified properties file.
     *
     * @throws MegatronException if global properties cannot be read.
     */
    private Map<String, String> loadPropertiesFile(String filename) throws MegatronException {
        // The class Properties cannot be used:
        //   1. Back-slashes in regular expressions are removed.
        //   2. Order is not preserved.

        // LinkedHashMap preserves insertion order.
        Map<String, String> result = new LinkedHashMap<String, String>();

        final String nameValueSeparator = "=";

        File file = new File(filename);
        BufferedReader in = null;
        try {
            in = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));

            int lineNo = 0;
            String line = null;
            while ((line = in.readLine()) != null) {
                ++lineNo;
                String trimmedLine = line.trim();
                if ((trimmedLine.length() == 0) || trimmedLine.startsWith(Constants.CONFIG_COMMENT_PREFIX)) {
                    continue;
                }

                // split line into name-value
                int index = line.indexOf(nameValueSeparator);
                if ((index == -1) || (index == 0)) {
                    String msg = "Parse error at line " + lineNo + ": Separator not found.";
                    throw new IOException(msg);
                }
                String name = line.substring(0, index);
                String value = ((index + 1) < line.length()) ? line.substring(index + 1, line.length()) : "";
                result.put(name, value);
            }
        } catch (IOException e) {
            String msg = "Cannot read properties file: " + file.getAbsolutePath();
            System.err.println(msg);
            throw new MegatronException(msg, e);
        } finally {
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e) {
                    String msg = "Cannot close properties file: " + file.getAbsolutePath();
                    System.err.println(msg);
                    throw new MegatronException(msg, e);
                }
            }
        }

        return result;
    }


    /**
     * Parsers specified command line arguments and adds them to properties map.
     */
    @SuppressWarnings("unused")
    private void parseCommandLine(String[] args) throws MegatronException, CommandLineParseException {
        if (args == null) {
            return;
        }

        inputFiles = new ArrayList<String>();
        for (int i = 0; i < args.length; i++) {
            String arg = args[i].trim();

            if (arg.startsWith("-")) {
                if (arg.equals("--version") || arg.equals("-v")) {
                    throw new CommandLineParseException(CommandLineParseException.SHOW_VERSION_ACTION);
                } else if (arg.equals("--help") || arg.equals("-h")) {
                    throw new CommandLineParseException(CommandLineParseException.SHOW_USAGE_ACTION);
                } else if (arg.equals("--slurp") || arg.equals("-s")) {
                    globalProps.put(TypedProperties.CLI_SLURP_KEY, TRUE);
                } else if (arg.equals("--export") || arg.equals("-e")) {
                    globalProps.put(TypedProperties.CLI_EXPORT_KEY, TRUE);
                } else if (arg.equals("--delete") || arg.equals("-d") || arg.equals("--delete-all") || arg.equals("-D")) {
                    // job specified after delete switch? 
                    int nextIndex = i + 1;
                    if ((nextIndex < args.length) && !args[nextIndex].startsWith("-")) {
                        i = nextIndex;
                        globalProps.put(TypedProperties.CLI_JOB_KEY, args[i]);
                    } else {
                        // otherwise --job must be present
                        List<String> argList = new ArrayList<String>(Arrays.asList(args));
                        if (!(argList.contains("--job") || argList.contains("--j"))) {
                            throw new CommandLineParseException("Job name not specified.");
                        }
                    }
                    
                    if (arg.equals("--delete") || arg.equals("-d")) {
                        globalProps.put(TypedProperties.CLI_DELETE_KEY, TRUE);
                    } else {
                        globalProps.put(TypedProperties.CLI_DELETE_ALL_KEY, TRUE);    
                    }
                } else if (arg.equals("--job") || arg.equals("-j")) {
                    ++i;
                    if (i == args.length) {
                        throw new CommandLineParseException("Missing argument to " + arg);
                    }
                    globalProps.put(TypedProperties.CLI_JOB_KEY, args[i]);
                } else if (arg.equals("--job-type") || arg.equals("-t")) {
                    ++i;
                    if (i == args.length) {
                        throw new CommandLineParseException("Missing argument to " + arg);
                    }
                    globalProps.put(TypedProperties.CLI_JOB_TYPE_KEY, args[i]);
                } else if (arg.equals("--output-dir") || arg.equals("-o")) {
                    ++i;
                    if (i == args.length) {
                        throw new CommandLineParseException("Missing argument to " + arg);
                    }
                    String outputDirStr = args[i];
                    File outputDir = new File(outputDirStr);
                    if (!outputDir.isDirectory()) {
                        throw new MegatronException("Output directory not found: " + outputDir.getAbsolutePath());
                    }
                    globalProps.put(TypedProperties.CLI_OUTPUT_DIR_KEY, outputDirStr);
                } else if (arg.equals("--id") || arg.equals("-i")) {
                    ++i;
                    if (i == args.length) {
                        throw new CommandLineParseException("Missing argument to " + arg);
                    }
                    String idStr = args[i];
                    try {
                        long id = Long.parseLong(idStr);
                        if (id <= 0L) {
                            throw new NumberFormatException("Id is zero or negative.");    
                        }
                    } catch (NumberFormatException e) {
                        throw new CommandLineParseException("Invalid RTIR id (--id): " + idStr, e);
                    }
                    globalProps.put(TypedProperties.CLI_ID_KEY, idStr);
                } else if (arg.equals("--prio") || arg.equals("-p")) {
                    ++i;
                    if (i == args.length) {
                        throw new CommandLineParseException("Missing argument to " + arg);
                    }
                    String prioStr = args[i];
// Skip check. OK with a list of intervals, e.g. --prio 55-60,70-75
//                    try {
//                        long prio = Long.parseLong(prioStr);
//                        if ((prio < 0L) || (prio > 100L)) {
//                            throw new NumberFormatException("Prio is not in range.");    
//                        }
//                    } catch (NumberFormatException e) {
//                        String msg = "Invalid prio (--prio): " + prioStr + ". Use --list-prios to list priorities.";
//                        throw new CommandLineParseException(msg, e);
//                    }
                    globalProps.put(TypedProperties.CLI_PRIO_KEY, prioStr);
                } else if (arg.equals("--list-prios") || arg.equals("-P")) {
                    globalProps.put(TypedProperties.CLI_LIST_PRIOS_KEY, TRUE);
                } else if (arg.equals("--list-jobs") || arg.equals("-l")) {
                    globalProps.put(TypedProperties.CLI_LIST_JOBS_KEY, TRUE);
                    // days specified after switch?
                    int nextIndex = i + 1;
                    if ((nextIndex < args.length) && !args[nextIndex].startsWith("-")) {
                        i = nextIndex;
                        String val = args[i];
                        try {
                            int days = Integer.parseInt(val);
                            if ((days <= 0) || (days > 1000)) {
                                throw new NumberFormatException("No. of days must be in the range 1-1000.");    
                            }
                        } catch (NumberFormatException e) {
                            throw new CommandLineParseException("Invalid no. of days (--list-jobs): " + val, e);
                        }
                        globalProps.put(TypedProperties.CLI_DAYS_IN_LIST_JOBS_KEY, val);
                    }
                } else if (arg.equals("--whois") || arg.equals("-w")) {
                    globalProps.put(TypedProperties.CLI_WHOIS_KEY, TRUE);
                    if (++i >= args.length) {
                        throw new CommandLineParseException("Missing list of IPs and hostnames (or file) to argument " + arg);
                    }
                    while (i < args.length) {
                        inputFiles.add(args[i]);                        
                        ++i;
                    }
                }  else if (arg.equals("--job-info") || arg.equals("-I")) {
                    // job specified after switch? 
                    int nextIndex = i + 1;
                    if ((nextIndex < args.length) && !args[nextIndex].startsWith("-")) {
                        i = nextIndex;
                        globalProps.put(TypedProperties.CLI_JOB_KEY, args[i]);
                    } else {
                        // otherwise --job must be present
                        List<String> argList = new ArrayList<String>(Arrays.asList(args));
                        if (!(argList.contains("--job") || argList.contains("--j"))) {
                            throw new CommandLineParseException("Job name not specified.");
                        }
                    }
                    globalProps.put(TypedProperties.CLI_JOB_INFO_KEY, TRUE);
                } else if (arg.equals("--no-db") || arg.equals("-n")) {
                    globalProps.put(TypedProperties.CLI_NO_DB_KEY, TRUE);
                } else if (arg.equals("--stdout") || arg.equals("-S")) {
                    globalProps.put(TypedProperties.CLI_STDOUT_KEY, TRUE);
                } else if (arg.equals("--import-contacts")) {
                    globalProps.put(TypedProperties.CLI_IMPORT_CONTACTS_KEY, TRUE);
                } else if (arg.equals("--import-bgp")) {
                    globalProps.put(TypedProperties.CLI_IMPORT_BGP_KEY, TRUE);
                } else if (arg.equals("--update-netname")) {
                    globalProps.put(TypedProperties.CLI_UPDATE_NETNAME_KEY, TRUE);
                } else if (arg.equals("--add-addresses") || arg.equals("--delete-addresses")) {
                    if (arg.equals("--delete-addresses")) {
                        globalProps.put(TypedProperties.CLI_DELETE_ADDRESSES_KEY, TRUE);
                    } else {
                        globalProps.put(TypedProperties.CLI_ADD_ADDRESSES_KEY, TRUE);
                    }
                    ++i;
                    if ((i < args.length) && !args[i].startsWith("-")) {
                        globalProps.put(TypedProperties.CLI_ADDRESSES_FILE_KEY, args[i]);
                    } else {
                        throw new CommandLineParseException("Missing address file to argument " + arg);
                    }
                } else if (arg.equals("--create-rss")) {
                    globalProps.put(TypedProperties.CLI_CREATE_STATS_RSS_KEY, TRUE);
                } else if (arg.equals("--create-xml")) {
                    // --create-xml is deprecated. Use --create-reports instead. 
                    globalProps.put(TypedProperties.CLI_CREATE_FLASH_XML_KEY, TRUE);
                } else if (arg.equals("--create-reports")) {
                    globalProps.put(TypedProperties.CLI_CREATE_REPORTS_KEY, TRUE);
                } else if (arg.equals("--create-report")) {
                    ++i;
                    if (i == args.length) {
                        throw new CommandLineParseException("Missing argument to " + arg);
                    }
                    String className = args[i];
                    try {
                        Class<?> reportGeneratorClass = Class.forName(className);
                        IReportGenerator reportGenerator = (IReportGenerator)reportGeneratorClass.newInstance();
                    } catch (Exception e) {
                        // ClassNotFoundException, InstantiationException, IllegalAccessException
                        String msg = "Argument for --create-report must be a Java-class that implements IReportGenerator: " + className;
                        throw new MegatronException(msg, e);
                    }
                    globalProps.put(TypedProperties.CLI_CREATE_REPORT_KEY, className);
                } else if (arg.equals("--ui-org")) {
                    globalProps.put(TypedProperties.CLI_UI_ORGANIZATIONS, TRUE);                    
                } else if (arg.equals("--mail-dry-run") || arg.equals("-1")) {
                    globalProps.put(TypedProperties.CLI_MAIL_DRY_RUN_KEY, TRUE);
                } else if (arg.equals("--mail-dry-run2") || arg.equals("-2")) {
                    globalProps.put(TypedProperties.CLI_MAIL_DRY_RUN2_KEY, TRUE);
                } else if (arg.equals("--mail") || arg.equals("-m")) {
                    globalProps.put(TypedProperties.CLI_MAIL_KEY, TRUE);
                } else if (arg.equals("--use-org2") || arg.equals("-b")) {
                    globalProps.put(TypedProperties.CLI_USE_ORG2_KEY, TRUE);
                } else {
                    throw new CommandLineParseException("Unrecognized option: " + arg);
                }
            } else {
                File file = new File(arg);
                if (!file.canRead()) {
                    throw new MegatronException("Cannot read input file: " + file.getAbsolutePath());
                }
                inputFiles.add(arg);
            }
        }
    }


    private List<File> listJobTypeFiles() throws MegatronException {
        String dirStr = globalProps.get(JOB_TYPE_CONFIG_DIR_KEY);
        if (StringUtil.isNullOrEmpty(dirStr)) {
            throw new MegatronException("Job-type config dir not specified in global properties.");
        }
        File file = new File(dirStr);
        File[] files = file.listFiles(new FileFilter() {
            @Override
            public boolean accept(File pathname) {
                return pathname.getName().endsWith(".properties");
            }
        });
        if (files == null) {
            throw new MegatronException("Cannot list job-type config files in directory: " + file.getAbsolutePath());
        }

        List<File> result = new ArrayList<File>(Arrays.asList(files));
        return result;
    }

} public class Interval implements Comparable<Interval> {
    private long start;
    private long end;
    
    
    public Interval(long start, long end) {
        this.start = start;
        this.end = end;
    }


    /**
     * Constructs an interval from specified string. Supports open
     * start and end, for example: "1-100", "-100", "50-". Only positive
     * numbers are valid. 
     */
    public Interval(String str) throws MegatronException {
        parse(str);
    }

    
    /**
     * Returns true if specified value is in this interval. 
     */
    public final boolean contains(long val) {
        return (val >= start) && (val <= end); 
    }

    
    /**
     * Returns true if specified interval intersects with this interval. 
     */
    public final boolean overlaps(Interval interval) {
        return contains(interval.getStart()) || contains(interval.getEnd()) ||
            interval.contains(start) || interval.contains(end);
    }
    
    
    public long getStart() {
        return start;
    }


    public long getEnd() {
        return end;
    }

    
    // implements Comparable
    @Override
    public int compareTo(Interval interval2) {
        long start2 = interval2.getStart();
        long end2 = interval2.getEnd();
        
        if (start < start2) {
            return -1;
        } else if (start > start2) {
            return 1;
        } else {
            // start == start2
            return (end < end2) ? -1 : ((end == end2) ? 0 : 1);
        }
    }

    
    // generated by Eclipse
    @Override
    public int hashCode() {
        final int prime = 31;
        int result = 1;
        result = prime * result + (int) (end ^ (end >>> 32));
        result = prime * result + (int) (start ^ (start >>> 32));
        return result;
    }

    
    @Override
    public boolean equals(Object obj) {
        boolean result = false;
        if (obj instanceof Interval) {
            Interval interval2 = (Interval)obj;
            result = (start == interval2.getStart()) && (end == interval2.getEnd());
        }
        return result;
    }

    
    @Override
    public String toString() {
        StringBuilder result = new StringBuilder();
        if (start != Long.MIN_VALUE) {
            result.append(start);
        }
        result.append('-');
        if (end != Long.MAX_VALUE) {
            result.append(end);
        }
        
        return result.toString();
    }

    
    private void parse(String str) throws MegatronException {
        if (StringUtil.isNullOrEmpty(str) || str.trim().equals("-")) {
            throw new MegatronException("Invalid interval: " + str);
        }
        
        String[] headTail = StringUtil.splitHeadTail(str, "-", false);
        if ((headTail == null) || str.equals(headTail[0])) {
            throw new MegatronException("Invalid interval; '-' not found: " + str);
        }

        try {
            if (headTail[0].length() > 0) {
                start = Long.parseLong(headTail[0]);
            } else {
                start = Long.MIN_VALUE;
            }
            if (headTail[1].length() > 0) {
                end = Long.parseLong(headTail[1]);
            } else {
                end = Long.MAX_VALUE;
            }
        } catch (NumberFormatException e) {
            throw new MegatronException("Invalid interval; cannot parse number: " + str, e);
        }
    }
    
} public class IntervalList {
    private ArrayList<Interval> intervals;
    private boolean sorted = false;
    
    
    public IntervalList() {
        this.intervals = new ArrayList<Interval>();
    }

    
    public IntervalList(IntervalList intervalList) {
        this.intervals = new ArrayList<Interval>(intervalList.intervals);
        this.sorted = intervalList.sorted;
    }

    
    /**
     * Creates an IntervalList-object from specified comma separated list of intervals.
     * 
     * @return created IntervalList-object, or null if intervalStrings is empty. 
     * @throws MegatronException if intervalStrings is invalid.
     */
    public static IntervalList createIntervalList(String[] intervalStrings) throws MegatronException {
        if ((intervalStrings == null) || (intervalStrings.length == 0)) {
            return null;
        }
        
        IntervalList result = new IntervalList();
        for (int i = 0; i < intervalStrings.length; i++) {
            try {
                result.add(new Interval(intervalStrings[i]));
            } catch (MegatronException e) {
                String msg = "Cannot create interval from string: " + intervalStrings[i]; 
                throw new MegatronException(msg, e);
            }
        }
    
        return result;
    }

    
    public void add(Interval interval) {
        intervals.add(interval);
        sorted = false;
    }

    
    public void sort() {
        if (!sorted) {
            Collections.sort(intervals);
            sorted = true;
        }
    }

    
    /**
     * Returns true if two or more intervals in this list intersects.
     */
    public boolean containsOverlaps() {
        if (intervals.size() <= 1) {
            return false;
        }
        
        sort();

        boolean result = false;
        for (int i = 0; !result && (i < intervals.size()); i++) {
            if (i > 0) {
                result = intervals.get(i).overlaps(intervals.get(i-1));
            }
            if (!result && (i < (intervals.size()-1))) {
                result = intervals.get(i).overlaps(intervals.get(i+1));
            }
        }
        
        return result;
    }

    
    /**
     * Returns first interval that overlaps with specified interval, or 
     * null if not found. 
     */
    public Interval findFirstInterval(Interval interval) {
        Interval result = null;

        for (Iterator<Interval> iterator = intervals.iterator(); (result == null) && iterator.hasNext(); ) {
            Interval candidate = iterator.next();
            if (candidate.overlaps(interval)) {
                result = candidate; 
            }
        }
        return result;
    }

    
    /**
     * Returns first interval in this list that contains specified value, or 
     * null if not found. 
     */
    public Interval findFirstInterval(long val) {
        List<Interval> intervals = findIntervalsInternal(val, true);
        return (intervals != null) ? intervals.get(0) : null;
    }

    
    /**
     * Returns intervals in this list that contains specified value.
     * If overlapping intervals exists, more than one element may
     * be returned. 
     */
    public List<Interval> findIntervals(long val) {
        List<Interval> intervals = findIntervalsInternal(val, false);
        return (intervals != null) ? intervals : new ArrayList<Interval>();
    }
    
    
    /**
     * Returns number of intervals specified value is a member of.
     */
    public int findNoOfIntervals(long val) {
        List<Interval> intervals = findIntervalsInternal(val, false);
        return (intervals != null) ? intervals.size() : 0;
    }
    
    
    /**
     * Returns number of intervals.
     */
    public int size() {
        return intervals.size();
    }
    
    
    @Override
    public String toString() {
        return (intervals != null) ? intervals.toString() : "null";
    }

    
    private List<Interval> findIntervalsInternal(long val, boolean onlyOneInterval) {
        if (intervals.size() <= 1) {
            if ((intervals.size() == 1) && intervals.get(0).contains(val)) {
                return Collections.singletonList(intervals.get(0));
            }
            return null;
        }

        sort();

        // Linear search with NO stop condition (just to be sure) 
        List<Interval> result = null;
        for (Iterator<Interval> iterator = intervals.iterator(); iterator.hasNext(); ) {
            Interval interval = iterator.next();
            if (interval.contains(val)) {
                if (result == null) {
                    result = new ArrayList<Interval>();
                }
                result.add(interval);
                if (onlyOneInterval) {
                    break;
                }
            }
        }
        
// Code block below seem to work, but the stop condition does not add a significant speed-up.  
//        // Linear search with stop condition (binary search was too complicated to implement)  
//        List<Interval> result = null;
//        Iterator<Interval> iterator = intervals.iterator();
//        Interval prevInterval = iterator.next();
//        if (prevInterval.contains(val)) {
//            if (result == null) {
//                result = new ArrayList<Interval>();
//            }
//            result.add(prevInterval);
//        }
//        boolean quit = false;
//        while (!quit && iterator.hasNext()) {
//            Interval interval = iterator.next();
//            if (interval.contains(val)) {
//                if (result == null) {
//                    result = new ArrayList<Interval>();
//                }
//                result.add(interval);
//            } else {
//                // is value in between of two intervals?
//                if ((prevInterval != null) && (prevInterval.getEnd() < val) && (val < interval.getStart())) {
//                    quit = true;
//                }
//            }
//            prevInterval = interval;
//        }

        return result;
    }
    
} public class JobContext {
    private TypedProperties props;
    private Job job;
    private DbManager dbManager;
    
    /** Time job was started (in ms). */
    private long startedTimestamp;

    /** Total number of lines in file. */
    private long noOfLines = -1L;

    /** Line number in file that is currently processing. */
    private long lineNo = 0L;

    /** As lineNo, but after optional ILineProcessor-step which merge or split lines. */
    private long lineNoAfterProcessor = 0L;
    
    /** No. of lines filtered. If a line processor is used this number may be skewed (filters may exist before and after a merger or splitter). */
    private long noOfFilteredLines = 0L;

    /** No. of parse errors. */
    private long noOfParseExceptions = 0L;

    /** No. of saved log entries. */
    private long noOfSavedEntries = 0L;

    /** No. of log entries connected to high priority organizations. */
    private long noOfHighPriorityEntries = 0L;

    /** No. of exported log entries. */
    private long noOfExportedEntries = 0L;

    /** Name-value table to store untyped data that needs to be shared e.g. between two decorators. */
    private Map<String, Object> additionalData;

    
    /**
     * Constructor.
     */
    public JobContext(TypedProperties props, Job job) {
        this.props = props;
        this.job = job;
    }

    
    /**
     * Displays message to operator. If CLI, the console is used.
     * If web application, the message is appended to a buffer that
     * is displayed at the end of the job.  
     */
    public void writeToConsole(String msg) {
        System.out.println(msg);
    
        // TODO add support for web application
    }

    
    public TypedProperties getProps() {
        return this.props;
    }

    
    public Job getJob() {
        return this.job;
    }

    
    public DbManager getDbManager() {
        return dbManager;
    }


    public void setDbManager(DbManager dbManager) {
        this.dbManager = dbManager;
    }

    
    public long getLineNo() {
        return lineNo;
    }


    public void incLineNo(int incValue) {
        this.lineNo += incValue;
    }

    
    public long getLineNoAfterProcessor() {
        return lineNoAfterProcessor;
    }


    public void incLineNoAfterProcessor(int incValue) {
        this.lineNoAfterProcessor += incValue;
    }

    
    public long getNoOfFilteredLines() {
        return noOfFilteredLines;
    }


    public void incNoOfFilteredLines(int incValue) {
        this.noOfFilteredLines += incValue;
    }


    public long getNoOfParseExceptions() {
        return noOfParseExceptions;
    }


    public void incNoOfParseExceptions(int incValue) {
        this.noOfParseExceptions += incValue;
    }

    
    public long getNoOfSavedEntries() {
        return noOfSavedEntries;
    }


    public void incNoOfSavedEntries(int incValue) {
        this.noOfSavedEntries += incValue;
    }

    
    public long getNoOfHighPriorityEntries() {
        return noOfHighPriorityEntries;
    }


    public void incNoOfHighPriorityEntries(int incValue) {
        this.noOfHighPriorityEntries += incValue;
    }

    
    public long getNoOfExportedEntries() {
        return noOfExportedEntries;
    }


    public void incNoOfExportedEntries(int incValue) {
        this.noOfExportedEntries += incValue;
    }

    
    public long getStartedTimestamp() {
        return startedTimestamp;
    }


    public void setStartedTimestamp(long startedTimestamp) {
        this.startedTimestamp = startedTimestamp;
    }


    public long getNoOfLines() {
        return noOfLines;
    }


    public void setNoOfLines(long noOfLines) {
        this.noOfLines = noOfLines;
    }
    
    
    public Object getAdditionalData(String key) {
        if (additionalData == null) {
            return null;
        }
        return additionalData.get(key);
    }
    
    
    public void addAdditionalData(String key, Object value) {
        if (additionalData == null) {
            additionalData = new HashMap<String, Object>();
        }
        additionalData.put(key, value);
    }


} public class DbManager {

    protected Session session = null;    
    protected Logger log = null;    
    protected DbManager()

            throws DbException { 

        this.log = Logger.getLogger(this.getClass());

        try {
            // This step will read hibernate.cfg.xml             
            //    and prepare hibernate for use
            SessionFactory sessionFactory = new Configuration().configure().buildSessionFactory();            
            session = sessionFactory.openSession();
        }
        catch (HibernateException he) {                    
            he.printStackTrace();
            throw new DbException("Failed to initialize DbManager", he);
        }              
    }


    public static DbManager createDbManager(TypedProperties props) 
            throws DbException { 

        DbManager dbm = null;

        if (props.isNoDb() || props.isMailDryRun() ||  props.isMailDryRun2()) {
            dbm = new ReadOnlyDbManager();            
        }    
        else {
            dbm = new DbManager();
        }
        return dbm;
    }


    protected Object loadLongObject (Class<?> clazz,  long objId) 
            throws DbException { 

        Object obj = session.get(clazz, objId);

        if (obj == null) {
            String msg = ("Invalid object reference: " + objId + ".");
            log.error(msg);
            throw new DbException(msg);
        }
        return obj;
    }

    protected Object loadIntegerObject (Class<?> clazz,  int objId) 
            throws DbException { 

        Object obj = session.get(clazz, objId);

        if (obj == null) {
            String msg = ("Invalid object reference: " + objId + ".");
            log.error(msg);
            throw new DbException(msg);
        }
        return obj;
    }


    public Object genericLoadObject(String className, String[] searchFields, Object[] values)
            throws DbException { 

        if (searchFields.length != values.length) {
            throw new DbException("Number of search fields and number of values are not maching");
        }
        if (searchFields.length < 1) {
            throw new DbException("Search fields are empty");
        }

        Object obj = null;

        String hql = "from " + className + " where ";                

        try {

            for (int i=0; i<searchFields.length; i++) {            
                hql = hql + ( i == 0 ? searchFields[i] + "= ?" :  " AND " +  searchFields[i]  + "= ?");
            }
            Query query = session.createQuery(hql);

            for (int i=0; i<searchFields.length; i++) {

                String typeName = values[i].getClass().getSimpleName();

                Method method = null;
                if (typeName.equals("Integer")) {
                    query.setInteger(i, ((Integer)values[i]).intValue());
                }
                else if (typeName.equals("Long")) {
                    query.setLong(i, ((Long)values[i]).longValue());
                }
                else {
                    method = query.getClass().getMethod("set" + typeName, Integer.TYPE, values[i].getClass());
                    Object[] args = {i, values[i]};
                    method.invoke(query, args);
                }                
            }                       
            obj = query.uniqueResult();

        } catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in genericLoadObject", e);
        }

        return obj;
    }



    public Object genericLoadObject(String className, String searchField, Object value)
            throws DbException { 

        Object obj = null;

        Query query = session.createQuery("from " + className + " where " + searchField + "= ?");

        String typeName = value.getClass().getSimpleName();

        try {

            Method method = null;
            if (typeName.equals("Integer")) {
                query.setInteger(0, ((Integer)value).intValue());
            }
            else if (typeName.equals("Long")) {
                query.setLong(0, ((Long)value).longValue());
            } 
            else {
                method = query.getClass().getMethod("set" + typeName, Integer.TYPE, value.getClass());
                Object[] args = {0, value};
                method.invoke(query, args);
            }            

            obj = query.uniqueResult();

        } catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in genericLoadObject", e);
        }

        return obj;

    }

    public void addPriority(Priority prio) 
            throws DbException { 

        try {                
            session.saveOrUpdate(prio);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in addPriority", e);
        }
    }


    public void addEntryType(EntryType entryType) 
            throws DbException { 
        try {                    
            session.saveOrUpdate(entryType);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in addEntryType", e);
        }
    }


    public void addLogEntry(LogEntry logEntry) 
            throws DbException { 
        try {                
            session.save(logEntry);
            session.flush();
            session.evict(logEntry);
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in addLogEntry", e);
        }
    }


    public void updateLogEntry(LogEntry logEntry) 
            throws DbException { 
        try {                
            session.update(logEntry);
            session.flush();
            session.evict(logEntry);
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in updateLogEntry", e);
        }
    }


    @SuppressWarnings("unchecked")
    public List<LogEntry> listLogEntries(long logJobId, int startIndex,
            int noOfRecords) 
                    throws DbException { 
        try {
            Query query = session.createQuery("from LogEntry where Job.Id = ? order by Id asc");
            query.setLong(0, logJobId);
            query.setMaxResults(noOfRecords);
            query.setFirstResult(startIndex);
            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in listLogEntries", e);
        }
    }


    @SuppressWarnings("unchecked")
    public List<LogEntry> listLogEntries(long logJobId, boolean usePrimaryOrg, int startIndex,
            int noOfRecords) 
                    throws DbException { 

        try {
            Query query = null;

            if (usePrimaryOrg) {
                query = session.createQuery("from LogEntry where Job.Id = ? order by Organization, Id asc");
            }
            else {
                query = session.createQuery("from LogEntry where Job.Id = ? order by Organization2, Id asc");
            }
            query.setLong(0, logJobId);
            query.setMaxResults(noOfRecords);
            query.setFirstResult(startIndex);

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in listLogEntries", e);
        }
    }


    /**
     * Returns high priority entries for specified job. Both primary and 
     * secondary organization is checked (org_id and org_id2).
     * 
     * Note, this method contains MySQL specific query parameters.
     * Must be checked if another database is used.
     * 
     * @param prio log entries linked to an organization with a priority equals
     *     or above this value will be included in the result. 
     * 
     * @return High priority entries, sorted by prio.
     */

    @SuppressWarnings({ "deprecation" })
    public List<LogEntry> listLogEntries(long logJobId, int prio, int startIndex, int noOfRecords) throws DbException {

        try {            
            ArrayList<LogEntry> entries = new ArrayList<LogEntry>();

            String sql = "select distinct l.id from log_entry l, organization o, prio p where l.job_id = ? and (l.org_id = o.id or l.org_id2 = o.id) and o.prio_id = p.id and p.prio >= ? order by prio desc limit ?, ?;";    
            PreparedStatement ps = session.connection().prepareStatement(sql);
            ps.setLong(1, logJobId);
            ps.setInt(2, prio);
            ps.setInt(3, startIndex);
            ps.setInt(4, startIndex + noOfRecords);

            boolean result = ps.execute();
            if (result) {
                while (ps.getResultSet().next()) {                        
                    long id = ps.getResultSet().getLong("id");
                    LogEntry entry = (LogEntry)this.loadLongObject(LogEntry.class, id);
                    entries.add(entry);                        
                }                             
            }
            ps.close();     

            return entries;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in listLogEntries", e);
        }                               
    }


    /**
     * Returns the log entry with the latest "created" timestamp, where 
     * ipAddress is equal to specified IP. Null is returned if not found.
     * Only ipAddress is checked, not ipAddress2 or ipRange.
     * 
     * @param ip ip-address for the log entry.
     * @param jobId log entries for this job will be ignored. 
     */
    @SuppressWarnings("deprecation")
    public LogEntry getLastSeenLogEntry(long ip, long jobId) throws DbException {

        LogEntry entry = null;

        try {
            String sql = "select id from log_entry where ip_address = ? and job_id != ? order by created desc";    
            PreparedStatement ps = session.connection().prepareStatement(sql);
            ps.setLong(1, ip);            
            ps.setLong(2, jobId);
            boolean result = ps.execute();
            if (result) {
                if (ps.getResultSet().first()) {
                    long id = ps.getResultSet().getLong("id");
                    entry = (LogEntry)this.loadLongObject(LogEntry.class, id);
                }                
            }
            ps.close();     
        }        
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getLastSeenLogEntry", e);
        }
        return entry;
    }


    /**
     * Returns the mail job with the latest "started" timestamp, which have
     * mailed specified IP. Null is returned if not found.
     * Only ipAddress is checked, not ipAddress2 or ipRange.
     */
    @SuppressWarnings("deprecation")
    public MailJob getLastSeenMailJob(long ip) throws DbException {

        MailJob mailJob = null;

        try {
            String sql = "select mj.id from mail_job mj, mail_job_log_entry_mapping mjlem, log_entry le " + 
                    "where mjlem.log_entry_id = le.id and le.ip_address = ? and mjlem.mail_job_id = mj.id order by mj.started desc";

            PreparedStatement ps = session.connection().prepareStatement(sql);
            ps.setLong(1, ip);                   
            boolean result = ps.execute();
            if (result) {
                if (ps.getResultSet().first()) {
                    long id = ps.getResultSet().getLong("id");
                    mailJob = (MailJob)this.loadLongObject(MailJob.class, id);
                }                
            }
            ps.close();     
        }       
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getLastSeenMailJob", e);
        }
        return mailJob;
    }


    public void addOrganization(Organization org, String modifiedBy) 
            throws DbException { 
        try {
            java.util.Date date = new java.util.Date();
            long time = SqlUtil.convertTimestamp(date);        
            org.setLastModified(time);
            org.setModifiedBy(modifiedBy);
            if (org.getCreated() == null) {
                org.setCreated(time);            
            }      
            session.save(org);
            session.flush();
        } 

        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " not possible to add organiation", e);
        }
    }



    public void updateOrganization(Organization org, String modifiedBy) 
            throws DbException { 
        try {
            java.util.Date date = new java.util.Date();
            long time = SqlUtil.convertTimestamp(date);        
            org.setLastModified(time);
            org.setModifiedBy(modifiedBy);
            session.update(org);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in updateOrganization", e);
        }
    }


    @SuppressWarnings("unchecked")
    public List<Organization> searchOrganizations(String name, int startIndex,
            int noOfRecords) 
                    throws DbException { 
        try {
            Query query = session.createQuery("from Organization where name like ?");
            query.setString(0, "%" + name +"%");

            query.setMaxResults(noOfRecords);
            query.setFirstResult(startIndex);

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrganizations", e);
        }
    }


    public Organization searchOrganization(String name) 
            throws DbException { 
        try {
            Query query = session.createQuery("from Organization where name = ?");
            query.setString(0, name);

            Organization org = (Organization)query.uniqueResult();

            return org;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrganization", e);
        }

    }


    @SuppressWarnings("unchecked")
    public List<Organization> listOrganizations(int startIndex, int noOfRecords) 
            throws DbException { 

        try {
            Query query = session.createQuery("from Organization order by Name asc");        
            query.setMaxResults(noOfRecords);
            query.setFirstResult(startIndex);                

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in listOrganizations", e);
        }

    }


    public Organization getOrganization(int orgId) 
            throws DbException { 

        try {
            Organization org = (Organization)loadIntegerObject(Organization.class, orgId);

            return org;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getOrganization", e);
        }

    }


    public void addMailJob(MailJob mailJob) 
            throws DbException { 

        try {
            mailJob.setStarted(SqlUtil.convertTimestamp(new Date()));
            session.saveOrUpdate(mailJob);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in addMailJob", e);
        }
    }


    /*        
        The updateMailJob is deprecated due to a problem when persisting the MailJob object after
        a change in the LogEntry collection. The reason could be due to some missconfiguration in the
        Hibernate object mapping for MailJob or a bug in Hibernate. 
        Use finishMailJob to persist the MailJob once it has been completed.      
     */
    @Deprecated
    public void updateMailJob(MailJob mailJob) 
            throws DbException { 

        try {
            session.update(mailJob);        
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in updateMailJob", e);
        }
    }


    public void finishMailJob(MailJob mailJob, String errorMsg) 
            throws DbException { 

        try {
            mailJob.setErrorMsg(errorMsg);
            mailJob.setFinished(SqlUtil.convertTimestamp(new Date()));
            session.update(mailJob);            
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in finishMailJob", e);
        }
    }


    public long deleteMailJob(MailJob mailJob) 
            throws DbException { 

        try {
            Query query = session.createSQLQuery(
                    "SELECT COUNT(*) FROM mail_job_log_entry_mapping WHERE mail_job_id = ?");
            query.setLong(0, mailJob.getId());
            long noLogEntriesInJob = ((BigInteger)query.list().get(0)).longValue();

            session.delete(mailJob);
            session.flush();

            long entriesLeft = ((BigInteger)query.list().get(0)).longValue();

            return noLogEntriesInJob - entriesLeft;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in deleteMailJob", e);
        }
    }


    // Search MailJob by time frame
    @SuppressWarnings("unchecked")
    public List<MailJob> searchMailJobs(Date startTime,
            Date endTime, int startIndex, int noOfRecords) 
                    throws DbException { 
        try {
            Query query = session.createQuery("from MailJob where started >= ? and started <= ? order by Id asc");
            query.setLong(0, SqlUtil.convertTimestamp(startTime));
            query.setLong(1, SqlUtil.convertTimestamp(endTime));
            query.setMaxResults(noOfRecords);
            query.setFirstResult(startIndex);

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchMailJobs", e);
        }
    }


    // Search MailJob by Job entity
    @SuppressWarnings("unchecked")
    public List<MailJob> searchMailJobs(Job job) 
            throws DbException { 

        try {
            Query query = session.createQuery("from MailJob where Job = ?");
            query.setEntity(0, job);

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchMailJobs", e);
        }
    }

    
    // Search MailJob by log entry
    public MailJob searchMailJob(LogEntry entry)
        throws DbException {
        
        try {
            Query query = session.createSQLQuery("select mail_job_id from mail_job_log_entry_mapping where log_entry_id = ?");
            query.setLong(0, entry.getId());
          
            Object result = query.uniqueResult();
            MailJob mailJob = null;
            if (result != null) {
                Long mailJobId = Long.valueOf(result.toString());
                mailJob = (MailJob)this.loadLongObject(MailJob.class, mailJobId.longValue());
            }    
            return mailJob;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchMailJob", e);
        }
    }


    public void addJobType(JobType jobType) 
            throws DbException { 

        try {                
            session.saveOrUpdate(jobType);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in addJobType", e);
        }
    }

    public void addLogJob(Job job) 
            throws DbException {

        try {
            session.save(job);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in addLogJob", e);
        }
    }

    public boolean existsFileHash(String hash)
            throws DbException {

        try {
            Query query = session.createQuery("from Job where FileHash = ?");
            query.setString(0, hash);

            return query.list().size() > 0;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in existsFileHash", e);
        }
    }

    @SuppressWarnings("unchecked")
    public List<Job> searchLogJobs(Date startTime, Date endTime, int startIndex,
            int noOfRecords) 
                    throws DbException { 

        try {

            Query query = session.createQuery("from Job where started >= ? and started <= ? order by Id asc");
            query.setLong(0, SqlUtil.convertTimestamp(startTime));
            query.setLong(1, SqlUtil.convertTimestamp(endTime));
            query.setMaxResults(noOfRecords);
            query.setFirstResult(startIndex);

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchLogJobs", e);
        }

    }                           


    public Job searchLogJob(String name)
            throws DbException { 

        try {
            Query query = session.createQuery("from Job where name = ?");
            query.setString(0, name);

            return (Job)query.uniqueResult();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchLogJob", e);
        }

    } 


    public void finishLogJob(Job job, String errorMsg) 
            throws DbException { 

        try {
            job.setFinished(SqlUtil.convertTimestamp(new Date()));
            job.setErrorMsg(errorMsg);
            session.update(job);
            session.flush();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in finishLogJob", e);
        }

    }


    @SuppressWarnings("deprecation")
    public long deleteLogJob(Job job) 
            throws DbException {

        long jobId = job.getId();        


        // Use JDBC connection to delete additional_item, free_text & original_log_entry
        try {

            // additional_item
            String sqlDelete = "delete from additional_item where log_entry_id in (select id from log_entry where job_id = ?)";       
            PreparedStatement ps = session.connection().prepareStatement(sqlDelete);
            ps.setLong(1, jobId);
            ps.execute();
            ps.close();        

            // free_text
            sqlDelete = "delete from free_text where log_entry_id in (select id from log_entry where job_id = ?)";
            ps = session.connection().prepareStatement(sqlDelete);
            ps.setLong(1, jobId);
            ps.execute();
            ps.close();

            // original_log_entry
            sqlDelete = "delete from original_log_entry where id in (select id from log_entry where job_id = ?)";
            ps = session.connection().prepareStatement(sqlDelete);
            ps.setLong(1, jobId);
            ps.execute();
            ps.close();

        }
        catch(SQLException sqle) {
            throw handleException("SQL exception in deleteLogJob", sqle);
        }


        try {
            // Delete LogEntry using hql
            String hqlDelete = "delete LogEntry where Job.Id = ?";
            long  numberOfDeletedLogEntries = session.createQuery( hqlDelete ).setLong(0, job.getId()).executeUpdate();

            session.evict(job);
            Job realJob = (Job)loadLongObject(job.getClass(), jobId);

            // Delete Job
            session.delete(realJob);        
            session.flush();


            return numberOfDeletedLogEntries;
        }

        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in deleteLogJob", e);
        }

    }


    public JobType searchJobType(String name) 
            throws DbException { 

        try {
            Query query = session.createQuery("from JobType where name = ?");
            query.setString(0, name);

            return (JobType)query.uniqueResult();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchJobType", e);
        }

    }


    public EntryType searchEntryType(String name) 
            throws DbException { 

        try {

            Query query = session.createQuery("from EntryType where name = ?");
            query.setString(0, name);

            return (EntryType)query.uniqueResult();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchEntryType", e);
        }

    }


    public Priority getPriority(int prio)
            throws DbException { 
        try {
            Query query = session.createQuery("from Priority where prio = ?");
            query.setInteger(0, prio);

            return (Priority)query.uniqueResult();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getPriority", e);
        }

    }


    @SuppressWarnings("unchecked")
    public List<Priority> getAllPriorities()
            throws DbException { 

        try {
            Query query = session.createQuery("from Priority order by Prio desc");


            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getAllPriorities", e);
        }

    }


    @SuppressWarnings("unchecked")
    public List<IpRange> getAllIpRanges(boolean includeDisabledOrgs)
            throws DbException { 

        try {

            Query query = null;
            if (includeDisabledOrgs) {
                query = session.createQuery("from IpRange");
            }
            else {
                query = session.createQuery("from IpRange where OrganizationId in (from Organization where Enabled = ?)");
                query.setBoolean(0, true);
            }

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getAllIpRanges", e);
        }

    }


    @SuppressWarnings("unchecked")
    public List<ASNumber> getAllASNumbers(boolean includeDisabledOrgs)
            throws DbException { 

        try {

            Query query = null;
            if (includeDisabledOrgs) {
                query = session.createQuery("from ASNumber");
            }
            else {
                query = session.createQuery("from ASNumber where OrganizationId in (from Organization where Enabled = ?)");
                query.setBoolean(0, true);
            }

            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getAllASNumbers", e);
        }

    }


    @SuppressWarnings("unchecked")
    public List<DomainName> getAllDomainNames(boolean includeDisabledOrgs)
            throws DbException { 

        try {

            Query query = null;
            if (includeDisabledOrgs) {
                query = session.createQuery("from DomainName");
            }
            else {
                query = session.createQuery("from DomainName where OrganizationId in (from Organization where Enabled = ?)");
                query.setBoolean(0, true);
            }
            return query.list();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getAllDomainNames", e);
        }

    }

    // Gets the organization with the given domain name
    public Organization getOrgByDomainName(String domainName)
            throws DbException { 

        try {

            Organization org = null;
            // Ugly to step search...
            Query query = session.createQuery("from DomainName where Name = ?");
            query.setString(0, domainName);

            DomainName dName = (DomainName) query.uniqueResult();

            if (dName != null) {
                org = (Organization)loadIntegerObject(Organization.class, dName.getOrganizationId());               
            }
            return org;

        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrgByDomainName", e);
        }

    }

    // Searches for organizations with domain names like the input name
    public List<Organization> searchOrgForDomainName(String domainName)
            throws DbException { 

        try {                       
            //Query query = session.createQuery("OrganizationId from DomainName where Name like '%?%'");
            //query.setString(0, domainName);

            String sql = "SELECT DISTINCT org_id FROM domain_name WHERE domain_name LIKE ? ORDER BY org_id ASC";            
            Query query = session.createSQLQuery(sql);
            query.setString(0, "%" + domainName + "%");
            @SuppressWarnings("unchecked")
            List<Object> orgIDs = query.list();
            List<Organization> orgs = new ArrayList<Organization>();    

            Iterator<Object> itr = orgIDs.iterator();           
            while (itr.hasNext()) { 
                Organization org = getOrganization(((Integer)itr.next()).intValue());
                if (org != null) {
                    orgs.add(org);
                }               
            }           
            return orgs;            
            
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrgForDomainName", e);
        }        
    }
    
    
    public List<Organization> searchOrganizationByPrioNumber(Integer prioNumber) 
            throws DbException { 
        
        try { 
            String sql = "SELECT DISTINCT o.id FROM organization o, prio p WHERE o.prio_id = p.id and p.prio = ? ORDER BY id ASC";
            Query query = session.createSQLQuery(sql);
            query.setInteger(0, prioNumber);
            
            @SuppressWarnings("unchecked")
            List<Object> orgIDs = query.list();
            List<Organization> orgs = new ArrayList<Organization>();    

            Iterator<Object> itr = orgIDs.iterator();           
            while (itr.hasNext()) { 
                Organization org = getOrganization(((Integer)itr.next()).intValue());
                if (org != null) {
                    orgs.add(org);
                }               
            }           
            return orgs;            
            
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrgByPrioNumber", e);
        }  
    }

    public Organization searchOrganizationByASNumber(int asn)
            throws DbException { 

        try {

            Organization org = null;
            // Ugly two step search...
            Query query = session.createQuery("from ASNumber where Number = ?");
            query.setInteger(0, asn);

            ASNumber asnum = (ASNumber) query.uniqueResult();

            if (asnum != null) {
                org = (Organization)loadIntegerObject(Organization.class, asnum.getOrganizationId());               
            }
            return org;

        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrgByASNumber", e);
        }

    }


    @SuppressWarnings("unchecked")
    public List<Organization> searchOrganizationsByIPrange(long start, long end) 
            throws DbException { 
        try {
            String sql = "SELECT DISTINCT org_id FROM ip_range WHERE (end_address >= ? AND end_address <= ?) OR (start_address <=  ? AND end_address >= ?) OR  (start_address >= ? AND start_address <= ?) OR (start_address >= ? AND end_address <= ?) ORDER BY org_id ASC";

            Query query = session.createSQLQuery(sql);
            query.setLong(0, start);   
            query.setLong(1, end);
            query.setLong(2, start);
            query.setLong(3, end);
            query.setLong(4, start);
            query.setLong(5, end);
            query.setLong(6, start);
            query.setLong(7, end);


            List<Object> orgIDs = query.list();
            List<Organization> orgs = new ArrayList<Organization>();    

            Iterator<Object> itr = orgIDs.iterator();           
            while (itr.hasNext()) { 
                Organization org = getOrganization(((Integer)itr.next()).intValue());
                if (org != null) {
                    orgs.add(org);
                }               
            }           
            return orgs;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrgsByIPrange", e);
        }           
    }   


    @SuppressWarnings("unchecked")
    public List<LogEntry> searchLogEntriesByIPrange(long start, long end) 
            throws DbException { 
        try {
            //String sql = "SELECT id FROM log_entry where (ip_address >= ? AND ip_address <= ?) OR (ip_address2 >= ? AND ip_address2 <= ?)";
            Query query = session.createQuery("from LogEntry where (IpAddress >= ? AND IpAddress <= ?) OR (IpAddress2 >= ? AND IpAddress2 <= ?)");

            query.setLong(0, start);   
            query.setLong(1, end);
            query.setLong(2, start);
            query.setLong(3, end);


            /*List<Object> entryIDs = query.list();
                List<LogEntry> entries = new ArrayList<LogEntry>(); 

                Iterator<Object> itr = entryIDs.iterator();         
                while (itr.hasNext()) {                                 
                    LogEntry entry = (LogEntry)loadLongObject(LogEntry.class, ((Long)itr.next()).longValue());                                  

                    if (entry != null) {
                        entries.add(entry);
                    }               
                }           */
            return query.list();
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchLogEntriesByIPrange", e);
        }           
    }   

    @SuppressWarnings("unchecked")
    public List<LogEntry> searchLogEntriesByOrgId(int orgId)            
            throws DbException { 
        try {
            Query query = session.createQuery("from LogEntry where Organization = ?");
            query.setInteger(0, orgId);

            return query.list();
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchLogEntriesByOrgId", e);
        }       
    }

    @SuppressWarnings("unchecked")
    public List<LogEntry> searchLogEntriesByASNumber(long asn) 
            throws DbException { 
        try {

            Query query = session.createQuery("from LogEntry where Asn = ? or Asn2 = ?");           
            query.setLong(0, asn);   
            query.setLong(1, asn);

            return query.list();
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchLogEntriesByASNumber", e);
        }           
    }   


    @SuppressWarnings("unchecked")
    public List<Organization> searchOrganizationByEmailAddress(String address) 
            throws DbException {

        List<Organization> orgs = new ArrayList<Organization>();

        try {
            Query query = session.createQuery("from Contact where EmailAddress like ?");
            query.setString(0, "%" + address +"%");
            List<Contact> contacts = query.list();
            for (Contact contact: contacts) {   
                Organization org = (Organization)loadIntegerObject(Organization.class, contact.getOrganizationId());                
                if (orgs.contains(org) == false) {              
                    orgs.add(org);
                }
            }       
            return orgs;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in searchOrganizationByEmailAddress", e);
        }

    }


    public long getNumberOfLogEntries()
            throws DbException { 
        try {

            Query query = session.createSQLQuery(
                    "SELECT COUNT(*) FROM log_entry");

            return ((BigInteger)query.list().get(0)).longValue();
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfLogEntries", e);
        }

    }


    public boolean existsMailForIp(long ip, Organization organization, long periodInSecs, boolean usePrimaryOrg)
            throws DbException { 

        try {        
            String orgIdColName = "org_id";
            String ipAddrColName = "ip_address";
            if (usePrimaryOrg != true) {
                orgIdColName = "org_id2";
                ipAddrColName = "ip_address2";
            }

            String select = "SELECT COUNT(*) FROM log_entry le, mail_job_log_entry_mapping map, mail_job mj where mj.started >= ? AND mj.finished <= ? AND mj.id = map.mail_job_id AND map.log_entry_id = le.id AND le." + orgIdColName + " = ? AND le." + ipAddrColName + " = ?";                  
            Query query = session.createSQLQuery(select); 

            long endTime = SqlUtil.convertTimestamp(new Date());        
            long startTime = endTime - periodInSecs;

            query.setLong(0, startTime);
            query.setLong(1, endTime);    
            query.setInteger(2, organization.getId());
            query.setLong(3, ip);

            return ((BigInteger)query.list().get(0)).intValue() > 0 ? true:false;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in existsMailForIp", e);
        }

    }


    public DbStatisticsData fetchStatistics(Date startDate, Date endDate)
            throws DbException { 
        try {

            DbStatisticsData dbStats = new DbStatisticsData(startDate, endDate);

            dbStats.setNoOfSuccessfulJobs(getNumberOfSuccessfulLogJobs(startDate, endDate));
            dbStats.setNoOfFailedJobs(getNumberOfFailedLogJobs(startDate, endDate));
            dbStats.setNoOfProcessedLines(getNumberOfProcessedLines(startDate, endDate));
            dbStats.setNoOfLogEntries(getNumberOfLogEntries(startDate, endDate));
            dbStats.setNoOfMatchedLogEntries(getNumberOfMatchedLogEntries(startDate, endDate));
            dbStats.setNoOfSentEmails(getNumberOfSentEmails(startDate, endDate));        
            dbStats.setNoOfNewOrganizations(getNumberOfNewOrganizations(startDate, endDate));
            dbStats.setNoOfModifiedOrganizations(getNumberOfModifiedOrganizations(startDate, endDate));

            return dbStats;

        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in fetchStatistics", e);
        }

    }

    /**
     * Fetches number of processed lines for successful jobs per job type.
     * 
     * @return Map with result, where key is job type and value is no. 
     *     of processed lines.
     */
    @SuppressWarnings("unchecked")
    public Map<String, Long> fetchNoOfProcessedLinesPerEntryType(Date startDate, Date endDate) 
            throws DbException { 
        try {

            Map<String, Long> result = new HashMap<String, Long>();

            String select = "SELECT et.name, sum(j.processed_lines) FROM job j, job_type jt, entry_type et WHERE job_type_id = jt.id AND jt.entry_type_id = et.id AND finished >= ? AND finished <= ? group by et.name;";

            Query query = session.createSQLQuery(select);        
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            List<Object[]> resultSet = query.list();

            if (resultSet != null) {
                for (Object[] obj: resultSet) {                
                    result.put(new String(obj[0].toString()), new Long(((java.math.BigDecimal)obj[1]).longValue()));                
                }
            }    

            return result;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in fetchNoOfProcessedLinesPerJobType", e);
        }
    }


    /**
     * Fetches number of log entries for successful jobs per job type.
     * 
     * @param onlyMatchedEntries If true, count only log entries where organization and 
     *     organization2 is not null. If false, count all log entries. 
     * 
     * @return Map with result, where key is job type and value is no. 
     *     of processed lines.
     */
    @SuppressWarnings("unchecked")
    public Map<String, Long> fetchNoOfLogEntriesPerEntryType(Date startDate, Date endDate, boolean onlyMatchedEntries) 
            throws DbException { 

        try {

            Map<String, Long> result = new HashMap<String, Long>();

            String select = "SELECT et.name, count(le.id) FROM log_entry le, job j, job_type jt, entry_type et WHERE le.job_id = j.id AND j.job_type_id = jt.id AND jt.entry_type_id = et.id AND j.finished >= ? AND j.finished <= ?";

            if (onlyMatchedEntries) {
                select = select + " AND (le.org_id IS NOT null OR le.org_id2 IS NOT null)";
            }
            select = select + " GROUP BY jt.entry_type_id";

            Query query = session.createSQLQuery(select);        
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            List<Object[]> resultSet = query.list();

            if (resultSet != null) {
                for (Object[] obj: resultSet) {                
                    result.put(new String(obj[0].toString()), new Long(((BigInteger)obj[1]).longValue()));                
                }
            }    

            return result;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in fetchNoOfLogEntriesPerJobType", e);
        }

    }


    /**
     * Fetches log entries for use in GeolocationXmlReportGenerator. Hibernate 
     * is not used due to memory constraints and complexity of the query.  
     * 
     * @param jobTypeKillList list if job types to exclude (the db-field "job_type.name").
     */
    @SuppressWarnings("deprecation")
    public List<LogEntry> fetchLogEntriesForGeolocation(Date startDate, Date endDate, List<String> jobTypeKillList) throws DbException {
        List<LogEntry> result = new ArrayList<LogEntry>(2048);
        PreparedStatement ps = null;
        try {
            // -- build query
            String queryStr = 
                    "select log_entry.id, log_timestamp, log_entry.created, ip_address, ip_range_start, port, hostname, asn, prio.name, additional_item.name, additional_item.value " +
                            "from (log_entry, job, job_type) " +
                            "left join additional_item on log_entry.id = additional_item.log_entry_id " +
                            "left join organization on log_entry.org_id = organization.id " +
                            "left join prio on organization.prio_id = prio.id " + 
                            "where " +
                            "log_entry.log_timestamp >= ? and log_entry.log_timestamp <= ? and " +
                            "log_entry.job_id = job.id and " +
                            "job.job_type_id = job_type.id and " +
                            "job_type.name not in (@inList@) " + 
                            "order by log_entry.log_timestamp, log_entry.id asc;";
            if ((jobTypeKillList == null) || jobTypeKillList.isEmpty()) {
                jobTypeKillList = new ArrayList<String>();
                jobTypeKillList.add("");
            }
            String inList = StringUtil.toSingleQuotedString(jobTypeKillList);
            queryStr = StringUtil.replace(queryStr, "@inList@", inList);

            // -- execute query
            log.debug("Executing query: " + queryStr);
            long t1 = System.currentTimeMillis();
            ps = session.connection().prepareStatement(queryStr);
            ps.setLong(1, SqlUtil.convertTimestamp(startDate));
            ps.setLong(2, SqlUtil.convertTimestamp(endDate));
            boolean successful = ps.execute();
            String durationStr = DateUtil.formatDuration(System.currentTimeMillis() - t1);
            log.debug("Query executed. Time: " + durationStr);

            // -- read result set; only one LogEntry per ip-address.
            final String dbIdKey = "dbId";
            final String timesSeenKey = "timesSeen";
            final String lastSeenKey = "lastSeen";
            final String prioNameKey = "prioName";
            final String infectionKey = "infection";
            if (successful) {
                long id = 0L;
                Map<Long, LogEntry> ipAddressMap = new HashMap<Long, LogEntry>();
                ResultSet resultSet = ps.getResultSet(); 
                while (resultSet.next()) {
                    Long ipAddress = resultSet.getLong(4);
                    if (ipAddress == null) {
                        ipAddress = resultSet.getLong(5);
                        if (ipAddress == null) {
                            continue;
                        }
                    }
                    LogEntry logEntry = ipAddressMap.get(ipAddress);
                    if (logEntry != null) {
                        int timesSeen = Integer.parseInt(logEntry.getAdditionalItems().get(timesSeenKey)); 
                        logEntry.getAdditionalItems().put(timesSeenKey, Integer.toString(++timesSeen));
                        long lastSeen = Long.parseLong(logEntry.getAdditionalItems().get(lastSeenKey));
                        long maxLastSeen = Math.max(lastSeen, resultSet.getLong(2));
                        logEntry.getAdditionalItems().put(lastSeenKey, Long.toString(maxLastSeen));
                    } 
                    else {
                        logEntry = new LogEntry(id++);
                        ipAddressMap.put(ipAddress, logEntry);
                        result.add(logEntry);
                        if (logEntry.getAdditionalItems() == null) {
                            logEntry.setAdditionalItems(new HashMap<String, String>());
                        }
                        logEntry.setIpAddress(ipAddress);
                        // logTimestamp == firstSeen
                        logEntry.setLogTimestamp(resultSet.getLong(2));
                        logEntry.setCreated(resultSet.getLong(3));
                        logEntry.setPort(resultSet.getInt(6));
                        logEntry.setHostname(resultSet.getString(7));
                        logEntry.setAsn(resultSet.getLong(8));
                        logEntry.getAdditionalItems().put(dbIdKey, Long.toString(resultSet.getLong(1))); 
                        logEntry.getAdditionalItems().put(timesSeenKey, "1");
                        logEntry.getAdditionalItems().put(lastSeenKey, logEntry.getLogTimestamp().toString());
                        logEntry.getAdditionalItems().put(prioNameKey, StringUtil.getNotNull(resultSet.getString(9)));
                    }
                    // add infection if missing
                    String infection = logEntry.getAdditionalItems().get(infectionKey);
                    if (StringUtil.isNullOrEmpty(infection)) {
                        infection = null;
                        String name = resultSet.getString(10);
                        if ((name != null) && name.equals(infectionKey)) {
                            infection = resultSet.getString(11);
                            if ((infection != null) && (infection.equals("irc") || infection.equalsIgnoreCase("sinkhole") || infection.equalsIgnoreCase("other"))) {
                                infection = null;
                            }
                        }
                        logEntry.getAdditionalItems().put(infectionKey, StringUtil.getNotNull(infection));
                    }
                }
            }
        } catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in fetchLogEntriesForGeolocation", e);
        } finally {
            try { if (ps != null) ps.close(); } catch (SQLException ignored) { }
        }

        return result;
    }


    /**
     * Fetches log entries for specified organization for use in OrganizationReportGenerator.
     * 
     * @param jobTypes list of job types (db-field "job_type.name") for log entries to be included in the result.
     */
    @SuppressWarnings("deprecation")
    public List<LogEntry> fetchLogEntriesForOrganizationReport(int orgId, Date startDate, Date endDate, List<String> jobTypes) 
            throws DbException {
        List<LogEntry> entries = new ArrayList<LogEntry>(2048);
        PreparedStatement ps = null;
        if ((jobTypes == null) || jobTypes.isEmpty()) {
            return entries;
        }

        try {
            String inList = StringUtil.toSingleQuotedString(jobTypes);
            String sql = 
                    "SELECT le.id from log_entry le, job j, job_type jt " +
                            "WHERE j.finished >= ? AND j.finished <= ? AND j.job_type_id = jt.id AND jt.name in (" + inList + ") AND le.job_id = j.id AND le.org_id = ?;";

            log.debug("Executing query: " + sql);
            ps = session.connection().prepareStatement(sql);            
            ps.setLong(1, SqlUtil.convertTimestamp(startDate));
            ps.setLong(2, SqlUtil.convertTimestamp(endDate));
            ps.setInt(3, orgId);
            boolean successful = ps.execute();

            if (successful) {
                ResultSet resultSet = ps.getResultSet(); 
                while (resultSet.next()) {
                    long entryID = resultSet.getLong(1);
                    LogEntry entry = (LogEntry)this.loadLongObject(LogEntry.class, entryID);
                    entries.add(entry);
                }
            }
        } catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in fetchLogEntriesForOrganizationReport", e);
        } finally {
            try { if (ps != null) ps.close(); } catch (SQLException ignored) { }
        }

        return entries;
    }


    public long getNumberOfSuccessfulLogJobs(Date startDate, Date endDate) 
            throws DbException {
        try {
            /** No. of successful log jobs (excluding mail jobs). */
            String sql = "SELECT COUNT(*) FROM job WHERE finished >= ? AND finished <= ? AND error_msg IS NULL";
            Query query = session.createSQLQuery(sql);

            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return (query.list() != null) ? ((BigInteger)query.list().get(0)).longValue() : 0;
        } 
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfSuccessfulLogJobs", e);
        }

    }


    public long getNumberOfFailedLogJobs(Date startDate, Date endDate) 
            throws DbException {
        try {    
            /** No. of failed log jobs (excluding mail jobs). */
            String sql = "SELECT COUNT(*) FROM job WHERE started >= ? AND started <= ? AND finished is null";
            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return (query.list() != null) ? ((BigInteger)query.list().get(0)).longValue() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfFailedLogJobs", e);
        }   
    }


    public long getNumberOfProcessedLines(Date startDate, Date endDate) 
            throws DbException { 
        try {
            /** No. of processed lines (after possible split or merge). */
            String sql = "SELECT SUM(processed_lines) FROM job WHERE finished >= ? AND finished <= ?";
            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return ((query.list() != null) && (query.list().get(0) != null)) ? ((java.math.BigDecimal)query.list().get(0)).longValue() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfProcessedLines", e);
        }
    }


    public long getNumberOfLogEntries(Date startDate, Date endDate) 
            throws DbException { 
        try {

            /** No. of processed lines (after possible split or merge). */
            String sql = "SELECT COUNT(*) FROM job j, log_entry l WHERE j.finished >= ? AND j.finished <= ? AND j.id = l.job_id" ;
            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return (query.list() != null) ? ((BigInteger)query.list().get(0)).longValue() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfLogEntries", e);
        }
    }


    public long getNumberOfMatchedLogEntries(Date startDate, Date endDate) 
            throws DbException { 
        try {

            /** No. of log entries that have been matched to an organisation. */
            String sql = "SELECT COUNT(*) FROM job j, log_entry l WHERE j.finished >= ? AND j.finished <= ? AND j.id = l.job_id AND (org_id IS NOT NULL or org_id2 IS NOT NULL)" ;
            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return (query.list() != null) ? ((BigInteger)query.list().get(0)).longValue() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfMatchedLogEntries", e);
        }
    }


    public long getNumberOfSentEmails(Date startDate, Date endDate) 
            throws DbException { 
        try {

            /** No. of sent emails in all mail jobs. */
            String sql = "SELECT DISTINCT le.org_id FROM log_entry le, mail_job mj, mail_job_log_entry_mapping mjle WHERE le.id = mjle.log_entry_id AND mj.id = mjle.mail_job_id AND mj.started >= ? AND mj.finished <= ?;";

            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            List<?> rs = query.list();

            return rs != null ? (long)rs.size() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfSentEmails", e);
        }
    }


    public long getNumberOfNewOrganizations(Date startDate, Date endDate) 
            throws DbException { 
        try {
            /** No. of created organizations. */        
            String sql = "SELECT COUNT(*) FROM organization WHERE created >= ? AND created <= ?";
            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return (query.list() != null) ? ((BigInteger)query.list().get(0)).longValue() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfNewOrganizations", e);
        }
    }


    public long getNumberOfModifiedOrganizations(Date startDate, Date endDate) 
            throws DbException { 
        try {    
            /** No. of modified organizations. */        
            String sql = "SELECT COUNT(*) FROM organization WHERE last_modified >= ? AND last_modified <= ? AND created <  last_modified";
            Query query = session.createSQLQuery(sql);
            query.setLong(0, SqlUtil.convertTimestamp(startDate));
            query.setLong(1, SqlUtil.convertTimestamp(endDate));

            return (query.list() != null) ? ((BigInteger)query.list().get(0)).longValue() : 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getNumberOfModifiedOrganizations", e);
        }   
    }


    public boolean isRangeOverlapping(long start, long end) 
            throws DbException { 
        try {
            String sql = "SELECT COUNT(*) FROM ip_range WHERE (end_address >= ? AND end_address <= ?) OR (start_address <=  ? AND end_address >= ?) OR  (start_address >= ? AND start_address <= ?) OR (start_address >= ? AND end_address <= ?)";

            Query query = session.createSQLQuery(sql);
            query.setLong(0, start);   
            query.setLong(1, end);
            query.setLong(2, start);
            query.setLong(3, end);
            query.setLong(4, start);
            query.setLong(5, end);
            query.setLong(6, start);
            query.setLong(7, end);

            long result = ((BigInteger)query.list().get(0)).longValue();

            return query.list() != null && result > 0;
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in isRangeOverlapping", e);
        }           
    }


    @SuppressWarnings("unchecked")
    public List<BigInteger> isRangeOverlappingOrgId(long start, long end)
            throws DbException {

        try {
            String sql = "SELECT ORG_ID FROM ip_range WHERE (end_address >= ? AND end_address <= ?) OR (start_address <=  ? AND end_address >= ?) OR  (start_address >= ? AND start_address <= ?) OR (start_address >= ? AND end_address <= ?)";

            Query query = session.createSQLQuery(sql);
            query.setLong(0, start);
            query.setLong(1, end);
            query.setLong(2, start);
            query.setLong(3, end);
            query.setLong(4, start);
            query.setLong(5, end);
            query.setLong(6, start);
            query.setLong(7, end);

            return query.list();

        } catch (Exception e) {
            throw handleException(e.getClass().getSimpleName()
                    + " exception in isRangeOverlappingOrgId", e);
        }
    }


    @SuppressWarnings("unchecked")
    public List<Contact> searchContactsByOrgIDsAndRoles(String[] orgIDs, String roles) 
            throws DbException {

        /*
         * Searches for contacts by orgId (if present) and roles (if present).
         * If no criterias are given, all contacts are returned.
         */
        ArrayList<Contact> contacts = new ArrayList<Contact>();
        Query query = null;


        if (orgIDs != null && orgIDs.length > 0) {
            for (String orgID : orgIDs) {           
                if (roles != null && roles.isEmpty() == false) {
                    // Search by orgId and roles
                    query = session.createQuery("from Contact where OrganizationId = ? and Role in (:roles)");              
                    query.setParameterList("roles", roles.split(","));

                    //System.out.println("1" + query.getQueryString() + " roles: " + roles);
                }   
                else {
                    // Search by orgId only
                    query = session.createQuery("from Contact where OrganizationId = ?");       
                    //System.out.println("2" + query.getQueryString());

                }
                query.setInteger(0, Integer.parseInt(orgID));
                //System.out.println("3" + query.getQueryString());
                //System.out.println("Found " + query.list().size() + " number of contacts.");
                if (query.list().isEmpty() == false) {              
                    contacts.addAll((List<Contact>)query.list());
                }
            }
        }
        else {

            if (roles != null && roles.isEmpty() == false) {    
                // Search by roles only
                query = session.createQuery("from Contact where Role in (:roles)");             
                query.setParameterList("roles", roles.split(","));
            }
            else {
                // Search all contacts
                query = session.createQuery("from Contact");
            }

            contacts.addAll((List<Contact>)query.list());
        }
        return contacts;

    }


    @SuppressWarnings("unchecked")
    public List<Contact>  searcContactsByPrioAndRoles(String prioHigh, String prioLow, String searchRoles)
            throws DbException {

        ArrayList<Contact> contacts = new ArrayList<Contact>();

        Query query = null;
        query = session.createQuery("from Organization where Priority.Prio <= ? and Priority.Prio >= ?");
        query.setString(0, prioHigh);
        query.setString(1, prioLow);

        if (query.list() != null) {         
            List<Organization> orgs = query.list();         
            Iterator<Organization> itr = orgs.iterator();           
            while (itr.hasNext()) {                     
                Set<Contact> orgContacts = ((Organization)itr.next()).getContacts();

                if (orgContacts != null && orgContacts.isEmpty() == false) {
                    if (searchRoles != null && searchRoles.isEmpty() == false) {
                        for (Contact contact : orgContacts) {
                            if (contact != null && contact.getRole() != null && searchRoles.contains(contact.getRole())) {

                                contacts.add(contact);                              
                            }
                        }
                    }
                    else {
                        contacts.addAll(orgContacts);
                    }
                }                               
            }                       
        }       

        return contacts;


    }

    @SuppressWarnings("unchecked")
    public List<String> getAllContactRoleNames() 
            throws DbException {
        try {
            String sql = "SELECT DISTINCT role FROM contact where role is not null ORDER BY role";
            Query query = session.createSQLQuery(sql);

            return (List<String>)query.list();

        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in getAllContactRoleNames", e);
        }

    }


    public void genericDelete(Object obj) 
            throws DbException {
        try {
            session.delete(obj);
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in genericDelete", e);
        }
    }


    public void flushSession()
            throws DbException {
        try {    
            session.flush();
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in flushSession", e);
        }
    }    

    public void close() 
            throws DbException {

        try {
            session.close();
        }
        catch (Exception e) {
            throw handleException(e.getClass().getSimpleName() + " exception in close", e);
        }
    }

    private DbException handleException (String msg, Exception e) {

        return new DbException(msg, e);

    }

} public class ASNumber extends BaseASNumber {
	private static final long serialVersionUID = 1L;

/*[CONSTRUCTOR MARKER BEGIN]*/
	public ASNumber () {
		super();
	}

	/**
	 * Constructor for primary key
	 */
	public ASNumber (java.lang.Integer id) {
		super(id);
	}

	/**
	 * Constructor for required fields
	 */
	public ASNumber (
		java.lang.Integer id,
		java.lang.Integer organizationId,
		java.lang.Long number) {

		super (
			id,
			organizationId,
			number);
	}
/*[CONSTRUCTOR MARKER END]*/
	
	public ASNumber (java.lang.Long asn) {
		super();
		this.setNumber(asn);
	}
	
	@Override
    public int compareTo(Object obj) {
		
		int result = 0;
		if (this.getNumber() == ((ASNumber)obj).getNumber()) {		
			result = 0;
		}
		else if (this.getNumber() < ((ASNumber)obj).getNumber()) {
			result = -1;
		}
		else {
			return 1;
		}
		return result;
	}


} public class DomainName extends BaseDomainName {
	private static final long serialVersionUID = 1L;

/*[CONSTRUCTOR MARKER BEGIN]*/
	public DomainName () {
		super();
	}

	/**
	 * Constructor for primary key
	 */
	public DomainName (java.lang.Integer id) {
		super(id);
	}

	/**
	 * Constructor for required fields
	 */
	public DomainName (
		java.lang.Integer id,
		java.lang.Integer organizationId,
		java.lang.String name) {

		super (
			id,
			organizationId,
			name);
	}

/*[CONSTRUCTOR MARKER END]*/	
	
	public DomainName(String name) {
		super();
		this.setName(name);
	}
	
	@Override
    public int compareTo(Object obj) {
		
		return this.getName().compareToIgnoreCase(((DomainName)obj).getName());
		
	}
	
	// Added to trim the domain name
	@Override
    public void setName (java.lang.String name) {
        super.setName(name.trim());
    }
	
} public class IpRange extends BaseIpRange {
	private static final long serialVersionUID = 1L;

/*[CONSTRUCTOR MARKER BEGIN]*/
	public IpRange () {
		super();
	}

	/**
	 * Constructor for primary key
	 */
	public IpRange (java.lang.Integer id) {
		super(id);
	}

	/**
	 * Constructor for required fields
	 */
	public IpRange (
		java.lang.Integer id,
		java.lang.Integer organizationId,
		java.lang.Long startAddress,
		java.lang.Long endAddress) {

		super (
			id,
			organizationId,
			startAddress,
			endAddress);
	}

/*[CONSTRUCTOR MARKER END]*/	
	
	@Override
    public int compareTo(Object obj) {

		int result = 0;
		if (this.getStartAddress() == ((IpRange)obj).getStartAddress()) {		
			result = 0;
		}
		else if (this.getStartAddress() < ((IpRange)obj).getStartAddress()) {
			result = -1;
		}
		else {
			return 1;
		}
		return result;
	}




} public class LogEntry extends BaseLogEntry {
	private static final long serialVersionUID = 1L;
	
	
/*[CONSTRUCTOR MARKER BEGIN]*/
	public LogEntry () {
		super();
	}

	/**
	 * Constructor for primary key
	 */
	public LogEntry (java.lang.Long id) {
		super(id);
	}

	/**
	 * Constructor for required fields
	 */
	public LogEntry (
		java.lang.Long id,
		java.lang.Long created,
		java.lang.Long logTimestamp) {

		super (
			id,
			created,
			logTimestamp);
	}

/*[CONSTRUCTOR MARKER END]*/
	
	@Override
    public int compareTo(Object obj) {

        int result = 0;
        
        if (this.getId() == 0 || ((LogEntry)obj).getId() == 0) {
            // Assert that only objects that have been persited are used. 
            throw new java.lang.AssertionError("LogEntry ID is undefined (0)");
        }
        		
		if (this.getId() == ((LogEntry)obj).getId()) {		
			result = 0;
		}
		else if (this.getId() < ((LogEntry)obj).getId()) {
			result = -1;
		}
		else {
			return 1;
		}
		return result;
	}
	
} public class Organization extends BaseOrganization {
    private static final long serialVersionUID = 1L;

    /*[CONSTRUCTOR MARKER BEGIN]*/
    public Organization () {
        super();
    }

    /**
     * Constructor for primary key
     */
    public Organization (java.lang.Integer id) {
        super(id);
    }

    /**
     * Constructor for required fields
     */
    public Organization (
            java.lang.Integer id,
            java.lang.String name,
            boolean enabled,
            java.lang.Long created,
            java.lang.Long lastModified,
            java.lang.String modifiedBy,
            boolean autoUpdateMatchFields) {

        super (
                id,
                name,
                enabled,
                created,
                lastModified,
                modifiedBy,            
                autoUpdateMatchFields);
    }

    /*[CONSTRUCTOR MARKER END]*/


    // Overloaded method from the base class that adds organization id to the Contact.
    @Override
    public void addToContacts (se.sitic.megatron.entity.Contact contact) {
        if (null == getContacts()) setContacts(new java.util.TreeSet<se.sitic.megatron.entity.Contact>());

        contact.setOrganizationId(this.getId());        
        super.addToContacts(contact);        
    }    
    // Overloaded method from the base class that adds organization id to the IP-range.
    @Override
    public void addToIpRanges (se.sitic.megatron.entity.IpRange ipRange) {
        if (null == getIpRanges()) setIpRanges(new java.util.TreeSet<se.sitic.megatron.entity.IpRange>());

        ipRange.setOrganizationId(this.getId());        
        super.addToIpRanges(ipRange);        
    }
    // Overloaded method from the base class that adds organization id to the DomainName.
    @Override
    public void addToDomainNames (se.sitic.megatron.entity.DomainName domainName) {
        if (null == getDomainNames()) setDomainNames(new java.util.TreeSet<se.sitic.megatron.entity.DomainName>());

        domainName.setOrganizationId(this.getId());
        super.addToDomainNames(domainName);        
    }
    // Overloaded method from the base class that adds organization id to the ASNumber.
    @Override
    public void addToASNumbers (se.sitic.megatron.entity.ASNumber asNumber) {
        if (null == getASNumbers()) setASNumbers(new java.util.TreeSet<se.sitic.megatron.entity.ASNumber>());

        asNumber.setOrganizationId(this.getId());
        super.addToASNumbers(asNumber);
    }
    
    // Returning all enabled email addresses
    public String getEmailAddresses(){        
        return this.getEmailAddresses(null, true);
    }

    // This method returns all email addresses for the organization that matches the given email type and 
    // the enabledOnly flag. If the email type is null or empty, all addresses matching the enabledOnly
    // flag are returned.

    public String getEmailAddresses(String emailType, boolean enabledOnly){

        java.util.Set<se.sitic.megatron.entity.Contact> allContacts = getContacts();
        java.util.Set<String> addresses = new java.util.TreeSet<String>(); 

        for (se.sitic.megatron.entity.Contact contact : allContacts) {
            if (contact.isEnabled() == true || enabledOnly == false) {
                if (emailType == null) {
                    addresses.add(contact.getEmailAddress());
                }
                else if (contact.getEmailType().toLowerCase().equals(emailType.toLowerCase())) {                    
                    addresses.add(contact.getEmailAddress());    
                }                
            }
        }
        if (addresses.isEmpty()) {
            return "";
        }
        else {
            String emails = java.util.Arrays.toString(addresses.toArray());        
            return emails.substring(1, emails.length()-1);
        }
    }
} public abstract class ObjectStringSorter implements Comparator<Object> {
    private static final Logger log = Logger.getLogger(ObjectStringSorter.class);

    /** String-comparer that is locale-sensitive */
    private Collator collator = null;
    /** Compare trailing numbers? */
    private boolean numberSensitive = true;
    /** Is a null value greater than a non-null value? */
    private boolean nullGreaterThanNotNull = true;


    /**
     * Returns default implementation of this class that uses toString() as
     * string representation.
     */
    public static synchronized ObjectStringSorter createDefaultSorter() {
        ObjectStringSorter result = new ObjectStringSorter() {
            @Override
            protected String getObjectString(Object obj) {
                return (obj != null) ? obj.toString() : null;
            }
        };

        return result;
    }

    /**
     * Same as createDefaultSorter(), but ignoring trailing numbers (better
     * performance).
     */
    public static synchronized ObjectStringSorter createBasicSorter() {
        ObjectStringSorter result = new ObjectStringSorter(false, true) {
            @Override
            protected String getObjectString(Object obj) {
                return (obj != null) ? obj.toString() : null;
            }
        };

        return result;
    }

    /**
     * Constructs a sorter that is number sensitive, i.e. trailing numbers
     * matters.
     */
    public ObjectStringSorter() {
        this.collator = Collator.getInstance();
    }

    /**
     * Constructor.
     *
     * @param numberSensitive
     *            Compare trailing numbers? If true, and the two strings to
     *            compare have numbers as suffixes and the same prefix, the
     *            numbers will be compared. Only integers are supported. Example
     *            when false: { "str1", "str10", "str2", "str20", "str3" }
     *            Example when true: { "str1", "str2", "str3", "str10", "str20" }
     * @param nullGreaterThanNotNull
     *            Is a null object string greater than a non-null value?
     */
    public ObjectStringSorter(boolean numberSensitive, boolean nullGreaterThanNotNull) {
        this();
        this.numberSensitive = numberSensitive;
        this.nullGreaterThanNotNull = nullGreaterThanNotNull;
    }

    /**
     * Compares string representation for specified objects, which may be null.
     * Implements Comparator.compare(Object, Object).
     */
    @Override
    public int compare(Object obj1, Object obj2) {
        String str1 = getObjectString(obj1);
        String str2 = getObjectString(obj2);

        if (str1 == str2)
            return 0;

        if (str1 == null)
            return this.nullGreaterThanNotNull ? 1 : -1;

        if (str2 == null)
            return this.nullGreaterThanNotNull ? -1 : 1;

        int result = collator.compare(str1, str2);

        if (this.numberSensitive && (result != 0) && isNumericSuffixAvailable(str1) && isNumericSuffixAvailable(str2)) {
            String[] headTailArray1 = splitNumericSuffixString(str1);
            String[] headTailArray2 = splitNumericSuffixString(str2);
            String prefix1 = headTailArray1[0];
            String prefix2 = headTailArray2[0];
            int prefixResult = collator.compare(prefix1, prefix2);
            // same prefix?
            if (prefixResult == 0) {
                try {
                    // compare as numbers
                    long long1 = Long.parseLong(headTailArray1[1]);
                    long long2 = Long.parseLong(headTailArray2[1]);
                    result = (long1 < long2) ? -1 : (long1 == long2) ? 0 : 1;
                } catch (NumberFormatException e) {
                    log.error("Internal Error: String should be a long.", e);
                }
            }
        }

        return result;
    }

    /** Gets the comparision string for the specified object */
    protected abstract String getObjectString(Object obj);

    /** Returns true if specifed string ends with a number. */
    private boolean isNumericSuffixAvailable(String str) {
        // assert: null-check already done
        if (str.length() == 0)
            return false;

        char lastChar = str.charAt(str.length() - 1);
        return (('0' <= lastChar) && (lastChar <= '9'));
    }

    /**
     * Splits specified string in two parts: head (string before numeric
     * suffix), and tail (numeric suffix or an empty string).
     * <p>
     * Example: "foo42" returns {"foo", "42"}, "foo" returns {"foo", ""}
     */
    private String[] splitNumericSuffixString(String str) {
        String[] result = new String[2];

        for (int i = (str.length() - 1); i >= 0; i--) {
            char ch = str.charAt(i);
            // is not a number?
            if (!(('0' <= ch) && (ch <= '9'))) {
                result[0] = str.substring(0, i + 1);
                result[1] = ((i + 1) < str.length()) ? str.substring(i + 1, str.length()) : "";
                return result;
            }
        }

        result[0] = "";
        result[1] = str;
        return result;
    }

} public abstract class StringUtil {


    /**
     * Replaces a substring in a string with another string. Replaces all
     * occurrences.
     * <p>
     * Example: replace("name=@name@", "@name@", "hubba") --> "name=hubba".
     *
     * @throws NullPointerException if the from- or to-argument is null.
     */
    public static String replace(String str, String from, String to) {
        return replaceInternal(str, from, to, false);
    }

    
    /**
     * As replace, but replace one the first occurrence.  
     */
    public static String replaceFirst(String str, String from, String to) {
        return replaceInternal(str, from, to, true);
    }


    /**
     * Removes prefix from specified string.<p>
     * Example: removePrefix("foobar.config", "foobar.") --> "config"
     */
    public static String removePrefix(String str, String prefix) {
        if ((str == null) || (prefix == null)) {
            return str;
        }

        String result = null;
        int hitIndex = str.indexOf(prefix);
        if (hitIndex == 0) {
            if (prefix.length() < str.length()) {
                result = str.substring(prefix.length());
            } else {
                result = "";
            }
        } else {
            result = str;
        }

        return result;
    }


    /**
     * Removes suffix from specified string.<p>
     * Example: removeSuffix("foobar.config", ".config") --> "foobar"
     */
    public static String removeSuffix(String str, String suffix) {
        if ((str == null) || (suffix == null)) {
            return str;
        }

        String result = str;
        int hitIndex = str.lastIndexOf(suffix);
        if ((hitIndex != -1) && ((hitIndex + suffix.length()) == str.length())) {
            result = result.substring(0, hitIndex);
        }
        return result;
    }


    /**
     * Splits specified string in two parts at delimiter position.
     *
     * @param str string to split.
     * @param delim delimiter that marks split between head and tail.
     * @param reverse is search for delimiter reverse (search starts at end of string)?
     * @return null if one argument is null, otherwise a String-array of length
     *         two in following format: { head, tail }. If delimiter is not
     *         found, then tail is empty if isReverse is false or
     *         head is empty if isReverse is true. Delimiter is not
     *         included.
     */
    public static String[] splitHeadTail(String str, String delim, boolean reverse) {
        // check param
        if ((str == null) || (delim == null)) {
            return null;
        }

        String[] result = new String[2];
        int hitPos = reverse ? str.lastIndexOf(delim) : str.indexOf(delim);
        if (hitPos != -1) {
            result[0] = str.substring(0, hitPos);
            int tailPos = hitPos + delim.length();
            if (tailPos < str.length()) {
                result[1] = str.substring(tailPos);
            } else {
                result[1] = "";
            }
        } else {
            if (reverse) {
                result[0] = "";
                result[1] = str;
            } else {
                result[0] = str;
                result[1] = "";
            }
        }

        return result;
    }


    /**
     * Returns true if specified string is null or empty.
     */
    public static boolean isNullOrEmpty(String str) {
        return str == null || str.length() == 0;
    }


    /**
     * Returns specified string if not null, otherwise an empty string.
     */
    public static String getNotNull(String str) {
        return (str != null) ? str : "";
    }


    /**
     * Returns specified string if not null, otherwise specified default value.
     */
    public static String getNotNull(String str, String defaultStr) {
        return (str != null) ? str : defaultStr;
    }


    /**
     * Converts specifed byte-array to hex-string.
     */
    public static String encode(byte[] buf) {
        StringBuffer result = new StringBuffer(2 * buf.length);

        for (int i = 0; i < buf.length; i++) {
            int byteAsInt = buf[i] & 0xFF;
            if (byteAsInt < 16) {
                result.append("0");
            }
            result.append(Integer.toString(byteAsInt, 16).toLowerCase());
        }

        return result.toString();
    }


    /**
     * Converts specified hex-string to byte-array.
     */
    public static byte[] decodeHexString(String hexStr) throws NumberFormatException {
        if (hexStr == null) {
            return null;
        }
        // is every digit of size two?
        if ((hexStr.length() % 2) != 0) {
            throw new NumberFormatException("Bad hex-string. Length of string must be even.");
        }

        byte[] result = new byte[hexStr.length() / 2];
        for (int i = 0; i < hexStr.length(); i += 2) {
            String hexDigit = hexStr.substring(i, i + 2);
            int byteValue = Integer.parseInt(hexDigit, 16);
            result[i / 2] = (byte)(byteValue & 0xFF);
        }

        return result;
    }


    /**
     * Encodes specifed text using named entities, e.g "<" converts
     * to "&amp;lt;".
     * <p>
     * The result is safe to include in XML or HTML.
     */
    public static String encodeCharacterEntities(String str) {
        if (str == null) {
            return null;
        }

        StringBuilder result = new StringBuilder(str.length() + 50);
        for (int i = 0; i < str.length(); i++) {
            char chr = str.charAt(i);
            switch (chr) {
                case '<':
                    result.append("&lt;");
                    break;
                case '>':
                    result.append("&gt;");
                    break;
                case '&':
                    result.append("&amp;");
                    break;
                case '"':
                    result.append("&quot;");
                    break;
                default:
                    result.append(chr);
            }
        }
        return result.toString();
    }

    
    /**
     * Removes line beaks from specified string.
     */
    public static String removeLineBreaks(String str, String replaceWith) {
        String result = str;
        result = replace(result, "\n", replaceWith);
        result = replace(result, "\r", replaceWith);
        return result;
    }


    /**
     * Removes trailing white spaces (including line breaks) from 
     * specified string.
     */
    public static String removeTrailingSpaces(String str) {
        if ((str == null) || (str.length() == 0)) {
            return str;
        }

        int endIndex = 0;
        for (int i = (str.length()-1); i >= 0; i--) {
            if (!Character.isWhitespace(str.charAt(i))) {
                endIndex = i + 1;
                break;
            }
        }
        
        return str.substring(0, endIndex);
    }
    
    
    /**
     * Removes enclosing characters from string.
     * Example: str="'foobar'", enclosingChars="'" will return "foobar". 
     */
    public static String removeEnclosingChars(String str, String enclosingChars) {
        if ((str == null) || !(str.startsWith(enclosingChars) && str.endsWith(enclosingChars))) {
            return str;
        }

        return removeSuffix(removePrefix(str, enclosingChars), enclosingChars); 
    }

    
    /**
     * Truncates specified string if longer than max length. "..." is added
     * as suffix to mark that the string have been truncated.
     */
    public static String truncateString(String str, int maxLength) {
        if ((str == null) || (str.length() <= maxLength)) {
            return str;
        }
        final String suffix = "...";
        if (maxLength < suffix.length()) {
            return suffix;
        }
        
        StringBuilder result = new StringBuilder(maxLength);
        int endIndex = maxLength - suffix.length();
        result.append(str.substring(0, endIndex)).append(suffix);

        return result.toString();
    }

    
    /**
     * Returns a string containing strToCopy duplicated
     * noOfCopies times.
     */
    public static String nCopyString(String strToCopy, int noOfCopies) {
        // check param
        if (strToCopy == null) {
            return null;
        }

        StringBuilder result = new StringBuilder(noOfCopies * strToCopy.length());
        for (int i = 0; i < noOfCopies; i++) {
            result.append(strToCopy);
        }

        return result.toString();
    }

    
    /**
     * Pads specified string in head with spaces.
     */
    public static String leftPad(String str, int len) {
        return pad(str, len, " ", true);
    }


    /**
     * Pads specified string in tail with spaces.
     */
    public static String rightPad(String str, int len) {
        return pad(str, len, " ", false);
    }


    /**
     * Pads specified string in head with pad-string.
     */
    public static String leftPad(String str, int len, String padString) {
        return pad(str, len, padString, true);
    }


    /**
     * Pads specified string in tail with pad-string.
     */
    public static String rightPad(String str, int len, String padString) {
        return pad(str, len, padString, false);
    }

    
    /**
     * Returns specified files as a string list. Returns never null.
     */
    public static String toString(List<File> files, boolean shortName) {
        if (files == null) {
            return "[null]";
        }

        StringBuilder result = new StringBuilder(256);
        for (Iterator<File> iterator = files.iterator(); iterator.hasNext(); ) {
            File file = iterator.next();
            if (result.length() > 0) {
                result.append(", ");
            }

            String filename = shortName ? file.getName() : file.getAbsolutePath();
            result.append(filename);
        }

        return result.toString();
    }

    
    /**
     * Returns specified list as a string where every element is quoted and
     * comma separated.
     */
    public static String toQuotedString(List<?> list) {
        return toQuotedString(list, "\"");
    }

    
    /**
     * As toQuotedString, but use a single quote instead of a double quote.
     */
    public static String toSingleQuotedString(List<?> list) {
        return toQuotedString(list, "\'");
    }

    
    /**
     * Returns specified list as a string where every element is quoted with
     * specified character and comma separated.
     */
    private static String toQuotedString(List<?> list, String quoteChar) {
        if ((list == null) || (list.size() == 0)) {
            return "";
        }
        
        StringBuffer result = new StringBuffer(256);
        for (Iterator<?> iterator = list.iterator(); iterator.hasNext(); ) {
            Object obj = iterator.next();
            String str = (obj != null) ? obj.toString() : "";
            if (result.length() > 0) {
                result.append(",");
            }
            result.append(quoteChar).append(str).append(quoteChar);
        }

        return result.toString();
    }

    
    /**
     * Pads specified string with pad-string.
     */
    private static String pad(String str, int len, String padString, boolean leftPad) {
        if ((str == null) || (str.length() >= len) || (padString == null)) {
            return str;
        }

        int n = (len - str.length()) / padString.length();
        String paddedString = nCopyString(padString, n);
        StringBuilder result = new StringBuilder(len);
        if (leftPad) {
            result.append(paddedString).append(str);
        } else {
            result.append(str).append(paddedString);
        }

        return result.toString();
    }
    
    
    @SuppressWarnings("null")
    private static String replaceInternal(String str, String from, String to, boolean replaceFirst) {
        if (str == null) {
            return null;
        }
        if ((from == null) || (to == null)) {
            throw new NullPointerException("from- or to-argument is null.");
        }

        StringBuilder result = null;
        int fromlength = 0;
        int index = 0;
        boolean quit = false;
        while (!quit) {
            int hitIndex = str.indexOf(from, index);
            if (hitIndex < 0) {
                break;
            }
            if (result == null) {
                result = new StringBuilder(str.length() + 40);
                fromlength = from.length();
            }
            result.append(str.substring(index, hitIndex));
            result.append(to);
            index = hitIndex + fromlength;
            quit = replaceFirst;
        }
        if (index == 0) {
            return str;
        }
        result.append(str.substring(index));
        return result.toString();
    }


}
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
import se.sitic.megatron.core.AppProperties;
import se.sitic.megatron.core.Interval;
import se.sitic.megatron.core.IntervalList;
import se.sitic.megatron.core.JobContext;
import se.sitic.megatron.core.MegatronException;
import se.sitic.megatron.db.DbException;
import se.sitic.megatron.db.DbManager;
import se.sitic.megatron.entity.ASNumber;
import se.sitic.megatron.entity.DomainName;
import se.sitic.megatron.entity.IpRange;
import se.sitic.megatron.entity.LogEntry;
import se.sitic.megatron.entity.Organization;
import se.sitic.megatron.util.DateUtil;
import se.sitic.megatron.util.ObjectStringSorter;
import se.sitic.megatron.util.StringUtil;
package se.sitic.megatron.decorator;





/**
 * Adds organization id for primary- and secondary organization matching the
 * following:<ul>
 * <li>IP ranges
 * <li>ASNs
 * <li>hostnames
 * </ul>
 */
public class OrganizationMatcherDecorator implements IDecorator {
    private static final Logger log = Logger.getLogger(OrganizationMatcherDecorator.class);

    // -- cache handling
    private static final long CACHE_TTL = 30*60*1000L;
    private static final Object cacheMutex = new Object();
    private static Map<Long, ASNumber> asnMapCached;
    private static Map<String, DomainName> domainNameMapCached;
    private static IntervalList ipIntervalsCached;