@JsonInclude(Include.NON_NULL)
public class AnalysisConfig
{
    /**
     * Serialisation names
     */
    public static final String BUCKET_SPAN = "bucketSpan";
    public static final String BATCH_SPAN = "batchSpan";
    public static final String CATEGORIZATION_FIELD_NAME = "categorizationFieldName";
    public static final String LATENCY = "latency";
    public static final String PERIOD = "period";
    public static final String SUMMARY_COUNT_FIELD_NAME = "summaryCountFieldName";
    public static final String DETECTORS = "detectors";
    public static final String INFLUENCERS = "influencers";
    public static final String OVERLAPPING_BUCKETS = "overlappingBuckets";
    public static final String RESULT_FINALIZATION_WINDOW = "resultFinalizationWindow";
    public static final String MULTIVARIATE_BY_FIELDS = "multivariateByFields";

    private static final String PRELERT_CATEGORY_FIELD = "prelertcategory";
    public static final Set<String> AUTO_CREATED_FIELDS = new HashSet<>(
            Arrays.asList(PRELERT_CATEGORY_FIELD));

    public static final long DEFAULT_RESULT_FINALIZATION_WINDOW = 2L;

    /**
     * These values apply to all detectors
     */
    private Long m_BucketSpan;
    private Long m_BatchSpan;
    private String m_CategorizationFieldName;
    private Long m_Latency = 0L;
    private Long m_Period;
    private String m_SummaryCountFieldName;
    private List<Detector> m_Detectors;
    private List<String> m_Influencers;
    private Boolean m_OverlappingBuckets;
    private Long m_ResultFinalizationWindow;
    private Boolean m_MultivariateByFields;

    /**
     * Default constructor
     */
    public AnalysisConfig()
    {
        m_Detectors = new ArrayList<>();
        m_Influencers = new ArrayList<>();
    }

    /**
     * The size of the interval the analysis is aggregated into measured in
     * seconds
     * @return The bucketspan or <code>null</code> if not set
     */
    public Long getBucketSpan()
    {
        return m_BucketSpan;
    }

    public void setBucketSpan(Long span)
    {
        m_BucketSpan = span;
    }

    /**
     * Interval into which to batch seasonal data measured in seconds
     * @return The batchspan or <code>null</code> if not set
     */
    public Long getBatchSpan()
    {
        return m_BatchSpan;
    }

    public void setBatchSpan(Long batchSpan)
    {
        m_BatchSpan = batchSpan;
    }

    public String getCategorizationFieldName()
    {
        return m_CategorizationFieldName;
    }

    public void setCategorizationFieldName(String categorizationFieldName)
    {
        m_CategorizationFieldName = categorizationFieldName;
    }

    /**
     * The latency interval (seconds) during which out-of-order records should be handled.
     * @return The latency interval (seconds) or <code>null</code> if not set
     */
    public Long getLatency()
    {
        return m_Latency;
    }

    /**
     * Set the latency interval during which out-of-order records should be handled.
     * @param latency the latency interval in seconds
     */
    public void setLatency(Long latency)
    {
        m_Latency = latency;
    }

    /**
     * The repeat interval for periodic data in multiples of
     * {@linkplain #getBatchSpan()}
     * @return The period or <code>null</code> if not set
     */
    public Long getPeriod()
    {
        return m_Period;
    }

    public void setPeriod(Long period)
    {
        m_Period = period;
    }

    /**
     * The name of the field that contains counts for pre-summarised input
     * @return The field name or <code>null</code> if not set
     */
    public String getSummaryCountFieldName()
    {
        return m_SummaryCountFieldName;
    }

    public void setSummaryCountFieldName(String summaryCountFieldName)
    {
        m_SummaryCountFieldName = summaryCountFieldName;
    }

    /**
     * The list of analysis detectors. In a valid configuration the list should
     * contain at least 1 {@link Detector}
     * @return The Detectors used in this job
     */
    public List<Detector> getDetectors()
    {
        return m_Detectors;
    }

    public void setDetectors(List<Detector> detectors)
    {
        m_Detectors = detectors;
    }

    /**
     * The list of influence field names
     */
    public List<String> getInfluencers()
    {
        return m_Influencers;
    }

    public void setInfluencers(List<String> influencers)
    {
        m_Influencers = influencers;
    }

    /**
     * Return the list of term fields.
     * These are the influencer fields, partition field,
     * by field and over field of each detector.
     * <code>null</code> and empty strings are filtered from the
     * config.
     *
     * @return Set of term fields - never <code>null</code>
     */
    public Set<String> termFields()
    {
        Set<String> termFields = new TreeSet<>();

        for (Detector d : getDetectors())
        {
            addIfNotNull(termFields, d.getByFieldName());
            addIfNotNull(termFields, d.getOverFieldName());
            addIfNotNull(termFields, d.getPartitionFieldName());
        }

        for (String i : getInfluencers())
        {
            addIfNotNull(termFields, i);
        }

        // remove empty strings
        termFields.remove("");

        return termFields;
    }

    public Boolean getOverlappingBuckets()
    {
        return m_OverlappingBuckets;
    }

    public void setOverlappingBuckets(Boolean b)
    {
        m_OverlappingBuckets = b;
    }

    public Long getResultFinalizationWindow()
    {
        return m_ResultFinalizationWindow;
    }

    public void setResultFinalizationWindow(Long l)
    {
        m_ResultFinalizationWindow = l;
    }

    public Boolean getMultivariateByFields()
    {
        return m_MultivariateByFields;
    }

    public void setMultivariateByFields(Boolean multivariateByFields)
    {
        m_MultivariateByFields = multivariateByFields;
    }

    /**
     * Return the list of fields required by the analysis.
     * These are the influencer fields, metric field, partition field,
     * by field and over field of each detector, plus the summary count
     * field and the categorization field name of the job.
     * <code>null</code> and empty strings are filtered from the
     * config.
     *
     * @return List of required analysis fields - never <code>null</code>
     */
    public List<String> analysisFields()
    {
        Set<String> analysisFields = termFields();

        addIfNotNull(analysisFields, m_CategorizationFieldName);
        addIfNotNull(analysisFields, m_SummaryCountFieldName);

        for (Detector d : getDetectors())
        {
            addIfNotNull(analysisFields, d.getFieldName());
        }

        // remove empty strings
        analysisFields.remove("");

        return new ArrayList<String>(analysisFields);
    }

    private static void addIfNotNull(Set<String> fields, String field)
    {
        if (field != null)
        {
            fields.add(field);
        }
    }

    public List<String> fields()
    {
        return collectNonNullAndNonEmptyDetectorFields(d -> d.getFieldName());
    }

    private List<String> collectNonNullAndNonEmptyDetectorFields(
            Function<Detector, String> fieldGetter)
    {
        Set<String> fields = new HashSet<>();

        for (Detector d : getDetectors())
        {
            addIfNotNull(fields, fieldGetter.apply(d));
        }

        // remove empty strings
        fields.remove("");

        return new ArrayList<String>(fields);
    }

    public List<String> byFields()
    {
        return collectNonNullAndNonEmptyDetectorFields(d -> d.getByFieldName());
    }

    public List<String> overFields()
    {
        return collectNonNullAndNonEmptyDetectorFields(d -> d.getOverFieldName());
    }


    public List<String> partitionFields()
    {
        return collectNonNullAndNonEmptyDetectorFields(d -> d.getPartitionFieldName());
    }

    /**
     * The array of detectors are compared for equality but they are not sorted
     * first so this test could fail simply because the detector arrays are in
     * different orders.
     */
    @Override
    public boolean equals(Object other)
    {
        if (this == other)
        {
            return true;
        }

        if (other instanceof AnalysisConfig == false)
        {
            return false;
        }

        AnalysisConfig that = (AnalysisConfig)other;

        if (this.m_Detectors.size() != that.m_Detectors.size())
        {
            return false;
        }

        for (int i=0; i<m_Detectors.size(); i++)
        {
            if (!this.m_Detectors.get(i).equals(that.m_Detectors.get(i)))
            {
                return false;
            }
        }

        return Objects.equals(this.m_BucketSpan, that.m_BucketSpan) &&
                Objects.equals(this.m_BatchSpan, that.m_BatchSpan) &&
                Objects.equals(this.m_CategorizationFieldName, that.m_CategorizationFieldName) &&
                Objects.equals(this.m_Latency, that.m_Latency) &&
                Objects.equals(this.m_Period, that.m_Period) &&
                Objects.equals(this.m_SummaryCountFieldName, that.m_SummaryCountFieldName) &&
                Objects.equals(this.m_Influencers, that.m_Influencers) &&
                Objects.equals(this.m_OverlappingBuckets, that.m_OverlappingBuckets) &&
                Objects.equals(this.m_ResultFinalizationWindow,  that.m_ResultFinalizationWindow) &&
                Objects.equals(this.m_MultivariateByFields, that.m_MultivariateByFields);
    }

    @Override
    public int hashCode()
    {
        return Objects.hash(m_Detectors, m_BucketSpan, m_BatchSpan, m_Latency, m_Period,
                m_SummaryCountFieldName, m_Influencers, m_OverlappingBuckets, m_ResultFinalizationWindow,
                m_MultivariateByFields);
    }

} @JsonInclude(Include.NON_NULL)
public class AnalysisLimits
{
    /**
     * Serialisation field names
     */
    public static final String MODEL_MEMORY_LIMIT = "modelMemoryLimit";
    public static final String CATEGORIZATION_EXAMPLES_LIMIT = "categorizationExamplesLimit";

    /**
     * It is initialised to 0.  A value of 0 indicates it was not set, which in
     * turn causes the C++ process to use its own default limit.  A negative
     * value means no limit.  All negative input values are stored as -1.
     */
    private long m_ModelMemoryLimit;

    /**
     * It is initialised to <code>null</code>.
     * A value of <code>null</code> indicates it was not set.
     * */
    private Long m_CategorizationExamplesLimit;

    public AnalysisLimits()
    {
        m_ModelMemoryLimit = 0;
        m_CategorizationExamplesLimit = null;
    }

    public AnalysisLimits(long modelMemoryLimit, Long categorizationExamplesLimit)
    {
        if (modelMemoryLimit < 0)
        {
            // All negative numbers mean "no limit"
            m_ModelMemoryLimit = -1;
        }
        else
        {
            m_ModelMemoryLimit = modelMemoryLimit;
        }
        m_CategorizationExamplesLimit = categorizationExamplesLimit;
    }

    /**
     * Maximum size of the model in MB before the anomaly detector
     * will drop new samples to prevent the model using any more
     * memory
     *
     * @return The set memory limit or 0 if not set
     */
    public long getModelMemoryLimit()
    {
        return m_ModelMemoryLimit;
    }

    public void setModelMemoryLimit(long value)
    {
        if (value < 0)
        {
            // All negative numbers mean "no limit"
            m_ModelMemoryLimit = -1;
        }
        else
        {
            m_ModelMemoryLimit = value;
        }
    }

    /**
     * Gets the limit to the number of examples that are stored per category
     * @return the limit or <code>null</code> if not set
     */
    public Long getCategorizationExamplesLimit()
    {
        return m_CategorizationExamplesLimit;
    }

    public void setCategorizationExamplesLimit(Long value)
    {
        m_CategorizationExamplesLimit = value;
    }

    /**
     * Overridden equality test
     */
    @Override
    public boolean equals(Object other)
    {
        if (this == other)
        {
            return true;
        }

        if (other instanceof AnalysisLimits == false)
        {
            return false;
        }

        AnalysisLimits that = (AnalysisLimits)other;
        return this.m_ModelMemoryLimit == that.m_ModelMemoryLimit
                && Objects.equals(this.m_CategorizationExamplesLimit,
                        that.m_CategorizationExamplesLimit);
    }

    @Override
    public int hashCode()
    {
        return Objects.hash(m_ModelMemoryLimit, m_CategorizationExamplesLimit);
    }

    public Map<String, Object> toMap()
    {
        Map<String, Object> map = new HashMap<>();
        map.put(MODEL_MEMORY_LIMIT, m_ModelMemoryLimit);
        if (m_CategorizationExamplesLimit != null)
        {
            map.put(CATEGORIZATION_EXAMPLES_LIMIT, m_CategorizationExamplesLimit);
        }
        return map;
    }
} @JsonInclude(Include.NON_NULL)
public class DataCounts
{
    public static final String BUCKET_COUNT = "bucketCount";
    public static final String PROCESSED_RECORD_COUNT = "processedRecordCount";
    public static final String PROCESSED_FIELD_COUNT = "processedFieldCount";
    public static final String INPUT_BYTES = "inputBytes";
    public static final String INPUT_RECORD_COUNT = "inputRecordCount";
    public static final String INPUT_FIELD_COUNT = "inputFieldCount";
    public static final String INVALID_DATE_COUNT = "invalidDateCount";
    public static final String MISSING_FIELD_COUNT = "missingFieldCount";
    public static final String OUT_OF_ORDER_TIME_COUNT = "outOfOrderTimeStampCount";
    public static final String FAILED_TRANSFORM_COUNT = "failedTransformCount";
    public static final String LATEST_RECORD_TIME = "latestRecordTimeStamp";
    public static final String EXCLUDED_RECORD_COUNT = "excludedRecordCount";


    private Long m_BucketCount;
    private long m_ProcessedRecordCount;
    private long m_ProcessedFieldCount;
    private long m_InputBytes;
    private long m_InputFieldCount;
    private long m_InvalidDateCount;
    private long m_MissingFieldCount;
    private long m_OutOfOrderTimeStampCount;
    private long m_FailedTransformCount;
    private long m_ExcludedRecordCount;
    private Date m_LatestRecordTimeStamp;

    public DataCounts()
    {
        m_BucketCount = new Long(0);
    }

    public DataCounts(DataCounts lhs)
    {
        m_BucketCount = lhs.m_BucketCount;
        m_ProcessedRecordCount = lhs.m_ProcessedRecordCount;
        m_ProcessedFieldCount = lhs.m_ProcessedFieldCount;
        m_InputBytes = lhs.m_InputBytes;
        m_InputFieldCount = lhs.m_InputFieldCount;
        m_InvalidDateCount = lhs.m_InvalidDateCount;
        m_MissingFieldCount = lhs.m_MissingFieldCount;
        m_OutOfOrderTimeStampCount = lhs.m_OutOfOrderTimeStampCount;
        m_FailedTransformCount = lhs.m_FailedTransformCount;
        m_LatestRecordTimeStamp = lhs.m_LatestRecordTimeStamp;
        m_ExcludedRecordCount = lhs.m_ExcludedRecordCount;
    }


    /**
     * The number of bucket results
     * @return May be <code>null</code>
     */
    public Long getBucketCount()
    {
        return m_BucketCount;
    }

    public void setBucketCount(Long count)
    {
        m_BucketCount = count;
    }

    /**
     * Number of records processed by this job.
     * This value is the number of records sent passed on to
     * the engine i.e. {@linkplain #getInputRecordCount()} minus
     * records with bad dates or out of order
     * @return Number of records processed by this job {@code long}
     */
    public long getProcessedRecordCount()
    {
        return m_ProcessedRecordCount;
    }

    public void setProcessedRecordCount(long count)
    {
        m_ProcessedRecordCount = count;
    }

    public void incrementProcessedRecordCount(long additional)
    {
        m_ProcessedRecordCount += additional;
    }

    /**
     * Number of data points (processed record count * the number
     * of analysed fields) processed by this job. This count does
     * not include the time field.
     * @return Number of data points processed by this job {@code long}
     */
    public long getProcessedFieldCount()
    {
        return m_ProcessedFieldCount;
    }

    public void setProcessedFieldCount(long count)
    {
        m_ProcessedFieldCount = count;
    }

    public void calcProcessedFieldCount(long analysisFieldsPerRecord)
    {
        m_ProcessedFieldCount =
                (m_ProcessedRecordCount * analysisFieldsPerRecord)
                - m_MissingFieldCount;

        // processedFieldCount could be a -ve value if no
        // records have been written in which case it should be 0
        m_ProcessedFieldCount = (m_ProcessedFieldCount < 0) ? 0 : m_ProcessedFieldCount;
    }

    /**
     * Total number of input records read.
     * This = processed record count + date parse error records count
     * + out of order record count.
     *
     * Records with missing fields are counted as they are still written.
     * @return Total number of input records read {@code long}
     */
    @JsonProperty
    public long getInputRecordCount()
    {
        return m_ProcessedRecordCount + m_OutOfOrderTimeStampCount
                                + m_InvalidDateCount + m_ExcludedRecordCount;
    }

    /**
     * Only present to keep jackson serialisation happy.
     * This property should not be deserialised
     * @param count
     */
    @JsonIgnore
    private void setInputRecordCount(long count)
    {
        throw new IllegalStateException();
    }

    /**
     * The total number of bytes sent to this job.
     * This value includes the bytes from any  records
     * that have been discarded for any  reason
     * e.g. because the date cannot be read
     * @return Volume in bytes
     */
    public long getInputBytes()
    {
        return m_InputBytes;
    }

    public void setInputBytes(long volume)
    {
        m_InputBytes = volume;
    }

    public void incrementInputBytes(long additional)
    {
        m_InputBytes += additional;
    }

    /**
     * The total number of fields sent to the job
     * including fields that aren't analysed.
     * @return The total number of fields sent to the job
     */
    public long getInputFieldCount()
    {
        return m_InputFieldCount;
    }

    public void setInputFieldCount(long count)
    {
        m_InputFieldCount = count;
    }

    public void incrementInputFieldCount(long additional)
    {
        m_InputFieldCount += additional;
    }

    /**
     * The number of records with an invalid date field that could
     * not be parsed or converted to epoch time.
     * @return The number of records with an invalid date field
     */
    public long getInvalidDateCount()
    {
        return m_InvalidDateCount;
    }

    public void setInvalidDateCount(long count)
    {
        m_InvalidDateCount = count;
    }

    public void incrementInvalidDateCount(long additional)
    {
        m_InvalidDateCount += additional;
    }


    /**
     * The number of missing fields that had been
     * configured for analysis.
     * @return The number of missing fields
     */
    public long getMissingFieldCount()
    {
        return m_MissingFieldCount;
    }

    public void setMissingFieldCount(long count)
    {
        m_MissingFieldCount = count;
    }

    public void incrementMissingFieldCount(long additional)
    {
        m_MissingFieldCount += additional;
    }

    /**
     * The number of records with a timestamp that is
     * before the time of the latest record. Records should
     * be in ascending chronological order
     * @return The number of records with a timestamp that is before the time of the latest record
     */
    public long getOutOfOrderTimeStampCount()
    {
        return m_OutOfOrderTimeStampCount;
    }

    public void setOutOfOrderTimeStampCount(long count)
    {
        m_OutOfOrderTimeStampCount = count;
    }

    public void incrementOutOfOrderTimeStampCount(long additional)
    {
        m_OutOfOrderTimeStampCount += additional;
    }


    /**
     * The number of transforms that failed.
     * In theory this could be more than the number of records
     * if multiple transforms are applied to each record
     * @return The number of transforms that failed
     */
    public long getFailedTransformCount()
    {
        return m_FailedTransformCount;
    }

    public void setFailedTransformCount(long failedTransformCount)
    {
        this.m_FailedTransformCount = failedTransformCount;
    }

    public void incrementFailedTransformCount(long additional)
    {
        m_FailedTransformCount += additional;
    }

    /**
     * The number of records excluded by a transform
     * @return Number of excluded records
     */
    public long getExcludedRecordCount()
    {
        return m_ExcludedRecordCount;
    }

    public void setExcludedRecordCount(long excludedRecordCount)
    {
        m_ExcludedRecordCount = excludedRecordCount;
    }

    public void incrementExcludedRecordCount(long additional)
    {
        m_ExcludedRecordCount += additional;
    }

    /**
     * The time of the latest record seen.
     * @return Latest record time
     */
    public Date getLatestRecordTimeStamp()
    {
        return m_LatestRecordTimeStamp;
    }

    public void setLatestRecordTimeStamp(Date latestRecordTime)
    {
        m_LatestRecordTimeStamp = latestRecordTime;
    }

    public Map<String, Object> toObjectMap()
    {
        Map<String, Object> map = new HashMap<>();
        map.put(BUCKET_COUNT, m_BucketCount);
        map.put(PROCESSED_RECORD_COUNT, m_ProcessedRecordCount);
        map.put(PROCESSED_FIELD_COUNT, m_ProcessedFieldCount);
        map.put(INPUT_BYTES, m_InputBytes);
        map.put(INPUT_RECORD_COUNT, getInputRecordCount());
        map.put(INPUT_FIELD_COUNT, m_InputFieldCount);
        map.put(INVALID_DATE_COUNT, m_InvalidDateCount);
        map.put(MISSING_FIELD_COUNT, m_MissingFieldCount);
        map.put(OUT_OF_ORDER_TIME_COUNT, m_OutOfOrderTimeStampCount);
        map.put(FAILED_TRANSFORM_COUNT, m_FailedTransformCount);
        map.put(LATEST_RECORD_TIME, m_LatestRecordTimeStamp);
        map.put(EXCLUDED_RECORD_COUNT, m_ExcludedRecordCount);
        return map;
    }

    /**
     * Equality test
     */
    @Override
    public boolean equals(Object other)
    {
        if (this == other)
        {
            return true;
        }

        if (other instanceof DataCounts == false)
        {
            return false;
        }

        DataCounts that = (DataCounts)other;

        return this.m_BucketCount.equals(that.m_BucketCount) &&
                this.m_ProcessedRecordCount == that.m_ProcessedRecordCount &&
                this.m_ProcessedFieldCount == that.m_ProcessedFieldCount &&
                this.m_InputBytes == that.m_InputBytes &&
                this.m_InputFieldCount == that.m_InputFieldCount &&
                this.m_InvalidDateCount == that.m_InvalidDateCount &&
                this.m_MissingFieldCount == that.m_MissingFieldCount &&
                this.m_OutOfOrderTimeStampCount == that.m_OutOfOrderTimeStampCount &&
                this.m_FailedTransformCount == that.m_FailedTransformCount &&
                this.m_ExcludedRecordCount == that.m_ExcludedRecordCount &&
                Objects.equals(this.m_LatestRecordTimeStamp, that.m_LatestRecordTimeStamp);
    }

    @Override
    public int hashCode()
    {
        return Objects.hash(m_BucketCount, m_ProcessedRecordCount, m_ProcessedFieldCount,
                m_InputBytes, m_InputFieldCount, m_InvalidDateCount, m_MissingFieldCount,
                m_OutOfOrderTimeStampCount, m_FailedTransformCount, m_ExcludedRecordCount,
                m_LatestRecordTimeStamp);
    }
} @JsonIgnoreProperties({"transformTime", "epochMs"})
@JsonInclude(Include.NON_NULL)
public class DataDescription
{
    /**
     * Enum of the acceptable data formats.
     */
    public enum DataFormat
    {
        JSON, DELIMITED, SINGLE_LINE, ELASTICSEARCH;

        /**
         * Delimited used to be called delineated. We keep supporting that for backwards
         * compatibility.
         */
        private static final String DEPRECATED_DELINEATED = "DELINEATED";

        /**
         * Case-insensitive from string method.
         * Works with either JSON, json, etc.
         *
         * @param value String representation
         * @return The data format
         */
        @JsonCreator
        public static DataFormat forString(String value)
        {
            String valueUpperCase = value.toUpperCase();
            return DEPRECATED_DELINEATED.equals(valueUpperCase) ? DELIMITED : DataFormat
                    .valueOf(valueUpperCase);
        }
    }

    /**
     * Special time format string for epoch times (seconds)
     */
    public static final String EPOCH = "epoch";

    /**
     * Special time format string for epoch times (milli-seconds)
     */
    public static final String EPOCH_MS = "epoch_ms";

    /**
     * The format field name
     */
    public static final String FORMAT = "format";
    /**
     * The time field name
     */
    public static final String TIME_FIELD_NAME = "timeField";

    /**
     * By default autodetect expects the timestamp in a field with this name
     */
    public static final String DEFAULT_TIME_FIELD = "time";

    /**
     * The timeFormat field name
     */
    public static final String TIME_FORMAT = "timeFormat";
    /**
     * The field delimiter field name
     */
    public static final String FIELD_DELIMITER = "fieldDelimiter";
    /**
     * The quote char field name
     */
    public static final String QUOTE_CHARACTER = "quoteCharacter";

    /**
     * The default field delimiter expected by the native autodetect_api
     * program.
     */
    public static final char DEFAULT_DELIMITER = '\t';

    /**
     * Csv data must have this line ending
     */
    public static final char LINE_ENDING = '\n';

    /**
     * The default quote character used to escape text in
     * delineated data formats
     */
    public static final char DEFAULT_QUOTE_CHAR = '"';

    private DataFormat m_DataFormat;
    private String m_TimeFieldName;
    private String m_TimeFormat;
    private char m_FieldDelimiter;
    private char m_QuoteCharacter;

    public DataDescription()
    {
        m_DataFormat = DataFormat.DELIMITED;
        m_TimeFieldName = DEFAULT_TIME_FIELD;
        m_TimeFormat = EPOCH;
        m_FieldDelimiter = DEFAULT_DELIMITER;
        m_QuoteCharacter = DEFAULT_QUOTE_CHAR;
    }

    /**
     * The format of the data to be processed.
     * Defaults to {@link DataDescription.DataFormat#DELIMITED}
     * @return The data format
     */
    public DataFormat getFormat()
    {
        return m_DataFormat;
    }

    public void setFormat(DataFormat format)
    {
        m_DataFormat = format;
    }

    /**
     * The name of the field containing the timestamp
     * @return A String if set or <code>null</code>
     */
    public String getTimeField()
    {
        return m_TimeFieldName;
    }

    public void setTimeField(String fieldName)
    {
        m_TimeFieldName = fieldName;
    }

    /**
     * Either {@value #EPOCH}, {@value #EPOCH_MS} or a SimpleDateTime format string.
     * If not set (is <code>null</code> or an empty string) or set to
     * {@value #EPOCH} (the default) then the date is assumed to be in
     * seconds from the epoch.
     * @return A String if set or <code>null</code>
     */
    public String getTimeFormat()
    {
        return m_TimeFormat;
    }

    public void setTimeFormat(String format)
    {
        m_TimeFormat = format;
    }

    /**
     * If the data is in a delineated format with a header e.g. csv or tsv
     * this is the delimiter character used. This is only applicable if
     * {@linkplain #getFormat()} is {@link DataDescription.DataFormat#DELIMITED}.
     * The default value is {@value #DEFAULT_DELIMITER}
     *
     * @return A char
     */
    public char getFieldDelimiter()
    {
        return m_FieldDelimiter;
    }

    public void setFieldDelimiter(char delimiter)
    {
        m_FieldDelimiter = delimiter;
    }

    /**
     * The quote character used in delineated formats.
     * Defaults to {@value #DEFAULT_QUOTE_CHAR}
     * @return The delineated format quote character
     */
    public char getQuoteCharacter()
    {
        return m_QuoteCharacter;
    }

    public void setQuoteCharacter(char value)
    {
        m_QuoteCharacter = value;
    }


    /**
     * Returns true if the data described by this object needs
     * transforming before processing by autodetect.
     * A transformation must be applied if either a timeformat is
     * not in seconds since the epoch or the data is in Json format.
     * @return True if the data should be transformed.
     */
    public boolean transform()
    {
        return m_DataFormat == DataFormat.JSON ||
                isTransformTime();
    }


    /**
     * Return true if the time is in a format that needs transforming.
     * Anytime format this isn't {@value #EPOCH} or <code>null</code>
     * needs transforming.
     * @return True if the time field needs to be transformed.
     */
    public boolean isTransformTime()
    {
        return m_TimeFormat != null && !EPOCH.equals(m_TimeFormat);
    }

    /**
     * Return true if the time format is {@value #EPOCH_MS}
     * @return True if the date is in milli-seconds since the epoch.
     */
    public boolean isEpochMs()
    {
        return EPOCH_MS.equals(m_TimeFormat);
    }

    /**
     * Overridden equality test
     */
    @Override
    public boolean equals(Object other)
    {
        if (this == other)
        {
            return true;
        }

        if (other instanceof DataDescription == false)
        {
            return false;
        }

        DataDescription that = (DataDescription)other;

        return this.m_DataFormat == that.m_DataFormat &&
                this.m_QuoteCharacter == that.m_QuoteCharacter &&
                Objects.equals(this.m_TimeFieldName, that.m_TimeFieldName) &&
                Objects.equals(this.m_TimeFormat, that.m_TimeFormat) &&
                Objects.equals(this.m_FieldDelimiter, that.m_FieldDelimiter);
    }

    @Override
    public int hashCode()
    {
        return Objects.hash(m_DataFormat, m_QuoteCharacter, m_TimeFieldName,
                m_TimeFormat, m_FieldDelimiter);
    }


} @JsonInclude(Include.NON_NULL)
public class Detector
{
    public static final String DETECTOR_DESCRIPTION = "detectorDescription";
    public static final String FUNCTION = "function";
    public static final String FIELD_NAME = "fieldName";
    public static final String BY_FIELD_NAME = "byFieldName";
    public static final String OVER_FIELD_NAME = "overFieldName";
    public static final String PARTITION_FIELD_NAME = "partitionFieldName";
    public static final String USE_NULL = "useNull";
    public static final String EXCLUDE_FREQUENT = "excludeFrequent";


    public static final String COUNT = "count";
    public static final String HIGH_COUNT = "high_count";
    public static final String LOW_COUNT = "low_count";
    public static final String NON_ZERO_COUNT = "non_zero_count";
    public static final String LOW_NON_ZERO_COUNT = "low_non_zero_count";
    public static final String HIGH_NON_ZERO_COUNT = "high_non_zero_count";
    public static final String NZC = "nzc";
    public static final String LOW_NZC = "low_nzc";
    public static final String HIGH_NZC = "high_nzc";
    public static final String DISTINCT_COUNT = "distinct_count";
    public static final String LOW_DISTINCT_COUNT = "low_distinct_count";
    public static final String HIGH_DISTINCT_COUNT = "high_distinct_count";
    public static final String DC = "dc";
    public static final String LOW_DC = "low_dc";
    public static final String HIGH_DC = "high_dc";
    public static final String RARE = "rare";
    public static final String FREQ_RARE = "freq_rare";
    public static final String INFO_CONTENT = "info_content";
    public static final String LOW_INFO_CONTENT = "low_info_content";
    public static final String HIGH_INFO_CONTENT = "high_info_content";
    public static final String METRIC = "metric";
    public static final String MEAN = "mean";
    public static final String MEDIAN = "median";
    public static final String HIGH_MEAN = "high_mean";
    public static final String LOW_MEAN = "low_mean";
    public static final String AVG = "avg";
    public static final String HIGH_AVG = "high_avg";
    public static final String LOW_AVG = "low_avg";
    public static final String MIN = "min";
    public static final String MAX = "max";
    public static final String SUM = "sum";
    public static final String LOW_SUM = "low_sum";
    public static final String HIGH_SUM = "high_sum";
    public static final String NON_NULL_SUM = "non_null_sum";
    public static final String LOW_NON_NULL_SUM = "low_non_null_sum";
    public static final String HIGH_NON_NULL_SUM = "high_non_null_sum";
    /**
     * Population variance is called varp to match Splunk
     */
    public static final String POPULATION_VARIANCE = "varp";
    public static final String LOW_POPULATION_VARIANCE = "low_varp";
    public static final String HIGH_POPULATION_VARIANCE = "high_varp";
    public static final String TIME_OF_DAY = "time_of_day";
    public static final String TIME_OF_WEEK = "time_of_week";
    public static final String LAT_LONG = "lat_long";


    /**
     * The set of valid function names.
     */
    public static final Set<String> ANALYSIS_FUNCTIONS =
            new HashSet<>(Arrays.asList(
                // The convention here is that synonyms (only) go on the same line
                COUNT,
                HIGH_COUNT,
                LOW_COUNT,
                NON_ZERO_COUNT, NZC,
                LOW_NON_ZERO_COUNT, LOW_NZC,
                HIGH_NON_ZERO_COUNT, HIGH_NZC,
                DISTINCT_COUNT, DC,
                LOW_DISTINCT_COUNT, LOW_DC,
                HIGH_DISTINCT_COUNT, HIGH_DC,
                RARE,
                FREQ_RARE,
                INFO_CONTENT,
                LOW_INFO_CONTENT,
                HIGH_INFO_CONTENT,
                METRIC,
                MEAN, AVG,
                HIGH_MEAN, HIGH_AVG,
                LOW_MEAN, LOW_AVG,
                MEDIAN,
                MIN,
                MAX,
                SUM,
                LOW_SUM,
                HIGH_SUM,
                NON_NULL_SUM,
                LOW_NON_NULL_SUM,
                HIGH_NON_NULL_SUM,
                POPULATION_VARIANCE,
                LOW_POPULATION_VARIANCE,
                HIGH_POPULATION_VARIANCE,
                TIME_OF_DAY,
                TIME_OF_WEEK,
                LAT_LONG
            ));

    /**
     * The set of functions that do not require a field, by field or over field
     */
    public static final Set<String> COUNT_WITHOUT_FIELD_FUNCTIONS =
            new HashSet<>(Arrays.asList(
                COUNT,
                HIGH_COUNT,
                LOW_COUNT,
                NON_ZERO_COUNT, NZC,
                LOW_NON_ZERO_COUNT, LOW_NZC,
                HIGH_NON_ZERO_COUNT, HIGH_NZC,
                TIME_OF_DAY,
                TIME_OF_WEEK
            ));

    /**
     * The set of functions that require a fieldname
     */
    public static final Set<String> FIELD_NAME_FUNCTIONS =
            new HashSet<>(Arrays.asList(
                DISTINCT_COUNT, DC,
                LOW_DISTINCT_COUNT, LOW_DC,
                HIGH_DISTINCT_COUNT, HIGH_DC,
                INFO_CONTENT,
                LOW_INFO_CONTENT,
                HIGH_INFO_CONTENT,
                METRIC,
                MEAN, AVG,
                HIGH_MEAN, HIGH_AVG,
                LOW_MEAN, LOW_AVG,
                MEDIAN,
                MIN,
                MAX,
                SUM,
                LOW_SUM,
                HIGH_SUM,
                NON_NULL_SUM,
                LOW_NON_NULL_SUM,
                HIGH_NON_NULL_SUM,
                POPULATION_VARIANCE,
                LOW_POPULATION_VARIANCE,
                HIGH_POPULATION_VARIANCE,
                LAT_LONG
            ));

    /**
     * The set of functions that require a by fieldname
     */
    public static final Set<String> BY_FIELD_NAME_FUNCTIONS =
            new HashSet<>(Arrays.asList(
                RARE,
                FREQ_RARE
            ));

    /**
     * The set of functions that require a over fieldname
     */
    public static final Set<String> OVER_FIELD_NAME_FUNCTIONS =
            new HashSet<>(Arrays.asList(
                FREQ_RARE
            ));

    /**
     * The set of functions that cannot have a by fieldname
     */
    public static final Set<String> NO_BY_FIELD_NAME_FUNCTIONS =
            new HashSet<>();

    /**
     * The set of functions that cannot have an over fieldname
     */
    public static final Set<String> NO_OVER_FIELD_NAME_FUNCTIONS =
            new HashSet<>(Arrays.asList(
                NON_ZERO_COUNT, NZC,
                LOW_NON_ZERO_COUNT, LOW_NZC,
                HIGH_NON_ZERO_COUNT, HIGH_NZC
            ));

    /**
     * The set of functions that must not be used with overlapping buckets
     */
    public static final Set<String> NO_OVERLAPPING_BUCKETS_FUNCTIONS =
            new HashSet<>(Arrays.asList(
                    RARE,
                    FREQ_RARE
            ));

    /**
     * The set of functions that should not be used with overlapping buckets
     * as they gain no benefit but have overhead
     */
    public static final Set<String> OVERLAPPING_BUCKETS_FUNCTIONS_NOT_NEEDED =
            new HashSet<>(Arrays.asList(
                    MIN,
                    MAX,
                    TIME_OF_DAY,
                    TIME_OF_WEEK
            ));

    /**
     * field names cannot contain any of these characters
     *     ", \
     */
    public static final Character [] PROHIBITED_FIELDNAME_CHARACTERS = {'"', '\\'};
    public static final String PROHIBITED = String.join(",",
            Arrays.stream(PROHIBITED_FIELDNAME_CHARACTERS).map(
                    c -> Character.toString(c)).collect(Collectors.toList()));


    private String m_DetectorDescription;
    private String m_Function;
    private String m_FieldName;
    private String m_ByFieldName;
    private String m_OverFieldName;
    private String m_PartitionFieldName;
    private Boolean m_UseNull;
    private String m_ExcludeFrequent;

    public Detector()
    {

    }

    public String getDetectorDescription()
    {
        return m_DetectorDescription;
    }

    public void setDetectorDescription(String description)
    {
        m_DetectorDescription = description;
    }

    /**
     * The analysis function used e.g. count, rare, min etc. There is no
     * validation to check this value is one a predefined set
     * @return The function or <code>null</code> if not set
     */
    public String getFunction()
    {
        return m_Function;
    }

    public void setFunction(String function)
    {
        this.m_Function = function;
    }

    /**
     * The Analysis field
     * @return The field to analyse
     */
    public String getFieldName()
    {
        return m_FieldName;
    }

    public void setFieldName(String fieldName)
    {
        this.m_FieldName = fieldName;
    }

    /**
     * The 'by' field or <code>null</code> if not set.
     * @return The 'by' field
     */
    public String getByFieldName()
    {
        return m_ByFieldName;
    }

    public void setByFieldName(String byFieldName)
    {
        this.m_ByFieldName = byFieldName;
    }

    /**
     * The 'over' field or <code>null</code> if not set.
     * @return The 'over' field
     */
    public String getOverFieldName()
    {
        return m_OverFieldName;
    }

    public void setOverFieldName(String overFieldName)
    {
        this.m_OverFieldName = overFieldName;
    }

    /**
     * Segments the analysis along another field to have completely
     * independent baselines for each instance of partitionfield
     *
     * @return The Partition Field
     */
    public String getPartitionFieldName()
    {
        return m_PartitionFieldName;
    }

    public void setPartitionFieldName(String partitionFieldName)
    {
        this.m_PartitionFieldName = partitionFieldName;
    }


    /**
     * Where there isn't a value for the 'by' or 'over' field should a new
     * series be used as the 'null' series.
     * @return true if the 'null' series should be created
     */
    public Boolean isUseNull()
    {
        return m_UseNull;
    }

    public void setUseNull(Boolean useNull)
    {
        this.m_UseNull = useNull;
    }

    /**
     * Excludes frequently-occuring metrics from the analysis;
     * can apply to 'by' field, 'over' field, or both
     *
     * @return the value that the user set
     */
    public String getExcludeFrequent()
    {
        return m_ExcludeFrequent;
    }

    public void setExcludeFrequent(String v)
    {
        m_ExcludeFrequent = v;
    }


    @Override
    public boolean equals(Object other)
    {
        if (this == other)
        {
            return true;
        }

        if (other instanceof Detector == false)
        {
            return false;
        }

        Detector that = (Detector)other;

        return Objects.equals(this.m_DetectorDescription, that.m_DetectorDescription) &&
               Objects.equals(this.m_Function, that.m_Function) &&
               Objects.equals(this.m_FieldName, that.m_FieldName) &&
               Objects.equals(this.m_ByFieldName, that.m_ByFieldName) &&
               Objects.equals(this.m_OverFieldName, that.m_OverFieldName) &&
               Objects.equals(this.m_PartitionFieldName, that.m_PartitionFieldName) &&
               Objects.equals(this.m_UseNull, that.m_UseNull) &&
               Objects.equals(this.m_ExcludeFrequent, that.m_ExcludeFrequent);
    }

    @Override
    public int hashCode()
    {
        return Objects.hash(m_DetectorDescription, m_Function, m_FieldName, m_ByFieldName,
                m_OverFieldName, m_PartitionFieldName, m_UseNull, m_ExcludeFrequent);
    }
} @JsonInclude(Include.NON_NULL)
public class JobDetails
{
    public static final long DEFAULT_TIMEOUT = 600;
    public static final long DEFAULT_BUCKETSPAN = 300;

    public static final String TYPE = "job";

    /*
     * Field names used in serialization
     */
    public static final String ANALYSIS_CONFIG = "analysisConfig";
    public static final String ANALYSIS_LIMITS = "analysisLimits";
    public static final String COUNTS = "counts";
    public static final String CREATE_TIME = "createTime";
    public static final String CUSTOM_SETTINGS = "customSettings";
    public static final String DATA_DESCRIPTION = "dataDescription";
    public static final String DESCRIPTION = "description";
    public static final String ENDPOINTS = "endpoints";
    public static final String FINISHED_TIME = "finishedTime";
    public static final String IGNORE_DOWNTIME = "ignoreDowntime";
    public static final String LAST_DATA_TIME = "lastDataTime";
    public static final String MODEL_DEBUG_CONFIG = "modelDebugConfig";
    public static final String SCHEDULER_CONFIG = "schedulerConfig";
    public static final String RENORMALIZATION_WINDOW_DAYS = "renormalizationWindowDays";
    public static final String BACKGROUND_PERSIST_INTERVAL = "backgroundPersistInterval";
    public static final String MODEL_SNAPSHOT_RETENTION_DAYS = "modelSnapshotRetentionDays";
    public static final String RESULTS_RETENTION_DAYS = "resultsRetentionDays";
    public static final String STATUS = "status";
    public static final String SCHEDULER_STATUS = "schedulerStatus";
    public static final String TIMEOUT = "timeout";
    public static final String TRANSFORMS = "transforms";

    /** Endpoints key names */
    public static final String ALERT_LONG_POLL_ENDPOINT_KEY= "alertsLongPoll";
    public static final String BUCKETS_ENDPOINT_KEY = "buckets";
    public static final String CATEGORY_DEFINITIONS_ENDPOINT_KEY = "categoryDefinitions";
    public static final String DATA_ENDPOINT_KEY = "data";
    public static final String LOGS_ENDPOINT_KEY = "logs";
    public static final String RECORDS_ENDPOINT_KEY = "records";
    public static final String MODEL_SNAPSHOTS_ENDPOINT_KEY = "modelSnapshots";

    private String m_JobId;
    private String m_Description;
    private JobStatus m_Status;
    private JobSchedulerStatus m_SchedulerStatus;

    private Date m_CreateTime;
    private Date m_FinishedTime;
    private Date m_LastDataTime;

    private long m_Timeout;

    private AnalysisConfig m_AnalysisConfig;
    private AnalysisLimits m_AnalysisLimits;
    private SchedulerConfig m_SchedulerConfig;
    private DataDescription m_DataDescription;
    private ModelSizeStats m_ModelSizeStats;
    private List<TransformConfig> m_Transforms;
    private ModelDebugConfig m_ModelDebugConfig;
    private DataCounts m_Counts;
    private IgnoreDowntime m_IgnoreDowntime;
    private Long m_RenormalizationWindowDays;
    private Long m_BackgroundPersistInterval;
    private Long m_ModelSnapshotRetentionDays;
    private Long m_ResultsRetentionDays;
    private Map<String, Object> m_CustomSettings;

    /* The endpoints should not be persisted in the storage */
    private URI m_Location;
    private Map<String, URI> m_Endpoints;

    /**
     * Default constructor required for serialisation
     */
    public JobDetails()
    {
        m_Counts = new DataCounts();
        m_Status = JobStatus.CLOSED;
        m_CreateTime = new Date();
    }

    /**
     * Create a new Job with the passed <code>jobId</code> and the
     * configuration parameters, where fields are not set in the
     * JobConfiguration defaults will be used.
     *
     * @param jobId the job id
     * @param jobConfig the job configuration
     */
    public JobDetails(String jobId, JobConfiguration jobConfig)
    {
        this();

        m_JobId = jobId;
        m_Description = jobConfig.getDescription();
        m_Timeout = (jobConfig.getTimeout() != null) ? jobConfig.getTimeout() : DEFAULT_TIMEOUT;

        m_AnalysisConfig = jobConfig.getAnalysisConfig();
        m_AnalysisLimits = jobConfig.getAnalysisLimits();
        m_SchedulerConfig = jobConfig.getSchedulerConfig();
        invokeIfNotNull(m_SchedulerConfig, sc -> sc.fillDefaults());
        m_Transforms = jobConfig.getTransforms();
        m_ModelDebugConfig = jobConfig.getModelDebugConfig();

        invokeIfNotNull(jobConfig.getDataDescription(), dd -> m_DataDescription = dd);
        m_IgnoreDowntime = jobConfig.getIgnoreDowntime();
        m_RenormalizationWindowDays = jobConfig.getRenormalizationWindowDays();
        m_BackgroundPersistInterval = jobConfig.getBackgroundPersistInterval();
        m_ModelSnapshotRetentionDays = jobConfig.getModelSnapshotRetentionDays();
        m_ResultsRetentionDays = jobConfig.getResultsRetentionDays();
        m_CustomSettings = jobConfig.getCustomSettings();
    }

    private <T> void invokeIfNotNull(T obj, Consumer<T> consumer)
    {
        if (obj != null)
        {
            consumer.accept(obj);
        }
    }

    /**
     * Return the Job Id.
     * This name is preferred when serialising to the REST API.
     * @return The job Id string
     */
    @JsonView(JsonViews.RestApiView.class)
    public String getId()
    {
        return m_JobId;
    }

    /**
     * Set the job's Id.
     * In general this method should not be used as the Id does not change
     * once set. This method is provided for the Jackson object mapper to
     * de-serialise this class from Json.
     *
     * @param id the job id
     */
    public void setId(String id)
    {
        m_JobId = id;
    }

    /**
     * Return the Job Id.
     * This name is preferred when serialising to the data store.
     * @return The job Id string
     */
    @JsonView(JsonViews.DatastoreView.class)
    public String getJobId()
    {
        return m_JobId;
    }

    /**
     * Set the job's Id.
     * In general this method should not be used as the Id does not change
     * once set. This method is provided for the Jackson object mapper to
     * de-serialise this class from Json.
     *
     * @param jobId the job id
     */
    public void setJobId(String jobId)
    {
        m_JobId = jobId;
    }

    /**
     * The job description
     * @return job description
     */
    public String getDescription()
    {
        return m_Description;
    }

    public void setDescription(String description)
    {
        m_Description = description;
    }

    public Map<String, URI> getEndpoints()
    {
        return m_Endpoints;
    }

    public void setEndpoints(Map<String, URI> endpoints)
    {
        m_Endpoints = endpoints;
    }

    public URI getLocation()
    {
        return m_Location;
    }

    public void setLocation(URI location)
    {
        m_Location = location;
    }

    /**
     * Return the Job Status. Jobs are initialised to {@link JobStatus#CLOSED}
     * when created and move into the @link JobStatus#RUNNING} state when
     * processing data. Once data has been processed the status will be
     * either {@link JobStatus#CLOSED} or {@link JobStatus#FAILED}
     *
     * @return The job's status
     */
    public JobStatus getStatus()
    {
        return m_Status;
    }

    public void setStatus(JobStatus status)
    {
        m_Status = status;
    }

    public JobSchedulerStatus getSchedulerStatus()
    {
        return m_SchedulerStatus;
    }

    public void setSchedulerStatus(JobSchedulerStatus schedulerStatus)
    {
        m_SchedulerStatus = schedulerStatus;
    }

    /**
     * The Job creation time.
     * This name is preferred when serialising to the REST API.
     * @return The date the job was created
     */
    @JsonView(JsonViews.RestApiView.class)
    public Date getCreateTime()
    {
        return m_CreateTime;
    }

    public void setCreateTime(Date time)
    {
        m_CreateTime = time;
    }

    /**
     * The Job creation time.
     * This name is preferred when serialising to the data store.
     * @return The date the job was created
     */
    @JsonView(JsonViews.DatastoreView.class)
    @JsonProperty("@timestamp")
    public Date getAtTimestamp()
    {
        return m_CreateTime;
    }

    @JsonProperty("@timestamp")
    public void setAtTimestamp(Date time)
    {
        m_CreateTime = time;
    }

    /**
     * The time the job was finished or <code>null</code> if not finished.
     * @return The date the job was last retired or <code>null</code>
     */
    public Date getFinishedTime()
    {
        return m_FinishedTime;
    }

    public void setFinishedTime(Date finishedTime)
    {
        m_FinishedTime = finishedTime;
    }

    /**
     * The last time data was uploaded to the job or <code>null</code>
     * if no data has been seen.
     * @return The date at which the last data was processed
     */
    public Date getLastDataTime()
    {
        return m_LastDataTime;
    }

    public void setLastDataTime(Date lastTime)
    {
        m_LastDataTime = lastTime;
    }

    /**
     * The job timeout setting in seconds. Jobs are retired if they do not
     * receive data for this period of time.
     * The default is 600 seconds
     * @return The timeout period in seconds
     */
    public long getTimeout()
    {
        return m_Timeout;
    }

    public void setTimeout(long timeout)
    {
        m_Timeout = timeout;
    }


    /**
     * The analysis configuration object
     * @return The AnalysisConfig
     */
    public AnalysisConfig getAnalysisConfig()
    {
        return m_AnalysisConfig;
    }

    public void setAnalysisConfig(AnalysisConfig config)
    {
        m_AnalysisConfig = config;
    }

    /**
     * The analysis options object
     * @return The AnalysisLimits
     */
    public AnalysisLimits getAnalysisLimits()
    {
        return m_AnalysisLimits;
    }

    public void setAnalysisLimits(AnalysisLimits analysisLimits)
    {
        m_AnalysisLimits = analysisLimits;
    }

    public IgnoreDowntime getIgnoreDowntime()
    {
        return m_IgnoreDowntime;
    }

    public void setIgnoreDowntime(IgnoreDowntime ignoreDowntime)
    {
        m_IgnoreDowntime = ignoreDowntime;
    }

    public SchedulerConfig getSchedulerConfig()
    {
        return m_SchedulerConfig;
    }

    public void setSchedulerConfig(SchedulerConfig schedulerConfig)
    {
        m_SchedulerConfig = schedulerConfig;
    }

    public ModelDebugConfig getModelDebugConfig()
    {
        return m_ModelDebugConfig;
    }

    public void setModelDebugConfig(ModelDebugConfig modelDebugConfig)
    {
        m_ModelDebugConfig = modelDebugConfig;
    }

    /**
    * The memory usage object
    * @return The ModelSizeStats
    */
    public ModelSizeStats getModelSizeStats()
    {
        return m_ModelSizeStats;
    }

    public void setModelSizeStats(ModelSizeStats modelSizeStats)
    {
        m_ModelSizeStats = modelSizeStats;
    }

    /**
     * If not set the input data is assumed to be csv with a '_time' field
     * in epoch format.
     * @return A DataDescription or <code>null</code>
     * @see DataDescription
     */
    public DataDescription getDataDescription()
    {
        return m_DataDescription;
    }

    public void setDataDescription(DataDescription dd)
    {
        m_DataDescription = dd;
    }

    public List<TransformConfig> getTransforms()
    {
        return m_Transforms;
    }

    public void setTransforms(List<TransformConfig> transforms)
    {
        m_Transforms = transforms;
    }


    /**
     * Processed records count
     * @return the processed records counts
     */
    public DataCounts getCounts()
    {
        return m_Counts;
    }

    /**
     * Processed records count
     * @param counts the counts {@code DataCounts}
     */
    public void setCounts(DataCounts counts)
    {
        m_Counts = counts;
    }

    /**
     * The duration of the renormalization window in days
     * @return renormalization window in days
     */
    public Long getRenormalizationWindowDays()
    {
        return m_RenormalizationWindowDays;
    }

    /**
     * Set the renormalization window duration
     * @param renormalizationWindowDays the renormalization window in days
     */
    public void setRenormalizationWindowDays(Long renormalizationWindowDays)
    {
        m_RenormalizationWindowDays = renormalizationWindowDays;
    }

    /**
     * The background persistence interval in seconds
     * @return background persistence interval in seconds
     */
    public Long getBackgroundPersistInterval()
    {
        return m_BackgroundPersistInterval;
    }

    /**
     * Set the background persistence interval
     * @param backgroundPersistInterval the persistence interval in seconds
     */
    public void setBackgroundPersistInterval(Long backgroundPersistInterval)
    {
        m_BackgroundPersistInterval = backgroundPersistInterval;
    }

    public Long getModelSnapshotRetentionDays()
    {
        return m_ModelSnapshotRetentionDays;
    }

    public void setModelSnapshotRetentionDays(Long modelSnapshotRetentionDays)
    {
        m_ModelSnapshotRetentionDays = modelSnapshotRetentionDays;
    }

    public Long getResultsRetentionDays()
    {
        return m_ResultsRetentionDays;
    }

    public void setResultsRetentionDays(Long resultsRetentionDays)
    {
        m_ResultsRetentionDays = resultsRetentionDays;
    }

    public Map<String, Object> getCustomSettings()
    {
        return m_CustomSettings;
    }

    public void setCustomSettings(Map<String, Object> customSettings)
    {
        m_CustomSettings = customSettings;
    }

    /**
     * Get a list of all input data fields mentioned in the job configuration,
     * namely analysis fields, time field and transform input fields.
     * @return the list of fields - never <code>null</code>
     */
    public List<String> allFields()
    {
        Set<String> allFields = new TreeSet<>();

        // analysis fields
        if (m_AnalysisConfig != null)
        {
            allFields.addAll(m_AnalysisConfig.analysisFields());
        }

        // transform input fields
        if (m_Transforms != null)
        {
            for (TransformConfig tc : m_Transforms)
            {
                 List<String> inputFields = tc.getInputs();
                 if (inputFields != null)
                 {
                     allFields.addAll(inputFields);
                 }
            }
        }

        // time field
        if (m_DataDescription != null)
        {
            String timeField = m_DataDescription.getTimeField();
            if (timeField != null)
            {
                allFields.add(timeField);
            }
        }

        // remove empty strings
        allFields.remove("");

        return new ArrayList<String>(allFields);
    }

    /**
     * Prints the more salient fields in a JSON-like format suitable for logging.
     * If every field was written it would spam the log file.
     */
    @Override
    public String toString()
    {
        StringBuilder sb = new StringBuilder("{");
        sb.append("id:").append(getId())
            .append(" description:").append(getDescription())
            .append(" status:").append(getStatus())
            .append(" createTime:").append(getCreateTime())
            .append(" lastDataTime:").append(getLastDataTime())
            .append("}");

        return sb.toString();
    }

    /**
     * Equality test
     */
    @Override
    public boolean equals(Object other)
    {
        if (this == other)
        {
            return true;
        }

        if (other instanceof JobDetails == false)
        {
            return false;
        }

        JobDetails that = (JobDetails)other;

        return Objects.equals(this.m_JobId, that.m_JobId) &&
                Objects.equals(this.m_Description, that.m_Description) &&
                (this.m_Status == that.m_Status) &&
                (this.m_SchedulerStatus == that.m_SchedulerStatus) &&
                Objects.equals(this.m_CreateTime, that.m_CreateTime) &&
                Objects.equals(this.m_FinishedTime, that.m_FinishedTime) &&
                Objects.equals(this.m_LastDataTime, that.m_LastDataTime) &&
                (this.m_Timeout == that.m_Timeout) &&
                Objects.equals(this.m_AnalysisConfig, that.m_AnalysisConfig) &&
                Objects.equals(this.m_AnalysisLimits, that.m_AnalysisLimits) &&
                Objects.equals(this.m_DataDescription, that.m_DataDescription) &&
                Objects.equals(this.m_ModelDebugConfig, that.m_ModelDebugConfig) &&
                Objects.equals(this.m_ModelSizeStats, that.m_ModelSizeStats) &&
                Objects.equals(this.m_Transforms, that.m_Transforms) &&
                Objects.equals(this.m_Counts, that.m_Counts) &&
                Objects.equals(this.m_IgnoreDowntime, that.m_IgnoreDowntime) &&
                Objects.equals(this.m_RenormalizationWindowDays, that.m_RenormalizationWindowDays) &&
                Objects.equals(this.m_BackgroundPersistInterval, that.m_BackgroundPersistInterval) &&
                Objects.equals(this.m_ModelSnapshotRetentionDays, that.m_ModelSnapshotRetentionDays) &&
                Objects.equals(this.m_ResultsRetentionDays, that.m_ResultsRetentionDays) &&
                Objects.equals(this.m_CustomSettings, that.m_CustomSettings) &&
                Objects.equals(this.m_Endpoints, that.m_Endpoints) &&
                Objects.equals(this.m_Location, that.m_Location);
    }

    @Override
    public int hashCode()
    {
        return Objects.hash(m_JobId, m_Description, m_Status, m_SchedulerStatus,m_CreateTime,
                m_FinishedTime, m_LastDataTime, m_Timeout, m_AnalysisConfig, m_AnalysisLimits,
                m_DataDescription, m_ModelDebugConfig, m_ModelSizeStats, m_Transforms, m_Counts,
                m_RenormalizationWindowDays, m_BackgroundPersistInterval, m_ModelSnapshotRetentionDays,
                m_ResultsRetentionDays, m_IgnoreDowntime, m_CustomSettings, m_Endpoints, m_Location);
    }
} @JsonIgnoreProperties({"enabled"})
@JsonInclude(Include.NON_NULL)
public class ModelDebugConfig
{
    /**
     * Enum of the acceptable output destinations.
     */
    public enum DebugDestination
    {
        FILE, DATA_STORE;

        /**
         * Case-insensitive from string method.
         * Works with FILE, File, file, etc.
         *
         * @param value String representation
         * @return The output destination
         */
        @JsonCreator
        public static DebugDestination forString(String value)
        {
            String valueUpperCase = value.toUpperCase();
            return DebugDestination.valueOf(valueUpperCase);
        }
    }

    public static final String TYPE = "modelDebugConfig";
    public static final String WRITE_TO = "writeTo";
    public static final String BOUNDS_PERCENTILE = "boundsPercentile";
    public static final String TERMS = "terms";

    private DebugDestination m_WriteTo;
    private Double m_BoundsPercentile;
    private String m_Terms;

    public ModelDebugConfig()
    {
        // NB: m_WriteTo defaults to null in this case, otherwise an update to
        // the bounds percentile could switch where the debug is written to
    }

    public ModelDebugConfig(Double boundsPercentile, String terms)
    {
        m_WriteTo = DebugDestination.FILE;
        m_BoundsPercentile = boundsPercentile;
        m_Terms = terms;
    }

    public ModelDebugConfig(DebugDestination writeTo, Double boundsPercentile, String terms)
    {
        m_WriteTo = writeTo;
        m_BoundsPercentile = boundsPercentile;
        m_Terms = terms;
    }

    public DebugDestination getWriteTo()
    {
        return m_WriteTo;
    }

    public void setWriteTo(DebugDestination writeTo)
    {
        m_WriteTo = writeTo;
    }

    public boolean isEnabled()
    {
        return m_BoundsPercentile != null;
    }

    public Double getBoundsPercentile()
    {
        return m_BoundsPercentile;
    }

    public void setBoundsPercentile(Double boundsPercentile)
    {
        m_BoundsPercentile = boundsPercentile;
    }

    public String getTerms()
    {
        return m_Terms;
    }

    public void setTerms(String terms)
    {
        m_Terms = terms;
    }

    @Override
    public boolean equals(Object other)
    {
        if (this == other)
        {
            return true;
        }

        if (other instanceof ModelDebugConfig == false)
        {
            return false;
        }

        ModelDebugConfig that = (ModelDebugConfig) other;
        return Objects.equals(this.m_WriteTo, that.m_WriteTo)
                && Objects.equals(this.m_BoundsPercentile, that.m_BoundsPercentile)
                && Objects.equals(this.m_Terms, that.m_Terms);
    }

    @Override
    public int hashCode()
    {
        return Objects.hash(m_WriteTo, m_BoundsPercentile, m_Terms);
    }
} @JsonIgnoreProperties({"modelSizeStatsId"})
public class ModelSizeStats
{
    /**
     * Field Names
     */
    public static final String MODEL_SIZE_STATS_ID = "modelSizeStatsId";
    public static final String MODEL_BYTES = "modelBytes";
    public static final String TOTAL_BY_FIELD_COUNT = "totalByFieldCount";
    public static final String TOTAL_OVER_FIELD_COUNT = "totalOverFieldCount";
    public static final String TOTAL_PARTITION_FIELD_COUNT = "totalPartitionFieldCount";
    public static final String BUCKET_ALLOCATION_FAILURES_COUNT = "bucketAllocationFailuresCount";
    public static final String MEMORY_STATUS = "memoryStatus";
    public static final String LOG_TIME = "logTime";
    public static final String BUCKET_TIME = "bucketTime";
    public static final String TIMESTAMP = "timestamp";

    /**
     * Elasticsearch type
     */
    public static final String TYPE = "modelSizeStats";

    /**
     * The status of the memory monitored by the ResourceMonitor.
     * OK is default, SOFT_LIMIT means that the models have done
     * some aggressive pruning to keep the memory below the limit,
     * and HARD_LIMIT means that samples have been dropped
     */
    public enum MemoryStatus { OK, SOFT_LIMIT, HARD_LIMIT }

    private long m_ModelBytes;
    private long m_TotalByFieldCount;
    private long m_TotalOverFieldCount;
    private long m_TotalPartitionFieldCount;
    private long m_BucketAllocationFailuresCount;
    private MemoryStatus m_MemoryStatus;
    private Date m_Timestamp;
    private Date m_LogTime;
    private String m_Id = TYPE;

    public ModelSizeStats()
    {
        m_ModelBytes = 0;
        m_TotalByFieldCount = 0;
        m_TotalOverFieldCount = 0;
        m_TotalPartitionFieldCount = 0;
        m_BucketAllocationFailuresCount = 0;
        m_MemoryStatus = MemoryStatus.OK;
        m_LogTime = new Date();
    }

    public String getModelSizeStatsId()
    {
        return m_Id;
    }

    public void setModelSizeStatsId(String id)
    {
        m_Id = id;
    }

    public void setModelBytes(long m)
    {
        m_ModelBytes = m;
    }

    public long getModelBytes()
    {
        return m_ModelBytes;
    }

    public void setTotalByFieldCount(long m)
    {
        m_TotalByFieldCount = m;
    }

    public long getTotalByFieldCount()
    {
        return m_TotalByFieldCount;
    }

    public void setTotalPartitionFieldCount(long m)
    {
        m_TotalPartitionFieldCount = m;
    }

    public long getTotalPartitionFieldCount()
    {
        return m_TotalPartitionFieldCount;
    }

    public void setTotalOverFieldCount(long m)
    {
        m_TotalOverFieldCount = m;
    }

    public long getTotalOverFieldCount()
    {
        return m_TotalOverFieldCount;
    }

    public void setBucketAllocationFailuresCount(long m)
    {
        m_BucketAllocationFailuresCount = m;
    }

    public long getBucketAllocationFailuresCount()
    {
        return m_BucketAllocationFailuresCount;
    }

    public void setMemoryStatus(String m)
    {
        if (m == null || m.isEmpty())
        {
            m_MemoryStatus = MemoryStatus.OK;
        }
        else
        {
            m_MemoryStatus = MemoryStatus.valueOf(m);
        }
    }

    public String getMemoryStatus()
    {
        return m_MemoryStatus.name();
    }

    public Date getTimestamp()
    {
        return m_Timestamp;
    }

    public void setTimestamp(Date d)
    {
        m_Timestamp = d;
    }

    public Date getLogTime()
    {
        return m_LogTime;
    }

    public void setLogTime(Date d)
    {
        m_LogTime = d;
    }

    @Override
    public int hashCode()
    {
        // m_Id excluded here as it is generated by the datastore
        return Objects.hash(m_ModelBytes, m_TotalByFieldCount, m_TotalOverFieldCount, m_TotalPartitionFieldCount,
                m_BucketAllocationFailuresCount, m_MemoryStatus, m_Timestamp, m_LogTime);
    }

    /**
     * Compare all the fields.
     */
    @Override
    public boolean equals(Object other)
    {
        if (this == other)
        {
            return true;
        }

        if (other instanceof ModelSizeStats == false)
        {
            return false;
        }

        ModelSizeStats that = (ModelSizeStats) other;

        // m_Id excluded here as it is generated by the datastore
        return this.m_ModelBytes == that.m_ModelBytes
                && this.m_TotalByFieldCount == that.m_TotalByFieldCount
                && this.m_TotalOverFieldCount == that.m_TotalOverFieldCount
                && this.m_TotalPartitionFieldCount == that.m_TotalPartitionFieldCount
                && this.m_BucketAllocationFailuresCount == that.m_BucketAllocationFailuresCount
                && Objects.equals(this.m_MemoryStatus, that.m_MemoryStatus)
                && Objects.equals(this.m_Timestamp, that.m_Timestamp)
                && Objects.equals(this.m_LogTime, that.m_LogTime);
    }
} @JsonInclude(Include.NON_NULL)
public class ModelSnapshot
{
    /**
     * Field Names
     */
    public static final String TIMESTAMP = "timestamp";
    public static final String DESCRIPTION = "description";
    public static final String RESTORE_PRIORITY = "restorePriority";
    public static final String SNAPSHOT_ID = "snapshotId";
    public static final String SNAPSHOT_DOC_COUNT = "snapshotDocCount";
    public static final String LATEST_RECORD_TIME = "latestRecordTimeStamp";
    public static final String LATEST_RESULT_TIME = "latestResultTimeStamp";

    /**
     * Elasticsearch type
     */
    public static final String TYPE = "modelSnapshot";

    private Date m_Timestamp;
    private String m_Description;
    private long m_RestorePriority;
    private String m_SnapshotId;
    private int m_SnapshotDocCount;
    private ModelSizeStats m_ModelSizeStats;
    private Date m_LatestRecordTimeStamp;
    private Date m_LatestResultTimeStamp;
    private Quantiles m_Quantiles;

    public Date getTimestamp()
    {
        return m_Timestamp;
    }

    public void setTimestamp(Date timestamp)
    {
        m_Timestamp = timestamp;
    }

    public String getDescription()
    {
        return m_Description;
    }

    public void setDescription(String description)
    {
        m_Description = description;
    }

    public long getRestorePriority()
    {
        return m_RestorePriority;
    }

    public void setRestorePriority(long restorePriority)
    {
        m_RestorePriority = restorePriority;
    }

    public String getSnapshotId()
    {
        return m_SnapshotId;
    }

    public void setSnapshotId(String snapshotId)
    {
        m_SnapshotId = snapshotId;
    }

    public int getSnapshotDocCount()
    {
        return m_SnapshotDocCount;
    }

    public void setSnapshotDocCount(int snapshotDocCount)
    {
        m_SnapshotDocCount = snapshotDocCount;
    }

    public ModelSizeStats getModelSizeStats()
    {
        return m_ModelSizeStats;
    }

    public void setModelSizeStats(ModelSizeStats modelSizeStats)
    {
        m_ModelSizeStats = modelSizeStats;
    }

    public Quantiles getQuantiles()
    {
        return m_Quantiles;
    }

    public void setQuantiles(Quantiles q)
    {
        m_Quantiles = q;
    }

    public Date getLatestRecordTimeStamp()
    {
        return m_LatestRecordTimeStamp;
    }

    public void setLatestRecordTimeStamp(Date latestRecordTimeStamp)
    {
        m_LatestRecordTimeStamp = latestRecordTimeStamp;
    }

    public Date getLatestResultTimeStamp()
    {
        return m_LatestResultTimeStamp;
    }

    public void setLatestResultTimeStamp(Date latestResultTimeStamp)
    {
        m_LatestResultTimeStamp = latestResultTimeStamp;
    }

    @Override
    public int hashCode()
    {
        return Objects.hash(m_Timestamp, m_Description, m_RestorePriority, m_SnapshotId, m_Quantiles,
                m_SnapshotDocCount, m_ModelSizeStats, m_LatestRecordTimeStamp, m_LatestResultTimeStamp);
    }

    /**
     * Compare all the fields.
     */
    @Override
    public boolean equals(Object other)
    {
        if (this == other)
        {
            return true;
        }

        if (other instanceof ModelSnapshot == false)
        {
            return false;
        }

        ModelSnapshot that = (ModelSnapshot) other;

        return Objects.equals(this.m_Timestamp, that.m_Timestamp)
                && Objects.equals(this.m_Description, that.m_Description)
                && this.m_RestorePriority == that.m_RestorePriority
                && Objects.equals(this.m_SnapshotId, that.m_SnapshotId)
                && this.m_SnapshotDocCount == that.m_SnapshotDocCount
                && Objects.equals(this.m_ModelSizeStats, that.m_ModelSizeStats)
                && Objects.equals(this.m_Quantiles,  that.m_Quantiles)
                && Objects.equals(this.m_LatestRecordTimeStamp, that.m_LatestRecordTimeStamp)
                && Objects.equals(this.m_LatestResultTimeStamp, that.m_LatestResultTimeStamp);
    }
} @JsonInclude(Include.NON_NULL)
public class SchedulerConfig implements PasswordStorage
{
    /**
     * Enum of the acceptable data sources.
     */
    public enum DataSource
    {
        FILE, ELASTICSEARCH;

        /**
         * Case-insensitive from string method.
         * Works with ELASTICSEARCH, Elasticsearch, ElasticSearch, etc.
         *
         * @param value String representation
         * @return The data source
         */
        @JsonCreator
        public static DataSource forString(String value)
        {
            String valueUpperCase = value.toUpperCase();
            return DataSource.valueOf(valueUpperCase);
        }
    }

    /**
     * The field name used to specify aggregation fields in Elasticsearch aggregations
     */
    private static final String FIELD = "field";

    /**
     * The field name used to specify document counts in Elasticsearch aggregations
     */
    public static final String DOC_COUNT = "doc_count";

    /**
     * The default query for elasticsearch searches
     */
    private static final String MATCH_ALL_ES_QUERY = "match_all";

    /**
     * Serialisation names
     */
    public static final String DATA_SOURCE = "dataSource";
    public static final String DATA_SOURCE_COMPATIBILITY = "dataSourceCompatibility";
    public static final String QUERY_DELAY = "queryDelay";
    public static final String FREQUENCY = "frequency";
    public static final String FILE_PATH = "filePath";
    public static final String TAIL_FILE = "tailFile";
    public static final String BASE_URL = "baseUrl";
    public static final String USERNAME = "username";
    public static final String PASSWORD = "password";
    public static final String ENCRYPTED_PASSWORD = "encryptedPassword";
    public static final String INDEXES = "indexes";
    public static final String TYPES = "types";
    public static final String QUERY = "query";
    public static final String RETRIEVE_WHOLE_SOURCE = "retrieveWholeSource";
    public static final String SCROLL_SIZE = "scrollSize";
    public static final String AGGREGATIONS = "aggregations";
    public static final String AGGS = "aggs";

    /**
     * Named to match Elasticsearch, hence lowercase_with_underscores instead
     * of camelCase
     */
    public static final String SCRIPT_FIELDS = "script_fields";

    private static final int DEFAULT_SCROLL_SIZE = 1000;
    private static final long DEFAULT_ELASTICSEARCH_QUERY_DELAY = 60L;

    private DataSource m_DataSource;
    private String m_DataSourceCompatibility;

    /**
     * The delay in seconds before starting to query a period of time
     */
    private Long m_QueryDelay;

    /**
     * The frequency in seconds with which queries are executed
     */
    private Long m_Frequency;

    /**
     * These values apply to the FILE data source
     */
    private String m_FilePath;
    private Boolean m_TailFile;

    /**
     * Used for data sources that require credentials.  May be null in the case
     * where credentials are sometimes needed and sometimes not (e.g. Elasticsearch).
     */
    private String m_Username;
    private String m_Password;
    private String m_EncryptedPassword;

    /**
     * These values apply to the ELASTICSEARCH data source
     */
    private String m_BaseUrl;
    private List<String> m_Indexes;
    private List<String> m_Types;
    private Map<String, Object> m_Query;
    private Map<String, Object> m_Aggregations;
    private Map<String, Object> m_Aggs;
    private Map<String, Object> m_ScriptFields;
    private Boolean m_RetrieveWholeSource;
    private Integer m_ScrollSize;

    /**
     * Default constructor
     */
    public SchedulerConfig()
    {
        m_DataSource = DataSource.FILE;
    }

    /**
     * The data source that the scheduler is to pull data from.
     * @return The data source.
     */
    public DataSource getDataSource()
    {
        return m_DataSource;
    }

    public void setDataSource(DataSource dataSource)
    {
        m_DataSource = dataSource;
    }

    /**
     * Gets the compatibility for the data source.
     * @return The data source compatibility.
     */
    public String getDataSourceCompatibility()
    {
        return m_DataSourceCompatibility;
    }

    public void setDataSourceCompatibility(String dataSourceCompatibility)
    {
        m_DataSourceCompatibility = dataSourceCompatibility;
    }

    public Long getQueryDelay()
    {
        return m_QueryDelay;
    }

    public void setQueryDelay(Long delay)
    {
        m_QueryDelay = delay;
    }

    public Long getFrequency()
    {
        return m_Frequency;
    }

    public void setFrequency(Long frequency)
    {
        m_Frequency = frequency;
    }

    /**
     * For the FILE data source only, the path to the file.
     * @return The path to the file, or <code>null</code> if not set.
     */
    public String getFilePath()
    {
        return m_FilePath;
    }

    public void setFilePath(String filePath)
    {
        m_FilePath = filePath;
    }

    /**
     * For the FILE data source only, should the file be tailed?  If not it will
     * just be read from once.
     * @return Should the file be tailed?  (<code>null</code> if not set.)
     */
    public Boolean getTailFile()
    {
        return m_TailFile;
    }

    public void setTailFile(Boolean tailFile)
    {
        m_TailFile = tailFile;
    }

    /**
     * For the ELASTICSEARCH data source only, the base URL to connect to
     * Elasticsearch on.
     * @return The URL, or <code>null</code> if not set.
     */
    public String getBaseUrl()
    {
        return m_BaseUrl;
    }

    public void setBaseUrl(String baseUrl)
    {
        m_BaseUrl = baseUrl;
    }

    /**
     * The username to use to connect to the data source (if any).
     * @return The username, or <code>null</code> if not set.
     */
    public String getUsername()
    {
        return m_Username;
    }

    public void setUsername(String username)
    {
        m_Username = username;
    }

    /**
     * The plain text password to use to connect to the data source (if any).
     * This is likely to return <code>null</code> most of the time, as the
     * intention is that it is only present it initial configurations, and gets
     * replaced with an encrypted password as soon as possible after receipt.
     * @return The password, or <code>null</code> if not set.
     */
    @Override
    public String getPassword()
    {
        return m_Password;
    }

    @Override
    public void setPassword(String password)
    {
        m_Password = password;
    }

    /**
     * The encrypted password to use to connect to the data source (if any).
     * A class outside this package is responsible for encrypting and decrypting
     * the password.
     * @return The password, or <code>null</code> if not set.
     */
    @Override
    public String getEncryptedPassword()
    {
        return m_EncryptedPassword;
    }

    @Override
    public void setEncryptedPassword(String encryptedPassword)
    {
        m_EncryptedPassword = encryptedPassword;
    }

    /**
     * For the ELASTICSEARCH data source only, one or more indexes to search for
     * input data.
     * @return The indexes to search, or <code>null</code> if not set.
     */
    public List<String> getIndexes()
    {
        return m_Indexes;
    }

    public void setIndexes(List<String> indexes)
    {
        m_Indexes = indexes;
    }

    /**
     * For the ELASTICSEARCH data source only, one or more types to search for
     * input data.
     * @return The types to search, or <code>null</code> if not set.
     */
    public List<String> getTypes()
    {
        return m_Types;
    }

    public void setTypes(List<String> types)
    {
        m_Types = types;
    }

    /**
     * For the ELASTICSEARCH data source only, the Elasticsearch query DSL
     * representing the query to submit to Elasticsearch to get the input data.
     * This should not include time bounds, as these are added separately.
     * This class does not attempt to interpret the query.  The map will be
     * converted back to an arbitrary JSON object.
     * @return The search query, or <code>null</code> if not set.
     */
    public Map<String, Object> getQuery()
    {
        return m_Query;
    }

    public void setQuery(Map<String, Object> query)
    {
        m_Query = query;
    }

    /**
     * For the ELASTICSEARCH data source only, should the whole _source document
     * be retrieved for analysis, or just the analysis fields?
     * @return Should the whole of _source be retrieved?  (<code>null</code> if not set.)
     */
    public Boolean getRetrieveWholeSource()
    {
        return m_RetrieveWholeSource;
    }

    public void setRetrieveWholeSource(Boolean retrieveWholeSource)
    {
        m_RetrieveWholeSource = retrieveWholeSource;
    }

    /**
     * For the ELASTICSEARCH data source only, get the size of documents to
     * be retrieved from each shard via a scroll search
     * @return The size of documents to be retrieved from each shard via a scroll search
     */
    public Integer getScrollSize()
    {
        return m_ScrollSize;
    }

    public void setScrollSize(Integer scrollSize)
    {
        m_ScrollSize = scrollSize;
    }

    /**
     * For the ELASTICSEARCH data source only, optional Elasticsearch
     * script_fields to add to the search to be submitted to Elasticsearch to
     * get the input data.  This class does not attempt to interpret the
     * script fields.  The map will be converted back to an arbitrary JSON object.
     * @return The script fields, or <code>null</code> if not set.
     */
    @JsonProperty("script_fields")
    public Map<String, Object> getScriptFields()
    {
        return m_ScriptFields;
    }

    @JsonProperty("script_fields")
    public void setScriptFields(Map<String, Object> scriptFields)
    {
        m_ScriptFields = scriptFields;
    }

    /**
     * For the ELASTICSEARCH data source only, optional Elasticsearch
     * aggregations to apply to the search to be submitted to Elasticsearch to
     * get the input data.  This class does not attempt to interpret the
     * aggregations.  The map will be converted back to an arbitrary JSON object.
     * Synonym for {@link getAggs()} (like Elasticsearch).
     * @return The aggregations, or <code>null</code> if not set.
     */
    public Map<String, Object> getAggregations()
    {
        return m_Aggregations;
    }

    public void setAggregations(Map<String, Object> aggregations)
    {
        // It's only expected that one of aggregations or aggs will be set,
        // having two member variables makes it easier to remember which the
        // user used so their input can be recreated
        m_Aggregations = aggregations;
    }

    /**
     * For the ELASTICSEARCH data source only, optional Elasticsearch
     * aggregations to apply to the search to be submitted to Elasticsearch to
     * get the input data.  This class does not attempt to interpret the
     * aggregations.  The map will be converted back to an arbitrary JSON object.
     * Synonym for {@link getAggregations()} (like Elasticsearch).
     * @return The aggregations, or <code>null</code> if not set.
     */
    public Map<String, Object> getAggs()
    {
        return m_Aggs;
    }

    public void setAggs(Map<String, Object> aggs)
    {
        // It's only expected that one of aggregations or aggs will be set,
        // having two member variables makes it easier to remember which the
        // user used so their input can be recreated
        m_Aggs = aggs;
    }

    /**
     * Convenience method to get either aggregations or aggs.
     * @return The aggregations (whether initially specified in aggregations
     * or aggs), or <code>null</code> if neither are set.
     */
    @JsonIgnore
    public Map<String, Object> getAggregationsOrAggs()
    {
        return (m_Aggregations != null) ? m_Aggregations : m_Aggs;
    }

    /**
     * Build the list of fields expected in the output from aggregations
     * submitted to Elasticsearch.
     * @return The list of fields, or empty list if there are no aggregations.
     */
    public List<String> buildAggregatedFieldList()
    {
        Map<String, Object> aggs = getAggregationsOrAggs();
        if (aggs == null)
        {
            return Collections.emptyList();
        }

        SortedMap<Integer, String> orderedFields = new TreeMap<>();

        scanSubLevel(aggs, 0, orderedFields);

        return new ArrayList<>(orderedFields.values());
    }

    @SuppressWarnings("unchecked")
    private void scanSubLevel(Map<String, Object> subLevel, int depth,
            SortedMap<Integer, String> orderedFields)
    {
        for (Map.Entry<String, Object> entry : subLevel.entrySet())
        {
            Object value = entry.getValue();
            if (value instanceof Map<?, ?>)
            {
                scanSubLevel((Map<String, Object>)value, depth + 1, orderedFields);
            }
            else if (value instanceof String && FIELD.equals(entry.getKey()))
            {
                orderedFields.put(depth, (String)value);
            }
        }
    }

    public void fillDefaults()
    {
        switch (m_DataSource)
        {
            case ELASTICSEARCH:
                fillElasticsearchDefaults();
                break;
            case FILE:
                fillFileDefaults();
                break;
            default:
                throw new IllegalStateException();
        }
    }

    private void fillElasticsearchDefaults()
    {
        if (m_Query == null)
        {
            m_Query = new HashMap<>();
            m_Query.put(MATCH_ALL_ES_QUERY, new HashMap<String, Object>());
        }
        if (m_QueryDelay == null)
        {
            m_QueryDelay = DEFAULT_ELASTICSEARCH_QUERY_DELAY;
        }
        if (m_RetrieveWholeSource == null)
        {
            m_RetrieveWholeSource = false;
        }
        if (m_ScrollSize == null)
        {
            m_ScrollSize = DEFAULT_SCROLL_SIZE;
        }
    }

    private void fillFileDefaults()
    {
        if (m_TailFile == null)
        {
            m_TailFile = false;
        }
    }

    /**
     * The lists of indexes and types are compared for equality but they are not
     * sorted first so this test could fail simply because the indexes and types
     * lists are in different orders.
     */
    @Override
    public boolean equals(Object other)
    {
        if (this == other)
        {
            return true;
        }

        if (other instanceof SchedulerConfig == false)
        {
            return false;
        }

        SchedulerConfig that = (SchedulerConfig)other;

        return Objects.equals(this.m_DataSource, that.m_DataSource) &&
                Objects.equals(this.m_DataSourceCompatibility, that.m_DataSourceCompatibility) &&
                Objects.equals(this.m_Frequency, that.m_Frequency) &&
                Objects.equals(this.m_QueryDelay, that.m_QueryDelay) &&
                Objects.equals(this.m_FilePath, that.m_FilePath) &&
                Objects.equals(this.m_TailFile, that.m_TailFile) &&
                Objects.equals(this.m_BaseUrl, that.m_BaseUrl) &&
                Objects.equals(this.m_Username, that.m_Username) &&
                Objects.equals(this.m_Password, that.m_Password) &&
                Objects.equals(this.m_EncryptedPassword, that.m_EncryptedPassword) &&
                Objects.equals(this.m_Indexes, that.m_Indexes) &&
                Objects.equals(this.m_Types, that.m_Types) &&
                Objects.equals(this.m_Query, that.m_Query) &&
                Objects.equals(this.m_RetrieveWholeSource, that.m_RetrieveWholeSource) &&
                Objects.equals(this.m_ScrollSize, that.m_ScrollSize) &&
                Objects.equals(this.getAggregationsOrAggs(), that.getAggregationsOrAggs()) &&
                Objects.equals(this.m_ScriptFields, that.m_ScriptFields);
    }

    @Override
    public int hashCode()
    {
        return Objects.hash(m_DataSource, m_DataSourceCompatibility, m_Frequency, m_QueryDelay,
                m_FilePath, m_TailFile, m_BaseUrl, m_Username, m_Password, m_EncryptedPassword,
                m_Indexes, m_Types, m_Query, m_RetrieveWholeSource, m_ScrollSize,
                getAggregationsOrAggs(), m_ScriptFields);
    }
} @JsonInclude(Include.NON_NULL)
public class Quantiles
{
    public static final String QUANTILES_ID = "hierarchical";

    /**
     * Field Names
     */
    public static final String TIMESTAMP = "timestamp";
    public static final String QUANTILE_STATE = "quantileState";

    /**
     * Elasticsearch type
     */
    public static final String TYPE = "quantiles";

    private Date m_Timestamp;
    private String m_QuantileState;

    public Date getTimestamp()
    {
        return m_Timestamp;
    }

    public void setTimestamp(Date timestamp)
    {
        m_Timestamp = timestamp;
    }

    public String getQuantileState()
    {
        return Strings.nullToEmpty(m_QuantileState);
    }

    public void setQuantileState(String quantileState)
    {
        m_QuantileState = quantileState;
    }

    @Override
    public int hashCode()
    {
        return Objects.hashCode(m_QuantileState);
    }

    /**
     * Compare all the fields.
     */
    @Override
    public boolean equals(Object other)
    {
        if (this == other)
        {
            return true;
        }

        if (other instanceof Quantiles == false)
        {
            return false;
        }

        Quantiles that = (Quantiles) other;

        return Objects.equals(this.m_QuantileState, that.m_QuantileState);
    }
} @JsonInclude(Include.NON_NULL)
public class TransformConfig
{
    // Serialisation strings
    public static final String TYPE = "transform";
    public static final String TRANSFORM = "transform";
    public static final String ARGUMENTS = "arguments";
    public static final String INPUTS = "inputs";
    public static final String OUTPUTS = "outputs";


    private List<String> m_Inputs;
    private String m_Name;
    private List<String> m_Arguments;
    private List<String> m_Outputs;
    private TransformType m_Type;
    private Condition m_Condition;


    public TransformConfig()
    {
        m_Arguments = Collections.emptyList();
    }

    public List<String> getInputs()
    {
        return m_Inputs;
    }

    public void setInputs(List<String> fields)
    {
        m_Inputs = fields;
    }

    /**
     * Transform name see {@linkplain TransformType.Names}
     * @return
     */
    public String getTransform()
    {
        return m_Name;
    }

    public void setTransform(String type)
    {
        m_Name = type;
    }

    public List<String> getArguments()
    {
        return m_Arguments;
    }

    public void setArguments(List<String> args)
    {
        m_Arguments = args;
    }

    public List<String> getOutputs()
    {
        if (m_Outputs == null || m_Outputs.isEmpty())
        {
            try
            {
                m_Outputs = type().defaultOutputNames();
            }
            catch (IllegalArgumentException e)
            {
                m_Outputs = Collections.emptyList();
            }
        }

        return m_Outputs;
    }

    public void setOutputs(List<String> outputs)
    {
        m_Outputs = outputs;
    }

    /**
     * The condition object which may or may not be defined for this
     * transform
     * @return May be <code>null</code>
     */
    public Condition getCondition()
    {
        return m_Condition;
    }

    public void setCondition(Condition condition)
    {
        m_Condition = condition;
    }

    /**
     * This field shouldn't be serialised as its created dynamically
     * Type may be null when the class is constructed.
     * @return
     */
    public TransformType type() throws IllegalArgumentException
    {
        if (m_Type == null)
        {
            m_Type = TransformType.fromString(m_Name);
        }

        return m_Type;
    }

    @Override
    public String toString()
    {
        return m_Name;
    }

    @Override
    public int hashCode()
    {
        return Objects.hash(m_Inputs, m_Name, m_Outputs, m_Type, m_Arguments, m_Condition);
    }

    @Override
    public boolean equals(Object obj)
    {
        if (this == obj)
        {
            return true;
        }
        if (obj == null)
        {
            return false;
        }

        if (getClass() != obj.getClass())
        {
            return false;
        }

        TransformConfig other = (TransformConfig) obj;

        return Objects.equals(this.m_Type, other.m_Type)
                && Objects.equals(this.m_Name, other.m_Name)
                && Objects.equals(this.m_Inputs, other.m_Inputs)
                && Objects.equals(this.m_Outputs, other.m_Outputs)
                && Objects.equals(this.m_Arguments, other.m_Arguments)
                && Objects.equals(this.m_Condition, other.m_Condition);
    }
} public class Usage
{
    public static final String TYPE = "usage";
    public static final String TIMESTAMP = "timestamp";
    public static final String INPUT_BYTES = "inputBytes";
    public static final String INPUT_FIELD_COUNT = "inputFieldCount";
    public static final String INPUT_RECORD_COUNT = "inputRecordCount";

    private Usage()
    {
    }
}
import com.prelert.job.SchedulerConfig;
import com.prelert.job.quantiles.Quantiles;
import com.prelert.job.transform.TransformConfig;
import com.prelert.job.usage.Usage;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import com.prelert.job.AnalysisConfig;
import com.prelert.job.AnalysisLimits;
import com.prelert.job.DataCounts;
import com.prelert.job.DataDescription;
import com.prelert.job.Detector;
import com.prelert.job.JobDetails;
import com.prelert.job.ModelDebugConfig;
import com.prelert.job.ModelSizeStats;
import com.prelert.job.ModelSnapshot;
        AnomalyRecord.PARTITION_FIELD_VALUE,
        AnomalyRecord.FUNCTION,
        AnomalyRecord.FUNCTION_DESCRIPTION,
        AnomalyRecord.TYPICAL,
        AnomalyRecord.ACTUAL,
        AnomalyRecord.IS_INTERIM,
        AnomalyRecord.INFLUENCERS,
        AnomalyRecord.FIELD_NAME,
        AnomalyRecord.OVER_FIELD_NAME,
        AnomalyRecord.OVER_FIELD_VALUE,
        AnomalyRecord.CAUSES,
        AnomalyRecord.ANOMALY_SCORE,
        AnomalyRecord.NORMALIZED_PROBABILITY,
        AnomalyRecord.INITIAL_NORMALIZED_PROBABILITY,
        AnomalyRecord.BUCKET_SPAN,

        Bucket.ANOMALY_SCORE,
        Bucket.BUCKET_SPAN,
        Bucket.MAX_NORMALIZED_PROBABILITY,
        Bucket.IS_INTERIM,
        Bucket.RECORD_COUNT,
        Bucket.EVENT_COUNT,
        Bucket.RECORDS,
        Bucket.BUCKET_INFLUENCERS,
        Bucket.INFLUENCERS,
        Bucket.INITIAL_ANOMALY_SCORE,

        BucketInfluencer.BUCKET_TIME,
        BucketInfluencer.INFLUENCER_FIELD_NAME,
        BucketInfluencer.INITIAL_ANOMALY_SCORE,
        BucketInfluencer.ANOMALY_SCORE,
        BucketInfluencer.RAW_ANOMALY_SCORE,
        BucketInfluencer.PROBABILITY,

        CategoryDefinition.CATEGORY_ID,
        CategoryDefinition.TERMS,
        CategoryDefinition.REGEX,
        CategoryDefinition.EXAMPLES,

        DataCounts.BUCKET_COUNT,
        DataCounts.PROCESSED_RECORD_COUNT,
        DataCounts.PROCESSED_FIELD_COUNT,
        DataCounts.INPUT_BYTES,
        DataCounts.INPUT_RECORD_COUNT,
        DataCounts.INPUT_FIELD_COUNT,
        DataCounts.INVALID_DATE_COUNT,
        DataCounts.MISSING_FIELD_COUNT,
        DataCounts.OUT_OF_ORDER_TIME_COUNT,
        DataCounts.FAILED_TRANSFORM_COUNT,
        DataCounts.EXCLUDED_RECORD_COUNT,
        DataCounts.LATEST_RECORD_TIME,

        DataDescription.FORMAT,
        DataDescription.TIME_FIELD_NAME,
        DataDescription.TIME_FORMAT,
        DataDescription.FIELD_DELIMITER,
        DataDescription.QUOTE_CHARACTER,

        Detector.DETECTOR_DESCRIPTION,
        Detector.FUNCTION,
        Detector.FIELD_NAME,
        Detector.BY_FIELD_NAME,
        Detector.OVER_FIELD_NAME,
        Detector.PARTITION_FIELD_NAME,
        Detector.USE_NULL,

        Influence.INFLUENCER_FIELD_NAME,
        Influence.INFLUENCER_FIELD_VALUES,

        Influencer.PROBABILITY,
        Influencer.INFLUENCER_FIELD_NAME,
        Influencer.INFLUENCER_FIELD_VALUE,
        Influencer.INITIAL_ANOMALY_SCORE,
        Influencer.ANOMALY_SCORE,

        // JobDetails.DESCRIPTION is not reserved because it is an analyzed string
        // JobDetails.STATUS is not reserved because it is an analyzed string
        JobDetails.DATA_DESCRIPTION,
        JobDetails.SCHEDULER_STATUS,
        JobDetails.SCHEDULER_CONFIG,
        JobDetails.FINISHED_TIME,
        JobDetails.LAST_DATA_TIME,
        JobDetails.COUNTS,
        JobDetails.TIMEOUT,
        JobDetails.RENORMALIZATION_WINDOW_DAYS,
        JobDetails.BACKGROUND_PERSIST_INTERVAL,
        JobDetails.MODEL_SNAPSHOT_RETENTION_DAYS,
        JobDetails.RESULTS_RETENTION_DAYS,
        JobDetails.ANALYSIS_CONFIG,
        JobDetails.ANALYSIS_LIMITS,
        JobDetails.TRANSFORMS,
        JobDetails.MODEL_DEBUG_CONFIG,
        JobDetails.IGNORE_DOWNTIME,
        JobDetails.CUSTOM_SETTINGS,

        ModelDebugConfig.WRITE_TO,
        ModelDebugConfig.BOUNDS_PERCENTILE,
        ModelDebugConfig.TERMS,

        ModelDebugOutput.PARTITION_FIELD_NAME,
        ModelDebugOutput.PARTITION_FIELD_VALUE,
        ModelDebugOutput.OVER_FIELD_NAME,
        ModelDebugOutput.OVER_FIELD_VALUE,
        ModelDebugOutput.BY_FIELD_NAME,
        ModelDebugOutput.BY_FIELD_VALUE,
        ModelDebugOutput.DEBUG_FEATURE,
        ModelDebugOutput.DEBUG_LOWER,
        ModelDebugOutput.DEBUG_UPPER,
        ModelDebugOutput.DEBUG_MEAN,
        ModelDebugOutput.ACTUAL,

        ModelSizeStats.MODEL_BYTES,
        ModelSizeStats.TOTAL_BY_FIELD_COUNT,
        ModelSizeStats.TOTAL_OVER_FIELD_COUNT,
        ModelSizeStats.TOTAL_PARTITION_FIELD_COUNT,
        ModelSizeStats.BUCKET_ALLOCATION_FAILURES_COUNT,
        ModelSizeStats.MEMORY_STATUS,
        ModelSizeStats.LOG_TIME,

        // ModelSnapshot.DESCRIPTION is not reserved because it is an analyzed string
        ModelSnapshot.RESTORE_PRIORITY,