Switch to counters for the job scheduling duration
Using gauges has the drawback that the last emitted value will persist even if the job is removed and no new metrics are emitted for this particular job.
@@ -30,7 +30,8 @@ type PrometheusEmitter struct {
schedulingFullDuration *prometheus.GaugeVec
schedulingLoadingDuration *prometheus.GaugeVec
- schedulingJobDuration *prometheus.GaugeVec
+ schedulingJobDurationTotalVec *prometheus.CounterVec
+ schedulingJobTotalVec *prometheus.CounterVec
dbQueriesTotal prometheus.Counter
dbConnections *prometheus.GaugeVec
@@ -183,16 +184,27 @@ func (config *PrometheusConfig) NewEmitter() (metric.Emitter, error) {
)
prometheus.MustRegister(schedulingLoadingDuration)
- schedulingJobDuration := prometheus.NewGaugeVec(
- prometheus.GaugeOpts{
+ schedulingJobDurationTotalVec := prometheus.NewCounterVec(
+ prometheus.CounterOpts{
Namespace: "concourse",
Subsystem: "scheduling",
- Name: "job_duration_seconds",
- Help: "Last time taken to calculate the set of valid input versions for a pipeline.",
+ Name: "job_duration_seconds_total",
+ Help: "Total amount of time spend calculating the set of valid input versions for a job.",
},
[]string{"pipeline", "job"},
)
- prometheus.MustRegister(schedulingJobDuration)
+ schedulingJobTotalVec := prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: "concourse",
+ Subsystem: "scheduling",
+ Name: "job_total",
+ Help: "Number of times the job was scheduled",
+ },
+ []string{"pipeline", "job"},
+ )
+
+ prometheus.MustRegister(schedulingJobDurationTotalVec)
+ prometheus.MustRegister(schedulingJobTotalVec)
dbQueriesTotal := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "concourse",
@@ -237,7 +249,8 @@ func (config *PrometheusConfig) NewEmitter() (metric.Emitter, error) {
schedulingFullDuration: schedulingFullDuration,
schedulingLoadingDuration: schedulingLoadingDuration,
- schedulingJobDuration: schedulingJobDuration,
+ schedulingJobDurationTotalVec: schedulingJobDurationTotalVec,
+ schedulingJobTotalVec: schedulingJobTotalVec,
dbQueriesTotal: dbQueriesTotal,
dbConnections: dbConnections,
@@ -392,8 +405,10 @@ func (emitter *PrometheusEmitter) schedulingMetrics(logger lager.Logger, event m
if !exists {
logger.Error("failed-to-find-job-in-event", fmt.Errorf("expected job to exist in event.Attributes"))
}
- // concourse_scheduling_job_duration_seconds
- emitter.schedulingJobDuration.WithLabelValues(pipeline, job).Set(duration / 1000)
+ // concourse_scheduling_job_duration_seconds_total
+ emitter.schedulingJobDurationTotalVec.WithLabelValues(pipeline, job).Add(duration / 1000)
+ // concourse_scheduling_job_total
+ emitter.schedulingJobTotalVec.WithLabelValues(pipeline, job).Inc()
default:
}
}
