Continue updating after a dashboard errors
@@ -96,18 +96,22 @@ func main() {
ctx := context.Background()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- updateOnce(ctx, opt)
+ if err := updateOnce(ctx, opt); err != nil {
+ logrus.WithError(err).Error("Failed update")
+ }
if opt.wait == 0 {
return
}
timer := time.NewTimer(opt.wait)
defer timer.Stop()
for range timer.C {
- updateOnce(ctx, opt)
+ if err := updateOnce(ctx, opt); err != nil {
+ logrus.WithError(err).Error("Failed update")
+ }
}
}
-func updateOnce(ctx context.Context, opt options) {
+func updateOnce(ctx context.Context, opt options) error {
client, err := gcs.ClientWithCreds(ctx, opt.creds)
if err != nil {
logrus.Fatalf("Failed to read storage client: %v", err)
@@ -137,6 +141,8 @@ func updateOnce(ctx context.Context, opt options) {
return group, reader, nil
}
+ errCh := make(chan error)
+
for i := 0; i < opt.concurrency; i++ {
wg.Add(1)
go func() {
@@ -145,7 +151,8 @@ func updateOnce(ctx context.Context, opt options) {
log.Info("Summarizing dashboard")
sum, err := updateDashboard(ctx, dash, groupFinder)
if err != nil {
- log.WithError(err).Fatal("Cannot summarize dashboard")
+ log.WithError(err).Error("Cannot summarize dashboard")
+ errCh <- errors.New(dash.Name)
continue
}
log.WithField("summary", sum).Info("summarized")
@@ -154,16 +161,37 @@ func updateOnce(ctx context.Context, opt options) {
}
path, err := opt.config.ResolveReference(&url.URL{Path: summaryPath(dash.Name)})
if err != nil {
- log.WithError(err).Fatal("Cannot resolve summary path")
+ log.WithError(err).Error("Cannot resolve summary path")
+ errCh <- errors.New(dash.Name)
+ continue
}
if err := writeSummary(ctx, client, *path, sum); err != nil {
- log.WithError(err).Fatal("Cannot write summary")
+ log.WithError(err).Error("Cannot write summary")
+ errCh <- errors.New(dash.Name)
+ continue
}
+ errCh <- nil
}
wg.Done()
}()
}
+ resultCh := make(chan error)
+ go func() {
+ var errs []string
+ for err := range errCh {
+ if err == nil {
+ continue
+ }
+ errs = append(errs, err.Error())
+ }
+ if n := len(errs); n > 0 {
+ resultCh <- fmt.Errorf("failed to update %d dashboards: %v", n, strings.Join(errs, ", "))
+ }
+ resultCh <- nil
+ close(resultCh)
+ }()
+
for _, d := range cfg.Dashboards {
if opt.dashboard != "" && opt.dashboard != d.Name {
logrus.WithField("dashboard", d.Name).Info("Skipping")
@@ -173,6 +201,8 @@ func updateOnce(ctx context.Context, opt options) {
}
close(dashboards)
wg.Wait()
+ close(errCh)
+ return <-resultCh
}
var (
