tsdb: commit data periodically in CreateBlock
To avoid building up data in memory, commit and make a new appender
periodically.
The number `commitAfter = 10000` was chosen arbitrarily; testing with
10x more or less gives slightly worse results.
@@ -44,6 +44,8 @@ func CreateBlock(series []storage.Series, dir string, chunkRange int64, logger l
}
}()
+ sampleCount := 0
+ const commitAfter = 10000
ctx := context.Background()
app := w.Appender(ctx)
@@ -57,10 +59,19 @@ func CreateBlock(series []storage.Series, dir string, chunkRange int64, logger l
if err != nil {
return "", err
}
+ sampleCount++
}
if it.Err() != nil {
return "", it.Err()
}
+ // Commit and make a new appender periodically, to avoid building up data in memory.
+ if sampleCount > commitAfter {
+ if err = app.Commit(); err != nil {
+ return "", err
+ }
+ app = w.Appender(ctx)
+ sampleCount = 0
+ }
}
if err = app.Commit(); err != nil {
