backend: dynamically set snapshotWarningTimeout based on db size
@@ -42,7 +42,8 @@ var (
plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "mvcc/backend")
- snapshotWarningTimeout = 30 * time.Second
+ // minSnapshotWarningTimeout is the minimum threshold to trigger a long running snapshot warning.
+ minSnapshotWarningTimeout = time.Duration(30 * time.Second)
)
type Backend interface {
@@ -165,16 +166,33 @@ func (b *backend) ForceCommit() {
}
func (b *backend) Snapshot() Snapshot {
+ b.batchTx.Commit()
+
+ b.mu.RLock()
+ defer b.mu.RUnlock()
+ tx, err := b.db.Begin(false)
+ if err != nil {
+ plog.Fatalf("cannot begin tx (%s)", err)
+ }
+
stopc, donec := make(chan struct{}), make(chan struct{})
+ dbBytes := tx.Size()
go func() {
defer close(donec)
+ // sendRateBytes is based on transferring snapshot data over a 1 gigabit/s connection
+ // assuming a min tcp throughput of 100MB/s.
+ var sendRateBytes int64 = 100 * 1024 * 1014
+ warningTimeout := time.Duration(int64((float64(dbBytes) / float64(sendRateBytes)) * float64(time.Second)))
+ if warningTimeout < minSnapshotWarningTimeout {
+ warningTimeout = minSnapshotWarningTimeout
+ }
start := time.Now()
- ticker := time.NewTicker(snapshotWarningTimeout)
+ ticker := time.NewTicker(warningTimeout)
defer ticker.Stop()
for {
select {
case <-ticker.C:
- plog.Warningf("snapshotting is taking more than %v seconds to finish [started at %v]", time.Since(start).Seconds(), start)
+ plog.Warningf("snapshotting is taking more than %v seconds to finish transferring %v MB [started at %v]", time.Since(start).Seconds(), float64(dbBytes)/float64(1024*1014), start)
case <-stopc:
snapshotDurations.Observe(time.Since(start).Seconds())
return
@@ -182,14 +200,6 @@ func (b *backend) Snapshot() Snapshot {
}
}()
- b.batchTx.Commit()
-
- b.mu.RLock()
- defer b.mu.RUnlock()
- tx, err := b.db.Begin(false)
- if err != nil {
- plog.Fatalf("cannot begin tx (%s)", err)
- }
return &snapshot{tx, stopc, donec}
}
