Recurring job is stuck in running state and never finishes if the volume is detached unexpectedly
Recurring cronjob now detects when a volume is detached unexpectedly,
and cancels the current running job. This make sure that it doesn't
block other running in the future.
Longhorn
@@ -35,6 +35,7 @@ const (
WaitInterval = 5 * time.Second
VolumeAttachTimeout = 300 // 5 minutes
+ BackupProcessStartTimeout = 90 // 1.5 minutes
jobTypeSnapshot = string("snapshot")
jobTypeBackup = string("backup")
@@ -445,6 +446,10 @@ func (job *Job) backupAndCleanup() (err error) {
return err
}
+ if err := job.waitForBackupProcessStart(BackupProcessStartTimeout); err != nil {
+ return err
+ }
+
// Wait for backup creation complete
for {
volume, err := volumeAPI.ById(volumeName)
@@ -459,8 +464,12 @@ func (job *Job) backupAndCleanup() (err error) {
}
}
+ if info == nil {
+ return fmt.Errorf("cannot find the status of the backup for snapshot %v. It might because the engine has restarted", snapshot)
+ }
+
complete := false
- if info != nil {
+
switch info.State {
case engineapi.BackupStateComplete:
complete = true
@@ -472,7 +481,6 @@ func (job *Job) backupAndCleanup() (err error) {
default:
return fmt.Errorf("invalid state %v for backup %v", info.State, info.BackupURL)
}
- }
if complete {
break
@@ -507,6 +515,29 @@ func (job *Job) backupAndCleanup() (err error) {
return nil
}
+// waitForBackupProcessStart timeout in second
+// Return nil if the backup progress has started; error if error or timeout
+func (job *Job) waitForBackupProcessStart(timeout int) error {
+ volumeAPI := job.api.Volume
+ volumeName := job.volumeName
+ snapshot := job.snapshotName
+
+ for i := 0; i < timeout; i++ {
+ volume, err := volumeAPI.ById(volumeName)
+ if err != nil {
+ return err
+ }
+
+ for _, status := range volume.BackupStatus {
+ if status.Snapshot == snapshot {
+ return nil
+ }
+ }
+ time.Sleep(WaitInterval)
+ }
+ return fmt.Errorf("timeout waiting for the backup of the snapshot %v of volume %v to start", snapshot, volumeName)
+}
+
// shouldDoRecurringBackup return whether the recurring backup should take place
// We should do recurring backup if there is new data since the last backup
func (job *Job) shouldDoRecurringBackup(volume *longhornclient.Volume) (bool, error) {
