Retry migrations for Unauthorized errors
The kube apiserver can sometimes return Unauthorized when running on the cloud
if there are temporary auth errors
@@ -38,6 +38,8 @@ const (
// StorkMigrationReplicasAnnotation is the annotation used to keep track of
// the number of replicas for an application when it was migrated
StorkMigrationReplicasAnnotation = "stork.libopenstorage.org/migrationReplicas"
+ // Max number of times to retry applying resources on the desination
+ maxApplyRetries = 10
)
// MigrationController reconciles migration objects
@@ -879,6 +881,8 @@ func (m *MigrationController) applyResources(
if !ok {
return fmt.Errorf("unable to cast object to unstructured: %v", o)
}
+ retries := 0
+ for {
_, err = dynamicClient.Create(unstructured)
if err != nil && (errors.IsAlreadyExists(err) || strings.Contains(err.Error(), portallocator.ErrAllocated.Error())) {
switch objectType.GetKind() {
@@ -902,9 +906,16 @@ func (m *MigrationController) applyResources(
} else {
log.MigrationLog(migration).Errorf("Error deleting %v %v during migrate: %v", objectType.GetKind(), metadata.GetName(), err)
}
- }
}
+ // Retry a few times for Unauthorized errors
+ if err != nil && errors.IsUnauthorized(err) && retries < maxApplyRetries {
+ retries++
+ continue
+ }
+ break
+ }
+ }
if err != nil {
m.updateResourceStatus(
migration,
