fix: retry CreateInstance requests on RESOURCE_EXHAUSTED
@@ -583,6 +583,7 @@ func createTestInstance(t *testing.T, projectID string) (instanceName string, cl
}
}
+ testutil.Retry(t, 20, time.Minute, func(r *testutil.R) {
op, err := instanceAdmin.CreateInstance(ctx, &instancepb.CreateInstanceRequest{
Parent: fmt.Sprintf("projects/%s", projectID),
InstanceId: instanceID,
@@ -597,12 +598,21 @@ func createTestInstance(t *testing.T, projectID string) (instanceName string, cl
},
})
if err != nil {
+ // Retry if the instance could not be created because there have
+ // been too many create requests in the past minute.
+ if spanner.ErrCode(err) == codes.ResourceExhausted && strings.Contains(err.Error(), "Quota exceeded for quota metric 'Instance create requests'") {
+ r.Errorf("could not create instance %s: %v", fmt.Sprintf("projects/%s/instances/%s", projectID, instanceID), err)
+ return
+ } else {
t.Fatalf("could not create instance %s: %v", fmt.Sprintf("projects/%s/instances/%s", projectID, instanceID), err)
}
+ }
_, err = op.Wait(ctx)
if err != nil {
t.Fatalf("waiting for instance creation to finish failed: %v", err)
}
+ })
+
return instanceName, func() {
deleteInstanceAndBackups(t, instanceName, instanceAdmin, databaseAdmin)
instanceAdmin.Close()
