api: Automatically retry on conflict caused API error
Also need to make sure there is no side-effort on retry
@@ -2,18 +2,36 @@ package api
import (
"net/http"
+ "time"
"github.com/Sirupsen/logrus"
"github.com/gorilla/mux"
+ "github.com/pkg/errors"
"github.com/rancher/go-rancher/api"
"github.com/rancher/go-rancher/client"
+
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+)
+
+var (
+ RetryCounts = 5
+ RetryInterval = 100 * time.Millisecond
)
type HandleFuncWithError func(http.ResponseWriter, *http.Request) error
func HandleError(s *client.Schemas, t HandleFuncWithError) http.Handler {
return api.ApiHandler(s, http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
- if err := t(rw, req); err != nil {
+ var err error
+ for i := 0; i < RetryCounts; i++ {
+ err = t(rw, req)
+ if !apierrors.IsConflict(errors.Cause(err)) {
+ break
+ }
+ logrus.Warnf("Retry API call due to conflict")
+ time.Sleep(RetryInterval)
+ }
+ if err != nil {
logrus.Warnf("HTTP handling error %v", err)
apiContext := api.GetApiContext(req)
apiContext.WriteErr(err)
