topgun: retry on ping timeout
hopefully this at least makes our pipeline more resilient until we can
get to the bottom of this.
adding the sleep didn't help because BOSH just nukes the VM after the
ping timeout, so there's nothing to investigate.
@@ -6,7 +6,6 @@ import (
"database/sql"
"fmt"
"io/ioutil"
- "log"
"net/http"
"os"
"path/filepath"
@@ -206,21 +205,21 @@ func Deploy(manifest string, args ...string) {
Expect(dbConn.Close()).To(Succeed())
}
+ for {
deploy := StartDeploy(manifest, args...)
<-deploy.Exited
if deploy.ExitCode() != 0 {
if strings.Contains(string(deploy.Out.Contents()), "Timed out pinging") {
- fmt.Fprintln(GinkgoWriter, "detected ping timeout; dumping vm info...")
- bosh("vms")
- log.Println("SLEEPING TO DEBUG PING TIMEOUT")
- log.Printf("DEPLOY OUTPUT:\n%s\n", deploy.Out.Contents())
- time.Sleep(24 * time.Hour)
- Fail("deploy failed due to ping timeout; possible iaas flake")
+ fmt.Fprintln(GinkgoWriter, "detected ping timeout; trying again...")
+ continue
}
Fail("deploy failed")
}
+ break
+ }
+
instances, jobInstances = loadJobInstances()
webInstance = JobInstance("web")
