Increase OVN upgrade timeout by 15m
The OVN upgrade jobs are expected to take longer than
OpenShiftSDN.
There is more context to this here:
@@ -270,14 +270,28 @@ func clusterUpgrade(f *framework.Framework, c configv1client.Interface, dc dynam
// this is very long. We should update the clusteroperator junit to give us a duration.
maximumDuration := 150 * time.Minute
- // if upgrades take longer than this, then we will have a junit marker indicating failure.
- durationToSoftFailure := 75 * time.Minute
+ baseDurationToSoftFailure := 75 * time.Minute
+ durationToSoftFailure := baseDurationToSoftFailure
infra, err := c.ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
framework.ExpectNoError(err)
if infra.Status.PlatformStatus.Type == configv1.AWSPlatformType {
// due to https://bugzilla.redhat.com/show_bug.cgi?id=1943804 upgrades take ~12 extra minutes on AWS
- durationToSoftFailure = 105 * time.Minute
+ // and see commit d69db34a816f3ce8a9ab567621d145c5cd2d257f which notes that some AWS upgrades can
+ // take close to 105 minutes total (75 is base duration, so adding 30 more if it's AWS)
+ durationToSoftFailure = (baseDurationToSoftFailure + 30) * time.Minute
+ } else {
+ // if the cluster is on AWS we've already bumped the timeout enough, but if not we need to check if
+ // the CNI is OVN and increase our timeout for that
+ network, err := c.ConfigV1().Networks().Get(context.Background(), "cluster", metav1.GetOptions{})
+ framework.ExpectNoError(err)
+ if network.Status.NetworkType == "OVNKubernetes" {
+ // deploying with OVN is expected to take longer. on average, ~15m longer
+ // some extra context to this increase which links to a jira showing which operators take longer:
+ // compared to OpenShiftSDN:
+ // https://bugzilla.redhat.com/show_bug.cgi?id=1942164
+ durationToSoftFailure = (baseDurationToSoftFailure + 15) * time.Minute
+ }
}
framework.Logf("Starting upgrade to version=%s image=%s attempt=%s", version.Version.String(), version.NodeImage, uid)
@@ -424,9 +438,9 @@ func clusterUpgrade(f *framework.Framework, c configv1client.Interface, dc dynam
upgradeEnded := time.Now()
upgradeDuration := upgradeEnded.Sub(upgradeStarted)
if upgradeDuration > durationToSoftFailure {
- disruption.RecordJUnitResult(f, "[sig-cluster-lifecycle] cluster upgrade should complete in 75m (105m on AWS)", upgradeDuration, fmt.Sprintf("%s to %s took too long: %0.2f minutes", action, versionString(desired), upgradeDuration.Minutes()))
+ disruption.RecordJUnitResult(f, fmt.Sprintf("[sig-cluster-lifecycle] cluster upgrade should complete in %v minutes", durationToSoftFailure), upgradeDuration, fmt.Sprintf("%s to %s took too long: %0.2f minutes", action, versionString(desired), upgradeDuration.Minutes()))
} else {
- disruption.RecordJUnitResult(f, "[sig-cluster-lifecycle] cluster upgrade should complete in 75m (105m on AWS)", upgradeDuration, "")
+ disruption.RecordJUnitResult(f, fmt.Sprintf("[sig-cluster-lifecycle] cluster upgrade should complete in %v minutes", durationToSoftFailure), upgradeDuration, "")
}
return nil
