Func Test to exercise crash loop backoff
@@ -1714,6 +1714,70 @@ var _ = Describe("[rfe_id:1177][crit:medium][vendor:cnv-qe@redhat.com][level:com
})
})
+
+ Context("crash loop backoff", func() {
+ It("should backoff attempting to create a new VMI when 'runStrategy: Always' during crash loop.", func() {
+ vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(cd.ContainerDiskFor(cd.ContainerDiskCirros), "echo Hi\n")
+
+ By("Creating VirtualMachine")
+ vm := tests.NewRandomVirtualMachine(vmi, true)
+ vm.Spec.Template.ObjectMeta.Annotations = map[string]string{
+ v1.FuncTestLauncherFailFastAnnotation: "",
+ }
+ newVM, err := virtClient.VirtualMachine(tests.NamespaceTestDefault).Create(vm)
+ Expect(err).ToNot(HaveOccurred())
+
+ Eventually(func() error {
+ vm, err := virtClient.VirtualMachine(newVM.Namespace).Get(newVM.Name, &k8smetav1.GetOptions{})
+ Expect(err).ToNot(HaveOccurred())
+
+ if vm.Status.StartFailure != nil && vm.Status.StartFailure.ConsecutiveFailCount > 0 {
+ return nil
+ }
+
+ return fmt.Errorf("Still waiting on crash loop to be detected")
+ }, 1*time.Minute, 5*time.Second).Should(BeNil())
+
+ By("Testing that the failure count is within the expected range over a period of time")
+ maxExpectedFailCount := 3
+ Consistently(func() error {
+ // get the VM and verify the failure count is less than 4 over a minute,
+ // indicating that backoff is occuring
+ vm, err := virtClient.VirtualMachine(newVM.Namespace).Get(newVM.Name, &k8smetav1.GetOptions{})
+ Expect(err).ToNot(HaveOccurred())
+
+ if vm.Status.StartFailure == nil {
+ return fmt.Errorf("start failure count not detected")
+ } else if vm.Status.StartFailure.ConsecutiveFailCount > maxExpectedFailCount {
+ return fmt.Errorf("consecutive fail count is higher than %d", maxExpectedFailCount)
+ }
+
+ return nil
+ }, 1*time.Minute, 5*time.Second).Should(BeNil())
+
+ By("Updating the VMI template to correct the crash loop")
+ Eventually(func() error {
+ updatedVM, err := virtClient.VirtualMachine(vm.Namespace).Get(vm.Name, &k8smetav1.GetOptions{})
+ Expect(err).ToNot(HaveOccurred())
+ delete(updatedVM.Spec.Template.ObjectMeta.Annotations, v1.FuncTestLauncherFailFastAnnotation)
+ _, err = virtClient.VirtualMachine(updatedVM.Namespace).Update(updatedVM)
+ return err
+ }, 30*time.Second, 1*time.Second).ShouldNot(HaveOccurred())
+
+ By("Waiting on crash loop status to be removed.")
+ Eventually(func() error {
+ vm, err := virtClient.VirtualMachine(newVM.Namespace).Get(newVM.Name, &k8smetav1.GetOptions{})
+ Expect(err).ToNot(HaveOccurred())
+
+ if vm.Status.StartFailure == nil {
+ return nil
+ }
+
+ return fmt.Errorf("Still waiting on crash loop status to be re-initialized")
+ }, 5*time.Minute, 5*time.Second).Should(BeNil())
+ })
+ })
+
})
func isKubemacpoolDeployed(virtClient kubecli.KubevirtClient) (bool, error) {
