UPSTREAM: 48757: Fix flaky test in reconciler_test
@@ -364,11 +364,7 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteMany(t *testing.
waitForTotalAttachCallCount(t, 2 /* expectedAttachCallCount */, fakePlugin)
verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin)
waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin)
-
- nodesForVolume := asw.GetNodesForVolume(generatedVolumeName)
- if len(nodesForVolume) != 2 {
- t.Fatal("Volume was not attached to both nodes")
- }
+ waitForAttachedToNodesCount(t, 2 /* expectedNodeCount */, generatedVolumeName, asw)
// Act
dsw.DeletePod(types.UniquePodName(podName1), generatedVolumeName, nodeName1)
@@ -455,13 +451,9 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing.
waitForTotalAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin)
verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin)
waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin)
+ waitForAttachedToNodesCount(t, 1 /* expectedNodeCount */, generatedVolumeName, asw)
nodesForVolume := asw.GetNodesForVolume(generatedVolumeName)
- if len(nodesForVolume) == 0 {
- t.Fatal("Volume was not attached to any node")
- } else if len(nodesForVolume) != 1 {
- t.Fatal("Volume was attached to multiple nodes")
- }
// check if multiattach is marked
// at least one volume should be marked with multiattach error
@@ -699,6 +691,39 @@ func waitForTotalDetachCallCount(
}
}
+func waitForAttachedToNodesCount(
+ t *testing.T,
+ expectedNodeCount int,
+ volumeName v1.UniqueVolumeName,
+ asw cache.ActualStateOfWorld) {
+
+ err := retryWithExponentialBackOff(
+ time.Duration(5*time.Millisecond),
+ func() (bool, error) {
+ count := len(asw.GetNodesForVolume(volumeName))
+ if count == expectedNodeCount {
+ return true, nil
+ }
+ t.Logf(
+ "Warning: Wrong number of nodes having <%v> attached. Expected: <%v> Actual: <%v>. Will retry.",
+ volumeName,
+ expectedNodeCount,
+ count)
+
+ return false, nil
+ },
+ )
+
+ if err != nil {
+ count := len(asw.GetNodesForVolume(volumeName))
+ t.Fatalf(
+ "Wrong number of nodes having <%v> attached. Expected: <%v> Actual: <%v>",
+ volumeName,
+ expectedNodeCount,
+ count)
+ }
+}
+
func verifyNewAttacherCallCount(
t *testing.T,
expectZeroNewAttacherCallCount bool,
