mantle/platform: Ignore some non-fatal unit failures in RHCOS
Special case to ignore some non-fatal unit failures frequently seen in
flakes. This is for RHEL 8 only as this has been fixed upstream in
systemd but not fixed in RHEL 8.
Tracked in
@@ -21,6 +21,7 @@ import (
"io"
"os/exec"
"path/filepath"
+ "strings"
"sync"
"time"
@@ -391,6 +392,39 @@ func NewMachines(c Cluster, userdata *conf.UserData, n int, options MachineOptio
return machs, nil
}
+// checkSystemdUnitFailures make sure no system unit is in failed state.
+func checkSystemdUnitFailures(output string, distribution string) error {
+ var ignoredUnits []string
+
+ // Temporarily ignore some non-fatal flakes on RHCOS. See:
+ // https://bugzilla.redhat.com/show_bug.cgi?id=1914362
+ if distribution == "rhcos" {
+ ignoredUnits = append(ignoredUnits, "user@1000.service")
+ ignoredUnits = append(ignoredUnits, "user-runtime-dir@1000.service")
+ }
+
+ var failedUnits []string
+ for _, unit := range strings.Split(output, "\n") {
+ u := strings.Fields(unit)[0]
+ ignore := false
+ for _, i := range ignoredUnits {
+ if u == i {
+ // Ignoring this unit
+ ignore = true
+ break
+ }
+ }
+ if !ignore {
+ failedUnits = append(failedUnits, u)
+ }
+ }
+ if len(failedUnits) > 0 {
+ return fmt.Errorf("some systemd units failed:\n%s", output)
+ }
+
+ return nil
+}
+
// CheckMachine tests a machine for various error conditions such as ssh
// being available and no systemd units failing at the time ssh is reachable.
// It also ensures the remote system is running Container Linux by CoreOS or
@@ -423,8 +457,13 @@ func CheckMachine(ctx context.Context, m Machine) error {
}
// ensure we're talking to a supported system
+ var distribution string
switch string(out) {
- case `coreos-`, `rhcos-`, `fedora-coreos`:
+ case `fedora-coreos`:
+ distribution = "fcos"
+ break
+ case `rhcos-`:
+ distribution = "rhcos"
break
default:
return fmt.Errorf("not a supported instance: %v", string(out))
@@ -437,7 +476,7 @@ func CheckMachine(ctx context.Context, m Machine) error {
return fmt.Errorf("systemctl: %s: %v: %s", out, err, stderr)
}
if len(out) > 0 {
- return fmt.Errorf("some systemd units failed:\n%s", out)
+ return checkSystemdUnitFailures(string(out), distribution)
}
}
