Add retry logic to handle ssm document during reboot
cr
@@ -115,17 +115,16 @@ func (d *DocumentFileMgr) GetDocumentState(log log.T, fileName, instanceID, loca
absoluteFileName := path.Join(filepath, fileName)
var commandState contracts.DocumentState
+ var count, retryLimit int = 0, 3
+
+ // retry to avoid sync problem, which arises when OfflineService and MessageDeliveryService try to access the file at the same time
+ for count < retryLimit {
err := jsonutil.UnmarshalFile(absoluteFileName, &commandState)
if err != nil {
log.Errorf("encountered error with message %v while reading Interim state of command from file - %v", err, fileName)
- if fileExists, _ := fileutil.LocalFileExist(absoluteFileName); fileExists {
- if documentContents, err := fileutil.ReadAllText(absoluteFileName); err == nil {
- log.Infof("Document contents: %v", documentContents)
- }
-
- d.MoveDocumentState(log, fileName, instanceID, locationFolder, appconfig.DefaultLocationOfCorrupt)
- }
-
+ count += 1
+ time.Sleep(500 * time.Millisecond)
+ continue
} else {
//logging interim state as read from the file
jsonString, err := jsonutil.Marshal(commandState)
@@ -134,6 +133,19 @@ func (d *DocumentFileMgr) GetDocumentState(log log.T, fileName, instanceID, loca
} else {
log.Tracef("interim CommandState read from file-system - %v", jsonutil.Indent(jsonString))
}
+ break
+ }
+ }
+
+ if count >= retryLimit {
+ if fileExists, _ := fileutil.LocalFileExist(absoluteFileName); fileExists {
+ if documentContents, err := fileutil.ReadAllText(absoluteFileName); err == nil {
+ log.Infof("Document contents: %v", documentContents)
+ }
+
+ d.MoveDocumentState(log, fileName, instanceID, locationFolder, appconfig.DefaultLocationOfCorrupt)
+ }
+
}
return commandState
