Add network connection logging
Run a tool called `ss` in the background while making s3 requests. Log the output iff reading from S3 fails in the manner that we're trying to diagnose in
@@ -7,6 +7,8 @@ package nbs
import (
"fmt"
"io"
+ "os"
+ "os/exec"
"github.com/attic-labs/noms/go/d"
"github.com/aws/aws-sdk-go/aws"
@@ -90,6 +92,12 @@ func (s3tr *s3TableReader) readRange(p []byte, rangeHeader string) (n int, err e
Key: aws.String(s3tr.hash().String()),
Range: aws.String(rangeHeader),
}
+ // TODO: take out this running of ss once BUG 3255 is fixed
+ shouldLog := make(chan struct{})
+ defer close(shouldLog)
+ ss, logFinished := newBg(shouldLog, "/usr/sbin/ss", "-ntp")
+ go ss.Run("Couldn't run ss to record network connections:")
+
// TODO: go back to just calling GetObject once BUG 3255 is fixed
result, reqID, reqID2, err := func() (*s3.GetObjectOutput, string, string, error) {
if impl, ok := s3tr.s3.(*s3.S3); ok {
@@ -106,8 +114,34 @@ func (s3tr *s3TableReader) readRange(p []byte, rangeHeader string) (n int, err e
n, err = io.ReadFull(result.Body, p)
if err != nil {
+ shouldLog <- struct{}{}
+ <-logFinished
d.Chk.Fail("Failed ranged read from S3\n", "req %s\nreq %s\n%s\nerror: %v", reqID, reqID2, input.GoString(), err)
}
return n, err
}
+
+type bgCmd struct {
+ cmd *exec.Cmd
+ startLog <-chan struct{}
+ logFinished chan<- struct{}
+}
+
+func newBg(startLog <-chan struct{}, path string, args ...string) (cmd *bgCmd, finished <-chan struct{}) {
+ f := make(chan struct{})
+ cmd = &bgCmd{exec.Command(path, args...), startLog, f}
+ return cmd, f
+}
+
+func (w *bgCmd) Run(msg string) {
+ defer close(w.logFinished)
+ out, err := w.cmd.CombinedOutput()
+ if err != nil {
+ fmt.Fprintln(os.Stderr, msg, err)
+ return
+ }
+ if _, log := <-w.startLog; log {
+ fmt.Fprintln(os.Stderr, out)
+ }
+}
