logtail: help the server be more efficient
Add content length hints to headers.
The server can use these hints to more efficiently select buffers.
Stop attempting to compress tiny requests.
The bandwidth savings are negligible (and sometimes negative!),
and it makes extra work for the server.
"io/ioutil"
"net/http"
"os"
+ "strconv"
"time"
"tailscale.com/logtail/backoff"
@@ -259,8 +260,17 @@ func (l *logger) uploading(ctx context.Context) {
for {
body := l.drainPending()
- if l.zstdEncoder != nil {
- body = l.zstdEncoder.EncodeAll(body, nil)
+ origlen := -1 // sentinel value: uncompressed
+ // Don't attempt to compress tiny bodies; not worth the CPU cycles.
+ if l.zstdEncoder != nil && len(body) > 256 {
+ zbody := l.zstdEncoder.EncodeAll(body, nil)
+ // Only send it compressed if the bandwidth savings are sufficient.
+ // Just the extra headers associated with enabling compression
+ // are 50 bytes by themselves.
+ if len(body)-len(zbody) > 64 {
+ origlen = len(body)
+ body = zbody
+ }
}
for len(body) > 0 {
@@ -269,7 +279,7 @@ func (l *logger) uploading(ctx context.Context) {
return
default:
}
- uploaded, err := l.upload(ctx, body)
+ uploaded, err := l.upload(ctx, body, origlen)
if err != nil {
fmt.Fprintf(l.stderr, "logtail: upload: %v\n", err)
}
@@ -287,7 +297,10 @@ func (l *logger) uploading(ctx context.Context) {
}
}
-func (l *logger) upload(ctx context.Context, body []byte) (uploaded bool, err error) {
+// upload uploads body to the log server.
+// origlen indicates the pre-compression body length.
+// origlen of -1 indicates that the body is not compressed.
+func (l *logger) upload(ctx context.Context, body []byte, origlen int) (uploaded bool, err error) {
req, err := http.NewRequest("POST", l.url, bytes.NewReader(body))
if err != nil {
// I know of no conditions under which this could fail.
@@ -295,8 +308,9 @@ func (l *logger) upload(ctx context.Context, body []byte) (uploaded bool, err er
// TODO record logs to disk
panic("logtail: cannot build http request: " + err.Error())
}
- if l.zstdEncoder != nil {
+ if origlen != -1 {
req.Header.Add("Content-Encoding", "zstd")
+ req.Header.Add("Orig-Content-Length", strconv.Itoa(origlen))
}
req.Header["User-Agent"] = nil // not worth writing one; save some bytes
@@ -306,7 +320,7 @@ func (l *logger) upload(ctx context.Context, body []byte) (uploaded bool, err er
req = req.WithContext(ctx)
compressedNote := "not-compressed"
- if l.zstdEncoder != nil {
+ if origlen != -1 {
compressedNote = "compressed"
}
