internal/frontend: add fetch endpoint to robots.txt
Disallow: /fetch/* is added to the robots.txt, so that search engines do
not crawl the endpoint and make unnecessary fetch requests.
Updates golang/go#36811
Updates golang/go#37002
@@ -113,6 +113,7 @@ func (s *Server) Install(handle func(string, http.Handler), redisClient *redis.C
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
http.ServeContent(w, r, "", time.Time{}, strings.NewReader(`User-agent: *
Disallow: /search?*
+Disallow: /fetch/*
`))
}))
}
