Limit file-fetching concurrency.
Summary:
For some very large compilations, it is possible to thrash the box with too
many goroutines. Although GOMAXPROCS limits the number of threads that will
handle goroutines, there isn't a cap on threads for syscalls.
Reviewers: shahms, #core_team
Subscribers: #core_team
@@ -232,11 +232,15 @@ func (c *Config) FetchInputs(ctx context.Context, paths []string) ([]*apb.FileDa
// Fetch concurrently. Each element of the proto slices is accessed by a
// single goroutine corresponding to its index.
+ throttle := make(chan struct{}, 256)
fileData := make([]*apb.FileData, len(paths))
var g errgroup.Group
for i, path := range paths {
i, path := i, path
g.Go(func() error {
+ throttle <- struct{}{}
+ defer func() { <-throttle }()
+
fd, err := c.readFileData(ctx, path)
if err != nil {
log.Printf("ERROR: Reading input file: %v", err)
