Skip to content
This repository was archived by the owner on Sep 9, 2020. It is now read-only.

Commit e4522e4

Browse files
committed
status: Place upper bounds on concurrency
dep status was handily busting through fd limits on larger projects by fanning out on all projects without limit. This adds a hardcoded limit that should still be sufficient to still get parallelism benefits, but will not exhaust the open files limits for any sane default setting on any OS. Fixes #1727 Fixes #1923
1 parent aefa7cc commit e4522e4

File tree

1 file changed

+12
-0
lines changed

1 file changed

+12
-0
lines changed

cmd/dep/status.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -955,13 +955,19 @@ func (cmd *statusCommand) runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Proje
955955
errListPkgCh := make(chan error, len(slp))
956956
errListVerCh := make(chan error, len(slp))
957957

958+
// Hardcode to a limit of 16 simultaneous projects. This is plenty high
959+
// enough to gain concurrency benefits, but low enough that we won't
960+
// hit an open fd limit on any OS.
961+
sem := make(chan struct{}, 16)
958962
var wg sync.WaitGroup
959963

960964
for i, proj := range slp {
961965
wg.Add(1)
962966
logger.Printf("(%d/%d) %s\n", i+1, len(slp), proj.Ident().ProjectRoot)
967+
sem <- struct{}{}
963968

964969
go func(proj verify.VerifiableProject) {
970+
defer func() { <-sem }()
965971
bs := BasicStatus{
966972
ProjectRoot: string(proj.Ident().ProjectRoot),
967973
PackageCount: len(proj.Packages()),
@@ -1327,14 +1333,20 @@ func collectConstraints(ctx *dep.Ctx, p *dep.Project, sm gps.SourceManager) (con
13271333
// Channel for receiving all the errors.
13281334
errCh := make(chan error, len(lp))
13291335

1336+
// Hardcode to a limit of 16 simultaneous projects. This is plenty high
1337+
// enough to gain concurrency benefits, but low enough that we won't
1338+
// hit an open fd limit on any OS.
1339+
sem := make(chan struct{}, 16)
13301340
var wg sync.WaitGroup
13311341

13321342
// Iterate through the locked projects and collect constraints of all the projects.
13331343
for i, proj := range lp {
13341344
wg.Add(1)
13351345
logger.Printf("(%d/%d) %s\n", i+1, len(lp), proj.Ident().ProjectRoot)
1346+
sem <- struct{}{}
13361347

13371348
go func(proj gps.LockedProject) {
1349+
defer func() { <-sem }()
13381350
defer wg.Done()
13391351

13401352
manifest, _, err := sm.GetManifestAndLock(proj.Ident(), proj.Version(), rootAnalyzer)

0 commit comments

Comments
 (0)