rebuild: refactor parallel scheduler
diff --git a/alldocs.go b/alldocs.go index 85e106b..bb5e889 100644 --- a/alldocs.go +++ b/alldocs.go
@@ -62,6 +62,8 @@ Flags: -precaire allow the use of insecure protocols. + -connections + count of parallel download connections. Update a local dependency
diff --git a/rebuild.go b/rebuild.go index b66bd0c..87f13cd 100644 --- a/rebuild.go +++ b/rebuild.go
@@ -6,6 +6,8 @@ "log" "os" "path/filepath" + "sync" + "sync/atomic" "github.com/FiloSottile/gvt/gbvendor" ) @@ -58,22 +60,30 @@ return fmt.Errorf("could not load manifest: %v", err) } - var semaphore = make(chan int, rbConnections) - var errorsChan = make(chan error, rbConnections) - - for _, dep := range m.Dependencies { - go func(d vendor.Dependency) { - semaphore <- 1 - errorsChan <- downloadDependency(d) - <-semaphore - }(dep) + var errors uint32 + var wg sync.WaitGroup + depC := make(chan vendor.Dependency) + for i := 0; i < int(rbConnections); i++ { + wg.Add(1) + go func() { + defer wg.Done() + for d := range depC { + if err := downloadDependency(d); err != nil { + log.Printf("%s: %v", d.Importpath, err) + atomic.AddUint32(&errors, 1) + } + } + }() } - for _ = range m.Dependencies { - err = <-errorsChan - if err != nil { - return err - } + for _, dep := range m.Dependencies { + depC <- dep + } + close(depC) + wg.Wait() + + if errors > 0 { + return fmt.Errorf("failed to fetch %d dependencies", errors) } return nil