Rebuild: make downloads of dependencies parallel (8 connections by default)

rebuild.go

diff --git a/rebuild.go b/rebuild.go
index 9d4b0fa..b66bd0c 100644
--- a/rebuild.go
+++ b/rebuild.go
@@ -11,11 +11,13 @@
 )
 
 var (
-	rbInsecure bool // Allow the use of insecure protocols
+	rbInsecure    bool // Allow the use of insecure protocols
+	rbConnections uint // Count of concurrent download connections
 )
 
 func addRebuildFlags(fs *flag.FlagSet) {
 	fs.BoolVar(&rbInsecure, "precaire", false, "allow the use of insecure protocols")
+	fs.UintVar(&rbConnections, "connections", 8, "count of parallel download connections")
 }
 
 var cmdRebuild = &Command{
@@ -36,6 +38,8 @@
 Flags:
 	-precaire
 		allow the use of insecure protocols.
+	-connections
+		count of parallel download connections.
 `,
 	Run: func(args []string) error {
 		switch len(args) {
@@ -54,35 +58,55 @@
 		return fmt.Errorf("could not load manifest: %v", err)
 	}
 
+	var semaphore = make(chan int, rbConnections)
+	var errorsChan = make(chan error, rbConnections)
+
 	for _, dep := range m.Dependencies {
-		dst := filepath.Join(vendorDir(), dep.Importpath)
-		if _, err := os.Stat(dst); err == nil {
-			if err := vendor.RemoveAll(dst); err != nil {
-				// TODO need to apply vendor.cleanpath here too
-				return fmt.Errorf("dependency could not be deleted: %v", err)
-			}
-		}
+		go func(d vendor.Dependency) {
+			semaphore <- 1
+			errorsChan <- downloadDependency(d)
+			<-semaphore
+		}(dep)
+	}
 
-		log.Printf("fetching %s", dep.Importpath)
-
-		repo, _, err := vendor.DeduceRemoteRepo(dep.Importpath, rbInsecure)
+	for _ = range m.Dependencies {
+		err = <-errorsChan
 		if err != nil {
 			return err
 		}
+	}
 
-		wc, err := repo.Checkout("", "", dep.Revision)
-		if err != nil {
-			return err
-		}
+	return nil
+}
 
-		src := filepath.Join(wc.Dir(), dep.Path)
-		if err := vendor.Copypath(dst, src); err != nil {
-			return err
+func downloadDependency(dep vendor.Dependency) error {
+	dst := filepath.Join(vendorDir(), dep.Importpath)
+	if _, err := os.Stat(dst); err == nil {
+		if err := vendor.RemoveAll(dst); err != nil {
+			// TODO need to apply vendor.cleanpath here too
+			return fmt.Errorf("dependency could not be deleted: %v", err)
 		}
+	}
 
-		if err := wc.Destroy(); err != nil {
-			return err
-		}
+	log.Printf("fetching %s", dep.Importpath)
+
+	repo, _, err := vendor.DeduceRemoteRepo(dep.Importpath, rbInsecure)
+	if err != nil {
+		return err
+	}
+
+	wc, err := repo.Checkout("", "", dep.Revision)
+	if err != nil {
+		return err
+	}
+
+	src := filepath.Join(wc.Dir(), dep.Path)
+	if err := vendor.Copypath(dst, src); err != nil {
+		return err
+	}
+
+	if err := wc.Destroy(); err != nil {
+		return err
 	}
 
 	return nil