install and version setting use concurrency now
diff --git a/cmd/get_imports.go b/cmd/get_imports.go index feae223..ff0d810 100644 --- a/cmd/get_imports.go +++ b/cmd/get_imports.go
@@ -9,6 +9,7 @@ "os/exec" "path/filepath" "sort" + "sync" //"log" "os" @@ -23,6 +24,9 @@ v "github.com/Masterminds/vcs" ) +// Used for the fan out/in pattern used with VCS calls. +var concurrentWorkers = 20 + //func init() { // Uncomment the line below and the log import to see the output // from the vcs commands executed for each project. @@ -152,23 +156,56 @@ // SetReference is a command to set the VCS reference (commit id, tag, etc) for // a project. func SetReference(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := p.Get("conf", nil).(*cfg.Config) + conf := p.Get("conf", nil).(*cfg.Config) cwd, err := VendorPath(c) if err != nil { return false, err } - if len(cfg.Imports) == 0 { + if len(conf.Imports) == 0 { Info("No references set.\n") return false, nil } + // + // for _, dep := range conf.Imports { + // if err := VcsVersion(dep, cwd); err != nil { + // Warn("Failed to set version on %s to %s: %s\n", dep.Name, dep.Reference, err) + // } + // } - for _, dep := range cfg.Imports { - if err := VcsVersion(dep, cwd); err != nil { - Warn("Failed to set version on %s to %s: %s\n", dep.Name, dep.Reference, err) - } + done := make(chan struct{}, concurrentWorkers) + in := make(chan *cfg.Dependency, concurrentWorkers) + var wg sync.WaitGroup + + for i := 0; i < concurrentWorkers; i++ { + go func(ch <-chan *cfg.Dependency) { + for { + select { + case dep := <-ch: + if err := VcsVersion(dep, cwd); err != nil { + Warn("Failed to set version on %s to %s: %s\n", dep.Name, dep.Reference, err) + } + wg.Done() + case <-done: + return + } + } + }(in) } + for _, dep := range conf.Imports { + wg.Add(1) + in <- dep + } + + wg.Wait() + // Close goroutines setting the version + for i := 0; i < concurrentWorkers; i++ { + done <- struct{}{} + } + // close(done) + // close(in) + return true, nil } @@ -218,7 +255,6 @@ // // VcsGet installs into the dest. func VcsGet(dep *cfg.Dependency, dest, home string, cache, cacheGopath, skipGopath bool) error { - // When not skipping the $GOPATH look in it for a copy of the package if !skipGopath { // Check if the $GOPATH has a viable version to use and if so copy to vendor @@ -414,7 +450,7 @@ if err == errCacheDisabled { Debug("Unable to cache default branch because caching is disabled") } else if err != nil { - Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) + Debug("Error saving %s to cache - Error: %s", repo.Remote(), err) } } }
diff --git a/cmd/install.go b/cmd/install.go index c58babd..b388947 100644 --- a/cmd/install.go +++ b/cmd/install.go
@@ -4,6 +4,7 @@ "errors" "io/ioutil" "os" + "sync" "github.com/Masterminds/cookoo" "github.com/Masterminds/glide/cfg" @@ -101,10 +102,41 @@ return false, nil } + // for _, dep := range newConf.Imports { + // if err := VcsUpdate(dep, cwd, home, force, cache, cacheGopath, skipGopath); err != nil { + // Warn("Update failed for %s: %s\n", dep.Name, err) + // } + // } + + done := make(chan struct{}, concurrentWorkers) + in := make(chan *cfg.Dependency, concurrentWorkers) + var wg sync.WaitGroup + + for i := 0; i < concurrentWorkers; i++ { + go func(ch <-chan *cfg.Dependency) { + for { + select { + case dep := <-ch: + if err := VcsUpdate(dep, cwd, home, force, cache, cacheGopath, skipGopath); err != nil { + Warn("Update failed for %s: %s\n", dep.Name, err) + } + wg.Done() + case <-done: + return + } + } + }(in) + } + for _, dep := range newConf.Imports { - if err := VcsUpdate(dep, cwd, home, force, cache, cacheGopath, skipGopath); err != nil { - Warn("Update failed for %s: %s\n", dep.Name, err) - } + wg.Add(1) + in <- dep + } + + wg.Wait() + // Close goroutines setting the version + for i := 0; i < concurrentWorkers; i++ { + done <- struct{}{} } return newConf, nil
diff --git a/cmd/msg.go b/cmd/msg.go index a054cbf..a6dcc14 100644 --- a/cmd/msg.go +++ b/cmd/msg.go
@@ -6,6 +6,7 @@ "fmt" "os" "strings" + "sync" ) // These contanstants map to color codes for shell scripts making them @@ -19,6 +20,8 @@ Pink = "1;35" ) +var outputLock sync.Mutex + // Color returns a string in a certain color. The first argument is a string // containing the color code or a constant from the table above mapped to a code. // @@ -36,8 +39,8 @@ if Quiet { return } - fmt.Fprint(os.Stderr, Color(Green, "[INFO] ")) - Msg(msg, args...) + i := fmt.Sprint(Color(Green, "[INFO] ")) + Msg(i+msg, args...) } // Debug logs debug information @@ -45,49 +48,53 @@ if Quiet || !IsDebugging { return } - fmt.Fprint(os.Stderr, "[DEBUG] ") - Msg(msg, args...) + i := fmt.Sprint("[DEBUG] ") + Msg(i+msg, args...) } // Warn logs a warning func Warn(msg string, args ...interface{}) { - fmt.Fprint(os.Stderr, Color(Yellow, "[WARN] ")) - ErrMsg(msg, args...) + i := fmt.Sprint(Color(Yellow, "[WARN] ")) + ErrMsg(i+msg, args...) } // Error logs and error. func Error(msg string, args ...interface{}) { - fmt.Fprint(os.Stderr, Color(Red, "[ERROR] ")) - ErrMsg(msg, args...) + i := fmt.Sprint(Color(Red, "[ERROR] ")) + ErrMsg(i+msg, args...) } // ErrMsg sends a message to Stderr func ErrMsg(msg string, args ...interface{}) { - if len(args) == 0 { - fmt.Fprint(os.Stderr, msg) - } else { - fmt.Fprintf(os.Stderr, msg, args...) - } + outputLock.Lock() + defer outputLock.Unlock() - // Get rid of the annoying fact that messages need \n at the end, but do - // it in a backward compatible way. + // If messages don't have a newline on the end we add one. + e := "" if !strings.HasSuffix(msg, "\n") { - fmt.Fprintln(os.Stderr) + e = "\n" + } + if len(args) == 0 { + fmt.Fprint(os.Stderr, msg+e) + } else { + fmt.Fprintf(os.Stderr, msg+e, args...) } } // Msg prints a message with optional arguments, that can be printed, of // varying types. func Msg(msg string, args ...interface{}) { - if len(args) == 0 { - fmt.Fprint(os.Stderr, msg) - } else { - fmt.Fprintf(os.Stderr, msg, args...) - } + outputLock.Lock() + defer outputLock.Unlock() - // Get rid of the annoying fact that messages need \n at the end, but do - // it in a backward compatible way. + // If messages don't have a newline on the end we add one. + e := "" if !strings.HasSuffix(msg, "\n") { - fmt.Fprintln(os.Stderr) + e = "\n" + } + if len(args) == 0 { + fmt.Fprint(os.Stderr, msg+e) + } else { + fmt.Fprintf(os.Stderr, msg+e, args...) } }
diff --git a/glide.go b/glide.go index b568270..fee0aea 100644 --- a/glide.go +++ b/glide.go
@@ -568,6 +568,7 @@ Does(cmd.Install, "icfg"). Using("conf").From("cxt:cfg"). Using("lock").From("cxt:lock"). + Using("home").From("cxt:home"). Does(cmd.SetReference, "version").Using("conf").From("cxt:icfg"). Does(cmd.VendoredCleanUp, "_"). Using("conf").From("cxt:icfg").