Issue #127: Updating the agressive caching to be opt-in - Will attempt to copy the dependency from the GOPATH if there. This can be skipped with the --skip-gopath flag. - Using the --cache flag will attempt to cache to the $HOME/.glide/cache folder. - Using the --cache-gopath flag will attempt to put a copy in the GOPATH. That can be pulled from on future runs.
diff --git a/cmd/flatten.go b/cmd/flatten.go index 3462cfe..abb3ce8 100644 --- a/cmd/flatten.go +++ b/cmd/flatten.go
@@ -29,6 +29,8 @@ skip := p.Get("skip", false).(bool) home := p.Get("home", "").(string) cache := p.Get("cache", false).(bool) + cacheGopath := p.Get("cacheGopath", false).(bool) + skipGopath := p.Get("skipGopath", false).(bool) if skip { return conf, nil @@ -53,7 +55,7 @@ f := &flattening{conf, vend, vend, deps, packages} - err := recFlatten(f, force, home, cache) + err := recFlatten(f, force, home, cache, cacheGopath, skipGopath) flattenSetRefs(f) Info("Project relies on %d dependencies.", len(deps)) exportFlattenedDeps(conf, deps) @@ -87,7 +89,7 @@ var updateCache = map[string]bool{} // refFlatten recursively flattens the vendor tree. -func recFlatten(f *flattening, force bool, home string, cache bool) error { +func recFlatten(f *flattening, force bool, home string, cache, cacheGopath, skipGopath bool) error { Debug("---> Inspecting %s for changes (%d packages).\n", f.curr, len(f.scan)) for _, imp := range f.scan { Debug("----> Scanning %s", imp) @@ -107,14 +109,14 @@ if len(mod) > 0 { Debug("----> Updating all dependencies for %q (%d)", imp, len(mod)) - flattenGlideUp(f, base, home, force, cache) + flattenGlideUp(f, base, home, force, cache, cacheGopath, skipGopath) f2 := &flattening{ conf: f.conf, top: f.top, curr: base, deps: f.deps, scan: mod} - recFlatten(f2, force, home, cache) + recFlatten(f2, force, home, cache, cacheGopath, skipGopath) } } @@ -126,7 +128,7 @@ // While this is expensive, it is also necessary to make sure we have the // correct version of all dependencies. We might be able to simplify by // marking packages dirty when they are added. -func flattenGlideUp(f *flattening, base, home string, force, cache bool) error { +func flattenGlideUp(f *flattening, base, home string, force, cache, cacheGopath, skipGopath bool) error { //vdir := path.Join(base, "vendor") for _, imp := range f.deps { wd := path.Join(f.top, imp.Name) @@ -136,7 +138,7 @@ continue } Debug("Updating project %s (%s)\n", imp.Name, wd) - if err := VcsUpdate(imp, f.top, home, force, cache); err != nil { + if err := VcsUpdate(imp, f.top, home, force, cache, cacheGopath, skipGopath); err != nil { // We can still go on just fine even if this fails. Warn("Skipped update %s: %s\n", imp.Name, err) continue @@ -144,7 +146,7 @@ updateCache[imp.Name] = true } else { Debug("Importing %s to project %s\n", imp.Name, wd) - if err := VcsGet(imp, wd, home, cache); err != nil { + if err := VcsGet(imp, wd, home, cache, cacheGopath, skipGopath); err != nil { Warn("Skipped getting %s: %v\n", imp.Name, err) continue }
diff --git a/cmd/get_imports.go b/cmd/get_imports.go index e3f01fa..fa55356 100644 --- a/cmd/get_imports.go +++ b/cmd/get_imports.go
@@ -46,6 +46,9 @@ insecure := p.Get("insecure", false).(bool) home := p.Get("home", "").(string) cache := p.Get("cache", false).(bool) + cacheGopath := p.Get("cacheGopath", false).(bool) + skipGopath := p.Get("skipGopath", false).(bool) + Info("Preparing to install %d package.", len(names)) deps := []*yaml.Dependency{} @@ -86,7 +89,7 @@ if len(subpkg) > 0 && subpkg != "/" { dep.Subpackages = []string{subpkg} } - if err := VcsGet(dep, dest, home, cache); err != nil { + if err := VcsGet(dep, dest, home, cache, cacheGopath, skipGopath); err != nil { return dep, err } @@ -111,6 +114,9 @@ plist := p.Get("packages", []string{}).([]string) home := p.Get("home", "").(string) cache := p.Get("cache", false).(bool) + cacheGopath := p.Get("cacheGopath", false).(bool) + skipGopath := p.Get("skipGopath", false).(bool) + pkgs := list2map(plist) restrict := len(pkgs) > 0 @@ -135,7 +141,7 @@ // flattening from causing unnecessary updates. updateCache[dep.Name] = true - if err := VcsUpdate(dep, cwd, home, force, cache); err != nil { + if err := VcsUpdate(dep, cwd, home, force, cache, cacheGopath, skipGopath); err != nil { Warn("Update failed for %s: %s\n", dep.Name, err) } } @@ -211,9 +217,10 @@ // VcsGet figures out how to fetch a dependency, and then gets it. // // VcsGet installs into the dest. -func VcsGet(dep *yaml.Dependency, dest, home string, cache bool) error { +func VcsGet(dep *yaml.Dependency, dest, home string, cache, cacheGopath, skipGopath bool) error { - if !cache { + // When not skipping the $GOPATH look in it for a copy of the package + if !skipGopath { // Check if the $GOPATH has a viable version to use and if so copy to vendor gps := Gopaths() for _, p := range gps { @@ -266,6 +273,10 @@ return nil } } + } + + // When opting in to cache in the GOPATH attempt to do put a copy there. + if cacheGopath { // Since we didn't find an existing copy in the GOPATHs try to clone there. gp := Gopath() @@ -308,65 +319,68 @@ } } - // Check if the cache has a viable version and try to use that. - var loc string - if dep.Repository != "" { - loc = dep.Repository - } else { - loc = "https://" + dep.Name - } - key, err := cacheCreateKey(loc) - if err == nil { - d := filepath.Join(home, "cache", "src", key) - - repo, err := dep.GetRepo(d) - if err != nil { - return err + // If opting in to caching attempt to put it in the cache folder + if cache { + // Check if the cache has a viable version and try to use that. + var loc string + if dep.Repository != "" { + loc = dep.Repository + } else { + loc = "https://" + dep.Name } - // If the directory does not exist this is a first cache. - if _, err = os.Stat(d); os.IsNotExist(err) { - Debug("Adding %s to the cache for the first time", dep.Name) - err = repo.Get() + key, err := cacheCreateKey(loc) + if err == nil { + d := filepath.Join(home, "cache", "src", key) + + repo, err := dep.GetRepo(d) if err != nil { return err } - branch := findCurrentBranch(repo) - if branch != "" { - // we know the default branch so we can store it in the cache - var loc string - if dep.Repository != "" { - loc = dep.Repository - } else { - loc = "https://" + dep.Name + // If the directory does not exist this is a first cache. + if _, err = os.Stat(d); os.IsNotExist(err) { + Debug("Adding %s to the cache for the first time", dep.Name) + err = repo.Get() + if err != nil { + return err } - key, err := cacheCreateKey(loc) - if err == nil { - Debug("Saving default branch for %s", repo.Remote()) - c := cacheRepoInfo{DefaultBranch: branch} - err = saveCacheRepoData(key, c, home) - if err != nil { - Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) + branch := findCurrentBranch(repo) + if branch != "" { + // we know the default branch so we can store it in the cache + var loc string + if dep.Repository != "" { + loc = dep.Repository + } else { + loc = "https://" + dep.Name + } + key, err := cacheCreateKey(loc) + if err == nil { + Debug("Saving default branch for %s", repo.Remote()) + c := cacheRepoInfo{DefaultBranch: branch} + err = saveCacheRepoData(key, c, home) + if err != nil { + Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) + } } } + + } else { + Debug("Updating %s in the cache", dep.Name) + err = repo.Update() + if err != nil { + return err + } } - } else { - Debug("Updating %s in the cache", dep.Name) - err = repo.Update() + Debug("Copying %s from the cache to %s", dep.Name, dest) + err = copyDir(d, dest) if err != nil { return err } - } - Debug("Copying %s from the cache to %s", dep.Name, dest) - err = copyDir(d, dest) - if err != nil { - return err + return nil + } else { + Warn("Cache key generation error: %s", err) } - - return nil - } else { - Warn("Cache key generation error: %s", err) } // If unable to cache pull directly into the vendor/ directory. @@ -375,11 +389,34 @@ return err } - return repo.Get() + gerr := repo.Get() + + // Attempt to cache the default branch + branch := findCurrentBranch(repo) + if branch != "" { + // we know the default branch so we can store it in the cache + var loc string + if dep.Repository != "" { + loc = dep.Repository + } else { + loc = "https://" + dep.Name + } + key, err := cacheCreateKey(loc) + if err == nil { + Debug("Saving default branch for %s", repo.Remote()) + c := cacheRepoInfo{DefaultBranch: branch} + err = saveCacheRepoData(key, c, home) + if err != nil { + Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) + } + } + } + + return gerr } // VcsUpdate updates to a particular checkout based on the VCS setting. -func VcsUpdate(dep *yaml.Dependency, vend, home string, force, cache bool) error { +func VcsUpdate(dep *yaml.Dependency, vend, home string, force, cache, cacheGopath, skipGopath bool) error { Info("Fetching updates for %s.\n", dep.Name) if filterArchOs(dep) { @@ -390,7 +427,7 @@ dest := path.Join(vend, dep.Name) // If destination doesn't exist we need to perform an initial checkout. if _, err := os.Stat(dest); os.IsNotExist(err) { - if err = VcsGet(dep, dest, home, cache); err != nil { + if err = VcsGet(dep, dest, home, cache, cacheGopath, skipGopath); err != nil { Warn("Unable to checkout %s\n", dep.Name) return err } @@ -427,7 +464,7 @@ if rerr != nil { return rerr } - if err = VcsGet(dep, dest, home, cache); err != nil { + if err = VcsGet(dep, dest, home, cache, cacheGopath, skipGopath); err != nil { Warn("Unable to checkout %s\n", dep.Name) return err }
diff --git a/glide.go b/glide.go index 6675b2c..59b1c82 100644 --- a/glide.go +++ b/glide.go
@@ -179,7 +179,15 @@ }, cli.BoolFlag{ Name: "cache", - Usage: "Setting will only and use files from the cache (excluding the GOPATH)", + Usage: "When downloading dependencies attempt to cache them.", + }, + cli.BoolFlag{ + Name: "cache-gopath", + Usage: "When downloading dependencies attempt to put them in the GOPATH, too.", + }, + cli.BoolFlag{ + Name: "skip-gopath", + Usage: "Skip attempting to copy a dependency from the GOPATH.", }, }, Action: func(c *cli.Context) { @@ -190,7 +198,9 @@ cxt.Put("packages", []string(c.Args())) cxt.Put("skipFlatten", !c.Bool("no-recursive")) cxt.Put("insecure", c.Bool("insecure")) - cxt.Put("forceCache", c.Bool("cache")) + cxt.Put("useCache", c.Bool("cache")) + cxt.Put("cacheGopath", c.Bool("cache-gopath")) + cxt.Put("skipGopath", c.Bool("skip-gopath")) // FIXME: Are these used anywhere? if c.Bool("import") { cxt.Put("importGodeps", true) @@ -354,7 +364,15 @@ }, cli.BoolFlag{ Name: "cache", - Usage: "Setting will only and use files from the cache (excluding the GOPATH)", + Usage: "When downloading dependencies attempt to cache them.", + }, + cli.BoolFlag{ + Name: "cache-gopath", + Usage: "When downloading dependencies attempt to put them in the GOPATH, too.", + }, + cli.BoolFlag{ + Name: "skip-gopath", + Usage: "Skip attempting to copy a dependency from the GOPATH.", }, }, Action: func(c *cli.Context) { @@ -364,7 +382,9 @@ cxt.Put("deleteFlatten", c.Bool("delete-flatten")) cxt.Put("toPath", c.String("file")) cxt.Put("toStdout", false) - cxt.Put("forceCache", c.Bool("cache")) + cxt.Put("useCache", c.Bool("cache")) + cxt.Put("cacheGopath", c.Bool("cache-gopath")) + cxt.Put("skipGopath", c.Bool("skip-gopath")) if c.Bool("import") { cxt.Put("importGodeps", true) cxt.Put("importGPM", true) @@ -469,12 +489,16 @@ Using("conf").From("cxt:cfg"). Using("insecure").From("cxt:insecure"). Using("home").From("cxt:home"). - Using("cache").From("cxt:forceCache"). + Using("cache").From("cxt:useCache"). + Using("cacheGopath").From("cxt:cacheGopath"). + Using("skipGopath").From("cxt:skipGopath"). Does(cmd.Flatten, "flatten").Using("conf").From("cxt:cfg"). Using("packages").From("cxt:packages"). Using("force").From("cxt:forceUpdate"). Using("home").From("cxt:home"). - Using("cache").From("cxt:forceCache"). + Using("cache").From("cxt:useCache"). + Using("cacheGopath").From("cxt:cacheGopath"). + Using("skipGopath").From("cxt:skipGopath"). Does(cmd.WriteYaml, "out"). Using("conf").From("cxt:cfg"). Using("filename").WithDefault("glide.yaml").From("cxt:yaml") @@ -502,14 +526,18 @@ Using("force").From("cxt:forceUpdate"). Using("packages").From("cxt:packages"). Using("home").From("cxt:home"). - Using("cache").From("cxt:forceCache"). + Using("cache").From("cxt:useCache"). + Using("cacheGopath").From("cxt:cacheGopath"). + Using("skipGopath").From("cxt:skipGopath"). Does(cmd.SetReference, "version").Using("conf").From("cxt:cfg"). Does(cmd.Flatten, "flattened").Using("conf").From("cxt:cfg"). Using("packages").From("cxt:packages"). Using("force").From("cxt:forceUpdate"). Using("skip").From("cxt:skipFlatten"). Using("home").From("cxt:home"). - Using("cache").From("cxt:forceCache"). + Using("cache").From("cxt:useCache"). + Using("cacheGopath").From("cxt:cacheGopath"). + Using("skipGopath").From("cxt:skipGopath"). Does(cmd.VendoredCleanUp, "_"). Using("conf").From("cxt:cfg"). Using("update").From("cxt:updateVendoredDeps").
diff --git a/glide.yaml b/glide.yaml index 05144a9..55f6078 100644 --- a/glide.yaml +++ b/glide.yaml
@@ -1,9 +1,6 @@ package: github.com/Masterminds/glide import: - - package: github.com/kylelemons/go-gypsy - subpackages: - - yaml - flatten: true + - package: gopkg.in/yaml.v2 - package: github.com/Masterminds/cookoo version: master repo: git@github.com:Masterminds/cookoo.git