Flatten dependency tree by default. Addresses issue #108.
diff --git a/cmd/flatten.go b/cmd/flatten.go new file mode 100644 index 0000000..84ed576 --- /dev/null +++ b/cmd/flatten.go
@@ -0,0 +1,242 @@ +package cmd + +import ( + "os" + "path" + + "github.com/Masterminds/cookoo" + "github.com/kylelemons/go-gypsy/yaml" +) + +// Flatten recurses through all dependent packages and flattens to a top level. +// +// Flattening involves determining a tree's dependencies and flattening them +// into a single large list. +// +// Params: +// - packages ([]string): The packages to read. If this is empty, it reads all +// packages. +// - force (bool): force git updates. +// - conf (*Config): The configuration. +// +// Returns: +// +func Flatten(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { + packages := p.Get("packages", []string{}).([]string) + conf := p.Get("conf", &Config{}).(*Config) + force := p.Get("force", true).(bool) + vend, _ := VendorPath(c) + + // If no packages are supplied, we do them all. + if len(packages) == 0 { + packages = make([]string, len(conf.Imports)) + for i, v := range conf.Imports { + packages[i] = v.Name + } + } + + // Build an initial dependency map. + deps := make(map[string]*Dependency, len(conf.Imports)) + for _, imp := range conf.Imports { + deps[imp.Name] = imp + } + + f := &flattening{conf, vend, vend, deps, packages} + + a, err := recFlatten(f, force) + flattenSetRefs(f) + Info("Project relies on %d dependencies.", len(deps)) + return a, err +} + +type flattening struct { + conf *Config + // Top vendor path, e.g. project/vendor + top string + // Current path + curr string + // Built list of dependencies + deps map[string]*Dependency + // Dependencies that need to be scanned. + scan []string +} + +// Hack: Cache record of updates so we don't have to keep doing git pulls. +var flattenUpdateCache = map[string]bool{} + +// refFlatten recursively flattens the vendor tree. +func recFlatten(f *flattening, force bool) (interface{}, error) { + Debug("---> Inspecting %s for changes (%d packages).\n", f.curr, len(f.scan)) + for _, imp := range f.scan { + Debug("----> Scanning %s", imp) + base := path.Join(f.top, imp) + mod := []string{} + if m, ok := mergeGlide(base, imp, f.deps); ok { + mod = m + } else if m, ok = mergeGodep(base, imp, f.deps); ok { + mod = m + } else if m, ok = mergeGPM(base, imp, f.deps); ok { + mod = m + } else if m, ok = mergeGb(base, imp, f.deps); ok { + mod = m + } else if m, ok = mergeGuess(base, imp, f.deps); ok { + mod = m + } + + if len(mod) > 0 { + Debug("----> Updating all dependencies for %q (%d)", imp, len(mod)) + flattenGlideUp(f, base, force) + f2 := &flattening{ + conf: f.conf, + top: f.top, + curr: base, + deps: f.deps, + scan: mod} + recFlatten(f2, force) + } + } + + // Stopped: Need to recurse down the next level. + return nil, nil +} + +// flattenGlideUp does a glide update in the middle of a flatten operation. +// +// While this is expensive, it is also necessary to make sure we have the +// correct version of all dependencies. We might be able to simplify by +// marking packages dirty when they are added. +func flattenGlideUp(f *flattening, base string, force bool) error { + //vdir := path.Join(base, "vendor") + for _, imp := range f.deps { + wd := path.Join(f.top, imp.Name) + if VcsExists(imp, wd) { + if flattenUpdateCache[imp.Name] { + Debug("----> Already updated %s", imp.Name) + continue + } + Debug("Updating project %s (%s)\n", imp.Name, wd) + if err := VcsUpdate(imp, f.top, force); err != nil { + // We can still go on just fine even if this fails. + Warn("Skipped update %s: %s\n", imp.Name, err) + continue + } + flattenUpdateCache[imp.Name] = true + } else { + Debug("Importing %s to project %s\n", imp.Name, wd) + if err := VcsGet(imp, wd); err != nil { + Warn("Skipped getting %s: %v\n", imp.Name, err) + continue + } + } + + // If a revision has been set use it. + err := VcsVersion(imp, f.top) + if err != nil { + Warn("Problem setting version on %s: %s\n", imp.Name, err) + } + } + + return nil +} + +// Set the references for all packages after a flatten is completed. +func flattenSetRefs(f *flattening) { + Debug("Setting final version for %d dependencies.", len(f.deps)) + for _, imp := range f.deps { + if err := VcsVersion(imp, f.top); err != nil { + Warn("Problem setting version on %s: %s (flatten)\n", imp.Name, err) + } + } +} + +func mergeGlide(dir, name string, deps map[string]*Dependency) ([]string, bool) { + gp := path.Join(dir, "glide.yaml") + if _, err := os.Stat(gp); err != nil { + return []string{}, false + } + f, err := yaml.ReadFile(gp) + if err != nil { + Warn("Found glide file %q, but can't parse: %s", gp, err) + return []string{}, false + } + + conf, err := FromYaml(f.Root) + if err != nil { + Warn("Found glide file %q, but can't use it: %s", gp, err) + return []string{}, false + } + + Info("Found glide.yaml in %s", gp) + + return mergeDeps(deps, conf.Imports), true +} + +// listGodep appends Godeps entries to the deps. +// +// It returns true if any dependencies were found (even if not added because +// they are duplicates). +func mergeGodep(dir, name string, deps map[string]*Dependency) ([]string, bool) { + Debug("Looking in %s/Godeps/ for a Godeps.json file.\n", dir) + d, err := parseGodepGodeps(dir) + if err != nil { + Warn("Looking for Godeps: %s\n", err) + return []string{}, false + } else if len(d) == 0 { + return []string{}, false + } + + Info("Found Godeps.json file for %q", name) + return mergeDeps(deps, d), true +} + +// listGb merges GB dependencies into the deps. +func mergeGb(dir, pkg string, deps map[string]*Dependency) ([]string, bool) { + Debug("Looking in %s/vendor/ for a manifest file.\n", dir) + d, err := parseGbManifest(dir) + if err != nil || len(d) == 0 { + return []string{}, false + } + Info("Found gb manifest file for %q", pkg) + return mergeDeps(deps, d), true +} + +// mergeGPM merges GPM Godeps files into deps. +func mergeGPM(dir, pkg string, deps map[string]*Dependency) ([]string, bool) { + d, err := parseGPMGodeps(dir) + if err != nil || len(d) == 0 { + return []string{}, false + } + Info("Found GPM file for %q", pkg) + return mergeDeps(deps, d), true +} + +// mergeGuess guesses dependencies and merges. +func mergeGuess(dir, pkg string, deps map[string]*Dependency) ([]string, bool) { + Info("%s manages its own dependencies.", pkg) + return []string{}, false +} + +// mergeDeps merges any dependency array into deps. +func mergeDeps(orig map[string]*Dependency, add []*Dependency) []string { + mod := []string{} + for _, dd := range add { + // Add it unless it's already there. + if existing, ok := orig[dd.Name]; !ok { + orig[dd.Name] = dd + Debug("Adding %s to the scan list", dd.Name) + mod = append(mod, dd.Name) + } else if existing.Reference == "" && dd.Reference != "" { + // If a nested dep has finer dependency references than outside, + // set the reference. + existing.Reference = dd.Reference + mod = append(mod, dd.Name) + } else if dd.Reference != "" && existing.Reference != "" && dd.Reference != existing.Reference { + // We can detect version conflicts, but we can't really do + // anything to correct, since we don't know the intentions of the + // authors. + Warn("Conflict: %s ref is %s, but also asked for %s", existing.Name, existing.Reference, dd.Reference) + Info("Keeping %s %s", existing.Name, existing.Reference) + } + } + return mod +}
diff --git a/cmd/get_imports.go b/cmd/get_imports.go index 9e17393..405cdc5 100644 --- a/cmd/get_imports.go +++ b/cmd/get_imports.go
@@ -30,6 +30,9 @@ // GetAll gets zero or more repos. // +// This takes a package name, normalizes it, finds the repo, and installs it. +// It's the workhorse behind `glide get`. +// // Params: // - packages ([]string): Package names to get. // - verbose (bool): default false
diff --git a/cmd/recursive_glide.go b/cmd/recursive_glide.go deleted file mode 100644 index 717249e..0000000 --- a/cmd/recursive_glide.go +++ /dev/null
@@ -1,246 +0,0 @@ -package cmd - -import ( - "github.com/Masterminds/cookoo" - "github.com/kylelemons/go-gypsy/yaml" - "io/ioutil" - "os" - "path" - "strings" -) - -// Recurse does glide installs on dependent packages. -// -// Recurse looks in all known packages for a glide.yaml files and installs for -// each one it finds. -// -// The packages scanned can be restricted (at the top level) by providing -// a list of packages to scan in the `packages` param. -// -// Params: -// - enable (bool) -// - importGodeps (bool) -// - importGPM (bool) -// - importGb (bool) -// - deleteFlatten (bool) -// - force (bool) -// - packages ([]string): Packages to recurse through. If empty, does all of them. -func Recurse(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - if !p.Get("enable", true).(bool) { - return nil, nil - } - force := p.Get("force", true).(bool) - plist := p.Get("packages", []string{}).([]string) - pkgs := list2map(plist) - - godeps, gpm, gb, deleteFlatten := false, false, false, false - if g, ok := p.Has("importGodeps"); ok { - godeps = g.(bool) - } - if g, ok := p.Has("importGPM"); ok { - gpm = g.(bool) - } - if g, ok := p.Has("importGb"); ok { - gb = g.(bool) - } - - if g, ok := p.Has("deleteFlatten"); ok { - deleteFlatten = g.(bool) - } - - Info("Checking dependencies for updates. Godeps: %v, GPM: %v, gb: %v\n", godeps, gpm, gb) - if deleteFlatten == true { - Info("Deleting flattened dependencies enabled\n") - } - conf := p.Get("conf", &Config{}).(*Config) - vend, _ := VendorPath(c) - - return recDepResolve(conf, pkgs, vend, godeps, gpm, gb, force, deleteFlatten) -} - -func recDepResolve(conf *Config, filter map[string]bool, vend string, godeps, gpm, gb, force, deleteFlatten bool) (interface{}, error) { - - Info("Inspecting %s.\n", vend) - - if len(conf.Imports) == 0 { - Info("No imports.\n") - } - - restrict := len(filter) > 0 - - // Look in each package to see whether it has a glide.yaml, and no vendor/ - for _, imp := range conf.Imports { - if restrict && !filter[imp.Name] { - Debug("===> Skipping %q", imp.Name) - continue - } - if imp.Flattened == true { - continue - } - base := path.Join(vend, imp.Name) - Info("Looking in %s for a glide.yaml file.\n", base) - if !needsGlideUp(base) { - if godeps { - importGodep(base, imp.Name) - } - if gpm { - importGPM(base, imp.Name) - } - if gb { - importGb(base, imp.Name) - } - if !needsGlideUp(base) { - Info("Package %s manages its own dependencies.\n", imp.Name) - continue - } - } - - if err := dependencyGlideUp(conf, base, godeps, gpm, gb, force, deleteFlatten); err != nil { - Warn("Failed to update dependency %s: %s", imp.Name, err) - } - } - - return nil, nil -} - -func dependencyGlideUp(parentConf *Config, base string, godep, gpm, gb, force, deleteFlatten bool) error { - Info("Doing a glide in %s\n", base) - fname := path.Join(base, "glide.yaml") - f, err := yaml.ReadFile(fname) - if err != nil { - return err - } - - conf, err := FromYaml(f.Root) - conf.Parent = parentConf - if err != nil { - return err - } - for _, imp := range conf.Imports { - vdir := path.Join(base, "vendor") - wd := path.Join(vdir, imp.Name) - // if our root glide.yaml says to flatten this, we skip it - if dep := conf.GetRoot().Imports.Get(imp.Name); dep != nil { - flatten := conf.GetRoot().Flatten - if flatten == true && dep.Flatten == false || - flatten == false && dep.Flatten == true { - flatten = dep.Flatten - } - if flatten == true { - Info("Skipping importing %s due to flatten being set in root import glide.yaml\n", imp.Name) - imp.Flattened = true - } - - if flatten == true && imp.Reference != dep.Reference { - Warn("Flattened package %s ref (%s) is diferent from sub vendored package ref (%s)\n", imp.Name, imp.Reference, dep.Reference) - } - - if imp.Flattened == true && deleteFlatten == true { - if exists, _ := fileExist(wd); exists == true || true { - remove := wd + string(os.PathSeparator) - Warn("Removing flattened sub vendored package: %s\n", strings.TrimPrefix(remove, base)) - rerr := os.RemoveAll(remove) - if rerr != nil { - return rerr - } - } - } - if imp.Flattened == true { - continue - } - } - - // We don't use the global var to find vendor dir name because the - // user may mis-use that var to modify the local vendor dir, and - // we don't want that to break the embedded vendor dirs. - - if err := ensureDir(wd); err != nil { - Warn("Skipped getting %s (vendor/ error): %s\n", imp.Name, err) - continue - } - - if VcsExists(imp, wd) { - Info("Updating project %s (%s)\n", imp.Name, wd) - if err := VcsUpdate(imp, vdir, force); err != nil { - // We can still go on just fine even if this fails. - Warn("Skipped update %s: %s\n", imp.Name, err) - continue - } - } else { - Info("Importing %s to project %s\n", imp.Name, base) - if err := VcsGet(imp, wd); err != nil { - Warn("Skipped getting %s: %v\n", imp.Name, err) - continue - } - } - - // If a revision has been set use it. - err = VcsVersion(imp, vdir) - if err != nil { - Warn("Problem setting version on %s: %s\n", imp.Name, err) - } - - //recDepResolve(conf, path.Join(wd, "vendor")) - } - // We only filter at the top level. - e := map[string]bool{} - recDepResolve(conf, e, path.Join(base, "vendor"), godep, gpm, gb, force, deleteFlatten) - return nil -} - -func ensureDir(dirpath string) error { - if fi, err := os.Stat(dirpath); err == nil && fi.IsDir() { - return nil - } - return os.MkdirAll(dirpath, 0755) -} - -func needsGlideUp(dir string) bool { - stat, err := os.Stat(path.Join(dir, "glide.yaml")) - if err != nil || stat.IsDir() { - return false - } - - // Should probably see if vendor is there and non-empty. - - return true -} - -func importGodep(dir, pkg string) error { - Info("Looking in %s/Godeps/ for a Godeps.json file.\n", dir) - d, err := parseGodepGodeps(dir) - if err != nil { - Warn("Looking for Godeps: %s\n", err) - return err - } - return quickDirtyYAMLWrite(dir, d, pkg) -} - -func importGPM(dir, pkg string) error { - d, err := parseGPMGodeps(dir) - if err != nil { - return err - } - return quickDirtyYAMLWrite(dir, d, pkg) -} - -func importGb(dir, pkg string) error { - Info("Looking in %s/vendor/ for a manifest file.\n", dir) - d, err := parseGbManifest(dir) - if err != nil { - return err - } - return quickDirtyYAMLWrite(dir, d, pkg) -} - -func quickDirtyYAMLWrite(dir string, d []*Dependency, pkg string) error { - if len(d) == 0 { - return nil - } - c := &Config{Name: pkg, Imports: d} - node := c.ToYaml() - data := yaml.Render(node) - f := path.Join(dir, "glide.yaml") - Info("Writing new glide.yaml file in %s\n", dir) - return ioutil.WriteFile(f, []byte(data), 0755) -}
diff --git a/cmd/yaml.go b/cmd/yaml.go index f0fbbdf..c629e1e 100644 --- a/cmd/yaml.go +++ b/cmd/yaml.go
@@ -432,6 +432,7 @@ UpdateAsVendored bool Flatten bool Flattened bool + dirty bool } // DependencyFromYaml creates a dependency from a yaml.Node.
diff --git a/glide.go b/glide.go index a71c5b5..88f62ee 100644 --- a/glide.go +++ b/glide.go
@@ -416,13 +416,9 @@ Using("packages").From("cxt:packages"). Using("conf").From("cxt:cfg"). Does(cmd.MergeToYaml, "merged").Using("conf").From("cxt:cfg"). - Does(cmd.Recurse, "recurse").Using("conf").From("cxt:cfg"). - Using("enable").From("cxt:recursiveDependencies"). - Using("importGodeps").From("cxt:importGodeps"). - Using("importGPM").From("cxt:importGPM"). - Using("importGb").From("cxt:importGb"). - Using("force").From("cxt:forceUpdate").WithDefault(false). + Does(cmd.Flatten, "flatten").Using("conf").From("cxt:cfg"). Using("packages").From("cxt:packages"). + Using("force").From("cxt:forceUpdate"). Does(cmd.WriteYaml, "out"). Using("yaml.Node").From("cxt:merged"). Using("filename").WithDefault("glide.yaml").From("cxt:yaml") @@ -450,14 +446,9 @@ Using("force").From("cxt:forceUpdate"). Using("packages").From("cxt:packages"). Does(cmd.SetReference, "version").Using("conf").From("cxt:cfg"). - Does(cmd.Recurse, "recurse").Using("conf").From("cxt:cfg"). - Using("deleteFlatten").From("cxt:deleteFlatten"). - Using("importGodeps").From("cxt:importGodeps"). - Using("importGPM").From("cxt:importGPM"). - Using("importGb").From("cxt:importGb"). - Using("enable").From("cxt:recursiveDependencies"). - Using("force").From("cxt:forceUpdate"). + Does(cmd.Flatten, "flatten").Using("conf").From("cxt:cfg"). Using("packages").From("cxt:packages"). + Using("force").From("cxt:forceUpdate"). Does(cmd.VendoredCleanUp, "_"). Using("conf").From("cxt:cfg"). Using("update").From("cxt:updateVendoredDeps")