Working on faster flattening.
diff --git a/cmd/flatten.go b/cmd/flatten.go index bb0f055..f49d3b5 100644 --- a/cmd/flatten.go +++ b/cmd/flatten.go
@@ -4,11 +4,14 @@ "io/ioutil" "os" "path" + "path/filepath" "strings" "github.com/Masterminds/cookoo" "github.com/Masterminds/glide/cfg" - "github.com/Masterminds/glide/util" + "github.com/Masterminds/glide/dependency" + "github.com/Masterminds/glide/msg" + //"github.com/Masterminds/glide/util" "github.com/Masterminds/semver" ) @@ -56,10 +59,17 @@ f := &flattening{conf, vend, vend, deps, packages} + pkgs, err := findAllProjects(f, strings.TrimSuffix(vend, "/vendor")) + if err != nil { + return conf, err + } + conf.Imports = pkgs + return conf, nil + // The assumption here is that once something has been scanned once in a // run, there is no need to scan it again. scanned := map[string]bool{} - err := recFlatten(f, force, home, cache, cacheGopath, skipGopath, scanned) + err = recFlatten(f, force, home, cache, cacheGopath, skipGopath, scanned) if err != nil { return conf, err } @@ -143,6 +153,40 @@ return nil } +// Get a list of all projects. +func findAllProjects(f *flattening, top string) ([]*cfg.Dependency, error) { + + seen := map[string]bool{} + for _, imp := range f.conf.Imports { + seen[imp.Name] = true + } + + res, err := dependency.NewResolver(top) + if err != nil { + return []*cfg.Dependency{}, err + } + + prjs, err := res.ResolveAll(f.conf.Imports) + if err != nil { + return []*cfg.Dependency{}, err + } + + out := make([]*cfg.Dependency, len(f.conf.Imports)) + copy(out, f.conf.Imports) + for _, p := range prjs { + p, err = filepath.Rel(f.top, p) + if err != nil { + Warn("Rel: %s", err) + } + + p, _ = NormalizeName(p) + if !seen[p] { + out = append(out, &cfg.Dependency{Name: p}) + } + } + return out, nil +} + // flattenGlideUp does a glide update in the middle of a flatten operation. // // While this is expensive, it is also necessary to make sure we have the @@ -264,65 +308,107 @@ // This always returns true because it always handles the job of searching // for dependencies. So generally it should be the last merge strategy // that you try. -func mergeGuess(dir, pkg string, deps map[string]*cfg.Dependency, vend string, scanned map[string]bool) ([]string, bool) { - Info("Scanning %s for dependencies.", pkg) - buildContext, err := GetBuildContext() +func mergeGuess(fullpath, pkg string, deps map[string]*cfg.Dependency, vend string, scanned map[string]bool) ([]string, bool) { + if scanned[pkg] { + return []string{}, true + } + scanned[pkg] = true + + vendor := strings.TrimSuffix(fullpath, pkg) + Info("Scanning %s for dependencies in %s.", pkg, vendor) + resolver, err := dependency.NewResolver(strings.TrimSuffix(vendor, "vendor/")) if err != nil { Warn("Could not scan package %q: %s", pkg, err) return []string{}, false } - res := []string{} - - if _, err := os.Stat(dir); err != nil { - Warn("Directory is missing: %s", dir) - return res, true + resolver.Handler = &InstallMissingPackagesHandler{ + Vendor: vendor, + Home: vendor, + UseCache: false, + UseCacheGopath: false, + SkipGopath: true, } - d := walkDeps(buildContext, dir, pkg) - for _, oname := range d { - if _, ok := scanned[oname]; ok { - //Info("===> Scanned %s already. Skipping", name) + resolved, err := resolver.ResolveAll([]*cfg.Dependency{&cfg.Dependency{Name: pkg}}) + if err != nil { + msg.Error("Failed to resolve %s: %s", pkg, err) + } + + cp := []string{} // := make([]string, 0, len(resolved)) + for _, d := range resolved { + d, err = filepath.Rel(vendor, d) + if err != nil { + msg.Warn("Failed to get relative path of %s", d) + } else if d == "." || d == "" { continue } - Debug("=> Scanning %s", oname) - name, _ := NormalizeName(oname) - //if _, ok := deps[name]; ok { - //scanned[oname] = true - //Debug("====> Seen %s already. Skipping", name) - //continue - //} + d, _ = NormalizeName(d) - found := findPkg(buildContext, name, dir) - switch found.PType { - case ptypeUnknown: - Info("==> Unknown %s (%s)", name, oname) - Debug("✨☆ Undownloaded dependency: %s", name) - repo := util.GetRootFromPackage(name) - nd := &cfg.Dependency{ - Name: name, - Repository: "https://" + repo, - } - deps[name] = nd - res = append(res, name) - case ptypeGoroot, ptypeCgo: - scanned[oname] = true - // Why do we break rather than continue? - break - default: - // We're looking for dependencies that might exist in $GOPATH - // but not be on vendor. We add any that are on $GOPATH. - if _, ok := deps[name]; !ok { - Debug("✨☆ GOPATH dependency: %s", name) - nd := &cfg.Dependency{Name: name} - deps[name] = nd - res = append(res, name) - } - scanned[oname] = true + // Just in case self comes up here, we skip it. + if d == pkg || d == "" { + continue } + cp = append(cp, d) + //scanned[d] = true } - return res, true + msg.Info("Returning %v (len: %d)", cp, len(cp)) + return cp, true + + /* + res := []string{} + + if _, err := os.Stat(dir); err != nil { + Warn("Directory is missing: %s", dir) + return res, true + } + + d := walkDeps(buildContext, dir, pkg) + for _, oname := range d { + if _, ok := scanned[oname]; ok { + //Info("===> Scanned %s already. Skipping", name) + continue + } + Debug("=> Scanning %s", oname) + name, _ := NormalizeName(oname) + //if _, ok := deps[name]; ok { + //scanned[oname] = true + //Debug("====> Seen %s already. Skipping", name) + //continue + //} + + found := findPkg(buildContext, name, dir) + switch found.PType { + case ptypeUnknown: + Info("==> Unknown %s (%s)", name, oname) + Debug("✨☆ Undownloaded dependency: %s", name) + repo := util.GetRootFromPackage(name) + nd := &cfg.Dependency{ + Name: name, + Repository: "https://" + repo, + } + deps[name] = nd + res = append(res, name) + case ptypeGoroot, ptypeCgo: + scanned[oname] = true + // Why do we break rather than continue? + break + default: + // We're looking for dependencies that might exist in $GOPATH + // but not be on vendor. We add any that are on $GOPATH. + if _, ok := deps[name]; !ok { + Debug("✨☆ GOPATH dependency: %s", name) + nd := &cfg.Dependency{Name: name} + deps[name] = nd + res = append(res, name) + } + scanned[oname] = true + } + } + + return res, true + */ } // mergeDeps merges any dependency array into deps. @@ -453,3 +539,23 @@ } return mod } + +// InstallMissingPackagesHandler is a missing package handler that installs missing packages. +type InstallMissingPackagesHandler struct { + Vendor, Home string + UseCache, UseCacheGopath, SkipGopath bool +} + +func (i *InstallMissingPackagesHandler) NotFound(pkg string) (bool, error) { + d := &cfg.Dependency{Name: pkg} + dest := filepath.Join(i.Vendor, pkg) + msg.Info("Cloning %s into %s", pkg, dest) + if err := VcsGet(d, dest, i.Home, i.UseCache, i.UseCacheGopath, i.SkipGopath); err != nil { + return false, err + } + return true, nil +} + +func (i *InstallMissingPackagesHandler) OnGopath(pkg string) (bool, error) { + return i.NotFound(pkg) +}
diff --git a/cmd/get_imports.go b/cmd/get_imports.go index ff0d810..590ebae 100644 --- a/cmd/get_imports.go +++ b/cmd/get_imports.go
@@ -254,6 +254,11 @@ // VcsGet figures out how to fetch a dependency, and then gets it. // // VcsGet installs into the dest. +// +// dep is the dependency to try to install. dest is the destination. home is the +// (what?). cache indicates whether to store in a local cache. cacheGopath indicates +// whether to store a copy in Gopath, and skipGopath prevents copying from Gopath +// into the present project. func VcsGet(dep *cfg.Dependency, dest, home string, cache, cacheGopath, skipGopath bool) error { // When not skipping the $GOPATH look in it for a copy of the package if !skipGopath {
diff --git a/cmd/update_references.go b/cmd/update_references.go index 92fe235..176d972 100644 --- a/cmd/update_references.go +++ b/cmd/update_references.go
@@ -1,10 +1,11 @@ package cmd import ( - "path" + "path/filepath" "github.com/Masterminds/cookoo" "github.com/Masterminds/glide/cfg" + "github.com/Masterminds/glide/dependency" ) // UpdateReferences updates the revision numbers on all of the imports. @@ -70,35 +71,76 @@ func discoverDependencyTree(f *flattening) error { Debug("---> Inspecting %s for dependencies (%d packages).\n", f.curr, len(f.scan)) - scanned := map[string]bool{} - for _, imp := range f.scan { - Debug("----> Scanning %s", imp) - base := path.Join(f.top, imp) - mod := []string{} - if m, ok := mergeGlide(base, imp, f.deps, f.top); ok { - mod = m - } else if m, ok = mergeGodep(base, imp, f.deps, f.top); ok { - mod = m - } else if m, ok = mergeGPM(base, imp, f.deps, f.top); ok { - mod = m - } else if m, ok = mergeGb(base, imp, f.deps, f.top); ok { - mod = m - } else if m, ok = mergeGuess(base, imp, f.deps, f.top, scanned); ok { - mod = m - } - if len(mod) > 0 { - Debug("----> Looking for dependencies in %q (%d)", imp, len(mod)) - f2 := &flattening{ - conf: f.conf, - top: f.top, - curr: base, - deps: f.deps, - scan: mod} - discoverDependencyTree(f2) + // projects tracks which projects we have seen. Initially, it's everytning + // in f.deps. As we go, we'll merge in all of the others that we find. + projects := f.deps + + // Get all of the packages that are used. + resolver, err := dependency.NewResolver(f.top) + if err != nil { + return err + } + + dlist := make([]*cfg.Dependency, 0, len(f.deps)) + for _, d := range f.deps { + dlist = append(dlist, d) + } + pkgs, err := resolver.ResolveAll(dlist) + if err != nil { + return err + } + + // From the packages, we just want the repositories. So we get a normalized + // list of dependencies. + for _, d := range pkgs { + d, err = filepath.Rel(f.top, d) + if err != nil { + Warn("Cannot resolve relative path: %s", err) + } + d, _ := NormalizeName(d) + + if _, ok := projects[d]; !ok { + projects[d] = &cfg.Dependency{Name: d} + Info("====> %s", d) } } + // At this point, we know that we have an exhaustive list of packages, so + // we can now just look for files that will tell us what version of each + // package to use. + + /* + scanned := map[string]bool{} + for _, imp := range f.scan { + Debug("----> Scanning %s", imp) + base := path.Join(f.top, imp) + mod := []string{} + if m, ok := mergeGlide(base, imp, f.deps, f.top); ok { + mod = m + } else if m, ok = mergeGodep(base, imp, f.deps, f.top); ok { + mod = m + } else if m, ok = mergeGPM(base, imp, f.deps, f.top); ok { + mod = m + } else if m, ok = mergeGb(base, imp, f.deps, f.top); ok { + mod = m + } else if m, ok = mergeGuess(base, imp, f.deps, f.top, scanned); ok { + mod = m + } + + if len(mod) > 0 { + Debug("----> Looking for dependencies in %q (%d)", imp, len(mod)) + f2 := &flattening{ + conf: f.conf, + top: f.top, + curr: base, + deps: f.deps, + scan: mod} + discoverDependencyTree(f2) + } + } + */ + return nil }
diff --git a/dependency/resolver.go b/dependency/resolver.go index 943c3aa..120afae 100644 --- a/dependency/resolver.go +++ b/dependency/resolver.go
@@ -137,7 +137,7 @@ // there. func (r *Resolver) Resolve(pkg, basepath string) ([]string, error) { target := filepath.Join(basepath, pkg) - //msg.Debug("Scanning %s", target) + msg.Info("Resolving %s", target) l := list.New() l.PushBack(target) return r.resolveList(l)