Merge pull request #240 from Masterminds/feat/resolve-imports
Resolve the imports being used
diff --git a/action/list.go b/action/list.go
index 067e3ca..a2d35c7 100644
--- a/action/list.go
+++ b/action/list.go
@@ -32,13 +32,14 @@
msg.Die("Error listing dependencies: %s", err)
}
+ msg.Info("Sorting...")
sort.Strings(sortable)
msg.Puts("INSTALLED packages:")
for _, k := range sortable {
v, err := filepath.Rel(basedir, k)
if err != nil {
- msg.Warn("Failed to Rel path: %s", err)
+ //msg.Warn("Failed to Rel path: %s", err)
v = k
}
msg.Puts("\t%s", v)
diff --git a/dependency/resolver.go b/dependency/resolver.go
index e08f738..e76f358 100644
--- a/dependency/resolver.go
+++ b/dependency/resolver.go
@@ -2,6 +2,7 @@
import (
"container/list"
+ //"go/build"
"os"
"path/filepath"
"strings"
@@ -115,15 +116,21 @@
type Resolver struct {
Handler MissingPackageHandler
VersionHandler VersionHandler
- basedir string
VendorDir string
BuildContext *util.BuildCtxt
- seen map[string]bool
Config *cfg.Config
+ // ResolveAllFiles toggles deep scanning.
+ // If this is true, resolve by scanning all files, not by walking the
+ // import tree.
+ ResolveAllFiles bool
+
// Items already in the queue.
alreadyQ map[string]bool
+ basedir string
+ seen map[string]bool
+
// findCache caches hits from Find. This reduces the number of filesystem
// touches that have to be done for dependency resolution.
findCache map[string]*PkgInfo
@@ -185,7 +192,12 @@
//msg.Debug("Scanning %s", target)
l := list.New()
l.PushBack(target)
- return r.resolveList(l)
+
+ // In this mode, walk the entire tree.
+ if r.ResolveAllFiles {
+ return r.resolveList(l)
+ }
+ return r.resolveImports(l)
}
// ResolveLocal resolves dependencies for the current project.
@@ -251,7 +263,10 @@
}
if deep {
- return r.resolveList(l)
+ if r.ResolveAllFiles {
+ return r.resolveList(l)
+ }
+ return r.resolveImports(l)
}
// If we're not doing a deep scan, we just convert the list into an
@@ -277,16 +292,164 @@
// an error is returned.
func (r *Resolver) ResolveAll(deps []*cfg.Dependency) ([]string, error) {
queue := sliceToQueue(deps, r.VendorDir)
- return r.resolveList(queue)
+
+ loc, err := r.ResolveLocal(false)
+ if err != nil {
+ return []string{}, err
+ }
+ for _, l := range loc {
+ msg.Debug("Adding local mport %s to queue", l)
+ queue.PushBack(l)
+ }
+
+ if r.ResolveAllFiles {
+ return r.resolveList(queue)
+ }
+ return r.resolveImports(queue)
+}
+
+// stripv strips the vendor/ prefix from vendored packages.
+func (r *Resolver) stripv(str string) string {
+ return strings.TrimPrefix(str, r.VendorDir+string(os.PathSeparator))
+}
+
+// vpath adds an absolute vendor path.
+func (r *Resolver) vpath(str string) string {
+ return filepath.Join(r.basedir, "vendor", str)
+}
+
+// resolveImports takes a list of existing packages and resolves their imports.
+//
+// It returns a list of all of the packages that it can determine are required
+// for the given code to function.
+//
+// The expectation is that each item in the queue is an absolute path to a
+// vendored package. This attempts to read that package, and then find
+// its referenced packages. Those packages are then added to the list
+// to be scanned next.
+//
+// The resolver's handler is used in the cases where a package cannot be
+// located.
+func (r *Resolver) resolveImports(queue *list.List) ([]string, error) {
+ for e := queue.Front(); e != nil; e = e.Next() {
+ vdep := e.Value.(string)
+ dep := r.stripv(vdep)
+
+ r.alreadyQ[dep] = true
+
+ // Skip ignored packages
+ if r.Config.HasIgnore(dep) {
+ msg.Info("Ignoring: %s", dep)
+ continue
+ }
+ r.VersionHandler.Process(dep)
+
+ // Here, we want to import the package and see what imports it has.
+ msg.Debug("Trying to open %s", vdep)
+ pkg, err := r.BuildContext.ImportDir(vdep, 0)
+ if err != nil {
+ msg.Warn("ImportDir error on %s: %s", vdep, err)
+ if strings.HasPrefix(err.Error(), "no buildable Go source") {
+ msg.Info("No subpackages declared. Skipping %s.", dep)
+ continue
+ }
+ if ok, err := r.Handler.NotFound(dep); ok {
+ r.alreadyQ[dep] = true
+ queue.PushBack(r.vpath(dep))
+ r.VersionHandler.SetVersion(dep)
+ } else if err != nil {
+ msg.Warn("Error looking for %s: %s", dep, err)
+ } else {
+ // TODO (mpb): Should we toss this into a Handler to
+ // see if this is on GOPATH and copy it?
+ msg.Info("Not found in vendor/: %s (1)", dep)
+ }
+ continue
+ }
+
+ // Range over all of the identified imports and see which ones we
+ // can locate.
+ for _, imp := range pkg.Imports {
+ pi := r.FindPkg(imp)
+ if pi.Loc != LocCgo && pi.Loc != LocGoroot {
+ msg.Debug("Package %s imports %s", dep, imp)
+ }
+ switch pi.Loc {
+ case LocVendor:
+ msg.Debug("In vendor: %s", imp)
+ if _, ok := r.alreadyQ[imp]; !ok {
+ msg.Debug("Marking %s to be scanned.", imp)
+ r.alreadyQ[imp] = true
+ queue.PushBack(r.vpath(imp))
+ r.VersionHandler.SetVersion(imp)
+ }
+ case LocUnknown:
+ msg.Debug("Missing %s. Trying to resolve.", imp)
+ if ok, err := r.Handler.NotFound(imp); ok {
+ r.alreadyQ[imp] = true
+ queue.PushBack(r.vpath(imp))
+ r.VersionHandler.SetVersion(imp)
+ } else if err != nil {
+ msg.Warn("Error looking for %s: %s", imp, err)
+ } else {
+ msg.Info("Not found: %s (2)", imp)
+ }
+ case LocGopath:
+ msg.Info("Found on GOPATH, not vendor: %s", imp)
+ if _, ok := r.alreadyQ[imp]; !ok {
+ // Only scan it if it gets moved into vendor/
+ if ok, _ := r.Handler.OnGopath(imp); ok {
+ r.alreadyQ[imp] = true
+ queue.PushBack(r.vpath(imp))
+ r.VersionHandler.SetVersion(imp)
+ }
+ }
+ }
+ }
+
+ }
+
+ // FIXME: From here to the end is a straight copy of the resolveList() func.
+ res := make([]string, 0, queue.Len())
+
+ // In addition to generating a list
+ for e := queue.Front(); e != nil; e = e.Next() {
+ t := r.stripv(e.Value.(string))
+ root, sp := util.NormalizeName(t)
+
+ // TODO(mattfarina): Need to eventually support devImport
+ existing := r.Config.Imports.Get(root)
+ if existing != nil {
+ if sp != "" && !existing.HasSubpackage(sp) {
+ existing.Subpackages = append(existing.Subpackages, sp)
+ }
+ } else {
+ newDep := &cfg.Dependency{
+ Name: root,
+ }
+ if sp != "" {
+ newDep.Subpackages = []string{sp}
+ }
+
+ r.Config.Imports = append(r.Config.Imports, newDep)
+ }
+ res = append(res, t)
+ }
+
+ return res, nil
}
// resolveList takes a list and resolves it.
+//
+// This walks the entire file tree for the given dependencies, not just the
+// parts that are imported directly. Using this will discover dependencies
+// regardless of OS, and arch.
func (r *Resolver) resolveList(queue *list.List) ([]string, error) {
var failedDep string
for e := queue.Front(); e != nil; e = e.Next() {
dep := e.Value.(string)
- t := strings.TrimPrefix(e.Value.(string), r.VendorDir+string(os.PathSeparator))
+ t := strings.TrimPrefix(dep, r.VendorDir+string(os.PathSeparator))
if r.Config.HasIgnore(t) {
msg.Info("Ignoring: %s", t)
continue
diff --git a/dependency/resolver_test.go b/dependency/resolver_test.go
index 0edb764..fa4ab6e 100644
--- a/dependency/resolver_test.go
+++ b/dependency/resolver_test.go
@@ -72,6 +72,10 @@
if len(l) != 1 {
t.Errorf("Expected 1 dep, got %d: %s", len(l), l[0])
}
+
+ if !strings.HasSuffix("github.com/codegangsta/cli", l[0]) {
+ t.Errorf("Unexpected package name: %s", l[0])
+ }
}
func TestResolveAll(t *testing.T) {
diff --git a/glide.go b/glide.go
index eb3b61b..714da14 100644
--- a/glide.go
+++ b/glide.go
@@ -183,6 +183,10 @@
Usage: "If there was a change in the repo or VCS switch to new one. Warning, changes will be lost.",
},
cli.BoolFlag{
+ Name: "all-dependencies",
+ Usage: "This will resolve all dependencies for all packages, not just those directly used.",
+ },
+ cli.BoolFlag{
Name: "update-vendored, u",
Usage: "Update vendored packages (without local VCS repo). Warning, changes will be lost.",
},
@@ -206,11 +210,12 @@
}
inst := &repo.Installer{
- Force: c.Bool("force"),
- UseCache: c.Bool("cache"),
- UseGopath: c.Bool("use-gopath"),
- UseCacheGopath: c.Bool("cache-gopath"),
- UpdateVendored: c.Bool("update-vendored"),
+ Force: c.Bool("force"),
+ UseCache: c.Bool("cache"),
+ UseGopath: c.Bool("use-gopath"),
+ UseCacheGopath: c.Bool("cache-gopath"),
+ UpdateVendored: c.Bool("update-vendored"),
+ ResolveAllFiles: c.Bool("all-dependencies"),
}
packages := []string(c.Args())
insecure := c.Bool("insecure")
@@ -437,6 +442,10 @@
Usage: "If there was a change in the repo or VCS switch to new one. Warning, changes will be lost.",
},
cli.BoolFlag{
+ Name: "all-dependencies",
+ Usage: "This will resolve all dependencies for all packages, not just those directly used.",
+ },
+ cli.BoolFlag{
Name: "update-vendored, u",
Usage: "Update vendored packages (without local VCS repo). Warning, changes will be lost.",
},
@@ -459,13 +468,14 @@
},
Action: func(c *cli.Context) {
installer := &repo.Installer{
- DeleteUnused: c.Bool("deleteOptIn"),
- UpdateVendored: c.Bool("update-vendored"),
- Force: c.Bool("force"),
- UseCache: c.Bool("cache"),
- UseCacheGopath: c.Bool("cache-gopath"),
- UseGopath: c.Bool("use-gopath"),
- Home: gpath.Home(),
+ DeleteUnused: c.Bool("deleteOptIn"),
+ UpdateVendored: c.Bool("update-vendored"),
+ ResolveAllFiles: c.Bool("all-dependencies"),
+ Force: c.Bool("force"),
+ UseCache: c.Bool("cache"),
+ UseCacheGopath: c.Bool("cache-gopath"),
+ UseGopath: c.Bool("use-gopath"),
+ Home: gpath.Home(),
}
action.Update(installer, c.Bool("no-recursive"))
@@ -486,7 +496,7 @@
},
{
Name: "list",
- Usage: "List prints all dependencies that Glide could discover.",
+ Usage: "List prints all dependencies that the present code references.",
Description: `List scans your code and lists all of the packages that are used.
It does not use the glide.yaml. Instead, it inspects the code to determine what packages are
diff --git a/repo/installer.go b/repo/installer.go
index 645d490..305f870 100644
--- a/repo/installer.go
+++ b/repo/installer.go
@@ -47,6 +47,11 @@
// imported pacakgage references this pacakage it does not need to be
// downloaded and searched out again.
RootPackage string
+
+ // ResolveAllFiles enables a resolver that will examine the dependencies
+ // of every file of every package, rather than only following imported
+ // packages.
+ ResolveAllFiles bool
}
// VendorPath returns the path to the location to put vendor packages
@@ -173,13 +178,16 @@
res.Config = conf
res.Handler = m
res.VersionHandler = v
+ res.ResolveAllFiles = i.ResolveAllFiles
msg.Info("Resolving imports")
_, err = allPackages(conf.Imports, res)
if err != nil {
msg.Die("Failed to retrieve a list of dependencies: %s", err)
}
- msg.Warn("devImports not resolved.")
+ if len(conf.DevImports) > 0 {
+ msg.Warn("dev imports not resolved.")
+ }
err = ConcurrentUpdate(conf.Imports, vpath, i)
@@ -207,6 +215,7 @@
}
res.Config = conf
res.VersionHandler = v
+ res.ResolveAllFiles = i.ResolveAllFiles
msg.Info("Resolving imports")
_, err = allPackages(conf.Imports, res)
@@ -214,7 +223,9 @@
msg.Die("Failed to retrieve a list of dependencies: %s", err)
}
- msg.Warn("devImports not resolved.")
+ if len(conf.DevImports) > 0 {
+ msg.Warn("dev imports not resolved.")
+ }
return conf.Imports
}
@@ -227,6 +238,8 @@
var lock sync.Mutex
var returnErr error
+ msg.Info("Downloading dependencies. Please wait...")
+
for ii := 0; ii < concurrentWorkers; ii++ {
go func(ch <-chan *cfg.Dependency) {
for {
@@ -316,10 +329,11 @@
// This package may have been placed on the list to look for when it wasn't
// downloaded but it has since been downloaded before coming to this entry.
if _, err := os.Stat(dest); err == nil {
+ msg.Debug("Found %s", dest)
return true, nil
}
- msg.Info("Fetching %s into %s", pkg, m.destination)
+ msg.Info("- Fetching %s into %s", pkg, m.destination)
d := m.Config.Imports.Get(root)
// If the dependency is nil it means the Config doesn't yet know about it.
diff --git a/repo/vcs.go b/repo/vcs.go
index 091b91c..6a3421c 100644
--- a/repo/vcs.go
+++ b/repo/vcs.go
@@ -30,7 +30,7 @@
return nil
}
- msg.Info("Fetching updates for %s.\n", dep.Name)
+ msg.Info("- Fetching updates for %s.\n", dep.Name)
if filterArchOs(dep) {
msg.Info("%s is not used for %s/%s.\n", dep.Name, runtime.GOOS, runtime.GOARCH)
