First pass at new dependency resolver.
diff --git a/dependency/resolver.go b/dependency/resolver.go new file mode 100644 index 0000000..dda96b9 --- /dev/null +++ b/dependency/resolver.go
@@ -0,0 +1,444 @@ +package dependency + +import ( + "container/list" + "go/build" + "os" + "path/filepath" + "strings" + + "github.com/Masterminds/glide/msg" + "github.com/Masterminds/glide/yaml" +) + +// MissingPackageHandler handles the case where a package is missing during scanning. +// +// It returns true if the package can be passed to the resolver, false otherwise. +// False may be returned even if error is nil. +type MissingPackageHandler interface { + // NotFound is called when the Resolver fails to find a package with the given name. + // + // NotFound returns true when the resolver should attempt to re-resole the + // dependency (e.g. when NotFound has gone and fetched the missing package). + // + // When NotFound returns false, the Resolver does not try to do any additional + // work on the missing package. + // + // NotFound only returns errors when it fails to perform its internal goals. + // When it returns false with no error, this indicates that the handler did + // its job, but the resolver should not do any additional work on the + // package. + NotFound(pkg string) (bool, error) + + // OnGopath is called when the Resolver finds a dependency, but it's only on GOPATH. + // + // OnGopath provides an opportunity to copy, move, warn, or ignore cases like this. + // + // OnGopath returns true when the resolver should attempt to re-resolve the + // dependency (e.g. when the dependency is copied to a new location). + // + // When OnGopath returns false, the Resolver does not try to do any additional + // work on the package. + // + // An error indicates that OnGopath cannot complete its intended operation. + // Not all false results are errors. + OnGopath(pkg string) (bool, error) +} + +// DefaultMissingPackageHandler is the default handler for missing packages. +// +// When asked to handle a missing package, it will report the miss as a warning, +// and then store the package in the Missing slice for later access. +type DefaultMissingPackageHandler struct { + Missing []string + Gopath []string +} + +// NotFound prints a warning and then stores the package name in Missing. +// +// It never returns an error, and it always returns false. +func (d *DefaultMissingPackageHandler) NotFound(pkg string) (bool, error) { + msg.Warn("Package %s is not installed", pkg) + d.Missing = append(d.Missing, pkg) + return false, nil +} + +func (d *DefaultMissingPackageHandler) OnGopath(pkg string) (bool, error) { + msg.Warn("Package %s is only on GOPATH.", pkg) + d.Gopath = append(d.Gopath, pkg) + return false, nil +} + +// Resolver resolves a dependency tree. +type Resolver struct { + Handler MissingPackageHandler + basedir string + VendorDir string + BuildContext build.Context + seen map[string]bool + + // Items already in the queue. + alreadyQ map[string]bool + + // findCache caches hits from Find. This reduces the number of filesystem + // touches that have to be done for dependency resolution. + findCache map[string]*PkgInfo +} + +// NewResolver returns a new Resolver initialized with the DefaultMissingPackageHandler. +// +// This will return an error if the given path does not meet the basic criteria +// for a Go source project. For example, basedir must have a vendor subdirectory. +// +// The BuildContext uses the "go/build".Default to resolve dependencies. +func NewResolver(basedir string) (*Resolver, error) { + + var err error + basedir, err = filepath.Abs(basedir) + if err != nil { + return nil, err + } + vdir := filepath.Join(basedir, "vendor") + + r := &Resolver{ + Handler: &DefaultMissingPackageHandler{Missing: []string{}}, + basedir: basedir, + VendorDir: vdir, + BuildContext: build.Default, + seen: map[string]bool{}, + alreadyQ: map[string]bool{}, + findCache: map[string]*PkgInfo{}, + } + + // TODO: Make sure the build context is correctly set up. Especially in + // regards to GOROOT, which is not always set. + + return r, nil +} + +// Resolve takes a package name and returns all of the imported package names. +// +// If a package is not found, this calls the Fetcher. If the Fetcher returns +// true, it will re-try traversing that package for dependencies. Otherwise it +// will add that package to the deps array and continue on without trying it. +// And if the Fetcher returns an error, this will stop resolution and return +// the error. +func (r *Resolver) Resolve(pkg string) ([]string, error) { + deps := []string{} + + // Get the path to the package. + + // Walk the package and collect all of its dependencies. + // filepath.Walk( + + //p, err := r.BuildContext.ImportDir(pkg, 0) + + return deps, nil +} + +// ResolveAll takes a list of packages and returns an inclusive list of all +// vendored dependencies. +// +// While this will scan all of the source code it can find, it will only return +// packages that were either explicitly passed in as deps, or were explicitly +// imported by the code. +// +// Packages that are either CGO or on GOROOT are ignored. Packages that are +// on GOPATH, but not vendored currently generate a warning. +// +// If one of the passed in packages does not exist in the vendor directory, +// an error is returned. +func (r *Resolver) ResolveAll(deps []*yaml.Dependency) ([]string, error) { + queue := sliceToQueue(deps, r.VendorDir) + + var failedDep string + for e := queue.Front(); e != nil; e = e.Next() { + dep := e.Value.(string) + msg.Warn("#### %s ####", dep) + msg.Info("Seen Count: %d", len(r.seen)) + // Catch the outtermost dependency. + failedDep = dep + err := filepath.Walk(dep, func(path string, fi os.FileInfo, err error) error { + if err != nil && err != filepath.SkipDir { + return err + } + + // Skip files. + if !fi.IsDir() { + return nil + } + // Skip dirs that are not source. + if !srcDir(fi) { + //msg.Debug("Skip resource %s", fi.Name()) + return filepath.SkipDir + } + + // Anything that comes through here has already been through + // the queue. + r.alreadyQ[path] = true + e := r.queueUnseen(path, queue) + if err != nil { + failedDep = path + //msg.Error("Failed to fetch dependency %s: %s", path, err) + } + return e + }) + if err != nil && err != filepath.SkipDir { + msg.Error("Dependency %s failed to resolve: %s.", failedDep, err) + return []string{}, err + } + } + + res := make([]string, 0, queue.Len()) + for e := queue.Front(); e != nil; e = e.Next() { + res = append(res, e.Value.(string)) + } + + return res, nil +} + +// queueUnseenImports scans a package's imports and adds any new ones to the +// processing queue. +func (r *Resolver) queueUnseen(pkg string, queue *list.List) error { + // A pkg is marked "seen" as soon as we have inspected it the first time. + // Seen means that we have added all of its imports to the list. + + // Already queued indicates that we've either already put it into the queue + // or intentionally not put it in the queue for fatal reasons (e.g. no + // buildable source). + + deps, err := r.imports(pkg) + if err != nil && !strings.HasPrefix(err.Error(), "no buildable Go source") { + msg.Error("Could not find %s: %s", pkg, err) + return err + } else if err != nil { + msg.Warn(err.Error()) + } + + for _, d := range deps { + if _, ok := r.alreadyQ[d]; !ok { + r.alreadyQ[d] = true + queue.PushBack(d) + } + //if _, ok := r.seen[d]; !ok { + //queue.PushBack(d) + ////r.seen[d] = true + //} + } + return nil +} + +// imports gets all of the imports for a given package. +// +// If the package is in GOROOT, this will return an empty list (but not +// an error). +// If it cannot resolve the pkg, it will return an error. +func (r *Resolver) imports(pkg string) ([]string, error) { + + // If this pkg is marked seen, we don't scan it again. + if _, ok := r.seen[pkg]; ok { + msg.Debug("Already saw %s", pkg) + return []string{}, nil + } + + // FIXME: On error this should try to NotFound to the dependency, and then import + // it again. + p, err := r.BuildContext.ImportDir(pkg, 0) + if err != nil { + return []string{}, err + } + + // It is okay to scan a package more than once. In some cases, this is + // desirable because the package can change between scans (e.g. as a result + // of a failed scan resolving the situation). + msg.Debug("=> Scanning %s (%s)", p.ImportPath, pkg) + r.seen[pkg] = true + + // Optimization: If it's in GOROOT, it has no imports worth scanning. + if p.Goroot { + return []string{}, nil + } + + // We are only looking for dependencies in vendor. No root, cgo, etc. + buf := []string{} + for _, imp := range p.Imports { + info := r.FindPkg(imp) + switch info.Loc { + case LocUnknown: + // Do we resolve here? + found, err := r.Handler.NotFound(imp) + if err != nil { + msg.Error("Failed to fetch %s: %s", imp, err) + } + if found { + buf = append(buf, filepath.Join(r.VendorDir, imp)) + continue + } + r.seen[info.Path] = true + case LocVendor: + //msg.Debug("Vendored: %s", imp) + buf = append(buf, info.Path) + case LocGopath: + found, err := r.Handler.OnGopath(imp) + if err != nil { + msg.Error("Failed to fetch %s: %s", imp, err) + } + // If the Handler marks this as found, we drop it into the buffer + // for subsequent processing. Otherwise, we assume that we're + // in a less-than-perfect, but functional, situation. + if found { + buf = append(buf, filepath.Join(r.VendorDir, imp)) + continue + } + msg.Warn("Package %s is on GOPATH, but not vendored. Ignoring.", imp) + r.seen[info.Path] = true + default: + // Local packages are an odd case. CGO cannot be scanned. + msg.Debug("===> Skipping %s", imp) + } + } + + return buf, nil +} + +// sliceToQueue is a special-purpose function for unwrapping a slice of +// dependencies into a queue of fully qualified paths. +func sliceToQueue(deps []*yaml.Dependency, basepath string) *list.List { + l := list.New() + for _, e := range deps { + l.PushBack(filepath.Join(basepath, e.Name)) + } + return l +} + +// PkgLoc describes the location of the package. +type PkgLoc uint8 + +const ( + // LocUnknown indicates the package location is unknown (probably not present) + LocUnknown PkgLoc = iota + // LocLocal inidcates that the package is in a local dir, not GOPATH or GOROOT. + LocLocal + // LocVendor indicates that the package is in a vendor/ dir + LocVendor + // LocGopath inidcates that the package is in GOPATH + LocGopath + // LocGoroot indicates that the package is in GOROOT + LocGoroot + // LocCgo indicates that the package is a a CGO package + LocCgo +) + +type PkgInfo struct { + Name, Path string + Vendored bool + Loc PkgLoc +} + +// FindPkg takes a package name and attempts to find it on the filesystem +// +// The resulting PkgInfo will indicate where it was found. +func (r *Resolver) FindPkg(name string) *PkgInfo { + // We cachae results for FindPkg to reduce the number of filesystem ops + // that we have to do. This is a little risky because certain directories, + // like GOPATH, can be modified while we're running an operation, and + // render the cache inaccurate. + // + // Unfound items (LocUnkown) are never cached because we assume that as + // part of the response, the Resolver may fetch that dependency. + if i, ok := r.findCache[name]; ok { + //msg.Info("Cache hit on %s", name) + return i + } + + // 502 individual packages scanned. + // No cache: + // glide -y etcd.yaml list 0.27s user 0.19s system 85% cpu 0.534 total + // With cache: + // glide -y etcd.yaml list 0.22s user 0.15s system 85% cpu 0.438 total + + var p string + info := &PkgInfo{ + Name: name, + } + + // Check _only_ if this dep is in the current vendor directory. + p = filepath.Join(r.VendorDir, name) + if pkgExists(p) { + info.Path = p + info.Loc = LocVendor + info.Vendored = true + r.findCache[name] = info + return info + } + + // TODO: Do we need this if we always flatten? + // Recurse backward to scan other vendor/ directories + //for wd := cwd; wd != "/"; wd = filepath.Dir(wd) { + //p = filepath.Join(wd, "vendor", name) + //if fi, err = os.Stat(p); err == nil && (fi.IsDir() || isLink(fi)) { + //info.Path = p + //info.PType = ptypeVendor + //info.Vendored = true + //return info + //} + //} + + // Check $GOPATH + for _, rr := range strings.Split(r.BuildContext.GOPATH, ":") { + p = filepath.Join(rr, "src", name) + if pkgExists(p) { + info.Path = p + info.Loc = LocGopath + r.findCache[name] = info + return info + } + } + + // Check $GOROOT + for _, rr := range strings.Split(r.BuildContext.GOROOT, ":") { + p = filepath.Join(rr, "src", name) + if pkgExists(p) { + info.Path = p + info.Loc = LocGoroot + r.findCache[name] = info + return info + } + } + + // Finally, if this is "C", we're dealing with cgo + if name == "C" { + info.Loc = LocCgo + r.findCache[name] = info + } + + return info +} + +func pkgExists(path string) bool { + fi, err := os.Stat(path) + return err == nil && (fi.IsDir() || isLink(fi)) +} + +// isLink returns true if the given FileInfo is a symbolic link. +func isLink(fi os.FileInfo) bool { + return fi.Mode()&os.ModeSymlink == os.ModeSymlink +} + +func srcDir(fi os.FileInfo) bool { + if !fi.IsDir() { + return false + } + + // Ignore _foo and .foo + if strings.HasPrefix(fi.Name(), "_") || strings.HasPrefix(fi.Name(), ".") { + return false + } + + // Ignore testdata. For now, ignore vendor. + if fi.Name() == "testdata" || fi.Name() == "vendor" { + return false + } + + return true +}
diff --git a/dependency/resolver_test.go b/dependency/resolver_test.go new file mode 100644 index 0000000..571d39e --- /dev/null +++ b/dependency/resolver_test.go
@@ -0,0 +1,40 @@ +package dependency + +import ( + "testing" + + "github.com/Masterminds/glide/yaml" +) + +func TestResolveAll(t *testing.T) { + // These are build dependencies of Glide, so we know they are here. + deps := []*yaml.Dependency{ + &yaml.Dependency{Name: "github.com/codegangsta/cli"}, + &yaml.Dependency{Name: "github.com/Masterminds/cookoo"}, + &yaml.Dependency{Name: "github.com/Masterminds/squirrel"}, + &yaml.Dependency{Name: "gopkg.in/yaml.v2"}, + } + + r, err := NewResolver("../") + if err != nil { + t.Fatalf("No new resolver: %s", err) + } + l, err := r.ResolveAll(deps) + if err != nil { + t.Fatalf("Failed to resolve: %s", err) + } + + if len(l) < 3 { + t.Errorf("Expected len=3, got %d", len(l)) + } + + println("SEEN") + for k := range r.seen { + println(k) + } + println("RESULT") + + for _, v := range l { + println(v) + } +}
diff --git a/glide.go b/glide.go index 565a80a..0df3a4b 100644 --- a/glide.go +++ b/glide.go
@@ -37,9 +37,13 @@ package main import ( + "io/ioutil" "path/filepath" "github.com/Masterminds/glide/cmd" + "github.com/Masterminds/glide/dependency" + "github.com/Masterminds/glide/msg" + "github.com/Masterminds/glide/yaml" "github.com/Masterminds/cookoo" "github.com/codegangsta/cli" @@ -425,7 +429,25 @@ vendor are only included if they are used by the project. `, Action: func(c *cli.Context) { - setupHandler(c, "list", cxt, router) + //setupHandler(c, "list", cxt, router) + fname := c.GlobalString("yaml") + //parse YAML + f, err := ioutil.ReadFile(fname) + if err != nil { + msg.Error("Failed to open %s: %s", fname, err) + } + gc, _ := yaml.FromYaml(string(f)) + r, err := dependency.NewResolver(".") + if err != nil { + msg.Error("Failed. %s", err) + } + res, err := r.ResolveAll(gc.Imports) + if err != nil { + msg.Error("Failed resolve: %s", err) + } + for _, r := range res { + msg.Puts("\t%s", r) + } }, }, {
diff --git a/msg/msg.go b/msg/msg.go new file mode 100644 index 0000000..a357a39 --- /dev/null +++ b/msg/msg.go
@@ -0,0 +1,92 @@ +// +build !windows + +package msg + +import ( + "fmt" + "strings" +) + +// These contanstants map to color codes for shell scripts making them +// human readable. +const ( + Blue = "0;34" + Red = "0;31" + Green = "0;32" + Yellow = "0;33" + Cyan = "0;36" + Pink = "1;35" +) + +// Color returns a string in a certain color. The first argument is a string +// containing the color code or a constant from the table above mapped to a code. +// +// The following will print the string "Foo" in yellow: +// fmt.Print(Color(Yellow, "Foo")) +func Color(code, msg string) string { + if NoColor { + return msg + } + return fmt.Sprintf("\033[%sm%s\033[m", code, msg) +} + +// Info logs information +func Info(msg string, args ...interface{}) { + if Quiet { + return + } + fmt.Fprint(Stderr, Color(Green, "[INFO] ")) + Msg(msg, args...) +} + +// Debug logs debug information +func Debug(msg string, args ...interface{}) { + if Quiet || !IsDebugging { + return + } + fmt.Fprint(Stderr, "[DEBUG] ") + Msg(msg, args...) +} + +// Warn logs a warning +func Warn(msg string, args ...interface{}) { + fmt.Fprint(Stderr, Color(Yellow, "[WARN] ")) + ErrMsg(msg, args...) +} + +// Error logs and error. +func Error(msg string, args ...interface{}) { + fmt.Fprint(Stderr, Color(Red, "[ERROR] ")) + ErrMsg(msg, args...) +} + +// ErrMsg sends a message to Stderr +func ErrMsg(msg string, args ...interface{}) { + if len(args) == 0 { + fmt.Fprint(Stderr, msg) + } else { + fmt.Fprintf(Stderr, msg, args...) + } + + // Get rid of the annoying fact that messages need \n at the end, but do + // it in a backward compatible way. + if !strings.HasSuffix(msg, "\n") { + fmt.Fprintln(Stderr) + } +} + +// Msg prints a message with optional arguments, that can be printed, of +// varying types. +func Msg(msg string, args ...interface{}) { + if len(args) == 0 { + fmt.Fprint(Stderr, msg) + } else { + fmt.Fprintf(Stderr, msg, args...) + } + + // Get rid of the annoying fact that messages need \n at the end, but do + // it in a backward compatible way. + if !strings.HasSuffix(msg, "\n") { + fmt.Fprintln(Stderr) + } +}
diff --git a/msg/msg_windows.go b/msg/msg_windows.go new file mode 100644 index 0000000..7a715f1 --- /dev/null +++ b/msg/msg_windows.go
@@ -0,0 +1,63 @@ +// +build windows + +package msg + +import ( + "fmt" + "strings" +) + +// Info logs information +func Info(msg string, args ...interface{}) { + if Quiet { + return + } + fmt.Print("[INFO] ") + Msg(msg, args...) +} + +// Debug logs debug information +func Debug(msg string, args ...interface{}) { + if Quiet || !IsDebugging { + return + } + fmt.Print("[DEBUG] ") + Msg(msg, args...) +} + +// Warn logs a warning +func Warn(msg string, args ...interface{}) { + fmt.Fprint(Stderr, "[WARN] ") + ErrMsg(msg, args...) +} + +// Error logs and error. +func Error(msg string, args ...interface{}) { + fmt.Fprint(Stderr, "[ERROR] ") + ErrMsg(msg, args...) +} + +// ErrMsg sends a message to Stderr +func ErrMsg(msg string, args ...interface{}) { + if len(args) == 0 { + fmt.Fprint(Stderr, msg) + return + } + fmt.Fprintf(Stderr, msg, args...) +} + +// Msg prints a message with optional arguments, that can be printed, of +// varying types. +func Msg(msg string, args ...interface{}) { + if len(args) == 0 { + fmt.Print(msg) + return + } + fmt.Printf(msg, args...) + + // Get rid of the annoying fact that messages need \n at the end, but do + // it in a backward compatible way. + if !strings.HasSuffix(msg, "\n") { + fmt.Println("") + } +}
diff --git a/msg/vars.go b/msg/vars.go new file mode 100644 index 0000000..129b53c --- /dev/null +++ b/msg/vars.go
@@ -0,0 +1,31 @@ +package msg + +import ( + "fmt" + "os" +) + +// Quiet, if true, suppresses chatty levels, like Info. +var Quiet = false + +// IsDebugging, if true, shows verbose levels, like Debug. +var IsDebugging = false + +// NoColor, if true, will not use color in the output. +var NoColor = false + +// Stdout is the location where this prints output. +var Stdout = os.Stdout + +// Stderr is the location where this prints logs. +var Stderr = os.Stderr + +// Puts formats a message and then prints to Stdout. +// +// It does not prefix the message, does not color it, or otherwise decorate it. +// +// It does add a line feed. +func Puts(msg string, args ...interface{}) { + fmt.Fprintf(Stdout, msg, args...) + fmt.Fprintln(Stdout) +}