Merging in lockfile support
diff --git a/.travis.yml b/.travis.yml index b66cd48..3d5193d 100644 --- a/.travis.yml +++ b/.travis.yml
@@ -17,7 +17,7 @@ # in the vendor directory. We don't need to test all dependent packages. # Only testing this project. script: - - GO15VENDOREXPERIMENT=1 go test -v . ./cmd ./gb ./util ./yaml + - GO15VENDOREXPERIMENT=1 go test -v . ./cmd ./gb ./util ./cfg notifications: irc: "irc.freenode.net#masterminds"
diff --git a/README.md b/README.md index 7890c50..ebfd0fa 100644 --- a/README.md +++ b/README.md
@@ -1,5 +1,8 @@ # Glide: Vendor Package Management for Golang +Are you used to tools such as Cargo, npm, Composer, Nuget, Pip, Maven, Bunder, +or other modern package managers? If so, Glide is the comparable Go tool. + *Manage your vendor and vendored packages with ease.* Glide is a tool for managing the `vendor` directory within a Go package. This feature, first introduced in Go 1.5, allows each package to have a `vendor` directory @@ -36,7 +39,10 @@ to the `vendor` directory. It then recursively walks through the downloaded packages looking for those with a `glide.yaml` file (or Godep, gb, or GPM config file) that don't already have a `vendor` directory and installing their -dependencies to their `vendor` directories. +dependencies to their `vendor` directories. Once Glide has downloaded and figured +out versions to use in the dependency tree it creates a `glide.lock` file +containing the complete dependency tree pinned to specific versions. To install +the correct versions use `glide install`. A projects is structured like this: @@ -45,6 +51,8 @@ | |-- glide.yaml | + |-- glide.lock + | |-- main.go (Your main go code can live here) | |-- mySubpackage (You can create your own subpackages, too) @@ -102,40 +110,46 @@ Check out the `glide.yaml` in this directory, or examples in the `docs/` directory. -### glide create [optional package name] +### glide create (aliased to init) -Initialize a new workspace. Among other things, this creates a stub -`glide.yaml` +Initialize a new workspace. Among other things, this creates a `glide.yaml` file +while attempting to guess the packages and versions to put in it. For example, +if your project is using Godep it will use the versions specified there. Glide +is smart enough to scan your codebase and detect the imports being used whether +they are specified with another package manager or not. ``` $ glide create -[INFO] Initialized. You can now edit 'glide.yaml' +[INFO] Generating a YAML configuration file and guessing the dependencies +[INFO] Attempting to import from other package managers (use --skip-import to skip) +[INFO] Found reference to github.com/Sirupsen/logrus +[INFO] Adding sub-package hooks/syslog to github.com/Sirupsen/logrus +[INFO] Found reference to github.com/boltdb/bolt +[INFO] Found reference to github.com/gorilla/websocket +[INFO] Found reference to github.com/mndrix/ps +[INFO] Found reference to github.com/spf13/cobra +[INFO] Found reference to github.com/spf13/pflag +[INFO] Found reference to github.com/tinylib/msgp/msgp +[INFO] Found reference to github.com/unrolled/secure +[INFO] Found reference to github.com/xeipuuv/gojsonschema +[INFO] Found reference to github.com/zenazn/goji/graceful +[INFO] Adding sub-package web to github.com/zenazn/goji +[INFO] Adding sub-package web/mutil to github.com/zenazn/goji ``` -If an optional package name is specified, Glide will add it to -glide.yaml as the name of your project. - ### glide get [package name] -You can download package to your `vendor` directory and have it added to your +You can download one or more packages to your `vendor` directory and have it added to your `glide.yaml` file with `glide get`. ``` $ glide get github.com/Masterminds/cookoo ``` -### glide guess and glide pin +When `glide get` is used it will introspect the listed package to resolve its +dependencies including using Godep, GPM, and GB config files. -To help with the creating and managing your `glide.yaml` files there are two -more helper commands. The `glide guess` command will look over your project, -read the imports, attempt to intelligently guess at the ones you need to list, -and create the text for a `glide.yaml` file. - -There are times you need to pin a dependency to a version, such as when you're -preparing to deploy to production. For that case there is the `glide pin` command -that will pin each dependency in the `glide.yaml` file to the current commit id. - -### glide up (aliased to update and install) +### glide update (aliased to up) Download or update all of the libraries listed in the `glide.yaml` file and put them in the `vendor` directory. It will also recursively walk through the @@ -148,6 +162,30 @@ This will recurse over the packages looking for other projects managed by Glide, Godep, gb, and GPM. When one is found those packages will be installed as needed. +A `glide.lock` file will be created or updated with the dependencies pinned to +specific versions. For example, if in the `glide.yaml` file a version was +specified as a range (e.g., `^1.2.3`) it will be set to a specific commit id in +the `glide.lock` file. That allows for reproducible installs (see `glide install`). + +### glide install + +When you want to install the specific versions from the `glide.lock` file use +`glide install`. + +``` +$ glide install +``` + +This will read the `glide.lock` file, making sure it's tied to the `glide.yaml` +file, and install the commit id specific versions there. + +When the `glide.lock` file doesn't tie to the `glide.yaml` file, such as there +being a change, it will provide an error. Running `glide up` will recreate the +`glide.lock` file when updating the dependency tree. + +If no `glide.lock` file is present `glide install` will perform an `update` and +generate a lock file. + ## glide novendor (aliased to nv) When you run commands like `go test ./...` it will iterate over all the @@ -238,21 +276,17 @@ ``` $ glide list -github.com/Masterminds/cookoo (Location: vendored) -github.com/Masterminds/cookoo/io (Location: vendored) -github.com/Masterminds/glide/cmd (Location: gopath) -github.com/Masterminds/glide/gb (Location: gopath) -github.com/Masterminds/glide/util (Location: gopath) -github.com/Masterminds/glide/yaml (Location: gopath) -github.com/Masterminds/semver (Location: vendored) -github.com/Masterminds/vcs (Location: vendored) -github.com/codegangsta/cli (Location: vendored) -gopkg.in/yaml.v2 (Location: vendored) +INSTALLED packages: + vendor/github.com/Masterminds/cookoo + vendor/github.com/Masterminds/cookoo/fmt + vendor/github.com/Masterminds/cookoo/io + vendor/github.com/Masterminds/cookoo/web + vendor/github.com/Masterminds/semver + vendor/github.com/Masterminds/vcs + vendor/github.com/codegangsta/cli + vendor/gopkg.in/yaml.v2 ``` -The possible locations for `list` are `vendored`, `gopath`, and `missing` (if -the package is not installed anywhere accessible). - ### glide help Print the glide help. @@ -267,7 +301,7 @@ ``` $ glide --version -glide version 0.7.0 +glide version 0.8.0 ``` ### glide.yaml @@ -347,7 +381,16 @@ The Git, SVN, Mercurial (Hg), and Bzr source control systems are supported. This happens through the [vcs package](https://github.com/masterminds/vcs). -## Troubleshooting +## Frequently Asked Questions (F.A.Q.) + +#### Q: Why does Glide have the concept of sub-packages when Go doesn't? + +In Go every directory is a package. This works well when you have one repo +containing all of your packages. When you have different packages in different +VCS locations things become a bit more complicated. A project containing a +collection of packages should be handled with the same information including +the version. By grouping packages this way we are able to manage the related +information. #### Q: bzr (or hg) is not working the way I expected. Why? @@ -379,7 +422,7 @@ You can write it to file like this: ``` -$ glide import godep > new-glide.yaml +$ glide import godep -f glide.yaml ``` #### Q: Can Glide fetch a package based on OS or Arch?
diff --git a/cfg/cfg.go b/cfg/cfg.go new file mode 100644 index 0000000..52874d2 --- /dev/null +++ b/cfg/cfg.go
@@ -0,0 +1,2 @@ +// Package cfg handles working with the Glide configuration files. +package cfg
diff --git a/cfg/config.go b/cfg/config.go new file mode 100644 index 0000000..63c3676 --- /dev/null +++ b/cfg/config.go
@@ -0,0 +1,399 @@ +package cfg + +import ( + "crypto/sha256" + "fmt" + "reflect" + "strings" + + "github.com/Masterminds/glide/util" + "github.com/Masterminds/vcs" + "gopkg.in/yaml.v2" +) + +// Config is the top-level configuration object. +type Config struct { + Name string `yaml:"package"` + Ignore []string `yaml:"ignore,omitempty"` + Imports Dependencies `yaml:"import"` + DevImports Dependencies `yaml:"devimport,omitempty"` +} + +// A transitive representation of a dependency for importing and exploting to yaml. +type cf struct { + Name string `yaml:"package"` + Ignore []string `yaml:"ignore,omitempty"` + Imports Dependencies `yaml:"import"` + DevImports Dependencies `yaml:"devimport,omitempty"` +} + +// ConfigFromYaml returns an instance of Config from YAML +func ConfigFromYaml(yml []byte) (*Config, error) { + cfg := &Config{} + err := yaml.Unmarshal([]byte(yml), &cfg) + return cfg, err +} + +// Marshal converts a Config instance to YAML +func (c *Config) Marshal() ([]byte, error) { + yml, err := yaml.Marshal(&c) + if err != nil { + return []byte{}, err + } + return yml, nil +} + +// UnmarshalYAML is a hook for gopkg.in/yaml.v2 in the unmarshalling process +func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { + newConfig := &cf{} + if err := unmarshal(&newConfig); err != nil { + return err + } + c.Name = newConfig.Name + c.Ignore = newConfig.Ignore + c.Imports = newConfig.Imports + c.DevImports = newConfig.DevImports + + // Cleanup the Config object now that we have it. + err := c.DeDupe() + + return err +} + +// MarshalYAML is a hook for gopkg.in/yaml.v2 in the marshaling process +func (c *Config) MarshalYAML() (interface{}, error) { + newConfig := &cf{ + Name: c.Name, + Ignore: c.Ignore, + } + i, err := c.Imports.Clone().DeDupe() + if err != nil { + return newConfig, err + } + + di, err := c.DevImports.Clone().DeDupe() + if err != nil { + return newConfig, err + } + + newConfig.Imports = i + newConfig.DevImports = di + + return newConfig, nil +} + +// HasDependency returns true if the given name is listed as an import or dev import. +func (c *Config) HasDependency(name string) bool { + for _, d := range c.Imports { + if d.Name == name { + return true + } + } + for _, d := range c.DevImports { + if d.Name == name { + return true + } + } + return false +} + +// HasIgnore returns true if the given name is listed on the ignore list. +func (c *Config) HasIgnore(name string) bool { + for _, v := range c.Ignore { + if v == name { + return true + } + } + + return false +} + +// Clone performs a deep clone of the Config instance +func (c *Config) Clone() *Config { + n := &Config{} + n.Name = c.Name + n.Ignore = c.Ignore + n.Imports = c.Imports.Clone() + n.DevImports = c.DevImports.Clone() + return n +} + +// DeDupe consolidates duplicate dependencies on a Config instance +func (c *Config) DeDupe() error { + + // Remove duplicates in the imports + var err error + c.Imports, err = c.Imports.DeDupe() + if err != nil { + return err + } + c.DevImports, err = c.DevImports.DeDupe() + if err != nil { + return err + } + + // If the name on the config object is part of the imports remove it. + found := -1 + for i, dep := range c.Imports { + if dep.Name == c.Name { + found = i + } + } + if found >= 0 { + c.Imports = append(c.Imports[:found], c.Imports[found+1:]...) + } + + found = -1 + for i, dep := range c.DevImports { + if dep.Name == c.Name { + found = i + } + } + if found >= 0 { + c.DevImports = append(c.DevImports[:found], c.DevImports[found+1:]...) + } + + // If something is on the ignore list remove it from the imports. + for _, v := range c.Ignore { + found = -1 + for k, d := range c.Imports { + if v == d.Name { + found = k + } + } + if found >= 0 { + c.Imports = append(c.Imports[:found], c.Imports[found+1:]...) + } + + found = -1 + for k, d := range c.DevImports { + if v == d.Name { + found = k + } + } + if found >= 0 { + c.DevImports = append(c.DevImports[:found], c.DevImports[found+1:]...) + } + } + + return nil +} + +// Hash generates a sha256 hash for a given Config +func (c *Config) Hash() (string, error) { + yml, err := c.Marshal() + if err != nil { + return "", err + } + + hash := sha256.New() + hash.Sum(yml) + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +// Dependencies is a collection of Dependency +type Dependencies []*Dependency + +// Get a dependency by name +func (d Dependencies) Get(name string) *Dependency { + for _, dep := range d { + if dep.Name == name { + return dep + } + } + return nil +} + +// Clone performs a deep clone of Dependencies +func (d Dependencies) Clone() Dependencies { + n := make(Dependencies, 0, 1) + for _, v := range d { + n = append(n, v.Clone()) + } + return n +} + +// DeDupe cleans up duplicates on a list of dependencies. +func (d Dependencies) DeDupe() (Dependencies, error) { + checked := map[string]int{} + imports := make(Dependencies, 0, 1) + i := 0 + for _, dep := range d { + // The first time we encounter a dependency add it to the list + if val, ok := checked[dep.Name]; !ok { + checked[dep.Name] = i + imports = append(imports, dep) + i++ + } else { + // In here we've encountered a dependency for the second time. + // Make sure the details are the same or return an error. + v := imports[val] + if dep.Reference != v.Reference { + return d, fmt.Errorf("Import %s repeated with different versions '%s' and '%s'", dep.Name, dep.Reference, v.Reference) + } + if dep.Repository != v.Repository || dep.VcsType != v.VcsType { + return d, fmt.Errorf("Import %s repeated with different Repository details", dep.Name) + } + if !reflect.DeepEqual(dep.Os, v.Os) || !reflect.DeepEqual(dep.Arch, v.Arch) { + return d, fmt.Errorf("Import %s repeated with different OS or Architecture filtering", dep.Name) + } + imports[checked[dep.Name]].Subpackages = stringArrayDeDupe(v.Subpackages, dep.Subpackages...) + } + } + + return imports, nil +} + +// Dependency describes a package that the present package depends upon. +type Dependency struct { + Name string `yaml:"package"` + Reference string `yaml:"version,omitempty"` + Pin string `yaml:"-"` + Repository string `yaml:"repo,omitempty"` + VcsType string `yaml:"vcs,omitempty"` + Subpackages []string `yaml:"subpackages,omitempty"` + Arch []string `yaml:"arch,omitempty"` + Os []string `yaml:"os,omitempty"` + UpdateAsVendored bool `yaml:"-"` +} + +// A transitive representation of a dependency for importing and exploting to yaml. +type dep struct { + Name string `yaml:"package"` + Reference string `yaml:"version,omitempty"` + Ref string `yaml:"ref,omitempty"` + Repository string `yaml:"repo,omitempty"` + VcsType string `yaml:"vcs,omitempty"` + Subpackages []string `yaml:"subpackages,omitempty"` + Arch []string `yaml:"arch,omitempty"` + Os []string `yaml:"os,omitempty"` +} + +// UnmarshalYAML is a hook for gopkg.in/yaml.v2 in the unmarshaling process +func (d *Dependency) UnmarshalYAML(unmarshal func(interface{}) error) error { + newDep := &dep{} + err := unmarshal(&newDep) + if err != nil { + return err + } + d.Name = newDep.Name + d.Reference = newDep.Reference + d.Repository = newDep.Repository + d.VcsType = newDep.VcsType + d.Subpackages = newDep.Subpackages + d.Arch = newDep.Arch + d.Os = newDep.Os + + if d.Reference == "" && newDep.Ref != "" { + d.Reference = newDep.Ref + } + + // Make sure only legitimate VCS are listed. + d.VcsType = filterVcsType(d.VcsType) + + // Get the root name for the package + o := d.Name + d.Name = util.GetRootFromPackage(d.Name) + subpkg := strings.TrimPrefix(o, d.Name) + if len(subpkg) > 0 && subpkg != o { + d.Subpackages = append(d.Subpackages, strings.TrimPrefix(subpkg, "/")) + } + + return nil +} + +// MarshalYAML is a hook for gopkg.in/yaml.v2 in the marshaling process +func (d *Dependency) MarshalYAML() (interface{}, error) { + + // Make sure we only write the correct vcs type to file + t := filterVcsType(d.VcsType) + newDep := &dep{ + Name: d.Name, + Reference: d.Reference, + Repository: d.Repository, + VcsType: t, + Subpackages: d.Subpackages, + Arch: d.Arch, + Os: d.Os, + } + + return newDep, nil +} + +// GetRepo retrieves a Masterminds/vcs repo object configured for the root +// of the package being retrieved. +func (d *Dependency) GetRepo(dest string) (vcs.Repo, error) { + + // The remote location is either the configured repo or the package + // name as an https url. + var remote string + if len(d.Repository) > 0 { + remote = d.Repository + } else { + remote = "https://" + d.Name + } + + // If the VCS type has a value we try that first. + if len(d.VcsType) > 0 && d.VcsType != "None" { + switch vcs.Type(d.VcsType) { + case vcs.Git: + return vcs.NewGitRepo(remote, dest) + case vcs.Svn: + return vcs.NewSvnRepo(remote, dest) + case vcs.Hg: + return vcs.NewHgRepo(remote, dest) + case vcs.Bzr: + return vcs.NewBzrRepo(remote, dest) + default: + return nil, fmt.Errorf("Unknown VCS type %s set for %s", d.VcsType, d.Name) + } + } + + // When no type set we try to autodetect. + return vcs.NewRepo(remote, dest) +} + +// Clone creates a clone of a Dependency +func (d *Dependency) Clone() *Dependency { + return &Dependency{ + Name: d.Name, + Reference: d.Reference, + Pin: d.Pin, + Repository: d.Repository, + VcsType: d.VcsType, + Subpackages: d.Subpackages, + Arch: d.Arch, + Os: d.Os, + UpdateAsVendored: d.UpdateAsVendored, + } +} + +func stringArrayDeDupe(s []string, items ...string) []string { + for _, item := range items { + exists := false + for _, v := range s { + if v == item { + exists = true + } + } + if !exists { + s = append(s, item) + } + } + return s +} + +func filterVcsType(vcs string) string { + switch vcs { + case "git", "hg", "bzr", "svn": + return vcs + case "mercurial": + return "hg" + case "bazaar": + return "bzr" + case "subversion": + return "svn" + default: + return "" + } +}
diff --git a/cfg/config_test.go b/cfg/config_test.go new file mode 100644 index 0000000..03ba08e --- /dev/null +++ b/cfg/config_test.go
@@ -0,0 +1,116 @@ +package cfg + +import ( + "testing" + + "gopkg.in/yaml.v2" +) + +var yml = ` +package: fake/testing +import: + - package: github.com/kylelemons/go-gypsy + subpackages: + - yaml + # Intentionally left spaces at end of next line. + - package: github.com/Masterminds/convert + repo: git@github.com:Masterminds/convert.git + vcs: git + ref: a9949121a2e2192ca92fa6dddfeaaa4a4412d955 + subpackages: + - color + - nautical + - radial + os: + - linux + arch: + - i386 + - arm + - package: github.com/Masterminds/structable + - package: github.com/Masterminds/cookoo/color + - package: github.com/Masterminds/cookoo/convert + +devimport: + - package: github.com/kylelemons/go-gypsy +` + +func TestManualConfigFromYaml(t *testing.T) { + cfg := &Config{} + err := yaml.Unmarshal([]byte(yml), &cfg) + if err != nil { + t.Errorf("Unable to Unmarshal config yaml") + } + + if cfg.Name != "fake/testing" { + t.Errorf("Inaccurate name found %s", cfg.Name) + } + + found := false + found2 := false + for _, i := range cfg.Imports { + if i.Name == "github.com/Masterminds/convert" { + found = true + ref := "a9949121a2e2192ca92fa6dddfeaaa4a4412d955" + if i.Reference != ref { + t.Errorf("Config reference for cookoo is inaccurate. Expected '%s' found '%s'", ref, i.Reference) + } + } + + if i.Name == "github.com/Masterminds/cookoo" { + found2 = true + if i.Subpackages[0] != "color" { + t.Error("Dependency separating package and subpackage not working") + } + } + } + if !found { + t.Error("Unable to find github.com/Masterminds/convert") + } + if !found2 { + t.Error("Unable to find github.com/Masterminds/cookoo") + } +} + +func TestClone(t *testing.T) { + cfg := &Config{} + err := yaml.Unmarshal([]byte(yml), &cfg) + if err != nil { + t.Errorf("Unable to Unmarshal config yaml") + } + + cfg2 := cfg.Clone() + if cfg2.Name != "fake/testing" { + t.Error("Config cloning failed") + } + cfg.Name = "foo" + + if cfg.Name == cfg2.Name { + t.Error("Cloning Config name failed") + } +} + +func TestConfigFromYaml(t *testing.T) { + c, err := ConfigFromYaml([]byte(yml)) + if err != nil { + t.Error("ConfigFromYaml failed to parse yaml") + } + + if c.Name != "fake/testing" { + t.Error("ConfigFromYaml failed to properly parse yaml") + } +} + +func TestHasDependency(t *testing.T) { + c, err := ConfigFromYaml([]byte(yml)) + if err != nil { + t.Error("ConfigFromYaml failed to parse yaml for HasDependency") + } + + if c.HasDependency("github.com/Masterminds/convert") != true { + t.Error("HasDependency failing to pickup depenency") + } + + if c.HasDependency("foo/bar/bar") != false { + t.Error("HasDependency picking up dependency it shouldn't") + } +}
diff --git a/cfg/lock.go b/cfg/lock.go new file mode 100644 index 0000000..921fe33 --- /dev/null +++ b/cfg/lock.go
@@ -0,0 +1,117 @@ +package cfg + +import ( + "sort" + "strings" + "time" + + "gopkg.in/yaml.v2" +) + +// Lockfile represents a glide.lock file. +type Lockfile struct { + Hash string `yaml:"hash"` + Updated time.Time `yaml:"updated"` + Imports Locks `yaml:"imports"` + DevImports Locks `yaml:"devImports"` +} + +// LockfileFromYaml returns an instance of Lockfile from YAML +func LockfileFromYaml(yml []byte) (*Lockfile, error) { + lock := &Lockfile{} + err := yaml.Unmarshal([]byte(yml), &lock) + return lock, err +} + +// Marshal converts a Config instance to YAML +func (lf *Lockfile) Marshal() ([]byte, error) { + yml, err := yaml.Marshal(&lf) + if err != nil { + return []byte{}, err + } + return yml, nil +} + +type Locks []*Lock + +// Len returns the length of the Locks. This is needed for sorting with +// the sort package. +func (l Locks) Len() int { + return len(l) +} + +// Less is needed for the sort interface. It compares two locks based on +// their name. +func (l Locks) Less(i, j int) bool { + + // Names are normalized to lowercase because case affects sorting order. For + // example, Masterminds comes before kylelemons. Making them lowercase + // causes kylelemons to come first which is what is expected. + return strings.ToLower(l[i].Name) < strings.ToLower(l[j].Name) +} + +// Swap is needed for the sort interface. It swaps the position of two +// locks. +func (l Locks) Swap(i, j int) { + l[i], l[j] = l[j], l[i] +} + +type Lock struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Repository string `yaml:"repo,omitempty"` + VcsType string `yaml:"vcs,omitempty"` + Subpackages []string `yaml:"subpackages,omitempty"` + Arch []string `yaml:"arch,omitempty"` + Os []string `yaml:"os,omitempty"` +} + +func NewLockfile(ds Dependencies, hash string) *Lockfile { + lf := &Lockfile{ + Hash: hash, + Updated: time.Now(), + Imports: make([]*Lock, len(ds)), + } + + for i := 0; i < len(ds); i++ { + lf.Imports[i] = &Lock{ + Name: ds[i].Name, + Version: ds[i].Pin, + Repository: ds[i].Repository, + VcsType: ds[i].VcsType, + Subpackages: ds[i].Subpackages, + Arch: ds[i].Arch, + Os: ds[i].Os, + } + } + + sort.Sort(lf.Imports) + + return lf +} + +func LockfileFromMap(ds map[string]*Dependency, hash string) *Lockfile { + lf := &Lockfile{ + Hash: hash, + Updated: time.Now(), + Imports: make([]*Lock, len(ds)), + } + + i := 0 + for name, dep := range ds { + lf.Imports[i] = &Lock{ + Name: name, + Version: dep.Pin, + Repository: dep.Repository, + VcsType: dep.VcsType, + Subpackages: dep.Subpackages, + Arch: dep.Arch, + Os: dep.Os, + } + i++ + } + + sort.Sort(lf.Imports) + + return lf +}
diff --git a/cfg/lock_test.go b/cfg/lock_test.go new file mode 100644 index 0000000..78dcee3 --- /dev/null +++ b/cfg/lock_test.go
@@ -0,0 +1,34 @@ +package cfg + +import ( + "sort" + "testing" +) + +func TestSortLocks(t *testing.T) { + c, err := ConfigFromYaml([]byte(yml)) + if err != nil { + t.Error("ConfigFromYaml failed to parse yaml for TestSortDependencies") + } + + ls := make(Locks, len(c.Imports)) + for i := 0; i < len(c.Imports); i++ { + ls[i] = &Lock{ + Name: c.Imports[i].Name, + Version: c.Imports[i].Reference, + } + } + + if ls[2].Name != "github.com/Masterminds/structable" { + t.Error("Initial dependencies are out of order prior to sort") + } + + sort.Sort(ls) + + if ls[0].Name != "github.com/kylelemons/go-gypsy" || + ls[1].Name != "github.com/Masterminds/convert" || + ls[2].Name != "github.com/Masterminds/cookoo" || + ls[3].Name != "github.com/Masterminds/structable" { + t.Error("Sorting of dependencies failed") + } +}
diff --git a/cmd/delete.go b/cmd/delete.go index 3752c9b..709dafd 100644 --- a/cmd/delete.go +++ b/cmd/delete.go
@@ -7,7 +7,7 @@ "strings" "github.com/Masterminds/cookoo" - "github.com/Masterminds/glide/yaml" + "github.com/Masterminds/glide/cfg" ) // DeleteUnusedPackages removes packages from vendor/ that are no longer used. @@ -27,7 +27,7 @@ } // Build directory tree of what to keep. - cfg := p.Get("conf", nil).(*yaml.Config) + cfg := p.Get("conf", nil).(*cfg.Config) var pkgList []string for _, dep := range cfg.Imports { pkgList = append(pkgList, dep.Name)
diff --git a/cmd/exec.go b/cmd/exec.go deleted file mode 100644 index 3cc0720..0000000 --- a/cmd/exec.go +++ /dev/null
@@ -1,41 +0,0 @@ -package cmd - -import ( - "fmt" - "github.com/Masterminds/cookoo" - "github.com/codegangsta/cli" - "os" - "os/exec" -) - -// ExecCmd executes a system command inside vendor -func ExecCmd(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - args := p.Get("args", nil).(cli.Args) - - if len(args) == 0 { - return nil, fmt.Errorf("No command to execute") - } - - gopath, err := VendorPath(c) - if err != nil { - return false, err - } - - err = os.Setenv("GOPATH", gopath) - if err != nil { - return false, err - } - - path := os.Getenv("PATH") - err = os.Setenv("PATH", gopath+"/bin:"+path) - if err != nil { - return false, err - } - - cmd := exec.Command(args[0], args[1:]...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Run() - - return true, nil -}
diff --git a/cmd/flatten.go b/cmd/flatten.go index cb55674..1bf9adc 100644 --- a/cmd/flatten.go +++ b/cmd/flatten.go
@@ -7,7 +7,8 @@ "strings" "github.com/Masterminds/cookoo" - "github.com/Masterminds/glide/yaml" + "github.com/Masterminds/glide/cfg" + "github.com/Masterminds/glide/util" "github.com/Masterminds/semver" ) @@ -20,12 +21,12 @@ // - packages ([]string): The packages to read. If this is empty, it reads all // packages. // - force (bool): force vcs updates. -// - conf (*yaml.Config): The configuration. +// - conf (*cfg.Config): The configuration. // // Returns: // func Flatten(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - conf := p.Get("conf", &yaml.Config{}).(*yaml.Config) + conf := p.Get("conf", &cfg.Config{}).(*cfg.Config) skip := p.Get("skip", false).(bool) home := p.Get("home", "").(string) cache := p.Get("cache", false).(bool) @@ -48,7 +49,7 @@ } // Build an initial dependency map. - deps := make(map[string]*yaml.Dependency, len(conf.Imports)) + deps := make(map[string]*cfg.Dependency, len(conf.Imports)) for _, imp := range conf.Imports { deps[imp.Name] = imp } @@ -69,6 +70,12 @@ flattenSetRefs(f) Info("Project relies on %d dependencies.", len(deps)) + hash, err := conf.Hash() + if err != nil { + return conf, err + } + c.Put("Lockfile", cfg.LockfileFromMap(deps, hash)) + // A shallow copy should be all that's needed. confcopy := conf.Clone() exportFlattenedDeps(confcopy, deps) @@ -76,8 +83,8 @@ return confcopy, err } -func exportFlattenedDeps(conf *yaml.Config, in map[string]*yaml.Dependency) { - out := make([]*yaml.Dependency, len(in)) +func exportFlattenedDeps(conf *cfg.Config, in map[string]*cfg.Dependency) { + out := make([]*cfg.Dependency, len(in)) i := 0 for _, v := range in { out[i] = v @@ -87,13 +94,13 @@ } type flattening struct { - conf *yaml.Config + conf *cfg.Config // Top vendor path, e.g. project/vendor top string // Current path curr string // Built list of dependencies - deps map[string]*yaml.Dependency + deps map[string]*cfg.Dependency // Dependencies that need to be scanned. scan []string } @@ -108,19 +115,17 @@ Debug("----> Scanning %s", imp) base := path.Join(f.top, imp) mod := []string{} - if m, ok := mergeGlide(base, imp, f.deps, f.top); ok { + if m, ok := mergeGlide(base, imp, f); ok { mod = m - } else if m, ok = mergeGodep(base, imp, f.deps, f.top); ok { + } else if m, ok = mergeGodep(base, imp, f); ok { mod = m - } else if m, ok = mergeGPM(base, imp, f.deps, f.top); ok { + } else if m, ok = mergeGPM(base, imp, f); ok { mod = m - } else if m, ok = mergeGb(base, imp, f.deps, f.top); ok { + } else if m, ok = mergeGb(base, imp, f); ok { mod = m - } else if m, ok = mergeGuess(base, imp, f.deps, f.top, scanned); ok { + } else if m, ok = mergeGuess(base, imp, f, scanned); ok { mod = m } - //mod, _ = mergeGuess(base, imp, f.deps, f.top, scanned) - //Info("Scanned: %v", scanned) if len(mod) > 0 { Debug("----> Updating all dependencies for %q (%d)", imp, len(mod)) @@ -192,7 +197,9 @@ } } -func mergeGlide(dir, name string, deps map[string]*yaml.Dependency, vend string) ([]string, bool) { +func mergeGlide(dir, name string, f *flattening) ([]string, bool) { + deps := f.deps + vend := f.top gp := path.Join(dir, "glide.yaml") if _, err := os.Stat(gp); err != nil { return []string{}, false @@ -204,7 +211,7 @@ return []string{}, false } - conf, err := yaml.FromYaml(string(yml)) + conf, err := cfg.ConfigFromYaml(yml) if err != nil { Warn("Found glide file %q, but can't use it: %s", gp, err) return []string{}, false @@ -212,14 +219,16 @@ Info("Found glide.yaml in %s", gp) - return mergeDeps(deps, conf.Imports, vend), true + return mergeDeps(deps, conf.Imports, vend, f), true } // listGodep appends Godeps entries to the deps. // // It returns true if any dependencies were found (even if not added because // they are duplicates). -func mergeGodep(dir, name string, deps map[string]*yaml.Dependency, vend string) ([]string, bool) { +func mergeGodep(dir, name string, f *flattening) ([]string, bool) { + deps := f.deps + vend := f.top Debug("Looking in %s/Godeps/ for a Godeps.json file.\n", dir) d, err := parseGodepGodeps(dir) if err != nil { @@ -230,28 +239,32 @@ } Info("Found Godeps.json file for %q", name) - return mergeDeps(deps, d, vend), true + return mergeDeps(deps, d, vend, f), true } // listGb merges GB dependencies into the deps. -func mergeGb(dir, pkg string, deps map[string]*yaml.Dependency, vend string) ([]string, bool) { +func mergeGb(dir, pkg string, f *flattening) ([]string, bool) { + deps := f.deps + vend := f.top Debug("Looking in %s/vendor/ for a manifest file.\n", dir) d, err := parseGbManifest(dir) if err != nil || len(d) == 0 { return []string{}, false } Info("Found gb manifest file for %q", pkg) - return mergeDeps(deps, d, vend), true + return mergeDeps(deps, d, vend, f), true } // mergeGPM merges GPM Godeps files into deps. -func mergeGPM(dir, pkg string, deps map[string]*yaml.Dependency, vend string) ([]string, bool) { +func mergeGPM(dir, pkg string, f *flattening) ([]string, bool) { + deps := f.deps + vend := f.top d, err := parseGPMGodeps(dir) if err != nil || len(d) == 0 { return []string{}, false } Info("Found GPM file for %q", pkg) - return mergeDeps(deps, d, vend), true + return mergeDeps(deps, d, vend, f), true } // mergeGuess guesses dependencies and merges. @@ -259,75 +272,80 @@ // This always returns true because it always handles the job of searching // for dependencies. So generally it should be the last merge strategy // that you try. -func mergeGuess(dir, pkg string, deps map[string]*yaml.Dependency, vend string, scanned map[string]bool) ([]string, bool) { - // Info("Scanning %s for dependencies.", pkg) - // buildContext, err := GetBuildContext() - // if err != nil { - // Warn("Could not scan package %q: %s", pkg, err) - // return []string{}, false - // } - // - // res := []string{} - // - // if _, err := os.Stat(dir); err != nil { - // Warn("Directory is missing: %s", dir) - // return res, true - // } - // - // d := walkDeps(buildContext, dir, pkg) - // for _, oname := range d { - // if _, ok := scanned[oname]; ok { - // //Info("===> Scanned %s already. Skipping", name) - // continue - // } - // Info("=> Scanning %s", oname) - // name, _ := NormalizeName(oname) - // //if _, ok := deps[name]; ok { - // //scanned[oname] = true - // //Debug("====> Seen %s already. Skipping", name) - // //continue - // //} - // - // repo := util.GetRootFromPackage(name) - // found := findPkg(buildContext, name, dir) - // switch found.PType { - // case ptypeUnknown: - // Info("==> Unknown %s (%s)", name, oname) - // Debug("✨☆ Undownloaded dependency: %s", name) - // nd := &yaml.Dependency{ - // Name: name, - // Repository: "https://" + repo, - // } - // deps[name] = nd - // res = append(res, name) - // case ptypeGoroot, ptypeCgo: - // scanned[oname] = true - // // Why do we break rather than continue? - // break - // default: - // // We're looking for dependencies that might exist in $GOPATH - // // but not be on vendor. We add any that are on $GOPATH. - // if _, ok := deps[name]; !ok { - // Debug("✨☆ GOPATH dependency: %s", name) - // nd := &yaml.Dependency{Name: name} - // deps[name] = nd - // res = append(res, name) - // } - // scanned[oname] = true - // } - // } - // - // return res, true - Info("Package %s manages its own dependencies", pkg) - return []string{}, true +func mergeGuess(dir, pkg string, f *flattening, scanned map[string]bool) ([]string, bool) { + deps := f.deps + Info("Scanning %s for dependencies.", pkg) + buildContext, err := GetBuildContext() + if err != nil { + Warn("Could not scan package %q: %s", pkg, err) + return []string{}, false + } + + res := []string{} + + if _, err := os.Stat(dir); err != nil { + Warn("Directory is missing: %s", dir) + return res, true + } + + d := walkDeps(buildContext, dir, pkg) + for _, oname := range d { + if _, ok := scanned[oname]; ok { + //Info("===> Scanned %s already. Skipping", name) + continue + } + Debug("=> Scanning %s", oname) + name, _ := NormalizeName(oname) + //if _, ok := deps[name]; ok { + //scanned[oname] = true + //Debug("====> Seen %s already. Skipping", name) + //continue + //} + if f.conf.HasIgnore(name) { + Debug("==> Skipping %s because it is on the ignore list", name) + continue + } + + found := findPkg(buildContext, name, dir) + switch found.PType { + case ptypeUnknown: + Info("==> Unknown %s (%s)", name, oname) + Debug("✨☆ Undownloaded dependency: %s", name) + repo := util.GetRootFromPackage(name) + nd := &cfg.Dependency{ + Name: name, + Repository: "https://" + repo, + } + deps[name] = nd + res = append(res, name) + case ptypeGoroot, ptypeCgo: + scanned[oname] = true + // Why do we break rather than continue? + break + default: + // We're looking for dependencies that might exist in $GOPATH + // but not be on vendor. We add any that are on $GOPATH. + if _, ok := deps[name]; !ok { + Debug("✨☆ GOPATH dependency: %s", name) + nd := &cfg.Dependency{Name: name} + deps[name] = nd + res = append(res, name) + } + scanned[oname] = true + } + } + + return res, true } // mergeDeps merges any dependency array into deps. -func mergeDeps(orig map[string]*yaml.Dependency, add []*yaml.Dependency, vend string) []string { +func mergeDeps(orig map[string]*cfg.Dependency, add []*cfg.Dependency, vend string, f *flattening) []string { mod := []string{} for _, dd := range add { - // Add it unless it's already there. - if existing, ok := orig[dd.Name]; !ok { + if f.conf.HasIgnore(dd.Name) { + Debug("Skipping %s because it is on the ignore list", dd.Name) + } else if existing, ok := orig[dd.Name]; !ok { + // Add it unless it's already there. orig[dd.Name] = dd Debug("Adding %s to the scan list", dd.Name) mod = append(mod, dd.Name)
diff --git a/cmd/gb.go b/cmd/gb.go index ac66aeb..35245d3 100644 --- a/cmd/gb.go +++ b/cmd/gb.go
@@ -6,8 +6,8 @@ "path/filepath" "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/cfg" "github.com/Masterminds/glide/gb" - "github.com/Masterminds/glide/yaml" ) func HasGbManifest(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { @@ -28,14 +28,14 @@ return parseGbManifest(dir) } -func parseGbManifest(dir string) ([]*yaml.Dependency, error) { +func parseGbManifest(dir string) ([]*cfg.Dependency, error) { path := filepath.Join(dir, "vendor/manifest") if fi, err := os.Stat(path); err != nil || fi.IsDir() { - return []*yaml.Dependency{}, nil + return []*cfg.Dependency{}, nil } Info("Found GB manifest file.\n") - buf := []*yaml.Dependency{} + buf := []*cfg.Dependency{} file, err := os.Open(path) if err != nil { return buf, err @@ -64,7 +64,7 @@ } } else { seen[pkg] = true - dep := &yaml.Dependency{ + dep := &cfg.Dependency{ Name: pkg, Reference: d.Revision, Repository: d.Repository,
diff --git a/cmd/get_imports.go b/cmd/get_imports.go index 35baee6..ab103eb 100644 --- a/cmd/get_imports.go +++ b/cmd/get_imports.go
@@ -9,6 +9,7 @@ "os/exec" "path/filepath" "sort" + "sync" //"log" "os" @@ -17,12 +18,15 @@ "strings" "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/cfg" "github.com/Masterminds/glide/util" - "github.com/Masterminds/glide/yaml" "github.com/Masterminds/semver" v "github.com/Masterminds/vcs" ) +// Used for the fan out/in pattern used with VCS calls. +var concurrentWorkers = 20 + //func init() { // Uncomment the line below and the log import to see the output // from the vcs commands executed for each project. @@ -42,7 +46,7 @@ // - []*Dependency: A list of constructed dependencies. func GetAll(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { names := p.Get("packages", []string{}).([]string) - cfg := p.Get("conf", nil).(*yaml.Config) + conf := p.Get("conf", nil).(*cfg.Config) insecure := p.Get("insecure", false).(bool) home := p.Get("home", "").(string) cache := p.Get("cache", false).(bool) @@ -51,7 +55,7 @@ Info("Preparing to install %d package.", len(names)) - deps := []*yaml.Dependency{} + deps := []*cfg.Dependency{} for _, name := range names { cwd, err := VendorPath(c) if err != nil { @@ -63,11 +67,16 @@ return nil, fmt.Errorf("Package name is required for %q.", name) } - if cfg.HasDependency(root) { + if conf.HasDependency(root) { Warn("Package %q is already in glide.yaml. Skipping", root) continue } + if conf.HasIgnore(root) { + Warn("Package %q is set to be ignored in glide.yaml. Skipping", root) + continue + } + dest := path.Join(cwd, root) if err != nil { @@ -75,7 +84,7 @@ return false, err } - dep := &yaml.Dependency{ + dep := &cfg.Dependency{ Name: root, } @@ -93,7 +102,7 @@ return dep, err } - cfg.Imports = append(cfg.Imports, dep) + conf.Imports = append(conf.Imports, dep) deps = append(deps, dep) @@ -106,10 +115,10 @@ // Params: // // - force (bool): force packages to update (default false) -// - conf (*yaml.Config): The configuration +// - conf (*cfg.Config): The configuration // - packages([]string): The packages to update. Default is all. func UpdateImports(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := p.Get("conf", nil).(*yaml.Config) + cfg := p.Get("conf", nil).(*cfg.Config) force := p.Get("force", true).(bool) plist := p.Get("packages", []string{}).([]string) home := p.Get("home", "").(string) @@ -152,29 +161,62 @@ // SetReference is a command to set the VCS reference (commit id, tag, etc) for // a project. func SetReference(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := p.Get("conf", nil).(*yaml.Config) + conf := p.Get("conf", nil).(*cfg.Config) cwd, err := VendorPath(c) if err != nil { return false, err } - if len(cfg.Imports) == 0 { + if len(conf.Imports) == 0 { Info("No references set.\n") return false, nil } + // + // for _, dep := range conf.Imports { + // if err := VcsVersion(dep, cwd); err != nil { + // Warn("Failed to set version on %s to %s: %s\n", dep.Name, dep.Reference, err) + // } + // } - for _, dep := range cfg.Imports { - if err := VcsVersion(dep, cwd); err != nil { - Warn("Failed to set version on %s to %s: %s\n", dep.Name, dep.Reference, err) - } + done := make(chan struct{}, concurrentWorkers) + in := make(chan *cfg.Dependency, concurrentWorkers) + var wg sync.WaitGroup + + for i := 0; i < concurrentWorkers; i++ { + go func(ch <-chan *cfg.Dependency) { + for { + select { + case dep := <-ch: + if err := VcsVersion(dep, cwd); err != nil { + Warn("Failed to set version on %s to %s: %s\n", dep.Name, dep.Reference, err) + } + wg.Done() + case <-done: + return + } + } + }(in) } + for _, dep := range conf.Imports { + wg.Add(1) + in <- dep + } + + wg.Wait() + // Close goroutines setting the version + for i := 0; i < concurrentWorkers; i++ { + done <- struct{}{} + } + // close(done) + // close(in) + return true, nil } // filterArchOs indicates a dependency should be filtered out because it is // the wrong GOOS or GOARCH. -func filterArchOs(dep *yaml.Dependency) bool { +func filterArchOs(dep *cfg.Dependency) bool { found := false if len(dep.Arch) > 0 { for _, a := range dep.Arch { @@ -205,7 +247,7 @@ } // VcsExists checks if the directory has a local VCS checkout. -func VcsExists(dep *yaml.Dependency, dest string) bool { +func VcsExists(dep *cfg.Dependency, dest string) bool { repo, err := dep.GetRepo(dest) if err != nil { return false @@ -217,8 +259,7 @@ // VcsGet figures out how to fetch a dependency, and then gets it. // // VcsGet installs into the dest. -func VcsGet(dep *yaml.Dependency, dest, home string, cache, cacheGopath, skipGopath bool) error { - +func VcsGet(dep *cfg.Dependency, dest, home string, cache, cacheGopath, skipGopath bool) error { // When not skipping the $GOPATH look in it for a copy of the package if !skipGopath { // Check if the $GOPATH has a viable version to use and if so copy to vendor @@ -414,7 +455,7 @@ if err == errCacheDisabled { Debug("Unable to cache default branch because caching is disabled") } else if err != nil { - Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) + Debug("Error saving %s to cache - Error: %s", repo.Remote(), err) } } } @@ -423,7 +464,7 @@ } // VcsUpdate updates to a particular checkout based on the VCS setting. -func VcsUpdate(dep *yaml.Dependency, vend, home string, force, cache, cacheGopath, skipGopath bool) error { +func VcsUpdate(dep *cfg.Dependency, vend, home string, force, cache, cacheGopath, skipGopath bool) error { Info("Fetching updates for %s.\n", dep.Name) if filterArchOs(dep) { @@ -513,14 +554,23 @@ } // VcsVersion set the VCS version for a checkout. -func VcsVersion(dep *yaml.Dependency, vend string) error { +func VcsVersion(dep *cfg.Dependency, vend string) error { + cwd := path.Join(vend, dep.Name) + // If there is no refernece configured there is nothing to set. if dep.Reference == "" { + // Before exiting update the pinned version + repo, err := dep.GetRepo(cwd) + if err != nil { + return err + } + dep.Pin, err = repo.Version() + if err != nil { + return err + } return nil } - cwd := path.Join(vend, dep.Name) - // When the directory is not empty and has no VCS directory it's // a vendored files situation. empty, err := isDirectoryEmpty(cwd) @@ -585,13 +635,17 @@ Error("Failed to set version to %s: %s\n", dep.Reference, err) return err } + dep.Pin, err = repo.Version() + if err != nil { + return err + } } return nil } // VcsLastCommit gets the last commit ID from the given dependency. -func VcsLastCommit(dep *yaml.Dependency, vend string) (string, error) { +func VcsLastCommit(dep *cfg.Dependency, vend string) (string, error) { cwd := path.Join(vend, dep.Name) repo, err := dep.GetRepo(cwd) if err != nil { @@ -754,6 +808,16 @@ return "" } +// list2map takes a list of packages names and creates a map of normalized names. +func list2map(in []string) map[string]bool { + out := make(map[string]bool, len(in)) + for _, v := range in { + v, _ := NormalizeName(v) + out[v] = true + } + return out +} + func envForDir(dir string) []string { env := os.Environ() return mergeEnvLists([]string{"PWD=" + dir}, env)
diff --git a/cmd/godeps.go b/cmd/godeps.go index 72c6d27..fc62dd7 100644 --- a/cmd/godeps.go +++ b/cmd/godeps.go
@@ -7,8 +7,8 @@ "strings" "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/cfg" "github.com/Masterminds/glide/util" - "github.com/Masterminds/glide/yaml" ) // This file contains commands for working with Godep. @@ -48,19 +48,19 @@ // Params: // - dir (string): the project's directory // -// Returns an []*yaml.Dependency +// Returns an []*cfg.Dependency func ParseGodepGodeps(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { dir := cookoo.GetString("dir", "", p) return parseGodepGodeps(dir) } -func parseGodepGodeps(dir string) ([]*yaml.Dependency, error) { +func parseGodepGodeps(dir string) ([]*cfg.Dependency, error) { path := filepath.Join(dir, "Godeps/Godeps.json") if _, err := os.Stat(path); err != nil { - return []*yaml.Dependency{}, nil + return []*cfg.Dependency{}, nil } Info("Found Godeps.json file.\n") - buf := []*yaml.Dependency{} + buf := []*cfg.Dependency{} godeps := new(Godeps) @@ -96,7 +96,7 @@ } } else { seen[pkg] = true - dep := &yaml.Dependency{Name: pkg, Reference: d.Rev} + dep := &cfg.Dependency{Name: pkg, Reference: d.Rev} if len(sub) > 0 { dep.Subpackages = []string{sub} }
diff --git a/cmd/gpm.go b/cmd/gpm.go index 82ed89d..eef097f 100644 --- a/cmd/gpm.go +++ b/cmd/gpm.go
@@ -7,7 +7,7 @@ "strings" "github.com/Masterminds/cookoo" - "github.com/Masterminds/glide/yaml" + "github.com/Masterminds/glide/cfg" ) // This file contains commands for working with GPM/GVP. @@ -25,22 +25,22 @@ // Params // - dir (string): Directory root. // -// Returns an []*yaml.Dependency +// Returns an []*cfg.Dependency func GPMGodeps(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { dir := cookoo.GetString("dir", "", p) return parseGPMGodeps(dir) } -func parseGPMGodeps(dir string) ([]*yaml.Dependency, error) { +func parseGPMGodeps(dir string) ([]*cfg.Dependency, error) { path := filepath.Join(dir, "Godeps") if i, err := os.Stat(path); err != nil { - return []*yaml.Dependency{}, nil + return []*cfg.Dependency{}, nil } else if i.IsDir() { Info("Godeps is a directory. This is probably a Godep project.\n") - return []*yaml.Dependency{}, nil + return []*cfg.Dependency{}, nil } Info("Found Godeps file.\n") - buf := []*yaml.Dependency{} + buf := []*cfg.Dependency{} file, err := os.Open(path) if err != nil { @@ -50,7 +50,7 @@ for scanner.Scan() { parts, ok := parseGodepsLine(scanner.Text()) if ok { - dep := &yaml.Dependency{Name: parts[0]} + dep := &cfg.Dependency{Name: parts[0]} if len(parts) > 1 { dep.Reference = parts[1] } @@ -70,11 +70,11 @@ dir := cookoo.GetString("dir", "", p) path := filepath.Join(dir, "Godeps-Git") if _, err := os.Stat(path); err != nil { - return []*yaml.Dependency{}, nil + return []*cfg.Dependency{}, nil } Info("Found Godeps-Git file.\n") - buf := []*yaml.Dependency{} + buf := []*cfg.Dependency{} file, err := os.Open(path) if err != nil { @@ -84,7 +84,7 @@ for scanner.Scan() { parts, ok := parseGodepsLine(scanner.Text()) if ok { - dep := &yaml.Dependency{Name: parts[1], Repository: parts[0]} + dep := &cfg.Dependency{Name: parts[1], Repository: parts[0]} if len(parts) > 2 { dep.Reference = parts[2] }
diff --git a/cmd/guess_deps.go b/cmd/guess_deps.go index 871dc6c..2b59533 100644 --- a/cmd/guess_deps.go +++ b/cmd/guess_deps.go
@@ -2,116 +2,122 @@ import ( "os" + "path/filepath" + "sort" "strings" "github.com/Masterminds/cookoo" - "github.com/Masterminds/glide/yaml" + "github.com/Masterminds/glide/cfg" + "github.com/Masterminds/glide/dependency" + "github.com/Masterminds/glide/util" ) // GuessDeps tries to get the dependencies for the current directory. // // Params -// - dirname (string): Directory to use as the base. Default: "." +// - dirname (string): Directory to use as the base. Default: "." +// - skipImport (book): Whether to skip importing from Godep, GPM, and gb func GuessDeps(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { buildContext, err := GetBuildContext() if err != nil { return nil, err } base := p.Get("dirname", ".").(string) - deps := make(map[string]bool) - err = findDeps(buildContext, deps, base, "") - deps = compactDeps(deps) - delete(deps, base) + skipImport := p.Get("skipImport", false).(bool) + name := guessPackageName(buildContext, base) + + Info("Generating a YAML configuration file and guessing the dependencies") + + config := new(cfg.Config) + + // Get the name of the top level package + config.Name = name + + // Import by looking at other package managers and looking over the + // entire directory structure. + + // Attempt to import from other package managers. + if !skipImport { + Info("Attempting to import from other package managers (use --skip-import to skip)") + deps := []*cfg.Dependency{} + absBase, err := filepath.Abs(base) + if err != nil { + return nil, err + } + + if d, ok := guessImportGodep(absBase); ok { + Info("Importing Godep configuration") + Warn("Godep uses commit id versions. Consider using Semantic Versions with Glide") + deps = d + } else if d, ok := guessImportGPM(absBase); ok { + Info("Importing GPM configuration") + deps = d + } else if d, ok := guessImportGB(absBase); ok { + Info("Importing GB configuration") + deps = d + } + + for _, i := range deps { + Info("Found imported reference to %s\n", i.Name) + config.Imports = append(config.Imports, i) + } + } + + // Resolve dependencies by looking at the tree. + r, err := dependency.NewResolver(base) if err != nil { return nil, err } - config := new(yaml.Config) + h := &dependency.DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}} + r.Handler = h - // Get the name of the top level package - config.Name = guessPackageName(buildContext, base) - config.Imports = make([]*yaml.Dependency, len(deps)) - i := 0 - for pa := range deps { - Info("Found reference to %s\n", pa) - d := &yaml.Dependency{ - Name: pa, - } - config.Imports[i] = d - i++ - } - - return config, nil -} - -// findDeps finds all of the dependenices. -// https://golang.org/src/cmd/go/pkg.go#485 -// -// As of Go 1.5 the go command knows about the vendor directory but the go/build -// package does not. It only knows about the GOPATH and GOROOT. In order to look -// for packages in the vendor/ directory we need to fake it for now. -func findDeps(b *BuildCtxt, soFar map[string]bool, name, vpath string) error { - cwd, err := os.Getwd() + sortable, err := r.ResolveLocal(false) if err != nil { - return err + return nil, err } - // Skip cgo pseudo-package. - if name == "C" { - return nil + sort.Strings(sortable) + + vpath := r.VendorDir + if !strings.HasSuffix(vpath, "/") { + vpath = vpath + string(os.PathSeparator) } - pkg, err := b.Import(name, cwd, 0) - if err != nil { - return err - } + for _, pa := range sortable { + n := strings.TrimPrefix(pa, vpath) + root := util.GetRootFromPackage(n) - if pkg.Goroot { - return nil - } - - if vpath == "" { - vpath = pkg.ImportPath - } - - // When the vendor/ directory is present make sure we strip it out before - // registering it as a guess. - realName := strings.TrimPrefix(pkg.ImportPath, vpath+"/vendor/") - - // Before adding a name to the list make sure it's not the name of the - // top level package. - lookupName, _ := NormalizeName(realName) - if vpath != lookupName { - soFar[realName] = true - } - for _, imp := range pkg.Imports { - if !soFar[imp] { - - // Try looking for a dependency as a vendor. If it's not there then - // fall back to a way where it will be found in the GOPATH or GOROOT. - if err := findDeps(b, soFar, vpath+"/vendor/"+imp, vpath); err != nil { - if err := findDeps(b, soFar, imp, vpath); err != nil { - return err + if !config.HasDependency(root) { + Info("Found reference to %s\n", n) + d := &cfg.Dependency{ + Name: root, + } + subpkg := strings.TrimPrefix(n, root) + if len(subpkg) > 0 && subpkg != "/" { + d.Subpackages = []string{subpkg} + } + config.Imports = append(config.Imports, d) + } else { + subpkg := strings.TrimPrefix(n, root) + if len(subpkg) > 0 && subpkg != "/" { + subpkg = strings.TrimPrefix(subpkg, "/") + d := config.Imports.Get(root) + f := false + for _, v := range d.Subpackages { + if v == subpkg { + f = true + } + } + if !f { + Info("Adding sub-package %s to %s\n", subpkg, root) + d.Subpackages = append(d.Subpackages, subpkg) } } } } - return nil -} -// compactDeps registers only top level packages. -// -// Minimize the package imports. For example, importing github.com/Masterminds/cookoo -// and github.com/Masterminds/cookoo/io should not import two packages. Only one -// package needs to be referenced. -func compactDeps(soFar map[string]bool) map[string]bool { - basePackages := make(map[string]bool, len(soFar)) - for k := range soFar { - base, _ := NormalizeName(k) - basePackages[base] = true - } - - return basePackages + return config, nil } // Attempt to guess at the package name at the top level. When unable to detect @@ -124,8 +130,40 @@ pkg, err := b.Import(base, cwd, 0) if err != nil { - return "main" + // There may not be any top level Go source files but the project may + // still be within the GOPATH. + if strings.HasPrefix(base, b.GOPATH) { + p := strings.TrimPrefix(base, b.GOPATH) + return strings.Trim(p, string(os.PathSeparator)) + } } return pkg.ImportPath } + +func guessImportGodep(dir string) ([]*cfg.Dependency, bool) { + d, err := parseGodepGodeps(dir) + if err != nil || len(d) == 0 { + return []*cfg.Dependency{}, false + } + + return d, true +} + +func guessImportGPM(dir string) ([]*cfg.Dependency, bool) { + d, err := parseGPMGodeps(dir) + if err != nil || len(d) == 0 { + return []*cfg.Dependency{}, false + } + + return d, true +} + +func guessImportGB(dir string) ([]*cfg.Dependency, bool) { + d, err := parseGbManifest(dir) + if err != nil || len(d) == 0 { + return []*cfg.Dependency{}, false + } + + return d, true +}
diff --git a/cmd/init_glide.go b/cmd/init_glide.go deleted file mode 100644 index 4d26008..0000000 --- a/cmd/init_glide.go +++ /dev/null
@@ -1,59 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - - "github.com/Masterminds/cookoo" -) - -var yamlTpl = `# Glide YAML configuration file -# Set this to your fully qualified package name, e.g. -# github.com/Masterminds/foo. This should be the -# top level package. -package: %s - -# Declare your project's dependencies. -import: - # Fetch package similar to 'go get': - #- package: github.com/Masterminds/cookoo - # Get and manage a package with Git: - #- package: github.com/Masterminds/cookoo - # # The repository URL - # repo: git@github.com:Masterminds/cookoo.git - # # A tag, branch, or SHA - # ref: 1.1.0 - # # the VCS type (compare to bzr, hg, svn). You should - # # set this if you know it. - # vcs: git -` - -// InitGlide initializes a new Glide project. -// -// Among other things, it creates a default glide.yaml. -// -// Params: -// - filename (string): The name of the glide YAML file. Default is glide.yaml. -// - project (string): The name of the project. Default is 'main'. -func InitGlide(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - fname := p.Get("filename", "glide.yaml").(string) - pname := p.Get("project", "main").(string) - vdir := c.Get("VendorDir", "vendor").(string) - - if _, err := os.Stat(fname); err == nil { - cwd, _ := os.Getwd() - return false, fmt.Errorf("Cowardly refusing to overwrite %s in %s", fname, cwd) - } - f, err := os.Create(fname) - if err != nil { - return false, err - } - - fmt.Fprintf(f, yamlTpl, pname) - f.Close() - - os.MkdirAll(vdir, 0755) - - Info("Initialized. You can now edit '%s'\n", fname) - return true, nil -}
diff --git a/cmd/install.go b/cmd/install.go new file mode 100644 index 0000000..b388947 --- /dev/null +++ b/cmd/install.go
@@ -0,0 +1,143 @@ +package cmd + +import ( + "errors" + "io/ioutil" + "os" + "sync" + + "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/cfg" +) + +// LockFileExists checks if a lock file exists. If not it jumps to the update +// command. +func LockFileExists(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { + fname := p.Get("filename", "glide.lock").(string) + if _, err := os.Stat(fname); err != nil { + Info("Lock file (glide.lock) does not exist. Performing update.") + return false, &cookoo.Reroute{"update"} + } + + return true, nil +} + +// LoadLockFile loads the lock file to the context and checks if it is correct +// for the loaded cfg file. +func LoadLockFile(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { + fname := p.Get("filename", "glide.lock").(string) + conf := p.Get("conf", nil).(*cfg.Config) + + yml, err := ioutil.ReadFile(fname) + if err != nil { + return nil, err + } + lock, err := cfg.LockfileFromYaml(yml) + if err != nil { + return nil, err + } + + hash, err := conf.Hash() + if err != nil { + return nil, err + } + + if hash != lock.Hash { + return nil, errors.New("Lock file does not match YAML configuration. Consider running 'update'") + } + + return lock, nil +} + +// Install installs the dependencies from a Lockfile. +func Install(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { + lock := p.Get("lock", nil).(*cfg.Lockfile) + conf := p.Get("conf", nil).(*cfg.Config) + force := p.Get("force", true).(bool) + home := p.Get("home", "").(string) + cache := p.Get("cache", false).(bool) + cacheGopath := p.Get("cacheGopath", false).(bool) + skipGopath := p.Get("skipGopath", false).(bool) + + cwd, err := VendorPath(c) + if err != nil { + return false, err + } + + // Create a config setup based on the Lockfile data to process with + // existing commands. + newConf := &cfg.Config{} + newConf.Name = conf.Name + + newConf.Imports = make(cfg.Dependencies, len(lock.Imports)) + for k, v := range lock.Imports { + newConf.Imports[k] = &cfg.Dependency{ + Name: v.Name, + Reference: v.Version, + Repository: v.Repository, + VcsType: v.VcsType, + Subpackages: v.Subpackages, + Arch: v.Arch, + Os: v.Os, + } + } + + newConf.DevImports = make(cfg.Dependencies, len(lock.DevImports)) + for k, v := range lock.DevImports { + newConf.DevImports[k] = &cfg.Dependency{ + Name: v.Name, + Reference: v.Version, + Repository: v.Repository, + VcsType: v.VcsType, + Subpackages: v.Subpackages, + Arch: v.Arch, + Os: v.Os, + } + } + + newConf.DeDupe() + + if len(newConf.Imports) == 0 { + Info("No dependencies found. Nothing installed.\n") + return false, nil + } + + // for _, dep := range newConf.Imports { + // if err := VcsUpdate(dep, cwd, home, force, cache, cacheGopath, skipGopath); err != nil { + // Warn("Update failed for %s: %s\n", dep.Name, err) + // } + // } + + done := make(chan struct{}, concurrentWorkers) + in := make(chan *cfg.Dependency, concurrentWorkers) + var wg sync.WaitGroup + + for i := 0; i < concurrentWorkers; i++ { + go func(ch <-chan *cfg.Dependency) { + for { + select { + case dep := <-ch: + if err := VcsUpdate(dep, cwd, home, force, cache, cacheGopath, skipGopath); err != nil { + Warn("Update failed for %s: %s\n", dep.Name, err) + } + wg.Done() + case <-done: + return + } + } + }(in) + } + + for _, dep := range newConf.Imports { + wg.Add(1) + in <- dep + } + + wg.Wait() + // Close goroutines setting the version + for i := 0; i < concurrentWorkers; i++ { + done <- struct{}{} + } + + return newConf, nil +}
diff --git a/cmd/link_package.go b/cmd/link_package.go index 71d1dd4..214b094 100644 --- a/cmd/link_package.go +++ b/cmd/link_package.go
@@ -7,13 +7,13 @@ "strings" "github.com/Masterminds/cookoo" - "github.com/Masterminds/glide/yaml" + "github.com/Masterminds/glide/cfg" ) // LinkPackage creates a symlink to the project within the GOPATH. func LinkPackage(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := c.Get("cfg", "").(*yaml.Config) - pname := p.Get("path", cfg.Name).(string) + conf := c.Get("cfg", "").(*cfg.Config) + pname := p.Get("path", conf.Name).(string) // Per issue #10, this may be nicer to work with in cases where repos are // moved.
diff --git a/cmd/msg.go b/cmd/msg.go index a054cbf..a6dcc14 100644 --- a/cmd/msg.go +++ b/cmd/msg.go
@@ -6,6 +6,7 @@ "fmt" "os" "strings" + "sync" ) // These contanstants map to color codes for shell scripts making them @@ -19,6 +20,8 @@ Pink = "1;35" ) +var outputLock sync.Mutex + // Color returns a string in a certain color. The first argument is a string // containing the color code or a constant from the table above mapped to a code. // @@ -36,8 +39,8 @@ if Quiet { return } - fmt.Fprint(os.Stderr, Color(Green, "[INFO] ")) - Msg(msg, args...) + i := fmt.Sprint(Color(Green, "[INFO] ")) + Msg(i+msg, args...) } // Debug logs debug information @@ -45,49 +48,53 @@ if Quiet || !IsDebugging { return } - fmt.Fprint(os.Stderr, "[DEBUG] ") - Msg(msg, args...) + i := fmt.Sprint("[DEBUG] ") + Msg(i+msg, args...) } // Warn logs a warning func Warn(msg string, args ...interface{}) { - fmt.Fprint(os.Stderr, Color(Yellow, "[WARN] ")) - ErrMsg(msg, args...) + i := fmt.Sprint(Color(Yellow, "[WARN] ")) + ErrMsg(i+msg, args...) } // Error logs and error. func Error(msg string, args ...interface{}) { - fmt.Fprint(os.Stderr, Color(Red, "[ERROR] ")) - ErrMsg(msg, args...) + i := fmt.Sprint(Color(Red, "[ERROR] ")) + ErrMsg(i+msg, args...) } // ErrMsg sends a message to Stderr func ErrMsg(msg string, args ...interface{}) { - if len(args) == 0 { - fmt.Fprint(os.Stderr, msg) - } else { - fmt.Fprintf(os.Stderr, msg, args...) - } + outputLock.Lock() + defer outputLock.Unlock() - // Get rid of the annoying fact that messages need \n at the end, but do - // it in a backward compatible way. + // If messages don't have a newline on the end we add one. + e := "" if !strings.HasSuffix(msg, "\n") { - fmt.Fprintln(os.Stderr) + e = "\n" + } + if len(args) == 0 { + fmt.Fprint(os.Stderr, msg+e) + } else { + fmt.Fprintf(os.Stderr, msg+e, args...) } } // Msg prints a message with optional arguments, that can be printed, of // varying types. func Msg(msg string, args ...interface{}) { - if len(args) == 0 { - fmt.Fprint(os.Stderr, msg) - } else { - fmt.Fprintf(os.Stderr, msg, args...) - } + outputLock.Lock() + defer outputLock.Unlock() - // Get rid of the annoying fact that messages need \n at the end, but do - // it in a backward compatible way. + // If messages don't have a newline on the end we add one. + e := "" if !strings.HasSuffix(msg, "\n") { - fmt.Fprintln(os.Stderr) + e = "\n" + } + if len(args) == 0 { + fmt.Fprint(os.Stderr, msg+e) + } else { + fmt.Fprintf(os.Stderr, msg+e, args...) } }
diff --git a/cmd/print_name.go b/cmd/print_name.go index 7245f1d..6a6d26d 100644 --- a/cmd/print_name.go +++ b/cmd/print_name.go
@@ -4,14 +4,14 @@ "fmt" "github.com/Masterminds/cookoo" - "github.com/Masterminds/glide/yaml" + "github.com/Masterminds/glide/cfg" ) // PrintName prints the name of the project. // // This comes from Config.Name. func PrintName(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := p.Get("conf", nil).(*yaml.Config) - fmt.Println(cfg.Name) + conf := p.Get("conf", nil).(*cfg.Config) + fmt.Println(conf.Name) return nil, nil }
diff --git a/cmd/rebuild.go b/cmd/rebuild.go index d13c414..a781503 100644 --- a/cmd/rebuild.go +++ b/cmd/rebuild.go
@@ -8,16 +8,16 @@ "strings" "github.com/Masterminds/cookoo" - "github.com/Masterminds/glide/yaml" + "github.com/Masterminds/glide/cfg" ) // Rebuild runs 'go build' in a directory. // // Params: -// - conf: the *yaml.Config. +// - conf: the *cfg.Config. // func Rebuild(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := p.Get("conf", nil).(*yaml.Config) + conf := p.Get("conf", nil).(*cfg.Config) vpath, err := VendorPath(c) if err != nil { return nil, err @@ -25,12 +25,12 @@ Info("Building dependencies.\n") - if len(cfg.Imports) == 0 { + if len(conf.Imports) == 0 { Info("No dependencies found. Nothing built.\n") return true, nil } - for _, dep := range cfg.Imports { + for _, dep := range conf.Imports { if err := buildDep(c, dep, vpath); err != nil { Warn("Failed to build %s: %s\n", dep.Name, err) } @@ -39,7 +39,7 @@ return true, nil } -func buildDep(c cookoo.Context, dep *yaml.Dependency, vpath string) error { +func buildDep(c cookoo.Context, dep *cfg.Dependency, vpath string) error { if len(dep.Subpackages) == 0 { buildPath(c, dep.Name) }
diff --git a/cmd/tree.go b/cmd/tree.go index 8983c4a..60181ec 100644 --- a/cmd/tree.go +++ b/cmd/tree.go
@@ -9,6 +9,8 @@ "strings" "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/dependency" + "github.com/Masterminds/glide/msg" ) // Tree prints a tree representing dependencies. @@ -40,40 +42,55 @@ // ListDeps lists all of the dependencies of the current project. // // Params: +// - dir (string): basedir +// - deep (bool): whether to do a deep scan or a shallow scan // // Returns: // func ListDeps(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - buildContext, err := GetBuildContext() - if err != nil { - return nil, err - } basedir := p.Get("dir", ".").(string) - myName := guessPackageName(buildContext, basedir) + deep := p.Get("deep", true).(bool) - basedir, err = filepath.Abs(basedir) + basedir, err := filepath.Abs(basedir) if err != nil { return nil, err } - direct := map[string]*pinfo{} - d := walkDeps(buildContext, basedir, myName) - for _, i := range d { - listDeps(buildContext, direct, i, basedir) + r, err := dependency.NewResolver(basedir) + if err != nil { + return nil, err } + h := &dependency.DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}} + r.Handler = h - sortable := make([]string, len(direct)) - i := 0 - for k := range direct { - sortable[i] = k - i++ + sortable, err := r.ResolveLocal(deep) + if err != nil { + return nil, err } sort.Strings(sortable) + fmt.Println("INSTALLED packages:") for _, k := range sortable { - t := direct[k].PType - fmt.Printf("%s (Location: %s)\n", k, ptypeString(t)) + v, err := filepath.Rel(basedir, k) + if err != nil { + msg.Warn("Failed to Rel path: %s", err) + v = k + } + fmt.Printf("\t%s\n", v) + } + + if len(h.Missing) > 0 { + fmt.Println("\nMISSING packages:") + for _, pkg := range h.Missing { + fmt.Printf("\t%s\n", pkg) + } + } + if len(h.Gopath) > 0 { + fmt.Println("\nGOPATH packages:") + for _, pkg := range h.Gopath { + fmt.Printf("\t%s\n", pkg) + } } return nil, nil @@ -170,13 +187,21 @@ } // Recurse backward to scan other vendor/ directories - for wd := cwd; wd != "/"; wd = filepath.Dir(wd) { - p = filepath.Join(wd, "vendor", name) - if fi, err = os.Stat(p); err == nil && (fi.IsDir() || isLink(fi)) { - info.Path = p - info.PType = ptypeVendor - info.Vendored = true - return info + // If the cwd isn't an absolute path walking upwards looking for vendor/ + // folders can get into an infinate loop. + abs, err := filepath.Abs(cwd) + if err != nil { + abs = cwd + } + if abs != "." { + for wd := abs; wd != "/"; wd = filepath.Dir(wd) { + p = filepath.Join(wd, "vendor", name) + if fi, err = os.Stat(p); err == nil && (fi.IsDir() || isLink(fi)) { + info.Path = p + info.PType = ptypeVendor + info.Vendored = true + return info + } } } // Check $GOPATH
diff --git a/cmd/update_references.go b/cmd/update_references.go deleted file mode 100644 index 3e602fa..0000000 --- a/cmd/update_references.go +++ /dev/null
@@ -1,113 +0,0 @@ -package cmd - -import ( - "path" - - "github.com/Masterminds/cookoo" - "github.com/Masterminds/glide/yaml" -) - -// UpdateReferences updates the revision numbers on all of the imports. -// -// If a `packages` list is supplied, only the given base packages will -// be updated. -// -// Params: -// - conf (*yaml.Config): Configuration -// - packages ([]string): A list of packages to update. Default is all packages. -func UpdateReferences(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := p.Get("conf", &yaml.Config{}).(*yaml.Config) - plist := p.Get("packages", []string{}).([]string) - vend, _ := VendorPath(c) - pkgs := list2map(plist) - restrict := len(pkgs) > 0 - - cwd, err := VendorPath(c) - if err != nil { - return false, err - } - - if len(cfg.Imports) == 0 { - return cfg, nil - } - - // Walk the dependency tree to discover all the packages to pin. - packages := make([]string, len(cfg.Imports)) - for i, v := range cfg.Imports { - packages[i] = v.Name - } - deps := make(map[string]*yaml.Dependency, len(cfg.Imports)) - for _, imp := range cfg.Imports { - deps[imp.Name] = imp - } - f := &flattening{cfg, vend, vend, deps, packages} - err = discoverDependencyTree(f) - if err != nil { - return cfg, err - } - - exportFlattenedDeps(cfg, deps) - - err = cfg.DeDupe() - if err != nil { - return cfg, err - } - - for _, imp := range cfg.Imports { - if restrict && !pkgs[imp.Name] { - Debug("===> Skipping %q", imp.Name) - continue - } - commit, err := VcsLastCommit(imp, cwd) - if err != nil { - Warn("Could not get commit on %s: %s", imp.Name, err) - } - imp.Reference = commit - } - - return cfg, nil -} - -func discoverDependencyTree(f *flattening) error { - Debug("---> Inspecting %s for dependencies (%d packages).\n", f.curr, len(f.scan)) - scanned := map[string]bool{} - for _, imp := range f.scan { - Debug("----> Scanning %s", imp) - base := path.Join(f.top, imp) - mod := []string{} - if m, ok := mergeGlide(base, imp, f.deps, f.top); ok { - mod = m - } else if m, ok = mergeGodep(base, imp, f.deps, f.top); ok { - mod = m - } else if m, ok = mergeGPM(base, imp, f.deps, f.top); ok { - mod = m - } else if m, ok = mergeGb(base, imp, f.deps, f.top); ok { - mod = m - } else if m, ok = mergeGuess(base, imp, f.deps, f.top, scanned); ok { - mod = m - } - - if len(mod) > 0 { - Debug("----> Looking for dependencies in %q (%d)", imp, len(mod)) - f2 := &flattening{ - conf: f.conf, - top: f.top, - curr: base, - deps: f.deps, - scan: mod} - discoverDependencyTree(f2) - } - } - - return nil -} - -// list2map takes a list of packages names and creates a map of normalized names. -func list2map(in []string) map[string]bool { - out := make(map[string]bool, len(in)) - for _, v := range in { - v, _ := NormalizeName(v) - out[v] = true - } - return out -}
diff --git a/cmd/vendored.go b/cmd/vendored.go index 8eb1b8f..b0e3f64 100644 --- a/cmd/vendored.go +++ b/cmd/vendored.go
@@ -5,7 +5,7 @@ "path" "github.com/Masterminds/cookoo" - "github.com/Masterminds/glide/yaml" + "github.com/Masterminds/glide/cfg" "github.com/Masterminds/vcs" ) @@ -15,17 +15,17 @@ // VendoredCleanUp should be a suffix to UpdateImports. func VendoredSetup(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { update := p.Get("update", true).(bool) - cfg := p.Get("conf", nil).(*yaml.Config) + conf := p.Get("conf", nil).(*cfg.Config) if update != true { - return cfg, nil + return conf, nil } vend, err := VendorPath(c) if err != nil { - return cfg, err + return conf, err } - for _, dep := range cfg.Imports { + for _, dep := range conf.Imports { cwd := path.Join(vend, dep.Name) // When the directory is not empty and has no VCS directory it's @@ -50,11 +50,11 @@ } } - return cfg, nil + return conf, nil } // VendoredCleanUp is a command that cleans up vendored codebases after an update. -// If enabled (via update) it removed the VCS info from updated vendored +// If enabled (via update) it removes the VCS info from updated vendored // packages. This should be a suffix to UpdateImports and VendoredSetup should // be a prefix to UpdateImports. func VendoredCleanUp(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { @@ -62,14 +62,14 @@ if update != true { return false, nil } - cfg := p.Get("conf", nil).(*yaml.Config) + conf := p.Get("conf", nil).(*cfg.Config) vend, err := VendorPath(c) if err != nil { return false, err } - for _, dep := range cfg.Imports { + for _, dep := range conf.Imports { if dep.UpdateAsVendored == true { Info("Cleaning up vendored package %s\n", dep.Name)
diff --git a/cmd/yaml.go b/cmd/yaml.go index fe302d6..ef41087 100644 --- a/cmd/yaml.go +++ b/cmd/yaml.go
@@ -8,7 +8,8 @@ "strings" "github.com/Masterminds/cookoo" - "github.com/Masterminds/glide/yaml" + "github.com/Masterminds/glide/cfg" + "github.com/Masterminds/glide/util" ) // ParseYaml parses the glide.yaml format and returns a Configuration object. @@ -17,7 +18,7 @@ // - filename (string): YAML filename as a string // // Returns: -// - *yaml.Config: The configuration. +// - *cfg.Config: The configuration. func ParseYaml(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { fname := p.Get("filename", "glide.yaml").(string) //conf := new(Config) @@ -25,12 +26,12 @@ if err != nil { return nil, err } - cfg, err := yaml.FromYaml(string(yml)) + conf, err := cfg.ConfigFromYaml(yml) if err != nil { return nil, err } - return cfg, nil + return conf, nil } // ParseYamlString parses a YAML string. This is similar but different to @@ -40,32 +41,44 @@ // - yaml (string): YAML as a string. // // Returns: -// - *yaml.Config: The configuration. +// - *cfg.Config: The configuration. func ParseYamlString(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { yamlString := p.Get("yaml", "").(string) - cfg, err := yaml.FromYaml(string(yamlString)) + conf, err := cfg.ConfigFromYaml([]byte(yamlString)) if err != nil { return nil, err } - return cfg, nil + return conf, nil } -// WriteYaml writes a yaml.Node to the console as a string. +// GuardYaml protects the glide yaml file from being overwritten. +func GuardYaml(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { + fname := p.Get("filename", "glide.yaml").(string) + if _, err := os.Stat(fname); err == nil { + cwd, _ := os.Getwd() + return false, fmt.Errorf("Cowardly refusing to overwrite %s in %s", fname, cwd) + } + + return true, nil +} + +// WriteYaml writes the config as YAML. // // Params: -// - conf: A *yaml.Config to render. +// - conf: A *cfg.Config to render. // - out (io.Writer): An output stream to write to. Default is os.Stdout. // - filename (string): If set, the file will be opened and the content will be written to it. func WriteYaml(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := p.Get("conf", nil).(*yaml.Config) + conf := p.Get("conf", nil).(*cfg.Config) toStdout := p.Get("toStdout", true).(bool) - yml, err := yaml.ToYaml(cfg) + data, err := conf.Marshal() if err != nil { return nil, err } + var out io.Writer if nn, ok := p.Has("filename"); ok && len(nn.(string)) > 0 { file, err := os.Create(nn.(string)) @@ -73,22 +86,51 @@ } defer file.Close() out = io.Writer(file) - fmt.Fprint(out, yml) + //fmt.Fprint(out, yml) + out.Write(data) } else if toStdout { out = p.Get("out", os.Stdout).(io.Writer) - fmt.Fprint(out, yml) + //fmt.Fprint(out, yml) + out.Write(data) } // Otherwise we supress output. return true, nil } -// AddDependencies adds a list of *Dependency objects to the given *yaml.Config. +// WriteLock writes the lock as YAML. +// +// Params: +// - lockfile: A *cfg.Lockfile to render. +// - out (io.Writer): An output stream to write to. Default is os.Stdout. +func WriteLock(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { + lockfile := p.Get("lockfile", nil).(*cfg.Lockfile) + + Info("Writing glide.lock file") + + data, err := lockfile.Marshal() + if err != nil { + return nil, err + } + + var out io.Writer + file, err := os.Create("glide.lock") + if err != nil { + return false, err + } + defer file.Close() + out = io.Writer(file) + out.Write(data) + + return true, nil +} + +// AddDependencies adds a list of *Dependency objects to the given *cfg.Config. // // This is used to merge in packages from other sources or config files. func AddDependencies(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - deps := p.Get("dependencies", []*yaml.Dependency{}).([]*yaml.Dependency) - config := p.Get("conf", nil).(*yaml.Config) + deps := p.Get("dependencies", []*cfg.Dependency{}).([]*cfg.Dependency) + config := p.Get("conf", nil).(*cfg.Config) // Make a set of existing package names for quick comparison. pkgSet := make(map[string]bool, len(config.Imports)) @@ -113,13 +155,24 @@ // For example, golang.org/x/crypto/ssh becomes golang.org/x/crypto. 'ssh' is // returned as extra data. func NormalizeName(name string) (string, string) { - parts := strings.SplitN(name, "/", 4) - extra := "" - if len(parts) < 3 { - return name, extra + root := util.GetRootFromPackage(name) + extra := strings.TrimPrefix(name, root) + if len(extra) > 0 && extra != "/" { + extra = strings.TrimPrefix(extra, "/") + } else { + // If extra is / (which is what it would be here) we want to return "" + extra = "" } - if len(parts) == 4 { - extra = parts[3] - } - return strings.Join(parts[0:3], "/"), extra + + return root, extra + + // parts := strings.SplitN(name, "/", 4) + // extra := "" + // if len(parts) < 3 { + // return name, extra + // } + // if len(parts) == 4 { + // extra = parts[3] + // } + // return strings.Join(parts[0:3], "/"), extra }
diff --git a/cmd/yaml_test.go b/cmd/yaml_test.go index 460d1ae..a1e3e6b 100644 --- a/cmd/yaml_test.go +++ b/cmd/yaml_test.go
@@ -4,7 +4,7 @@ "testing" "github.com/Masterminds/cookoo" - "github.com/Masterminds/glide/yaml" + "github.com/Masterminds/glide/cfg" ) var yamlFile = ` @@ -32,55 +32,36 @@ - package: github.com/kylelemons/go-gypsy ` -var childYamlFile = ` -package: fake/testing/more -import: - - package: github.com/kylelemons/go-gypsy - subpackages: - - yaml -` - func TestFromYaml(t *testing.T) { reg, router, cxt := cookoo.Cookoo() reg.Route("t", "Testing"). - Does(ParseYamlString, "cfg").Using("yaml").WithDefault(yamlFile). - Does(ParseYamlString, "childCfg").Using("yaml").WithDefault(childYamlFile) + Does(ParseYamlString, "cfg").Using("yaml").WithDefault(yamlFile) if err := router.HandleRequest("t", cxt, false); err != nil { t.Errorf("Failed to parse YAML: %s", err) } - cfg := cxt.Get("cfg", nil).(*yaml.Config) - cfgChild := cxt.Get("childCfg", nil).(*yaml.Config) - cfgChild.Parent = cfg + conf := cxt.Get("cfg", nil).(*cfg.Config) - if cfg.Name != "fake/testing" { - t.Errorf("Expected name to be 'fake/teting', not '%s'", cfg.Name) + if conf.Name != "fake/testing" { + t.Errorf("Expected name to be 'fake/teting', not '%s'", conf.Name) } - if len(cfg.Imports) != 3 { - t.Errorf("Expected 3 imports, got %d", len(cfg.Imports)) + if len(conf.Imports) != 3 { + t.Errorf("Expected 3 imports, got %d", len(conf.Imports)) } - if cfg.Parent != nil { - t.Error("Expected root glide Parent to be nil") - } - - if cfg.Imports.Get("github.com/Masterminds/convert") == nil { + if conf.Imports.Get("github.com/Masterminds/convert") == nil { t.Error("Expected Imports.Get to return Dependency") } - if cfg.Imports.Get("github.com/doesnot/exist") != nil { + if conf.Imports.Get("github.com/doesnot/exist") != nil { t.Error("Execpted Imports.Get to return nil") } - if cfgChild.HasRecursiveDependency("github.com/Masterminds/convert") == false { - t.Errorf("Expected to find a recursive dependency") - } - - var imp *yaml.Dependency - for _, d := range cfg.Imports { + var imp *cfg.Dependency + for _, d := range conf.Imports { if d.Name == "github.com/Masterminds/convert" { imp = d } @@ -119,7 +100,7 @@ t.Errorf("Got wrong reference.") } - if len(cfg.DevImports) != 1 { + if len(conf.DevImports) != 1 { t.Errorf("Expected one dev import.") } @@ -129,9 +110,9 @@ packages := map[string]string{ "github.com/Masterminds/cookoo/web/io/foo": "github.com/Masterminds/cookoo", "golang.org/x/crypto/ssh": "golang.org/x/crypto", - "technosophos.me/x/totally/fake/package": "technosophos.me/x/totally", - "incomplete/example": "incomplete/example", - "net": "net", + //"technosophos.me/x/totally/fake/package": "technosophos.me/x/totally", + "incomplete/example": "incomplete/example", + "net": "net", } for start, expected := range packages { if finish, extra := NormalizeName(start); expected != finish {
diff --git a/dependency/resolver.go b/dependency/resolver.go new file mode 100644 index 0000000..943c3aa --- /dev/null +++ b/dependency/resolver.go
@@ -0,0 +1,530 @@ +package dependency + +import ( + "container/list" + "go/build" + "os" + "path/filepath" + "strings" + + "github.com/Masterminds/glide/cfg" + "github.com/Masterminds/glide/msg" +) + +// MissingPackageHandler handles the case where a package is missing during scanning. +// +// It returns true if the package can be passed to the resolver, false otherwise. +// False may be returned even if error is nil. +type MissingPackageHandler interface { + // NotFound is called when the Resolver fails to find a package with the given name. + // + // NotFound returns true when the resolver should attempt to re-resole the + // dependency (e.g. when NotFound has gone and fetched the missing package). + // + // When NotFound returns false, the Resolver does not try to do any additional + // work on the missing package. + // + // NotFound only returns errors when it fails to perform its internal goals. + // When it returns false with no error, this indicates that the handler did + // its job, but the resolver should not do any additional work on the + // package. + NotFound(pkg string) (bool, error) + + // OnGopath is called when the Resolver finds a dependency, but it's only on GOPATH. + // + // OnGopath provides an opportunity to copy, move, warn, or ignore cases like this. + // + // OnGopath returns true when the resolver should attempt to re-resolve the + // dependency (e.g. when the dependency is copied to a new location). + // + // When OnGopath returns false, the Resolver does not try to do any additional + // work on the package. + // + // An error indicates that OnGopath cannot complete its intended operation. + // Not all false results are errors. + OnGopath(pkg string) (bool, error) +} + +// DefaultMissingPackageHandler is the default handler for missing packages. +// +// When asked to handle a missing package, it will report the miss as a warning, +// and then store the package in the Missing slice for later access. +type DefaultMissingPackageHandler struct { + Missing []string + Gopath []string +} + +// NotFound prints a warning and then stores the package name in Missing. +// +// It never returns an error, and it always returns false. +func (d *DefaultMissingPackageHandler) NotFound(pkg string) (bool, error) { + msg.Warn("Package %s is not installed", pkg) + d.Missing = append(d.Missing, pkg) + return false, nil +} + +func (d *DefaultMissingPackageHandler) OnGopath(pkg string) (bool, error) { + msg.Warn("Package %s is only on GOPATH.", pkg) + d.Gopath = append(d.Gopath, pkg) + return false, nil +} + +// Resolver resolves a dependency tree. +// +// It operates in two modes: +// - local resolution (ResolveLocal) determines the dependencies of the local project. +// - vendor resolving (Resolve, ResolveAll) determines the dependencies of vendored +// projects. +// +// Local resolution is for guessing initial dependencies. Vendor resolution is +// for determining vendored dependencies. +type Resolver struct { + Handler MissingPackageHandler + basedir string + VendorDir string + BuildContext build.Context + seen map[string]bool + + // Items already in the queue. + alreadyQ map[string]bool + + // findCache caches hits from Find. This reduces the number of filesystem + // touches that have to be done for dependency resolution. + findCache map[string]*PkgInfo +} + +// NewResolver returns a new Resolver initialized with the DefaultMissingPackageHandler. +// +// This will return an error if the given path does not meet the basic criteria +// for a Go source project. For example, basedir must have a vendor subdirectory. +// +// The BuildContext uses the "go/build".Default to resolve dependencies. +func NewResolver(basedir string) (*Resolver, error) { + + var err error + basedir, err = filepath.Abs(basedir) + if err != nil { + return nil, err + } + vdir := filepath.Join(basedir, "vendor") + + r := &Resolver{ + Handler: &DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}}, + basedir: basedir, + VendorDir: vdir, + BuildContext: build.Default, + seen: map[string]bool{}, + alreadyQ: map[string]bool{}, + findCache: map[string]*PkgInfo{}, + } + + // TODO: Make sure the build context is correctly set up. Especially in + // regards to GOROOT, which is not always set. + + return r, nil +} + +// Resolve takes a package name and returns all of the imported package names. +// +// If a package is not found, this calls the Fetcher. If the Fetcher returns +// true, it will re-try traversing that package for dependencies. Otherwise it +// will add that package to the deps array and continue on without trying it. +// And if the Fetcher returns an error, this will stop resolution and return +// the error. +// +// If basepath is set to $GOPATH, this will start from that package's root there. +// If basepath is set to a project's vendor path, the scanning will begin from +// there. +func (r *Resolver) Resolve(pkg, basepath string) ([]string, error) { + target := filepath.Join(basepath, pkg) + //msg.Debug("Scanning %s", target) + l := list.New() + l.PushBack(target) + return r.resolveList(l) +} + +// ResolveLocal resolves dependencies for the current project. +// +// This begins with the project, builds up a list of external dependencies. +// +// If the deep flag is set to true, this will then resolve all of the dependencies +// of the dependencies it has found. If not, it will return just the packages that +// the base project relies upon. +func (r *Resolver) ResolveLocal(deep bool) ([]string, error) { + // We build a list of local source to walk, then send this list + // to resolveList. + l := list.New() + alreadySeen := map[string]bool{} + err := filepath.Walk(r.basedir, func(path string, fi os.FileInfo, err error) error { + if err != nil && err != filepath.SkipDir { + return err + } + if !fi.IsDir() { + return nil + } + if !srcDir(fi) { + return filepath.SkipDir + } + + // Scan for dependencies, and anything that's not part of the local + // package gets added to the scan list. + p, err := r.BuildContext.ImportDir(path, 0) + if err != nil { + if strings.HasPrefix(err.Error(), "no buildable Go source") { + return nil + } + return err + } + + // We are only looking for dependencies in vendor. No root, cgo, etc. + for _, imp := range p.Imports { + if alreadySeen[imp] { + continue + } + alreadySeen[imp] = true + info := r.FindPkg(imp) + switch info.Loc { + case LocUnknown, LocVendor: + l.PushBack(filepath.Join(r.VendorDir, imp)) // Do we need a path on this? + case LocGopath: + if !strings.HasPrefix(info.Path, r.basedir) { + // FIXME: This is a package outside of the project we're + // scanning. It should really be on vendor. But we don't + // want it to reference GOPATH. We want it to be detected + // and moved. + l.PushBack(filepath.Join(r.VendorDir, imp)) + } + } + } + + return nil + }) + + if err != nil { + msg.Error("Failed to build an initial list of packages to scan: %s", err) + return []string{}, err + } + + if deep { + return r.resolveList(l) + } + + // If we're not doing a deep scan, we just convert the list into an + // array and return. + res := make([]string, 0, l.Len()) + for e := l.Front(); e != nil; e = e.Next() { + res = append(res, e.Value.(string)) + } + return res, nil +} + +// ResolveAll takes a list of packages and returns an inclusive list of all +// vendored dependencies. +// +// While this will scan all of the source code it can find, it will only return +// packages that were either explicitly passed in as deps, or were explicitly +// imported by the code. +// +// Packages that are either CGO or on GOROOT are ignored. Packages that are +// on GOPATH, but not vendored currently generate a warning. +// +// If one of the passed in packages does not exist in the vendor directory, +// an error is returned. +func (r *Resolver) ResolveAll(deps []*cfg.Dependency) ([]string, error) { + queue := sliceToQueue(deps, r.VendorDir) + return r.resolveList(queue) +} + +// resolveList takes a list and resolves it. +func (r *Resolver) resolveList(queue *list.List) ([]string, error) { + + var failedDep string + for e := queue.Front(); e != nil; e = e.Next() { + dep := e.Value.(string) + //msg.Warn("#### %s ####", dep) + //msg.Info("Seen Count: %d", len(r.seen)) + // Catch the outtermost dependency. + failedDep = dep + err := filepath.Walk(dep, func(path string, fi os.FileInfo, err error) error { + if err != nil && err != filepath.SkipDir { + return err + } + + // Skip files. + if !fi.IsDir() { + return nil + } + // Skip dirs that are not source. + if !srcDir(fi) { + //msg.Debug("Skip resource %s", fi.Name()) + return filepath.SkipDir + } + + // Anything that comes through here has already been through + // the queue. + r.alreadyQ[path] = true + e := r.queueUnseen(path, queue) + if err != nil { + failedDep = path + //msg.Error("Failed to fetch dependency %s: %s", path, err) + } + return e + }) + if err != nil && err != filepath.SkipDir { + msg.Error("Dependency %s failed to resolve: %s.", failedDep, err) + return []string{}, err + } + } + + res := make([]string, 0, queue.Len()) + for e := queue.Front(); e != nil; e = e.Next() { + res = append(res, e.Value.(string)) + } + + return res, nil +} + +// queueUnseenImports scans a package's imports and adds any new ones to the +// processing queue. +func (r *Resolver) queueUnseen(pkg string, queue *list.List) error { + // A pkg is marked "seen" as soon as we have inspected it the first time. + // Seen means that we have added all of its imports to the list. + + // Already queued indicates that we've either already put it into the queue + // or intentionally not put it in the queue for fatal reasons (e.g. no + // buildable source). + + deps, err := r.imports(pkg) + if err != nil && !strings.HasPrefix(err.Error(), "no buildable Go source") { + msg.Error("Could not find %s: %s", pkg, err) + return err + // NOTE: If we uncomment this, we get lots of "no buildable Go source" errors, + // which don't ever seem to be helpful. They don't actually indicate an error + // condition, and it's perfectly okay to run into that condition. + //} else if err != nil { + // msg.Warn(err.Error()) + } + + for _, d := range deps { + if _, ok := r.alreadyQ[d]; !ok { + r.alreadyQ[d] = true + queue.PushBack(d) + } + } + return nil +} + +// imports gets all of the imports for a given package. +// +// If the package is in GOROOT, this will return an empty list (but not +// an error). +// If it cannot resolve the pkg, it will return an error. +func (r *Resolver) imports(pkg string) ([]string, error) { + + // If this pkg is marked seen, we don't scan it again. + if _, ok := r.seen[pkg]; ok { + msg.Debug("Already saw %s", pkg) + return []string{}, nil + } + + // FIXME: On error this should try to NotFound to the dependency, and then import + // it again. + p, err := r.BuildContext.ImportDir(pkg, 0) + if err != nil { + return []string{}, err + } + + // It is okay to scan a package more than once. In some cases, this is + // desirable because the package can change between scans (e.g. as a result + // of a failed scan resolving the situation). + msg.Debug("=> Scanning %s (%s)", p.ImportPath, pkg) + r.seen[pkg] = true + + // Optimization: If it's in GOROOT, it has no imports worth scanning. + if p.Goroot { + return []string{}, nil + } + + // We are only looking for dependencies in vendor. No root, cgo, etc. + buf := []string{} + for _, imp := range p.Imports { + info := r.FindPkg(imp) + switch info.Loc { + case LocUnknown: + // Do we resolve here? + found, err := r.Handler.NotFound(imp) + if err != nil { + msg.Error("Failed to fetch %s: %s", imp, err) + } + if found { + buf = append(buf, filepath.Join(r.VendorDir, imp)) + continue + } + r.seen[info.Path] = true + case LocVendor: + //msg.Debug("Vendored: %s", imp) + buf = append(buf, info.Path) + case LocGopath: + found, err := r.Handler.OnGopath(imp) + if err != nil { + msg.Error("Failed to fetch %s: %s", imp, err) + } + // If the Handler marks this as found, we drop it into the buffer + // for subsequent processing. Otherwise, we assume that we're + // in a less-than-perfect, but functional, situation. + if found { + buf = append(buf, filepath.Join(r.VendorDir, imp)) + continue + } + msg.Warn("Package %s is on GOPATH, but not vendored. Ignoring.", imp) + r.seen[info.Path] = true + default: + // Local packages are an odd case. CGO cannot be scanned. + msg.Debug("===> Skipping %s", imp) + } + } + + return buf, nil +} + +// sliceToQueue is a special-purpose function for unwrapping a slice of +// dependencies into a queue of fully qualified paths. +func sliceToQueue(deps []*cfg.Dependency, basepath string) *list.List { + l := list.New() + for _, e := range deps { + l.PushBack(filepath.Join(basepath, e.Name)) + } + return l +} + +// PkgLoc describes the location of the package. +type PkgLoc uint8 + +const ( + // LocUnknown indicates the package location is unknown (probably not present) + LocUnknown PkgLoc = iota + // LocLocal inidcates that the package is in a local dir, not GOPATH or GOROOT. + LocLocal + // LocVendor indicates that the package is in a vendor/ dir + LocVendor + // LocGopath inidcates that the package is in GOPATH + LocGopath + // LocGoroot indicates that the package is in GOROOT + LocGoroot + // LocCgo indicates that the package is a a CGO package + LocCgo +) + +type PkgInfo struct { + Name, Path string + Vendored bool + Loc PkgLoc +} + +// FindPkg takes a package name and attempts to find it on the filesystem +// +// The resulting PkgInfo will indicate where it was found. +func (r *Resolver) FindPkg(name string) *PkgInfo { + // We cachae results for FindPkg to reduce the number of filesystem ops + // that we have to do. This is a little risky because certain directories, + // like GOPATH, can be modified while we're running an operation, and + // render the cache inaccurate. + // + // Unfound items (LocUnkown) are never cached because we assume that as + // part of the response, the Resolver may fetch that dependency. + if i, ok := r.findCache[name]; ok { + //msg.Info("Cache hit on %s", name) + return i + } + + // 502 individual packages scanned. + // No cache: + // glide -y etcd.yaml list 0.27s user 0.19s system 85% cpu 0.534 total + // With cache: + // glide -y etcd.yaml list 0.22s user 0.15s system 85% cpu 0.438 total + + var p string + info := &PkgInfo{ + Name: name, + } + + // Check _only_ if this dep is in the current vendor directory. + p = filepath.Join(r.VendorDir, name) + if pkgExists(p) { + info.Path = p + info.Loc = LocVendor + info.Vendored = true + r.findCache[name] = info + return info + } + + // TODO: Do we need this if we always flatten? + // Recurse backward to scan other vendor/ directories + //for wd := cwd; wd != "/"; wd = filepath.Dir(wd) { + //p = filepath.Join(wd, "vendor", name) + //if fi, err = os.Stat(p); err == nil && (fi.IsDir() || isLink(fi)) { + //info.Path = p + //info.PType = ptypeVendor + //info.Vendored = true + //return info + //} + //} + + // Check $GOPATH + for _, rr := range strings.Split(r.BuildContext.GOPATH, ":") { + p = filepath.Join(rr, "src", name) + if pkgExists(p) { + info.Path = p + info.Loc = LocGopath + r.findCache[name] = info + return info + } + } + + // Check $GOROOT + for _, rr := range strings.Split(r.BuildContext.GOROOT, ":") { + p = filepath.Join(rr, "src", name) + if pkgExists(p) { + info.Path = p + info.Loc = LocGoroot + r.findCache[name] = info + return info + } + } + + // Finally, if this is "C", we're dealing with cgo + if name == "C" { + info.Loc = LocCgo + r.findCache[name] = info + } + + return info +} + +func pkgExists(path string) bool { + fi, err := os.Stat(path) + return err == nil && (fi.IsDir() || isLink(fi)) +} + +// isLink returns true if the given FileInfo is a symbolic link. +func isLink(fi os.FileInfo) bool { + return fi.Mode()&os.ModeSymlink == os.ModeSymlink +} + +func srcDir(fi os.FileInfo) bool { + if !fi.IsDir() { + return false + } + + // Ignore _foo and .foo + if strings.HasPrefix(fi.Name(), "_") || strings.HasPrefix(fi.Name(), ".") { + return false + } + + // Ignore testdata. For now, ignore vendor. + if fi.Name() == "testdata" || fi.Name() == "vendor" { + return false + } + + return true +}
diff --git a/dependency/resolver_test.go b/dependency/resolver_test.go new file mode 100644 index 0000000..46268f2 --- /dev/null +++ b/dependency/resolver_test.go
@@ -0,0 +1,99 @@ +package dependency + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Masterminds/glide/cfg" +) + +func TestResolveLocalShallow(t *testing.T) { + r, err := NewResolver("../") + if err != nil { + t.Fatal(err) + } + + l, err := r.ResolveLocal(false) + if err != nil { + t.Fatalf("Failed to resolve: %s", err) + } + + expect := []string{ + "github.com/Masterminds/cookoo", + "github.com/Masterminds/semver", + "github.com/Masterminds/vcs", + "gopkg.in/yaml.v2", + "github.com/codegangsta/cli", + } + + for _, p := range expect { + found := false + for _, li := range l { + if strings.HasSuffix(li, p) { + found = true + break + } + } + if !found { + t.Errorf("Could not find %s in resolved list.", p) + } + } +} + +func TestResolveLocalDeep(t *testing.T) { + r, err := NewResolver("../") + if err != nil { + t.Fatal(err) + } + + l, err := r.ResolveLocal(true) + if err != nil { + t.Fatalf("Failed to resolve: %s", err) + } + + if len(l) < 8 { + t.Errorf("Expected at least 8 deps, got %d: %s", len(l)) + } +} + +func TestResolve(t *testing.T) { + r, err := NewResolver("../") + if err != nil { + t.Fatal(err) + } + + base := filepath.Join(os.Getenv("GOPATH"), "src/github.com/Masterminds/glide/vendor") + l, err := r.Resolve("github.com/codegangsta/cli", base) + if err != nil { + t.Fatalf("Failed to resolve: %s", err) + } + + if len(l) != 1 { + t.Errorf("Expected 1 dep, got %d: %s", len(l), l[0]) + } +} + +func TestResolveAll(t *testing.T) { + // These are build dependencies of Glide, so we know they are here. + deps := []*cfg.Dependency{ + &cfg.Dependency{Name: "github.com/codegangsta/cli"}, + &cfg.Dependency{Name: "github.com/Masterminds/cookoo"}, + &cfg.Dependency{Name: "github.com/Masterminds/semver"}, + &cfg.Dependency{Name: "gopkg.in/yaml.v2"}, + } + + r, err := NewResolver("../") + if err != nil { + t.Fatalf("No new resolver: %s", err) + } + l, err := r.ResolveAll(deps) + if err != nil { + t.Fatalf("Failed to resolve: %s", err) + } + + if len(l) < len(deps) { + t.Errorf("Expected at least %d deps, got %d", len(deps), len(l)) + } +}
diff --git a/glide.go b/glide.go index 565a80a..fee0aea 100644 --- a/glide.go +++ b/glide.go
@@ -124,28 +124,28 @@ { Name: "create", ShortName: "init", - Usage: "Initialize a new project, creating a template glide.yaml", + Usage: "Initialize a new project, creating a glide.yaml file", Description: `This command starts from a project without Glide and - sets it up. Once this step is done, you may edit the glide.yaml file and then - you may run 'glide install' to fetch your initial dependencies. + sets it up. It generates a glide.yaml file, parsing your codebase to guess + the dependencies to include. Once this step is done you may edit the + glide.yaml file to update imported dependency properties such as the version + or version range to include. - By default, the project name is 'main'. You can specify an alternative on - the commandline: - - $ glide create github.com/Masterminds/foo - - For a project that already has a glide.yaml file, you may skip 'glide create' - and instead run 'glide up'.`, + To fetch the dependencies you may run 'glide install'.`, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "skip-import", + Usage: "When initializing skip importing from other package managers.", + }, + }, Action: func(c *cli.Context) { - if len(c.Args()) >= 1 { - cxt.Put("project", c.Args()[0]) - } + cxt.Put("skipImport", c.Bool("skip-import")) setupHandler(c, "create", cxt, router) }, }, { Name: "get", - Usage: "Install one or more package into `vendor/` and add dependency to glide.yaml.", + Usage: "Install one or more packages into `vendor/` and add dependency to glide.yaml.", Description: `Gets one or more package (like 'go get') and then adds that file to the glide.yaml file. Multiple package names can be specified on one line. @@ -154,19 +154,14 @@ The above will install the project github.com/Masterminds/cookoo and add the subpackage 'web'. - If a fetched dependency has a glide.yaml file, 'get' will also install - all of the dependencies for that dependency. Those are installed in a scoped - vendor directory. So dependency vendor/foo/bar has its dependencies stored - in vendor/foo/bar/vendor. This behavior can be disabled using - '--no-recursive' + If a fetched dependency has a glide.yaml file, configuration from Godep, + GPM, or GB Glide that configuration will be used to find the dependencies + and versions to fetch. If those are not available the dependent packages will + be fetched as either a version specified elsewhere or the latest version. - If '--import' is set, this will also read the dependency projects, looking - for gb, Godep and GPM files. When it finds them, it will build a comparable - glide.yaml file, and then fetch all of the necessary dependencies. The - dependencies are then vendored in the appropriate project. Subsequent calls - to 'glide up' will use the glide.yaml to maintain those dependencies. - However, only if you call 'glide up --import' will the glide file be - rebuilt. When '--no-recursive' is used, '--import' does nothing. + When adding a new dependency Glide will perform an update to work out the + the versions to use from the dependency tree. This will generate an updated + glide.lock file with specific locked versions to use. `, Flags: []cli.Flag{ cli.BoolFlag{ @@ -317,10 +312,75 @@ }, }, { + Name: "install", + ShortName: "i", + Usage: "Install a project's dependencies", + Description: `This uses the native VCS of each packages to install + the appropriate version. There are two ways a projects dependencies can + be installed. When there is a glide.yaml file defining the dependencies but + no lock file (glide.lock) the dependencies are installed using the "update" + command and a glide.lock file is generated pinning all dependencies. If a + glide.lock file is already present the dependencies are installed or updated + from the lock file.`, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "delete", + Usage: "Delete vendor packages not specified in config.", + }, + cli.BoolFlag{ + Name: "no-recursive, quick", + Usage: "Disable updating dependencies' dependencies. Only update things in glide.yaml.", + }, + cli.BoolFlag{ + Name: "force", + Usage: "If there was a change in the repo or VCS switch to new one. Warning, changes will be lost.", + }, + cli.BoolFlag{ + Name: "update-vendored, u", + Usage: "Update vendored packages (without local VCS repo). Warning, changes will be lost.", + }, + cli.StringFlag{ + Name: "file, f", + Usage: "Save all of the discovered dependencies to a Glide YAML file.", + }, + cli.BoolFlag{ + Name: "cache", + Usage: "When downloading dependencies attempt to cache them.", + }, + cli.BoolFlag{ + Name: "cache-gopath", + Usage: "When downloading dependencies attempt to put them in the GOPATH, too.", + }, + cli.BoolFlag{ + Name: "skip-gopath", + Usage: "Skip attempting to copy a dependency from the GOPATH.", + }, + }, + Action: func(c *cli.Context) { + cxt.Put("deleteOptIn", c.Bool("delete")) + cxt.Put("forceUpdate", c.Bool("force")) + cxt.Put("skipFlatten", c.Bool("no-recursive")) + cxt.Put("deleteFlatten", c.Bool("delete-flatten")) + cxt.Put("toPath", c.String("file")) + cxt.Put("toStdout", false) + cxt.Put("useCache", c.Bool("cache")) + cxt.Put("cacheGopath", c.Bool("cache-gopath")) + cxt.Put("skipGopath", c.Bool("skip-gopath")) + if c.Bool("import") { + cxt.Put("importGodeps", true) + cxt.Put("importGPM", true) + cxt.Put("importGb", true) + } + cxt.Put("updateVendoredDeps", c.Bool("update-vendored")) + + cxt.Put("packages", []string(c.Args())) + setupHandler(c, "install", cxt, router) + }, + }, + { Name: "update", ShortName: "up", - Aliases: []string{"install"}, - Usage: "Update or install a project's dependencies", + Usage: "Update a project's dependencies", Description: `This uses the native VCS of each package to try to pull the most applicable updates. Packages with fixed refs (Versions or tags) will not be updated. Packages with no ref or with a branch ref will @@ -429,27 +489,6 @@ }, }, { - Name: "guess", - Usage: "Guess dependencies for existing source.", - Description: `This looks through existing source and dependencies, - and tries to guess all of the dependent packages. - - By default, 'glide guess' writes to standard output. But if a filename - is supplied, the results are written to the file: - - $ glide guess glide.yaml - - The above will overwrite the glide.yaml file.`, - Action: func(c *cli.Context) { - outfile := "" - if len(c.Args()) == 1 { - outfile = c.Args()[0] - } - cxt.Put("toPath", outfile) - setupHandler(c, "guess", cxt, router) - }, - }, - { Name: "about", Usage: "Learn about Glide", Action: func(c *cli.Context) { @@ -508,14 +547,32 @@ Using("skipGopath").From("cxt:skipGopath"). Does(cmd.WriteYaml, "out"). Using("conf").From("cxt:cfg"). - Using("filename").WithDefault("glide.yaml").From("cxt:yaml") + Using("filename").WithDefault("glide.yaml").From("cxt:yaml"). + Does(cmd.WriteLock, "lock"). + Using("lockfile").From("cxt:Lockfile") - reg.Route("exec", "Execute command with GOPATH set."). + reg.Route("install", "Install dependencies."). Includes("@startup"). Includes("@ready"). - Does(cmd.ExecCmd, "cmd"). - Using("args").From("cxt:cliArgs"). - Using("filename").From("cxt:yaml") + Does(cmd.CowardMode, "_"). + Does(cmd.LockFileExists, "_"). + Does(cmd.LoadLockFile, "lock"). + Using("conf").From("cxt:cfg"). + Does(cmd.Mkdir, "dir").Using("dir").WithDefault(VendorDir). + Does(cmd.DeleteUnusedPackages, "deleted"). + Using("conf").From("cxt:cfg"). + Using("optIn").From("cxt:deleteOptIn"). + Does(cmd.VendoredSetup, "cfg"). + Using("conf").From("cxt:cfg"). + Using("update").From("cxt:updateVendoredDeps"). + Does(cmd.Install, "icfg"). + Using("conf").From("cxt:cfg"). + Using("lock").From("cxt:lock"). + Using("home").From("cxt:home"). + Does(cmd.SetReference, "version").Using("conf").From("cxt:icfg"). + Does(cmd.VendoredCleanUp, "_"). + Using("conf").From("cxt:icfg"). + Using("update").From("cxt:updateVendoredDeps") reg.Route("update", "Update dependencies."). Includes("@startup"). @@ -551,7 +608,9 @@ Does(cmd.WriteYaml, "out"). Using("conf").From("cxt:cfg"). Using("filename").From("cxt:toPath"). - Using("toStdout").From("cxt:toStdout") + Using("toStdout").From("cxt:toStdout"). + Does(cmd.WriteLock, "lock"). + Using("lockfile").From("cxt:Lockfile") //Does(cmd.Rebuild, "rebuild").Using("conf").From("cxt:cfg") @@ -564,10 +623,22 @@ reg.Route("pin", "Print a YAML file with all of the packages pinned to the current version."). Includes("@startup"). Includes("@ready"). - Does(cmd.UpdateReferences, "refs").Using("conf").From("cxt:cfg"). + Does(cmd.Flatten, "flattened").Using("conf").From("cxt:cfg"). + Using("packages").From("cxt:packages"). + Using("force").From("cxt:forceUpdate"). + Using("skip").From("cxt:skipFlatten"). + Using("home").From("cxt:home"). + Using("cache").From("cxt:useCache"). + Using("cacheGopath").From("cxt:cacheGopath"). + Using("skipGopath").From("cxt:skipGopath"). + //Does(cmd.VendoredCleanUp, "_"). + //Using("conf").From("cxt:flattened"). + //Using("update").From("cxt:updateVendoredDeps"). + // Write the Lockfile Does(cmd.WriteYaml, "out"). - Using("conf").From("cxt:cfg"). - Using("filename").From("cxt:toPath") + Using("conf").From("cxt:Lockfile"). + Using("filename").From("cxt:toPath"). + Using("toStdout").From("cxt:toStdout") reg.Route("import gpm", "Read a Godeps file"). Includes("@startup"). @@ -605,18 +676,15 @@ Does(cmd.WriteYaml, "out").Using("conf").From("cxt:cfg"). Using("filename").From("cxt:toPath") - reg.Route("guess", "Guess dependencies"). - Includes("@ready"). + reg.Route("create", "Guess dependencies"). + Includes("@startup"). + Does(cmd.GuardYaml, "_"). + Using("filename").From("cxt:yaml"). Does(cmd.GuessDeps, "cfg"). + Using("skipImport").From("cxt:skipImport"). Does(cmd.WriteYaml, "out"). Using("conf").From("cxt:cfg"). - Using("filename").From("cxt:toPath") - - reg.Route("create", "Initialize Glide"). - Includes("@startup"). - Does(cmd.InitGlide, "init"). - Using("filename").From("cxt:yaml"). - Using("project").From("cxt:project").WithDefault("main") + Using("filename").From("cxt:yaml") reg.Route("name", "Print environment"). Includes("@startup").
diff --git a/glide.lock b/glide.lock new file mode 100644 index 0000000..a61af79 --- /dev/null +++ b/glide.lock
@@ -0,0 +1,18 @@ +hash: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +updated: 2015-12-07T11:25:07.016186764-05:00 +imports: +- name: github.com/codegangsta/cli + version: 0302d3914d2a6ad61404584cdae6e6dbc9c03599 +- name: github.com/Masterminds/cookoo + version: 623f8762b2474f1ad6c2cac6bf331b8871591379 + repo: git@github.com:Masterminds/cookoo.git + vcs: git + subpackages: + - . +- name: github.com/Masterminds/semver + version: 6333b7bd29aad1d79898ff568fd90a8aa533ae82 +- name: github.com/Masterminds/vcs + version: eaee272c8fa4514e1572e182faecff5be20e792a +- name: gopkg.in/yaml.v2 + version: f7716cbe52baa25d2e9b0d0da546fcf909fc16b4 +devImports: []
diff --git a/glide.yaml b/glide.yaml index 55f6078..caa6878 100644 --- a/glide.yaml +++ b/glide.yaml
@@ -2,7 +2,7 @@ import: - package: gopkg.in/yaml.v2 - package: github.com/Masterminds/cookoo - version: master + version: ^1.2.0 repo: git@github.com:Masterminds/cookoo.git vcs: git subpackages:
diff --git a/msg/msg.go b/msg/msg.go new file mode 100644 index 0000000..a357a39 --- /dev/null +++ b/msg/msg.go
@@ -0,0 +1,92 @@ +// +build !windows + +package msg + +import ( + "fmt" + "strings" +) + +// These contanstants map to color codes for shell scripts making them +// human readable. +const ( + Blue = "0;34" + Red = "0;31" + Green = "0;32" + Yellow = "0;33" + Cyan = "0;36" + Pink = "1;35" +) + +// Color returns a string in a certain color. The first argument is a string +// containing the color code or a constant from the table above mapped to a code. +// +// The following will print the string "Foo" in yellow: +// fmt.Print(Color(Yellow, "Foo")) +func Color(code, msg string) string { + if NoColor { + return msg + } + return fmt.Sprintf("\033[%sm%s\033[m", code, msg) +} + +// Info logs information +func Info(msg string, args ...interface{}) { + if Quiet { + return + } + fmt.Fprint(Stderr, Color(Green, "[INFO] ")) + Msg(msg, args...) +} + +// Debug logs debug information +func Debug(msg string, args ...interface{}) { + if Quiet || !IsDebugging { + return + } + fmt.Fprint(Stderr, "[DEBUG] ") + Msg(msg, args...) +} + +// Warn logs a warning +func Warn(msg string, args ...interface{}) { + fmt.Fprint(Stderr, Color(Yellow, "[WARN] ")) + ErrMsg(msg, args...) +} + +// Error logs and error. +func Error(msg string, args ...interface{}) { + fmt.Fprint(Stderr, Color(Red, "[ERROR] ")) + ErrMsg(msg, args...) +} + +// ErrMsg sends a message to Stderr +func ErrMsg(msg string, args ...interface{}) { + if len(args) == 0 { + fmt.Fprint(Stderr, msg) + } else { + fmt.Fprintf(Stderr, msg, args...) + } + + // Get rid of the annoying fact that messages need \n at the end, but do + // it in a backward compatible way. + if !strings.HasSuffix(msg, "\n") { + fmt.Fprintln(Stderr) + } +} + +// Msg prints a message with optional arguments, that can be printed, of +// varying types. +func Msg(msg string, args ...interface{}) { + if len(args) == 0 { + fmt.Fprint(Stderr, msg) + } else { + fmt.Fprintf(Stderr, msg, args...) + } + + // Get rid of the annoying fact that messages need \n at the end, but do + // it in a backward compatible way. + if !strings.HasSuffix(msg, "\n") { + fmt.Fprintln(Stderr) + } +}
diff --git a/msg/msg_windows.go b/msg/msg_windows.go new file mode 100644 index 0000000..7a715f1 --- /dev/null +++ b/msg/msg_windows.go
@@ -0,0 +1,63 @@ +// +build windows + +package msg + +import ( + "fmt" + "strings" +) + +// Info logs information +func Info(msg string, args ...interface{}) { + if Quiet { + return + } + fmt.Print("[INFO] ") + Msg(msg, args...) +} + +// Debug logs debug information +func Debug(msg string, args ...interface{}) { + if Quiet || !IsDebugging { + return + } + fmt.Print("[DEBUG] ") + Msg(msg, args...) +} + +// Warn logs a warning +func Warn(msg string, args ...interface{}) { + fmt.Fprint(Stderr, "[WARN] ") + ErrMsg(msg, args...) +} + +// Error logs and error. +func Error(msg string, args ...interface{}) { + fmt.Fprint(Stderr, "[ERROR] ") + ErrMsg(msg, args...) +} + +// ErrMsg sends a message to Stderr +func ErrMsg(msg string, args ...interface{}) { + if len(args) == 0 { + fmt.Fprint(Stderr, msg) + return + } + fmt.Fprintf(Stderr, msg, args...) +} + +// Msg prints a message with optional arguments, that can be printed, of +// varying types. +func Msg(msg string, args ...interface{}) { + if len(args) == 0 { + fmt.Print(msg) + return + } + fmt.Printf(msg, args...) + + // Get rid of the annoying fact that messages need \n at the end, but do + // it in a backward compatible way. + if !strings.HasSuffix(msg, "\n") { + fmt.Println("") + } +}
diff --git a/msg/vars.go b/msg/vars.go new file mode 100644 index 0000000..129b53c --- /dev/null +++ b/msg/vars.go
@@ -0,0 +1,31 @@ +package msg + +import ( + "fmt" + "os" +) + +// Quiet, if true, suppresses chatty levels, like Info. +var Quiet = false + +// IsDebugging, if true, shows verbose levels, like Debug. +var IsDebugging = false + +// NoColor, if true, will not use color in the output. +var NoColor = false + +// Stdout is the location where this prints output. +var Stdout = os.Stdout + +// Stderr is the location where this prints logs. +var Stderr = os.Stderr + +// Puts formats a message and then prints to Stdout. +// +// It does not prefix the message, does not color it, or otherwise decorate it. +// +// It does add a line feed. +func Puts(msg string, args ...interface{}) { + fmt.Fprintf(Stdout, msg, args...) + fmt.Fprintln(Stdout) +}
diff --git a/util/util.go b/util/util.go index d84b13f..dd624b2 100644 --- a/util/util.go +++ b/util/util.go
@@ -53,6 +53,11 @@ // ?go-get=1 to the url. func getRootFromGoGet(pkg string) string { + p, found := checkRemotePackageCache(pkg) + if found { + return p + } + vcsURL := "https://" + pkg u, err := url.Parse(vcsURL) if err != nil { @@ -66,20 +71,42 @@ checkURL := u.String() resp, err := http.Get(checkURL) if err != nil { + addToRemotePackageCache(pkg) return pkg } defer resp.Body.Close() nu, err := parseImportFromBody(u, resp.Body) if err != nil { + addToRemotePackageCache(pkg) return pkg } else if nu == "" { + addToRemotePackageCache(pkg) return pkg } + addToRemotePackageCache(nu) return nu } +// The caching is not concurrency safe but should be made to be that way. +// This implementation is far too much of a hack... rewrite needed. +var remotePackageCache = make(map[string]bool) + +func checkRemotePackageCache(pkg string) (string, bool) { + for k := range remotePackageCache { + if strings.HasPrefix(pkg, k) { + return k, true + } + } + + return pkg, false +} + +func addToRemotePackageCache(pkg string) { + remotePackageCache[pkg] = true +} + func parseImportFromBody(ur *url.URL, r io.ReadCloser) (u string, err error) { d := xml.NewDecoder(r) d.CharsetReader = charsetReader
diff --git a/yaml/yaml.go b/yaml/yaml.go deleted file mode 100644 index 5648e22..0000000 --- a/yaml/yaml.go +++ /dev/null
@@ -1,304 +0,0 @@ -// Package yaml provides the ability to work with glide.yaml files. -package yaml - -import ( - "fmt" - "reflect" - "strings" - - "github.com/Masterminds/glide/util" - "github.com/Masterminds/vcs" - "gopkg.in/yaml.v2" -) - -// FromYaml takes a yaml string and converts it to a Config instance. -func FromYaml(yml string) (*Config, error) { - c := &Config{} - err := yaml.Unmarshal([]byte(yml), &c) - if err != nil { - return nil, err - } - - // The ref property is for the legacy yaml file structure. - // This sets the currect version to the ref if a version isn't - // already set. - for _, v := range c.Imports { - if v.Reference == "" && v.Ref != "" { - v.Reference = v.Ref - } - v.Ref = "" - - // Make sure only legitimate VCS are listed. - v.VcsType = filterVcsType(v.VcsType) - - // Get the root name for the package - o := v.Name - v.Name = util.GetRootFromPackage(v.Name) - subpkg := strings.TrimPrefix(o, v.Name) - if len(subpkg) > 0 && subpkg != o { - v.Subpackages = append(v.Subpackages, strings.TrimPrefix(subpkg, "/")) - } - } - for _, v := range c.DevImports { - if v.Reference == "" && v.Ref != "" { - v.Reference = v.Ref - } - v.Ref = "" - - v.VcsType = filterVcsType(v.VcsType) - - // Get the root name for the package - o := v.Name - v.Name = util.GetRootFromPackage(v.Name) - subpkg := strings.TrimPrefix(o, v.Name) - if len(subpkg) > 0 && subpkg != o { - v.Subpackages = append(v.Subpackages, subpkg) - } - } - - c.DeDupe() - if err != nil { - return c, err - } - - return c, nil -} - -// ToYaml takes a *Config instance and converts it into a yaml string. -func ToYaml(cfg *Config) (string, error) { - yml, err := yaml.Marshal(&cfg) - if err != nil { - return "", err - } - return string(yml), nil -} - -// Config is the top-level configuration object. -type Config struct { - Parent *Config `yaml:"-"` - Name string `yaml:"package"` - Imports Dependencies `yaml:"import"` - DevImports Dependencies `yaml:"devimport,omitempty"` -} - -// HasDependency returns true if the given name is listed as an import or dev import. -func (c *Config) HasDependency(name string) bool { - for _, d := range c.Imports { - if d.Name == name { - return true - } - } - for _, d := range c.DevImports { - if d.Name == name { - return true - } - } - return false -} - -// HasRecursiveDependency returns true if this config or one of it's parents has this dependency -func (c *Config) HasRecursiveDependency(name string) bool { - if c.HasDependency(name) == true { - return true - } else if c.Parent != nil { - return c.Parent.HasRecursiveDependency(name) - } - return false -} - -func (c *Config) DeDupe() error { - - // Remove duplicates in the imports - var err error - c.Imports, err = c.Imports.DeDupe() - if err != nil { - return err - } - c.DevImports, err = c.DevImports.DeDupe() - if err != nil { - return err - } - - // If the name on the config object is part of the imports remove it. - found := -1 - for i, dep := range c.Imports { - if dep.Name == c.Name { - found = i - } - } - if found >= 0 { - c.Imports = append(c.Imports[:found], c.Imports[found+1:]...) - } - - found = -1 - for i, dep := range c.DevImports { - if dep.Name == c.Name { - found = i - } - } - if found >= 0 { - c.Imports = append(c.DevImports[:found], c.DevImports[found+1:]...) - } - - return nil -} - -// GetRoot follows the Parent down to the top node -func (c *Config) GetRoot() *Config { - if c.Parent != nil { - return c.Parent.GetRoot() - } - return c -} - -// Clone performs a deep clone of the Config instance -func (c *Config) Clone() *Config { - n := &Config{} - n.Name = c.Name - n.Imports = c.Imports.Clone() - n.DevImports = c.DevImports.Clone() - return n -} - -// Dependencies is a collection of Dependency -type Dependencies []*Dependency - -// Dependency describes a package that the present package depends upon. -type Dependency struct { - Name string `yaml:"package"` - Reference string `yaml:"version,omitempty"` - Ref string `yaml:"ref,omitempty"` - Pin string `yaml:"pin,omitempty"` - Repository string `yaml:"repo,omitempty"` - VcsType string `yaml:"vcs,omitempty"` - Subpackages []string `yaml:"subpackages,omitempty"` - Arch []string `yaml:"arch,omitempty"` - Os []string `yaml:"os,omitempty"` - UpdateAsVendored bool `yaml:"-"` -} - -// GetRepo retrieves a Masterminds/vcs repo object configured for the root -// of the package being retrieved. -func (d *Dependency) GetRepo(dest string) (vcs.Repo, error) { - - // The remote location is either the configured repo or the package - // name as an https url. - var remote string - if len(d.Repository) > 0 { - remote = d.Repository - } else { - remote = "https://" + d.Name - } - - // If the VCS type has a value we try that first. - if len(d.VcsType) > 0 && d.VcsType != "None" { - switch vcs.Type(d.VcsType) { - case vcs.Git: - return vcs.NewGitRepo(remote, dest) - case vcs.Svn: - return vcs.NewSvnRepo(remote, dest) - case vcs.Hg: - return vcs.NewHgRepo(remote, dest) - case vcs.Bzr: - return vcs.NewBzrRepo(remote, dest) - default: - return nil, fmt.Errorf("Unknown VCS type %s set for %s", d.VcsType, d.Name) - } - } - - // When no type set we try to autodetect. - return vcs.NewRepo(remote, dest) -} - -func (d *Dependency) Clone() *Dependency { - return &Dependency{ - Name: d.Name, - Reference: d.Reference, - Pin: d.Pin, - Repository: d.Repository, - VcsType: d.VcsType, - Subpackages: d.Subpackages, - Arch: d.Arch, - Os: d.Os, - UpdateAsVendored: d.UpdateAsVendored, - } -} - -// Get a dependency by name -func (d Dependencies) Get(name string) *Dependency { - for _, dep := range d { - if dep.Name == name { - return dep - } - } - return nil -} - -func (d Dependencies) Clone() Dependencies { - n := make(Dependencies, 0, 1) - for _, v := range d { - n = append(n, v.Clone()) - } - return n -} - -// DeDupe cleans up duplicates on a list of dependencies. -func (d Dependencies) DeDupe() (Dependencies, error) { - checked := map[string]int{} - imports := make(Dependencies, 0, 1) - i := 0 - for _, dep := range d { - // The first time we encounter a dependency add it to the list - if val, ok := checked[dep.Name]; !ok { - checked[dep.Name] = i - imports = append(imports, dep) - i++ - } else { - // In here we've encountered a dependency for the second time. - // Make sure the details are the same or return an error. - v := imports[val] - if dep.Reference != v.Reference { - return d, fmt.Errorf("Import %s repeated with different versions '%s' and '%s'", dep.Name, dep.Reference, v.Reference) - } - if dep.Repository != v.Repository || dep.VcsType != v.VcsType { - return d, fmt.Errorf("Import %s repeated with different Repository details", dep.Name) - } - if !reflect.DeepEqual(dep.Os, v.Os) || !reflect.DeepEqual(dep.Arch, v.Arch) { - return d, fmt.Errorf("Import %s repeated with different OS or Architecture filtering", dep.Name) - } - imports[checked[dep.Name]].Subpackages = stringArrayDeDupe(v.Subpackages, dep.Subpackages...) - } - } - - return imports, nil -} - -func stringArrayDeDupe(s []string, items ...string) []string { - for _, item := range items { - exists := false - for _, v := range s { - if v == item { - exists = true - } - } - if !exists { - s = append(s, item) - } - } - return s -} - -func filterVcsType(vcs string) string { - switch vcs { - case "git", "hg", "bzr", "svn": - return vcs - case "mercurial": - return "hg" - case "bazaar": - return "bzr" - case "subversion": - return "svn" - default: - return "" - } -}
diff --git a/yaml/yaml_test.go b/yaml/yaml_test.go deleted file mode 100644 index 94d160d..0000000 --- a/yaml/yaml_test.go +++ /dev/null
@@ -1,66 +0,0 @@ -package yaml - -import "testing" - -var yml = ` -package: fake/testing -import: - - package: github.com/kylelemons/go-gypsy - subpackages: - - yaml - # Intentionally left spaces at end of next line. - - package: github.com/Masterminds/convert - repo: git@github.com:Masterminds/convert.git - ref: a9949121a2e2192ca92fa6dddfeaaa4a4412d955 - subpackages: - - color - - nautical - - radial - os: - - linux - arch: - - i386 - - arm - - package: github.com/Masterminds/structable - - package: github.com/Masterminds/cookoo/color - -devimport: - - package: github.com/kylelemons/go-gypsy -` - -func TestFromYaml(t *testing.T) { - cfg, err := FromYaml(yml) - if err != nil { - t.Errorf("Unexpected error parsing yaml %s", err) - } - - if cfg.Name != "fake/testing" { - t.Errorf("Inaccurate name found %s", cfg.Name) - } - - found := false - for _, i := range cfg.Imports { - if i.Name == "github.com/Masterminds/cookoo" { - found = true - } - } - if !found { - t.Error("Unable to find github.com/Masterminds/cookoo") - } -} - -func TestToYaml(t *testing.T) { - cfg, err := FromYaml(yml) - if err != nil { - t.Errorf("Unexpected error parsing yaml %s", err) - } - - o, err := ToYaml(cfg) - if err != nil { - t.Errorf("Unexpected error converting cfg to yaml %s", err) - } - - if o == "" { - t.Error("Yaml output not generated when expected") - } -}