Updated Config handling (glide.yaml) - Moved to gopkg.in/yaml.v2 to parse and generate yaml - Moved yaml handling into new subpackage separate from cmd. This allows others to import the config handling and keep it separate from the commands that use it. - Moved some shared util code to the util sub-package
diff --git a/cmd/delete.go b/cmd/delete.go index c6cdd36..3752c9b 100644 --- a/cmd/delete.go +++ b/cmd/delete.go
@@ -2,10 +2,12 @@ import ( "errors" - "github.com/Masterminds/cookoo" "os" "path/filepath" "strings" + + "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/yaml" ) // DeleteUnusedPackages removes packages from vendor/ that are no longer used. @@ -25,7 +27,7 @@ } // Build directory tree of what to keep. - cfg := p.Get("conf", nil).(*Config) + cfg := p.Get("conf", nil).(*yaml.Config) var pkgList []string for _, dep := range cfg.Imports { pkgList = append(pkgList, dep.Name)
diff --git a/cmd/flatten.go b/cmd/flatten.go index 51655af..c5174ea 100644 --- a/cmd/flatten.go +++ b/cmd/flatten.go
@@ -1,13 +1,14 @@ package cmd import ( + "io/ioutil" "os" "path" "strings" "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/yaml" "github.com/Masterminds/semver" - "github.com/kylelemons/go-gypsy/yaml" ) // Flatten recurses through all dependent packages and flattens to a top level. @@ -19,12 +20,12 @@ // - packages ([]string): The packages to read. If this is empty, it reads all // packages. // - force (bool): force vcs updates. -// - conf (*Config): The configuration. +// - conf (*yaml.Config): The configuration. // // Returns: // func Flatten(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - conf := p.Get("conf", &Config{}).(*Config) + conf := p.Get("conf", &yaml.Config{}).(*yaml.Config) skip := p.Get("skip", false).(bool) if skip { return conf, nil @@ -42,7 +43,7 @@ } // Build an initial dependency map. - deps := make(map[string]*Dependency, len(conf.Imports)) + deps := make(map[string]*yaml.Dependency, len(conf.Imports)) for _, imp := range conf.Imports { deps[imp.Name] = imp } @@ -57,8 +58,8 @@ return conf, err } -func exportFlattenedDeps(conf *Config, in map[string]*Dependency) { - out := make([]*Dependency, len(in)) +func exportFlattenedDeps(conf *yaml.Config, in map[string]*yaml.Dependency) { + out := make([]*yaml.Dependency, len(in)) i := 0 for _, v := range in { out[i] = v @@ -68,13 +69,13 @@ } type flattening struct { - conf *Config + conf *yaml.Config // Top vendor path, e.g. project/vendor top string // Current path curr string // Built list of dependencies - deps map[string]*Dependency + deps map[string]*yaml.Dependency // Dependencies that need to be scanned. scan []string } @@ -166,18 +167,19 @@ } } -func mergeGlide(dir, name string, deps map[string]*Dependency, vend string) ([]string, bool) { +func mergeGlide(dir, name string, deps map[string]*yaml.Dependency, vend string) ([]string, bool) { gp := path.Join(dir, "glide.yaml") if _, err := os.Stat(gp); err != nil { return []string{}, false } - f, err := yaml.ReadFile(gp) + + yml, err := ioutil.ReadFile(gp) if err != nil { - Warn("Found glide file %q, but can't parse: %s", gp, err) + Warn("Found glide file %q, but can't read: %s", gp, err) return []string{}, false } - conf, err := FromYaml(f.Root) + conf, err := yaml.FromYaml(string(yml)) if err != nil { Warn("Found glide file %q, but can't use it: %s", gp, err) return []string{}, false @@ -192,7 +194,7 @@ // // It returns true if any dependencies were found (even if not added because // they are duplicates). -func mergeGodep(dir, name string, deps map[string]*Dependency, vend string) ([]string, bool) { +func mergeGodep(dir, name string, deps map[string]*yaml.Dependency, vend string) ([]string, bool) { Debug("Looking in %s/Godeps/ for a Godeps.json file.\n", dir) d, err := parseGodepGodeps(dir) if err != nil { @@ -207,7 +209,7 @@ } // listGb merges GB dependencies into the deps. -func mergeGb(dir, pkg string, deps map[string]*Dependency, vend string) ([]string, bool) { +func mergeGb(dir, pkg string, deps map[string]*yaml.Dependency, vend string) ([]string, bool) { Debug("Looking in %s/vendor/ for a manifest file.\n", dir) d, err := parseGbManifest(dir) if err != nil || len(d) == 0 { @@ -218,7 +220,7 @@ } // mergeGPM merges GPM Godeps files into deps. -func mergeGPM(dir, pkg string, deps map[string]*Dependency, vend string) ([]string, bool) { +func mergeGPM(dir, pkg string, deps map[string]*yaml.Dependency, vend string) ([]string, bool) { d, err := parseGPMGodeps(dir) if err != nil || len(d) == 0 { return []string{}, false @@ -232,7 +234,7 @@ // This always returns true because it always handles the job of searching // for dependencies. So generally it should be the last merge strategy // that you try. -func mergeGuess(dir, pkg string, deps map[string]*Dependency, vend string) ([]string, bool) { +func mergeGuess(dir, pkg string, deps map[string]*yaml.Dependency, vend string) ([]string, bool) { /* Info("Scanning %s for dependencies.", pkg) buildContext, err := GetBuildContext() @@ -288,7 +290,7 @@ } // mergeDeps merges any dependency array into deps. -func mergeDeps(orig map[string]*Dependency, add []*Dependency, vend string) []string { +func mergeDeps(orig map[string]*yaml.Dependency, add []*yaml.Dependency, vend string) []string { mod := []string{} for _, dd := range add { // Add it unless it's already there.
diff --git a/cmd/gb.go b/cmd/gb.go index 6f52dfe..ac66aeb 100644 --- a/cmd/gb.go +++ b/cmd/gb.go
@@ -7,6 +7,7 @@ "github.com/Masterminds/cookoo" "github.com/Masterminds/glide/gb" + "github.com/Masterminds/glide/yaml" ) func HasGbManifest(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { @@ -27,14 +28,14 @@ return parseGbManifest(dir) } -func parseGbManifest(dir string) ([]*Dependency, error) { +func parseGbManifest(dir string) ([]*yaml.Dependency, error) { path := filepath.Join(dir, "vendor/manifest") if fi, err := os.Stat(path); err != nil || fi.IsDir() { - return []*Dependency{}, nil + return []*yaml.Dependency{}, nil } Info("Found GB manifest file.\n") - buf := []*Dependency{} + buf := []*yaml.Dependency{} file, err := os.Open(path) if err != nil { return buf, err @@ -63,7 +64,7 @@ } } else { seen[pkg] = true - dep := &Dependency{ + dep := &yaml.Dependency{ Name: pkg, Reference: d.Revision, Repository: d.Repository,
diff --git a/cmd/get_imports.go b/cmd/get_imports.go index 973d4c5..3740cc9 100644 --- a/cmd/get_imports.go +++ b/cmd/get_imports.go
@@ -1,34 +1,27 @@ package cmd import ( - "encoding/xml" "fmt" "sort" //"log" - "io" - "net/http" - "net/url" + "os" "path" - "regexp" "runtime" "strings" "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/util" + "github.com/Masterminds/glide/yaml" "github.com/Masterminds/semver" v "github.com/Masterminds/vcs" ) -func init() { - // Precompile the regular expressions used to check VCS locations. - for _, v := range vcsList { - v.regex = regexp.MustCompile(v.pattern) - } - - // Uncomment the line below and the log import to see the output - // from the vcs commands executed for each project. - //v.Logger = log.New(os.Stdout, "go-vcs", log.LstdFlags) -} +//func init() { +// Uncomment the line below and the log import to see the output +// from the vcs commands executed for each project. +//v.Logger = log.New(os.Stdout, "go-vcs", log.LstdFlags) +//} // GetAll gets zero or more repos. // @@ -43,19 +36,19 @@ // - []*Dependency: A list of constructed dependencies. func GetAll(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { names := p.Get("packages", []string{}).([]string) - cfg := p.Get("conf", nil).(*Config) + cfg := p.Get("conf", nil).(*yaml.Config) insecure := p.Get("insecure", false).(bool) Info("Preparing to install %d package.", len(names)) - deps := []*Dependency{} + deps := []*yaml.Dependency{} for _, name := range names { cwd, err := VendorPath(c) if err != nil { return nil, err } - root := getRepoRootFromPackage(name) + root := util.GetRootFromPackage(name) if len(root) == 0 { return nil, fmt.Errorf("Package name is required for %q.", name) } @@ -79,7 +72,7 @@ return false, err } - dep := &Dependency{ + dep := &yaml.Dependency{ Name: root, } @@ -108,7 +101,7 @@ // GetImports iterates over the imported packages and gets them. func GetImports(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := p.Get("conf", nil).(*Config) + cfg := p.Get("conf", nil).(*yaml.Config) cwd, err := VendorPath(c) if err != nil { Error("Failed to prepare vendor directory: %s", err) @@ -134,10 +127,10 @@ // Params: // // - force (bool): force packages to update (default false) -// - conf (*Config): The configuration +// - conf (*yaml.Config): The configuration // - packages([]string): The packages to update. Default is all. func UpdateImports(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := p.Get("conf", nil).(*Config) + cfg := p.Get("conf", nil).(*yaml.Config) force := p.Get("force", true).(bool) plist := p.Get("packages", []string{}).([]string) pkgs := list2map(plist) @@ -175,7 +168,7 @@ // SetReference is a command to set the VCS reference (commit id, tag, etc) for // a project. func SetReference(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := p.Get("conf", nil).(*Config) + cfg := p.Get("conf", nil).(*yaml.Config) cwd, err := VendorPath(c) if err != nil { return false, err @@ -197,7 +190,7 @@ // filterArchOs indicates a dependency should be filtered out because it is // the wrong GOOS or GOARCH. -func filterArchOs(dep *Dependency) bool { +func filterArchOs(dep *yaml.Dependency) bool { found := false if len(dep.Arch) > 0 { for _, a := range dep.Arch { @@ -228,7 +221,7 @@ } // VcsExists checks if the directory has a local VCS checkout. -func VcsExists(dep *Dependency, dest string) bool { +func VcsExists(dep *yaml.Dependency, dest string) bool { repo, err := dep.GetRepo(dest) if err != nil { return false @@ -240,7 +233,7 @@ // VcsGet figures out how to fetch a dependency, and then gets it. // // VcsGet installs into the dest. -func VcsGet(dep *Dependency, dest string) error { +func VcsGet(dep *yaml.Dependency, dest string) error { repo, err := dep.GetRepo(dest) if err != nil { @@ -251,7 +244,7 @@ } // VcsUpdate updates to a particular checkout based on the VCS setting. -func VcsUpdate(dep *Dependency, vend string, force bool) error { +func VcsUpdate(dep *yaml.Dependency, vend string, force bool) error { Info("Fetching updates for %s.\n", dep.Name) if filterArchOs(dep) { @@ -341,7 +334,7 @@ } // VcsVersion set the VCS version for a checkout. -func VcsVersion(dep *Dependency, vend string) error { +func VcsVersion(dep *yaml.Dependency, vend string) error { // If there is no refernece configured there is nothing to set. if dep.Reference == "" { return nil @@ -419,7 +412,7 @@ } // VcsLastCommit gets the last commit ID from the given dependency. -func VcsLastCommit(dep *Dependency, vend string) (string, error) { +func VcsLastCommit(dep *yaml.Dependency, vend string) (string, error) { cwd := path.Join(vend, dep.Name) repo, err := dep.GetRepo(cwd) if err != nil { @@ -437,174 +430,3 @@ return version, nil } - -// From a package name find the root repo. For example, -// the package github.com/Masterminds/cookoo/io has a root repo -// at github.com/Masterminds/cookoo -func getRepoRootFromPackage(pkg string) string { - for _, v := range vcsList { - m := v.regex.FindStringSubmatch(pkg) - if m == nil { - continue - } - - if m[1] != "" { - return m[1] - } - } - - // There are cases where a package uses the special go get magic for - // redirects. If we've not discovered the location already try that. - pkg = getRepoRootFromGoGet(pkg) - - return pkg -} - -// Pages like https://golang.org/x/net provide an html document with -// meta tags containing a location to work with. The go tool uses -// a meta tag with the name go-import which is what we use here. -// godoc.org also has one call go-source that we do not need to use. -// The value of go-import is in the form "prefix vcs repo". The prefix -// should match the vcsURL and the repo is a location that can be -// checked out. Note, to get the html document you you need to add -// ?go-get=1 to the url. -func getRepoRootFromGoGet(pkg string) string { - - vcsURL := "https://" + pkg - u, err := url.Parse(vcsURL) - if err != nil { - return pkg - } - if u.RawQuery == "" { - u.RawQuery = "go-get=1" - } else { - u.RawQuery = u.RawQuery + "+go-get=1" - } - checkURL := u.String() - resp, err := http.Get(checkURL) - if err != nil { - return pkg - } - defer resp.Body.Close() - - nu, err := parseImportFromBody(u, resp.Body) - if err != nil { - return pkg - } else if nu == "" { - return pkg - } - - return nu -} - -func parseImportFromBody(ur *url.URL, r io.ReadCloser) (u string, err error) { - d := xml.NewDecoder(r) - d.CharsetReader = charsetReader - d.Strict = false - var t xml.Token - for { - t, err = d.Token() - if err != nil { - if err == io.EOF { - // If we hit the end of the markup and don't have anything - // we return an error. - err = v.ErrCannotDetectVCS - } - return - } - if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") { - return - } - if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") { - return - } - e, ok := t.(xml.StartElement) - if !ok || !strings.EqualFold(e.Name.Local, "meta") { - continue - } - if attrValue(e.Attr, "name") != "go-import" { - continue - } - if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 { - - // If the prefix supplied by the remote system isn't a prefix to the - // url we're fetching return continue looking for more go-imports. - // This will work for exact matches and prefixes. For example, - // golang.org/x/net as a prefix will match for golang.org/x/net and - // golang.org/x/net/context. - vcsURL := ur.Host + ur.Path - if !strings.HasPrefix(vcsURL, f[0]) { - continue - } else { - u = f[0] - return - } - - } - } -} - -func charsetReader(charset string, input io.Reader) (io.Reader, error) { - switch strings.ToLower(charset) { - case "ascii": - return input, nil - default: - return nil, fmt.Errorf("can't decode XML document using charset %q", charset) - } -} - -func attrValue(attrs []xml.Attr, name string) string { - for _, a := range attrs { - if strings.EqualFold(a.Name.Local, name) { - return a.Value - } - } - return "" -} - -type vcsInfo struct { - host string - pattern string - regex *regexp.Regexp -} - -var vcsList = []*vcsInfo{ - { - host: "github.com", - pattern: `^(?P<rootpkg>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`, - }, - { - host: "bitbucket.org", - pattern: `^(?P<rootpkg>bitbucket\.org/([A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, - }, - { - host: "launchpad.net", - pattern: `^(?P<rootpkg>launchpad\.net/(([A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, - }, - { - host: "git.launchpad.net", - pattern: `^(?P<rootpkg>git\.launchpad\.net/(([A-Za-z0-9_.\-]+)|~[A-Za-z0-9_.\-]+/(\+git|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))$`, - }, - { - host: "go.googlesource.com", - pattern: `^(?P<rootpkg>go\.googlesource\.com/[A-Za-z0-9_.\-]+/?)$`, - }, - // TODO: Once Google Code becomes fully deprecated this can be removed. - { - host: "code.google.com", - pattern: `^(?P<rootpkg>code\.google\.com/[pr]/([a-z0-9\-]+)(\.([a-z0-9\-]+))?)(/[A-Za-z0-9_.\-]+)*$`, - }, - // Alternative Google setup for SVN. This is the previous structure but it still works... until Google Code goes away. - { - pattern: `^(?P<rootpkg>[a-z0-9_\-.]+\.googlecode\.com/svn(/.*)?)$`, - }, - // Alternative Google setup. This is the previous structure but it still works... until Google Code goes away. - { - pattern: `^(?P<rootpkg>[a-z0-9_\-.]+\.googlecode\.com/(git|hg))(/.*)?$`, - }, - // If none of the previous detect the type they will fall to this looking for the type in a generic sense - // by the extension to the path. - { - pattern: `^(?P<rootpkg>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(bzr|git|hg|svn))(/[A-Za-z0-9_.\-]+)*$`, - }, -}
diff --git a/cmd/get_imports_test.go b/cmd/get_imports_test.go index 4b1975e..9deddf4 100644 --- a/cmd/get_imports_test.go +++ b/cmd/get_imports_test.go
@@ -4,12 +4,13 @@ "testing" "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/yaml" ) func TestGetImportsEmptyConfig(t *testing.T) { _, _, c := cookoo.Cookoo() SilenceLogs(c) - cfg := new(Config) + cfg := new(yaml.Config) p := cookoo.NewParamsWithValues(map[string]interface{}{"conf": cfg}) res, it := GetImports(c, p) if it != nil { @@ -25,36 +26,3 @@ p := cookoo.NewParamsWithValues(map[string]interface{}{"quiet": true}) BeQuiet(c, p) } - -func TestGetRepoRootFromPackage(t *testing.T) { - urlList := map[string]string{ - "github.com/Masterminds/VCSTestRepo": "github.com/Masterminds/VCSTestRepo", - "bitbucket.org/mattfarina/testhgrepo": "bitbucket.org/mattfarina/testhgrepo", - "launchpad.net/govcstestbzrrepo/trunk": "launchpad.net/govcstestbzrrepo/trunk", - "launchpad.net/~mattfarina/+junk/mygovcstestbzrrepo": "launchpad.net/~mattfarina/+junk/mygovcstestbzrrepo", - "launchpad.net/~mattfarina/+junk/mygovcstestbzrrepo/trunk": "launchpad.net/~mattfarina/+junk/mygovcstestbzrrepo", - "git.launchpad.net/govcstestgitrepo": "git.launchpad.net/govcstestgitrepo", - "git.launchpad.net/~mattfarina/+git/mygovcstestgitrepo": "git.launchpad.net/~mattfarina/+git/mygovcstestgitrepo", - "farbtastic.googlecode.com/svn/": "farbtastic.googlecode.com/svn/", - "farbtastic.googlecode.com/svn/trunk": "farbtastic.googlecode.com/svn/trunk", - "code.google.com/p/farbtastic": "code.google.com/p/farbtastic", - "code.google.com/p/plotinum": "code.google.com/p/plotinum", - "example.com/foo/bar.git": "example.com/foo/bar.git", - "example.com/foo/bar.svn": "example.com/foo/bar.svn", - "example.com/foo/bar/baz.bzr": "example.com/foo/bar/baz.bzr", - "example.com/foo/bar/baz.hg": "example.com/foo/bar/baz.hg", - "gopkg.in/mgo.v2": "gopkg.in/mgo.v2", - "gopkg.in/mgo.v2/txn": "gopkg.in/mgo.v2", - "gopkg.in/nowk/assert.v2": "gopkg.in/nowk/assert.v2", - "gopkg.in/nowk/assert.v2/tests": "gopkg.in/nowk/assert.v2", - "golang.org/x/net": "golang.org/x/net", - "golang.org/x/net/context": "golang.org/x/net", - } - - for u, c := range urlList { - repo := getRepoRootFromPackage(u) - if repo != c { - t.Errorf("getRepoRootFromPackage expected %s but got %s", c, repo) - } - } -}
diff --git a/cmd/godeps.go b/cmd/godeps.go index ac4b4ed..9b8036b 100644 --- a/cmd/godeps.go +++ b/cmd/godeps.go
@@ -7,6 +7,8 @@ "strings" "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/util" + "github.com/Masterminds/glide/yaml" ) // This file contains commands for working with Godep. @@ -46,19 +48,19 @@ // Params: // - dir (string): the project's directory // -// Returns an []*Dependency +// Returns an []*yaml.Dependency func ParseGodepGodeps(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { dir := cookoo.GetString("dir", "", p) return parseGodepGodeps(dir) } -func parseGodepGodeps(dir string) ([]*Dependency, error) { +func parseGodepGodeps(dir string) ([]*yaml.Dependency, error) { path := filepath.Join(dir, "Godeps/Godeps.json") if _, err := os.Stat(path); err != nil { - return []*Dependency{}, nil + return []*yaml.Dependency{}, nil } Info("Found Godeps.json file.\n") - buf := []*Dependency{} + buf := []*yaml.Dependency{} godeps := new(Godeps) @@ -79,7 +81,7 @@ for _, d := range godeps.Deps { // Info("Adding package %s\n", d.ImportPath) - pkg := getRepoRootFromPackage(d.ImportPath) + pkg := util.GetRootFromPackage(d.ImportPath) sub := strings.TrimPrefix(d.ImportPath, pkg) if _, ok := seen[pkg]; ok { if len(sub) == 0 { @@ -93,7 +95,7 @@ } } else { seen[pkg] = true - dep := &Dependency{Name: pkg, Reference: d.Rev} + dep := &yaml.Dependency{Name: pkg, Reference: d.Rev} if len(sub) > 0 { dep.Subpackages = []string{sub} }
diff --git a/cmd/gpm.go b/cmd/gpm.go index b0eff89..82ed89d 100644 --- a/cmd/gpm.go +++ b/cmd/gpm.go
@@ -7,6 +7,7 @@ "strings" "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/yaml" ) // This file contains commands for working with GPM/GVP. @@ -24,22 +25,22 @@ // Params // - dir (string): Directory root. // -// Returns an []*Dependency +// Returns an []*yaml.Dependency func GPMGodeps(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { dir := cookoo.GetString("dir", "", p) return parseGPMGodeps(dir) } -func parseGPMGodeps(dir string) ([]*Dependency, error) { +func parseGPMGodeps(dir string) ([]*yaml.Dependency, error) { path := filepath.Join(dir, "Godeps") if i, err := os.Stat(path); err != nil { - return []*Dependency{}, nil + return []*yaml.Dependency{}, nil } else if i.IsDir() { Info("Godeps is a directory. This is probably a Godep project.\n") - return []*Dependency{}, nil + return []*yaml.Dependency{}, nil } Info("Found Godeps file.\n") - buf := []*Dependency{} + buf := []*yaml.Dependency{} file, err := os.Open(path) if err != nil { @@ -49,7 +50,7 @@ for scanner.Scan() { parts, ok := parseGodepsLine(scanner.Text()) if ok { - dep := &Dependency{Name: parts[0]} + dep := &yaml.Dependency{Name: parts[0]} if len(parts) > 1 { dep.Reference = parts[1] } @@ -69,11 +70,11 @@ dir := cookoo.GetString("dir", "", p) path := filepath.Join(dir, "Godeps-Git") if _, err := os.Stat(path); err != nil { - return []*Dependency{}, nil + return []*yaml.Dependency{}, nil } Info("Found Godeps-Git file.\n") - buf := []*Dependency{} + buf := []*yaml.Dependency{} file, err := os.Open(path) if err != nil { @@ -83,7 +84,7 @@ for scanner.Scan() { parts, ok := parseGodepsLine(scanner.Text()) if ok { - dep := &Dependency{Name: parts[1], Repository: parts[0]} + dep := &yaml.Dependency{Name: parts[1], Repository: parts[0]} if len(parts) > 2 { dep.Reference = parts[2] }
diff --git a/cmd/guess_deps.go b/cmd/guess_deps.go index 15735c4..871dc6c 100644 --- a/cmd/guess_deps.go +++ b/cmd/guess_deps.go
@@ -1,9 +1,11 @@ package cmd import ( - "github.com/Masterminds/cookoo" "os" "strings" + + "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/yaml" ) // GuessDeps tries to get the dependencies for the current directory. @@ -24,15 +26,15 @@ return nil, err } - config := new(Config) + config := new(yaml.Config) // Get the name of the top level package config.Name = guessPackageName(buildContext, base) - config.Imports = make([]*Dependency, len(deps)) + config.Imports = make([]*yaml.Dependency, len(deps)) i := 0 for pa := range deps { Info("Found reference to %s\n", pa) - d := &Dependency{ + d := &yaml.Dependency{ Name: pa, } config.Imports[i] = d
diff --git a/cmd/link_package.go b/cmd/link_package.go index 3519ed3..71d1dd4 100644 --- a/cmd/link_package.go +++ b/cmd/link_package.go
@@ -7,11 +7,12 @@ "strings" "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/yaml" ) // LinkPackage creates a symlink to the project within the GOPATH. func LinkPackage(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := c.Get("cfg", "").(*Config) + cfg := c.Get("cfg", "").(*yaml.Config) pname := p.Get("path", cfg.Name).(string) // Per issue #10, this may be nicer to work with in cases where repos are
diff --git a/cmd/print_name.go b/cmd/print_name.go index bc6a901..7245f1d 100644 --- a/cmd/print_name.go +++ b/cmd/print_name.go
@@ -4,13 +4,14 @@ "fmt" "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/yaml" ) // PrintName prints the name of the project. // // This comes from Config.Name. func PrintName(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := p.Get("conf", nil).(*Config) + cfg := p.Get("conf", nil).(*yaml.Config) fmt.Println(cfg.Name) return nil, nil }
diff --git a/cmd/rebuild.go b/cmd/rebuild.go index 716d8f5..d13c414 100644 --- a/cmd/rebuild.go +++ b/cmd/rebuild.go
@@ -8,15 +8,16 @@ "strings" "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/yaml" ) // Rebuild runs 'go build' in a directory. // // Params: -// - conf: the *Config. +// - conf: the *yaml.Config. // func Rebuild(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := p.Get("conf", nil).(*Config) + cfg := p.Get("conf", nil).(*yaml.Config) vpath, err := VendorPath(c) if err != nil { return nil, err @@ -38,7 +39,7 @@ return true, nil } -func buildDep(c cookoo.Context, dep *Dependency, vpath string) error { +func buildDep(c cookoo.Context, dep *yaml.Dependency, vpath string) error { if len(dep.Subpackages) == 0 { buildPath(c, dep.Name) }
diff --git a/cmd/update_references.go b/cmd/update_references.go index 5536e64..33c8eeb 100644 --- a/cmd/update_references.go +++ b/cmd/update_references.go
@@ -2,6 +2,7 @@ import ( "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/yaml" ) // UpdateReferences updates the revision numbers on all of the imports. @@ -10,10 +11,10 @@ // be updated. // // Params: -// - conf (*Config): Configuration +// - conf (*yaml.Config): Configuration // - packages ([]string): A list of packages to update. Default is all packages. func UpdateReferences(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - cfg := p.Get("conf", &Config{}).(*Config) + cfg := p.Get("conf", &yaml.Config{}).(*yaml.Config) plist := p.Get("packages", []string{}).([]string) pkgs := list2map(plist)
diff --git a/cmd/vendored.go b/cmd/vendored.go index 8a24f8c..8eb1b8f 100644 --- a/cmd/vendored.go +++ b/cmd/vendored.go
@@ -1,10 +1,12 @@ package cmd import ( - "github.com/Masterminds/cookoo" - "github.com/Masterminds/vcs" "os" "path" + + "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/yaml" + "github.com/Masterminds/vcs" ) // VendoredSetup is a command that does the setup for vendored directories. @@ -13,7 +15,7 @@ // VendoredCleanUp should be a suffix to UpdateImports. func VendoredSetup(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { update := p.Get("update", true).(bool) - cfg := p.Get("conf", nil).(*Config) + cfg := p.Get("conf", nil).(*yaml.Config) if update != true { return cfg, nil } @@ -60,7 +62,7 @@ if update != true { return false, nil } - cfg := p.Get("conf", nil).(*Config) + cfg := p.Get("conf", nil).(*yaml.Config) vend, err := VendorPath(c) if err != nil {
diff --git a/cmd/yaml.go b/cmd/yaml.go index b44e054..fe302d6 100644 --- a/cmd/yaml.go +++ b/cmd/yaml.go
@@ -1,16 +1,14 @@ package cmd import ( - "bytes" "fmt" "io" + "io/ioutil" "os" - "reflect" "strings" "github.com/Masterminds/cookoo" - v "github.com/Masterminds/vcs" - "github.com/kylelemons/go-gypsy/yaml" + "github.com/Masterminds/glide/yaml" ) // ParseYaml parses the glide.yaml format and returns a Configuration object. @@ -18,21 +16,21 @@ // Params: // - filename (string): YAML filename as a string // -// Context: -// - yaml.File: This puts the parsed YAML file into the context. -// // Returns: -// - *Config: The configuration. +// - *yaml.Config: The configuration. func ParseYaml(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { fname := p.Get("filename", "glide.yaml").(string) //conf := new(Config) - f, err := yaml.ReadFile(fname) + yml, err := ioutil.ReadFile(fname) + if err != nil { + return nil, err + } + cfg, err := yaml.FromYaml(string(yml)) if err != nil { return nil, err } - c.Put("yaml.File", f) - return FromYaml(f.Root) + return cfg, nil } // ParseYamlString parses a YAML string. This is similar but different to @@ -42,28 +40,32 @@ // - yaml (string): YAML as a string. // // Returns: -// - *Config: The configuration. +// - *yaml.Config: The configuration. func ParseYamlString(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { yamlString := p.Get("yaml", "").(string) - // Unfortunately, this does not wrap the root in a YAML file object. - root, err := yaml.Parse(bytes.NewBufferString(yamlString)) + cfg, err := yaml.FromYaml(string(yamlString)) if err != nil { return nil, err } - return FromYaml(root) + return cfg, nil } // WriteYaml writes a yaml.Node to the console as a string. // // Params: -// - yaml.Node (yaml.Node): A yaml.Node to render. +// - conf: A *yaml.Config to render. // - out (io.Writer): An output stream to write to. Default is os.Stdout. // - filename (string): If set, the file will be opened and the content will be written to it. func WriteYaml(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - top := p.Get("yaml.Node", yaml.Scalar("nothing to print")).(yaml.Node) + cfg := p.Get("conf", nil).(*yaml.Config) toStdout := p.Get("toStdout", true).(bool) + + yml, err := yaml.ToYaml(cfg) + if err != nil { + return nil, err + } var out io.Writer if nn, ok := p.Has("filename"); ok && len(nn.(string)) > 0 { file, err := os.Create(nn.(string)) @@ -71,88 +73,22 @@ } defer file.Close() out = io.Writer(file) - fmt.Fprint(out, yaml.Render(top)) + fmt.Fprint(out, yml) } else if toStdout { out = p.Get("out", os.Stdout).(io.Writer) - fmt.Fprint(out, yaml.Render(top)) + fmt.Fprint(out, yml) } - // Otherwise we supress output. + // Otherwise we supress output. return true, nil } -// MergeToYaml converts a Config object and a yaml.File to a single yaml.File. -// -// Params: -// - conf (*Config): The configuration to merge. -// - overwriteImports (bool, default true): If this is true, old config will -// overwritten. If false, we attempt to merge the old and new config, with -// preference to the old. -// -// Returns: -// - The root yaml.Node of the modified config. -// -// Uses: -// - cxt.Get("yaml.File") as the source for the YAML file. -func MergeToYaml(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - root := c.Get("yaml.File", nil).(*yaml.File).Root - cfg := p.Get("conf", nil).(*Config) - overwrite := p.Get("overwriteImports", true).(bool) - - rootMap, ok := root.(yaml.Map) - if !ok { - return nil, fmt.Errorf("Expected root node to be a map.") - } - - if len(cfg.Name) > 0 { - rootMap["package"] = yaml.Scalar(cfg.Name) - } - - if overwrite { - // Imports - imports := make([]yaml.Node, len(cfg.Imports)) - for i, imp := range cfg.Imports { - imports[i] = imp.ToYaml() - } - rootMap["import"] = yaml.List(imports) - } else { - var err error - rootMap, err = mergeImports(rootMap, cfg) - if err != nil { - Warn("Problem merging imports: %s\n", err) - } - } - - return root, nil -} - -// mergeImports merges the imports on a *Config into an existing YAML doc. -func mergeImports(root yaml.Map, cfg *Config) (yaml.Map, error) { - left, err := FromYaml(root) - if err != nil { - return root, err - } - - leftnames := make(map[string]bool, len(left.Imports)) - for _, i := range left.Imports { - leftnames[i.Name] = true - } - - for _, right := range cfg.Imports { - if _, ok := leftnames[right.Name]; !ok { - left.Imports = append(left.Imports, right) - } - } - - return left.ToYaml().(yaml.Map), nil -} - -// AddDependencies adds a list of *Dependency objects to the given *Config. +// AddDependencies adds a list of *Dependency objects to the given *yaml.Config. // // This is used to merge in packages from other sources or config files. func AddDependencies(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { - deps := p.Get("dependencies", []*Dependency{}).([]*Dependency) - config := p.Get("conf", nil).(*Config) + deps := p.Get("dependencies", []*yaml.Dependency{}).([]*yaml.Dependency) + config := p.Get("conf", nil).(*yaml.Config) // Make a set of existing package names for quick comparison. pkgSet := make(map[string]bool, len(config.Imports)) @@ -172,90 +108,6 @@ return true, nil } -func valOrEmpty(key string, store map[string]yaml.Node) string { - val, ok := store[key] - if !ok { - return "" - } - return strings.TrimSpace(val.(yaml.Scalar).String()) -} - -// boolOrDefault returns a bool, with the dft returned if there is an error or the value is not true/false -func boolOrDefault(key string, store map[string]yaml.Node, dft bool) bool { - val, ok := store[key] - if !ok { - return dft - } - switch val.(yaml.Scalar).String() { - case "true": - return true - case "false": - return false - default: - return dft - } -} - -// valOrList gets a single value or a list of values. -// -// Supports syntaxes like: -// -// subpkg: foo -// -// and -// -// supkpg: -// -foo -// -bar -func valOrList(key string, store map[string]yaml.Node) []string { - val, ok := store[key] - - subpackages := []string{} - if !ok { - return subpackages - } - - pkgs, ok := val.(yaml.List) - - if !ok { - - // Special case: Allow 'subpackages: justOne' - if one, ok := val.(yaml.Scalar); ok { - return []string{one.String()} - } - - Warn("Expected list of subpackages.\n") - return subpackages - } - - for _, pkg := range pkgs { - subpackages = append(subpackages, pkg.(yaml.Scalar).String()) - } - return subpackages -} - -func getVcsType(store map[string]yaml.Node) string { - val, ok := store["vcs"] - if !ok { - return string(v.NoVCS) - } - - name := val.(yaml.Scalar).String() - - switch name { - case "git", "hg", "bzr", "svn": - return name - case "mercurial": - return "hg" - case "bazaar": - return "bzr" - case "subversion": - return "svn" - default: - return "" - } -} - // NormalizeName takes a package name and normalizes it to the top level package. // // For example, golang.org/x/crypto/ssh becomes golang.org/x/crypto. 'ssh' is @@ -271,325 +123,3 @@ } return strings.Join(parts[0:3], "/"), extra } - -// Config is the top-level configuration object. -type Config struct { - Parent *Config - Name string - Imports Dependencies - DevImports Dependencies -} - -// HasDependency returns true if the given name is listed as an import or dev import. -func (c *Config) HasDependency(name string) bool { - for _, d := range c.Imports { - if d.Name == name { - return true - } - } - for _, d := range c.DevImports { - if d.Name == name { - return true - } - } - return false -} - -// HasRecursiveDependency returns true if this config or one of it's parents has this dependency -func (c *Config) HasRecursiveDependency(name string) bool { - if c.HasDependency(name) == true { - return true - } else if c.Parent != nil { - return c.Parent.HasRecursiveDependency(name) - } - return false -} - -// GetRoot follows the Parent down to the top node -func (c *Config) GetRoot() *Config { - if c.Parent != nil { - return c.Parent.GetRoot() - } - return c -} - -// FromYaml creates a *Config from a YAML node. -func FromYaml(top yaml.Node) (*Config, error) { - conf := new(Config) - - vals, ok := top.(yaml.Map) - if !ok { - return conf, fmt.Errorf("Top YAML node must be a map.") - } - - if name, ok := vals["package"]; ok { - conf.Name = name.(yaml.Scalar).String() - } else { - Warn("The 'package' directive is required in Glide YAML.\n") - conf.Name = "main" - } - - conf.Imports = make(Dependencies, 0, 1) - if imp, ok := vals["import"]; ok { - imports, ok := imp.(yaml.List) - - if ok { - for _, v := range imports { - dep, err := DependencyFromYaml(v) - if err != nil { - Warn("Could not add a dependency: %s\n", err) - } - conf.Imports = append(conf.Imports, dep) - } - } - } - - i, err := conf.Imports.DeDupe() - if err != nil { - return conf, err - } - conf.Imports = i - - // Same for (experimental) devimport. - // These are currently unused. Not sure what we'll do with it yet. - conf.DevImports = make(Dependencies, 0, 0) - if imp, ok := vals["devimport"]; ok { - imports, ok := imp.(yaml.List) - if ok { - for _, v := range imports { - dep, err := DependencyFromYaml(v) - if err != nil { - Warn("Could not add a dependency: %s\n", err) - } - conf.DevImports = append(conf.DevImports, dep) - } - } - } - - conf.DevImports, err = conf.DevImports.DeDupe() - if err != nil { - return conf, err - } - - return conf, nil -} - -// ToYaml returns a yaml.Map containing the data from Config. -func (c *Config) ToYaml() yaml.Node { - cfg := make(map[string]yaml.Node, 5) - - cfg["package"] = yaml.Scalar(c.Name) - - imps := make([]yaml.Node, len(c.Imports)) - for i, imp := range c.Imports { - imps[i] = imp.ToYaml() - } - devimps := make([]yaml.Node, len(c.DevImports)) - for i, dimp := range c.DevImports { - devimps[i] = dimp.ToYaml() - } - - // Fixed in 0.5.0. Prior to that, these were not being printed. Worried - // that the "fix" might introduce an unintended side effect. - if len(imps) > 0 { - cfg["import"] = yaml.List(imps) - } - if len(devimps) > 0 { - cfg["devimport"] = yaml.List(devimps) - } - - return yaml.Map(cfg) -} - -// Dependency describes a package that the present package depends upon. -type Dependency struct { - Name, Reference, Pin, Repository string - VcsType string - Subpackages, Arch, Os []string - UpdateAsVendored bool -} - -// DependencyFromYaml creates a dependency from a yaml.Node. -func DependencyFromYaml(node yaml.Node) (*Dependency, error) { - pkg, ok := node.(yaml.Map) - if !ok { - return &Dependency{}, fmt.Errorf("Expected yaml.Node to be a dependency map.") - } - dep := &Dependency{ - Name: valOrEmpty("package", pkg), - Reference: valOrEmpty("version", pkg), - Pin: valOrEmpty("pin", pkg), - VcsType: getVcsType(pkg), - Repository: valOrEmpty("repo", pkg), - Subpackages: valOrList("subpackages", pkg), - Arch: valOrList("arch", pkg), - Os: valOrList("os", pkg), - } - - // Continue to support the legacy ref property for the version. To remove - // support remove the following block. - if dep.Reference == "" { - dep.Reference = valOrEmpty("ref", pkg) - } - - if dep.Name != "" { - orig := dep.Name - dep.Name = getRepoRootFromPackage(orig) - - // The package name listed was actually a sub-package. Modify the - // config to reflect reality. - if orig != dep.Name { - subpkg := strings.TrimPrefix(orig, dep.Name) - if len(subpkg) > 0 && subpkg != "/" { - dep.Subpackages = append(dep.Subpackages, strings.TrimPrefix(subpkg, "/")) - } - } - } - - return dep, nil -} - -// GetRepo retrieves a Masterminds/vcs repo object configured for the root -// of the package being retrieved. -func (d *Dependency) GetRepo(dest string) (v.Repo, error) { - - // The remote location is either the configured repo or the package - // name as an https url. - var remote string - if len(d.Repository) > 0 { - remote = d.Repository - } else { - remote = "https://" + d.Name - } - - // If the VCS type has a value we try that first. - if len(d.VcsType) > 0 && d.VcsType != "None" { - switch v.Type(d.VcsType) { - case v.Git: - return v.NewGitRepo(remote, dest) - case v.Svn: - return v.NewSvnRepo(remote, dest) - case v.Hg: - return v.NewHgRepo(remote, dest) - case v.Bzr: - return v.NewBzrRepo(remote, dest) - default: - return nil, fmt.Errorf("Unknown VCS type %s set for %s", d.VcsType, d.Name) - } - } - - // When no type set we try to autodetect. - return v.NewRepo(remote, dest) -} - -func stripScheme(u string) string { - parts := strings.Split(u, "://") - if len(parts) > 1 { - return parts[1] - } - return u -} - -// ToYaml converts a *Dependency to a YAML Map node. -func (d *Dependency) ToYaml() yaml.Node { - dep := make(map[string]yaml.Node, 8) - dep["package"] = yaml.Scalar(d.Name) - - if len(d.Subpackages) > 0 { - subp := make([]yaml.Node, len(d.Subpackages)) - for i, item := range d.Subpackages { - subp[i] = yaml.Scalar(item) - } - - dep["subpackages"] = yaml.List(subp) - } - vcs := d.VcsType - if len(vcs) > 0 { - dep["vcs"] = yaml.Scalar(vcs) - } - if len(d.Reference) > 0 { - dep["version"] = yaml.Scalar(d.Reference) - } - if len(d.Pin) > 0 { - dep["pin"] = yaml.Scalar(d.Pin) - } - if len(d.Repository) > 0 { - dep["repo"] = yaml.Scalar(d.Repository) - } - - if len(d.Arch) > 0 { - archs := make([]yaml.Node, len(d.Arch)) - for i, a := range d.Arch { - archs[i] = yaml.Scalar(a) - } - dep["arch"] = yaml.List(archs) - } - if len(d.Os) > 0 { - oses := make([]yaml.Node, len(d.Os)) - for i, a := range d.Os { - oses[i] = yaml.Scalar(a) - } - dep["os"] = yaml.List(oses) - } - - return yaml.Map(dep) -} - -// Dependencies is a collection of Dependency -type Dependencies []*Dependency - -// Get a dependency by name -func (d Dependencies) Get(name string) *Dependency { - for _, dep := range d { - if dep.Name == name { - return dep - } - } - return nil -} - -// DeDupe cleans up duplicates on a list of dependencies. -func (d Dependencies) DeDupe() (Dependencies, error) { - checked := map[string]*Dependency{} - for _, dep := range d { - // The first time we encounter a dependency add it to the list - if val, ok := checked[dep.Name]; !ok { - checked[dep.Name] = dep - } else { - // In here we've encountered a dependency for the second time. - // Make sure the details are the same or return an error. - if dep.Reference != val.Reference { - return d, fmt.Errorf("Import %s repeated with different versions '%s' and '%s'", dep.Name, dep.Reference, val.Reference) - } - if dep.Repository != val.Repository || dep.VcsType != val.VcsType { - return d, fmt.Errorf("Import %s repeated with different Repository details", dep.Name) - } - if !reflect.DeepEqual(dep.Os, val.Os) || !reflect.DeepEqual(dep.Arch, val.Arch) { - return d, fmt.Errorf("Import %s repeated with different OS or Architecture filtering", dep.Name) - } - checked[dep.Name].Subpackages = stringArrayDeDupe(checked[dep.Name].Subpackages, dep.Subpackages...) - } - } - - imports := make(Dependencies, 0, 1) - for _, dep := range checked { - imports = append(imports, dep) - } - - return imports, nil -} - -func stringArrayDeDupe(s []string, items ...string) []string { - for _, item := range items { - exists := false - for _, v := range s { - if v == item { - exists = true - } - } - if !exists { - s = append(s, item) - } - } - return s -}
diff --git a/cmd/yaml_test.go b/cmd/yaml_test.go index 46564ee..460d1ae 100644 --- a/cmd/yaml_test.go +++ b/cmd/yaml_test.go
@@ -4,13 +4,15 @@ "testing" "github.com/Masterminds/cookoo" + "github.com/Masterminds/glide/yaml" ) var yamlFile = ` package: fake/testing import: - package: github.com/kylelemons/go-gypsy - subpackages: yaml + subpackages: + - yaml # Intentionally left spaces at end of next line. - package: github.com/Masterminds/convert repo: git@github.com:Masterminds/convert.git @@ -19,7 +21,8 @@ - color - nautical - radial - os: linux + os: + - linux arch: - i386 - arm @@ -33,7 +36,8 @@ package: fake/testing/more import: - package: github.com/kylelemons/go-gypsy - subpackages: yaml + subpackages: + - yaml ` func TestFromYaml(t *testing.T) { @@ -47,8 +51,8 @@ t.Errorf("Failed to parse YAML: %s", err) } - cfg := cxt.Get("cfg", nil).(*Config) - cfgChild := cxt.Get("childCfg", nil).(*Config) + cfg := cxt.Get("cfg", nil).(*yaml.Config) + cfgChild := cxt.Get("childCfg", nil).(*yaml.Config) cfgChild.Parent = cfg if cfg.Name != "fake/testing" { @@ -75,7 +79,7 @@ t.Errorf("Expected to find a recursive dependency") } - var imp *Dependency + var imp *yaml.Dependency for _, d := range cfg.Imports { if d.Name == "github.com/Masterminds/convert" { imp = d @@ -109,7 +113,7 @@ } if imp.Repository != "git@github.com:Masterminds/convert.git" { - t.Errorf("Got wrong repo") + t.Errorf("Got wrong repo %s on %s", imp.Repository, imp.Name) } if imp.Reference != "a9949121a2e2192ca92fa6dddfeaaa4a4412d955" { t.Errorf("Got wrong reference.")
diff --git a/glide.go b/glide.go index e868953..e1456a0 100644 --- a/glide.go +++ b/glide.go
@@ -447,12 +447,11 @@ Using("packages").From("cxt:packages"). Using("conf").From("cxt:cfg"). Using("insecure").From("cxt:insecure"). - Does(cmd.MergeToYaml, "merged").Using("conf").From("cxt:cfg"). Does(cmd.Flatten, "flatten").Using("conf").From("cxt:cfg"). Using("packages").From("cxt:packages"). Using("force").From("cxt:forceUpdate"). Does(cmd.WriteYaml, "out"). - Using("yaml.Node").From("cxt:merged"). + Using("conf").From("cxt:cfg"). Using("filename").WithDefault("glide.yaml").From("cxt:yaml") reg.Route("exec", "Execute command with GOPATH set."). @@ -485,9 +484,8 @@ Does(cmd.VendoredCleanUp, "_"). Using("conf").From("cxt:cfg"). Using("update").From("cxt:updateVendoredDeps"). - Does(cmd.MergeToYaml, "merged").Using("conf").From("cxt:cfg"). Does(cmd.WriteYaml, "out"). - Using("yaml.Node").From("cxt:merged"). + Using("conf").From("cxt:cfg"). Using("filename").From("cxt:toPath"). Using("toStdout").From("cxt:toStdout") @@ -503,9 +501,8 @@ Includes("@startup"). Includes("@ready"). Does(cmd.UpdateReferences, "refs").Using("conf").From("cxt:cfg"). - Does(cmd.MergeToYaml, "merged").Using("conf").From("cxt:cfg"). Does(cmd.WriteYaml, "out"). - Using("yaml.Node").From("cxt:merged"). + Using("conf").From("cxt:cfg"). Using("filename").From("cxt:toPath") reg.Route("import gpm", "Read a Godeps file"). @@ -520,8 +517,7 @@ Using("dependencies").From("cxt:godepsGit"). Using("conf").From("cxt:cfg"). // Does(cmd.UpdateReferences, "refs").Using("conf").From("cxt:cfg"). - Does(cmd.MergeToYaml, "merged").Using("conf").From("cxt:cfg"). - Does(cmd.WriteYaml, "out").Using("yaml.Node").From("cxt:merged"). + Does(cmd.WriteYaml, "out").Using("conf").From("cxt:cfg"). Using("filename").From("cxt:toPath") reg.Route("import godep", "Read a Godeps.json file"). @@ -532,8 +528,7 @@ Using("dependencies").From("cxt:godeps"). Using("conf").From("cxt:cfg"). // Does(cmd.UpdateReferences, "refs").Using("conf").From("cxt:cfg"). - Does(cmd.MergeToYaml, "merged").Using("conf").From("cxt:cfg"). - Does(cmd.WriteYaml, "out").Using("yaml.Node").From("cxt:merged"). + Does(cmd.WriteYaml, "out").Using("conf").From("cxt:cfg"). Using("filename").From("cxt:toPath") reg.Route("import gb", "Read a vendor/manifest file"). @@ -543,16 +538,14 @@ Does(cmd.AddDependencies, "addGodeps"). Using("dependencies").From("cxt:manifest"). Using("conf").From("cxt:cfg"). - Does(cmd.MergeToYaml, "merged").Using("conf").From("cxt:cfg"). - Does(cmd.WriteYaml, "out").Using("yaml.Node").From("cxt:merged"). + Does(cmd.WriteYaml, "out").Using("conf").From("cxt:cfg"). Using("filename").From("cxt:toPath") reg.Route("guess", "Guess dependencies"). Includes("@ready"). Does(cmd.GuessDeps, "cfg"). - Does(cmd.MergeToYaml, "merged").Using("conf").From("cxt:cfg"). Does(cmd.WriteYaml, "out"). - Using("yaml.Node").From("cxt:merged"). + Using("conf").From("cxt:cfg"). Using("filename").From("cxt:toPath") reg.Route("create", "Initialize Glide").
diff --git a/util/util.go b/util/util.go new file mode 100644 index 0000000..d84b13f --- /dev/null +++ b/util/util.go
@@ -0,0 +1,193 @@ +package util + +import ( + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + + "github.com/Masterminds/vcs" +) + +func init() { + // Precompile the regular expressions used to check VCS locations. + for _, v := range vcsList { + v.regex = regexp.MustCompile(v.pattern) + } +} + +// GetRootFromPackage retrives the top level package from a name. +// +// From a package name find the root repo. For example, +// the package github.com/Masterminds/cookoo/io has a root repo +// at github.com/Masterminds/cookoo +func GetRootFromPackage(pkg string) string { + for _, v := range vcsList { + m := v.regex.FindStringSubmatch(pkg) + if m == nil { + continue + } + + if m[1] != "" { + return m[1] + } + } + + // There are cases where a package uses the special go get magic for + // redirects. If we've not discovered the location already try that. + pkg = getRootFromGoGet(pkg) + + return pkg +} + +// Pages like https://golang.org/x/net provide an html document with +// meta tags containing a location to work with. The go tool uses +// a meta tag with the name go-import which is what we use here. +// godoc.org also has one call go-source that we do not need to use. +// The value of go-import is in the form "prefix vcs repo". The prefix +// should match the vcsURL and the repo is a location that can be +// checked out. Note, to get the html document you you need to add +// ?go-get=1 to the url. +func getRootFromGoGet(pkg string) string { + + vcsURL := "https://" + pkg + u, err := url.Parse(vcsURL) + if err != nil { + return pkg + } + if u.RawQuery == "" { + u.RawQuery = "go-get=1" + } else { + u.RawQuery = u.RawQuery + "+go-get=1" + } + checkURL := u.String() + resp, err := http.Get(checkURL) + if err != nil { + return pkg + } + defer resp.Body.Close() + + nu, err := parseImportFromBody(u, resp.Body) + if err != nil { + return pkg + } else if nu == "" { + return pkg + } + + return nu +} + +func parseImportFromBody(ur *url.URL, r io.ReadCloser) (u string, err error) { + d := xml.NewDecoder(r) + d.CharsetReader = charsetReader + d.Strict = false + var t xml.Token + for { + t, err = d.Token() + if err != nil { + if err == io.EOF { + // If we hit the end of the markup and don't have anything + // we return an error. + err = vcs.ErrCannotDetectVCS + } + return + } + if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") { + return + } + if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") { + return + } + e, ok := t.(xml.StartElement) + if !ok || !strings.EqualFold(e.Name.Local, "meta") { + continue + } + if attrValue(e.Attr, "name") != "go-import" { + continue + } + if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 { + + // If the prefix supplied by the remote system isn't a prefix to the + // url we're fetching return continue looking for more go-imports. + // This will work for exact matches and prefixes. For example, + // golang.org/x/net as a prefix will match for golang.org/x/net and + // golang.org/x/net/context. + vcsURL := ur.Host + ur.Path + if !strings.HasPrefix(vcsURL, f[0]) { + continue + } else { + u = f[0] + return + } + + } + } +} + +func charsetReader(charset string, input io.Reader) (io.Reader, error) { + switch strings.ToLower(charset) { + case "ascii": + return input, nil + default: + return nil, fmt.Errorf("can't decode XML document using charset %q", charset) + } +} + +func attrValue(attrs []xml.Attr, name string) string { + for _, a := range attrs { + if strings.EqualFold(a.Name.Local, name) { + return a.Value + } + } + return "" +} + +type vcsInfo struct { + host string + pattern string + regex *regexp.Regexp +} + +var vcsList = []*vcsInfo{ + { + host: "github.com", + pattern: `^(?P<rootpkg>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`, + }, + { + host: "bitbucket.org", + pattern: `^(?P<rootpkg>bitbucket\.org/([A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, + }, + { + host: "launchpad.net", + pattern: `^(?P<rootpkg>launchpad\.net/(([A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, + }, + { + host: "git.launchpad.net", + pattern: `^(?P<rootpkg>git\.launchpad\.net/(([A-Za-z0-9_.\-]+)|~[A-Za-z0-9_.\-]+/(\+git|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))$`, + }, + { + host: "go.googlesource.com", + pattern: `^(?P<rootpkg>go\.googlesource\.com/[A-Za-z0-9_.\-]+/?)$`, + }, + // TODO: Once Google Code becomes fully deprecated this can be removed. + { + host: "code.google.com", + pattern: `^(?P<rootpkg>code\.google\.com/[pr]/([a-z0-9\-]+)(\.([a-z0-9\-]+))?)(/[A-Za-z0-9_.\-]+)*$`, + }, + // Alternative Google setup for SVN. This is the previous structure but it still works... until Google Code goes away. + { + pattern: `^(?P<rootpkg>[a-z0-9_\-.]+\.googlecode\.com/svn(/.*)?)$`, + }, + // Alternative Google setup. This is the previous structure but it still works... until Google Code goes away. + { + pattern: `^(?P<rootpkg>[a-z0-9_\-.]+\.googlecode\.com/(git|hg))(/.*)?$`, + }, + // If none of the previous detect the type they will fall to this looking for the type in a generic sense + // by the extension to the path. + { + pattern: `^(?P<rootpkg>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(bzr|git|hg|svn))(/[A-Za-z0-9_.\-]+)*$`, + }, +}
diff --git a/util/util_test.go b/util/util_test.go new file mode 100644 index 0000000..f324293 --- /dev/null +++ b/util/util_test.go
@@ -0,0 +1,36 @@ +package util + +import "testing" + +func TestGetRootFromPackage(t *testing.T) { + urlList := map[string]string{ + "github.com/Masterminds/VCSTestRepo": "github.com/Masterminds/VCSTestRepo", + "bitbucket.org/mattfarina/testhgrepo": "bitbucket.org/mattfarina/testhgrepo", + "launchpad.net/govcstestbzrrepo/trunk": "launchpad.net/govcstestbzrrepo/trunk", + "launchpad.net/~mattfarina/+junk/mygovcstestbzrrepo": "launchpad.net/~mattfarina/+junk/mygovcstestbzrrepo", + "launchpad.net/~mattfarina/+junk/mygovcstestbzrrepo/trunk": "launchpad.net/~mattfarina/+junk/mygovcstestbzrrepo", + "git.launchpad.net/govcstestgitrepo": "git.launchpad.net/govcstestgitrepo", + "git.launchpad.net/~mattfarina/+git/mygovcstestgitrepo": "git.launchpad.net/~mattfarina/+git/mygovcstestgitrepo", + "farbtastic.googlecode.com/svn/": "farbtastic.googlecode.com/svn/", + "farbtastic.googlecode.com/svn/trunk": "farbtastic.googlecode.com/svn/trunk", + "code.google.com/p/farbtastic": "code.google.com/p/farbtastic", + "code.google.com/p/plotinum": "code.google.com/p/plotinum", + "example.com/foo/bar.git": "example.com/foo/bar.git", + "example.com/foo/bar.svn": "example.com/foo/bar.svn", + "example.com/foo/bar/baz.bzr": "example.com/foo/bar/baz.bzr", + "example.com/foo/bar/baz.hg": "example.com/foo/bar/baz.hg", + "gopkg.in/mgo.v2": "gopkg.in/mgo.v2", + "gopkg.in/mgo.v2/txn": "gopkg.in/mgo.v2", + "gopkg.in/nowk/assert.v2": "gopkg.in/nowk/assert.v2", + "gopkg.in/nowk/assert.v2/tests": "gopkg.in/nowk/assert.v2", + "golang.org/x/net": "golang.org/x/net", + "golang.org/x/net/context": "golang.org/x/net", + } + + for u, c := range urlList { + repo := GetRootFromPackage(u) + if repo != c { + t.Errorf("getRepoRootFromPackage expected %s but got %s", c, repo) + } + } +}
diff --git a/yaml/yaml.go b/yaml/yaml.go new file mode 100644 index 0000000..de0fcb2 --- /dev/null +++ b/yaml/yaml.go
@@ -0,0 +1,230 @@ +// Package yaml provides the ability to work with glide.yaml files. +package yaml + +import ( + "fmt" + "reflect" + "strings" + + "github.com/Masterminds/glide/util" + "github.com/Masterminds/vcs" + "gopkg.in/yaml.v2" +) + +// FromYaml takes a yaml string and converts it to a Config instance. +func FromYaml(yml string) (*Config, error) { + c := &Config{} + err := yaml.Unmarshal([]byte(yml), &c) + if err != nil { + return nil, err + } + + // The ref property is for the legacy yaml file structure. + // This sets the currect version to the ref if a version isn't + // already set. + for _, v := range c.Imports { + if v.Reference == "" && v.Ref != "" { + v.Reference = v.Ref + } + v.Ref = "" + + // Make sure only legitimate VCS are listed. + v.VcsType = filterVcsType(v.VcsType) + + // Get the root name for the package + o := v.Name + v.Name = util.GetRootFromPackage(v.Name) + subpkg := strings.TrimPrefix(o, v.Name) + if len(subpkg) > 0 && subpkg != o { + v.Subpackages = append(v.Subpackages, strings.TrimPrefix(subpkg, "/")) + } + } + for _, v := range c.DevImports { + if v.Reference == "" && v.Ref != "" { + v.Reference = v.Ref + } + v.Ref = "" + + v.VcsType = filterVcsType(v.VcsType) + + // Get the root name for the package + o := v.Name + v.Name = util.GetRootFromPackage(v.Name) + subpkg := strings.TrimPrefix(o, v.Name) + if len(subpkg) > 0 && subpkg != o { + v.Subpackages = append(v.Subpackages, subpkg) + } + } + return c, nil +} + +// ToYaml takes a *Config instance and converts it into a yaml string. +func ToYaml(cfg *Config) (string, error) { + yml, err := yaml.Marshal(&cfg) + if err != nil { + return "", err + } + return string(yml), nil +} + +// Config is the top-level configuration object. +type Config struct { + Parent *Config + Name string `yaml:"package"` + Imports Dependencies `yaml:"import"` + DevImports Dependencies `yaml:"devimport"` +} + +// HasDependency returns true if the given name is listed as an import or dev import. +func (c *Config) HasDependency(name string) bool { + for _, d := range c.Imports { + if d.Name == name { + return true + } + } + for _, d := range c.DevImports { + if d.Name == name { + return true + } + } + return false +} + +// HasRecursiveDependency returns true if this config or one of it's parents has this dependency +func (c *Config) HasRecursiveDependency(name string) bool { + if c.HasDependency(name) == true { + return true + } else if c.Parent != nil { + return c.Parent.HasRecursiveDependency(name) + } + return false +} + +// GetRoot follows the Parent down to the top node +func (c *Config) GetRoot() *Config { + if c.Parent != nil { + return c.Parent.GetRoot() + } + return c +} + +// Dependencies is a collection of Dependency +type Dependencies []*Dependency + +// Dependency describes a package that the present package depends upon. +type Dependency struct { + Name string `yaml:"package"` + Reference string `yaml:"version,omitempty"` + Ref string `yaml:"ref,omitempty"` + Pin string `yaml:"pin,omitempty"` + Repository string `yaml:"repo,omitempty"` + VcsType string `yaml:"vcs,omitempty"` + Subpackages []string `yaml:"subpackages,omitempty"` + Arch []string `yaml:"arch,omitempty"` + Os []string `yaml:"os,omitempty"` + UpdateAsVendored bool `yaml:"-"` +} + +// GetRepo retrieves a Masterminds/vcs repo object configured for the root +// of the package being retrieved. +func (d *Dependency) GetRepo(dest string) (vcs.Repo, error) { + + // The remote location is either the configured repo or the package + // name as an https url. + var remote string + if len(d.Repository) > 0 { + remote = d.Repository + } else { + remote = "https://" + d.Name + } + + // If the VCS type has a value we try that first. + if len(d.VcsType) > 0 && d.VcsType != "None" { + switch vcs.Type(d.VcsType) { + case vcs.Git: + return vcs.NewGitRepo(remote, dest) + case vcs.Svn: + return vcs.NewSvnRepo(remote, dest) + case vcs.Hg: + return vcs.NewHgRepo(remote, dest) + case vcs.Bzr: + return vcs.NewBzrRepo(remote, dest) + default: + return nil, fmt.Errorf("Unknown VCS type %s set for %s", d.VcsType, d.Name) + } + } + + // When no type set we try to autodetect. + return vcs.NewRepo(remote, dest) +} + +// Get a dependency by name +func (d Dependencies) Get(name string) *Dependency { + for _, dep := range d { + if dep.Name == name { + return dep + } + } + return nil +} + +// DeDupe cleans up duplicates on a list of dependencies. +func (d Dependencies) DeDupe() (Dependencies, error) { + checked := map[string]*Dependency{} + for _, dep := range d { + // The first time we encounter a dependency add it to the list + if val, ok := checked[dep.Name]; !ok { + checked[dep.Name] = dep + } else { + // In here we've encountered a dependency for the second time. + // Make sure the details are the same or return an error. + if dep.Reference != val.Reference { + return d, fmt.Errorf("Import %s repeated with different versions '%s' and '%s'", dep.Name, dep.Reference, val.Reference) + } + if dep.Repository != val.Repository || dep.VcsType != val.VcsType { + return d, fmt.Errorf("Import %s repeated with different Repository details", dep.Name) + } + if !reflect.DeepEqual(dep.Os, val.Os) || !reflect.DeepEqual(dep.Arch, val.Arch) { + return d, fmt.Errorf("Import %s repeated with different OS or Architecture filtering", dep.Name) + } + checked[dep.Name].Subpackages = stringArrayDeDupe(checked[dep.Name].Subpackages, dep.Subpackages...) + } + } + + imports := make(Dependencies, 0, 1) + for _, dep := range checked { + imports = append(imports, dep) + } + + return imports, nil +} + +func stringArrayDeDupe(s []string, items ...string) []string { + for _, item := range items { + exists := false + for _, v := range s { + if v == item { + exists = true + } + } + if !exists { + s = append(s, item) + } + } + return s +} + +func filterVcsType(vcs string) string { + switch vcs { + case "git", "hg", "bzr", "svn": + return vcs + case "mercurial": + return "hg" + case "bazaar": + return "bzr" + case "subversion": + return "svn" + default: + return "" + } +}
diff --git a/yaml/yaml_test.go b/yaml/yaml_test.go new file mode 100644 index 0000000..94d160d --- /dev/null +++ b/yaml/yaml_test.go
@@ -0,0 +1,66 @@ +package yaml + +import "testing" + +var yml = ` +package: fake/testing +import: + - package: github.com/kylelemons/go-gypsy + subpackages: + - yaml + # Intentionally left spaces at end of next line. + - package: github.com/Masterminds/convert + repo: git@github.com:Masterminds/convert.git + ref: a9949121a2e2192ca92fa6dddfeaaa4a4412d955 + subpackages: + - color + - nautical + - radial + os: + - linux + arch: + - i386 + - arm + - package: github.com/Masterminds/structable + - package: github.com/Masterminds/cookoo/color + +devimport: + - package: github.com/kylelemons/go-gypsy +` + +func TestFromYaml(t *testing.T) { + cfg, err := FromYaml(yml) + if err != nil { + t.Errorf("Unexpected error parsing yaml %s", err) + } + + if cfg.Name != "fake/testing" { + t.Errorf("Inaccurate name found %s", cfg.Name) + } + + found := false + for _, i := range cfg.Imports { + if i.Name == "github.com/Masterminds/cookoo" { + found = true + } + } + if !found { + t.Error("Unable to find github.com/Masterminds/cookoo") + } +} + +func TestToYaml(t *testing.T) { + cfg, err := FromYaml(yml) + if err != nil { + t.Errorf("Unexpected error parsing yaml %s", err) + } + + o, err := ToYaml(cfg) + if err != nil { + t.Errorf("Unexpected error converting cfg to yaml %s", err) + } + + if o == "" { + t.Error("Yaml output not generated when expected") + } +}