Add a cache for DeduceRemoteRepo This skips the network ping, and speeds up for example gvt fetch camlistore.org/pkg/client by many minutes, since all fetches from camlistore.org after the first become local.
diff --git a/downloader.go b/downloader.go index 3fb76aa..c9081d6 100644 --- a/downloader.go +++ b/downloader.go
@@ -1,6 +1,7 @@ package main import ( + "strings" "sync" "github.com/FiloSottile/gvt/gbvendor" @@ -19,14 +20,20 @@ // Downloader acts as a cache for downloaded repositories type Downloader struct { - mu sync.Mutex - m map[cacheKey]*cacheEntry + wcsMu sync.Mutex + wcs map[cacheKey]*cacheEntry + + reposMu sync.RWMutex + repos map[string]vendor.RemoteRepo + reposI map[string]vendor.RemoteRepo } var GlobalDownloader = Downloader{} func init() { - GlobalDownloader.m = make(map[cacheKey]*cacheEntry) + GlobalDownloader.wcs = make(map[cacheKey]*cacheEntry) + GlobalDownloader.repos = make(map[string]vendor.RemoteRepo) + GlobalDownloader.reposI = make(map[string]vendor.RemoteRepo) } // Get returns a cached WorkingCopy, or runs RemoteRepo.Checkout @@ -35,17 +42,17 @@ url: repo.URL(), repoType: repo.Type(), branch: branch, tag: tag, revision: revision, } - d.mu.Lock() - if entry, ok := d.m[key]; ok { - d.mu.Unlock() + d.wcsMu.Lock() + if entry, ok := d.wcs[key]; ok { + d.wcsMu.Unlock() entry.wg.Wait() return entry.v, entry.err } entry := &cacheEntry{} entry.wg.Add(1) - d.m[key] = entry - d.mu.Unlock() + d.wcs[key] = entry + d.wcsMu.Unlock() entry.v, entry.err = repo.Checkout(branch, tag, revision) entry.wg.Done() @@ -53,10 +60,10 @@ } func (d *Downloader) Flush() error { - d.mu.Lock() - defer d.mu.Unlock() + d.wcsMu.Lock() + defer d.wcsMu.Unlock() - for _, entry := range d.m { + for _, entry := range d.wcs { entry.wg.Wait() if entry.err != nil { continue @@ -67,3 +74,37 @@ } return nil } + +// DeduceRemoteRepo is a cached version of vendor.DeduceRemoteRepo +func (d *Downloader) DeduceRemoteRepo(path string, insecure bool) (vendor.RemoteRepo, string, error) { + cache := d.repos + if insecure { + cache = d.reposI + } + + d.reposMu.RLock() + for p, repo := range cache { + if path == p || strings.HasPrefix(path, p+"/") { + d.reposMu.RUnlock() + extra := strings.Trim(strings.TrimPrefix(path, p), "/") + return repo, extra, nil + } + } + d.reposMu.RUnlock() + + repo, extra, err := vendor.DeduceRemoteRepo(path, insecure) + if err != nil { + return repo, extra, err + } + + if !strings.HasSuffix(path, extra) { + // Shouldn't happen, but in case just bypass the cache + return repo, extra, err + } + basePath := strings.Trim(strings.TrimSuffix(path, extra), "/") + d.reposMu.Lock() + cache[basePath] = repo + d.reposMu.Unlock() + + return repo, extra, err +}
diff --git a/fetch.go b/fetch.go index cb9814c..089a1ac 100644 --- a/fetch.go +++ b/fetch.go
@@ -159,7 +159,7 @@ // Find and download the repository - repo, extra, err := vendor.DeduceRemoteRepo(fullPath, insecure) + repo, extra, err := GlobalDownloader.DeduceRemoteRepo(fullPath, insecure) if err != nil { return err }