make num concurrent downloads and download queue size configurable
diff --git a/apidGatewayDeploy_suite_test.go b/apidGatewayDeploy_suite_test.go index 7411e6f..4604588 100644 --- a/apidGatewayDeploy_suite_test.go +++ b/apidGatewayDeploy_suite_test.go
@@ -50,6 +50,8 @@ bundleCleanupDelay = time.Millisecond bundleRetryDelay = 10 * time.Millisecond bundleDownloadTimeout = 50 * time.Millisecond + concurrentDownloads = 1 + downloadQueueSize = 1 router := apid.API().Router() // fake an unreliable bundle repo
diff --git a/bundle.go b/bundle.go index 651576c..6f0c208 100644 --- a/bundle.go +++ b/bundle.go
@@ -17,11 +17,12 @@ "time" ) -var numConcurrentDownloads = 15 -var bundleRetryDelay time.Duration = time.Second -var bundleDownloadTimeout time.Duration = 10 * time.Minute -var downloadQueue = make(chan *DownloadRequest, 200) -var workerQueue = make(chan chan *DownloadRequest, numConcurrentDownloads) +var ( + bundleRetryDelay time.Duration = time.Second + bundleDownloadTimeout time.Duration = 10 * time.Minute + downloadQueue = make(chan *DownloadRequest, downloadQueueSize) + workerQueue = make(chan chan *DownloadRequest, concurrentDownloads) +) // simple doubling back-off func createBackoff(retryIn, maxBackOff time.Duration) func() { @@ -245,7 +246,7 @@ func initializeBundleDownloading() { // create workers - for i := 0; i < numConcurrentDownloads; i++ { + for i := 0; i < concurrentDownloads; i++ { worker := BundleDownloader{ id: i + 1, workChan: make(chan *DownloadRequest),
diff --git a/init.go b/init.go index 68d75e6..f507357 100644 --- a/init.go +++ b/init.go
@@ -18,18 +18,22 @@ configApiServerBaseURI = "apigeesync_proxy_server_base" configApidInstanceID = "apigeesync_apid_instance_id" configApidClusterID = "apigeesync_cluster_id" + configConcurrentDownloads = "apigeesync_concurrent_downloads" + configDownloadQueueSize = "apigeesync_download_queue_size" ) var ( - services apid.Services - log apid.LogService - data apid.DataService - bundlePath string - debounceDuration time.Duration - bundleCleanupDelay time.Duration - apiServerBaseURI *url.URL - apidInstanceID string - apidClusterID string + services apid.Services + log apid.LogService + data apid.DataService + bundlePath string + debounceDuration time.Duration + bundleCleanupDelay time.Duration + apiServerBaseURI *url.URL + apidInstanceID string + apidClusterID string + downloadQueueSize int + concurrentDownloads int ) func init() { @@ -66,6 +70,8 @@ config.SetDefault(configDebounceDuration, time.Second) config.SetDefault(configBundleCleanupDelay, time.Minute) config.SetDefault(configBundleDownloadTimeout, 5*time.Minute) + config.SetDefault(configConcurrentDownloads, 15) + config.SetDefault(configDownloadQueueSize, 2000) debounceDuration = config.GetDuration(configDebounceDuration) if debounceDuration < time.Millisecond { @@ -84,6 +90,8 @@ data = services.Data() + concurrentDownloads = config.GetInt(configConcurrentDownloads) + downloadQueueSize = config.GetInt(configDownloadQueueSize) relativeBundlePath := config.GetString(configBundleDirKey) storagePath := config.GetString("local_storage_path") bundlePath = path.Join(storagePath, relativeBundlePath)