make num concurrent downloads and download queue size configurable
diff --git a/apidGatewayDeploy_suite_test.go b/apidGatewayDeploy_suite_test.go
index 7411e6f..4604588 100644
--- a/apidGatewayDeploy_suite_test.go
+++ b/apidGatewayDeploy_suite_test.go
@@ -50,6 +50,8 @@
bundleCleanupDelay = time.Millisecond
bundleRetryDelay = 10 * time.Millisecond
bundleDownloadTimeout = 50 * time.Millisecond
+ concurrentDownloads = 1
+ downloadQueueSize = 1
router := apid.API().Router()
// fake an unreliable bundle repo
diff --git a/bundle.go b/bundle.go
index 651576c..6f0c208 100644
--- a/bundle.go
+++ b/bundle.go
@@ -17,11 +17,12 @@
"time"
)
-var numConcurrentDownloads = 15
-var bundleRetryDelay time.Duration = time.Second
-var bundleDownloadTimeout time.Duration = 10 * time.Minute
-var downloadQueue = make(chan *DownloadRequest, 200)
-var workerQueue = make(chan chan *DownloadRequest, numConcurrentDownloads)
+var (
+ bundleRetryDelay time.Duration = time.Second
+ bundleDownloadTimeout time.Duration = 10 * time.Minute
+ downloadQueue = make(chan *DownloadRequest, downloadQueueSize)
+ workerQueue = make(chan chan *DownloadRequest, concurrentDownloads)
+)
// simple doubling back-off
func createBackoff(retryIn, maxBackOff time.Duration) func() {
@@ -245,7 +246,7 @@
func initializeBundleDownloading() {
// create workers
- for i := 0; i < numConcurrentDownloads; i++ {
+ for i := 0; i < concurrentDownloads; i++ {
worker := BundleDownloader{
id: i + 1,
workChan: make(chan *DownloadRequest),
diff --git a/init.go b/init.go
index 68d75e6..f507357 100644
--- a/init.go
+++ b/init.go
@@ -18,18 +18,22 @@
configApiServerBaseURI = "apigeesync_proxy_server_base"
configApidInstanceID = "apigeesync_apid_instance_id"
configApidClusterID = "apigeesync_cluster_id"
+ configConcurrentDownloads = "apigeesync_concurrent_downloads"
+ configDownloadQueueSize = "apigeesync_download_queue_size"
)
var (
- services apid.Services
- log apid.LogService
- data apid.DataService
- bundlePath string
- debounceDuration time.Duration
- bundleCleanupDelay time.Duration
- apiServerBaseURI *url.URL
- apidInstanceID string
- apidClusterID string
+ services apid.Services
+ log apid.LogService
+ data apid.DataService
+ bundlePath string
+ debounceDuration time.Duration
+ bundleCleanupDelay time.Duration
+ apiServerBaseURI *url.URL
+ apidInstanceID string
+ apidClusterID string
+ downloadQueueSize int
+ concurrentDownloads int
)
func init() {
@@ -66,6 +70,8 @@
config.SetDefault(configDebounceDuration, time.Second)
config.SetDefault(configBundleCleanupDelay, time.Minute)
config.SetDefault(configBundleDownloadTimeout, 5*time.Minute)
+ config.SetDefault(configConcurrentDownloads, 15)
+ config.SetDefault(configDownloadQueueSize, 2000)
debounceDuration = config.GetDuration(configDebounceDuration)
if debounceDuration < time.Millisecond {
@@ -84,6 +90,8 @@
data = services.Data()
+ concurrentDownloads = config.GetInt(configConcurrentDownloads)
+ downloadQueueSize = config.GetInt(configDownloadQueueSize)
relativeBundlePath := config.GetString(configBundleDirKey)
storagePath := config.GetString("local_storage_path")
bundlePath = path.Join(storagePath, relativeBundlePath)
