Merge pull request #10 from 30x/XAPID-1070
Xapid 1070
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..120c8e0
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,15 @@
+language: go
+
+go:
+ - 1.8.x
+
+before_install:
+ - sudo add-apt-repository ppa:masterminds/glide -y
+ - sudo apt-get update -q
+ - sudo apt-get install glide -y
+
+install:
+ - glide install --strip-vendor
+
+script:
+ - go test
diff --git a/bundle.go b/bundle.go
index b2206ea..622c8ed 100644
--- a/bundle.go
+++ b/bundle.go
@@ -48,12 +48,12 @@
apiMan apiManagerInterface
concurrentDownloads int
markDeploymentFailedAfter time.Duration
- bundleDownloadConnTimeout time.Duration
bundleRetryDelay time.Duration
bundleCleanupDelay time.Duration
downloadQueue chan *DownloadRequest
isClosed *int32
workers []*BundleDownloader
+ client *http.Client
}
type blobServerResponse struct {
@@ -103,7 +103,7 @@
blobId: id,
backoffFunc: createBackoff(retryIn, maxBackOff),
markFailedAt: markFailedAt,
- connTimeout: bm.bundleDownloadConnTimeout,
+ client: bm.client,
}
}
@@ -158,8 +158,8 @@
blobId string
backoffFunc func()
markFailedAt time.Time
- connTimeout time.Duration
blobServerURL string
+ client *http.Client
}
func (r *DownloadRequest) downloadBundle() error {
@@ -172,7 +172,7 @@
}
}
- downloadedFile, err := downloadFromURI(r.blobServerURL, r.blobId, r.connTimeout)
+ downloadedFile, err := downloadFromURI(r.client, r.blobServerURL, r.blobId)
if err != nil {
log.Errorf("Unable to download blob file blobId=%s err:%v", r.blobId, err)
@@ -210,7 +210,7 @@
return path.Join(bundlePath, base64.StdEncoding.EncodeToString([]byte(blobId)))
}
-func getSignedURL(blobServerURL string, blobId string, bundleDownloadConnTimeout time.Duration) (string, error) {
+func getSignedURL(client *http.Client, blobServerURL string, blobId string) (string, error) {
blobUri, err := url.Parse(blobServerURL)
if err != nil {
@@ -224,7 +224,7 @@
uri := blobUri.String()
- surl, err := getURIReader(uri, bundleDownloadConnTimeout)
+ surl, err := getUriReaderWithAuth(client, uri)
if err != nil {
log.Errorf("Unable to get signed URL from BlobServer %s: %v", uri, err)
return "", err
@@ -248,12 +248,12 @@
// downloadFromURI involves retrieving the signed URL for the blob, and storing the resource locally
// after downloading the resource from GCS (via the signed URL)
-func downloadFromURI(blobServerURL string, blobId string, bundleDownloadConnTimeout time.Duration) (tempFileName string, err error) {
+func downloadFromURI(client *http.Client, blobServerURL string, blobId string) (tempFileName string, err error) {
var tempFile *os.File
log.Debugf("Downloading bundle: %s", blobId)
- uri, err := getSignedURL(blobServerURL, blobId, bundleDownloadConnTimeout)
+ uri, err := getSignedURL(client, blobServerURL, blobId)
if err != nil {
log.Errorf("Unable to get signed URL for blobId {%s}, error : {%v}", blobId, err)
return
@@ -268,7 +268,7 @@
tempFileName = tempFile.Name()
var confReader io.ReadCloser
- confReader, err = getURIReader(uri, bundleDownloadConnTimeout)
+ confReader, err = getUriReaderWithAuth(client, uri)
if err != nil {
log.Errorf("Unable to retrieve bundle %s: %v", uri, err)
return
@@ -286,12 +286,14 @@
}
// retrieveBundle retrieves bundle data from a URI
-func getURIReader(uriString string, bundleDownloadConnTimeout time.Duration) (io.ReadCloser, error) {
-
- client := http.Client{
- Timeout: bundleDownloadConnTimeout,
+func getUriReaderWithAuth(client *http.Client, uriString string) (io.ReadCloser, error) {
+ req, err := http.NewRequest("GET", uriString, nil)
+ if err != nil {
+ return nil, err
}
- res, err := client.Get(uriString)
+ // add Auth
+ req.Header.Add("Authorization", getBearerToken())
+ res, err := client.Do(req)
if err != nil {
return nil, err
}
diff --git a/bundle_test.go b/bundle_test.go
index 0b8b535..7946f96 100644
--- a/bundle_test.go
+++ b/bundle_test.go
@@ -72,11 +72,16 @@
apiMan: dummyApiMan,
concurrentDownloads: concurrentDownloads,
markDeploymentFailedAfter: 5 * time.Second,
- bundleDownloadConnTimeout: time.Second,
bundleRetryDelay: time.Second,
bundleCleanupDelay: 5 * time.Second,
downloadQueue: make(chan *DownloadRequest, downloadQueueSize),
isClosed: new(int32),
+ client: &http.Client{
+ Timeout: time.Second,
+ Transport: &http.Transport{
+ MaxIdleConnsPerHost: 10,
+ },
+ },
}
testBundleMan.initializeBundleDownloading()
time.Sleep(100 * time.Millisecond)
@@ -101,7 +106,7 @@
// setup timeout
atomic.StoreInt32(blobServer.signedTimeout, 1)
atomic.StoreInt32(blobServer.blobTimeout, 1)
- testBundleMan.bundleDownloadConnTimeout = 500 * time.Millisecond
+ testBundleMan.client.Timeout = 500 * time.Millisecond
testBundleMan.bundleRetryDelay = 50 * time.Millisecond
// download blobs
@@ -116,7 +121,7 @@
// setup timeout
atomic.StoreInt32(blobServer.signedTimeout, 1)
atomic.StoreInt32(blobServer.blobTimeout, 1)
- testBundleMan.bundleDownloadConnTimeout = 100 * time.Millisecond
+ testBundleMan.client.Timeout = 100 * time.Millisecond
testBundleMan.bundleRetryDelay = 100 * time.Millisecond
testBundleMan.markDeploymentFailedAfter = 200 * time.Millisecond
diff --git a/glide.yaml b/glide.yaml
new file mode 100644
index 0000000..0dc35e8
--- /dev/null
+++ b/glide.yaml
@@ -0,0 +1,25 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package: github.com/30x/apidGatewayConfDeploy
+import:
+- package: github.com/30x/apid-core
+ version: master
+- package: github.com/apigee-labs/transicator/common
+ version: master
+- package: github.com/gorilla/mux
+ version: master
+testImport:
+- package: github.com/onsi/ginkgo
+- package: github.com/onsi/gomega
diff --git a/init.go b/init.go
index d27b566..1c4e16d 100644
--- a/init.go
+++ b/init.go
@@ -21,6 +21,7 @@
"time"
"github.com/30x/apid-core"
+ "net/http"
"sync"
)
@@ -40,6 +41,9 @@
configDownloadQueueSize = "apigeesync_download_queue_size"
configBlobServerBaseURI = "apigeesync_blob_server_base"
configStoragePath = "local_storage_path"
+ maxIdleConnsPerHost = 50
+ httpTimeout = time.Minute
+ configBearerToken = "apigeesync_bearer_token"
)
var (
@@ -107,6 +111,18 @@
log.Debug("apiServerBaseURI = " + apiServerBaseURI.String())
+ // initialize tracker client
+ httpClient := &http.Client{
+ Transport: &http.Transport{
+ MaxIdleConnsPerHost: maxIdleConnsPerHost,
+ },
+ Timeout: httpTimeout,
+ CheckRedirect: func(req *http.Request, _ []*http.Request) error {
+ req.Header.Set("Authorization", getBearerToken())
+ return nil
+ },
+ }
+
// initialize db manager
dbMan := &dbManager{
@@ -145,11 +161,11 @@
apiMan: apiMan,
concurrentDownloads: concurrentDownloads,
markDeploymentFailedAfter: markDeploymentFailedAfter,
- bundleDownloadConnTimeout: bundleDownloadConnTimeout,
bundleRetryDelay: time.Second,
bundleCleanupDelay: bundleCleanupDelay,
downloadQueue: make(chan *DownloadRequest, downloadQueueSize),
isClosed: new(int32),
+ client: httpClient,
}
bundleMan.initializeBundleDownloading()
@@ -171,3 +187,7 @@
return pluginData, nil
}
+
+func getBearerToken() string {
+ return "Bearer " + config.GetString(configBearerToken)
+}
