[ISSUE-67015020] query by config id, type filter for "/configurations"
diff --git a/api.go b/api.go index f676aaa..746a11c 100644 --- a/api.go +++ b/api.go
@@ -15,6 +15,7 @@ import ( "bytes" + "database/sql" "encoding/json" "fmt" "github.com/gorilla/mux" @@ -22,6 +23,7 @@ "io/ioutil" "net/http" "net/url" + "regexp" "strconv" "sync/atomic" "time" @@ -38,14 +40,17 @@ ) const ( - deploymentsEndpoint = "/configurations" - blobEndpointPath = "/blobs" - blobEndpoint = blobEndpointPath + "/{blobId}" + deploymentsEndpoint = "/configurations" + blobEndpointPath = "/blobs" + blobEndpoint = blobEndpointPath + "/{blobId}" + deploymentIdEndpoint = deploymentsEndpoint + "/{configId}" ) const ( API_ERR_BAD_BLOCK = iota + 1 API_ERR_INTERNAL + API_ERR_BAD_CONFIG_ID + API_ERR_NOT_FOUND ) const ( @@ -102,14 +107,15 @@ } type apiManager struct { - dbMan dbManagerInterface - deploymentsEndpoint string - blobEndpoint string - eTag int64 - deploymentsChanged chan interface{} - addSubscriber chan chan deploymentsResult - removeSubscriber chan chan deploymentsResult - apiInitialized bool + dbMan dbManagerInterface + deploymentsEndpoint string + blobEndpoint string + deploymentIdEndpoint string + eTag int64 + deploymentsChanged chan interface{} + addSubscriber chan chan deploymentsResult + removeSubscriber chan chan deploymentsResult + apiInitialized bool } func (a *apiManager) InitAPI() { @@ -118,6 +124,7 @@ } services.API().HandleFunc(a.deploymentsEndpoint, a.apiGetCurrentDeployments).Methods("GET") services.API().HandleFunc(a.blobEndpoint, a.apiReturnBlobData).Methods("GET") + services.API().HandleFunc(a.deploymentIdEndpoint, a.apiHandleConfigId).Methods("GET") a.apiInitialized = true log.Debug("API endpoints initialized") } @@ -231,6 +238,47 @@ } +func (a *apiManager) apiHandleConfigId(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + configId := vars["configId"] + if !isValidUuid(configId) { + a.writeError(w, http.StatusBadRequest, API_ERR_BAD_CONFIG_ID, "configuration id is invalid") + return + } + config, err := a.dbMan.getConfigById(configId) + if err != nil { + if err == sql.ErrNoRows { + a.writeError(w, http.StatusNotFound, API_ERR_NOT_FOUND, "cannot find the configuration") + } else { + log.Errorf("apiHandleConfigId: %v", err) + a.writeInternalError(w, err.Error()) + } + return + } + configDetail := ApiDeploymentDetails{ + Self: getHttpHost() + a.deploymentsEndpoint + "/" + config.ID, + Name: config.Name, + Type: config.Type, + Revision: config.Revision, + BeanBlobUrl: getBlobUrl(config.BlobID), + Org: config.OrgID, + Env: config.EnvID, + ResourceBlobUrl: getBlobUrl(config.BlobResourceID), + Path: config.Path, + Created: convertTime(config.Created), + Updated: convertTime(config.Updated), + } + + b, err := json.Marshal(configDetail) + if err != nil { + log.Errorf("unable to marshal config: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + log.Debugf("sending configuration %s", b) + w.Write(b) +} + func (a *apiManager) apiGetCurrentDeployments(w http.ResponseWriter, r *http.Request) { // If returning without a bundle (immediately or after timeout), status = 404 @@ -240,6 +288,7 @@ // If timeout > 0 AND there is no deployment (or new deployment) available (per If-None-Match), then // block for up to the specified number of seconds until a new deployment becomes available. b := r.URL.Query().Get("block") + typeFilter := r.URL.Query().Get("type") var timeout int if b != "" { var err error @@ -266,7 +315,7 @@ // send results if different eTag if eTag != ifNoneMatch { - a.sendReadyDeployments(w) + a.sendReadyDeployments(typeFilter, w) return } @@ -285,7 +334,7 @@ if result.err != nil { a.writeInternalError(w, "Database error") } else { - a.sendDeployments(w, result.deployments, result.eTag) + a.sendDeployments(w, result.deployments, result.eTag, typeFilter) } case <-time.After(time.Duration(timeout) * time.Second): @@ -294,22 +343,22 @@ if ifNoneMatch != "" { w.WriteHeader(http.StatusNotModified) } else { - a.sendReadyDeployments(w) + a.sendReadyDeployments(typeFilter, w) } } } -func (a *apiManager) sendReadyDeployments(w http.ResponseWriter) { +func (a *apiManager) sendReadyDeployments(typeFilter string, w http.ResponseWriter) { eTag := a.getETag() - deployments, err := a.dbMan.getReadyDeployments() + deployments, err := a.dbMan.getReadyDeployments(typeFilter) if err != nil { a.writeInternalError(w, fmt.Sprintf("Database error: %s", err.Error())) return } - a.sendDeployments(w, deployments, eTag) + a.sendDeployments(w, deployments, eTag, typeFilter) } -func (a *apiManager) sendDeployments(w http.ResponseWriter, dataDeps []DataDeployment, eTag string) { +func (a *apiManager) sendDeployments(w http.ResponseWriter, dataDeps []DataDeployment, eTag string, typeFilter string) { apiDeps := ApiDeploymentResponse{} apiDepDetails := make([]ApiDeploymentDetails, 0) @@ -334,6 +383,10 @@ } apiDeps.ApiDeploymentsResponse = apiDepDetails + if typeFilter != "" { + apiDeps.Self += "?type=" + typeFilter + } + b, err := json.Marshal(apiDeps) if err != nil { log.Errorf("unable to marshal deployments: %v", err) @@ -380,6 +433,11 @@ return t } +func isValidUuid(uuid string) bool { + r := regexp.MustCompile("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[8|9|aA|bB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$") + return r.MatchString(uuid) +} + func getHttpHost() string { configuredEndpoint := config.GetString(configBundleBlobDownloadEndpoint)
diff --git a/api_test.go b/api_test.go index ccd3b0c..c5454c1 100644 --- a/api_test.go +++ b/api_test.go
@@ -20,12 +20,14 @@ "net/url" "crypto/rand" + "database/sql" "fmt" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" mathrand "math/rand" "os" "strconv" + "strings" "time" ) @@ -43,13 +45,14 @@ testCount += 1 dummyDbMan = &dummyDbManager{} testApiMan = &apiManager{ - dbMan: dummyDbMan, - deploymentsEndpoint: deploymentsEndpoint + strconv.Itoa(testCount), - blobEndpoint: blobEndpointPath + strconv.Itoa(testCount) + "/{blobId}", - eTag: int64(testCount * 10), - deploymentsChanged: make(chan interface{}, 5), - addSubscriber: make(chan chan deploymentsResult), - removeSubscriber: make(chan chan deploymentsResult), + dbMan: dummyDbMan, + deploymentsEndpoint: deploymentsEndpoint + strconv.Itoa(testCount), + blobEndpoint: blobEndpointPath + strconv.Itoa(testCount) + "/{blobId}", + deploymentIdEndpoint: deploymentsEndpoint + strconv.Itoa(testCount) + "/{configId}", + eTag: int64(testCount * 10), + deploymentsChanged: make(chan interface{}, 5), + addSubscriber: make(chan chan deploymentsResult), + removeSubscriber: make(chan chan deploymentsResult), } testApiMan.InitAPI() time.Sleep(100 * time.Millisecond) @@ -114,6 +117,40 @@ }) + It("should get configs by filter", func() { + typeFilter := "ORGANIZATION" + // setup http client + uri, err := url.Parse(apiTestUrl) + Expect(err).Should(Succeed()) + uri.Path = deploymentsEndpoint + strconv.Itoa(testCount) + uri.RawQuery = "type=" + typeFilter + // set test data + dep := makeTestDeployment() + + dummyDbMan.configurations = make(map[string]*DataDeployment) + dummyDbMan.configurations[typeFilter] = dep + detail := makeExpectedDetail(dep, strings.Split(uri.String(), "?")[0]) + + // http get + res, err := http.Get(uri.String()) + Expect(err).Should(Succeed()) + defer res.Body.Close() + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + + // parse response + var depRes ApiDeploymentResponse + body, err := ioutil.ReadAll(res.Body) + Expect(err).Should(Succeed()) + err = json.Unmarshal(body, &depRes) + Expect(err).Should(Succeed()) + + // verify response + Expect(depRes.Kind).Should(Equal(kindCollection)) + Expect(depRes.Self).Should(Equal(uri.String())) + Expect(depRes.ApiDeploymentsResponse).Should(Equal([]ApiDeploymentDetails{*detail})) + + }) + It("should get 304 for no change", func() { // setup http client @@ -274,6 +311,91 @@ }) }) + Context("GET /configurations/{configId}", func() { + It("should get configuration according to {configId}", func() { + // setup http client + uri, err := url.Parse(apiTestUrl) + Expect(err).Should(Succeed()) + uri.Path = deploymentsEndpoint + strconv.Itoa(testCount) + "/3ecd351c-1173-40bf-b830-c194e5ef9038" + + //setup test data + dummyDbMan.err = nil + dummyDbMan.configurations = make(map[string]*DataDeployment) + expectedConfig := &DataDeployment{ + ID: "3ecd351c-1173-40bf-b830-c194e5ef9038", + OrgID: "73fcac6c-5d9f-44c1-8db0-333efda3e6e8", + EnvID: "ada76573-68e3-4f1a-a0f9-cbc201a97e80", + BlobID: "gcs:SHA-512:8fcc902465ccb32ceff25fa9f6fb28e3b314dbc2874c0f8add02f4e29c9e2798d344c51807aa1af56035cf09d39c800cf605d627ba65723f26d8b9c83c82d2f2", + BlobResourceID: "gcs:SHA-512:0c648779da035bfe0ac21f6268049aa0ae74d9d6411dadefaec33991e55c2d66c807e06f7ef84e0947f7c7d63b8c9e97cf0684cbef9e0a86b947d73c74ae7455", + Type: "ENVIRONMENT", + Name: "test", + Revision: "", + Path: "/organizations/Org1//environments/test/", + Created: "2017-06-27 03:14:46.018+00:00", + CreatedBy: "defaultUser", + Updated: "2017-06-27 03:14:46.018+00:00", + UpdatedBy: "defaultUser", + } + dummyDbMan.configurations[expectedConfig.ID] = expectedConfig + // http get + res, err := http.Get(uri.String()) + Expect(err).Should(Succeed()) + defer res.Body.Close() + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + + // parse response + var depRes ApiDeploymentDetails + body, err := ioutil.ReadAll(res.Body) + Expect(err).Should(Succeed()) + err = json.Unmarshal(body, &depRes) + Expect(err).Should(Succeed()) + + // verify response + Expect(depRes.Self).Should(ContainSubstring(expectedConfig.ID)) + Expect(depRes.Org).Should(Equal(expectedConfig.OrgID)) + Expect(depRes.Name).Should(Equal(expectedConfig.Name)) + Expect(depRes.Type).Should(Equal(expectedConfig.Type)) + Expect(depRes.Revision).Should(Equal(expectedConfig.Revision)) + Expect(depRes.BeanBlobUrl).Should(ContainSubstring(expectedConfig.BlobID)) + Expect(depRes.ResourceBlobUrl).Should(ContainSubstring(expectedConfig.BlobResourceID)) + Expect(depRes.Path).Should(Equal(expectedConfig.Path)) + Expect(depRes.Created).Should(Equal(convertTime(expectedConfig.Created))) + Expect(depRes.Updated).Should(Equal(convertTime(expectedConfig.Updated))) + }) + + It("should get error responses", func() { + // setup http client + uri, err := url.Parse(apiTestUrl) + Expect(err).Should(Succeed()) + + //setup test data + testData := [][]interface{}{ + {"invalid-uuid", nil}, + {GenerateUUID(), sql.ErrNoRows}, + {GenerateUUID(), fmt.Errorf("test error")}, + } + expectedCode := []int{ + http.StatusBadRequest, + http.StatusNotFound, + http.StatusInternalServerError, + } + + for i, data := range testData { + if data[1] != nil { + dummyDbMan.err = data[1].(error) + } + dummyDbMan.configurations = make(map[string]*DataDeployment) + dummyDbMan.configurations[data[0].(string)] = &DataDeployment{} + // http get + uri.Path = deploymentsEndpoint + strconv.Itoa(testCount) + "/" + data[0].(string) + res, err := http.Get(uri.String()) + Expect(err).Should(Succeed()) + Expect(res.StatusCode).Should(Equal(expectedCode[i])) + res.Body.Close() + } + }) + }) + }) func setTestDeployments(dummyDbMan *dummyDbManager, self string) []ApiDeploymentDetails { @@ -338,6 +460,8 @@ localFSLocation string fileResponse chan string version string + configurations map[string]*DataDeployment + err error } func (d *dummyDbManager) setDbVersion(version string) { @@ -352,8 +476,11 @@ return d.unreadyBlobIds, nil } -func (d *dummyDbManager) getReadyDeployments() ([]DataDeployment, error) { - return d.readyDeployments, nil +func (d *dummyDbManager) getReadyDeployments(typeFilter string) ([]DataDeployment, error) { + if typeFilter == "" { + return d.readyDeployments, nil + } + return []DataDeployment{*(d.configurations[typeFilter])}, nil } func (d *dummyDbManager) updateLocalFsLocation(blobId, localFsLocation string) error { @@ -374,6 +501,10 @@ return d.localFSLocation, nil } +func (d *dummyDbManager) getConfigById(id string) (*DataDeployment, error) { + return d.configurations[id], d.err +} + func GenerateUUID() string { buff := make([]byte, 16)
diff --git a/data.go b/data.go index 564ce6d..d424e2b 100644 --- a/data.go +++ b/data.go
@@ -49,9 +49,10 @@ setDbVersion(string) initDb() error getUnreadyBlobs() ([]string, error) - getReadyDeployments() ([]DataDeployment, error) + getReadyDeployments(typeFilter string) ([]DataDeployment, error) updateLocalFsLocation(string, string) error getLocalFSLocation(string) (string, error) + getConfigById(string) (*DataDeployment, error) } type dbManager struct { @@ -99,6 +100,31 @@ return nil } +func (dbc *dbManager) getConfigById(id string) (config *DataDeployment, err error) { + row := dbc.getDb().QueryRow(` + SELECT a.id, + a.organization_id, + a.environment_id, + a.bean_blob_id, + a.resource_blob_id, + a.type, + a.name, + a.revision, + a.path, + a.created_at, + a.created_by, + a.updated_at, + a.updated_by + FROM metadata_runtime_entity_metadata as a + WHERE a.id = ?; + `, id) + config, err = dataDeploymentsFromRow(row) + if err != nil { + return nil, err + } + return config, nil +} + // getUnreadyDeployments() returns array of resources that are not yet to be processed func (dbc *dbManager) getUnreadyBlobs() (ids []string, err error) { @@ -135,11 +161,15 @@ return } -func (dbc *dbManager) getReadyDeployments() ([]DataDeployment, error) { +func (dbc *dbManager) getReadyDeployments(typeFilter string) ([]DataDeployment, error) { // An alternative statement is in get_ready_deployments.sql // Need testing with large data volume to determine which is better - rows, err := dbc.getDb().Query(` + + var rows *sql.Rows + var err error + if typeFilter == "" { + rows, err = dbc.getDb().Query(` SELECT a.id, a.organization_id, a.environment_id, @@ -179,6 +209,49 @@ ) ; `) + } else { + rows, err = dbc.getDb().Query(` + SELECT a.id, + a.organization_id, + a.environment_id, + a.bean_blob_id, + a.resource_blob_id, + a.type, + a.name, + a.revision, + a.path, + a.created_at, + a.created_by, + a.updated_at, + a.updated_by + FROM metadata_runtime_entity_metadata as a + WHERE a.type = ? + AND a.id IN ( + SELECT + a.id + FROM metadata_runtime_entity_metadata as a + INNER JOIN apid_blob_available as b + ON a.resource_blob_id = b.id + WHERE a.resource_blob_id IS NOT NULL AND a.resource_blob_id != "" + INTERSECT + SELECT + a.id + FROM metadata_runtime_entity_metadata as a + INNER JOIN apid_blob_available as b + ON a.bean_blob_id = b.id + WHERE a.resource_blob_id IS NOT NULL AND a.resource_blob_id != "" + + UNION + SELECT + a.id + FROM metadata_runtime_entity_metadata as a + INNER JOIN apid_blob_available as b + ON a.bean_blob_id = b.id + WHERE a.resource_blob_id IS NULL OR a.resource_blob_id = "" + ) + ; + `, typeFilter) + } if err != nil { log.Errorf("DB Query for project_runtime_blob_metadata failed %v", err) @@ -186,7 +259,7 @@ } defer rows.Close() - deployments, err := dataDeploymentsFromRow(rows) + deployments, err := dataDeploymentsFromRows(rows) if err != nil { return nil, err } @@ -244,7 +317,7 @@ return } -func dataDeploymentsFromRow(rows *sql.Rows) ([]DataDeployment, error) { +func dataDeploymentsFromRows(rows *sql.Rows) ([]DataDeployment, error) { tmp, err := structFromRows(reflect.TypeOf((*DataDeployment)(nil)).Elem(), rows) if err != nil { return nil, err @@ -275,3 +348,35 @@ } return slice.Interface(), nil } + +func dataDeploymentsFromRow(row *sql.Row) (*DataDeployment, error) { + tmp, err := structFromRow(reflect.TypeOf((*DataDeployment)(nil)).Elem(), row) + if err != nil { + if err != sql.ErrNoRows { + log.Errorf("Error in dataDeploymentsFromRow: %v", err) + } + return nil, err + } + config := tmp.(DataDeployment) + return &config, nil +} + +func structFromRow(t reflect.Type, row *sql.Row) (interface{}, error) { + num := t.NumField() + cols := make([]interface{}, num) + for i := range cols { + cols[i] = new(sql.NullString) + } + v := reflect.New(t).Elem() + err := row.Scan(cols...) + if err != nil { + return nil, err + } + for i := range cols { + p := cols[i].(*sql.NullString) + if p.Valid { + v.Field(i).SetString(p.String) + } + } + return v.Interface(), nil +}
diff --git a/data_test.go b/data_test.go index abd9fc9..fbbb0a7 100644 --- a/data_test.go +++ b/data_test.go
@@ -19,6 +19,7 @@ "github.com/apid/apid-core/data" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "reflect" "strconv" "sync" "time" @@ -90,7 +91,7 @@ }) It("should get empty slice if no deployments are ready", func() { - deps, err := testDbMan.getReadyDeployments() + deps, err := testDbMan.getReadyDeployments("") Expect(err).Should(Succeed()) Expect(len(deps)).Should(BeZero()) }) @@ -123,14 +124,41 @@ Expect(location).Should(Equal(testBlobLocalFsPrefix + testBlobId)) }) - It("should succefully get all ready deployments", func() { + It("should get configuration by Id", func() { + config, err := testDbMan.getConfigById("3ecd351c-1173-40bf-b830-c194e5ef9038") + Expect(err).Should(Succeed()) + expectedResponse := &DataDeployment{ + ID: "3ecd351c-1173-40bf-b830-c194e5ef9038", + OrgID: "73fcac6c-5d9f-44c1-8db0-333efda3e6e8", + EnvID: "ada76573-68e3-4f1a-a0f9-cbc201a97e80", + BlobID: "gcs:SHA-512:8fcc902465ccb32ceff25fa9f6fb28e3b314dbc2874c0f8add02f4e29c9e2798d344c51807aa1af56035cf09d39c800cf605d627ba65723f26d8b9c83c82d2f2", + BlobResourceID: "gcs:SHA-512:0c648779da035bfe0ac21f6268049aa0ae74d9d6411dadefaec33991e55c2d66c807e06f7ef84e0947f7c7d63b8c9e97cf0684cbef9e0a86b947d73c74ae7455", + Type: "ENVIRONMENT", + Name: "test", + Revision: "", + Path: "/organizations/Org1//environments/test/", + Created: "2017-06-27 03:14:46.018+00:00", + CreatedBy: "defaultUser", + Updated: "2017-06-27 03:14:46.018+00:00", + UpdatedBy: "defaultUser", + } + Expect(config).ShouldNot(BeNil()) + Expect(reflect.DeepEqual(expectedResponse, config)).Should(BeTrue()) + }) + + It("should get non-nil error for nonexistent Id", func() { + _, err := testDbMan.getConfigById("3ecd351c-aaaa-40bf-b830-c194e5ef9038") + Expect(err).ShouldNot(Succeed()) + }) + + It("should successfully get all ready configurations", func() { err := testDbMan.updateLocalFsLocation(readyBlobId, testBlobLocalFsPrefix+readyBlobId) Expect(err).Should(Succeed()) err = testDbMan.updateLocalFsLocation(readyResourceId, testBlobLocalFsPrefix+readyResourceId) Expect(err).Should(Succeed()) - deps, err := testDbMan.getReadyDeployments() + deps, err := testDbMan.getReadyDeployments("") Expect(err).Should(Succeed()) Expect(len(deps)).Should(Equal(2)) for _, dep := range deps { @@ -141,6 +169,28 @@ } }) + It("should get ready configurations by type filter", func() { + + err := testDbMan.updateLocalFsLocation(readyBlobId, testBlobLocalFsPrefix+readyBlobId) + Expect(err).Should(Succeed()) + err = testDbMan.updateLocalFsLocation(readyResourceId, testBlobLocalFsPrefix+readyResourceId) + Expect(err).Should(Succeed()) + + deps, err := testDbMan.getReadyDeployments("ORGANIZATION") + Expect(err).Should(Succeed()) + Expect(len(deps)).Should(Equal(1)) + Expect(deps[0].ID).Should(Equal("319963ff-217e-4ecc-8d6e-c3665e962d1e")) + + deps, err = testDbMan.getReadyDeployments("ENVIRONMENT") + Expect(err).Should(Succeed()) + Expect(len(deps)).Should(Equal(1)) + Expect(deps[0].ID).Should(Equal("1dc4895e-6494-4b59-979f-5f4c89c073b4")) + + deps, err = testDbMan.getReadyDeployments("INVALID-TYPE") + Expect(err).Should(Succeed()) + Expect(len(deps)).Should(Equal(0)) + }) + It("should succefully get all unready blob ids", func() { err := testDbMan.updateLocalFsLocation(readyBlobId, testBlobLocalFsPrefix+readyBlobId) @@ -193,10 +243,10 @@ '', 'gcs:SHA-512:39ca7ae89bb9468af34df8bc873748b4035210c91bcc01359c092c1d51364b5f3df06bc69a40621acfaa46791af9ea41bc0f3429a84738ba1a7c8d394859601a', NULL, - 'ORGANIZATION', + 'ENVIRONMENT', 'Org1', '', - '/organizations/Org1/', + '/organizations/edgex01//environments/prod/', '2017-06-27 03:14:45.748+00:00', 'defaultUser', '2017-06-27 03:15:03.557+00:00',
diff --git a/init.go b/init.go index 037e5ea..bf85583 100644 --- a/init.go +++ b/init.go
@@ -133,14 +133,15 @@ // initialize api manager apiMan := &apiManager{ - dbMan: dbMan, - deploymentsEndpoint: deploymentsEndpoint, - blobEndpoint: blobEndpoint, - eTag: 0, - deploymentsChanged: make(chan interface{}, 5), - addSubscriber: make(chan chan deploymentsResult), - removeSubscriber: make(chan chan deploymentsResult), - apiInitialized: false, + dbMan: dbMan, + deploymentsEndpoint: deploymentsEndpoint, + blobEndpoint: blobEndpoint, + deploymentIdEndpoint: deploymentIdEndpoint, + eTag: 0, + deploymentsChanged: make(chan interface{}, 5), + addSubscriber: make(chan chan deploymentsResult), + removeSubscriber: make(chan chan deploymentsResult), + apiInitialized: false, } // initialize bundle manager