[XAPID-1074] add tests for offline mode, minor refactor
diff --git a/apigeeSync_suite_test.go b/apigeeSync_suite_test.go index f754843..a9370c2 100644 --- a/apigeeSync_suite_test.go +++ b/apigeeSync_suite_test.go
@@ -44,10 +44,12 @@ wipeDBAferTest = true }) -var _ = BeforeEach(func(done Done) { +var _ = BeforeEach(func() { apid.Initialize(factory.DefaultServicesFactory()) config = apid.Config() + dataService = apid.Data() + events = apid.Events() var err error tmpDir, err = ioutil.TempDir("", "api_test") @@ -66,25 +68,12 @@ config.Set(configConsumerSecret, "YYYYYYY") block = "0" - log = apid.Log() - - _initPlugin(apid.AllServices()) - createManagers() - close(done) + log = apid.Log().ForModule("apigeeSync") }, 3) var _ = AfterEach(func() { apid.Events().Close() - lastSequence = "" - - if wipeDBAferTest { - db, err := dataService.DB() - Expect(err).NotTo(HaveOccurred()) - _, err = db.Exec("DELETE FROM APID") - Expect(err).NotTo(HaveOccurred()) - } - wipeDBAferTest = true }) var _ = AfterSuite(func() {
diff --git a/apigee_sync.go b/apigee_sync.go index a78b0ab..895447c 100644 --- a/apigee_sync.go +++ b/apigee_sync.go
@@ -41,8 +41,7 @@ } if apidInfo.LastSnapshot != "" { - snapshot := startOnLocalSnapshot(apidInfo.LastSnapshot) - processSnapshot(snapshot) + snapshot := apidSnapshotManager.startOnLocalSnapshot(apidInfo.LastSnapshot) events.EmitWithCallback(ApigeeSyncEventSelector, snapshot, func(event apid.Event) { apidChangeManager.pollChangeWithBackoff() })
diff --git a/apigee_sync_test.go b/apigee_sync_test.go index d81ec32..a3d4036 100644 --- a/apigee_sync_test.go +++ b/apigee_sync_test.go
@@ -24,8 +24,87 @@ ) var _ = Describe("Sync", func() { + Context("offline mode", func() { + var ( + testInstanceID = GenerateUUID() + testInstanceName = "offline-instance-name" + testClusterID = "offline-cluster-id" + testLastSnapshot = "offline-last-snapshot" + testChangeMan *dummyChangeManager + ) - Context("Sync", func() { + var _ = BeforeEach(func() { + config.Set(configOnlineMode, offlineMode) + config.Set(configApidClusterId, testClusterID) + _initPlugin(apid.AllServices()) + apidSnapshotManager = &dummySnapshotManager{} + testChangeMan = &dummyChangeManager{ + pollChangeWithBackoffChan: make(chan bool, 1), + } + apidChangeManager = testChangeMan + apidTokenManager = &dummyTokenManager{} + apidInfo = apidInstanceInfo{ + InstanceID: testInstanceID, + InstanceName: testInstanceName, + ClusterID: testClusterID, + LastSnapshot: testLastSnapshot, + } + updateApidInstanceInfo() + + }) + + var _ = AfterEach(func() { + config.Set(configOnlineMode, "") + if wipeDBAferTest { + db, err := dataService.DB() + Expect(err).NotTo(HaveOccurred()) + _, err = db.Exec("DELETE FROM APID") + Expect(err).NotTo(HaveOccurred()) + } + wipeDBAferTest = true + newInstanceID = true + isOfflineMode = false + }) + + It("offline mode should bootstrap from local DB", func(done Done) { + + Expect(apidInfo.LastSnapshot).NotTo(BeEmpty()) + + apid.Events().ListenFunc(ApigeeSyncEventSelector, func(event apid.Event) { + + if s, ok := event.(*common.Snapshot); ok { + // In this test, the changeManager.pollChangeWithBackoff() has not been launched when changeManager closed + // This is because the changeManager.pollChangeWithBackoff() in bootstrap() happened after this handler + Expect(s.SnapshotInfo).Should(Equal(testLastSnapshot)) + Expect(s.Tables).To(BeNil()) + close(done) + } + }) + pie := apid.PluginsInitializedEvent{ + Description: "plugins initialized", + } + pie.Plugins = append(pie.Plugins, pluginData) + postInitPlugins(pie) + + }, 3) + + }) + + Context("online mode", func() { + var _ = BeforeEach(func() { + _initPlugin(apid.AllServices()) + createManagers() + }) + + var _ = AfterEach(func() { + if wipeDBAferTest { + db, err := dataService.DB() + Expect(err).NotTo(HaveOccurred()) + _, err = db.Exec("DELETE FROM APID") + Expect(err).NotTo(HaveOccurred()) + } + wipeDBAferTest = true + }) const expectedDataScopeId1 = "dataScope1" const expectedDataScopeId2 = "dataScope2"
diff --git a/change_test.go b/change_test.go index 67331c4..617129d 100644 --- a/change_test.go +++ b/change_test.go
@@ -18,6 +18,7 @@ "github.com/30x/apid-core" "github.com/apigee-labs/transicator/common" . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" "net/http/httptest" "net/url" "os" @@ -31,24 +32,13 @@ var createTestDb = func(sqlfile string, dbId string) common.Snapshot { initDb(sqlfile, "./mockdb_change.sqlite3") file, err := os.Open("./mockdb_change.sqlite3") - if err != nil { - Fail("Failed to open mock db for test") - } - + Expect(err).Should(Succeed()) s := common.Snapshot{} err = processSnapshotServerFileResponse(dbId, file, &s) - if err != nil { - Fail("Error processing test snapshots") - } + Expect(err).Should(Succeed()) return s } - BeforeEach(func() { - event := createTestDb("./sql/init_mock_db.sql", "test_change") - processSnapshot(&event) - knownTables = extractTablesFromDB(getDB()) - }) - var initializeContext = func() { testRouter = apid.API().Router() testServer = httptest.NewServer(testRouter) @@ -80,8 +70,23 @@ config.Set(configPollInterval, 10*time.Millisecond) } - AfterEach(func() { + var _ = BeforeEach(func() { + _initPlugin(apid.AllServices()) + createManagers() + event := createTestDb("./sql/init_mock_db.sql", "test_change") + processSnapshot(&event) + knownTables = extractTablesFromDB(getDB()) + }) + + var _ = AfterEach(func() { restoreContext() + if wipeDBAferTest { + db, err := dataService.DB() + Expect(err).Should(Succeed()) + _, err = db.Exec("DELETE FROM APID") + Expect(err).Should(Succeed()) + } + wipeDBAferTest = true }) It("test change agent with authorization failure", func() { @@ -218,3 +223,9 @@ func (s *dummySnapshotManager) downloadSnapshot(isBoot bool, scopes []string, snapshot *common.Snapshot) error { return nil } + +func (s *dummySnapshotManager) startOnLocalSnapshot(snapshot string) *common.Snapshot { + return &common.Snapshot{ + SnapshotInfo: snapshot, + } +}
diff --git a/data_test.go b/data_test.go index 11a4a6c..438a1d4 100644 --- a/data_test.go +++ b/data_test.go
@@ -15,45 +15,52 @@ package apidApigeeSync import ( + "github.com/30x/apid-core/data" "github.com/apigee-labs/transicator/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "sort" + "strconv" ) var _ = Describe("data access tests", func() { + testCount := 1 - BeforeEach(func() { - db := getDB() - + var _ = BeforeEach(func() { + testCount++ + db, err := dataService.DBVersion("data_test_" + strconv.Itoa(testCount)) + Expect(err).Should(Succeed()) + initDB(db) //all tests in this file operate on the api_product table. Create the necessary tables for this here - getDB().Exec("CREATE TABLE _transicator_tables " + + db.Exec("CREATE TABLE _transicator_tables " + "(tableName varchar not null, columnName varchar not null, " + "typid integer, primaryKey bool);") - getDB().Exec("DELETE from _transicator_tables") - getDB().Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','id',2950,1)") - getDB().Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','tenant_id',1043,1)") - getDB().Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','description',1043,0)") - getDB().Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','api_resources',1015,0)") - getDB().Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','approval_type',1043,0)") - getDB().Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','scopes',1015,0)") - getDB().Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','proxies',1015,0)") - getDB().Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','environments',1015,0)") - getDB().Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','created_at',1114,1)") - getDB().Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','created_by',1043,0)") - getDB().Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','updated_at',1114,1)") - getDB().Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','updated_by',1043,0)") - getDB().Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','_change_selector',1043,0)") + db.Exec("DELETE from _transicator_tables") + db.Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','id',2950,1)") + db.Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','tenant_id',1043,1)") + db.Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','description',1043,0)") + db.Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','api_resources',1015,0)") + db.Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','approval_type',1043,0)") + db.Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','scopes',1015,0)") + db.Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','proxies',1015,0)") + db.Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','environments',1015,0)") + db.Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','created_at',1114,1)") + db.Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','created_by',1043,0)") + db.Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','updated_at',1114,1)") + db.Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','updated_by',1043,0)") + db.Exec("INSERT INTO _transicator_tables VALUES('kms_api_product','_change_selector',1043,0)") - getDB().Exec("CREATE TABLE kms_api_product (id text,tenant_id text,name text, description text, " + + db.Exec("CREATE TABLE kms_api_product (id text,tenant_id text,name text, description text, " + "api_resources text,approval_type text,scopes text,proxies text, environments text," + "created_at blob, created_by text,updated_at blob,updated_by text,_change_selector text, " + "primary key (id,tenant_id,created_at,updated_at));") - getDB().Exec("DELETE from kms_api_product") + db.Exec("DELETE from kms_api_product") setDB(db) - initDB(db) + }) + var _ = AfterEach(func() { + data.Delete(data.VersionedDBID("common", "data_test_"+strconv.Itoa(testCount))) }) Context("Update processing", func() {
diff --git a/init.go b/init.go index 6b1be34..eb25a95 100644 --- a/init.go +++ b/init.go
@@ -96,9 +96,7 @@ log.Debugf("Using %s as display name", config.GetString(configName)) } -func initVariables(services apid.Services) error { - dataService = services.Data() - events = services.Events() +func initVariables() error { tr := &http.Transport{ MaxIdleConnsPerHost: maxIdleConnsPerHost, @@ -173,13 +171,13 @@ /* initialization */ func _initPlugin(services apid.Services) error { - SetLogger(services.Log().ForModule("apigeeSync")) log.Debug("start init") config = services.Config() initConfigDefaults() if strings.EqualFold(config.GetString(configOnlineMode), offlineMode) { + log.Warn("offline mode!") isOfflineMode = true } @@ -188,7 +186,7 @@ return err } - err = initVariables(services) + err = initVariables() if err != nil { return err } @@ -197,6 +195,9 @@ } func initPlugin(services apid.Services) (apid.PluginData, error) { + SetLogger(services.Log().ForModule("apigeeSync")) + dataService = services.Data() + events = services.Events() err := _initPlugin(services) if err != nil {
diff --git a/init_test.go b/init_test.go index dcdf89e..1f4ca21 100644 --- a/init_test.go +++ b/init_test.go
@@ -15,11 +15,15 @@ package apidApigeeSync import ( + "github.com/30x/apid-core" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("init", func() { + var _ = BeforeEach(func() { + _initPlugin(apid.AllServices()) + }) Context("Apid Instance display name", func() {
diff --git a/listener_test.go b/listener_test.go index d48c70e..13ea0bf 100644 --- a/listener_test.go +++ b/listener_test.go
@@ -18,6 +18,7 @@ . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/30x/apid-core" "github.com/apigee-labs/transicator/common" "os" "reflect" @@ -37,6 +38,20 @@ return s } + var _ = BeforeEach(func() { + _initPlugin(apid.AllServices()) + }) + + var _ = AfterEach(func() { + if wipeDBAferTest { + db, err := dataService.DB() + Expect(err).Should(Succeed()) + _, err = db.Exec("DELETE FROM APID") + Expect(err).Should(Succeed()) + } + wipeDBAferTest = true + }) + Context("ApigeeSync snapshot event", func() { It("should fail if more than one apid_cluster rows", func() {
diff --git a/managerInterfaces.go b/managerInterfaces.go index 8ef331c..a93e950 100644 --- a/managerInterfaces.go +++ b/managerInterfaces.go
@@ -36,6 +36,7 @@ downloadDataSnapshot() storeDataSnapshot(snapshot *common.Snapshot) downloadSnapshot(isBoot bool, scopes []string, snapshot *common.Snapshot) error + startOnLocalSnapshot(snapshot string) *common.Snapshot } type changeManager interface {
diff --git a/snapshot.go b/snapshot.go index 90c03a2..ebcc901 100644 --- a/snapshot.go +++ b/snapshot.go
@@ -205,22 +205,24 @@ } // Skip Downloading snapshot if there is already a snapshot available from previous run -func startOnLocalSnapshot(snapshot string) *common.Snapshot { - log.Infof("Starting on local snapshot: %s", snapshot) +func (s *simpleSnapShotManager) startOnLocalSnapshot(snapshotName string) *common.Snapshot { + log.Infof("Starting on local snapshot: %s", snapshotName) // ensure DB version will be accessible on behalf of dependant plugins - db, err := dataService.DBVersion(snapshot) + db, err := dataService.DBVersion(snapshotName) if err != nil { log.Panicf("Database inaccessible: %v", err) } knownTables = extractTablesFromDB(db) + snapshot := &common.Snapshot{ + SnapshotInfo: snapshotName, + } + processSnapshot(snapshot) // allow plugins (including this one) to start immediately on existing database // Note: this MUST have no tables as that is used as an indicator - return &common.Snapshot{ - SnapshotInfo: snapshot, - } + return snapshot } // a blocking method @@ -364,3 +366,22 @@ func (o *offlineSnapshotManager) downloadSnapshot(isBoot bool, scopes []string, snapshot *common.Snapshot) error { return nil } +func (o *offlineSnapshotManager) startOnLocalSnapshot(snapshotName string) *common.Snapshot { + log.Infof("Starting on local snapshot: %s", snapshotName) + + // ensure DB version will be accessible on behalf of dependant plugins + db, err := dataService.DBVersion(snapshotName) + if err != nil { + log.Panicf("Database inaccessible: %v", err) + } + + knownTables = extractTablesFromDB(db) + snapshot := &common.Snapshot{ + SnapshotInfo: snapshotName, + } + processSnapshot(snapshot) + + // allow plugins (including this one) to start immediately on existing database + // Note: this MUST have no tables as that is used as an indicator + return snapshot +}
diff --git a/test_mock_test.go b/test_mock_test.go new file mode 100644 index 0000000..e62d0a5 --- /dev/null +++ b/test_mock_test.go
@@ -0,0 +1,28 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package apidApigeeSync + +type dummyChangeManager struct { + pollChangeWithBackoffChan chan bool +} + +func (d *dummyChangeManager) close() <-chan bool { + c := make(chan bool, 1) + c <- true + return c +} + +func (d *dummyChangeManager) pollChangeWithBackoff() { + d.pollChangeWithBackoffChan <- true +}
diff --git a/token.go b/token.go index 46e52cd..4c344e9 100644 --- a/token.go +++ b/token.go
@@ -169,6 +169,7 @@ if newInstanceID { req.Header.Set("created_at_apid", time.Now().Format(time.RFC3339)) + newInstanceID = false } else { req.Header.Set("updated_at_apid", time.Now().Format(time.RFC3339)) } @@ -208,15 +209,17 @@ log.Debugf("Got new token: %#v", token) - if newInstanceID { - newInstanceID = false - err = updateApidInstanceInfo() - if err != nil { - log.Errorf("unable to unmarshal update apid instance info : %v", string(body), err) - return err + /* + if newInstanceID { + newInstanceID = false + err = updateApidInstanceInfo() + if err != nil { + log.Errorf("unable to unmarshal update apid instance info : %v", string(body), err) + return err + } } - } + */ t.token = &token config.Set(configBearerToken, token.AccessToken)
diff --git a/token_test.go b/token_test.go index 85b71c7..a3c7d51 100644 --- a/token_test.go +++ b/token_test.go
@@ -185,11 +185,9 @@ count++ if count == 1 { - Expect(newInstanceID).To(BeTrue()) Expect(r.Header.Get("created_at_apid")).NotTo(BeEmpty()) Expect(r.Header.Get("updated_at_apid")).To(BeEmpty()) } else { - Expect(newInstanceID).To(BeFalse()) Expect(r.Header.Get("created_at_apid")).To(BeEmpty()) Expect(r.Header.Get("updated_at_apid")).NotTo(BeEmpty()) finished <- true