add map to store columns
diff --git a/apigee_sync_test.go b/apigee_sync_test.go index d81ec32..b885593 100644 --- a/apigee_sync_test.go +++ b/apigee_sync_test.go
@@ -84,7 +84,7 @@ apid.Events().ListenFunc(ApigeeSyncEventSelector, func(event apid.Event) { if s, ok := event.(*common.Snapshot); ok { - Expect(16).To(Equal(len(knownTables))) + Expect(12).To(Equal(len(knownTables))) Expect(changesRequireDDLSync(expectedSnapshotTables)).To(BeFalse()) lastSnapshot = s
diff --git a/changes.go b/changes.go index 993613f..9f26466 100644 --- a/changes.go +++ b/changes.go
@@ -317,6 +317,27 @@ } /* + * Determine if any columns added/dropped in any table + */ +func changesHavecolumnsChanged(a map[string]bool, changes []common.Change) bool { + + //nil maps should not be passed in. Making the distinction between nil map and empty map + if a == nil { + log.Warn("Nil map passed to function changesHaveNewTables, may be bug") + return true + } + + for _, change := range changes { + if !a[normalizeTableName(change.Table)] { + log.Infof("Unable to find %s table in current known tables", change.Table) + return true + } + } + + return false +} + +/* * seqCurr.Compare() will return 1, if its newer than seqPrev, * else will return 0, if same, or -1 if older. */
diff --git a/snapshot.go b/snapshot.go index 960b4fc..ae889e3 100644 --- a/snapshot.go +++ b/snapshot.go
@@ -201,6 +201,46 @@ } +func extractTableColumnsFromSnapshot(snapshot *common.Snapshot) (map[string][]string) { + + columns := make(map[string][]string) + tables := make([]string, 0) + + log.Debug("Extracting table names from snapshot") + db, err := dataService.DBVersion(snapshot.SnapshotInfo) + if err != nil { + log.Panicf("Database inaccessible: %v", err) + } + rows, err := db.Query("SELECT DISTINCT tableName FROM _transicator_tables;") + if err != nil { + log.Panicf("Unable to read in known snapshot tables from sqlite file") + } + defer rows.Close() + for rows.Next() { + var tableName string + rows.Scan(&tableName) + if err != nil { + log.Panic("Error scaning tableNames from _transicator_tables") + } + tables = append(tables, tableName) + } + + for _, tableName := range tables { + + dummyRows, err := db.Query("SELECT * FROM " + tableName + " LIMIT 0;") + if err != nil { + log.Panicf("Get table info failed: %v", err) + } + defer dummyRows.Close() + cols, err := dummyRows.Columns() + if err != nil { + log.Panicf("Get table columns failed: %v", err) + } + columns[tableName] = cols + } + return columns +} + func extractTablesFromDB(db apid.DB) (tables map[string]bool) { tables = make(map[string]bool)
diff --git a/snapshot_test.go b/snapshot_test.go new file mode 100644 index 0000000..ffad61a --- /dev/null +++ b/snapshot_test.go
@@ -0,0 +1,61 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package apidApigeeSync +import ( + "github.com/apigee-labs/transicator/common" + . "github.com/onsi/ginkgo" + "os" + "strings" +) + +var _ = Describe("Change Agent", func() { + + const testDbId = "test_snapshot" + + Context("Change Agent Unit Tests", func() { + testHandler := handler{} + + var createTestDb = func(sqlfile string, dbId string) common.Snapshot { + initDb(sqlfile, "./mockdb_snapshot.sqlite3") + file, err := os.Open("./mockdb_snapshot.sqlite3") + if err != nil { + Fail("Failed to open mock db for test") + } + + s := common.Snapshot{} + err = processSnapshotServerFileResponse(dbId, file, &s) + if err != nil { + Fail("Error processing test snapshots") + } + return s + } + + BeforeEach(func() { + event := createTestDb("./sql/init_mock_db.sql", testDbId) + testHandler.Handle(&event) + knownTables = extractTablesFromDB(getDB()) + }) + + It("test extract table columns", func() { + s := &common.Snapshot{ + SnapshotInfo: testDbId, + } + columns := extractTableColumnsFromSnapshot(s) + for table, cols := range columns { + log.Error("snapshot TABLE: " + table + " COLUMN: " + strings.Join(cols, "|")) + } + }) + + }) +}) \ No newline at end of file
diff --git a/sql/init_mock_db.sql b/sql/init_mock_db.sql index f1b8471..4df6654 100644 --- a/sql/init_mock_db.sql +++ b/sql/init_mock_db.sql
@@ -22,17 +22,17 @@ columnName varchar not null, typid integer, primaryKey bool); -INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','id',1043,1); -INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','data_scope_id',1043,1); -INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','name',25,0); -INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','uri',25,0); -INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','checksumtype',25,0); -INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','checksum',25,0); -INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','created',1114,0); -INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','created_by',25,0); -INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','updated',1114,0); -INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','updated_by',25,0); -INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','crc',25,0); +-- INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','id',1043,1); +-- INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','data_scope_id',1043,1); +-- INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','name',25,0); +-- INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','uri',25,0); +-- INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','checksumtype',25,0); +-- INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','checksum',25,0); +-- INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','created',1114,0); +-- INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','created_by',25,0); +-- INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','updated',1114,0); +-- INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','updated_by',25,0); +-- INSERT INTO "_transicator_tables" VALUES('kms_bundle_config','crc',25,0); INSERT INTO "_transicator_tables" VALUES('kms_deployment','id',1043,1); INSERT INTO "_transicator_tables" VALUES('kms_deployment','bundle_config_id',1043,1); INSERT INTO "_transicator_tables" VALUES('kms_deployment','apid_cluster_id',1043,1); @@ -107,21 +107,6 @@ INSERT INTO "_transicator_tables" VALUES('edgex_apid_cluster','updated',1114,0); INSERT INTO "_transicator_tables" VALUES('edgex_apid_cluster','updated_by',25,0); INSERT INTO "_transicator_tables" VALUES('edgex_apid_cluster','_change_selector',25,1); -INSERT INTO "_transicator_tables" VALUES('kms_deployment_history','id',2950,1); -INSERT INTO "_transicator_tables" VALUES('kms_deployment_history','name',1043,0); -INSERT INTO "_transicator_tables" VALUES('kms_deployment_history','ext_ref_id',1043,1); -INSERT INTO "_transicator_tables" VALUES('kms_deployment_history','display_name',1043,0); -INSERT INTO "_transicator_tables" VALUES('kms_deployment_history','description',1043,0); -INSERT INTO "_transicator_tables" VALUES('kms_deployment_history','created_at',1114,1); -INSERT INTO "_transicator_tables" VALUES('kms_deployment_history','created_by',1043,0); -INSERT INTO "_transicator_tables" VALUES('kms_deployment_history','updated_at',1114,1); -INSERT INTO "_transicator_tables" VALUES('kms_deployment_history','updated_by',1043,0); -INSERT INTO "_transicator_tables" VALUES('configuration','id',1043,1); -INSERT INTO "_transicator_tables" VALUES('configuration','body',25,0); -INSERT INTO "_transicator_tables" VALUES('configuration','created',1114,0); -INSERT INTO "_transicator_tables" VALUES('configuration','created_by',25,0); -INSERT INTO "_transicator_tables" VALUES('configuration','updated',1114,0); -INSERT INTO "_transicator_tables" VALUES('configuration','updated_by',25,0); INSERT INTO "_transicator_tables" VALUES('edgex_data_scope','id',1043,1); INSERT INTO "_transicator_tables" VALUES('edgex_data_scope','apid_cluster_id',1043,1); INSERT INTO "_transicator_tables" VALUES('edgex_data_scope','scope',25,0); @@ -156,18 +141,6 @@ INSERT INTO "_transicator_tables" VALUES('kms_company_developer','updated_at',1114,0); INSERT INTO "_transicator_tables" VALUES('kms_company_developer','updated_by',1043,0); INSERT INTO "_transicator_tables" VALUES('kms_company_developer','_change_selector',1043,0); -INSERT INTO "_transicator_tables" VALUES('edgex_deployment_history','id',1043,1); -INSERT INTO "_transicator_tables" VALUES('edgex_deployment_history','deployment_id',1043,0); -INSERT INTO "_transicator_tables" VALUES('edgex_deployment_history','action',25,0); -INSERT INTO "_transicator_tables" VALUES('edgex_deployment_history','bundle_config_id',1043,0); -INSERT INTO "_transicator_tables" VALUES('edgex_deployment_history','apid_cluster_id',1043,0); -INSERT INTO "_transicator_tables" VALUES('edgex_deployment_history','data_scope_id',1043,0); -INSERT INTO "_transicator_tables" VALUES('edgex_deployment_history','bundle_config_json',25,0); -INSERT INTO "_transicator_tables" VALUES('edgex_deployment_history','config_json',25,0); -INSERT INTO "_transicator_tables" VALUES('edgex_deployment_history','created',1114,0); -INSERT INTO "_transicator_tables" VALUES('edgex_deployment_history','created_by',25,0); -INSERT INTO "_transicator_tables" VALUES('edgex_deployment_history','updated',1114,0); -INSERT INTO "_transicator_tables" VALUES('edgex_deployment_history','updated_by',25,0); INSERT INTO "_transicator_tables" VALUES('kms_app','id',2950,1); INSERT INTO "_transicator_tables" VALUES('kms_app','tenant_id',1043,1); INSERT INTO "_transicator_tables" VALUES('kms_app','name',1043,1);