add column maps
diff --git a/apigee_sync.go b/apigee_sync.go index 10745de..1f6e38a 100644 --- a/apigee_sync.go +++ b/apigee_sync.go
@@ -26,7 +26,7 @@ maxIdleConnsPerHost = 10 ) -var knownTables = make(map[string]bool) +var knownTables = make(map[string]map[string]bool) /* * Start from existing snapshot if possible
diff --git a/apigee_sync_test.go b/apigee_sync_test.go index b885593..ad79a6b 100644 --- a/apigee_sync_test.go +++ b/apigee_sync_test.go
@@ -235,38 +235,39 @@ It("should correctly identify non-proper subsets with respect to maps", func() { + testMap := map[string]map[string]bool{"a": make(map[string]bool), "b": make(map[string]bool)} //test b proper subset of a - Expect(changesHaveNewTables(map[string]bool{"a": true, "b": true}, + Expect(changesHaveNewTables(testMap, []common.Change{common.Change{Table: "b"}}, )).To(BeFalse()) //test a == b - Expect(changesHaveNewTables(map[string]bool{"a": true, "b": true}, + Expect(changesHaveNewTables(testMap, []common.Change{common.Change{Table: "a"}, common.Change{Table: "b"}}, )).To(BeFalse()) //test b superset of a - Expect(changesHaveNewTables(map[string]bool{"a": true, "b": true}, + Expect(changesHaveNewTables(testMap, []common.Change{common.Change{Table: "a"}, common.Change{Table: "b"}, common.Change{Table: "c"}}, )).To(BeTrue()) //test b not subset of a - Expect(changesHaveNewTables(map[string]bool{"a": true, "b": true}, + Expect(changesHaveNewTables(testMap, []common.Change{common.Change{Table: "c"}}, )).To(BeTrue()) //test a empty - Expect(changesHaveNewTables(map[string]bool{}, + Expect(changesHaveNewTables(map[string]map[string]bool{}, []common.Change{common.Change{Table: "a"}}, )).To(BeTrue()) //test b empty - Expect(changesHaveNewTables(map[string]bool{"a": true, "b": true}, + Expect(changesHaveNewTables(testMap, []common.Change{}, )).To(BeFalse()) //test b nil - Expect(changesHaveNewTables(map[string]bool{"a": true, "b": true}, nil)).To(BeFalse()) + Expect(changesHaveNewTables(testMap, nil)).To(BeFalse()) //test a nil Expect(changesHaveNewTables(nil,
diff --git a/change_test.go b/change_test.go index 383bff6..ec9b82f 100644 --- a/change_test.go +++ b/change_test.go
@@ -46,7 +46,7 @@ BeforeEach(func() { event := createTestDb("./sql/init_mock_db.sql", "test_change") processSnapshot(&event) - knownTables = extractTablesFromDB(getDB()) + knownTables = extractTableColsFromDB(getDB()) }) var initializeContext = func() {
diff --git a/changes.go b/changes.go index 9f26466..9141927 100644 --- a/changes.go +++ b/changes.go
@@ -298,7 +298,7 @@ /* * Determine if any tables in changes are not present in known tables */ -func changesHaveNewTables(a map[string]bool, changes []common.Change) bool { +func changesHaveNewTables(a map[string]map[string]bool, changes []common.Change) bool { //nil maps should not be passed in. Making the distinction between nil map and empty map if a == nil { @@ -307,7 +307,7 @@ } for _, change := range changes { - if !a[normalizeTableName(change.Table)] { + if _, ok := a[normalizeTableName(change.Table)]; !ok { log.Infof("Unable to find %s table in current known tables", change.Table) return true }
diff --git a/snapshot.go b/snapshot.go index ae889e3..66108eb 100644 --- a/snapshot.go +++ b/snapshot.go
@@ -147,7 +147,7 @@ } func (s *simpleSnapShotManager) storeDataSnapshot(snapshot *common.Snapshot) { - knownTables = extractTablesFromSnapshot(snapshot) + knownTables = extractTableColumnsFromSnapshot(snapshot) _, err := dataService.DBVersion(snapshot.SnapshotInfo) if err != nil { @@ -167,50 +167,20 @@ } -func extractTablesFromSnapshot(snapshot *common.Snapshot) (tables map[string]bool) { - - tables = make(map[string]bool) - - log.Debug("Extracting table names from snapshot") - if snapshot.Tables == nil { - //if this panic ever fires, it's a bug - db, err := dataService.DBVersion(snapshot.SnapshotInfo) - if err != nil { - log.Panicf("Database inaccessible: %v", err) - } - rows, err := db.Query("SELECT DISTINCT tableName FROM _transicator_tables;") - if err != nil { - log.Panicf("Unable to read in known snapshot tables from sqlite file") - } - for rows.Next() { - var tableName string - rows.Scan(&tableName) - if err != nil { - log.Panic("Error scaning tableNames from _transicator_tables") - } - tables[tableName] = true - } - - } else { - - for _, table := range snapshot.Tables { - tables[table.Name] = true - } - } - return tables - -} - -func extractTableColumnsFromSnapshot(snapshot *common.Snapshot) (map[string][]string) { - - columns := make(map[string][]string) - tables := make([]string, 0) - +func extractTableColumnsFromSnapshot(snapshot *common.Snapshot) map[string]map[string]bool { log.Debug("Extracting table names from snapshot") db, err := dataService.DBVersion(snapshot.SnapshotInfo) if err != nil { log.Panicf("Database inaccessible: %v", err) } + + return extractTableColsFromDB(db) +} + +func extractTableColsFromDB(db apid.DB) map[string]map[string]bool { + + columns := make(map[string]map[string]bool) + tables := make([]string, 0) rows, err := db.Query("SELECT DISTINCT tableName FROM _transicator_tables;") if err != nil { log.Panicf("Unable to read in known snapshot tables from sqlite file") @@ -226,7 +196,7 @@ } for _, tableName := range tables { - + columns[tableName] = make(map[string]bool) dummyRows, err := db.Query("SELECT * FROM " + tableName + " LIMIT 0;") if err != nil { log.Panicf("Get table info failed: %v", err) @@ -236,35 +206,14 @@ if err != nil { log.Panicf("Get table columns failed: %v", err) } - columns[tableName] = cols + for _, col := range cols { + columns[tableName][col] = true + } + } return columns } -func extractTablesFromDB(db apid.DB) (tables map[string]bool) { - - tables = make(map[string]bool) - - log.Debug("Extracting table names from existing DB") - rows, err := db.Query("SELECT DISTINCT tableName FROM _transicator_tables;") - defer rows.Close() - - if err != nil { - log.Panicf("Error reading current set of tables: %v", err) - } - - for rows.Next() { - var table string - if err := rows.Scan(&table); err != nil { - log.Panicf("Error reading current set of tables: %v", err) - } - log.Debugf("Table %s found in existing db", table) - - tables[table] = true - } - return tables -} - // Skip Downloading snapshot if there is already a snapshot available from previous run func startOnLocalSnapshot(snapshot string) *common.Snapshot { log.Infof("Starting on local snapshot: %s", snapshot) @@ -275,7 +224,7 @@ log.Panicf("Database inaccessible: %v", err) } - knownTables = extractTablesFromDB(db) + knownTables = extractTableColsFromDB(db) // allow plugins (including this one) to start immediately on existing database // Note: this MUST have no tables as that is used as an indicator
diff --git a/snapshot_test.go b/snapshot_test.go index ffad61a..bcbafd5 100644 --- a/snapshot_test.go +++ b/snapshot_test.go
@@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. package apidApigeeSync + import ( "github.com/apigee-labs/transicator/common" . "github.com/onsi/ginkgo" @@ -24,7 +25,6 @@ const testDbId = "test_snapshot" Context("Change Agent Unit Tests", func() { - testHandler := handler{} var createTestDb = func(sqlfile string, dbId string) common.Snapshot { initDb(sqlfile, "./mockdb_snapshot.sqlite3") @@ -43,8 +43,8 @@ BeforeEach(func() { event := createTestDb("./sql/init_mock_db.sql", testDbId) - testHandler.Handle(&event) - knownTables = extractTablesFromDB(getDB()) + processSnapshot(&event) + knownTables = extractTableColsFromDB(getDB()) }) It("test extract table columns", func() { @@ -52,10 +52,14 @@ SnapshotInfo: testDbId, } columns := extractTableColumnsFromSnapshot(s) - for table, cols := range columns { + for table, colMap := range columns { + cols := []string{} + for col := range colMap { + cols = append(cols, col) + } log.Error("snapshot TABLE: " + table + " COLUMN: " + strings.Join(cols, "|")) } }) }) -}) \ No newline at end of file +})