add column maps
diff --git a/apigee_sync.go b/apigee_sync.go
index 10745de..1f6e38a 100644
--- a/apigee_sync.go
+++ b/apigee_sync.go
@@ -26,7 +26,7 @@
maxIdleConnsPerHost = 10
)
-var knownTables = make(map[string]bool)
+var knownTables = make(map[string]map[string]bool)
/*
* Start from existing snapshot if possible
diff --git a/apigee_sync_test.go b/apigee_sync_test.go
index b885593..ad79a6b 100644
--- a/apigee_sync_test.go
+++ b/apigee_sync_test.go
@@ -235,38 +235,39 @@
It("should correctly identify non-proper subsets with respect to maps", func() {
+ testMap := map[string]map[string]bool{"a": make(map[string]bool), "b": make(map[string]bool)}
//test b proper subset of a
- Expect(changesHaveNewTables(map[string]bool{"a": true, "b": true},
+ Expect(changesHaveNewTables(testMap,
[]common.Change{common.Change{Table: "b"}},
)).To(BeFalse())
//test a == b
- Expect(changesHaveNewTables(map[string]bool{"a": true, "b": true},
+ Expect(changesHaveNewTables(testMap,
[]common.Change{common.Change{Table: "a"}, common.Change{Table: "b"}},
)).To(BeFalse())
//test b superset of a
- Expect(changesHaveNewTables(map[string]bool{"a": true, "b": true},
+ Expect(changesHaveNewTables(testMap,
[]common.Change{common.Change{Table: "a"}, common.Change{Table: "b"}, common.Change{Table: "c"}},
)).To(BeTrue())
//test b not subset of a
- Expect(changesHaveNewTables(map[string]bool{"a": true, "b": true},
+ Expect(changesHaveNewTables(testMap,
[]common.Change{common.Change{Table: "c"}},
)).To(BeTrue())
//test a empty
- Expect(changesHaveNewTables(map[string]bool{},
+ Expect(changesHaveNewTables(map[string]map[string]bool{},
[]common.Change{common.Change{Table: "a"}},
)).To(BeTrue())
//test b empty
- Expect(changesHaveNewTables(map[string]bool{"a": true, "b": true},
+ Expect(changesHaveNewTables(testMap,
[]common.Change{},
)).To(BeFalse())
//test b nil
- Expect(changesHaveNewTables(map[string]bool{"a": true, "b": true}, nil)).To(BeFalse())
+ Expect(changesHaveNewTables(testMap, nil)).To(BeFalse())
//test a nil
Expect(changesHaveNewTables(nil,
diff --git a/change_test.go b/change_test.go
index 383bff6..ec9b82f 100644
--- a/change_test.go
+++ b/change_test.go
@@ -46,7 +46,7 @@
BeforeEach(func() {
event := createTestDb("./sql/init_mock_db.sql", "test_change")
processSnapshot(&event)
- knownTables = extractTablesFromDB(getDB())
+ knownTables = extractTableColsFromDB(getDB())
})
var initializeContext = func() {
diff --git a/changes.go b/changes.go
index 9f26466..9141927 100644
--- a/changes.go
+++ b/changes.go
@@ -298,7 +298,7 @@
/*
* Determine if any tables in changes are not present in known tables
*/
-func changesHaveNewTables(a map[string]bool, changes []common.Change) bool {
+func changesHaveNewTables(a map[string]map[string]bool, changes []common.Change) bool {
//nil maps should not be passed in. Making the distinction between nil map and empty map
if a == nil {
@@ -307,7 +307,7 @@
}
for _, change := range changes {
- if !a[normalizeTableName(change.Table)] {
+ if _, ok := a[normalizeTableName(change.Table)]; !ok {
log.Infof("Unable to find %s table in current known tables", change.Table)
return true
}
diff --git a/snapshot.go b/snapshot.go
index ae889e3..66108eb 100644
--- a/snapshot.go
+++ b/snapshot.go
@@ -147,7 +147,7 @@
}
func (s *simpleSnapShotManager) storeDataSnapshot(snapshot *common.Snapshot) {
- knownTables = extractTablesFromSnapshot(snapshot)
+ knownTables = extractTableColumnsFromSnapshot(snapshot)
_, err := dataService.DBVersion(snapshot.SnapshotInfo)
if err != nil {
@@ -167,50 +167,20 @@
}
-func extractTablesFromSnapshot(snapshot *common.Snapshot) (tables map[string]bool) {
-
- tables = make(map[string]bool)
-
- log.Debug("Extracting table names from snapshot")
- if snapshot.Tables == nil {
- //if this panic ever fires, it's a bug
- db, err := dataService.DBVersion(snapshot.SnapshotInfo)
- if err != nil {
- log.Panicf("Database inaccessible: %v", err)
- }
- rows, err := db.Query("SELECT DISTINCT tableName FROM _transicator_tables;")
- if err != nil {
- log.Panicf("Unable to read in known snapshot tables from sqlite file")
- }
- for rows.Next() {
- var tableName string
- rows.Scan(&tableName)
- if err != nil {
- log.Panic("Error scaning tableNames from _transicator_tables")
- }
- tables[tableName] = true
- }
-
- } else {
-
- for _, table := range snapshot.Tables {
- tables[table.Name] = true
- }
- }
- return tables
-
-}
-
-func extractTableColumnsFromSnapshot(snapshot *common.Snapshot) (map[string][]string) {
-
- columns := make(map[string][]string)
- tables := make([]string, 0)
-
+func extractTableColumnsFromSnapshot(snapshot *common.Snapshot) map[string]map[string]bool {
log.Debug("Extracting table names from snapshot")
db, err := dataService.DBVersion(snapshot.SnapshotInfo)
if err != nil {
log.Panicf("Database inaccessible: %v", err)
}
+
+ return extractTableColsFromDB(db)
+}
+
+func extractTableColsFromDB(db apid.DB) map[string]map[string]bool {
+
+ columns := make(map[string]map[string]bool)
+ tables := make([]string, 0)
rows, err := db.Query("SELECT DISTINCT tableName FROM _transicator_tables;")
if err != nil {
log.Panicf("Unable to read in known snapshot tables from sqlite file")
@@ -226,7 +196,7 @@
}
for _, tableName := range tables {
-
+ columns[tableName] = make(map[string]bool)
dummyRows, err := db.Query("SELECT * FROM " + tableName + " LIMIT 0;")
if err != nil {
log.Panicf("Get table info failed: %v", err)
@@ -236,35 +206,14 @@
if err != nil {
log.Panicf("Get table columns failed: %v", err)
}
- columns[tableName] = cols
+ for _, col := range cols {
+ columns[tableName][col] = true
+ }
+
}
return columns
}
-func extractTablesFromDB(db apid.DB) (tables map[string]bool) {
-
- tables = make(map[string]bool)
-
- log.Debug("Extracting table names from existing DB")
- rows, err := db.Query("SELECT DISTINCT tableName FROM _transicator_tables;")
- defer rows.Close()
-
- if err != nil {
- log.Panicf("Error reading current set of tables: %v", err)
- }
-
- for rows.Next() {
- var table string
- if err := rows.Scan(&table); err != nil {
- log.Panicf("Error reading current set of tables: %v", err)
- }
- log.Debugf("Table %s found in existing db", table)
-
- tables[table] = true
- }
- return tables
-}
-
// Skip Downloading snapshot if there is already a snapshot available from previous run
func startOnLocalSnapshot(snapshot string) *common.Snapshot {
log.Infof("Starting on local snapshot: %s", snapshot)
@@ -275,7 +224,7 @@
log.Panicf("Database inaccessible: %v", err)
}
- knownTables = extractTablesFromDB(db)
+ knownTables = extractTableColsFromDB(db)
// allow plugins (including this one) to start immediately on existing database
// Note: this MUST have no tables as that is used as an indicator
diff --git a/snapshot_test.go b/snapshot_test.go
index ffad61a..bcbafd5 100644
--- a/snapshot_test.go
+++ b/snapshot_test.go
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package apidApigeeSync
+
import (
"github.com/apigee-labs/transicator/common"
. "github.com/onsi/ginkgo"
@@ -24,7 +25,6 @@
const testDbId = "test_snapshot"
Context("Change Agent Unit Tests", func() {
- testHandler := handler{}
var createTestDb = func(sqlfile string, dbId string) common.Snapshot {
initDb(sqlfile, "./mockdb_snapshot.sqlite3")
@@ -43,8 +43,8 @@
BeforeEach(func() {
event := createTestDb("./sql/init_mock_db.sql", testDbId)
- testHandler.Handle(&event)
- knownTables = extractTablesFromDB(getDB())
+ processSnapshot(&event)
+ knownTables = extractTableColsFromDB(getDB())
})
It("test extract table columns", func() {
@@ -52,10 +52,14 @@
SnapshotInfo: testDbId,
}
columns := extractTableColumnsFromSnapshot(s)
- for table, cols := range columns {
+ for table, colMap := range columns {
+ cols := []string{}
+ for col := range colMap {
+ cols = append(cols, col)
+ }
log.Error("snapshot TABLE: " + table + " COLUMN: " + strings.Join(cols, "|"))
}
})
})
-})
\ No newline at end of file
+})
