Add DB configuration pool settings, as options to set, with defaults set.
diff --git a/README.md b/README.md index e3519f2..7a4d0cd 100644 --- a/README.md +++ b/README.md
@@ -52,3 +52,8 @@ ## Running Tests go test $(glide novendor) + +## apid.Data() service +This service provides the primitives to perform SQL operations on the database. It also provides the +provision to alter DB connection pool settings via ConfigDBMaxConns, ConfigDBIdleConns andonfigDBConnsTimeout configuration parameters. They currently are defaulted to 1000, 1000 and 120 seconds respectively. +More details on this can be found at https://golang.org/pkg/database/sql
diff --git a/api/api.go b/api/api.go index 32a28aa..d378de1 100644 --- a/api/api.go +++ b/api/api.go
@@ -27,10 +27,16 @@ ) const ( - configAPIListen = "api_listen" - configExpVarPath = "api_expvar_path" - configReadyPath = "api_ready" - configHealthyPath = "api_healthy" + configAPIListen = "api_listen" + configExpVarPath = "api_expvar_path" + configReadyPath = "api_ready" + configHealthyPath = "api_healthy" + ConfigDBMaxConns = "db_config_max_conns" + ConfigDBIdleConns = "db_config_idle_conns" + ConfigDBConnsTimeout = "db_config_conns_timeout_seconds" + dbDefaultMaxConnsLimit = 1000 + dbDefaultIdleConnsLimit = 1000 + dbMaxConnTimeoutLimit = 120 ) var log apid.LogService @@ -45,6 +51,10 @@ config.SetDefault(configReadyPath, "/ready") config.SetDefault(configHealthyPath, "/healthy") + config.SetDefault(ConfigDBMaxConns, dbDefaultMaxConnsLimit) + config.SetDefault(ConfigDBIdleConns, dbDefaultIdleConnsLimit) + config.SetDefault(ConfigDBConnsTimeout, dbMaxConnTimeoutLimit) + listen := config.GetString(configAPIListen) h, p, err := net.SplitHostPort(listen) if err != nil {
diff --git a/data/data.go b/data/data.go index 61aabb9..7fed43d 100644 --- a/data/data.go +++ b/data/data.go
@@ -18,11 +18,15 @@ "database/sql" "fmt" "github.com/30x/apid-core" + "github.com/30x/apid-core/api" "github.com/30x/apid-core/data/wrap" + "github.com/30x/apid-core/logger" + "github.com/Sirupsen/logrus" "github.com/mattn/go-sqlite3" "os" "path" "runtime" + "strings" "sync" "time" ) @@ -41,7 +45,12 @@ var log, dbTraceLog apid.LogService var config apid.ConfigService -var dbMap = make(map[string]*sql.DB) +type dbMapInfo struct { + db *sql.DB + closed chan bool +} + +var dbMap = make(map[string]*dbMapInfo) var dbMapSync sync.RWMutex func CreateDataService() apid.DataService { @@ -97,12 +106,17 @@ dbMapSync.Lock() defer dbMapSync.Unlock() - db := dbMap[versionedID] - if db != nil { - dbMap[versionedID] = nil + dbm := dbMap[versionedID] + if dbm != nil && dbm.db != nil { + if strings.EqualFold(config.GetString(logger.ConfigLevel), logrus.DebugLevel.String()) { + dbm.closed <- true + } log.Warn("SETTING FINALIZER") finalizer := Delete(versionedID) - runtime.SetFinalizer(db, finalizer) + runtime.SetFinalizer(dbm.db, finalizer) + dbMap[versionedID] = nil + } else { + log.Error("Cannot find DB handle for ver {%s} to release", version) } return @@ -110,23 +124,19 @@ func (d *dataService) dbVersionForID(id, version string) (db *sql.DB, err error) { + var stoplogchan chan bool versionedID := VersionedDBID(id, version) dbMapSync.RLock() - db = dbMap[versionedID] + dbm := dbMap[versionedID] dbMapSync.RUnlock() - if db != nil { - return + if dbm != nil && dbm.db != nil { + return dbm.db, nil } dbMapSync.Lock() defer dbMapSync.Unlock() - db = dbMap[versionedID] - if db != nil { - return - } - dataPath := DBPath(versionedID) if err = os.MkdirAll(path.Dir(dataPath), 0700); err != nil { @@ -171,11 +181,16 @@ log.Errorf("error enabling foreign_keys: %s", err) return } - dbLvl := config.GetString("log_level") - if dbLvl == "Debug" { - go printDBStatsInfo(db, versionedID) + if strings.EqualFold(config.GetString(logger.ConfigLevel), + logrus.DebugLevel.String()) { + stoplogchan = logDBInfo(versionedID, db) } - dbMap[versionedID] = db + + db.SetMaxOpenConns(config.GetInt(api.ConfigDBMaxConns)) + db.SetMaxIdleConns(config.GetInt(api.ConfigDBIdleConns)) + db.SetConnMaxLifetime(time.Duration(config.GetInt(api.ConfigDBConnsTimeout)) * time.Second) + dbInfo := dbMapInfo{db: db, closed: stoplogchan} + dbMap[versionedID] = &dbInfo return } @@ -204,9 +219,19 @@ return path.Join(storagePath, relativeDataPath, id, "sqlite3") } -func printDBStatsInfo(db *sql.DB, versionedId string) { - tick := time.Duration(statCollectionInterval * time.Second) - for _ = range time.Tick(tick) { - log.Debugf("Current number of open DB connections for ver {%s} is {%d}", versionedId, db.Stats().OpenConnections) - } +func logDBInfo(versionedId string, db *sql.DB) chan bool { + stop := make(chan bool) + go func() { + for { + select { + case <-time.After(time.Duration(statCollectionInterval * time.Second)): + log.Debugf("Current number of open DB connections for ver {%s} is {%d}", + versionedId, db.Stats().OpenConnections) + case <-stop: + log.Debugf("Stop DB conn. logging for ver {%s}", versionedId) + return + } + } + }() + return stop }
diff --git a/logger/logger.go b/logger/logger.go index 39f6e8b..7c16b17 100644 --- a/logger/logger.go +++ b/logger/logger.go
@@ -24,7 +24,7 @@ ) const ( - configLevel = "log_level" + ConfigLevel = "log_level" defaultLevel = logrus.ErrorLevel @@ -42,10 +42,10 @@ func Base() apid.LogService { if std == nil { config = apid.Config() - config.SetDefault(configLevel, defaultLevel.String()) - logLevel := config.GetString(configLevel) + config.SetDefault(ConfigLevel, defaultLevel.String()) + logLevel := config.GetString(ConfigLevel) fmt.Printf("Base log level: %s\n", logLevel) - std = NewLogger(configLevel, logLevel) + std = NewLogger(ConfigLevel, logLevel) } return std } @@ -62,7 +62,7 @@ // note: config module xx log level using config var: xx_log_level = "debug" func (l *logger) ForModule(name string) apid.LogService { - configKey := fmt.Sprintf("%s_%s", name, configLevel) + configKey := fmt.Sprintf("%s_%s", name, ConfigLevel) log := NewLogger(configKey, config.GetString(configKey)).WithField(moduleField, name) std.Debugf("created logger '%s' at level %s", name, log.(loggerPlus).Level()) return log