Merge pull request #10 from 30x/XAPID-1089
Add periodic DB Conn Info.
diff --git a/README.md b/README.md
index e3519f2..7a4d0cd 100644
--- a/README.md
+++ b/README.md
@@ -52,3 +52,8 @@
## Running Tests
go test $(glide novendor)
+
+## apid.Data() service
+This service provides the primitives to perform SQL operations on the database. It also provides the
+provision to alter DB connection pool settings via ConfigDBMaxConns, ConfigDBIdleConns andonfigDBConnsTimeout configuration parameters. They currently are defaulted to 1000, 1000 and 120 seconds respectively.
+More details on this can be found at https://golang.org/pkg/database/sql
diff --git a/api/api.go b/api/api.go
index 32a28aa..d378de1 100644
--- a/api/api.go
+++ b/api/api.go
@@ -27,10 +27,16 @@
)
const (
- configAPIListen = "api_listen"
- configExpVarPath = "api_expvar_path"
- configReadyPath = "api_ready"
- configHealthyPath = "api_healthy"
+ configAPIListen = "api_listen"
+ configExpVarPath = "api_expvar_path"
+ configReadyPath = "api_ready"
+ configHealthyPath = "api_healthy"
+ ConfigDBMaxConns = "db_config_max_conns"
+ ConfigDBIdleConns = "db_config_idle_conns"
+ ConfigDBConnsTimeout = "db_config_conns_timeout_seconds"
+ dbDefaultMaxConnsLimit = 1000
+ dbDefaultIdleConnsLimit = 1000
+ dbMaxConnTimeoutLimit = 120
)
var log apid.LogService
@@ -45,6 +51,10 @@
config.SetDefault(configReadyPath, "/ready")
config.SetDefault(configHealthyPath, "/healthy")
+ config.SetDefault(ConfigDBMaxConns, dbDefaultMaxConnsLimit)
+ config.SetDefault(ConfigDBIdleConns, dbDefaultIdleConnsLimit)
+ config.SetDefault(ConfigDBConnsTimeout, dbMaxConnTimeoutLimit)
+
listen := config.GetString(configAPIListen)
h, p, err := net.SplitHostPort(listen)
if err != nil {
diff --git a/data/data.go b/data/data.go
index 0c24d65..7fed43d 100644
--- a/data/data.go
+++ b/data/data.go
@@ -17,23 +17,27 @@
import (
"database/sql"
"fmt"
+ "github.com/30x/apid-core"
+ "github.com/30x/apid-core/api"
+ "github.com/30x/apid-core/data/wrap"
+ "github.com/30x/apid-core/logger"
+ "github.com/Sirupsen/logrus"
+ "github.com/mattn/go-sqlite3"
"os"
"path"
"runtime"
+ "strings"
"sync"
-
- "github.com/30x/apid-core"
- "github.com/30x/apid-core/data/wrap"
- "github.com/mattn/go-sqlite3"
+ "time"
)
const (
- configDataDriverKey = "data_driver"
- configDataSourceKey = "data_source"
- configDataPathKey = "data_path"
-
- commonDBID = "common"
- commonDBVersion = "base"
+ configDataDriverKey = "data_driver"
+ configDataSourceKey = "data_source"
+ configDataPathKey = "data_path"
+ statCollectionInterval = 10
+ commonDBID = "common"
+ commonDBVersion = "base"
defaultTraceLevel = "warn"
)
@@ -41,7 +45,12 @@
var log, dbTraceLog apid.LogService
var config apid.ConfigService
-var dbMap = make(map[string]*sql.DB)
+type dbMapInfo struct {
+ db *sql.DB
+ closed chan bool
+}
+
+var dbMap = make(map[string]*dbMapInfo)
var dbMapSync sync.RWMutex
func CreateDataService() apid.DataService {
@@ -97,12 +106,17 @@
dbMapSync.Lock()
defer dbMapSync.Unlock()
- db := dbMap[versionedID]
- if db != nil {
- dbMap[versionedID] = nil
+ dbm := dbMap[versionedID]
+ if dbm != nil && dbm.db != nil {
+ if strings.EqualFold(config.GetString(logger.ConfigLevel), logrus.DebugLevel.String()) {
+ dbm.closed <- true
+ }
log.Warn("SETTING FINALIZER")
finalizer := Delete(versionedID)
- runtime.SetFinalizer(db, finalizer)
+ runtime.SetFinalizer(dbm.db, finalizer)
+ dbMap[versionedID] = nil
+ } else {
+ log.Error("Cannot find DB handle for ver {%s} to release", version)
}
return
@@ -110,23 +124,19 @@
func (d *dataService) dbVersionForID(id, version string) (db *sql.DB, err error) {
+ var stoplogchan chan bool
versionedID := VersionedDBID(id, version)
dbMapSync.RLock()
- db = dbMap[versionedID]
+ dbm := dbMap[versionedID]
dbMapSync.RUnlock()
- if db != nil {
- return
+ if dbm != nil && dbm.db != nil {
+ return dbm.db, nil
}
dbMapSync.Lock()
defer dbMapSync.Unlock()
- db = dbMap[versionedID]
- if db != nil {
- return
- }
-
dataPath := DBPath(versionedID)
if err = os.MkdirAll(path.Dir(dataPath), 0700); err != nil {
@@ -171,8 +181,16 @@
log.Errorf("error enabling foreign_keys: %s", err)
return
}
+ if strings.EqualFold(config.GetString(logger.ConfigLevel),
+ logrus.DebugLevel.String()) {
+ stoplogchan = logDBInfo(versionedID, db)
+ }
- dbMap[versionedID] = db
+ db.SetMaxOpenConns(config.GetInt(api.ConfigDBMaxConns))
+ db.SetMaxIdleConns(config.GetInt(api.ConfigDBIdleConns))
+ db.SetConnMaxLifetime(time.Duration(config.GetInt(api.ConfigDBConnsTimeout)) * time.Second)
+ dbInfo := dbMapInfo{db: db, closed: stoplogchan}
+ dbMap[versionedID] = &dbInfo
return
}
@@ -200,3 +218,20 @@
relativeDataPath := config.GetString(configDataPathKey)
return path.Join(storagePath, relativeDataPath, id, "sqlite3")
}
+
+func logDBInfo(versionedId string, db *sql.DB) chan bool {
+ stop := make(chan bool)
+ go func() {
+ for {
+ select {
+ case <-time.After(time.Duration(statCollectionInterval * time.Second)):
+ log.Debugf("Current number of open DB connections for ver {%s} is {%d}",
+ versionedId, db.Stats().OpenConnections)
+ case <-stop:
+ log.Debugf("Stop DB conn. logging for ver {%s}", versionedId)
+ return
+ }
+ }
+ }()
+ return stop
+}
diff --git a/logger/logger.go b/logger/logger.go
index 39f6e8b..7c16b17 100644
--- a/logger/logger.go
+++ b/logger/logger.go
@@ -24,7 +24,7 @@
)
const (
- configLevel = "log_level"
+ ConfigLevel = "log_level"
defaultLevel = logrus.ErrorLevel
@@ -42,10 +42,10 @@
func Base() apid.LogService {
if std == nil {
config = apid.Config()
- config.SetDefault(configLevel, defaultLevel.String())
- logLevel := config.GetString(configLevel)
+ config.SetDefault(ConfigLevel, defaultLevel.String())
+ logLevel := config.GetString(ConfigLevel)
fmt.Printf("Base log level: %s\n", logLevel)
- std = NewLogger(configLevel, logLevel)
+ std = NewLogger(ConfigLevel, logLevel)
}
return std
}
@@ -62,7 +62,7 @@
// note: config module xx log level using config var: xx_log_level = "debug"
func (l *logger) ForModule(name string) apid.LogService {
- configKey := fmt.Sprintf("%s_%s", name, configLevel)
+ configKey := fmt.Sprintf("%s_%s", name, ConfigLevel)
log := NewLogger(configKey, config.GetString(configKey)).WithField(moduleField, name)
std.Debugf("created logger '%s' at level %s", name, log.(loggerPlus).Level())
return log
