merge upstream/master
diff --git a/README.md b/README.md
index 09a3d50..6051f6a 100644
--- a/README.md
+++ b/README.md
@@ -8,6 +8,8 @@
Viper is a complete configuration solution. Designed to work within an
application to handle file based configuration and seamlessly marry that with
command line flags which can also be used to control application behavior.
+Viper also supports retrieving configuration values from remote key/value stores.
+Etcd and Consul are supported.
## Why Viper?
@@ -26,10 +28,8 @@
Viper believes that:
1. command line flags take precedence over options set in config files
-2. config files take precedence over defaults
-
-Config files often can be found in multiple locations. Viper allows you to set
-multiple paths to search for the config file in.
+2. config files take precedence over options set in remote key/value stores
+3. remote key/value stores take precedence over defaults
Viper configuration keys are case insensitive.
@@ -70,6 +70,46 @@
fmt.Println("verbose enabled")
}
+### Remote Key/Value Store Support
+Viper will read a config string (as JSON, TOML, or YAML) retrieved from a
+path in a Key/Value store such as Etcd or Consul. These values take precedence
+over default values, but are overriden by configuration values retrieved from disk,
+flags, or environment variables.
+
+Viper uses [crypt](https://github.com/xordataexchange/crypt) to retrieve configuration
+from the k/v store, which means that you can store your configuration values
+encrypted and have them automatically decrypted if you have the correct
+gpg keyring. Encryption is optional.
+
+You can use remote configuration in conjunction with local configuration, or
+independently of it.
+
+`crypt` has a command-line helper that you can use to put configurations
+in your k/v store. `crypt` defaults to etcd on http://127.0.0.1:4001.
+
+ go get github.com/xordataexchange/crypt/bin/crypt
+ crypt set -plaintext /config/hugo.json /Users/hugo/settings/config.json
+
+Confirm that your value was set:
+
+ crypt get -plaintext /config/hugo.json
+
+See the `crypt` documentation for examples of how to set encrypted values, or how
+to use Consul.
+
+### Remote Key/Value Store Example - Unencrypted
+
+ viper.AddRemoteProvider("etcd", "http://127.0.0.1:4001","/config/hugo.json")
+ viper.SetConfigType("json") // because there is no file extension in a stream of bytes
+ err := viper.ReadRemoteConfig()
+
+### Remote Key/Value Store Example - Encrypted
+
+ viper.AddSecureRemoteProvier("etcd","http://127.0.0.1:4001","/config/hugo.json","/etc/secrets/mykeyring.gpg")
+ viper.SetConfigType("json") // because there is no file extension in a stream of bytes
+ err := viper.ReadRemoteConfig()
+
+
## Q & A
diff --git a/viper.go b/viper.go
index 720b759..aa81a0c 100644
--- a/viper.go
+++ b/viper.go
@@ -5,13 +5,15 @@
// Viper is a application configuration system.
// It believes that applications can be configured a variety of ways
-// via flags, ENVIRONMENT variables, configuration files.
+// via flags, ENVIRONMENT variables, configuration files retrieved
+// from the file system, or a remote key/value store.
// Each item takes precedence over the item below it:
// flag
// env
// config
+// key/value store
// default
package viper
@@ -25,6 +27,7 @@
"os"
"path"
"path/filepath"
+ "reflect"
"runtime"
"strings"
"time"
@@ -35,17 +38,33 @@
"github.com/spf13/cast"
jww "github.com/spf13/jwalterweatherman"
"github.com/spf13/pflag"
+ crypt "github.com/xordataexchange/crypt/config"
"gopkg.in/yaml.v1"
)
+// remoteProvider stores the configuration necessary
+// to connect to a remote key/value store.
+// Optional secretKeyring to unencrypt encrypted values
+// can be provided.
+type remoteProvider struct {
+ provider string
+ endpoint string
+ path string
+ secretKeyring string
+}
+
// A set of paths to look for the config file in
var configPaths []string
+// A set of remote providers to search for the configuration
+var remoteProviders []*remoteProvider
+
// Name of file to look for inside the path
var configName string = "config"
// extensions Supported
var SupportedExts []string = []string{"json", "toml", "yaml", "yml"}
+var SupportedRemoteProviders []string = []string{"etcd", "consul"}
var configFile string
var configType string
@@ -53,6 +72,7 @@
var override map[string]interface{} = make(map[string]interface{})
var env map[string]string = make(map[string]string)
var defaults map[string]interface{} = make(map[string]interface{})
+var kvstore map[string]interface{} = make(map[string]interface{})
var pflags map[string]*pflag.Flag = make(map[string]*pflag.Flag)
var aliases map[string]string = make(map[string]string)
@@ -81,6 +101,76 @@
}
}
+// AddRemoteProvider adds a remote configuration source.
+// Remote Providers are searched in the order they are added.
+// provider is a string value, "etcd" or "consul" are currently supported.
+// endpoint is the url. etcd requires http://ip:port consul requires ip:port
+// path is the path in the k/v store to retrieve configuration
+// To retrieve a config file called myapp.json from /configs/myapp.json
+// you should set path to /configs and set config name (SetConfigName()) to
+// "myapp"
+func AddRemoteProvider(provider, endpoint, path string) error {
+ if !stringInSlice(provider, SupportedRemoteProviders) {
+ return UnsupportedRemoteProviderError(provider)
+ }
+ if provider != "" && endpoint != "" {
+ jww.INFO.Printf("adding %s:%s to remote provider list", provider, endpoint)
+ rp := &remoteProvider{
+ endpoint: endpoint,
+ provider: provider,
+ path: path,
+ }
+ if !providerPathExists(rp) {
+ remoteProviders = append(remoteProviders, rp)
+ }
+ }
+ return nil
+}
+
+// AddSecureRemoteProvider adds a remote configuration source.
+// Secure Remote Providers are searched in the order they are added.
+// provider is a string value, "etcd" or "consul" are currently supported.
+// endpoint is the url. etcd requires http://ip:port consul requires ip:port
+// secretkeyring is the filepath to your openpgp secret keyring. e.g. /etc/secrets/myring.gpg
+// path is the path in the k/v store to retrieve configuration
+// To retrieve a config file called myapp.json from /configs/myapp.json
+// you should set path to /configs and set config name (SetConfigName()) to
+// "myapp"
+// Secure Remote Providers are implemented with github.com/xordataexchange/crypt
+func AddSecureRemoteProvider(provider, endpoint, path, secretkeyring string) error {
+ if !stringInSlice(provider, SupportedRemoteProviders) {
+ return UnsupportedRemoteProviderError(provider)
+ }
+ if provider != "" && endpoint != "" {
+ jww.INFO.Printf("adding %s:%s to remote provider list", provider, endpoint)
+ rp := &remoteProvider{
+ endpoint: endpoint,
+ provider: provider,
+ path: path,
+ }
+ if !providerPathExists(rp) {
+ remoteProviders = append(remoteProviders, rp)
+ }
+ }
+ return nil
+}
+
+func providerPathExists(p *remoteProvider) bool {
+
+ for _, y := range remoteProviders {
+ if reflect.DeepEqual(y, p) {
+ return true
+ }
+ }
+ return false
+}
+
+type UnsupportedRemoteProviderError string
+
+func (str UnsupportedRemoteProviderError) Error() string {
+ return fmt.Sprintf("Unsupported Remote Provider Type %q", string(str))
+}
+
func GetString(key string) string {
return cast.ToString(Get(key))
}
@@ -132,6 +222,10 @@
if err != nil {
return err
}
+ err = mapstructure.Decode(kvstore, rawVal)
+ if err != nil {
+ return err
+ }
insensativiseMaps()
@@ -221,6 +315,12 @@
return val
}
+ val, exists = kvstore[key]
+ if exists {
+ jww.TRACE.Println(key, "found in key/value store:", val)
+ return val
+ }
+
val, exists = defaults[key]
if exists {
jww.TRACE.Println(key, "found in defaults:", val)
@@ -289,6 +389,10 @@
delete(config, alias)
config[key] = val
}
+ if val, ok := kvstore[alias]; ok {
+ delete(kvstore, alias)
+ kvstore[key] = val
+ }
if val, ok := defaults[alias]; ok {
delete(defaults, alias)
defaults[key] = val
@@ -331,7 +435,8 @@
}
// The user provided value (via flag)
-// Will be used instead of values obtained via config file, ENV or default
+// Will be used instead of values obtained via
+// config file, ENV, default, or key/value store
func Set(key string, value interface{}) {
// If alias passed in, then set the proper override
key = realKey(strings.ToLower(key))
@@ -345,7 +450,7 @@
}
// Viper will discover and load the configuration file from disk
-// searching in one of the defined paths.
+// and key/value stores, searching in one of the defined paths.
func ReadInConfig() error {
jww.INFO.Println("Attempting to read in config file")
if !stringInSlice(getConfigType(), SupportedExts) {
@@ -357,38 +462,98 @@
return err
}
- MarshallReader(bytes.NewReader(file))
+ MarshallReader(bytes.NewReader(file), config)
return nil
}
-
-func MarshallReader(in io.Reader) {
+func ReadRemoteConfig() error {
+ err := getKeyValueConfig()
+ if err != nil {
+ return err
+ }
+ return nil
+}
+func MarshallReader(in io.Reader, c map[string]interface{}) {
buf := new(bytes.Buffer)
buf.ReadFrom(in)
switch getConfigType() {
case "yaml", "yml":
- if err := yaml.Unmarshal(buf.Bytes(), &config); err != nil {
+ if err := yaml.Unmarshal(buf.Bytes(), &c); err != nil {
jww.ERROR.Fatalf("Error parsing config: %s", err)
}
case "json":
- if err := json.Unmarshal(buf.Bytes(), &config); err != nil {
+ if err := json.Unmarshal(buf.Bytes(), &c); err != nil {
jww.ERROR.Fatalf("Error parsing config: %s", err)
}
case "toml":
- if _, err := toml.Decode(buf.String(), &config); err != nil {
+ if _, err := toml.Decode(buf.String(), &c); err != nil {
jww.ERROR.Fatalf("Error parsing config: %s", err)
}
}
- insensativiseMap(config)
+ insensativiseMap(c)
}
func insensativiseMaps() {
insensativiseMap(config)
insensativiseMap(defaults)
insensativiseMap(override)
+ insensativiseMap(kvstore)
+}
+
+// retrieve the first found remote configuration
+func getKeyValueConfig() error {
+ for _, rp := range remoteProviders {
+ val, err := getRemoteConfig(rp)
+ if err != nil {
+ continue
+ }
+ kvstore = val
+ return nil
+ }
+ return RemoteConfigError("No Files Found")
+}
+
+type RemoteConfigError string
+
+func (rce RemoteConfigError) Error() string {
+ return fmt.Sprintf("Remote Configurations Error: %s", string(rce))
+}
+
+func getRemoteConfig(provider *remoteProvider) (map[string]interface{}, error) {
+ var cm crypt.ConfigManager
+ var err error
+
+ if provider.secretKeyring != "" {
+ kr, err := os.Open(provider.secretKeyring)
+ defer kr.Close()
+ if err != nil {
+ return nil, err
+ }
+ if provider.provider == "etcd" {
+ cm, err = crypt.NewEtcdConfigManager([]string{provider.endpoint}, kr)
+ } else {
+ cm, err = crypt.NewConsulConfigManager([]string{provider.endpoint}, kr)
+ }
+ } else {
+ if provider.provider == "etcd" {
+ cm, err = crypt.NewStandardEtcdConfigManager([]string{provider.endpoint})
+ } else {
+ cm, err = crypt.NewStandardConsulConfigManager([]string{provider.endpoint})
+ }
+ }
+ if err != nil {
+ return nil, err
+ }
+ b, err := cm.Get(provider.path)
+ if err != nil {
+ return nil, err
+ }
+ reader := bytes.NewReader(b)
+ MarshallReader(reader, kvstore)
+ return kvstore, err
}
func insensativiseMap(m map[string]interface{}) {
@@ -412,6 +577,10 @@
m[key] = struct{}{}
}
+ for key, _ := range kvstore {
+ m[key] = struct{}{}
+ }
+
for key, _ := range override {
m[key] = struct{}{}
}
@@ -594,6 +763,8 @@
func Debug() {
fmt.Println("Config:")
pretty.Println(config)
+ fmt.Println("Key/Value Store:")
+ pretty.Println(kvstore)
fmt.Println("Env:")
pretty.Println(env)
fmt.Println("Defaults:")
@@ -613,6 +784,7 @@
configFile = ""
configType = ""
+ kvstore = make(map[string]interface{})
config = make(map[string]interface{})
override = make(map[string]interface{})
env = make(map[string]string)
diff --git a/viper_test.go b/viper_test.go
index 09d8c78..bc4059f 100644
--- a/viper_test.go
+++ b/viper_test.go
@@ -54,6 +54,12 @@
}
}`)
+var remoteExample = []byte(`{
+"id":"0002",
+"type":"cronut",
+"newkey":"remote"
+}`)
+
//stubs for PFlag Values
type stringValue string
@@ -89,7 +95,7 @@
SetConfigType("yaml")
r := bytes.NewReader(yamlExample)
- MarshallReader(r)
+ MarshallReader(r, config)
assert.True(t, InConfig("name"))
assert.False(t, InConfig("state"))
assert.Equal(t, "steve", Get("name"))
@@ -130,7 +136,7 @@
SetConfigType("yml")
r := bytes.NewReader(yamlExample)
- MarshallReader(r)
+ MarshallReader(r, config)
assert.Equal(t, "steve", Get("name"))
}
@@ -138,7 +144,7 @@
SetConfigType("json")
r := bytes.NewReader(jsonExample)
- MarshallReader(r)
+ MarshallReader(r, config)
assert.Equal(t, "0001", Get("id"))
}
@@ -146,14 +152,30 @@
SetConfigType("toml")
r := bytes.NewReader(tomlExample)
- MarshallReader(r)
+ MarshallReader(r, config)
assert.Equal(t, "TOML Example", Get("title"))
}
+func TestRemotePrecedence(t *testing.T) {
+ SetConfigType("json")
+ r := bytes.NewReader(jsonExample)
+ MarshallReader(r, config)
+ remote := bytes.NewReader(remoteExample)
+ assert.Equal(t, "0001", Get("id"))
+ MarshallReader(remote, kvstore)
+ assert.Equal(t, "0001", Get("id"))
+ assert.NotEqual(t, "cronut", Get("type"))
+ assert.Equal(t, "remote", Get("newkey"))
+ Set("newkey", "newvalue")
+ assert.NotEqual(t, "remote", Get("newkey"))
+ assert.Equal(t, "newvalue", Get("newkey"))
+ Set("newkey", "remote")
+}
+
func TestEnv(t *testing.T) {
SetConfigType("json")
r := bytes.NewReader(jsonExample)
- MarshallReader(r)
+ MarshallReader(r, config)
BindEnv("id")
BindEnv("f", "FOOD")
@@ -171,9 +193,9 @@
}
func TestAllKeys(t *testing.T) {
- ks := sort.StringSlice{"title", "owner", "name", "beard", "ppu", "batters", "hobbies", "clothing", "age", "hacker", "id", "type", "eyes"}
+ ks := sort.StringSlice{"title", "newkey", "owner", "name", "beard", "ppu", "batters", "hobbies", "clothing", "age", "hacker", "id", "type", "eyes"}
dob, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z")
- all := map[string]interface{}{"hacker": true, "beard": true, "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "ppu": 0.55, "clothing": map[interface{}]interface{}{"jacket": "leather", "trousers": "denim"}, "name": "crunk", "owner": map[string]interface{}{"organization": "MongoDB", "Bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "id": "13", "title": "TOML Example", "age": 35, "type": "donut", "eyes": "brown"}
+ all := map[string]interface{}{"hacker": true, "beard": true, "newkey": "remote", "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "ppu": 0.55, "clothing": map[interface{}]interface{}{"jacket": "leather", "trousers": "denim"}, "name": "crunk", "owner": map[string]interface{}{"organization": "MongoDB", "Bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "id": "13", "title": "TOML Example", "age": 35, "type": "donut", "eyes": "brown"}
var allkeys sort.StringSlice
allkeys = AllKeys()
