Issue #10: Support loading from urls
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b82f095..a34b828 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,7 +2,9 @@
### [1.6.1](https://github.com/magiconair/properties/tags/v1.6.1) - unreleased
- * [Issue #11](https://github.com/magiconair/properties/issues/11): Add [LoadString](http://godoc.org/github.com/magiconair/properties#Properties.LoadString) method to load properties from an UTF8 string.
+ * [Issue #10](https://github.com/magiconair/properties/issues/10): Add [LoadURL,LoadURLs,MustLoadURL,MustLoadURLs](http://godoc.org/github.com/magiconair/properties#Properties.LoadURL) method to load properties from a URL.
+ * [Issue #11](https://github.com/magiconair/properties/issues/11): Add [LoadString,MustLoadString](http://godoc.org/github.com/magiconair/properties#Properties.LoadString) method to load properties from an UTF8 string.
+ * [PR #8](https://github.com/magiconair/properties/pull/8): Add [MustFlag](http://godoc.org/github.com/magiconair/properties#Properties.MustFlag) method to provide overrides via command line flags. (@pascaldekloe)
### [1.6.0](https://github.com/magiconair/properties/tags/v1.6.0) - 11 Dec 2015
@@ -14,7 +16,7 @@
### [1.5.5](https://github.com/magiconair/properties/tags/v1.5.5) - 31 Jul 2015
- * [Pull Request #6](https://github.com/magiconair/properties/pull/6): Add [Delete](http://godoc.org/github.com/magiconair/properties#Properties.Delete) method to remove keys including comments. (@gerbenjacobs)
+ * [PR #6](https://github.com/magiconair/properties/pull/6): Add [Delete](http://godoc.org/github.com/magiconair/properties#Properties.Delete) method to remove keys including comments. (@gerbenjacobs)
### [1.5.4](https://github.com/magiconair/properties/tags/v1.5.4) - 23 Jun 2015
diff --git a/README.md b/README.md
index b3b2e5c..1dd13d7 100644
--- a/README.md
+++ b/README.md
@@ -5,12 +5,11 @@
properties is a Go library for reading and writing properties files.
-It supports reading from multiple files and Spring style recursive property
-expansion of expressions like `${key}` to their corresponding value.
-Value expressions can refer to other keys like in `${key}` or to
-environment variables like in `${USER}`.
-Filenames can also contain environment variables like in
-`/home/${USER}/myapp.properties`.
+It supports reading from multiple files or URLs and Spring style recursive
+property expansion of expressions like `${key}` to their corresponding value.
+Value expressions can refer to other keys like in `${key}` or to environment
+variables like in `${USER}`. Filenames can also contain environment variables
+like in `/home/${USER}/myapp.properties`.
Properties can be decoded into structs, maps, arrays and values through
struct tags.
@@ -56,6 +55,9 @@
// or via flags
p.MustFlag(flag.CommandLine)
+
+ // or via url
+ p = properties.MustLoadURL("http://host/path")
}
```
diff --git a/load.go b/load.go
index e74d77b..3915c73 100644
--- a/load.go
+++ b/load.go
@@ -5,9 +5,12 @@
package properties
import (
+ "bytes"
"fmt"
"io/ioutil"
+ "net/http"
"os"
+ "strings"
)
// Encoding specifies encoding of the input data.
@@ -43,6 +46,26 @@
return loadFiles(filenames, enc, ignoreMissing)
}
+// LoadURL reads the content of the URL into a Properties struct.
+//
+// The encoding is determined via the Content-Type header which
+// should be set to 'text/plain'. If the 'charset' parameter is
+// missing, 'iso-8859-1' or 'latin1' the encoding is set to
+// ISO-8859-1. If the 'charset' parameter is set to 'utf-8' the
+// encoding is set to UTF-8. A missing content type header is
+// interpreted as 'text/plain; charset=utf-8'.
+func LoadURL(url string) (*Properties, error) {
+ return loadURLs([]string{url}, false)
+}
+
+// LoadURLs reads the content of multiple URLs in the given order into a
+// Properties struct. If 'ignoreMissing' is true then a 404 status code will
+// not be reported as error. See LoadURL for the Content-Type header
+// and the encoding.
+func LoadURLs(urls []string, ignoreMissing bool) (*Properties, error) {
+ return loadURLs(urls, ignoreMissing)
+}
+
// MustLoadString reads an UTF8 string into a Properties struct and
// panics on error.
func MustLoadString(s string) *Properties {
@@ -62,6 +85,19 @@
return must(LoadFiles(filenames, enc, ignoreMissing))
}
+// MustLoadURL reads the content of a URL into a Properties struct and
+// panics on error.
+func MustLoadURL(url string) *Properties {
+ return must(LoadURL(url))
+}
+
+// MustLoadFiles reads the content of multiple URLs in the given order into a
+// Properties struct and panics on error. If 'ignoreMissing' is true then a 404
+// status code will not be reported as error.
+func MustLoadURLs(urls []string, ignoreMissing bool) *Properties {
+ return must(LoadURLs(urls, ignoreMissing))
+}
+
func loadBuf(buf []byte, enc Encoding) (*Properties, error) {
p, err := parse(convert(buf, enc))
if err != nil {
@@ -71,18 +107,17 @@
}
func loadFiles(filenames []string, enc Encoding, ignoreMissing bool) (*Properties, error) {
- buff := make([]byte, 0, 4096)
-
+ var buf bytes.Buffer
for _, filename := range filenames {
f, err := expandFilename(filename)
if err != nil {
return nil, err
}
- buf, err := ioutil.ReadFile(f)
+ data, err := ioutil.ReadFile(f)
if err != nil {
if ignoreMissing && os.IsNotExist(err) {
- // TODO(frank): should we log that we are skipping the file?
+ LogPrintf("properties: %s not found. skipping", filename)
continue
}
return nil, err
@@ -90,10 +125,47 @@
// concatenate the buffers and add a new line in case
// the previous file didn't end with a new line
- buff = append(append(buff, buf...), '\n')
+ buf.Write(data)
+ buf.WriteRune('\n')
}
+ return loadBuf(buf.Bytes(), enc)
+}
- return loadBuf(buff, enc)
+func loadURLs(urls []string, ignoreMissing bool) (*Properties, error) {
+ var buf bytes.Buffer
+ for _, u := range urls {
+ resp, err := http.Get(u)
+ if err != nil {
+ return nil, fmt.Errorf("properties: error fetching %q. %s", u, err)
+ }
+ if resp.StatusCode == 404 && ignoreMissing {
+ LogPrintf("properties: %s returned %d. skipping", u, resp.StatusCode)
+ continue
+ }
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("properties: %s returned %d", u, resp.StatusCode)
+ }
+ body, err := ioutil.ReadAll(resp.Body)
+ resp.Body.Close()
+ if err != nil {
+ return nil, fmt.Errorf("properties: %s error reading response. %s", u, err)
+ }
+
+ ct := resp.Header.Get("Content-Type")
+ var enc Encoding
+ switch strings.ToLower(ct) {
+ case "text/plain", "text/plain; charset=iso-8859-1", "text/plain; charset=latin1":
+ enc = ISO_8859_1
+ case "", "text/plain; charset=utf-8":
+ enc = UTF8
+ default:
+ return nil, fmt.Errorf("properties: invalid content type %s", ct)
+ }
+
+ buf.WriteString(convert(body, enc))
+ buf.WriteRune('\n')
+ }
+ return loadBuf(buf.Bytes(), UTF8)
}
func must(p *Properties, err error) *Properties {
diff --git a/load_test.go b/load_test.go
index 0983af6..f95b948 100644
--- a/load_test.go
+++ b/load_test.go
@@ -7,6 +7,8 @@
import (
"fmt"
"io/ioutil"
+ "net/http"
+ "net/http/httptest"
"os"
"strings"
@@ -73,6 +75,55 @@
assertKeyValues(c, "", p, "key", "value", "key2", "value2")
}
+func (s *LoadSuite) TestLoadURL(c *C) {
+ srv := testServer()
+ defer srv.Close()
+ p := MustLoadURL(srv.URL + "/a")
+ assertKeyValues(c, "", p, "key", "value")
+}
+
+func (s *LoadSuite) TestLoadURLs(c *C) {
+ srv := testServer()
+ defer srv.Close()
+ p := MustLoadURLs([]string{srv.URL + "/a", srv.URL + "/b"}, false)
+ assertKeyValues(c, "", p, "key", "value", "key2", "value2")
+}
+
+func (s *LoadSuite) TestLoadURLsAndFailMissing(c *C) {
+ srv := testServer()
+ defer srv.Close()
+ p, err := LoadURLs([]string{srv.URL + "/a", srv.URL + "/c"}, false)
+ c.Assert(p, IsNil)
+ c.Assert(err, ErrorMatches, ".*returned 404.*")
+}
+
+func (s *LoadSuite) TestLoadURLsAndIgnoreMissing(c *C) {
+ srv := testServer()
+ defer srv.Close()
+ p := MustLoadURLs([]string{srv.URL + "/a", srv.URL + "/b", srv.URL + "/c"}, true)
+ assertKeyValues(c, "", p, "key", "value", "key2", "value2")
+}
+
+func (s *LoadSuite) TestLoadURLEncoding(c *C) {
+ srv := testServer()
+ defer srv.Close()
+
+ uris := []string{"/none", "/utf8", "/plain", "/latin1", "/iso88591"}
+ for i, uri := range uris {
+ p := MustLoadURL(srv.URL + uri)
+ c.Assert(p.GetString("key", ""), Equals, "äöü", Commentf("%d", i))
+ }
+}
+
+func (s *LoadSuite) TestLoadURLFailInvalidEncoding(c *C) {
+ srv := testServer()
+ defer srv.Close()
+
+ p, err := LoadURL(srv.URL + "/json")
+ c.Assert(p, IsNil)
+ c.Assert(err, ErrorMatches, ".*invalid content type.*")
+}
+
func (s *LoadSuite) SetUpSuite(c *C) {
s.tempFiles = make([]string, 0)
}
@@ -118,3 +169,36 @@
return f.Name()
}
+
+func testServer() *httptest.Server {
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ send := func(data []byte, contentType string) {
+ w.Header().Set("Content-Type", contentType)
+ w.Write(data)
+ }
+
+ utf8 := []byte("key=äöü")
+ iso88591 := []byte{0x6b, 0x65, 0x79, 0x3d, 0xe4, 0xf6, 0xfc} // key=äöü
+
+ switch r.RequestURI {
+ case "/a":
+ send([]byte("key=value"), "")
+ case "/b":
+ send([]byte("key2=value2"), "")
+ case "/none":
+ send(utf8, "")
+ case "/utf8":
+ send(utf8, "text/plain; charset=utf-8")
+ case "/json":
+ send(utf8, "application/json; charset=utf-8")
+ case "/plain":
+ send(iso88591, "text/plain")
+ case "/latin1":
+ send(iso88591, "text/plain; charset=latin1")
+ case "/iso88591":
+ send(iso88591, "text/plain; charset=iso-8859-1")
+ default:
+ w.WriteHeader(404)
+ }
+ }))
+}
diff --git a/properties.go b/properties.go
index 90e2955..884ef4e 100644
--- a/properties.go
+++ b/properties.go
@@ -26,7 +26,11 @@
// ErrorHandler is the function which handles failures of the MustXXX()
// functions. The default is LogFatalHandler.
-var ErrorHandler = LogFatalHandler
+var ErrorHandler ErrorHandlerFunc = LogFatalHandler
+
+type LogHandlerFunc func(fmt string, args ...interface{})
+
+var LogPrintf LogHandlerFunc = log.Printf
// LogFatalHandler handles the error by logging a fatal error and exiting.
func LogFatalHandler(err error) {
