Issue #10: Support loading from urls
diff --git a/CHANGELOG.md b/CHANGELOG.md index b82f095..a34b828 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md
@@ -2,7 +2,9 @@ ### [1.6.1](https://github.com/magiconair/properties/tags/v1.6.1) - unreleased - * [Issue #11](https://github.com/magiconair/properties/issues/11): Add [LoadString](http://godoc.org/github.com/magiconair/properties#Properties.LoadString) method to load properties from an UTF8 string. + * [Issue #10](https://github.com/magiconair/properties/issues/10): Add [LoadURL,LoadURLs,MustLoadURL,MustLoadURLs](http://godoc.org/github.com/magiconair/properties#Properties.LoadURL) method to load properties from a URL. + * [Issue #11](https://github.com/magiconair/properties/issues/11): Add [LoadString,MustLoadString](http://godoc.org/github.com/magiconair/properties#Properties.LoadString) method to load properties from an UTF8 string. + * [PR #8](https://github.com/magiconair/properties/pull/8): Add [MustFlag](http://godoc.org/github.com/magiconair/properties#Properties.MustFlag) method to provide overrides via command line flags. (@pascaldekloe) ### [1.6.0](https://github.com/magiconair/properties/tags/v1.6.0) - 11 Dec 2015 @@ -14,7 +16,7 @@ ### [1.5.5](https://github.com/magiconair/properties/tags/v1.5.5) - 31 Jul 2015 - * [Pull Request #6](https://github.com/magiconair/properties/pull/6): Add [Delete](http://godoc.org/github.com/magiconair/properties#Properties.Delete) method to remove keys including comments. (@gerbenjacobs) + * [PR #6](https://github.com/magiconair/properties/pull/6): Add [Delete](http://godoc.org/github.com/magiconair/properties#Properties.Delete) method to remove keys including comments. (@gerbenjacobs) ### [1.5.4](https://github.com/magiconair/properties/tags/v1.5.4) - 23 Jun 2015
diff --git a/README.md b/README.md index b3b2e5c..1dd13d7 100644 --- a/README.md +++ b/README.md
@@ -5,12 +5,11 @@ properties is a Go library for reading and writing properties files. -It supports reading from multiple files and Spring style recursive property -expansion of expressions like `${key}` to their corresponding value. -Value expressions can refer to other keys like in `${key}` or to -environment variables like in `${USER}`. -Filenames can also contain environment variables like in -`/home/${USER}/myapp.properties`. +It supports reading from multiple files or URLs and Spring style recursive +property expansion of expressions like `${key}` to their corresponding value. +Value expressions can refer to other keys like in `${key}` or to environment +variables like in `${USER}`. Filenames can also contain environment variables +like in `/home/${USER}/myapp.properties`. Properties can be decoded into structs, maps, arrays and values through struct tags. @@ -56,6 +55,9 @@ // or via flags p.MustFlag(flag.CommandLine) + + // or via url + p = properties.MustLoadURL("http://host/path") } ```
diff --git a/load.go b/load.go index e74d77b..3915c73 100644 --- a/load.go +++ b/load.go
@@ -5,9 +5,12 @@ package properties import ( + "bytes" "fmt" "io/ioutil" + "net/http" "os" + "strings" ) // Encoding specifies encoding of the input data. @@ -43,6 +46,26 @@ return loadFiles(filenames, enc, ignoreMissing) } +// LoadURL reads the content of the URL into a Properties struct. +// +// The encoding is determined via the Content-Type header which +// should be set to 'text/plain'. If the 'charset' parameter is +// missing, 'iso-8859-1' or 'latin1' the encoding is set to +// ISO-8859-1. If the 'charset' parameter is set to 'utf-8' the +// encoding is set to UTF-8. A missing content type header is +// interpreted as 'text/plain; charset=utf-8'. +func LoadURL(url string) (*Properties, error) { + return loadURLs([]string{url}, false) +} + +// LoadURLs reads the content of multiple URLs in the given order into a +// Properties struct. If 'ignoreMissing' is true then a 404 status code will +// not be reported as error. See LoadURL for the Content-Type header +// and the encoding. +func LoadURLs(urls []string, ignoreMissing bool) (*Properties, error) { + return loadURLs(urls, ignoreMissing) +} + // MustLoadString reads an UTF8 string into a Properties struct and // panics on error. func MustLoadString(s string) *Properties { @@ -62,6 +85,19 @@ return must(LoadFiles(filenames, enc, ignoreMissing)) } +// MustLoadURL reads the content of a URL into a Properties struct and +// panics on error. +func MustLoadURL(url string) *Properties { + return must(LoadURL(url)) +} + +// MustLoadFiles reads the content of multiple URLs in the given order into a +// Properties struct and panics on error. If 'ignoreMissing' is true then a 404 +// status code will not be reported as error. +func MustLoadURLs(urls []string, ignoreMissing bool) *Properties { + return must(LoadURLs(urls, ignoreMissing)) +} + func loadBuf(buf []byte, enc Encoding) (*Properties, error) { p, err := parse(convert(buf, enc)) if err != nil { @@ -71,18 +107,17 @@ } func loadFiles(filenames []string, enc Encoding, ignoreMissing bool) (*Properties, error) { - buff := make([]byte, 0, 4096) - + var buf bytes.Buffer for _, filename := range filenames { f, err := expandFilename(filename) if err != nil { return nil, err } - buf, err := ioutil.ReadFile(f) + data, err := ioutil.ReadFile(f) if err != nil { if ignoreMissing && os.IsNotExist(err) { - // TODO(frank): should we log that we are skipping the file? + LogPrintf("properties: %s not found. skipping", filename) continue } return nil, err @@ -90,10 +125,47 @@ // concatenate the buffers and add a new line in case // the previous file didn't end with a new line - buff = append(append(buff, buf...), '\n') + buf.Write(data) + buf.WriteRune('\n') } + return loadBuf(buf.Bytes(), enc) +} - return loadBuf(buff, enc) +func loadURLs(urls []string, ignoreMissing bool) (*Properties, error) { + var buf bytes.Buffer + for _, u := range urls { + resp, err := http.Get(u) + if err != nil { + return nil, fmt.Errorf("properties: error fetching %q. %s", u, err) + } + if resp.StatusCode == 404 && ignoreMissing { + LogPrintf("properties: %s returned %d. skipping", u, resp.StatusCode) + continue + } + if resp.StatusCode != 200 { + return nil, fmt.Errorf("properties: %s returned %d", u, resp.StatusCode) + } + body, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return nil, fmt.Errorf("properties: %s error reading response. %s", u, err) + } + + ct := resp.Header.Get("Content-Type") + var enc Encoding + switch strings.ToLower(ct) { + case "text/plain", "text/plain; charset=iso-8859-1", "text/plain; charset=latin1": + enc = ISO_8859_1 + case "", "text/plain; charset=utf-8": + enc = UTF8 + default: + return nil, fmt.Errorf("properties: invalid content type %s", ct) + } + + buf.WriteString(convert(body, enc)) + buf.WriteRune('\n') + } + return loadBuf(buf.Bytes(), UTF8) } func must(p *Properties, err error) *Properties {
diff --git a/load_test.go b/load_test.go index 0983af6..f95b948 100644 --- a/load_test.go +++ b/load_test.go
@@ -7,6 +7,8 @@ import ( "fmt" "io/ioutil" + "net/http" + "net/http/httptest" "os" "strings" @@ -73,6 +75,55 @@ assertKeyValues(c, "", p, "key", "value", "key2", "value2") } +func (s *LoadSuite) TestLoadURL(c *C) { + srv := testServer() + defer srv.Close() + p := MustLoadURL(srv.URL + "/a") + assertKeyValues(c, "", p, "key", "value") +} + +func (s *LoadSuite) TestLoadURLs(c *C) { + srv := testServer() + defer srv.Close() + p := MustLoadURLs([]string{srv.URL + "/a", srv.URL + "/b"}, false) + assertKeyValues(c, "", p, "key", "value", "key2", "value2") +} + +func (s *LoadSuite) TestLoadURLsAndFailMissing(c *C) { + srv := testServer() + defer srv.Close() + p, err := LoadURLs([]string{srv.URL + "/a", srv.URL + "/c"}, false) + c.Assert(p, IsNil) + c.Assert(err, ErrorMatches, ".*returned 404.*") +} + +func (s *LoadSuite) TestLoadURLsAndIgnoreMissing(c *C) { + srv := testServer() + defer srv.Close() + p := MustLoadURLs([]string{srv.URL + "/a", srv.URL + "/b", srv.URL + "/c"}, true) + assertKeyValues(c, "", p, "key", "value", "key2", "value2") +} + +func (s *LoadSuite) TestLoadURLEncoding(c *C) { + srv := testServer() + defer srv.Close() + + uris := []string{"/none", "/utf8", "/plain", "/latin1", "/iso88591"} + for i, uri := range uris { + p := MustLoadURL(srv.URL + uri) + c.Assert(p.GetString("key", ""), Equals, "äöü", Commentf("%d", i)) + } +} + +func (s *LoadSuite) TestLoadURLFailInvalidEncoding(c *C) { + srv := testServer() + defer srv.Close() + + p, err := LoadURL(srv.URL + "/json") + c.Assert(p, IsNil) + c.Assert(err, ErrorMatches, ".*invalid content type.*") +} + func (s *LoadSuite) SetUpSuite(c *C) { s.tempFiles = make([]string, 0) } @@ -118,3 +169,36 @@ return f.Name() } + +func testServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + send := func(data []byte, contentType string) { + w.Header().Set("Content-Type", contentType) + w.Write(data) + } + + utf8 := []byte("key=äöü") + iso88591 := []byte{0x6b, 0x65, 0x79, 0x3d, 0xe4, 0xf6, 0xfc} // key=äöü + + switch r.RequestURI { + case "/a": + send([]byte("key=value"), "") + case "/b": + send([]byte("key2=value2"), "") + case "/none": + send(utf8, "") + case "/utf8": + send(utf8, "text/plain; charset=utf-8") + case "/json": + send(utf8, "application/json; charset=utf-8") + case "/plain": + send(iso88591, "text/plain") + case "/latin1": + send(iso88591, "text/plain; charset=latin1") + case "/iso88591": + send(iso88591, "text/plain; charset=iso-8859-1") + default: + w.WriteHeader(404) + } + })) +}
diff --git a/properties.go b/properties.go index 90e2955..884ef4e 100644 --- a/properties.go +++ b/properties.go
@@ -26,7 +26,11 @@ // ErrorHandler is the function which handles failures of the MustXXX() // functions. The default is LogFatalHandler. -var ErrorHandler = LogFatalHandler +var ErrorHandler ErrorHandlerFunc = LogFatalHandler + +type LogHandlerFunc func(fmt string, args ...interface{}) + +var LogPrintf LogHandlerFunc = log.Printf // LogFatalHandler handles the error by logging a fatal error and exiting. func LogFatalHandler(err error) {