Merge pull request #37 from pelletier/pelletier/better_keys_parsing
Update keys parsing
diff --git a/keysparsing.go b/keysparsing.go
new file mode 100644
index 0000000..528f1de
--- /dev/null
+++ b/keysparsing.go
@@ -0,0 +1,56 @@
+// Parsing keys handling both bare and quoted keys.
+
+package toml
+
+import (
+ "bytes"
+ "fmt"
+ "unicode"
+)
+
+func parseKey(key string) ([]string, error) {
+ groups := []string{}
+ var buffer bytes.Buffer
+ inQuotes := false
+ escapeNext := false
+ for _, char := range key {
+ if escapeNext {
+ buffer.WriteRune(char)
+ escapeNext = false
+ continue
+ }
+ switch char {
+ case '\\':
+ escapeNext = true
+ continue
+ case '"':
+ inQuotes = !inQuotes
+ case '.':
+ if inQuotes {
+ buffer.WriteRune(char)
+ } else {
+ groups = append(groups, buffer.String())
+ buffer.Reset()
+ }
+ default:
+ if !inQuotes && !isValidBareChar(char) {
+ return nil, fmt.Errorf("invalid bare character: %c", char)
+ }
+ buffer.WriteRune(char)
+ }
+ }
+ if inQuotes {
+ return nil, fmt.Errorf("mismatched quotes")
+ }
+ if escapeNext {
+ return nil, fmt.Errorf("unfinished escape sequence")
+ }
+ if buffer.Len() > 0 {
+ groups = append(groups, buffer.String())
+ }
+ return groups, nil
+}
+
+func isValidBareChar(r rune) bool {
+ return isAlphanumeric(r) || r == '-' || unicode.IsNumber(r)
+}
diff --git a/keysparsing_test.go b/keysparsing_test.go
new file mode 100644
index 0000000..2d5379f
--- /dev/null
+++ b/keysparsing_test.go
@@ -0,0 +1,44 @@
+package toml
+
+import (
+ "fmt"
+ "testing"
+)
+
+func testResult(t *testing.T, key string, expected []string) {
+ parsed, err := parseKey(key)
+ if err != nil {
+ t.Fatal("Unexpected error:", err)
+ }
+ if len(expected) != len(parsed) {
+ t.Fatal("Expected length", len(expected), "but", len(parsed), "parsed")
+ }
+ for index, expectedKey := range expected {
+ if expectedKey != parsed[index] {
+ t.Fatal("Expected", expectedKey, "at index", index, "but found", parsed[index])
+ }
+ }
+}
+
+func testError(t *testing.T, key string, expectedError string) {
+ _, err := parseKey(key)
+ if fmt.Sprintf("%s", err) != expectedError {
+ t.Fatalf("Expected error \"%s\", but got \"%s\".", expectedError, err)
+ }
+}
+
+func TestBareKeyBasic(t *testing.T) {
+ testResult(t, "test", []string{"test"})
+}
+
+func TestBareKeyDotted(t *testing.T) {
+ testResult(t, "this.is.a.key", []string{"this", "is", "a", "key"})
+}
+
+func TestDottedKeyBasic(t *testing.T) {
+ testResult(t, "\"a.dotted.key\"", []string{"a.dotted.key"})
+}
+
+func TestBaseKeyPound(t *testing.T) {
+ testError(t, "hello#world", "invalid bare character: #")
+}
diff --git a/lexer.go b/lexer.go
index e6c3566..7cb789b 100644
--- a/lexer.go
+++ b/lexer.go
@@ -253,9 +253,16 @@
func (l *tomlLexer) lexKey() tomlLexStateFn {
l.ignore()
- for r := l.next(); isKeyChar(r); r = l.next() {
- if r == '#' {
- return l.errorf("keys cannot contain # character")
+ inQuotes := false
+ for r := l.next(); isKeyChar(r) || r == '\n'; r = l.next() {
+ if r == '"' {
+ inQuotes = !inQuotes
+ } else if r == '\n' {
+ return l.errorf("keys cannot contain new lines")
+ } else if isSpace(r) && !inQuotes {
+ break
+ } else if !isValidBareChar(r) && !inQuotes {
+ return l.errorf("keys cannot contain %c character", r)
}
}
l.backup()
diff --git a/lexer_test.go b/lexer_test.go
index 683d7f7..d1cd130 100644
--- a/lexer_test.go
+++ b/lexer_test.go
@@ -124,10 +124,7 @@
func TestKeyWithSymbolsAndEqual(t *testing.T) {
testFlow(t, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
- token{Position{1, 1}, tokenKey, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:'"},
- token{Position{1, 38}, tokenEqual, "="},
- token{Position{1, 40}, tokenInteger, "5"},
- token{Position{1, 41}, tokenEOF, ""},
+ token{Position{1, 1}, tokenError, "keys cannot contain ~ character"},
})
}
@@ -549,3 +546,18 @@
token{Position{1, 8}, tokenEOF, ""},
})
}
+
+func TestQuotedKey(t *testing.T) {
+ testFlow(t, "\"a b\" = 42", []token{
+ token{Position{1, 1}, tokenKey, "\"a b\""},
+ token{Position{1, 7}, tokenEqual, "="},
+ token{Position{1, 9}, tokenInteger, "42"},
+ token{Position{1, 11}, tokenEOF, ""},
+ })
+}
+
+func TestKeyNewline(t *testing.T) {
+ testFlow(t, "a\n= 4", []token{
+ token{Position{1, 1}, tokenError, "keys cannot contain new lines"},
+ })
+}
diff --git a/parser.go b/parser.go
index e83647d..e550967 100644
--- a/parser.go
+++ b/parser.go
@@ -98,7 +98,10 @@
}
// get or create group array element at the indicated part in the path
- keys := strings.Split(key.val, ".")
+ keys, err := parseKey(key.val)
+ if err != nil {
+ p.raiseError(key, "invalid group array key: %s", err)
+ }
p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries
destTree := p.tree.GetPath(keys)
var array []*TomlTree
@@ -153,7 +156,10 @@
}
p.seenGroupKeys = append(p.seenGroupKeys, key.val)
- keys := strings.Split(key.val, ".")
+ keys, err := parseKey(key.val)
+ if err != nil {
+ p.raiseError(key, "invalid group array key: %s", err)
+ }
if err := p.tree.createSubTree(keys, startToken.Position); err != nil {
p.raiseError(key, "%s", err)
}
@@ -186,13 +192,21 @@
}
// assign value to the found group
- localKey := []string{key.val}
- finalKey := append(groupKey, key.val)
+ keyVals, err := parseKey(key.val)
+ if err != nil {
+ p.raiseError(key, "%s", err)
+ }
+ if len(keyVals) != 1 {
+ p.raiseError(key, "Invalid key")
+ }
+ keyVal := keyVals[0]
+ localKey := []string{keyVal}
+ finalKey := append(groupKey, keyVal)
if targetNode.GetPath(localKey) != nil {
p.raiseError(key, "The following key was defined twice: %s",
strings.Join(finalKey, "."))
}
- targetNode.values[key.val] = &tomlValue{value, key.Position}
+ targetNode.values[keyVal] = &tomlValue{value, key.Position}
return p.parseStart
}
@@ -222,7 +236,7 @@
}
return val
case tokenDate:
- val, err := time.Parse(time.RFC3339Nano, tok.val)
+ val, err := time.ParseInLocation(time.RFC3339Nano, tok.val, time.UTC)
if err != nil {
p.raiseError(tok, "%s", err)
}
diff --git a/parser_test.go b/parser_test.go
index dc051ea..2998913 100644
--- a/parser_test.go
+++ b/parser_test.go
@@ -51,12 +51,10 @@
})
}
-// NOTE: from the BurntSushi test suite
-// NOTE: this test is pure evil due to the embedded '.'
-func TestSpecialKV(t *testing.T) {
- tree, err := Load("~!@$^&*()_+-`1234567890[]\\|/?><.,;: = 1")
+func TestNumberInKey(t *testing.T) {
+ tree, err := Load("hello2 = 42")
assertTree(t, tree, err, map[string]interface{}{
- "~!@$^&*()_+-`1234567890[]\\|/?><.,;:": int64(1),
+ "hello2": int64(42),
})
}
@@ -91,14 +89,14 @@
func TestDateOffset(t *testing.T) {
tree, err := Load("a = 1979-05-27T00:32:00-07:00")
assertTree(t, tree, err, map[string]interface{}{
- "a": time.Date(1979, time.May, 27, 0, 32, 0, 0, time.FixedZone("PDT", -7*60*60)),
+ "a": time.Date(1979, time.May, 27, 0, 32, 0, 0, time.FixedZone("", -7*60*60)),
})
}
func TestDateNano(t *testing.T) {
tree, err := Load("a = 1979-05-27T00:32:00.999999999-07:00")
assertTree(t, tree, err, map[string]interface{}{
- "a": time.Date(1979, time.May, 27, 0, 32, 0, 999999999, time.FixedZone("PDT", -7*60*60)),
+ "a": time.Date(1979, time.May, 27, 0, 32, 0, 999999999, time.FixedZone("", -7*60*60)),
})
}
@@ -109,6 +107,13 @@
})
}
+func TestSpaceKey(t *testing.T) {
+ tree, err := Load("\"a b\" = \"hello world\"")
+ assertTree(t, tree, err, map[string]interface{}{
+ "a b": "hello world",
+ })
+}
+
func TestStringEscapables(t *testing.T) {
tree, err := Load("a = \"a \\n b\"")
assertTree(t, tree, err, map[string]interface{}{
@@ -461,3 +466,10 @@
"foo.bar.b": Position{3, 1},
})
}
+
+func TestInvalidGroupArray(t *testing.T) {
+ _, err := Load("[key#group]\nanswer = 42")
+ if err == nil {
+ t.Error("Should error")
+ }
+}
diff --git a/token.go b/token.go
index 6c0bcc6..f0fbac9 100644
--- a/token.go
+++ b/token.go
@@ -120,7 +120,7 @@
// Keys start with the first character that isn't whitespace or [ and end
// with the last non-whitespace character before the equals sign. Keys
// cannot contain a # character."
- return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '=')
+ return !(r == '\r' || r == '\n' || r == eof || r == '=')
}
func isKeyStartChar(r rune) bool {
diff --git a/toml.go b/toml.go
index 2896da7..bf35cd3 100644
--- a/toml.go
+++ b/toml.go
@@ -65,7 +65,11 @@
if key == "" {
return t
}
- return t.GetPath(strings.Split(key, "."))
+ comps, err := parseKey(key)
+ if err != nil {
+ return nil
+ }
+ return t.GetPath(comps)
}
// GetPath returns the element in the tree indicated by 'keys'.
