Merge pull request #14 from pelletier/pelletier/go_test_support
Add BurntSushi's test suite
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..f1b6190
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+test_program/test_program_bin
diff --git a/.travis.yml b/.travis.yml
index 17a616a..ea2966c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,5 @@
language: go
+script: "./test.sh"
go:
- 1.0
- 1.1
diff --git a/README.md b/README.md
index 334d927..756ed51 100644
--- a/README.md
+++ b/README.md
@@ -56,6 +56,14 @@
[pelletier/go-toml](https://github.com/pelletier/go-toml). Any feedback would be
much appreciated!
+### Run tests
+
+You have to make sure two kind of tests run:
+
+1. The Go unit tests: `go test`
+2. The TOML examples base: `./test_program/go-test.sh`
+
+You can run both of them using `./test.sh`.
## License
diff --git a/lexer.go b/lexer.go
index 2593466..5dfff38 100644
--- a/lexer.go
+++ b/lexer.go
@@ -68,7 +68,9 @@
}
func isKeyChar(r rune) bool {
- return isAlphanumeric(r) || r == '-'
+ // "Keys start with the first non-whitespace character and end with the last
+ // non-whitespace character before the equals sign."
+ return !(isSpace(r) || r == '\n' || r == eof || r == '=')
}
func isDigit(r rune) bool {
@@ -167,10 +169,6 @@
return lexEqual
}
- if isAlphanumeric(next) {
- return lexKey
- }
-
if isSpace(next) {
l.ignore()
}
@@ -179,6 +177,10 @@
return lexRvalue
}
+ if isKeyChar(next) {
+ return lexKey
+ }
+
if l.next() == eof {
break
}
@@ -192,6 +194,10 @@
for {
next := l.peek()
switch next {
+ case '.':
+ return l.errorf("cannot start float with a dot")
+ case '=':
+ return l.errorf("cannot have multiple equals for the same key")
case '[':
l.depth += 1
return lexLeftBracket
@@ -328,6 +334,15 @@
} else if l.follow("\\n") {
l.pos += 1
growing_string += "\n"
+ } else if l.follow("\\b") {
+ l.pos += 1
+ growing_string += "\b"
+ } else if l.follow("\\f") {
+ l.pos += 1
+ growing_string += "\f"
+ } else if l.follow("\\/") {
+ l.pos += 1
+ growing_string += "/"
} else if l.follow("\\t") {
l.pos += 1
growing_string += "\t"
@@ -354,6 +369,9 @@
return l.errorf("invalid unicode escape: \\u" + code)
}
growing_string += string(rune(intcode))
+ } else if l.follow("\\") {
+ l.pos += 1
+ return l.errorf("invalid escape sequence: \\" + string(l.peek()))
} else {
growing_string += string(l.peek())
}
@@ -409,6 +427,12 @@
for {
next := l.next()
if next == '.' {
+ if point_seen {
+ return l.errorf("cannot have two dots in one float")
+ }
+ if !isDigit(l.peek()) {
+ return l.errorf("float cannot end with a dot")
+ }
point_seen = true
} else if isDigit(next) {
digit_seen = true
@@ -416,6 +440,9 @@
l.backup()
break
}
+ if point_seen && !digit_seen {
+ return l.errorf("cannot start float with a dot")
+ }
}
if !digit_seen {
diff --git a/lexer_test.go b/lexer_test.go
index e5846bd..8582520 100644
--- a/lexer_test.go
+++ b/lexer_test.go
@@ -113,6 +113,23 @@
})
}
+func TestKeyWithSharpAndEqual(t *testing.T) {
+ testFlow(t, "key#name = 5", []token{
+ token{tokenKey, "key#name"},
+ token{tokenEqual, "="},
+ token{tokenInteger, "5"},
+ token{tokenEOF, ""},
+ })
+}
+func TestKeyWithSymbolsAndEqual(t *testing.T) {
+ testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
+ token{tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'"},
+ token{tokenEqual, "="},
+ token{tokenInteger, "5"},
+ token{tokenEOF, ""},
+ })
+}
+
func TestKeyEqualStringEscape(t *testing.T) {
testFlow(t, "foo = \"hello\\\"\"", []token{
token{tokenKey, "foo"},
@@ -268,6 +285,52 @@
})
}
+func TestFloatEndingWithDot(t *testing.T) {
+ testFlow(t, "foo = 42.", []token{
+ token{tokenKey, "foo"},
+ token{tokenEqual, "="},
+ token{tokenError, "float cannot end with a dot"},
+ })
+}
+
+func TestFloatWithTwoDots(t *testing.T) {
+ testFlow(t, "foo = 4.2.", []token{
+ token{tokenKey, "foo"},
+ token{tokenEqual, "="},
+ token{tokenError, "cannot have two dots in one float"},
+ })
+}
+
+func TestDoubleEqualKey(t *testing.T) {
+ testFlow(t, "foo= = 2", []token{
+ token{tokenKey, "foo"},
+ token{tokenEqual, "="},
+ token{tokenError, "cannot have multiple equals for the same key"},
+ })
+}
+
+func TestInvalidEsquapeSequence(t *testing.T) {
+ testFlow(t, "foo = \"\\x\"", []token{
+ token{tokenKey, "foo"},
+ token{tokenEqual, "="},
+ token{tokenError, "invalid escape sequence: \\x"},
+ })
+}
+
+func TestNestedArrays(t *testing.T) {
+ testFlow(t, "foo = [[[]]]", []token{
+ token{tokenKey, "foo"},
+ token{tokenEqual, "="},
+ token{tokenLeftBracket, "["},
+ token{tokenLeftBracket, "["},
+ token{tokenLeftBracket, "["},
+ token{tokenRightBracket, "]"},
+ token{tokenRightBracket, "]"},
+ token{tokenRightBracket, "]"},
+ token{tokenEOF, ""},
+ })
+}
+
func TestKeyEqualNumber(t *testing.T) {
testFlow(t, "foo = 42", []token{
token{tokenKey, "foo"},
diff --git a/parser.go b/parser.go
index e9c9f52..672bb35 100644
--- a/parser.go
+++ b/parser.go
@@ -4,15 +4,18 @@
import (
"fmt"
+ "reflect"
"strconv"
+ "strings"
"time"
)
type parser struct {
- flow chan token
- tree *TomlTree
- tokensBuffer []token
- currentGroup string
+ flow chan token
+ tree *TomlTree
+ tokensBuffer []token
+ currentGroup []string
+ seenGroupKeys []string
}
type parserStateFn func(*parser) parserStateFn
@@ -86,9 +89,15 @@
if key.typ != tokenKeyGroup {
panic(fmt.Sprintf("unexpected token %s, was expecting a key group", key))
}
+ for _, item := range p.seenGroupKeys {
+ if item == key.val {
+ panic("duplicated tables")
+ }
+ }
+ p.seenGroupKeys = append(p.seenGroupKeys, key.val)
p.tree.createSubTree(key.val)
p.assume(tokenRightBracket)
- p.currentGroup = key.val
+ p.currentGroup = strings.Split(key.val, ".")
return parseStart(p)
}
@@ -96,11 +105,17 @@
key := p.getToken()
p.assume(tokenEqual)
value := parseRvalue(p)
- final_key := key.val
- if p.currentGroup != "" {
- final_key = p.currentGroup + "." + key.val
+ var final_key []string
+ if len(p.currentGroup) > 0 {
+ final_key = p.currentGroup
+ } else {
+ final_key = make([]string, 0)
}
- p.tree.Set(final_key, value)
+ final_key = append(final_key, key.val)
+ if p.tree.GetPath(final_key) != nil {
+ panic(fmt.Sprintf("the following key was defined twice: %s", strings.Join(final_key, ".")))
+ }
+ p.tree.SetPath(final_key, value)
return parseStart(p)
}
@@ -137,6 +152,8 @@
return val
case tokenLeftBracket:
return parseArray(p)
+ case tokenError:
+ panic(tok.val)
}
panic("never reached")
@@ -146,6 +163,7 @@
func parseArray(p *parser) []interface{} {
array := make([]interface{}, 0)
+ arrayType := reflect.TypeOf(nil)
for {
follow := p.peek()
if follow == nil || follow.typ == tokenEOF {
@@ -156,14 +174,18 @@
return array
}
val := parseRvalue(p)
+ if arrayType == nil {
+ arrayType = reflect.TypeOf(val)
+ }
+ if reflect.TypeOf(val) != arrayType {
+ panic("mixed types in array")
+ }
array = append(array, val)
follow = p.peek()
if follow == nil {
panic("unterminated array")
}
if follow.typ != tokenRightBracket && follow.typ != tokenComma {
- fmt.Println(follow.typ)
- fmt.Println(follow.val)
panic("missing comma")
}
if follow.typ == tokenComma {
@@ -176,10 +198,11 @@
func parse(flow chan token) *TomlTree {
result := make(TomlTree)
parser := &parser{
- flow: flow,
- tree: &result,
- tokensBuffer: make([]token, 0),
- currentGroup: "",
+ flow: flow,
+ tree: &result,
+ tokensBuffer: make([]token, 0),
+ currentGroup: make([]string, 0),
+ seenGroupKeys: make([]string, 0),
}
parser.run()
return parser.tree
diff --git a/parser_test.go b/parser_test.go
index 629e0a6..238c3c5 100644
--- a/parser_test.go
+++ b/parser_test.go
@@ -12,9 +12,15 @@
return
}
for k, v := range ref {
- if fmt.Sprintf("%v", tree.Get(k)) != fmt.Sprintf("%v", v) {
- t.Log("was expecting", v, "at", k, "but got", tree.Get(k))
- t.Error()
+ node := tree.Get(k)
+ switch node.(type) {
+ case *TomlTree:
+ assertTree(t, node.(*TomlTree), err, v.(map[string]interface{}))
+ default:
+ if fmt.Sprintf("%v", node) != fmt.Sprintf("%v", v) {
+ t.Log("was expecting", v, "at", k, "but got", node)
+ t.Error()
+ }
}
}
}
@@ -142,6 +148,26 @@
})
}
+func TestNestedEmptyArrays(t *testing.T) {
+ tree, err := Load("a = [[[]]]")
+ assertTree(t, tree, err, map[string]interface{}{
+ "a": [][][]interface{}{[][]interface{}{[]interface{}{}}},
+ })
+}
+
+
+func TestArrayMixedTypes(t *testing.T) {
+ _, err := Load("a = [42, 16.0]")
+ if err.Error() != "mixed types in array" {
+ t.Error("Bad error message:", err.Error())
+ }
+
+ _, err = Load("a = [42, \"hello\"]")
+ if err.Error() != "mixed types in array" {
+ t.Error("Bad error message:", err.Error())
+ }
+}
+
func TestArrayNestedStrings(t *testing.T) {
tree, err := Load("data = [ [\"gamma\", \"delta\"], [\"Foo\"] ]")
assertTree(t, tree, err, map[string]interface{}{
@@ -184,6 +210,53 @@
})
}
+func TestDuplicateGroups(t *testing.T) {
+ _, err := Load("[foo]\na=2\n[foo]b=3")
+ if err.Error() != "duplicated tables" {
+ t.Error("Bad error message:", err.Error())
+ }
+}
+
+func TestDuplicateKeys(t *testing.T) {
+ _, err := Load("foo = 2\nfoo = 3")
+ if err.Error() != "the following key was defined twice: foo" {
+ t.Error("Bad error message:", err.Error())
+ }
+}
+
+func TestEmptyIntermediateTable(t *testing.T) {
+ _, err := Load("[foo..bar]")
+ if err.Error() != "empty intermediate table" {
+ t.Error("Bad error message:", err.Error())
+ }
+}
+
+func TestImplicitDeclarationBefore(t *testing.T) {
+ tree, err := Load("[a.b.c]\nanswer = 42\n[a]\nbetter = 43")
+ assertTree(t, tree, err, map[string]interface{}{
+ "a": map[string]interface{}{
+ "b": map[string]interface{}{
+ "c": map[string]interface{}{
+ "answer": int64(42),
+ },
+ },
+ "better": int64(43),
+ },
+ })
+}
+
+func TestFloatsWithoutLeadingZeros(t *testing.T) {
+ _, err := Load("a = .42")
+ if err.Error() != "cannot start float with a dot" {
+ t.Error("Bad error message:", err.Error())
+ }
+
+ _, err = Load("a = -.42")
+ if err.Error() != "cannot start float with a dot" {
+ t.Error("Bad error message:", err.Error())
+ }
+}
+
func TestMissingFile(t *testing.T) {
_, err := LoadFile("foo.toml")
if err.Error() != "open foo.toml: no such file or directory" {
diff --git a/test.sh b/test.sh
new file mode 100755
index 0000000..b6ed4b4
--- /dev/null
+++ b/test.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Run basic go unit tests
+go test -v ./...
+result=$?
+
+# Run example-based toml tests
+cd test_program && ./go-test.sh
+result="$(( result || $? ))"
+
+exit $result
diff --git a/test_program/go-test.sh b/test_program/go-test.sh
new file mode 100755
index 0000000..71293a3
--- /dev/null
+++ b/test_program/go-test.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+go get github.com/BurntSushi/toml-test # install test suite
+go get github.com/BurntSushi/toml/toml-test-go # install my parser
+go build -o test_program_bin github.com/pelletier/go-toml/test_program
+
+toml_test_wrapper() {
+ ret=0
+ if hash toml-test 2>/dev/null; then # test availability in $PATH
+ toml-test "$@"
+ ret=$?
+ else
+ p="$HOME/gopath/bin/toml-test" # try in Travi's place
+ if [ -f "$p" ]; then
+ "$p" "$@"
+ ret=$?
+ else
+ "$GOPATH/bin/toml-test" "$@"
+ ret=$?
+ fi
+ fi
+}
+
+toml_test_wrapper ./test_program_bin | tee test_out
+ret="$([ `tail -n 1 test_out | sed -E 's/^.+([0-9]+) failed$/\1/'` -eq 0 ])"
+exit $ret
diff --git a/test_program/test_program.go b/test_program/test_program.go
new file mode 100644
index 0000000..8912c05
--- /dev/null
+++ b/test_program/test_program.go
@@ -0,0 +1,75 @@
+package main
+
+import (
+ "io/ioutil"
+ "os"
+ "github.com/pelletier/go-toml"
+ "encoding/json"
+ "fmt"
+ "log"
+ "time"
+)
+
+func main() {
+ bytes, err := ioutil.ReadAll(os.Stdin)
+ if err != nil {
+ os.Exit(2)
+ }
+ tree, err := toml.Load(string(bytes))
+ if err != nil {
+ os.Exit(1)
+ }
+
+ typedTree := translate((map[string]interface{})(*tree))
+
+ if err := json.NewEncoder(os.Stdout).Encode(typedTree); err != nil {
+ log.Fatalf("Error encoding JSON: %s", err)
+ }
+
+ os.Exit(0)
+}
+
+func translate(tomlData interface{}) interface{} {
+
+ switch orig := tomlData.(type) {
+ case map[string]interface{}:
+ typed := make(map[string]interface{}, len(orig))
+ for k, v := range orig {
+ typed[k] = translate(v)
+ }
+ return typed
+ case *toml.TomlTree:
+ return translate((map[string]interface{})(*orig))
+ case []map[string]interface{}:
+ typed := make([]map[string]interface{}, len(orig))
+ for i, v := range orig {
+ typed[i] = translate(v).(map[string]interface{})
+ }
+ return typed
+ case []interface{}:
+ typed := make([]interface{}, len(orig))
+ for i, v := range orig {
+ typed[i] = translate(v)
+ }
+ return tag("array", typed)
+ case time.Time:
+ return tag("datetime", orig.Format("2006-01-02T15:04:05Z"))
+ case bool:
+ return tag("bool", fmt.Sprintf("%v", orig))
+ case int64:
+ return tag("integer", fmt.Sprintf("%d", orig))
+ case float64:
+ return tag("float", fmt.Sprintf("%v", orig))
+ case string:
+ return tag("string", orig)
+ }
+
+ panic(fmt.Sprintf("Unknown type: %T", tomlData))
+}
+
+func tag(typeName string, data interface{}) map[string]interface{} {
+ return map[string]interface{}{
+ "type": typeName,
+ "value": data,
+ }
+}
diff --git a/toml.go b/toml.go
index 410bab6..dbf8ae8 100644
--- a/toml.go
+++ b/toml.go
@@ -15,6 +15,18 @@
// This is the result of the parsing of a TOML file.
type TomlTree map[string]interface{}
+// Has returns a boolean indicating if the toplevel tree contains the given
+// key.
+func (t *TomlTree) Has(key string) bool {
+ mp := (map[string]interface{})(*t)
+ for k, _ := range mp {
+ if k == key {
+ return true
+ }
+ }
+ return false
+}
+
// Keys returns the keys of the toplevel tree.
// Warning: this is a costly operation.
func (t *TomlTree) Keys() []string {
@@ -30,8 +42,11 @@
// Key is a dot-separated path (e.g. a.b.c).
// Returns nil if the path does not exist in the tree.
func (t *TomlTree) Get(key string) interface{} {
+ return t.GetPath(strings.Split(key, "."))
+}
+
+func (t *TomlTree) GetPath(keys []string) interface{} {
subtree := t
- keys := strings.Split(key, ".")
for _, intermediate_key := range keys[:len(keys)-1] {
_, exists := (*subtree)[intermediate_key]
if !exists {
@@ -55,8 +70,11 @@
// Key is a dot-separated path (e.g. a.b.c).
// Creates all necessary intermediates trees, if needed.
func (t *TomlTree) Set(key string, value interface{}) {
+ t.SetPath(strings.Split(key, "."), value)
+}
+
+func (t *TomlTree) SetPath(keys []string, value interface{}) {
subtree := t
- keys := strings.Split(key, ".")
for _, intermediate_key := range keys[:len(keys)-1] {
_, exists := (*subtree)[intermediate_key]
if !exists {
@@ -76,6 +94,9 @@
func (t *TomlTree) createSubTree(key string) {
subtree := t
for _, intermediate_key := range strings.Split(key, ".") {
+ if intermediate_key == "" {
+ panic("empty intermediate table")
+ }
_, exists := (*subtree)[intermediate_key]
if !exists {
var new_tree TomlTree = make(TomlTree)
@@ -109,6 +130,5 @@
s := string(buff)
tree, err = Load(s)
}
-
return
}
