Merge branch 'master' of github.com:pelletier/go-toml
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f1b6190 --- /dev/null +++ b/.gitignore
@@ -0,0 +1 @@ +test_program/test_program_bin
diff --git a/.travis.yml b/.travis.yml index 90958b0..23b0214 100644 --- a/.travis.yml +++ b/.travis.yml
@@ -1,4 +1,5 @@ language: go +script: "./test.sh" go: - 1.0 - 1.1
diff --git a/README.md b/README.md index 334d927..756ed51 100644 --- a/README.md +++ b/README.md
@@ -56,6 +56,14 @@ [pelletier/go-toml](https://github.com/pelletier/go-toml). Any feedback would be much appreciated! +### Run tests + +You have to make sure two kind of tests run: + +1. The Go unit tests: `go test` +2. The TOML examples base: `./test_program/go-test.sh` + +You can run both of them using `./test.sh`. ## License
diff --git a/lexer.go b/lexer.go index 2593466..5dfff38 100644 --- a/lexer.go +++ b/lexer.go
@@ -68,7 +68,9 @@ } func isKeyChar(r rune) bool { - return isAlphanumeric(r) || r == '-' + // "Keys start with the first non-whitespace character and end with the last + // non-whitespace character before the equals sign." + return !(isSpace(r) || r == '\n' || r == eof || r == '=') } func isDigit(r rune) bool { @@ -167,10 +169,6 @@ return lexEqual } - if isAlphanumeric(next) { - return lexKey - } - if isSpace(next) { l.ignore() } @@ -179,6 +177,10 @@ return lexRvalue } + if isKeyChar(next) { + return lexKey + } + if l.next() == eof { break } @@ -192,6 +194,10 @@ for { next := l.peek() switch next { + case '.': + return l.errorf("cannot start float with a dot") + case '=': + return l.errorf("cannot have multiple equals for the same key") case '[': l.depth += 1 return lexLeftBracket @@ -328,6 +334,15 @@ } else if l.follow("\\n") { l.pos += 1 growing_string += "\n" + } else if l.follow("\\b") { + l.pos += 1 + growing_string += "\b" + } else if l.follow("\\f") { + l.pos += 1 + growing_string += "\f" + } else if l.follow("\\/") { + l.pos += 1 + growing_string += "/" } else if l.follow("\\t") { l.pos += 1 growing_string += "\t" @@ -354,6 +369,9 @@ return l.errorf("invalid unicode escape: \\u" + code) } growing_string += string(rune(intcode)) + } else if l.follow("\\") { + l.pos += 1 + return l.errorf("invalid escape sequence: \\" + string(l.peek())) } else { growing_string += string(l.peek()) } @@ -409,6 +427,12 @@ for { next := l.next() if next == '.' { + if point_seen { + return l.errorf("cannot have two dots in one float") + } + if !isDigit(l.peek()) { + return l.errorf("float cannot end with a dot") + } point_seen = true } else if isDigit(next) { digit_seen = true @@ -416,6 +440,9 @@ l.backup() break } + if point_seen && !digit_seen { + return l.errorf("cannot start float with a dot") + } } if !digit_seen {
diff --git a/lexer_test.go b/lexer_test.go index e5846bd..8582520 100644 --- a/lexer_test.go +++ b/lexer_test.go
@@ -113,6 +113,23 @@ }) } +func TestKeyWithSharpAndEqual(t *testing.T) { + testFlow(t, "key#name = 5", []token{ + token{tokenKey, "key#name"}, + token{tokenEqual, "="}, + token{tokenInteger, "5"}, + token{tokenEOF, ""}, + }) +} +func TestKeyWithSymbolsAndEqual(t *testing.T) { + testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{ + token{tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'"}, + token{tokenEqual, "="}, + token{tokenInteger, "5"}, + token{tokenEOF, ""}, + }) +} + func TestKeyEqualStringEscape(t *testing.T) { testFlow(t, "foo = \"hello\\\"\"", []token{ token{tokenKey, "foo"}, @@ -268,6 +285,52 @@ }) } +func TestFloatEndingWithDot(t *testing.T) { + testFlow(t, "foo = 42.", []token{ + token{tokenKey, "foo"}, + token{tokenEqual, "="}, + token{tokenError, "float cannot end with a dot"}, + }) +} + +func TestFloatWithTwoDots(t *testing.T) { + testFlow(t, "foo = 4.2.", []token{ + token{tokenKey, "foo"}, + token{tokenEqual, "="}, + token{tokenError, "cannot have two dots in one float"}, + }) +} + +func TestDoubleEqualKey(t *testing.T) { + testFlow(t, "foo= = 2", []token{ + token{tokenKey, "foo"}, + token{tokenEqual, "="}, + token{tokenError, "cannot have multiple equals for the same key"}, + }) +} + +func TestInvalidEsquapeSequence(t *testing.T) { + testFlow(t, "foo = \"\\x\"", []token{ + token{tokenKey, "foo"}, + token{tokenEqual, "="}, + token{tokenError, "invalid escape sequence: \\x"}, + }) +} + +func TestNestedArrays(t *testing.T) { + testFlow(t, "foo = [[[]]]", []token{ + token{tokenKey, "foo"}, + token{tokenEqual, "="}, + token{tokenLeftBracket, "["}, + token{tokenLeftBracket, "["}, + token{tokenLeftBracket, "["}, + token{tokenRightBracket, "]"}, + token{tokenRightBracket, "]"}, + token{tokenRightBracket, "]"}, + token{tokenEOF, ""}, + }) +} + func TestKeyEqualNumber(t *testing.T) { testFlow(t, "foo = 42", []token{ token{tokenKey, "foo"},
diff --git a/parser.go b/parser.go index e9c9f52..672bb35 100644 --- a/parser.go +++ b/parser.go
@@ -4,15 +4,18 @@ import ( "fmt" + "reflect" "strconv" + "strings" "time" ) type parser struct { - flow chan token - tree *TomlTree - tokensBuffer []token - currentGroup string + flow chan token + tree *TomlTree + tokensBuffer []token + currentGroup []string + seenGroupKeys []string } type parserStateFn func(*parser) parserStateFn @@ -86,9 +89,15 @@ if key.typ != tokenKeyGroup { panic(fmt.Sprintf("unexpected token %s, was expecting a key group", key)) } + for _, item := range p.seenGroupKeys { + if item == key.val { + panic("duplicated tables") + } + } + p.seenGroupKeys = append(p.seenGroupKeys, key.val) p.tree.createSubTree(key.val) p.assume(tokenRightBracket) - p.currentGroup = key.val + p.currentGroup = strings.Split(key.val, ".") return parseStart(p) } @@ -96,11 +105,17 @@ key := p.getToken() p.assume(tokenEqual) value := parseRvalue(p) - final_key := key.val - if p.currentGroup != "" { - final_key = p.currentGroup + "." + key.val + var final_key []string + if len(p.currentGroup) > 0 { + final_key = p.currentGroup + } else { + final_key = make([]string, 0) } - p.tree.Set(final_key, value) + final_key = append(final_key, key.val) + if p.tree.GetPath(final_key) != nil { + panic(fmt.Sprintf("the following key was defined twice: %s", strings.Join(final_key, "."))) + } + p.tree.SetPath(final_key, value) return parseStart(p) } @@ -137,6 +152,8 @@ return val case tokenLeftBracket: return parseArray(p) + case tokenError: + panic(tok.val) } panic("never reached") @@ -146,6 +163,7 @@ func parseArray(p *parser) []interface{} { array := make([]interface{}, 0) + arrayType := reflect.TypeOf(nil) for { follow := p.peek() if follow == nil || follow.typ == tokenEOF { @@ -156,14 +174,18 @@ return array } val := parseRvalue(p) + if arrayType == nil { + arrayType = reflect.TypeOf(val) + } + if reflect.TypeOf(val) != arrayType { + panic("mixed types in array") + } array = append(array, val) follow = p.peek() if follow == nil { panic("unterminated array") } if follow.typ != tokenRightBracket && follow.typ != tokenComma { - fmt.Println(follow.typ) - fmt.Println(follow.val) panic("missing comma") } if follow.typ == tokenComma { @@ -176,10 +198,11 @@ func parse(flow chan token) *TomlTree { result := make(TomlTree) parser := &parser{ - flow: flow, - tree: &result, - tokensBuffer: make([]token, 0), - currentGroup: "", + flow: flow, + tree: &result, + tokensBuffer: make([]token, 0), + currentGroup: make([]string, 0), + seenGroupKeys: make([]string, 0), } parser.run() return parser.tree
diff --git a/parser_test.go b/parser_test.go index 629e0a6..238c3c5 100644 --- a/parser_test.go +++ b/parser_test.go
@@ -12,9 +12,15 @@ return } for k, v := range ref { - if fmt.Sprintf("%v", tree.Get(k)) != fmt.Sprintf("%v", v) { - t.Log("was expecting", v, "at", k, "but got", tree.Get(k)) - t.Error() + node := tree.Get(k) + switch node.(type) { + case *TomlTree: + assertTree(t, node.(*TomlTree), err, v.(map[string]interface{})) + default: + if fmt.Sprintf("%v", node) != fmt.Sprintf("%v", v) { + t.Log("was expecting", v, "at", k, "but got", node) + t.Error() + } } } } @@ -142,6 +148,26 @@ }) } +func TestNestedEmptyArrays(t *testing.T) { + tree, err := Load("a = [[[]]]") + assertTree(t, tree, err, map[string]interface{}{ + "a": [][][]interface{}{[][]interface{}{[]interface{}{}}}, + }) +} + + +func TestArrayMixedTypes(t *testing.T) { + _, err := Load("a = [42, 16.0]") + if err.Error() != "mixed types in array" { + t.Error("Bad error message:", err.Error()) + } + + _, err = Load("a = [42, \"hello\"]") + if err.Error() != "mixed types in array" { + t.Error("Bad error message:", err.Error()) + } +} + func TestArrayNestedStrings(t *testing.T) { tree, err := Load("data = [ [\"gamma\", \"delta\"], [\"Foo\"] ]") assertTree(t, tree, err, map[string]interface{}{ @@ -184,6 +210,53 @@ }) } +func TestDuplicateGroups(t *testing.T) { + _, err := Load("[foo]\na=2\n[foo]b=3") + if err.Error() != "duplicated tables" { + t.Error("Bad error message:", err.Error()) + } +} + +func TestDuplicateKeys(t *testing.T) { + _, err := Load("foo = 2\nfoo = 3") + if err.Error() != "the following key was defined twice: foo" { + t.Error("Bad error message:", err.Error()) + } +} + +func TestEmptyIntermediateTable(t *testing.T) { + _, err := Load("[foo..bar]") + if err.Error() != "empty intermediate table" { + t.Error("Bad error message:", err.Error()) + } +} + +func TestImplicitDeclarationBefore(t *testing.T) { + tree, err := Load("[a.b.c]\nanswer = 42\n[a]\nbetter = 43") + assertTree(t, tree, err, map[string]interface{}{ + "a": map[string]interface{}{ + "b": map[string]interface{}{ + "c": map[string]interface{}{ + "answer": int64(42), + }, + }, + "better": int64(43), + }, + }) +} + +func TestFloatsWithoutLeadingZeros(t *testing.T) { + _, err := Load("a = .42") + if err.Error() != "cannot start float with a dot" { + t.Error("Bad error message:", err.Error()) + } + + _, err = Load("a = -.42") + if err.Error() != "cannot start float with a dot" { + t.Error("Bad error message:", err.Error()) + } +} + func TestMissingFile(t *testing.T) { _, err := LoadFile("foo.toml") if err.Error() != "open foo.toml: no such file or directory" {
diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..b6ed4b4 --- /dev/null +++ b/test.sh
@@ -0,0 +1,11 @@ +#!/bin/bash + +# Run basic go unit tests +go test -v ./... +result=$? + +# Run example-based toml tests +cd test_program && ./go-test.sh +result="$(( result || $? ))" + +exit $result
diff --git a/test_program/go-test.sh b/test_program/go-test.sh new file mode 100755 index 0000000..71293a3 --- /dev/null +++ b/test_program/go-test.sh
@@ -0,0 +1,26 @@ +#!/bin/bash + +go get github.com/BurntSushi/toml-test # install test suite +go get github.com/BurntSushi/toml/toml-test-go # install my parser +go build -o test_program_bin github.com/pelletier/go-toml/test_program + +toml_test_wrapper() { + ret=0 + if hash toml-test 2>/dev/null; then # test availability in $PATH + toml-test "$@" + ret=$? + else + p="$HOME/gopath/bin/toml-test" # try in Travi's place + if [ -f "$p" ]; then + "$p" "$@" + ret=$? + else + "$GOPATH/bin/toml-test" "$@" + ret=$? + fi + fi +} + +toml_test_wrapper ./test_program_bin | tee test_out +ret="$([ `tail -n 1 test_out | sed -E 's/^.+([0-9]+) failed$/\1/'` -eq 0 ])" +exit $ret
diff --git a/test_program/test_program.go b/test_program/test_program.go new file mode 100644 index 0000000..8912c05 --- /dev/null +++ b/test_program/test_program.go
@@ -0,0 +1,75 @@ +package main + +import ( + "io/ioutil" + "os" + "github.com/pelletier/go-toml" + "encoding/json" + "fmt" + "log" + "time" +) + +func main() { + bytes, err := ioutil.ReadAll(os.Stdin) + if err != nil { + os.Exit(2) + } + tree, err := toml.Load(string(bytes)) + if err != nil { + os.Exit(1) + } + + typedTree := translate((map[string]interface{})(*tree)) + + if err := json.NewEncoder(os.Stdout).Encode(typedTree); err != nil { + log.Fatalf("Error encoding JSON: %s", err) + } + + os.Exit(0) +} + +func translate(tomlData interface{}) interface{} { + + switch orig := tomlData.(type) { + case map[string]interface{}: + typed := make(map[string]interface{}, len(orig)) + for k, v := range orig { + typed[k] = translate(v) + } + return typed + case *toml.TomlTree: + return translate((map[string]interface{})(*orig)) + case []map[string]interface{}: + typed := make([]map[string]interface{}, len(orig)) + for i, v := range orig { + typed[i] = translate(v).(map[string]interface{}) + } + return typed + case []interface{}: + typed := make([]interface{}, len(orig)) + for i, v := range orig { + typed[i] = translate(v) + } + return tag("array", typed) + case time.Time: + return tag("datetime", orig.Format("2006-01-02T15:04:05Z")) + case bool: + return tag("bool", fmt.Sprintf("%v", orig)) + case int64: + return tag("integer", fmt.Sprintf("%d", orig)) + case float64: + return tag("float", fmt.Sprintf("%v", orig)) + case string: + return tag("string", orig) + } + + panic(fmt.Sprintf("Unknown type: %T", tomlData)) +} + +func tag(typeName string, data interface{}) map[string]interface{} { + return map[string]interface{}{ + "type": typeName, + "value": data, + } +}
diff --git a/toml.go b/toml.go index 410bab6..dbf8ae8 100644 --- a/toml.go +++ b/toml.go
@@ -15,6 +15,18 @@ // This is the result of the parsing of a TOML file. type TomlTree map[string]interface{} +// Has returns a boolean indicating if the toplevel tree contains the given +// key. +func (t *TomlTree) Has(key string) bool { + mp := (map[string]interface{})(*t) + for k, _ := range mp { + if k == key { + return true + } + } + return false +} + // Keys returns the keys of the toplevel tree. // Warning: this is a costly operation. func (t *TomlTree) Keys() []string { @@ -30,8 +42,11 @@ // Key is a dot-separated path (e.g. a.b.c). // Returns nil if the path does not exist in the tree. func (t *TomlTree) Get(key string) interface{} { + return t.GetPath(strings.Split(key, ".")) +} + +func (t *TomlTree) GetPath(keys []string) interface{} { subtree := t - keys := strings.Split(key, ".") for _, intermediate_key := range keys[:len(keys)-1] { _, exists := (*subtree)[intermediate_key] if !exists { @@ -55,8 +70,11 @@ // Key is a dot-separated path (e.g. a.b.c). // Creates all necessary intermediates trees, if needed. func (t *TomlTree) Set(key string, value interface{}) { + t.SetPath(strings.Split(key, "."), value) +} + +func (t *TomlTree) SetPath(keys []string, value interface{}) { subtree := t - keys := strings.Split(key, ".") for _, intermediate_key := range keys[:len(keys)-1] { _, exists := (*subtree)[intermediate_key] if !exists { @@ -76,6 +94,9 @@ func (t *TomlTree) createSubTree(key string) { subtree := t for _, intermediate_key := range strings.Split(key, ".") { + if intermediate_key == "" { + panic("empty intermediate table") + } _, exists := (*subtree)[intermediate_key] if !exists { var new_tree TomlTree = make(TomlTree) @@ -109,6 +130,5 @@ s := string(buff) tree, err = Load(s) } - return }