Implement parser
diff --git a/README.md b/README.md index d44c653..2477554 100644 --- a/README.md +++ b/README.md
@@ -1,3 +1,5 @@ # go-toml Go library for the [TOML](https://github.com/mojombo/toml) format. + +This library supports TOML version [e3656ad493400895f4460f1244a25f8f8e31a32a](https://github.com/mojombo/toml/tree/e3656ad493400895f4460f1244a25f8f8e31a32a)
diff --git a/src/toml/lexer.go b/src/toml/lexer.go index 7484b7b..876817e 100644 --- a/src/toml/lexer.go +++ b/src/toml/lexer.go
@@ -39,6 +39,7 @@ tokenDate tokenKeyGroup tokenComma + tokenEOL ) type token struct { @@ -196,6 +197,9 @@ case ',': return lexComma case '\n': + l.ignore() + l.pos += 1 + l.emit(tokenEOF) return lexVoid }
diff --git a/src/toml/parser.go b/src/toml/parser.go index 0441423..e4188a0 100644 --- a/src/toml/parser.go +++ b/src/toml/parser.go
@@ -3,27 +3,165 @@ package toml import ( - "strings" + "fmt" + "strconv" + "time" ) -// createSubTree takes a tree and a key andcreate the necessary intermediate -// subtrees to create a subtree at that point. In-place. -// -// e.g. passing a.b.c will create (assuming tree is empty) tree[a], tree[a][b] -// and tree[a][b][c] -func createSubTree(tree *TomlTree, key string) { - subtree := tree - for _, intermediate_key := range strings.Split(key, ".") { - _, exists := (*subtree)[intermediate_key] - if !exists { - (*subtree)[intermediate_key] = make(TomlTree) - } - subtree = (*subtree)[intermediate_key].(*TomlTree) + +type parser struct { + flow chan token + tree *TomlTree + tokensBuffer []token + currentGroup string +} + +type parserStateFn func(*parser) parserStateFn + +func (p *parser) run() { + for state := parseStart; state != nil; { + state = state(p) } } +func (p *parser) peek() *token { + if len(p.tokensBuffer) != 0 { + return &(p.tokensBuffer[0]) + } -func parse(chan token) *TomlTree { + tok, ok := <- p.flow + if !ok { + return nil + } + p.tokensBuffer = append(p.tokensBuffer, tok) + return &tok +} + +func (p *parser) assume(typ tokenType) { + tok := p.getToken() + if tok == nil { + panic(fmt.Sprintf("was expecting token %s, but token stream is empty", typ)) + } + if tok.typ != typ { + panic(fmt.Sprintf("was expecting token %s, but got %s", typ, tok.typ)) + } +} + +func (p *parser) getToken() *token { + if len(p.tokensBuffer) != 0 { + tok := p.tokensBuffer[0] + p.tokensBuffer = p.tokensBuffer[1:] + return &tok + } + tok, ok := <- p.flow + if !ok { + return nil + } + return &tok +} + +func parseStart(p *parser) parserStateFn { + tok := p.peek() + + // end of stream, parsing is finished + if tok == nil { + return nil + } + + switch tok.typ { + case tokenLeftBracket: + return parseGroup + case tokenKey: + return parseAssign + default: + panic("unexpected token") + } + return nil +} + +func parseGroup(p *parser) parserStateFn { + p.getToken() // discard the [ + key := p.getToken() + if key.typ != tokenKey { + panic(fmt.Sprintf("unexpected token %s, was expecting a key", key)) + } + p.tree.createSubTree(key.val) + p.assume(tokenRightBracket) + p.currentGroup = key.val + return parseStart(p) +} + +func parseAssign(p *parser) parserStateFn { + key := p.getToken() + p.assume(tokenEqual) + value := parseRvalue(p) + p.tree.Set(key.val, value) + return parseStart(p) +} + +func parseRvalue(p *parser) interface{} { + tok := p.getToken() + if tok == nil { + panic("expecting a value") + } + + switch tok.typ { + case tokenString: + return tok.val + case tokenTrue: + return true + case tokenFalse: + return false + case tokenInteger: + val, err := strconv.ParseInt(tok.val, 10, 64) + if err != nil { panic(err) } + return val + case tokenFloat: + val, err := strconv.ParseFloat(tok.val, 64) + if err != nil { panic(err) } + return val + case tokenDate: + val, err := time.Parse(time.RFC3339, tok.val) + if err != nil { panic(err) } + return val + case tokenLeftBracket: + return parseArray(p) + } + + panic("never reached") + + return nil +} + +func parseArray(p *parser) []interface{} { + array := make([]interface{}, 0) + for { + follow := p.peek() + if follow == nil { panic("unterminated array") } + if follow.typ == tokenRightBracket { + p.getToken() + return array + } + val := parseRvalue(p) + array = append(array, val) + follow = p.peek() + if follow == nil { panic("unterminated array") } + if follow.typ != tokenRightBracket && follow.typ != tokenComma { + panic("missing comma") + } + } + return array +} + + +func parse(flow chan token) *TomlTree { result := make(TomlTree) - return &result + parser := &parser { + flow: flow, + tree: &result, + tokensBuffer: make([]token, 0), + currentGroup: "", + } + parser.run() + return parser.tree }
diff --git a/src/toml/parser_test.go b/src/toml/parser_test.go index f61135d..00b86bc 100644 --- a/src/toml/parser_test.go +++ b/src/toml/parser_test.go
@@ -3,11 +3,44 @@ import "testing" +func assertTree(t *testing.T, tree *TomlTree, ref map[string]interface{}) { + for k, v := range ref { + if tree.Get(k) != v { + t.Log("was expecting", v, "at", k, "but got", tree.Get(k)) + t.Fail() + } + } +} + func testCreateSubTree(t *testing.T) { tree := make(TomlTree) - createSubTree(&tree, "a.b.c") + tree.createSubTree("a.b.c") tree.Set("a.b.c", 42) if tree.Get("a.b.c") != 42 { t.Fail() } } + + +func testSimpleKV(t *testing.T) { + tree := Load("a = 42") + assertTree(t, tree, map[string]interface{}{ + "a": 42, + }) + + tree = Load("a = 42\nb = 21") + assertTree(t, tree, map[string]interface{}{ + "a": 42, + "b": 21, + }) +} + +func testSimpleIntegers(t *testing.T) { + tree := Load("a = +42\nb = -21\nc = +4.2\nd = -2.1") + assertTree(t, tree, map[string]interface{}{ + "a": 42, + "b": -21, + "c": 4.2, + "d": -4.2, + }) +}
diff --git a/src/toml/toml.go b/src/toml/toml.go index 717e81f..9c84455 100644 --- a/src/toml/toml.go +++ b/src/toml/toml.go
@@ -39,8 +39,24 @@ (*subtree)[keys[len(key) - 1]] = value } +// createSubTree takes a tree and a key andcreate the necessary intermediate +// subtrees to create a subtree at that point. In-place. +// +// e.g. passing a.b.c will create (assuming tree is empty) tree[a], tree[a][b] +// and tree[a][b][c] +func (t *TomlTree) createSubTree(key string) { + subtree := t + for _, intermediate_key := range strings.Split(key, ".") { + _, exists := (*subtree)[intermediate_key] + if !exists { + (*subtree)[intermediate_key] = make(TomlTree) + } + subtree = (*subtree)[intermediate_key].(*TomlTree) + } +} -func Load() TomlTree { - result := make(TomlTree) - return result + +func Load(content string) *TomlTree { + _, flow := lex(content) + return parse(flow) }