Merge branch 'toml-0.3.1'
diff --git a/lexer.go b/lexer.go index c1b4998..cc4dc08 100644 --- a/lexer.go +++ b/lexer.go
@@ -138,7 +138,7 @@ return l.lexRvalue } - if isKeyChar(next) { + if isKeyStartChar(next) { return l.lexKey } @@ -169,6 +169,8 @@ return l.lexComment case '"': return l.lexString + case '\'': + return l.lexLiteralString case ',': return l.lexComma case '\n': @@ -192,7 +194,10 @@ return l.lexKey } - if dateRegexp.FindString(l.input[l.pos:]) != "" { + dateMatch := dateRegexp.FindString(l.input[l.pos:]) + if dateMatch != "" { + l.ignore() + l.pos += len(dateMatch) return l.lexDate } @@ -214,8 +219,6 @@ } func (l *tomlLexer) lexDate() tomlLexStateFn { - l.ignore() - l.pos += 20 // Fixed size of a date in TOML l.emit(tokenDate) return l.lexRvalue } @@ -250,7 +253,10 @@ func (l *tomlLexer) lexKey() tomlLexStateFn { l.ignore() - for isKeyChar(l.next()) { + for r := l.next(); isKeyChar(r); r = l.next() { + if (r == '#') { + return l.errorf("keys cannot contain # character") + } } l.backup() l.emit(tokenKey) @@ -275,6 +281,29 @@ return l.lexRvalue } +func (l *tomlLexer) lexLiteralString() tomlLexStateFn { + l.pos++ + l.ignore() + growingString := "" + + for { + if l.peek() == '\'' { + l.emitWithValue(tokenString, growingString) + l.pos++ + l.ignore() + return l.lexRvalue + } + + growingString += string(l.peek()) + + if l.next() == eof { + break + } + } + + return l.errorf("unclosed string") +} + func (l *tomlLexer) lexString() tomlLexStateFn { l.pos++ l.ignore() @@ -418,6 +447,7 @@ l.accept("-") } pointSeen := false + expSeen := false digitSeen := false for { next := l.next() @@ -429,6 +459,11 @@ return l.errorf("float cannot end with a dot") } pointSeen = true + } else if next == 'e' || next == 'E' { + expSeen = true + if !l.accept("+") { + l.accept("-") + } } else if isDigit(next) { digitSeen = true } else { @@ -443,7 +478,7 @@ if !digitSeen { return l.errorf("no digit in that number") } - if pointSeen { + if pointSeen || expSeen { l.emit(tokenFloat) } else { l.emit(tokenInteger) @@ -452,7 +487,7 @@ } func init() { - dateRegexp = regexp.MustCompile("^\\d{1,4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z") + dateRegexp = regexp.MustCompile("^\\d{1,4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{1,9})?(Z|[+-]\\d{2}:\\d{2})") } // Entry point
diff --git a/lexer_test.go b/lexer_test.go index 5114223..2555a35 100644 --- a/lexer_test.go +++ b/lexer_test.go
@@ -118,19 +118,16 @@ func TestKeyWithSharpAndEqual(t *testing.T) { testFlow(t, "key#name = 5", []token{ - token{Position{1, 1}, tokenKey, "key#name"}, - token{Position{1, 10}, tokenEqual, "="}, - token{Position{1, 12}, tokenInteger, "5"}, - token{Position{1, 13}, tokenEOF, ""}, + token{Position{1, 1}, tokenError, "keys cannot contain # character"}, }) } func TestKeyWithSymbolsAndEqual(t *testing.T) { - testFlow(t, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{ - token{Position{1, 1}, tokenKey, "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:'"}, - token{Position{1, 39}, tokenEqual, "="}, - token{Position{1, 41}, tokenInteger, "5"}, - token{Position{1, 42}, tokenEOF, ""}, + testFlow(t, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{ + token{Position{1, 1}, tokenKey, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:'"}, + token{Position{1, 38}, tokenEqual, "="}, + token{Position{1, 40}, tokenInteger, "5"}, + token{Position{1, 41}, tokenEOF, ""}, }) } @@ -276,7 +273,13 @@ func TestDateRegexp(t *testing.T) { if dateRegexp.FindString("1979-05-27T07:32:00Z") == "" { - t.Fail() + t.Error("basic lexing") + } + if dateRegexp.FindString("1979-05-27T00:32:00-07:00") == "" { + t.Error("offset lexing") + } + if dateRegexp.FindString("1979-05-27T00:32:00.999999-07:00") == "" { + t.Error("nano precision lexing") } } @@ -287,6 +290,18 @@ token{Position{1, 7}, tokenDate, "1979-05-27T07:32:00Z"}, token{Position{1, 27}, tokenEOF, ""}, }) + testFlow(t, "foo = 1979-05-27T00:32:00-07:00", []token{ + token{Position{1, 1}, tokenKey, "foo"}, + token{Position{1, 5}, tokenEqual, "="}, + token{Position{1, 7}, tokenDate, "1979-05-27T00:32:00-07:00"}, + token{Position{1, 32}, tokenEOF, ""}, + }) + testFlow(t, "foo = 1979-05-27T00:32:00.999999-07:00", []token{ + token{Position{1, 1}, tokenKey, "foo"}, + token{Position{1, 5}, tokenEqual, "="}, + token{Position{1, 7}, tokenDate, "1979-05-27T00:32:00.999999-07:00"}, + token{Position{1, 39}, tokenEOF, ""}, + }) } func TestFloatEndingWithDot(t *testing.T) { @@ -305,6 +320,51 @@ }) } +func TestFloatWithExponent1(t *testing.T) { + testFlow(t, "a = 5e+22", []token{ + token{Position{1, 1}, tokenKey, "a"}, + token{Position{1, 3}, tokenEqual, "="}, + token{Position{1, 5}, tokenFloat, "5e+22"}, + token{Position{1, 10}, tokenEOF, ""}, + }) +} + +func TestFloatWithExponent2(t *testing.T) { + testFlow(t, "a = 5E+22", []token{ + token{Position{1, 1}, tokenKey, "a"}, + token{Position{1, 3}, tokenEqual, "="}, + token{Position{1, 5}, tokenFloat, "5E+22"}, + token{Position{1, 10}, tokenEOF, ""}, + }) +} + +func TestFloatWithExponent3(t *testing.T) { + testFlow(t, "a = -5e+22", []token{ + token{Position{1, 1}, tokenKey, "a"}, + token{Position{1, 3}, tokenEqual, "="}, + token{Position{1, 5}, tokenFloat, "-5e+22"}, + token{Position{1, 11}, tokenEOF, ""}, + }) +} + +func TestFloatWithExponent4(t *testing.T) { + testFlow(t, "a = -5e-22", []token{ + token{Position{1, 1}, tokenKey, "a"}, + token{Position{1, 3}, tokenEqual, "="}, + token{Position{1, 5}, tokenFloat, "-5e-22"}, + token{Position{1, 11}, tokenEOF, ""}, + }) +} + +func TestFloatWithExponent5(t *testing.T) { + testFlow(t, "a = 6.626e-34", []token{ + token{Position{1, 1}, tokenKey, "a"}, + token{Position{1, 3}, tokenEqual, "="}, + token{Position{1, 5}, tokenFloat, "6.626e-34"}, + token{Position{1, 14}, tokenEOF, ""}, + }) +} + func TestDoubleEqualKey(t *testing.T) { testFlow(t, "foo= = 2", []token{ token{Position{1, 1}, tokenKey, "foo"}, @@ -400,6 +460,33 @@ }) } +func TestLiteralString(t *testing.T) { + testFlow(t, `foo = 'C:\Users\nodejs\templates'`, []token{ + token{Position{1, 1}, tokenKey, "foo"}, + token{Position{1, 5}, tokenEqual, "="}, + token{Position{1, 8}, tokenString, `C:\Users\nodejs\templates`}, + token{Position{1, 34}, tokenEOF, ""}, + }) + testFlow(t, `foo = '\\ServerX\admin$\system32\'`, []token{ + token{Position{1, 1}, tokenKey, "foo"}, + token{Position{1, 5}, tokenEqual, "="}, + token{Position{1, 8}, tokenString, `\\ServerX\admin$\system32\`}, + token{Position{1, 35}, tokenEOF, ""}, + }) + testFlow(t, `foo = 'Tom "Dubs" Preston-Werner'`, []token{ + token{Position{1, 1}, tokenKey, "foo"}, + token{Position{1, 5}, tokenEqual, "="}, + token{Position{1, 8}, tokenString, `Tom "Dubs" Preston-Werner`}, + token{Position{1, 34}, tokenEOF, ""}, + }) + testFlow(t, `foo = '<\i\c*\s*>'`, []token{ + token{Position{1, 1}, tokenKey, "foo"}, + token{Position{1, 5}, tokenEqual, "="}, + token{Position{1, 8}, tokenString, `<\i\c*\s*>`}, + token{Position{1, 19}, tokenEOF, ""}, + }) +} + func TestUnicodeString(t *testing.T) { testFlow(t, `foo = "hello ♥ world"`, []token{ token{Position{1, 1}, tokenKey, "foo"},
diff --git a/parser.go b/parser.go index 60918cc..e83647d 100644 --- a/parser.go +++ b/parser.go
@@ -222,7 +222,7 @@ } return val case tokenDate: - val, err := time.Parse(time.RFC3339, tok.val) + val, err := time.Parse(time.RFC3339Nano, tok.val) if err != nil { p.raiseError(tok, "%s", err) }
diff --git a/parser_test.go b/parser_test.go index 8160905..afaabd3 100644 --- a/parser_test.go +++ b/parser_test.go
@@ -54,9 +54,9 @@ // NOTE: from the BurntSushi test suite // NOTE: this test is pure evil due to the embedded '.' func TestSpecialKV(t *testing.T) { - tree, err := Load("~!@#$^&*()_+-`1234567890[]\\|/?><.,;: = 1") + tree, err := Load("~!@$^&*()_+-`1234567890[]\\|/?><.,;: = 1") assertTree(t, tree, err, map[string]interface{}{ - "~!@#$^&*()_+-`1234567890[]\\|/?><.,;:": int64(1), + "~!@$^&*()_+-`1234567890[]\\|/?><.,;:": int64(1), }) } @@ -70,6 +70,17 @@ }) } +func TestFloatsWithExponents(t *testing.T) { + tree, err := Load("a = 5e+22\nb = 5E+22\nc = -5e+22\nd = -5e-22\ne = 6.626e-34") + assertTree(t, tree, err, map[string]interface{}{ + "a": float64(5e+22), + "b": float64(5E+22), + "c": float64(-5e+22), + "d": float64(-5e-22), + "e": float64(6.626e-34), + }) +} + func TestSimpleDate(t *testing.T) { tree, err := Load("a = 1979-05-27T07:32:00Z") assertTree(t, tree, err, map[string]interface{}{ @@ -77,6 +88,21 @@ }) } +func TestDateOffset(t *testing.T) { + tree, err := Load("a = 1979-05-27T00:32:00-07:00") + assertTree(t, tree, err, map[string]interface{}{ + "a": time.Date(1979, time.May, 27, 0, 32, 0, 0, time.FixedZone("", -7 * 60 * 60)), + }) +} + +func TestDateNano(t *testing.T) { + tree, err := Load("a = 1979-05-27T00:32:00.999999999-07:00") + assertTree(t, tree, err, map[string]interface{}{ + "a": time.Date(1979, time.May, 27, 0, 32, 0, 999999999, time.FixedZone("", -7 * 60 * 60)), + }) +} + + func TestSimpleString(t *testing.T) { tree, err := Load("a = \"hello world\"") assertTree(t, tree, err, map[string]interface{}{
diff --git a/token.go b/token.go index 22ad37a..6c0bcc6 100644 --- a/token.go +++ b/token.go
@@ -117,11 +117,16 @@ } func isKeyChar(r rune) bool { - // "Keys start with the first non-whitespace character and end with the last - // non-whitespace character before the equals sign." + // Keys start with the first character that isn't whitespace or [ and end + // with the last non-whitespace character before the equals sign. Keys + // cannot contain a # character." return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '=') } +func isKeyStartChar(r rune) bool { + return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '[') +} + func isDigit(r rune) bool { return unicode.IsNumber(r) }