Fix support for CRLF line ending
diff --git a/example-crlf.toml b/example-crlf.toml new file mode 100644 index 0000000..12950a1 --- /dev/null +++ b/example-crlf.toml
@@ -0,0 +1,29 @@ +# This is a TOML document. Boom. + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T07:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it
diff --git a/lexer.go b/lexer.go index df0596b..0bffb0d 100644 --- a/lexer.go +++ b/lexer.go
@@ -132,6 +132,8 @@ return l.lexComment case '=': return l.lexEqual + case '\r': + fallthrough case '\n': l.skip() continue @@ -185,6 +187,8 @@ return l.lexLiteralString case ',': return l.lexComma + case '\r': + fallthrough case '\n': l.skip() if l.depth == 0 { @@ -277,7 +281,7 @@ func (l *tomlLexer) lexKey() tomlLexStateFn { inQuotes := false - for r := l.peek(); isKeyChar(r) || r == '\n'; r = l.peek() { + for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() { if r == '"' { inQuotes = !inQuotes } else if r == '\n' { @@ -295,6 +299,9 @@ func (l *tomlLexer) lexComment() tomlLexStateFn { for next := l.peek(); next != '\n' && next != eof; next = l.peek() { + if (next == '\r' && l.follow("\r\n")) { + break + } l.next() } l.ignore() @@ -319,7 +326,10 @@ terminator = "'''" // special case: discard leading newline - if l.peek() == '\n' { + if l.follow("\r\n") { + l.skip() + l.skip() + } else if l.peek() == '\n' { l.skip() } } @@ -355,7 +365,10 @@ terminator = "\"\"\"" // special case: discard leading newline - if l.peek() == '\n' { + if l.follow("\r\n") { + l.skip() + l.skip() + } else if l.peek() == '\n' { l.skip() } }
diff --git a/lexer_test.go b/lexer_test.go index 9fa8be8..6183061 100644 --- a/lexer_test.go +++ b/lexer_test.go
@@ -87,6 +87,19 @@ }) } + +func TestSimpleWindowsCRLF(t *testing.T) { + testFlow(t, "a=4\r\nb=2", []token{ + token{Position{1, 1}, tokenKey, "a"}, + token{Position{1, 2}, tokenEqual, "="}, + token{Position{1, 3}, tokenInteger, "4"}, + token{Position{2, 1}, tokenKey, "b"}, + token{Position{2, 2}, tokenEqual, "="}, + token{Position{2, 3}, tokenInteger, "2"}, + token{Position{2, 4}, tokenEOF, ""}, + }) +} + func TestBasicKey(t *testing.T) { testFlow(t, "hello", []token{ token{Position{1, 1}, tokenKey, "hello"},
diff --git a/parser_test.go b/parser_test.go index f9191b6..d21604a 100644 --- a/parser_test.go +++ b/parser_test.go
@@ -494,6 +494,42 @@ }) } +func TestParseFileCRLF(t *testing.T) { + tree, err := LoadFile("example-crlf.toml") + + assertTree(t, tree, err, map[string]interface{}{ + "title": "TOML Example", + "owner": map[string]interface{}{ + "name": "Tom Preston-Werner", + "organization": "GitHub", + "bio": "GitHub Cofounder & CEO\nLikes tater tots and beer.", + "dob": time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC), + }, + "database": map[string]interface{}{ + "server": "192.168.1.1", + "ports": []int64{8001, 8001, 8002}, + "connection_max": 5000, + "enabled": true, + }, + "servers": map[string]interface{}{ + "alpha": map[string]interface{}{ + "ip": "10.0.0.1", + "dc": "eqdc10", + }, + "beta": map[string]interface{}{ + "ip": "10.0.0.2", + "dc": "eqdc10", + }, + }, + "clients": map[string]interface{}{ + "data": []interface{}{ + []string{"gamma", "delta"}, + []int64{1, 2}, + }, + }, + }) +} + func TestParseKeyGroupArray(t *testing.T) { tree, err := Load("[[foo.bar]] a = 42\n[[foo.bar]] a = 69") assertTree(t, tree, err, map[string]interface{}{