Merge pull request #51 from pelletier/pelletier/fix-crlf-support
Fix support for CRLF line ending
diff --git a/example-crlf.toml b/example-crlf.toml
new file mode 100644
index 0000000..12950a1
--- /dev/null
+++ b/example-crlf.toml
@@ -0,0 +1,29 @@
+# This is a TOML document. Boom.
+
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+organization = "GitHub"
+bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
+dob = 1979-05-27T07:32:00Z # First class dates? Why not?
+
+[database]
+server = "192.168.1.1"
+ports = [ 8001, 8001, 8002 ]
+connection_max = 5000
+enabled = true
+
+[servers]
+
+ # You can indent as you please. Tabs or spaces. TOML don't care.
+ [servers.alpha]
+ ip = "10.0.0.1"
+ dc = "eqdc10"
+
+ [servers.beta]
+ ip = "10.0.0.2"
+ dc = "eqdc10"
+
+[clients]
+data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it
diff --git a/lexer.go b/lexer.go
index df0596b..0bffb0d 100644
--- a/lexer.go
+++ b/lexer.go
@@ -132,6 +132,8 @@
return l.lexComment
case '=':
return l.lexEqual
+ case '\r':
+ fallthrough
case '\n':
l.skip()
continue
@@ -185,6 +187,8 @@
return l.lexLiteralString
case ',':
return l.lexComma
+ case '\r':
+ fallthrough
case '\n':
l.skip()
if l.depth == 0 {
@@ -277,7 +281,7 @@
func (l *tomlLexer) lexKey() tomlLexStateFn {
inQuotes := false
- for r := l.peek(); isKeyChar(r) || r == '\n'; r = l.peek() {
+ for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() {
if r == '"' {
inQuotes = !inQuotes
} else if r == '\n' {
@@ -295,6 +299,9 @@
func (l *tomlLexer) lexComment() tomlLexStateFn {
for next := l.peek(); next != '\n' && next != eof; next = l.peek() {
+ if (next == '\r' && l.follow("\r\n")) {
+ break
+ }
l.next()
}
l.ignore()
@@ -319,7 +326,10 @@
terminator = "'''"
// special case: discard leading newline
- if l.peek() == '\n' {
+ if l.follow("\r\n") {
+ l.skip()
+ l.skip()
+ } else if l.peek() == '\n' {
l.skip()
}
}
@@ -355,7 +365,10 @@
terminator = "\"\"\""
// special case: discard leading newline
- if l.peek() == '\n' {
+ if l.follow("\r\n") {
+ l.skip()
+ l.skip()
+ } else if l.peek() == '\n' {
l.skip()
}
}
diff --git a/lexer_test.go b/lexer_test.go
index 9fa8be8..6183061 100644
--- a/lexer_test.go
+++ b/lexer_test.go
@@ -87,6 +87,19 @@
})
}
+
+func TestSimpleWindowsCRLF(t *testing.T) {
+ testFlow(t, "a=4\r\nb=2", []token{
+ token{Position{1, 1}, tokenKey, "a"},
+ token{Position{1, 2}, tokenEqual, "="},
+ token{Position{1, 3}, tokenInteger, "4"},
+ token{Position{2, 1}, tokenKey, "b"},
+ token{Position{2, 2}, tokenEqual, "="},
+ token{Position{2, 3}, tokenInteger, "2"},
+ token{Position{2, 4}, tokenEOF, ""},
+ })
+}
+
func TestBasicKey(t *testing.T) {
testFlow(t, "hello", []token{
token{Position{1, 1}, tokenKey, "hello"},
diff --git a/parser_test.go b/parser_test.go
index f9191b6..d21604a 100644
--- a/parser_test.go
+++ b/parser_test.go
@@ -494,6 +494,42 @@
})
}
+func TestParseFileCRLF(t *testing.T) {
+ tree, err := LoadFile("example-crlf.toml")
+
+ assertTree(t, tree, err, map[string]interface{}{
+ "title": "TOML Example",
+ "owner": map[string]interface{}{
+ "name": "Tom Preston-Werner",
+ "organization": "GitHub",
+ "bio": "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+ "dob": time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+ },
+ "database": map[string]interface{}{
+ "server": "192.168.1.1",
+ "ports": []int64{8001, 8001, 8002},
+ "connection_max": 5000,
+ "enabled": true,
+ },
+ "servers": map[string]interface{}{
+ "alpha": map[string]interface{}{
+ "ip": "10.0.0.1",
+ "dc": "eqdc10",
+ },
+ "beta": map[string]interface{}{
+ "ip": "10.0.0.2",
+ "dc": "eqdc10",
+ },
+ },
+ "clients": map[string]interface{}{
+ "data": []interface{}{
+ []string{"gamma", "delta"},
+ []int64{1, 2},
+ },
+ },
+ })
+}
+
func TestParseKeyGroupArray(t *testing.T) {
tree, err := Load("[[foo.bar]] a = 42\n[[foo.bar]] a = 69")
assertTree(t, tree, err, map[string]interface{}{
