Changes to support #29 - Support multi-line literal strings * Added error output to test_program.go * Added multi-line literal string support to lexer * Added multi-line string supprt to lexer * Added unit-test for new string support * Modified test.sh to take an optional parameter to run an individual BurntSushi test suite. * Fixed formatting
diff --git a/cmd/test_program.go b/cmd/test_program.go index 65dd8a3..971465e 100644 --- a/cmd/test_program.go +++ b/cmd/test_program.go
@@ -13,10 +13,12 @@ func main() { bytes, err := ioutil.ReadAll(os.Stdin) if err != nil { + log.Fatalf("Error during TOML read: %s", err) os.Exit(2) } tree, err := toml.Load(string(bytes)) if err != nil { + log.Fatalf("Error during TOML load: %s", err) os.Exit(1) } @@ -24,6 +26,7 @@ if err := json.NewEncoder(os.Stdout).Encode(typedTree); err != nil { log.Fatalf("Error encoding JSON: %s", err) + os.Exit(3) } os.Exit(0)
diff --git a/lexer.go b/lexer.go index cc4dc08..abb6551 100644 --- a/lexer.go +++ b/lexer.go
@@ -254,7 +254,7 @@ func (l *tomlLexer) lexKey() tomlLexStateFn { l.ignore() for r := l.next(); isKeyChar(r); r = l.next() { - if (r == '#') { + if r == '#' { return l.errorf("keys cannot contain # character") } } @@ -286,11 +286,26 @@ l.ignore() growingString := "" - for { - if l.peek() == '\'' { - l.emitWithValue(tokenString, growingString) + // handle special case for triple-quote + terminator := "'" + if l.follow("''") { + l.pos += 2 + l.ignore() + terminator = "'''" + + // special case: discard leading newline + if l.peek() == '\n' { l.pos++ l.ignore() + } + } + + // find end of string + for { + if l.follow(terminator) { + l.emitWithValue(tokenString, growingString) + l.pos += len(terminator) + l.ignore() return l.lexRvalue } @@ -309,58 +324,80 @@ l.ignore() growingString := "" - for { - if l.peek() == '"' { - l.emitWithValue(tokenString, growingString) + // handle special case for triple-quote + terminator := "\"" + if l.follow("\"\"") { + l.pos += 2 + l.ignore() + terminator = "\"\"\"" + + // special case: discard leading newline + if l.peek() == '\n' { l.pos++ l.ignore() + } + } + + for { + if l.follow(terminator) { + l.emitWithValue(tokenString, growingString) + l.pos += len(terminator) + l.ignore() return l.lexRvalue } - if l.follow("\\\"") { + if l.follow("\\") { l.pos++ - growingString += "\"" - } else if l.follow("\\n") { - l.pos++ - growingString += "\n" - } else if l.follow("\\b") { - l.pos++ - growingString += "\b" - } else if l.follow("\\f") { - l.pos++ - growingString += "\f" - } else if l.follow("\\/") { - l.pos++ - growingString += "/" - } else if l.follow("\\t") { - l.pos++ - growingString += "\t" - } else if l.follow("\\r") { - l.pos++ - growingString += "\r" - } else if l.follow("\\\\") { - l.pos++ - growingString += "\\" - } else if l.follow("\\u") { - l.pos += 2 - code := "" - for i := 0; i < 4; i++ { - c := l.peek() + switch l.peek() { + case '\r': + fallthrough + case '\n': + fallthrough + case '\t': + fallthrough + case ' ': + // skip all whitespace chars following backslash l.pos++ - if !isHexDigit(c) { - return l.errorf("unfinished unicode escape") + for strings.ContainsRune("\r\n\t ", l.peek()) { + l.pos++ } - code = code + string(c) + l.pos-- + case '"': + growingString += "\"" + case 'n': + growingString += "\n" + case 'b': + growingString += "\b" + case 'f': + growingString += "\f" + case '/': + growingString += "/" + case 't': + growingString += "\t" + case 'r': + growingString += "\r" + case '\\': + growingString += "\\" + case 'u': + l.pos++ + code := "" + for i := 0; i < 4; i++ { + c := l.peek() + l.pos++ + if !isHexDigit(c) { + return l.errorf("unfinished unicode escape") + } + code = code + string(c) + } + l.pos-- + intcode, err := strconv.ParseInt(code, 16, 32) + if err != nil { + return l.errorf("invalid unicode escape: \\u" + code) + } + growingString += string(rune(intcode)) + default: + return l.errorf("invalid escape sequence: \\" + string(l.peek())) } - l.pos-- - intcode, err := strconv.ParseInt(code, 16, 32) - if err != nil { - return l.errorf("invalid unicode escape: \\u" + code) - } - growingString += string(rune(intcode)) - } else if l.follow("\\") { - l.pos++ - return l.errorf("invalid escape sequence: \\" + string(l.peek())) } else { growingString += string(l.peek()) }
diff --git a/lexer_test.go b/lexer_test.go index 2555a35..e9d0f14 100644 --- a/lexer_test.go +++ b/lexer_test.go
@@ -487,6 +487,45 @@ }) } +func TestMultilineLiteralString(t *testing.T) { + testFlow(t, `foo = '''hello 'literal' world'''`, []token{ + token{Position{1, 1}, tokenKey, "foo"}, + token{Position{1, 5}, tokenEqual, "="}, + token{Position{1, 10}, tokenString, `hello 'literal' world`}, + token{Position{1, 34}, tokenEOF, ""}, + }) + + testFlow(t, "foo = '''\nhello\n'literal'\nworld'''", []token{ + token{Position{1, 1}, tokenKey, "foo"}, + token{Position{1, 5}, tokenEqual, "="}, + token{Position{2, 1}, tokenString, "hello\n'literal'\nworld"}, + token{Position{4, 9}, tokenEOF, ""}, + }) +} + +func TestMultilineString(t *testing.T) { + testFlow(t, `foo = """hello "literal" world"""`, []token{ + token{Position{1, 1}, tokenKey, "foo"}, + token{Position{1, 5}, tokenEqual, "="}, + token{Position{1, 10}, tokenString, `hello "literal" world`}, + token{Position{1, 34}, tokenEOF, ""}, + }) + + testFlow(t, "foo = \"\"\"\nhello\n\"literal\"\nworld\"\"\"", []token{ + token{Position{1, 1}, tokenKey, "foo"}, + token{Position{1, 5}, tokenEqual, "="}, + token{Position{2, 1}, tokenString, "hello\n\"literal\"\nworld"}, + token{Position{4, 9}, tokenEOF, ""}, + }) + + testFlow(t, "foo = \"\"\"\\\n \\\n \\\n hello\nmultiline\nworld\"\"\"", []token{ + token{Position{1, 1}, tokenKey, "foo"}, + token{Position{1, 5}, tokenEqual, "="}, + token{Position{1, 10}, tokenString, "hello\nmultiline\nworld"}, + token{Position{6, 9}, tokenEOF, ""}, + }) +} + func TestUnicodeString(t *testing.T) { testFlow(t, `foo = "hello ♥ world"`, []token{ token{Position{1, 1}, tokenKey, "foo"},
diff --git a/parser_test.go b/parser_test.go index afaabd3..8110495 100644 --- a/parser_test.go +++ b/parser_test.go
@@ -91,18 +91,17 @@ func TestDateOffset(t *testing.T) { tree, err := Load("a = 1979-05-27T00:32:00-07:00") assertTree(t, tree, err, map[string]interface{}{ - "a": time.Date(1979, time.May, 27, 0, 32, 0, 0, time.FixedZone("", -7 * 60 * 60)), + "a": time.Date(1979, time.May, 27, 0, 32, 0, 0, time.FixedZone("", -7*60*60)), }) } func TestDateNano(t *testing.T) { tree, err := Load("a = 1979-05-27T00:32:00.999999999-07:00") assertTree(t, tree, err, map[string]interface{}{ - "a": time.Date(1979, time.May, 27, 0, 32, 0, 999999999, time.FixedZone("", -7 * 60 * 60)), + "a": time.Date(1979, time.May, 27, 0, 32, 0, 999999999, time.FixedZone("", -7*60*60)), }) } - func TestSimpleString(t *testing.T) { tree, err := Load("a = \"hello world\"") assertTree(t, tree, err, map[string]interface{}{
diff --git a/test.sh b/test.sh index 07aa52e..410838b 100755 --- a/test.sh +++ b/test.sh
@@ -5,15 +5,26 @@ # set the path to the present working directory export GOPATH=`pwd` -# Vendorize the BurntSushi test suite -# NOTE: this gets a specific release to avoid versioning issues -if [ ! -d 'src/github.com/BurntSushi/toml-test' ]; then - mkdir -p src/github.com/BurntSushi - git clone https://github.com/BurntSushi/toml-test.git src/github.com/BurntSushi/toml-test -fi -pushd src/github.com/BurntSushi/toml-test -git reset --hard '0.2.0' # use the released version, NOT tip -popd +function git_clone() { + path=$1 + branch=$2 + version=$3 + if [ ! -d "src/$path" ]; then + mkdir -p src/$path + git clone https://$path.git src/$path + fi + pushd src/$path + git checkout "$branch" + git reset --hard "$version" + popd +} + +# get code for BurntSushi TOML validation +# pinning all to 'HEAD' for version 0.3.x work (TODO: pin to commit hash when tests stabilize) +git_clone github.com/BurntSushi/toml master HEAD +git_clone github.com/BurntSushi/toml-test master HEAD #was: 0.2.0 HEAD + +# build the BurntSushi test application go build -o toml-test github.com/BurntSushi/toml-test # vendorize the current lib for testing @@ -23,6 +34,42 @@ cp cmd/*.go src/github.com/pelletier/go-toml/cmd go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go -# Run basic unit tests and then the BurntSushi test suite +# Run basic unit tests go test -v github.com/pelletier/go-toml -./toml-test ./test_program_bin | tee test_out + +# run the entire BurntSushi test suite +if [[ $# -eq 0 ]] ; then + echo "Running all BurntSushi tests" + ./toml-test ./test_program_bin | tee test_out +else + # run a specific test + test=$1 + test_path='src/github.com/BurntSushi/toml-test/tests' + valid_test="$test_path/valid/$test" + invalid_test="$test_path/invalid/$test" + + if [ -e "$valid_test.toml" ]; then + echo "Valid Test TOML for $test:" + echo "====" + cat "$valid_test.toml" + + echo "Valid Test JSON for $test:" + echo "====" + cat "$valid_test.json" + + echo "Go-TOML Output for $test:" + echo "====" + cat "$valid_test.toml" | ./test_program_bin + fi + + if [ -e "$invalid_test.toml" ]; then + echo "Invalid Test TOML for $test:" + echo "====" + cat "$invalid_test.toml" + + echo "Go-TOML Output for $test:" + echo "====" + echo "go-toml Output:" + cat "$invalid_test.toml" | ./test_program_bin + fi +fi