Fixed endless loop for comments which end on EOF. Reorganized tests into basic, complex and error tests. Added support for delimiter-less key/value pairs. Started work on support for \r\n EOL (not done).
diff --git a/lex.go b/lex.go index 45b963c..09312ce 100644 --- a/lex.go +++ b/lex.go
@@ -44,7 +44,6 @@ const ( itemError itemType = iota // error occurred; value is text of error itemEOF - itemDelim // a = or : delimiter char itemKey // a key itemValue // a value ) @@ -182,6 +181,7 @@ // lexBeforeKey scans until a key begins. func lexBeforeKey(l *lexer) stateFn { + // fmt.Println("lexBeforeKey") switch r := l.next(); { case isEOF(r): l.emit(itemEOF) @@ -192,9 +192,7 @@ return lexBeforeKey case isComment(r): - l.acceptRunUntil('\n') - l.ignore() - return lexBeforeKey + return lexComment case isWhitespace(r): l.acceptRun(whitespace) @@ -205,11 +203,27 @@ l.backup() return lexKey } +} +// lexComment scans a comment line. The comment character has already been scanned. +func lexComment(l *lexer) stateFn { + for { + switch r := l.next(); { + case isEOF(r): + l.ignore() + l.emit(itemEOF) + return nil + case isEOL(r): + l.ignore() + return lexBeforeKey + } + } } // lexKey scans the key up to a delimiter func lexKey(l *lexer) stateFn { + // fmt.Println("lexKey") + Loop: for { switch r := l.next(); { @@ -220,7 +234,7 @@ return l.errorf(err.Error()) } - case isKeyTerminationCharacter(r): + case isEndOfKey(r): l.backup() break Loop @@ -237,28 +251,26 @@ } // ignore trailing spaces - l.acceptRun(" ") + l.acceptRun(whitespace) l.ignore() - return lexDelim + return lexBeforeValue } // lexDelim scans the delimiter. We expect to be just before the delimiter. -func lexDelim(l *lexer) stateFn { - if l.next() == eof { - return l.errorf("premature EOF") - } - l.emit(itemDelim) +func lexBeforeValue(l *lexer) stateFn { + // fmt.Println("lexBeforeValue") + l.acceptRun(whitespace) + l.accept(":=") + l.acceptRun(whitespace) + l.ignore() return lexValue } // lexValue scans text until the end of the line. We expect to be just after the delimiter. func lexValue(l *lexer) stateFn { - // ignore leading whitespace - l.acceptRun(whitespace) - l.ignore() + // fmt.Println("lexValue") - // TODO: handle multiline with indent on subsequent lines for { switch r := l.next(); { case isEscape(r): @@ -360,6 +372,11 @@ return r == '#' || r == '!' } +// isEndOfKey reports whether the rune terminates the current key. +func isEndOfKey(r rune) bool { + return strings.ContainsRune(" \f\t\r\n:=", r) +} + // isEOF reports whether we are at EOF. func isEOF(r rune) bool { return r == eof @@ -382,11 +399,6 @@ return strings.ContainsRune(" :=nrt", r) } -// isKeyTerminationCharacter reports whether the rune terminates the current key. -func isKeyTerminationCharacter(r rune) bool { - return strings.ContainsRune(" :=", r) -} - // isWhitespace reports whether the rune is a whitespace character. func isWhitespace(r rune) bool { return strings.ContainsRune(whitespace, r)
diff --git a/parser.go b/parser.go index 448239b..ba3dbed 100644 --- a/parser.go +++ b/parser.go
@@ -29,8 +29,11 @@ break } key := token.val - p.expect(itemDelim) - token = p.expect(itemValue) + token = p.expectOneOf(itemValue, itemEOF) + if token.typ == itemEOF { + props.Set(key, "") + break + } props.Set(key, token.val) }
diff --git a/properties_test.go b/properties_test.go index a0c8102..025aa3f 100644 --- a/properties_test.go +++ b/properties_test.go
@@ -3,7 +3,9 @@ package properties import ( + "flag" "fmt" + "os" "strings" "testing" @@ -12,77 +14,75 @@ func Test(t *testing.T) { TestingT(t) } -type LoadSuite struct{} +type TestSuite struct{} -var _ = Suite(&LoadSuite{}) +var ( + _ = Suite(&TestSuite{}) + verbose = flag.Bool("verbose", false, "Verbose output") +) -func (l *LoadSuite) TestKeyWithEmptyValue(c *C) { - testAllDelimiterCombinations(c, "key", "") +// define test cases in the form of +// {"input", "key1", "value1", "key2", "value2", ...} +var complexTests = [][]string{ + // whitespace prefix + {" key=value", "key", "value"}, // SPACE prefix + {"\fkey=value", "key", "value"}, // FF prefix + {"\tkey=value", "key", "value"}, // TAB prefix + {" \f\tkey=value", "key", "value"}, // mix prefix + + // multiple keys + {"key1=value1\nkey2=value2", "key1", "value1", "key2", "value2"}, + + // blank lines + {"\n\nkey=value\n\n", "key", "value"}, // leading and trailing new lines + + // escaped chars + {"k\\ e\\:y\\= = value", "k e:y=", "value"}, // escaped chars in key + {"key = v\\ a\\:lu\\=e\\n\\r\\t", "key", "v a:lu=e\n\r\t"}, // escaped chars in value + + // unicode literals + {"key\\u2318 = value", "key⌘", "value"}, // unicode literal in key + + // multiline values + {"key = valueA,\\\n valueB", "key", "valueA,valueB"}, // SPACE indent + {"key = valueA,\\\n\f\f\fvalueB", "key", "valueA,valueB"}, // FF indent + {"key = valueA,\\\n\t\t\tvalueB", "key", "valueA,valueB"}, // TAB indent + {"key = valueA,\\\n \f\tvalueB", "key", "valueA,valueB"}, // mix indent + + // comments + {"# this is a comment\n! and so is this\nkey1=value1\nkey#2=value#2\n\nkey!3=value!3\n# and another one\n! and the final one", "key1", "value1", "key#2", "value#2", "key!3", "value!3"}, } -func (l *LoadSuite) TestOneKeyValue(c *C) { - testAllDelimiterCombinations(c, "key", "value") +// define error test cases in the form of +// {"input", "expected error message"} +var errorTests = [][]string{ + {"key", "premature EOF"}, + {"key\\ugh32 = value", "invalid unicode literal"}, } -func (l *LoadSuite) TestTwoKeysAndValues(c *C) { - testKeyValue(c, "key1=value1\nkey2=value2", "key1", "value1", "key2", "value2") +// tests basic single key/value combinations with all possible whitespace, delimiter and newline permutations. +func (l *TestSuite) TestBasic(c *C) { + testAllCombinations(c, "key", "") + testAllCombinations(c, "key", "value") + testAllCombinations(c, "key", "value ") } -func (l *LoadSuite) TestWithBlankLines(c *C) { - testKeyValue(c, "\n\nkey=value\n\n", "key", "value") +func (l *TestSuite) TestComplex(c *C) { + for i, test := range complexTests { + printf("[C%02d] %q %q\n", i, test[0], test[1:]) + testKeyValue(c, test[0], test[1:]...) + } } -func (l *LoadSuite) TestKeyWithWhitespacePrefix(c *C) { - testKeyValue(c, " key=value", "key", "value") - testKeyValue(c, "\fkey=value", "key", "value") - testKeyValue(c, "\tkey=value", "key", "value") - testKeyValue(c, " \f\tkey=value", "key", "value") +func (l *TestSuite) TestErrors(c *C) { + for i, test := range errorTests { + input, msg := test[0], test[1] + printf("[E%02d] %q %q\n", i, input, msg) + testError(c, input, msg) + } } -func (l *LoadSuite) TestWithComments(c *C) { - input := ` -# this is a comment -! and so is this -key1=value1 -key#2=value#2 -key!3=value!3 -# and another one -! and the final one -` - testKeyValue(c, input, "key1", "value1", "key#2", "value#2", "key!3", "value!3") -} - -func (l *LoadSuite) TestValueWithTrailingSpaces(c *C) { - testAllDelimiterCombinations(c, "key", "value ") -} - -func (l *LoadSuite) TestEscapedCharsInKey(c *C) { - testKeyValue(c, "k\\ e\\:y\\= = value", "k e:y=", "value") -} - -func (l *LoadSuite) TestUnicodeLiteralInKey(c *C) { - testKeyValue(c, "key\\u2318 = value", "key⌘", "value") -} - -func (l *LoadSuite) TestEscapedCharsInValue(c *C) { - testKeyValue(c, "key = v\\ a\\:lu\\=e\\n\\r\\t", "key", "v a:lu=e\n\r\t") -} - -func (l *LoadSuite) TestMultilineValue(c *C) { - testKeyValue(c, "key = valueA,\\\n valueB", "key", "valueA,valueB") - testKeyValue(c, "key = valueA,\\\n\fvalueB", "key", "valueA,valueB") - testKeyValue(c, "key = valueA,\\\n\tvalueB", "key", "valueA,valueB") -} - -func (l *LoadSuite) TestFailWithPrematureEOF(c *C) { - testError(c, "key", "premature EOF") -} - -func (l *LoadSuite) TestFailWithInvalidUnicodeLiteralInKey(c *C) { - testError(c, "key\\ugh32 = value", "invalid unicode literal") -} - -func BenchmarkNewPropertiesFromString(b *testing.B) { +func BenchmarkDecoder(b *testing.B) { input := "" for i := 0; i < 1000; i++ { input += fmt.Sprintf("key%d=value%d\n", i, i) @@ -95,11 +95,21 @@ } // tests all combinations of delimiters plus leading and/or trailing spaces. -func testAllDelimiterCombinations(c *C, key, value string) { - delimiters := []string{"=", " =", "= ", " = ", ":", " :", ": ", " : "} - for _, delim := range delimiters { - testKeyValue(c, fmt.Sprintf("%s%s%s", key, delim, value), key, value) - testKeyValue(c, fmt.Sprintf("%s%s%s\n", key, delim, value), key, value) +func testAllCombinations(c *C, key, value string) { + whitespace := []string{" ", "\f", "\t"} + delimiters := []string{"", "=", ":"} + // newlines := []string{"", "\r", "\n", "\r\n"} + newlines := []string{"", "\n"} + for _, dl := range delimiters { + for _, ws1 := range whitespace { + for _, ws2 := range whitespace { + for _, nl := range newlines { + input := fmt.Sprintf("%s%s%s%s%s%s", key, ws1, dl, ws2, value, nl) + printf("%q\n", input) + testKeyValue(c, input, key, value) + } + } + } } } @@ -109,9 +119,12 @@ p, err := d.Decode() c.Assert(err, IsNil) c.Assert(p, NotNil) - c.Assert(p.Len(), Equals, len(keyvalues)/2) + c.Assert(p.Len(), Equals, len(keyvalues)/2, Commentf("Odd number of key/value pairs.")) for i := 0; i < len(keyvalues)/2; i += 2 { - assertKeyValue(c, p, keyvalues[i], keyvalues[i+1]) + key, value := keyvalues[i], keyvalues[i+1] + v, ok := p.Get(key) + c.Assert(ok, Equals, true, Commentf("No key %q for input %q", key, input)) + c.Assert(v, Equals, value, Commentf("Value %q does not match input %q", value, input)) } } @@ -123,8 +136,8 @@ c.Assert(strings.Contains(err.Error(), msg), Equals, true) } -func assertKeyValue(c *C, p *Properties, key, value string) { - v, ok := p.Get(key) - c.Assert(ok, Equals, true) - c.Assert(v, Equals, value) +func printf(format string, args ...interface{}) { + if *verbose { + fmt.Fprintf(os.Stderr, format, args...) + } }