Query interface with callback functions * Added public Query interface * Added filter function callback support * Added "script" function callback support Queries are generated via Compile(), which then may be run via Execute() as many times as needed. Much like compiling a regex, this is done to elide the need to re-parse and build the funciton tree for each execution. The distinction between 'filter' and 'script' is borrowed from their syntactic equivalents in jsonpath. Right now, these accept no arguments in the query, and instead merely pass the current node to the callback. Filters return a bool and determine if the node is kept or culled out. 'Scripts' return a string or an in64, which is in turn used in an index or key filter (respectively) on the current node's data. A few callbacks are provided by default, with the ability to add additional callbacks before calling Execute() on a compiled query.
diff --git a/jpath/match.go b/jpath/match.go index ffde749..345bbf4 100644 --- a/jpath/match.go +++ b/jpath/match.go
@@ -2,61 +2,9 @@ import ( . "github.com/pelletier/go-toml" + "fmt" ) -// result set for storage of results -type pathResult struct { - Values []interface{} -} - -func newPathResult() *pathResult { - return &pathResult { - Values: []interface{}{}, - } -} - -func (r *pathResult) Append(value interface{}) { - r.Values = append(r.Values, value) -} - -// generic path functor interface -type PathFn interface{ - SetNext(next PathFn) - Call(context interface{}, results *pathResult) -} - -// contains a functor chain -type QueryPath struct { - root PathFn - tail PathFn -} - -func newQueryPath() *QueryPath { - return &QueryPath { - root: nil, - tail: nil, - } -} - -func (path *QueryPath) Append(next PathFn) { - if path.root == nil { - path.root = next - } else { - path.tail.SetNext(next) - } - path.tail = next - next.SetNext(newTerminatingFn()) // init the next functor -} - -func (path *QueryPath) Call(context interface{}) []interface{} { - results := newPathResult() - if path.root == nil { - results.Append(context) // identity query for no predicates - } else { - path.root.Call(context, results) - } - return results.Values -} // base match type matchBase struct { @@ -80,8 +28,8 @@ // do nothing } -func (f *terminatingFn) Call(context interface{}, results *pathResult) { - results.Append(context) +func (f *terminatingFn) Call(node interface{}, ctx *queryContext) { + ctx.appendResult(node) } // shim to ease functor writing @@ -99,11 +47,11 @@ return &matchKeyFn{ Name: name } } -func (f *matchKeyFn) Call(context interface{}, results *pathResult) { - if tree, ok := context.(*TomlTree); ok { +func (f *matchKeyFn) Call(node interface{}, ctx *queryContext) { + if tree, ok := node.(*TomlTree); ok { item := treeValue(tree, f.Name) if item != nil { - f.next.Call(item, results) + f.next.Call(item, ctx) } } } @@ -118,10 +66,10 @@ return &matchIndexFn{ Idx: idx } } -func (f *matchIndexFn) Call(context interface{}, results *pathResult) { - if arr, ok := context.([]interface{}); ok { +func (f *matchIndexFn) Call(node interface{}, ctx *queryContext) { + if arr, ok := node.([]interface{}); ok { if f.Idx < len(arr) && f.Idx >= 0 { - f.next.Call(arr[f.Idx], results) + f.next.Call(arr[f.Idx], ctx) } } } @@ -136,8 +84,8 @@ return &matchSliceFn{ Start: start, End: end, Step: step } } -func (f *matchSliceFn) Call(context interface{}, results *pathResult) { - if arr, ok := context.([]interface{}); ok { +func (f *matchSliceFn) Call(node interface{}, ctx *queryContext) { + if arr, ok := node.([]interface{}); ok { // adjust indexes for negative values, reverse ordering realStart, realEnd := f.Start, f.End if realStart < 0 { @@ -151,7 +99,7 @@ } // loop and gather for idx := realStart; idx < realEnd; idx += f.Step { - f.next.Call(arr[idx], results) + f.next.Call(arr[idx], ctx) } } } @@ -159,18 +107,17 @@ // match anything type matchAnyFn struct { matchBase - // empty } func newMatchAnyFn() *matchAnyFn { return &matchAnyFn{} } -func (f *matchAnyFn) Call(context interface{}, results *pathResult) { - if tree, ok := context.(*TomlTree); ok { +func (f *matchAnyFn) Call(node interface{}, ctx *queryContext) { + if tree, ok := node.(*TomlTree); ok { for _, key := range tree.Keys() { item := treeValue(tree, key) - f.next.Call(item, results) + f.next.Call(item, ctx) } } } @@ -186,9 +133,9 @@ } } -func (f *matchUnionFn) Call(context interface{}, results *pathResult) { +func (f *matchUnionFn) Call(node interface{}, ctx *queryContext) { for _, fn := range f.Union { - fn.Call(context, results) + fn.Call(node, ctx) } } @@ -201,13 +148,13 @@ return &matchRecursiveFn{} } -func (f *matchRecursiveFn) Call(context interface{}, results *pathResult) { - if tree, ok := context.(*TomlTree); ok { +func (f *matchRecursiveFn) Call(node interface{}, ctx *queryContext) { + if tree, ok := node.(*TomlTree); ok { var visit func(tree *TomlTree) visit = func(tree *TomlTree) { for _, key := range tree.Keys() { item := treeValue(tree, key) - f.next.Call(item, results) + f.next.Call(item, ctx) switch node := item.(type) { case *TomlTree: visit(node) @@ -221,3 +168,67 @@ visit(tree) } } + +// match based on an externally provided functional filter +type matchFilterFn struct { + matchBase + Pos Position + Name string +} + +func newMatchFilterFn(name string, pos Position) *matchFilterFn { + return &matchFilterFn{ Name: name, Pos: pos } +} + +func (f *matchFilterFn) Call(node interface{}, ctx *queryContext) { + fn, ok := (*ctx.filters)[f.Name] + if !ok { + panic(fmt.Sprintf("%s: query context does not have filter '%s'", + f.Pos, f.Name)) + } + switch castNode := node.(type) { + case *TomlTree: + for _, k := range castNode.Keys() { + v := castNode.GetPath([]string{k}) + if fn(v) { + f.next.Call(v, ctx) + } + } + case []interface{}: + for _, v := range castNode { + if fn(v) { + f.next.Call(v, ctx) + } + } + } +} + +// match based using result of an externally provided functional filter +type matchScriptFn struct { + matchBase + Pos Position + Name string +} + +func newMatchScriptFn(name string, pos Position) *matchScriptFn { + return &matchScriptFn{ Name: name, Pos: pos } +} + +func (f *matchScriptFn) Call(node interface{}, ctx *queryContext) { + fn, ok := (*ctx.scripts)[f.Name] + if !ok { + panic(fmt.Sprintf("%s: query context does not have script '%s'", + f.Pos, f.Name)) + } + switch result := fn(node).(type) { + case string: + nextMatch := newMatchKeyFn(result) + nextMatch.SetNext(f.next) + nextMatch.Call(node, ctx) + case int: + nextMatch := newMatchIndexFn(result) + nextMatch.SetNext(f.next) + nextMatch.Call(node, ctx) + //TODO: support other return types? + } +}
diff --git a/jpath/match_test.go b/jpath/match_test.go index 514962b..9509ae9 100644 --- a/jpath/match_test.go +++ b/jpath/match_test.go
@@ -1,6 +1,7 @@ package jpath import ( + . "github.com/pelletier/go-toml" "fmt" "math" "testing" @@ -8,7 +9,7 @@ // dump path tree to a string func pathString(root PathFn) string { - result := fmt.Sprintf("%T:") + result := fmt.Sprintf("%T:", root) switch fn := root.(type) { case *terminatingFn: result += "{}" @@ -28,21 +29,27 @@ case *matchUnionFn: result += "{[" for _, v := range fn.Union { - result += pathString(v) + result += pathString(v) + ", " } result += "]}" case *matchRecursiveFn: result += "{}" result += pathString(fn.next) + case *matchFilterFn: + result += fmt.Sprintf("{%s}", fn.Name) + result += pathString(fn.next) + case *matchScriptFn: + result += fmt.Sprintf("{%s}", fn.Name) + result += pathString(fn.next) } return result } -func assertPathMatch(t *testing.T, path, ref *QueryPath) bool { +func assertPathMatch(t *testing.T, path, ref *Query) bool { pathStr := pathString(path.root) refStr := pathString(ref.root) if pathStr != refStr { - t.Errorf("paths do not match: %v vs %v") + t.Errorf("paths do not match") t.Log("test:", pathStr) t.Log("ref: ", refStr) return false @@ -50,18 +57,18 @@ return true } -func assertPath(t *testing.T, query string, ref *QueryPath) { +func assertPath(t *testing.T, query string, ref *Query) { _, flow := lex(query) path := parse(flow) assertPathMatch(t, path, ref) } -func buildPath(parts... PathFn) *QueryPath { - path := newQueryPath() +func buildPath(parts... PathFn) *Query { + query := newQuery() for _, v := range parts { - path.Append(v) + query.appendPath(v) } - return path + return query } func TestPathRoot(t *testing.T) { @@ -187,3 +194,25 @@ newMatchRecursiveFn(), )) } + +func TestPathFilterExpr(t *testing.T) { + assertPath(t, + "$[?('foo'),?(bar)]", + buildPath( + &matchUnionFn{ []PathFn { + newMatchFilterFn("foo", Position{}), + newMatchFilterFn("bar", Position{}), + }}, + )) +} + +func TestPathScriptExpr(t *testing.T) { + assertPath(t, + "$[('foo'),(bar)]", + buildPath( + &matchUnionFn{ []PathFn { + newMatchScriptFn("foo", Position{}), + newMatchScriptFn("bar", Position{}), + }}, + )) +}
diff --git a/jpath/parser.go b/jpath/parser.go index 7b842a4..f59ba0c 100644 --- a/jpath/parser.go +++ b/jpath/parser.go
@@ -15,7 +15,7 @@ type parser struct { flow chan token tokensBuffer []token - path *QueryPath + path *Query union []PathFn } @@ -102,12 +102,12 @@ tok := p.getToken() switch tok.typ { case tokenDotDot: - p.path.Append(&matchRecursiveFn{}) + p.path.appendPath(&matchRecursiveFn{}) // nested parse for '..' tok := p.getToken() switch tok.typ { case tokenKey: - p.path.Append(newMatchKeyFn(tok.val)) + p.path.appendPath(newMatchKeyFn(tok.val)) return parseMatchExpr case tokenLBracket: return parseBracketExpr @@ -121,10 +121,10 @@ tok := p.getToken() switch tok.typ { case tokenKey: - p.path.Append(newMatchKeyFn(tok.val)) + p.path.appendPath(newMatchKeyFn(tok.val)) return parseMatchExpr case tokenStar: - p.path.Append(&matchAnyFn{}) + p.path.appendPath(&matchAnyFn{}) return parseMatchExpr } @@ -149,6 +149,8 @@ } func parseUnionExpr(p *parser) parserStateFn { + var tok *token + // this state can be traversed after some sub-expressions // so be careful when setting up state in the parser if p.union == nil { @@ -157,8 +159,21 @@ loop: // labeled loop for easy breaking for { + if len(p.union) > 0 { + // parse delimiter or terminator + tok = p.getToken() + switch tok.typ { + case tokenComma: + // do nothing + case tokenRBracket: + break loop + default: + p.raiseError(tok, "expected ',' or ']', not '%s'", tok.val) + } + } + // parse sub expression - tok := p.getToken() + tok = p.getToken() switch tok.typ { case tokenInteger: p.union = append(p.union, newMatchIndexFn(tok.Int())) @@ -171,25 +186,15 @@ case tokenLParen: return parseScriptExpr default: - p.raiseError(tok, "expected union sub expression, not '%s'", tok.val) - } - // parse delimiter or terminator - tok = p.getToken() - switch tok.typ { - case tokenComma: - continue - case tokenRBracket: - break loop - default: - p.raiseError(tok, "expected ',' or ']'") + p.raiseError(tok, "expected union sub expression, not '%s', %d", tok.val, len(p.union)) } } // if there is only one sub-expression, use that instead if len(p.union) == 1 { - p.path.Append(p.union[0]) + p.path.appendPath(p.union[0]) }else { - p.path.Append(&matchUnionFn{p.union}) + p.path.appendPath(&matchUnionFn{p.union}) } p.union = nil // clear out state @@ -217,7 +222,7 @@ tok = p.getToken() } if tok.typ == tokenRBracket { - p.path.Append(newMatchSliceFn(start, end, step)) + p.path.appendPath(newMatchSliceFn(start, end, step)) return parseMatchExpr } if tok.typ != tokenColon { @@ -237,25 +242,47 @@ p.raiseError(tok, "expected ']'") } - p.path.Append(newMatchSliceFn(start, end, step)) + p.path.appendPath(newMatchSliceFn(start, end, step)) return parseMatchExpr } func parseFilterExpr(p *parser) parserStateFn { - p.raiseError(p.peek(), "filter expressions are unsupported") - return nil + tok := p.getToken() + if tok.typ != tokenLParen { + p.raiseError(tok, "expected left-parenthesis for filter expression") + } + tok = p.getToken() + if tok.typ != tokenKey && tok.typ != tokenString { + p.raiseError(tok, "expected key or string for filter funciton name") + } + name := tok.val + tok = p.getToken() + if tok.typ != tokenRParen { + p.raiseError(tok, "expected right-parenthesis for filter expression") + } + p.union = append(p.union, newMatchFilterFn(name, tok.Position)) + return parseUnionExpr } func parseScriptExpr(p *parser) parserStateFn { - p.raiseError(p.peek(), "script expressions are unsupported") - return nil + tok := p.getToken() + if tok.typ != tokenKey && tok.typ != tokenString { + p.raiseError(tok, "expected key or string for script funciton name") + } + name := tok.val + tok = p.getToken() + if tok.typ != tokenRParen { + p.raiseError(tok, "expected right-parenthesis for script expression") + } + p.union = append(p.union, newMatchScriptFn(name, tok.Position)) + return parseUnionExpr } -func parse(flow chan token) *QueryPath { +func parse(flow chan token) *Query { parser := &parser{ flow: flow, tokensBuffer: []token{}, - path: newQueryPath(), + path: newQuery(), } parser.run() return parser.path
diff --git a/jpath/parser_test.go b/jpath/parser_test.go index 3103970..6d3c941 100644 --- a/jpath/parser_test.go +++ b/jpath/parser_test.go
@@ -12,14 +12,8 @@ t.Errorf("Non-nil toml parse error: %v", err) return } - _, flow := lex(query) - if err != nil { - t.Errorf("Non-nil query lex error: %v", err) - return - } - path := parse(flow) - result := path.Call(tree) - assertValue(t, result, ref, "((" + query + ")) -> ") + results := Compile(query).Execute(tree) + assertValue(t, results, ref, "((" + query + ")) -> ") } func assertValue(t *testing.T, result, ref interface{}, location string) { @@ -233,3 +227,38 @@ }, }) } + +func TestQueryScriptFnLast(t *testing.T) { + assertQuery(t, + "[foo]\na = [0,1,2,3,4,5,6,7,8,9]", + "$.foo.a[(last)]", + []interface{}{ + int64(9), + }) +} + +func TestQueryFilterFnOdd(t *testing.T) { + assertQuery(t, + "[foo]\na = [0,1,2,3,4,5,6,7,8,9]", + "$.foo.a[?(odd)]", + []interface{}{ + int64(1), + int64(3), + int64(5), + int64(7), + int64(9), + }) +} + +func TestQueryFilterFnEven(t *testing.T) { + assertQuery(t, + "[foo]\na = [0,1,2,3,4,5,6,7,8,9]", + "$.foo.a[?(even)]", + []interface{}{ + int64(0), + int64(2), + int64(4), + int64(6), + int64(8), + }) +}
diff --git a/jpath/query.go b/jpath/query.go new file mode 100644 index 0000000..082fa69 --- /dev/null +++ b/jpath/query.go
@@ -0,0 +1,117 @@ +package jpath + +import ( + _ "github.com/pelletier/go-toml" +) + +type nodeFilterFn func(node interface{}) bool +type nodeFn func(node interface{}) interface{} + +// runtime context for executing query paths +type queryContext struct { + filters *map[string]nodeFilterFn + scripts *map[string]nodeFn + results []interface{} +} + +func (c *queryContext) appendResult(value interface{}) { + c.results = append(c.results, value) +} + +// generic path functor interface +type PathFn interface { + SetNext(next PathFn) + Call(node interface{}, ctx *queryContext) +} + +// encapsulates a query functor chain and script callbacks +type Query struct { + root PathFn + tail PathFn + filters *map[string]nodeFilterFn + scripts *map[string]nodeFn +} + +func newQuery() *Query { + return &Query { + root: nil, + tail: nil, + filters: &defaultFilterFunctions, + scripts: &defaultScriptFunctions, + } +} + +func (q *Query) appendPath(next PathFn) { + if q.root == nil { + q.root = next + } else { + q.tail.SetNext(next) + } + q.tail = next + next.SetNext(newTerminatingFn()) // init the next functor +} + +func Compile(path string) *Query { + _, flow := lex(path) + return parse(flow) +} + +func (q *Query) Execute(node interface{}) interface{} { + if q.root == nil { + return []interface{}{node} // identity query for no predicates + } + ctx := &queryContext { + filters: q.filters, + scripts: q.scripts, + results: []interface{}{}, + } + q.root.Call(node, ctx) + return ctx.results +} + +func (q *Query) SetFilter(name string, fn nodeFilterFn) { + if q.filters == &defaultFilterFunctions { + // clone the static table + q.filters = &map[string]nodeFilterFn{} + for k, v := range defaultFilterFunctions { + (*q.filters)[k] = v + } + } + (*q.filters)[name] = fn +} + +func (q *Query) SetScript(name string, fn nodeFn) { + if q.scripts == &defaultScriptFunctions { + // clone the static table + q.scripts = &map[string]nodeFn{} + for k, v := range defaultScriptFunctions { + (*q.scripts)[k] = v + } + } + (*q.scripts)[name] = fn +} + +var defaultFilterFunctions = map[string]nodeFilterFn { + "odd": func(node interface{}) bool { + if ii, ok := node.(int64); ok { + return (ii & 1) == 1 + } + return false + }, + "even": func(node interface{}) bool { + if ii, ok := node.(int64); ok { + return (ii & 1) == 0 + } + return false + }, +} + +var defaultScriptFunctions = map[string]nodeFn { + "last": func(node interface{}) interface{} { + if arr, ok := node.([]interface{}); ok { + return len(arr)-1 + } + return nil + }, +} +