Replace element position with brackets To be consistent with view based indexes, we allow the user to access array elements via position. We convert that position to [] for the underlying indexed field. For example, myarray.0.1, would be converted to myarray.[].[]. This converted field will then be used for the text search. The results will then be filtered by the match function.
Bugzid: 46268 Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/460fcd47 Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/460fcd47 Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/460fcd47 Branch: refs/heads/2787-merge-repos Commit: 460fcd4723a70ba24aee2b0254ba02602260431a Parents: f481b56 Author: Tony Sun <tony....@cloudant.com> Authored: Wed Aug 26 01:30:27 2015 -0700 Committer: Tony Sun <tony....@cloudant.com> Committed: Wed Aug 26 01:30:27 2015 -0700 ---------------------------------------------------------------------- src/mango_doc.erl | 47 +++------------------------------- src/mango_fields.erl | 2 +- src/mango_selector_text.erl | 54 +++++++++++++++++++++++++++++++++++----- src/mango_util.erl | 47 ++++++++++++++++++++++++++++++++-- test/06-basic-text-test.py | 22 ++++++++++++++++ test/user_docs.py | 4 ++- 6 files changed, 122 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/460fcd47/src/mango_doc.erl ---------------------------------------------------------------------- diff --git a/src/mango_doc.erl b/src/mango_doc.erl index 479a8ad..c22b155 100644 --- a/src/mango_doc.erl +++ b/src/mango_doc.erl @@ -22,7 +22,6 @@ get_field/2, get_field/3, - parse_field/1, rem_field/2, set_field/3 ]). @@ -373,7 +372,7 @@ get_field(Props, Field) -> get_field(Props, Field, Validator) when is_binary(Field) -> - {ok, Path} = parse_field(Field), + {ok, Path} = mango_util:parse_field(Field), get_field(Props, Path, Validator); get_field(Props, [], no_validation) -> Props; @@ -411,7 +410,7 @@ get_field(_, [_|_], _) -> rem_field(Props, Field) when is_binary(Field) -> - {ok, Path} = parse_field(Field), + {ok, Path} = mango_util:parse_field(Field), rem_field(Props, Path); rem_field({Props}, [Name]) -> case lists:keytake(Name, 1, Props) of @@ -472,7 +471,7 @@ rem_field(_, [_|_]) -> set_field(Props, Field, Value) when is_binary(Field) -> - {ok, Path} = parse_field(Field), + {ok, Path} = mango_util:parse_field(Field), set_field(Props, Path, Value); set_field({Props}, [Name], Value) -> {lists:keystore(Name, 1, Props, {Name, Value})}; @@ -536,43 +535,3 @@ set_elem(1, [_ | Rest], Value) -> [Value | Rest]; set_elem(I, [Item | Rest], Value) when I > 1 -> [Item | set_elem(I-1, Rest, Value)]. - -parse_field(Field) -> - case binary:match(Field, <<"\\">>, []) of - nomatch -> - % Fast path, no regex required - {ok, check_non_empty(Field, binary:split(Field, <<".">>, [global]))}; - _ -> - parse_field_slow(Field) - end. - -parse_field_slow(Field) -> - Path = lists:map(fun - (P) when P =:= <<>> -> - ?MANGO_ERROR({invalid_field_name, Field}); - (P) -> - re:replace(P, <<"\\\\">>, <<>>, [global, {return, binary}]) - end, re:split(Field, <<"(?<!\\\\)\\.">>)), - {ok, Path}. - -check_non_empty(Field, Parts) -> - case lists:member(<<>>, Parts) of - true -> - ?MANGO_ERROR({invalid_field_name, Field}); - false -> - Parts - end. - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -parse_field_test() -> - ?assertEqual({ok, [<<"ab">>]}, parse_field(<<"ab">>)), - ?assertEqual({ok, [<<"a">>, <<"b">>]}, parse_field(<<"a.b">>)), - ?assertEqual({ok, [<<"a.b">>]}, parse_field(<<"a\\.b">>)), - ?assertEqual({ok, [<<"a">>, <<"b">>, <<"c">>]}, parse_field(<<"a.b.c">>)), - ?assertEqual({ok, [<<"a">>, <<"b.c">>]}, parse_field(<<"a.b\\.c">>)), - Exception = {mango_error, ?MODULE, {invalid_field_name, <<"a..b">>}}, - ?assertThrow(Exception, parse_field(<<"a..b">>)). - --endif. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/460fcd47/src/mango_fields.erl ---------------------------------------------------------------------- diff --git a/src/mango_fields.erl b/src/mango_fields.erl index 8b6a00b..2732560 100644 --- a/src/mango_fields.erl +++ b/src/mango_fields.erl @@ -35,7 +35,7 @@ extract(Doc, all_fields) -> Doc; extract(Doc, Fields) -> lists:foldl(fun(F, NewDoc) -> - {ok, Path} = mango_doc:parse_field(F), + {ok, Path} = mango_util:parse_field(F), case mango_doc:get_field(Doc, Path) of not_found -> NewDoc; http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/460fcd47/src/mango_selector_text.erl ---------------------------------------------------------------------- diff --git a/src/mango_selector_text.erl b/src/mango_selector_text.erl index b7d78e4..e61f759 100644 --- a/src/mango_selector_text.erl +++ b/src/mango_selector_text.erl @@ -25,6 +25,12 @@ -include("mango.hrl"). +%% Regex for <<"\\.">> +-define(PERIOD, {re_pattern,0,0,<<69,82,67,80,57,0,0,0,0,0,0,0,2,0,0,0, +0,0,0,0,46,0,0,0,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,93,0, +5,27,46,84,0,5,0>>}). + + convert(Object) -> TupleTree = convert([], Object), iolist_to_binary(to_query(TupleTree)). @@ -159,12 +165,35 @@ convert(Path, {[{<<"$size">>, Arg}]}) -> convert(_Path, {[{<<"$", _/binary>>=Op, _}]}) -> ?MANGO_ERROR({invalid_operator, Op}); -% We've hit a field name specifier. We need to break the name -% into path parts and continue our conversion. -convert(Path, {[{Field, Cond}]}) -> - NewPathParts = re:split(Field, <<"\\.">>), - NewPath = lists:reverse(NewPathParts) ++ Path, - convert(NewPath, Cond); +% We've hit a field name specifier. Check if the field name is accessing +% arrays. Convert occurrences of element position references to .[]. Then we +% need to break the name into path parts and continue our conversion. +convert(Path, {[{Field0, Cond}]}) -> + {ok, PP0} = case Field0 of + <<>> -> + {ok, []}; + _ -> + mango_util:parse_field(Field0) + end, + % Later on, we perform a lucene_escape_user call on the + % final Path, which calls parse_field again. Calling the function + % twice converts <<"a\\.b">> to [<<"a">>,<<"b">>]. This leads to + % an incorrect query since we need [<<"a.b">>]. Without breaking + % our escaping mechanism, we simply revert this first parse_field + % effect and replace instances of "." to "\\.". + PP1 = [re:replace(P, ?PERIOD, <<"\\\\.">>, + [global,{return,binary}]) || P <- PP0], + {PP2, HasInteger} = replace_array_indexes(PP1, [], false), + NewPath = PP2 ++ Path, + case HasInteger of + true -> + OldPath = lists:reverse(PP1, Path), + OldParts = convert(OldPath, Cond), + NewParts = convert(NewPath, Cond), + {op_or, [OldParts, NewParts]}; + false -> + convert(NewPath, Cond) + end; %% For $in convert(Path, Val) when is_binary(Val); is_number(Val); is_boolean(Val) -> @@ -362,3 +391,16 @@ get_sort_types(Field, {[{_, Cond}]}, Acc) when is_tuple(Cond)-> get_sort_types(_Field, _, Acc) -> Acc. + + +replace_array_indexes([], NewPartsAcc, HasIntAcc) -> + {NewPartsAcc, HasIntAcc}; +replace_array_indexes([Part | Rest], NewPartsAcc, HasIntAcc) -> + {NewPart, HasInt} = try + _ = list_to_integer(binary_to_list(Part)), + {<<"[]">>, true} + catch _:_ -> + {Part, false} + end, + replace_array_indexes(Rest, [NewPart | NewPartsAcc], + HasInt or HasIntAcc). http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/460fcd47/src/mango_util.erl ---------------------------------------------------------------------- diff --git a/src/mango_util.erl b/src/mango_util.erl index 0e06f66..aa11310 100644 --- a/src/mango_util.erl +++ b/src/mango_util.erl @@ -38,7 +38,9 @@ has_suffix/2, - join/2 + join/2, + + parse_field/1 ]). @@ -343,7 +345,7 @@ lucene_escape_qv(<<C, Rest/binary>>) -> lucene_escape_user(Field) -> - {ok, Path} = mango_doc:parse_field(Field), + {ok, Path} = parse_field(Field), Escaped = [mango_util:lucene_escape_field(P) || P <- Path], iolist_to_binary(join(".", Escaped)). @@ -377,3 +379,44 @@ is_number_string(Value) when is_list(Value)-> _ -> true end. + + +parse_field(Field) -> + case binary:match(Field, <<"\\">>, []) of + nomatch -> + % Fast path, no regex required + {ok, check_non_empty(Field, binary:split(Field, <<".">>, [global]))}; + _ -> + parse_field_slow(Field) + end. + +parse_field_slow(Field) -> + Path = lists:map(fun + (P) when P =:= <<>> -> + ?MANGO_ERROR({invalid_field_name, Field}); + (P) -> + re:replace(P, <<"\\\\">>, <<>>, [global, {return, binary}]) + end, re:split(Field, <<"(?<!\\\\)\\.">>)), + {ok, Path}. + +check_non_empty(Field, Parts) -> + case lists:member(<<>>, Parts) of + true -> + ?MANGO_ERROR({invalid_field_name, Field}); + false -> + Parts + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +parse_field_test() -> + ?assertEqual({ok, [<<"ab">>]}, parse_field(<<"ab">>)), + ?assertEqual({ok, [<<"a">>, <<"b">>]}, parse_field(<<"a.b">>)), + ?assertEqual({ok, [<<"a.b">>]}, parse_field(<<"a\\.b">>)), + ?assertEqual({ok, [<<"a">>, <<"b">>, <<"c">>]}, parse_field(<<"a.b.c">>)), + ?assertEqual({ok, [<<"a">>, <<"b.c">>]}, parse_field(<<"a.b\\.c">>)), + Exception = {mango_error, ?MODULE, {invalid_field_name, <<"a..b">>}}, + ?assertThrow(Exception, parse_field(<<"a..b">>)). + +-endif. http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/460fcd47/test/06-basic-text-test.py ---------------------------------------------------------------------- diff --git a/test/06-basic-text-test.py b/test/06-basic-text-test.py index 28538ad..d44e340 100644 --- a/test/06-basic-text-test.py +++ b/test/06-basic-text-test.py @@ -59,6 +59,28 @@ class BasicTextTests(mango.UserDocsTextTests): assert docs[0]["name"]["first"] == "Stephanie" assert docs[0]["favorites"] == faves + def test_array_ref(self): + docs = self.db.find({"favorites.1": "Python"}) + assert len(docs) == 4 + for d in docs: + assert "Python" in d["favorites"] + + # Nested Level + docs = self.db.find({"favorites.0.2": "Python"}) + print len(docs) + assert len(docs) == 1 + for d in docs: + assert "Python" in d["favorites"][0][2] + + def test_number_ref(self): + docs = self.db.find({"11111": "number_field"}) + assert len(docs) == 1 + assert docs[0]["11111"] == "number_field" + + docs = self.db.find({"22222.33333": "nested_number_field"}) + assert len(docs) == 1 + assert docs[0]["22222"]["33333"] == "nested_number_field" + def test_lt(self): docs = self.db.find({"age": {"$lt": 22}}) assert len(docs) == 0 http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/460fcd47/test/user_docs.py ---------------------------------------------------------------------- diff --git a/test/user_docs.py b/test/user_docs.py index baf83f7..e2f1705 100644 --- a/test/user_docs.py +++ b/test/user_docs.py @@ -197,7 +197,9 @@ DOCS = [ "Erlang", "C", "Erlang" - ] + ], + "11111": "number_field", + "22222": {"33333" : "nested_number_field"} }, { "_id": "8e1c90c0-ac18-4832-8081-40d14325bde0",
