couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tonysu...@apache.org
Subject [01/17] couchdb-mango git commit: Revert "Remove reference to _text indexes"
Date Wed, 26 Aug 2015 16:39:02 GMT
Repository: couchdb-mango
Updated Branches:
  refs/heads/2787-merge-repos [created] 8d496c1e8


Revert "Remove reference to _text indexes"

This reverts commit 955a42c3bbd0502f1623bec29edad59eddd7b2ea.

Conflicts:
	src/mango_error.erl
COUCHDB-2787


Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/fc1e36f3
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/fc1e36f3
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/fc1e36f3

Branch: refs/heads/2787-merge-repos
Commit: fc1e36f354475981a4fce044a160f14251922421
Parents: 677cd2a
Author: Tony Sun <tony.sun@cloudant.com>
Authored: Mon Aug 24 14:17:01 2015 -0700
Committer: Tony Sun <tony.sun@cloudant.com>
Committed: Mon Aug 24 14:17:01 2015 -0700

----------------------------------------------------------------------
 src/mango_cursor.erl      |   3 +-
 src/mango_cursor_text.erl | 307 +++++++++++++++++++++++++++++++++++++++++
 src/mango_error.erl       |  39 ++++++
 src/mango_idx.erl         |  10 +-
 4 files changed, 355 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/fc1e36f3/src/mango_cursor.erl
----------------------------------------------------------------------
diff --git a/src/mango_cursor.erl b/src/mango_cursor.erl
index 0b58c12..545a863 100644
--- a/src/mango_cursor.erl
+++ b/src/mango_cursor.erl
@@ -124,7 +124,8 @@ group_indexes_by_type(Indexes) ->
     % don't suddenly switch indexes for existing client
     % queries.
     CursorModules = [
-        mango_cursor_view
+        mango_cursor_view,
+        mango_cursor_text
     ],
     lists:flatmap(fun(CMod) ->
         case dict:find(CMod, IdxDict) of

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/fc1e36f3/src/mango_cursor_text.erl
----------------------------------------------------------------------
diff --git a/src/mango_cursor_text.erl b/src/mango_cursor_text.erl
new file mode 100644
index 0000000..c774c82
--- /dev/null
+++ b/src/mango_cursor_text.erl
@@ -0,0 +1,307 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_cursor_text).
+
+-export([
+    create/4,
+    explain/1,
+    execute/3
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("dreyfus/include/dreyfus.hrl").
+-include("mango_cursor.hrl").
+-include("mango.hrl").
+
+
+-record(cacc, {
+    selector,
+    dbname,
+    ddocid,
+    idx_name,
+    query_args,
+    bookmark,
+    limit,
+    skip,
+    user_fun,
+    user_acc
+}).
+
+
+create(Db, Indexes, Selector, Opts0) ->
+    Index = case Indexes of
+        [Index0] ->
+            Index0;
+        _ ->
+            ?MANGO_ERROR(multiple_text_indexes)
+    end,
+
+    Opts = unpack_bookmark(Db#db.name, Opts0),
+
+    % Limit the result set size to 50 for Clouseau's
+    % sake. We may want to revisit this.
+    Limit0 = couch_util:get_value(limit, Opts, 50),
+    Limit = if Limit0 < 50 -> Limit0; true -> 50 end,
+    Skip = couch_util:get_value(skip, Opts, 0),
+    Fields = couch_util:get_value(fields, Opts, all_fields),
+
+    {ok, #cursor{
+        db = Db,
+        index = Index,
+        ranges = null,
+        selector = Selector,
+        opts = Opts,
+        limit = Limit,
+        skip = Skip,
+        fields = Fields
+    }}.
+
+
+explain(Cursor) ->
+    #cursor{
+        selector = Selector,
+        opts = Opts
+    } = Cursor,
+    [
+        {'query', mango_selector_text:convert(Selector)},
+        {sort, sort_query(Opts, Selector)}
+    ].
+
+
+execute(Cursor, UserFun, UserAcc) ->
+    #cursor{
+        db = Db,
+        index = Idx,
+        limit = Limit,
+        skip = Skip,
+        selector = Selector,
+        opts = Opts
+    } = Cursor,
+    QueryArgs = #index_query_args{
+        q = mango_selector_text:convert(Selector),
+        sort = sort_query(Opts, Selector),
+        raw_bookmark = true
+    },
+    CAcc = #cacc{
+        selector = Selector,
+        dbname = Db#db.name,
+        ddocid = ddocid(Idx),
+        idx_name = mango_idx:name(Idx),
+        bookmark = get_bookmark(Opts),
+        limit = Limit,
+        skip = Skip,
+        query_args = QueryArgs,
+        user_fun = UserFun,
+        user_acc = UserAcc
+    },
+    try
+        execute(CAcc)
+    catch
+        throw:{stop, FinalCAcc} ->
+            #cacc{
+                bookmark = FinalBM,
+                user_fun = UserFun,
+                user_acc = LastUserAcc
+            } = FinalCAcc,
+            JsonBM = dreyfus_bookmark:pack(FinalBM),
+            Arg = {add_key, bookmark, JsonBM},
+            {_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc),
+            {ok, FinalUserAcc}
+    end.
+
+
+execute(CAcc) ->
+    case search_docs(CAcc) of
+        {ok, Bookmark, []} ->
+            % If we don't have any results from the
+            % query it means the request has paged through
+            % all possible results and the request is over.
+            NewCAcc = CAcc#cacc{bookmark = Bookmark},
+            throw({stop, NewCAcc});
+        {ok, Bookmark, Hits} ->
+            NewCAcc = CAcc#cacc{bookmark = Bookmark},
+            HitDocs = get_json_docs(CAcc#cacc.dbname, Hits),
+            {ok, FinalCAcc} = handle_hits(NewCAcc, HitDocs),
+            execute(FinalCAcc)
+    end.
+
+
+search_docs(CAcc) ->
+    #cacc{
+        dbname = DbName,
+        ddocid = DDocId,
+        idx_name = IdxName
+    } = CAcc,
+    QueryArgs = update_query_args(CAcc),
+    case dreyfus_fabric_search:go(DbName, DDocId, IdxName, QueryArgs) of
+        {ok, Bookmark, _, Hits, _, _} ->
+            {ok, Bookmark, Hits};
+        {error, Reason} ->
+            ?MANGO_ERROR({text_search_error, {error, Reason}})
+    end.
+
+
+handle_hits(CAcc, []) ->
+    {ok, CAcc};
+
+handle_hits(CAcc0, [{Sort, Doc} | Rest]) ->
+    CAcc1 = handle_hit(CAcc0, Sort, Doc),
+    handle_hits(CAcc1, Rest).
+
+
+handle_hit(CAcc0, Sort, Doc) ->
+    #cacc{
+        limit = Limit,
+        skip = Skip
+    } = CAcc0,
+    CAcc1 = update_bookmark(CAcc0, Sort),
+    case mango_selector:match(CAcc1#cacc.selector, Doc) of
+        true when Skip > 0 ->
+            CAcc1#cacc{skip = Skip - 1};
+        true when Limit == 0 ->
+            % We hit this case if the user spcified with a
+            % zero limit. Notice that in this case we need
+            % to return the bookmark from before this match
+            throw({stop, CAcc0});
+        true when Limit == 1 ->
+            NewCAcc = apply_user_fun(CAcc1, Doc),
+            throw({stop, NewCAcc});
+        true when Limit > 1 ->
+            NewCAcc = apply_user_fun(CAcc1, Doc),
+            NewCAcc#cacc{limit = Limit - 1};
+        false ->
+            CAcc1
+    end.
+
+
+apply_user_fun(CAcc, Doc) ->
+    #cacc{
+        user_fun = UserFun,
+        user_acc = UserAcc
+    } = CAcc,
+    case UserFun({row, Doc}, UserAcc) of
+        {ok, NewUserAcc} ->
+            CAcc#cacc{user_acc = NewUserAcc};
+        {stop, NewUserAcc} ->
+            throw({stop, CAcc#cacc{user_acc = NewUserAcc}})
+    end.
+
+
+%% Convert Query to Dreyfus sort specifications
+%% Covert <<"Field">>, <<"desc">> to <<"-Field">>
+%% and append to the dreyfus query
+sort_query(Opts, Selector) ->
+    {sort, {Sort}} = lists:keyfind(sort, 1, Opts),
+    SortList = lists:map(fun(SortField) ->
+        {Dir, RawSortField}  = case SortField of
+            {Field, <<"asc">>} -> {asc, Field};
+            {Field, <<"desc">>} -> {desc, Field};
+            Field when is_binary(Field) -> {asc, Field}
+        end,
+        SField = mango_selector_text:append_sort_type(RawSortField, Selector),
+        case Dir of
+            asc ->
+                SField;
+            desc ->
+                <<"-", SField/binary>>
+        end
+    end, Sort),
+    case SortList of
+        [] -> relevance;
+        _ -> SortList
+    end.
+
+
+get_bookmark(Opts) ->
+    case lists:keyfind(bookmark, 1, Opts) of
+        {_, BM} when is_list(BM), BM /= [] ->
+            BM;
+        _ ->
+            nil
+    end.
+
+
+update_bookmark(CAcc, Sortable) ->
+    BM = CAcc#cacc.bookmark,
+    QueryArgs = CAcc#cacc.query_args,
+    Sort = QueryArgs#index_query_args.sort,
+    NewBM = dreyfus_bookmark:update(Sort, BM, [Sortable]),
+    CAcc#cacc{bookmark = NewBM}.
+
+
+pack_bookmark(Bookmark) ->
+    case dreyfus_bookmark:pack(Bookmark) of
+        null -> nil;
+        Enc -> Enc
+    end.
+
+
+unpack_bookmark(DbName, Opts) ->
+    NewBM = case lists:keyfind(bookmark, 1, Opts) of
+        {_, nil} ->
+            [];
+        {_, Bin} ->
+            try
+                dreyfus_bookmark:unpack(DbName, Bin)
+            catch _:_ ->
+                ?MANGO_ERROR({invalid_bookmark, Bin})
+            end
+    end,
+    lists:keystore(bookmark, 1, Opts, {bookmark, NewBM}).
+
+
+ddocid(Idx) ->
+    case mango_idx:ddoc(Idx) of
+        <<"_design/", Rest/binary>> ->
+            Rest;
+        Else ->
+            Else
+    end.
+
+
+update_query_args(CAcc) ->
+    #cacc{
+        bookmark = Bookmark,
+        query_args = QueryArgs
+    } = CAcc,
+    QueryArgs#index_query_args{
+        bookmark = pack_bookmark(Bookmark),
+        limit = get_limit(CAcc)
+    }.
+
+
+get_limit(CAcc) ->
+    Total = CAcc#cacc.limit + CAcc#cacc.skip,
+    if
+        Total < 25 -> 25;
+        Total > 100 -> 100;
+        true -> Total
+    end.
+
+
+get_json_docs(DbName, Hits) ->
+    Ids = lists:map(fun(#sortable{item = Item}) ->
+        couch_util:get_value(<<"_id">>, Item#hit.fields)
+    end, Hits),
+    {ok, IdDocs} = dreyfus_fabric:get_json_docs(DbName, Ids),
+    lists:map(fun(#sortable{item = Item} = Sort) ->
+        Id = couch_util:get_value(<<"_id">>, Item#hit.fields),
+        case lists:keyfind(Id, 1, IdDocs) of
+            {Id, {doc, Doc}} ->
+                {Sort, Doc};
+            false ->
+                {Sort, not_found}
+        end
+    end, Hits).
+

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/fc1e36f3/src/mango_error.erl
----------------------------------------------------------------------
diff --git a/src/mango_error.erl b/src/mango_error.erl
index 69767cf..cf117ab 100644
--- a/src/mango_error.erl
+++ b/src/mango_error.erl
@@ -46,6 +46,32 @@ info(mango_cursor, {no_usable_index, selector_unsupported}) ->
         <<"There is no index available for this selector.">>
     };
 
+info(mango_cursor_text, {invalid_bookmark, BadBookmark}) ->
+    {
+        400,
+        <<"invalid_bookmark">>,
+        fmt("Invalid boomkark value: ~s", [?JSON_ENCODE(BadBookmark)])
+    };
+info(mango_cursor_text, multiple_text_indexes) ->
+    {
+        400,
+        <<"multiple_text_indexes">>,
+        <<"You must specify an index with the `use_index` parameter.">>
+    };
+info(mango_cursor_text, {text_search_error, {error, {bad_request, Msg}}}) 
+        when is_binary(Msg) ->
+    {
+        400,
+        <<"text_search_error">>,
+        Msg
+    };
+info(mango_cursor_text, {text_search_error, {error, Error}}) ->
+    {
+        400,
+        <<"text_search_error">>,
+        fmt("Error performing text search: ~p", [Error])
+    };
+
 info(mango_fields, {invalid_fields_json, BadFields}) ->
     {
         400,
@@ -122,6 +148,19 @@ info(mango_idx_view, {index_not_found, BadIdx}) ->
         fmt("JSON index ~s not found in this design doc.", [BadIdx])
     };
 
+info(mango_idx_text, {invalid_index_text, BadIdx}) ->
+    {
+        400,
+        <<"invalid_index">>,
+        fmt("Text indexes must be an object, not: ~w", [BadIdx])
+    };
+info(mango_idx_text, {index_not_found, BadIdx}) ->
+    {
+        404,
+        <<"index_not_found">>,
+        fmt("Text index ~s not found in this design doc.", [BadIdx])
+    };
+
 info(mango_opts, {invalid_bulk_docs, Val}) ->
     {
         400,

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/fc1e36f3/src/mango_idx.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx.erl b/src/mango_idx.erl
index f6e688b..71a55a9 100644
--- a/src/mango_idx.erl
+++ b/src/mango_idx.erl
@@ -167,7 +167,7 @@ from_ddoc(Db, {Props}) ->
             ?MANGO_ERROR(invalid_query_ddoc_language)
     end,
 
-    IdxMods = [mango_idx_view],
+    IdxMods = [mango_idx_view, mango_idx_text],
     Idxs = lists:flatmap(fun(Mod) -> Mod:from_ddoc({Props}) end, IdxMods),
     lists:map(fun(Idx) ->
         Idx#idx{
@@ -241,13 +241,17 @@ end_key(#idx{}=Idx, Ranges) ->
 cursor_mod(#idx{type = <<"json">>}) ->
     mango_cursor_view;
 cursor_mod(#idx{def = all_docs, type= <<"special">>}) ->
-    mango_cursor_view.
+    mango_cursor_view;
+cursor_mod(#idx{type = <<"text">>}) ->
+    mango_cursor_text.
 
 
 idx_mod(#idx{type = <<"json">>}) ->
     mango_idx_view;
 idx_mod(#idx{type = <<"special">>}) ->
-    mango_idx_special.
+    mango_idx_special;
+idx_mod(#idx{type = <<"text">>}) ->
+    mango_idx_text.
 
 
 db_to_name(#db{name=Name}) ->


Mime
View raw message