couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chewbra...@apache.org
Subject couch-mrview commit: updated refs/heads/1993-bigcouch-couch-mrview to 22ec9d9
Date Fri, 14 Feb 2014 20:38:05 GMT
Updated Branches:
  refs/heads/1993-bigcouch-couch-mrview f980f2503 -> 22ec9d906 (forced update)


WIP: support multi query views

COUCHDB-523
COUCHDB-1993

This is a functional implementation of multi query views on the single
node interface. This is a WIP commit as the implementation is a bit
awkward in some places. In particular, this uses the
couch_mrview_http:view_cb/2 for the entirety of the http response,
which is nice, but it requires introducing state to indicate when
we've reached the last view query, to know when we can complete the
chunked response. Is this better than having multi_query_view/5 finish
the final http chunk and end the response? Pros and cons to either
approach. Once I get some feedback on approach there I'll update
fabric to support this as well.


Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/22ec9d90
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/22ec9d90
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/22ec9d90

Branch: refs/heads/1993-bigcouch-couch-mrview
Commit: 22ec9d906251010eced428765d24ba883e511b91
Parents: 11eb1b3
Author: Russell Branca <chewbranca@apache.org>
Authored: Fri Feb 14 11:58:18 2014 -0800
Committer: Russell Branca <chewbranca@apache.org>
Committed: Fri Feb 14 12:38:08 2014 -0800

----------------------------------------------------------------------
 include/couch_mrview.hrl  |   3 +-
 src/couch_mrview_http.erl | 151 ++++++++++++++++++++++++++++++++---------
 src/couch_mrview_show.erl |   2 +-
 3 files changed, 122 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/22ec9d90/include/couch_mrview.hrl
----------------------------------------------------------------------
diff --git a/include/couch_mrview.hrl b/include/couch_mrview.hrl
index ce1547f..8791249 100644
--- a/include/couch_mrview.hrl
+++ b/include/couch_mrview.hrl
@@ -86,5 +86,6 @@
     req,
     resp,
     prepend,
-    etag
+    etag,
+    multi_query
 }).

http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/22ec9d90/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 71c3845..87a56b0 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -19,7 +19,7 @@
     handle_info_req/3,
     handle_compact_req/3,
     handle_cleanup_req/2,
-    parse_qs/2,
+    parse_params/2,
     view_cb/2,
     row_to_json/1,
     row_to_json/2,
@@ -49,9 +49,24 @@ handle_view_req(#httpd{method='GET'}=Req, Db, DDoc) ->
     design_doc_view(Req, Db, DDoc, ViewName, undefined);
 handle_view_req(#httpd{method='POST'}=Req, Db, DDoc) ->
     [_, _, _, _, ViewName] = Req#httpd.path_parts,
-    Keys = get_view_keys(couch_httpd:json_body_obj(Req)),
-    couch_stats_collector:increment({httpd, view_reads}),
-    design_doc_view(Req, Db, DDoc, ViewName, Keys);
+    Props = couch_httpd:json_body_obj(Req),
+    Keys = get_view_keys(Props),
+    Queries = get_view_queries(Props),
+    case {Queries, Keys} of
+        {Queries, undefined} when is_list(Queries) ->
+            [couch_stats_collector:increment({httpd, view_reads}) || _I <- Queries],
+            multi_query_view(Req, Db, DDoc, ViewName, Queries);
+        {undefined, Keys} when is_list(Keys) ->
+            couch_stats_collector:increment({httpd, view_reads}),
+            design_doc_view(Req, Db, DDoc, ViewName, Keys);
+        {undefined, undefined} ->
+            throw({
+                bad_request,
+                "POST body must contain `keys` or `queries` field"
+            });
+        {_, _} ->
+            throw({bad_request, "`keys` and `queries` are mutually exclusive"})
+    end;
 handle_view_req(Req, _Db, _DDoc) ->
     couch_httpd:send_method_not_allowed(Req, "GET,POST,HEAD").
 
@@ -85,7 +100,7 @@ handle_compact_req(#httpd{method='POST'}=Req, Db, DDoc) ->
     ok = couch_mrview:compact(Db, DDoc),
     couch_httpd:send_json(Req, 202, {[{ok, true}]});
 handle_compact_req(Req, _Db, _DDoc) ->
-    couch_httpd:send_method_not_allowd(Req, "POST").
+    couch_httpd:send_method_not_allowed(Req, "POST").
 
 
 handle_cleanup_req(#httpd{method='POST'}=Req, Db) ->
@@ -113,7 +128,7 @@ all_docs_req(Req, Db, Keys) ->
 
 
 do_all_docs_req(Req, Db, Keys) ->
-    Args0 = parse_qs(Req, Keys),
+    Args0 = parse_params(Req, Keys),
     ETagFun = fun(Sig, Acc0) ->
         check_view_etag(Sig, Acc0, Req)
     end,
@@ -129,7 +144,7 @@ do_all_docs_req(Req, Db, Keys) ->
 
 
 design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
-    Args0 = parse_qs(Req, Keys),
+    Args0 = parse_params(Req, Keys),
     ETagFun = fun(Sig, Acc0) ->
         check_view_etag(Sig, Acc0, Req)
     end,
@@ -144,10 +159,58 @@ design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
     end.
 
 
-view_cb({meta, Meta}, #vacc{resp=undefined}=Acc) ->
+multi_query_view(Req, Db, DDoc, ViewName, Queries) ->
+    Args0 = parse_params(Req, undefined),
+    %% ETagFun = fun(Sig, Acc0) ->
+    %%     check_view_etag(Sig, Acc0, Req)
+    %% end,
+    %% Args = Args0#mrargs{preflight_fun=ETagFun},
+    %% Args2 = couch_util:with_db(
+    %%     DbName,
+    %%     fun(WDb) ->
+    %%         {ok, _VInfo, _Sig, Args1} =
+    %%             couch_mrview_util:get_view(WDb, DDoc, ViewName, Args0),
+    %%         Args1
+    %%     end
+    %% ),
+    {ok, _, _, Args1} = couch_mrview_util:get_view(Db, DDoc, ViewName, Args0),
+    ArgQueries = lists:map(fun({Query}) ->
+        QueryArg = parse_params(Query, undefined, Args1),
+        couch_mrview_util:validate_args(QueryArg)
+    end, Queries),
+    {ok, Resp2} = couch_httpd:etag_maybe(Req, fun() ->
+        VAcc0 = #vacc{db=Db, req=Req, multi_query=true},
+        Headers = [{"ETag", VAcc0#vacc.etag}],
+        FirstChunk = "{\"results\":[",
+        {ok, Resp0} = chttpd:start_delayed_json_response(VAcc0#vacc.req, 200, Headers, FirstChunk),
+        VAcc1 = VAcc0#vacc{resp=Resp0},
+        VAcc2 = lists:foldl(fun(Args, Acc0) ->
+            {ok, Acc1} = couch_mrview:query_view(Db, DDoc, ViewName, Args, fun view_cb/2,
Acc0),
+            Acc1
+        end, VAcc1, ArgQueries),
+        {ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"),
+        {ok, Resp2} = chttpd:end_delayed_json_response(Resp1),
+        {ok, VAcc2#vacc{resp=Resp2}}
+    end),
+    case is_record(Resp2, vacc) of
+        true -> {ok, Resp2#vacc.resp};
+        _ -> {ok, Resp2}
+    end.
+
+
+view_cb({meta, Meta}, #vacc{resp=undefined, multi_query=undefined}=Acc) ->
+    % Map function starting
     Headers = [{"ETag", Acc#vacc.etag}],
     {ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers),
-    % Map function starting
+    view_cb({meta, Meta}, Acc#vacc{resp=Resp});
+view_cb({meta, Meta}, #vacc{resp=Resp, prepend=Prepend0}=Acc) ->
+    % Sending metadata
+    Prepend = case Prepend0 of
+        undefined ->
+            "";
+        _ ->
+            Prepend0
+    end,
     Parts = case couch_util:get_value(total, Meta) of
         undefined -> [];
         Total -> [io_lib:format("\"total_rows\":~p", [Total])]
@@ -158,7 +221,7 @@ view_cb({meta, Meta}, #vacc{resp=undefined}=Acc) ->
         undefined -> [];
         UpdateSeq -> [io_lib:format("\"update_seq\":~p", [UpdateSeq])]
     end ++ ["\"rows\":["],
-    Chunk = lists:flatten("{" ++ string:join(Parts, ",") ++ "\r\n"),
+    Chunk = lists:flatten(Prepend ++ "{" ++ string:join(Parts, ",") ++ "\r\n"),
     {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk),
     {ok, Acc#vacc{resp=Resp1, prepend=""}};
 view_cb({row, Row}, #vacc{resp=undefined}=Acc) ->
@@ -167,7 +230,7 @@ view_cb({row, Row}, #vacc{resp=undefined}=Acc) ->
     {ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers),
     Chunk = ["{\"rows\":[\r\n", row_to_json(Row)],
     {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk),
-    {ok, #vacc{resp=Resp1, prepend=",\r\n"}};
+    {ok, Acc#vacc{resp=Resp1, prepend=",\r\n"}};
 view_cb({row, Row}, Acc) ->
     % Adding another row
     Chunk = [Acc#vacc.prepend, row_to_json(Row)],
@@ -177,6 +240,10 @@ view_cb(complete, #vacc{resp=undefined}=Acc) ->
     % Nothing in view
     {ok, Resp} = chttpd:send_json(Acc#vacc.req, 200, {[{rows, []}]}),
     {ok, Acc#vacc{resp=Resp}};
+view_cb(complete, #vacc{resp=Resp0, multi_query=true}=Acc) ->
+    % Finish current query of a multiquery view output
+    {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "\r\n]}"),
+    {ok, Acc#vacc{resp=Resp1, prepend=",\r\n"}};
 view_cb(complete, #vacc{resp=Resp}=Acc) ->
     % Finish view output
     chttpd:send_delayed_chunk(Resp, "\r\n]}"),
@@ -223,47 +290,67 @@ get_view_keys({Props}) ->
     end.
 
 
-parse_qs(Req, Keys) ->
-    Args = #mrargs{keys=Keys},
+get_view_queries({Props}) ->
+    case couch_util:get_value(<<"queries">>, Props) of
+        undefined ->
+            undefined;
+        Queries when is_list(Queries) ->
+            Queries;
+        _ ->
+            throw({bad_request, "`queries` member must be a array."})
+    end.
+
+
+parse_params(#httpd{}=Req, Keys) ->
+    parse_params(couch_httpd:qs(Req), Keys);
+parse_params(Props, Keys) ->
+    Args = #mrargs{},
+    parse_params(Props, Keys, Args).
+
+
+parse_params(Props, Keys, #mrargs{}=Args0) ->
+    Args = Args0#mrargs{keys=Keys},
     lists:foldl(fun({K, V}, Acc) ->
-        parse_qs(K, V, Acc)
-    end, Args, couch_httpd:qs(Req)).
+        parse_param(K, V, Acc)
+    end, Args, Props).
 
 
-parse_qs(Key, Val, Args) ->
+parse_param(Key, Val, Args) when is_binary(Key) ->
+    parse_param(binary_to_list(Key), Val, Args);
+parse_param(Key, Val, Args) ->
     case Key of
         "" ->
             Args;
         "reduce" ->
             Args#mrargs{reduce=parse_boolean(Val)};
         "key" ->
-            JsonKey = ?JSON_DECODE(Val),
+            JsonKey = parse_json(Val),
             Args#mrargs{start_key=JsonKey, end_key=JsonKey};
         "keys" ->
-            Args#mrargs{keys=?JSON_DECODE(Val)};
+            Args#mrargs{keys=parse_json(Val)};
         "startkey" ->
-            Args#mrargs{start_key=?JSON_DECODE(Val)};
+            Args#mrargs{start_key=parse_json(Val)};
         "start_key" ->
-            Args#mrargs{start_key=?JSON_DECODE(Val)};
+            Args#mrargs{start_key=parse_json(Val)};
         "startkey_docid" ->
-            Args#mrargs{start_key_docid=list_to_binary(Val)};
+            Args#mrargs{start_key_docid=couch_util:to_binary(Val)};
         "start_key_doc_id" ->
-            Args#mrargs{start_key_docid=list_to_binary(Val)};
+            Args#mrargs{start_key_docid=couch_util:to_binary(Val)};
         "endkey" ->
-            Args#mrargs{end_key=?JSON_DECODE(Val)};
+            Args#mrargs{end_key=parse_json(Val)};
         "end_key" ->
-            Args#mrargs{end_key=?JSON_DECODE(Val)};
+            Args#mrargs{end_key=parse_json(Val)};
         "endkey_docid" ->
-            Args#mrargs{end_key_docid=list_to_binary(Val)};
+            Args#mrargs{end_key_docid=couch_util:to_binary(Val)};
         "end_key_doc_id" ->
-            Args#mrargs{end_key_docid=list_to_binary(Val)};
+            Args#mrargs{end_key_docid=couch_util:to_binary(Val)};
         "limit" ->
             Args#mrargs{limit=parse_pos_int(Val)};
         "count" ->
             throw({query_parse_error, <<"QS param `count` is not `limit`">>});
-        "stale" when Val == "ok" ->
+        "stale" when Val == "ok" orelse Val == <<"ok">> ->
             Args#mrargs{stale=ok};
-        "stale" when Val == "update_after" ->
+        "stale" when Val == "update_after" orelse Val == "update_after" ->
             Args#mrargs{stale=update_after};
         "stale" ->
             throw({query_parse_error, <<"Invalid value for `stale`.">>});
@@ -290,12 +377,12 @@ parse_qs(Key, Val, Args) ->
         "conflicts" ->
             Args#mrargs{conflicts=parse_boolean(Val)};
         "list" ->
-            Args#mrargs{list=list_to_binary(Val)};
+            Args#mrargs{list=couch_util:to_binary(Val)};
         "callback" ->
-            Args#mrargs{callback=list_to_binary(Val)};
+            Args#mrargs{callback=couch_util:to_binary(Val)};
         _ ->
-            BKey = list_to_binary(Key),
-            BVal = list_to_binary(Val),
+            BKey = couch_util:to_binary(Key),
+            BVal = couch_util:to_binary(Val),
             Args#mrargs{extra=[{BKey, BVal} | Args#mrargs.extra]}
     end.
 

http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/22ec9d90/src/couch_mrview_show.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_show.erl b/src/couch_mrview_show.erl
index 1be96d5..0b42da5 100644
--- a/src/couch_mrview_show.erl
+++ b/src/couch_mrview_show.erl
@@ -193,7 +193,7 @@ handle_view_list_req(Req, _Db, _DDoc) ->
 
 
 handle_view_list(Req, Db, DDoc, LName, VDDoc, VName, Keys) ->
-    Args0 = couch_mrview_http:parse_qs(Req, Keys),
+    Args0 = couch_mrview_http:parse_params(Req, Keys),
     ETagFun = fun(BaseSig, Acc0) ->
         UserCtx = Req#httpd.user_ctx,
         Name = UserCtx#user_ctx.name,


Mime
View raw message