couchdb-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Robert Newson <rnew...@apache.org>
Subject Re: git commit: updated refs/heads/1334-revert-feature-view-server-pipelining to 6ffc527
Date Tue, 23 Jul 2013 11:33:06 GMT
We're looking to see if, instead of reverting, we can make Windows
fall back to the old, slower behavior instead.

B.


On 23 July 2013 12:29, Jan Lehnardt <jan@apache.org> wrote:
> Context:
>
> This patch used to be in master before 1.3.x branched.
>
> In the course of releasing 1.3.0 we found that this patch breaks on Windows and we didn’t
manage to find a fix in time for the release so we decided to back this out of 1.3.x to unblock
the release. We left it in master in the hopes that a quick fix would follow. That didn’t
happen and now it is time to ship 1.4.0, so Dave backed the patch out then.
>
> Best
> Jan
> --
>
>
>
> On Jul 23, 2013, at 13:17 , dch@apache.org wrote:
>
>> Updated Branches:
>>  refs/heads/1334-revert-feature-view-server-pipelining [created] 6ffc52796
>>
>>
>> COUCHDB-1334 - revert "More efficient communication with the view server"
>>
>> This reverts commit a851c6e
>> - COUCHDB-1334 breaks with Windows + couchjs in unexplained ways
>> - reducing to 1 concurrent query server is not sufficient
>> - Testing with open_port options overlapped_io was not in itself sufficient
>> - http://erlang.org/doc/man/erlang.html find overlapped_io
>> - Refer history in COUCHDB-1346
>>
>>
>> Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
>> Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/6ffc5279
>> Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/6ffc5279
>> Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/6ffc5279
>>
>> Branch: refs/heads/1334-revert-feature-view-server-pipelining
>> Commit: 6ffc52796182a8d3673f3648c59c7e9abddf4fa2
>> Parents: 635022b
>> Author: Dave Cottlehuber <dch@apache.org>
>> Authored: Wed Dec 12 21:37:18 2012 +0100
>> Committer: Dave Cottlehuber <dch@apache.org>
>> Committed: Tue Jul 23 13:14:33 2013 +0200
>>
>> ----------------------------------------------------------------------
>> src/couch_mrview/src/couch_mrview_updater.erl | 46 +++++++++-------------
>> src/couchdb/couch_native_process.erl          | 11 +-----
>> src/couchdb/couch_os_process.erl              | 38 +-----------------
>> src/couchdb/couch_query_servers.erl           | 17 +++-----
>> 4 files changed, 25 insertions(+), 87 deletions(-)
>> ----------------------------------------------------------------------
>>
>>
>> http://git-wip-us.apache.org/repos/asf/couchdb/blob/6ffc5279/src/couch_mrview/src/couch_mrview_updater.erl
>> ----------------------------------------------------------------------
>> diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl
>> index 3014664..9604ea9 100644
>> --- a/src/couch_mrview/src/couch_mrview_updater.erl
>> +++ b/src/couch_mrview/src/couch_mrview_updater.erl
>> @@ -130,42 +130,32 @@ map_docs(Parent, State0) ->
>>             couch_query_servers:stop_doc_map(State0#mrst.qserver),
>>             couch_work_queue:close(State0#mrst.write_queue);
>>         {ok, Dequeued} ->
>> +            % Run all the non deleted docs through the view engine and
>> +            % then pass the results on to the writer process.
>>             State1 = case State0#mrst.qserver of
>>                 nil -> start_query_server(State0);
>>                 _ -> State0
>>             end,
>> -            {ok, MapResults} = compute_map_results(State1, Dequeued),
>> -            couch_work_queue:queue(State1#mrst.write_queue, MapResults),
>> +            QServer = State1#mrst.qserver,
>> +            DocFun = fun
>> +                ({nil, Seq, _}, {SeqAcc, Results}) ->
>> +                    {erlang:max(Seq, SeqAcc), Results};
>> +                ({Id, Seq, deleted}, {SeqAcc, Results}) ->
>> +                    {erlang:max(Seq, SeqAcc), [{Id, []} | Results]};
>> +                ({Id, Seq, Doc}, {SeqAcc, Results}) ->
>> +                    {ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc),
>> +                    {erlang:max(Seq, SeqAcc), [{Id, Res} | Results]}
>> +            end,
>> +            FoldFun = fun(Docs, Acc) ->
>> +                update_task(length(Docs)),
>> +                lists:foldl(DocFun, Acc, Docs)
>> +            end,
>> +            Results = lists:foldl(FoldFun, {0, []}, Dequeued),
>> +            couch_work_queue:queue(State1#mrst.write_queue, Results),
>>             map_docs(Parent, State1)
>>     end.
>>
>>
>> -compute_map_results(#mrst{qserver = Qs}, Dequeued) ->
>> -    % Run all the non deleted docs through the view engine and
>> -    % then pass the results on to the writer process.
>> -    DocFun = fun
>> -        ({nil, Seq, _}, {SeqAcc, AccDel, AccNotDel}) ->
>> -            {erlang:max(Seq, SeqAcc), AccDel, AccNotDel};
>> -        ({Id, Seq, deleted}, {SeqAcc, AccDel, AccNotDel}) ->
>> -            {erlang:max(Seq, SeqAcc), [{Id, []} | AccDel], AccNotDel};
>> -        ({_Id, Seq, Doc}, {SeqAcc, AccDel, AccNotDel}) ->
>> -            {erlang:max(Seq, SeqAcc), AccDel, [Doc | AccNotDel]}
>> -    end,
>> -    FoldFun = fun(Docs, Acc) ->
>> -        lists:foldl(DocFun, Acc, Docs)
>> -    end,
>> -    {MaxSeq, DeletedResults, Docs} =
>> -        lists:foldl(FoldFun, {0, [], []}, Dequeued),
>> -    {ok, MapResultList} = couch_query_servers:map_docs_raw(Qs, Docs),
>> -    NotDeletedResults = lists:zipwith(
>> -        fun(#doc{id = Id}, MapResults) -> {Id, MapResults} end,
>> -        Docs,
>> -        MapResultList),
>> -    AllMapResults = DeletedResults ++ NotDeletedResults,
>> -    update_task(length(AllMapResults)),
>> -    {ok, {MaxSeq, AllMapResults}}.
>> -
>> -
>> write_results(Parent, State) ->
>>     case couch_work_queue:dequeue(State#mrst.write_queue) of
>>         closed ->
>>
>> http://git-wip-us.apache.org/repos/asf/couchdb/blob/6ffc5279/src/couchdb/couch_native_process.erl
>> ----------------------------------------------------------------------
>> diff --git a/src/couchdb/couch_native_process.erl b/src/couchdb/couch_native_process.erl
>> index b1d51ed..5a32e75 100644
>> --- a/src/couchdb/couch_native_process.erl
>> +++ b/src/couchdb/couch_native_process.erl
>> @@ -42,7 +42,7 @@
>>
>> -export([start_link/0,init/1,terminate/2,handle_call/3,handle_cast/2,code_change/3,
>>          handle_info/2]).
>> --export([set_timeout/2, prompt/2, prompt_many/2]).
>> +-export([set_timeout/2, prompt/2]).
>>
>> -define(STATE, native_proc_state).
>> -record(evstate, {ddocs, funs=[], query_config=[], list_pid=nil, timeout=5000}).
>> @@ -62,15 +62,6 @@ set_timeout(Pid, TimeOut) ->
>> prompt(Pid, Data) when is_list(Data) ->
>>     gen_server:call(Pid, {prompt, Data}).
>>
>> -prompt_many(Pid, DataList) ->
>> -    prompt_many(Pid, DataList, []).
>> -
>> -prompt_many(_Pid, [], Acc) ->
>> -    {ok, lists:reverse(Acc)};
>> -prompt_many(Pid, [Data | Rest], Acc) ->
>> -    Result = prompt(Pid, Data),
>> -    prompt_many(Pid, Rest, [Result | Acc]).
>> -
>> % gen_server callbacks
>> init([]) ->
>>     {ok, #evstate{ddocs=dict:new()}}.
>>
>> http://git-wip-us.apache.org/repos/asf/couchdb/blob/6ffc5279/src/couchdb/couch_os_process.erl
>> ----------------------------------------------------------------------
>> diff --git a/src/couchdb/couch_os_process.erl b/src/couchdb/couch_os_process.erl
>> index 3a267be..db62d49 100644
>> --- a/src/couchdb/couch_os_process.erl
>> +++ b/src/couchdb/couch_os_process.erl
>> @@ -14,7 +14,7 @@
>> -behaviour(gen_server).
>>
>> -export([start_link/1, start_link/2, start_link/3, stop/1]).
>> --export([set_timeout/2, prompt/2, prompt_many/2]).
>> +-export([set_timeout/2, prompt/2]).
>> -export([send/2, writeline/2, readline/1, writejson/2, readjson/1]).
>> -export([init/1, terminate/2, handle_call/3, handle_cast/2, handle_info/2, code_change/3]).
>>
>> @@ -57,40 +57,6 @@ prompt(Pid, Data) ->
>>             throw(Error)
>>     end.
>>
>> -prompt_many(Pid, DataList) ->
>> -    OsProc = gen_server:call(Pid, get_os_proc, infinity),
>> -    true = port_connect(OsProc#os_proc.port, self()),
>> -    try
>> -        send_many(OsProc, DataList),
>> -        receive_many(length(DataList), OsProc, [])
>> -    after
>> -        % Can throw badarg error, when OsProc Pid is dead or port was closed
>> -        % by the readline function on error/timeout.
>> -        (catch port_connect(OsProc#os_proc.port, Pid)),
>> -        unlink(OsProc#os_proc.port),
>> -        drop_port_messages(OsProc#os_proc.port)
>> -    end.
>> -
>> -send_many(_OsProc, []) ->
>> -    ok;
>> -send_many(#os_proc{writer = Writer} = OsProc, [Data | Rest]) ->
>> -    Writer(OsProc, Data),
>> -    send_many(OsProc, Rest).
>> -
>> -receive_many(0, _OsProc, Acc) ->
>> -    {ok, lists:reverse(Acc)};
>> -receive_many(N, #os_proc{reader = Reader} = OsProc, Acc) ->
>> -    Line = Reader(OsProc),
>> -    receive_many(N - 1, OsProc, [Line | Acc]).
>> -
>> -drop_port_messages(Port) ->
>> -    receive
>> -    {Port, _} ->
>> -        drop_port_messages(Port)
>> -    after 0 ->
>> -        ok
>> -    end.
>> -
>> % Utility functions for reading and writing
>> % in custom functions
>> writeline(OsProc, Data) when is_record(OsProc, os_proc) ->
>> @@ -209,8 +175,6 @@ terminate(_Reason, #os_proc{port=Port}) ->
>>     catch port_close(Port),
>>     ok.
>>
>> -handle_call(get_os_proc, _From, OsProc) ->
>> -    {reply, OsProc, OsProc};
>> handle_call({set_timeout, TimeOut}, _From, OsProc) ->
>>     {reply, ok, OsProc#os_proc{timeout=TimeOut}};
>> handle_call({prompt, Data}, _From, OsProc) ->
>>
>> http://git-wip-us.apache.org/repos/asf/couchdb/blob/6ffc5279/src/couchdb/couch_query_servers.erl
>> ----------------------------------------------------------------------
>> diff --git a/src/couchdb/couch_query_servers.erl b/src/couchdb/couch_query_servers.erl
>> index e29f23b..3b58cbe 100644
>> --- a/src/couchdb/couch_query_servers.erl
>> +++ b/src/couchdb/couch_query_servers.erl
>> @@ -16,7 +16,7 @@
>> -export([start_link/0, config_change/1]).
>>
>> -export([init/1, terminate/2, handle_call/3, handle_cast/2, handle_info/2,code_change/3]).
>> --export([start_doc_map/3, map_docs/2, map_docs_raw/2, stop_doc_map/1, raw_to_ejson/1]).
>> +-export([start_doc_map/3, map_docs/2, map_doc_raw/2, stop_doc_map/1, raw_to_ejson/1]).
>> -export([reduce/3, rereduce/3,validate_doc_update/5]).
>> -export([filter_docs/5]).
>> -export([filter_view/3]).
>> @@ -33,7 +33,6 @@
>>     lang,
>>     ddoc_keys = [],
>>     prompt_fun,
>> -    prompt_many_fun,
>>     set_timeout_fun,
>>     stop_fun
>> }).
>> @@ -84,15 +83,10 @@ map_docs(Proc, Docs) ->
>>         Docs),
>>     {ok, Results}.
>>
>> -map_docs_raw(Proc, DocList) ->
>> -    {Mod, Fun} = Proc#proc.prompt_many_fun,
>> -    CommandList = lists:map(
>> -        fun(Doc) ->
>> -            EJson = couch_doc:to_json_obj(Doc, []),
>> -            [<<"map_doc">>, EJson]
>> -        end,
>> -        DocList),
>> -    Mod:Fun(Proc#proc.pid, CommandList).
>> +map_doc_raw(Proc, Doc) ->
>> +    Json = couch_doc:to_json_obj(Doc, []),
>> +    {ok, proc_prompt_raw(Proc, [<<"map_doc">>, Json])}.
>> +
>>
>> stop_doc_map(nil) ->
>>     ok;
>> @@ -487,7 +481,6 @@ new_process(Langs, LangLimits, Lang) ->
>>                        pid=Pid,
>>                        % Called via proc_prompt, proc_set_timeout, and proc_stop
>>                        prompt_fun={Mod, prompt},
>> -                       prompt_many_fun={Mod, prompt_many},
>>                        set_timeout_fun={Mod, set_timeout},
>>                        stop_fun={Mod, stop}}};
>>         _ ->
>>
>

Mime
View raw message