couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From beno...@apache.org
Subject [39/57] [abbrv] [partial] inital move to rebar compilation
Date Tue, 07 Jan 2014 00:36:59 GMT
http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_mrview/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/apps/couch_mrview/src/couch_mrview_util.erl b/apps/couch_mrview/src/couch_mrview_util.erl
new file mode 100644
index 0000000..f7946d1
--- /dev/null
+++ b/apps/couch_mrview/src/couch_mrview_util.erl
@@ -0,0 +1,802 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_mrview_util).
+
+-export([get_view/4]).
+-export([ddoc_to_mrst/2, init_state/4, reset_index/3]).
+-export([make_header/1]).
+-export([index_file/2, compaction_file/2, open_file/1]).
+-export([delete_files/2, delete_index_file/2, delete_compaction_file/2]).
+-export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]).
+-export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
+-export([fold/4, fold_reduce/4]).
+-export([temp_view_to_ddoc/1]).
+-export([calculate_data_size/2]).
+-export([validate_args/1]).
+-export([maybe_load_doc/3, maybe_load_doc/4]).
+-export([maybe_update_index_file/1]).
+
+-define(MOD, couch_mrview_index).
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+
+
+get_view(Db, DDoc, ViewName, Args0) ->
+    ArgCheck = fun(InitState) ->
+        Args1 = set_view_type(Args0, ViewName, InitState#mrst.views),
+        {ok, validate_args(Args1)}
+    end,
+    {ok, Pid, Args2} = couch_index_server:get_index(?MOD, Db, DDoc, ArgCheck),
+    DbUpdateSeq = couch_util:with_db(Db, fun(WDb) ->
+        couch_db:get_update_seq(WDb)
+    end),
+    MinSeq = case Args2#mrargs.stale of
+        ok -> 0; update_after -> 0; _ -> DbUpdateSeq
+    end,
+    {ok, State} = case couch_index:get_state(Pid, MinSeq) of
+        {ok, _} = Resp -> Resp;
+        Error -> throw(Error)
+    end,
+    couch_ref_counter:add(State#mrst.refc),
+    if Args2#mrargs.stale == update_after ->
+        spawn(fun() -> catch couch_index:get_state(Pid, DbUpdateSeq) end);
+        true -> ok
+    end,
+    #mrst{language=Lang, views=Views} = State,
+    {Type, View, Args3} = extract_view(Lang, Args2, ViewName, Views),
+    check_range(Args3, view_cmp(View)),
+    Sig = view_sig(Db, State, View, Args3),
+    {ok, {Type, View}, Sig, Args3}.
+
+
+ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
+    MakeDict = fun({Name, {MRFuns}}, DictBySrcAcc) ->
+        case couch_util:get_value(<<"map">>, MRFuns) of
+            MapSrc when is_binary(MapSrc) ->
+                RedSrc = couch_util:get_value(<<"reduce">>, MRFuns, null),
+                {ViewOpts} = couch_util:get_value(<<"options">>, MRFuns, {[]}),
+                View = case dict:find({MapSrc, ViewOpts}, DictBySrcAcc) of
+                    {ok, View0} -> View0;
+                    error -> #mrview{def=MapSrc, options=ViewOpts}
+                end,
+                {MapNames, RedSrcs} = case RedSrc of
+                    null ->
+                        MNames = [Name | View#mrview.map_names],
+                        {MNames, View#mrview.reduce_funs};
+                    _ ->
+                        RedFuns = [{Name, RedSrc} | View#mrview.reduce_funs],
+                        {View#mrview.map_names, RedFuns}
+                end,
+                View2 = View#mrview{map_names=MapNames, reduce_funs=RedSrcs},
+                dict:store({MapSrc, ViewOpts}, View2, DictBySrcAcc);
+            undefined ->
+                DictBySrcAcc
+        end
+    end,
+    {RawViews} = couch_util:get_value(<<"views">>, Fields, {[]}),
+    BySrc = lists:foldl(MakeDict, dict:new(), RawViews),
+
+    NumViews = fun({_, View}, N) -> {View#mrview{id_num=N}, N+1} end,
+    {Views, _} = lists:mapfoldl(NumViews, 0, lists:sort(dict:to_list(BySrc))),
+
+    Language = couch_util:get_value(<<"language">>, Fields, <<"javascript">>),
+    {DesignOpts} = couch_util:get_value(<<"options">>, Fields, {[]}),
+    {RawViews} = couch_util:get_value(<<"views">>, Fields, {[]}),
+    Lib = couch_util:get_value(<<"lib">>, RawViews, {[]}),
+
+    IdxState = #mrst{
+        db_name=DbName,
+        idx_name=Id,
+        lib=Lib,
+        views=Views,
+        language=Language,
+        design_opts=DesignOpts
+    },
+    SigInfo = {Views, Language, DesignOpts, couch_index_util:sort_lib(Lib)},
+    {ok, IdxState#mrst{sig=couch_util:md5(term_to_binary(SigInfo))}}.
+
+
+set_view_type(_Args, _ViewName, []) ->
+    throw({not_found, missing_named_view});
+set_view_type(Args, ViewName, [View | Rest]) ->
+    RedNames = [N || {N, _} <- View#mrview.reduce_funs],
+    case lists:member(ViewName, RedNames) of
+        true ->
+            case Args#mrargs.reduce of
+                false -> Args#mrargs{view_type=map};
+                _ -> Args#mrargs{view_type=red}
+            end;
+        false ->
+            case lists:member(ViewName, View#mrview.map_names) of
+                true -> Args#mrargs{view_type=map};
+                false -> set_view_type(Args, ViewName, Rest)
+            end
+    end.
+
+
+extract_view(_Lang, _Args, _ViewName, []) ->
+    throw({not_found, missing_named_view});
+extract_view(Lang, #mrargs{view_type=map}=Args, Name, [View | Rest]) ->
+    Names = View#mrview.map_names ++ [N || {N, _} <- View#mrview.reduce_funs],
+    case lists:member(Name, Names) of
+        true -> {map, View, Args};
+        _ -> extract_view(Lang, Args, Name, Rest)
+    end;
+extract_view(Lang, #mrargs{view_type=red}=Args, Name, [View | Rest]) ->
+    RedNames = [N || {N, _} <- View#mrview.reduce_funs],
+    case lists:member(Name, RedNames) of
+        true -> {red, {index_of(Name, RedNames), Lang, View}, Args};
+        false -> extract_view(Lang, Args, Name, Rest)
+    end.
+
+
+view_sig(Db, State, View, #mrargs{include_docs=true}=Args) ->
+    BaseSig = view_sig(Db, State, View, Args#mrargs{include_docs=false}),
+    UpdateSeq = couch_db:get_update_seq(Db),
+    PurgeSeq = couch_db:get_purge_seq(Db),
+    Bin = term_to_binary({BaseSig, UpdateSeq, PurgeSeq}),
+    couch_index_util:hexsig(couch_util:md5(Bin));
+view_sig(Db, State, {_Nth, _Lang, View}, Args) ->
+    view_sig(Db, State, View, Args);
+view_sig(_Db, State, View, Args0) ->
+    Sig = State#mrst.sig,
+    UpdateSeq = View#mrview.update_seq,
+    PurgeSeq = View#mrview.purge_seq,
+    Args = Args0#mrargs{
+        preflight_fun=undefined,
+        extra=[]
+    },
+    Bin = term_to_binary({Sig, UpdateSeq, PurgeSeq, Args}),
+    couch_index_util:hexsig(couch_util:md5(Bin)).
+
+
+init_state(Db, Fd, #mrst{views=Views}=State, nil) ->
+    Header = #mrheader{
+        seq=0,
+        purge_seq=couch_db:get_purge_seq(Db),
+        id_btree_state=nil,
+        view_states=[{nil, 0, 0} || _ <- Views]
+    },
+    init_state(Db, Fd, State, Header);
+% read <= 1.2.x header record and transpile it to >=1.3.x
+% header record
+init_state(Db, Fd, State, #index_header{
+    seq=Seq,
+    purge_seq=PurgeSeq,
+    id_btree_state=IdBtreeState,
+    view_states=ViewStates}) ->
+    init_state(Db, Fd, State, #mrheader{
+        seq=Seq,
+        purge_seq=PurgeSeq,
+        id_btree_state=IdBtreeState,
+        view_states=ViewStates
+        });
+init_state(Db, Fd, State, Header) ->
+    #mrst{language=Lang, views=Views} = State,
+    #mrheader{
+        seq=Seq,
+        purge_seq=PurgeSeq,
+        id_btree_state=IdBtreeState,
+        view_states=ViewStates
+    } = Header,
+
+    StateUpdate = fun
+        ({_, _, _}=St) -> St;
+        (St) -> {St, 0, 0}
+    end,
+    ViewStates2 = lists:map(StateUpdate, ViewStates),
+
+    IdBtOpts = [{compression, couch_db:compression(Db)}],
+    {ok, IdBtree} = couch_btree:open(IdBtreeState, Fd, IdBtOpts),
+
+    OpenViewFun = fun(St, View) -> open_view(Db, Fd, Lang, St, View) end,
+    Views2 = lists:zipwith(OpenViewFun, ViewStates2, Views),
+
+    State#mrst{
+        fd=Fd,
+        update_seq=Seq,
+        purge_seq=PurgeSeq,
+        id_btree=IdBtree,
+        views=Views2
+    }.
+
+
+open_view(Db, Fd, Lang, {BTState, USeq, PSeq}, View) ->
+    FunSrcs = [FunSrc || {_Name, FunSrc} <- View#mrview.reduce_funs],
+    ReduceFun =
+        fun(reduce, KVs) ->
+            KVs2 = detuple_kvs(expand_dups(KVs, []), []),
+            {ok, Result} = couch_query_servers:reduce(Lang, FunSrcs, KVs2),
+            {length(KVs2), Result};
+        (rereduce, Reds) ->
+            Count = lists:sum([Count0 || {Count0, _} <- Reds]),
+            UsrReds = [UsrRedsList || {_, UsrRedsList} <- Reds],
+            {ok, Result} = couch_query_servers:rereduce(Lang, FunSrcs, UsrReds),
+            {Count, Result}
+        end,
+
+    Less = case couch_util:get_value(<<"collation">>, View#mrview.options) of
+        <<"raw">> -> fun(A, B) -> A < B end;
+        _ -> fun couch_ejson_compare:less_json_ids/2
+    end,
+
+    ViewBtOpts = [
+        {less, Less},
+        {reduce, ReduceFun},
+        {compression, couch_db:compression(Db)}
+    ],
+    {ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts),
+    View#mrview{btree=Btree, update_seq=USeq, purge_seq=PSeq}.
+
+
+temp_view_to_ddoc({Props}) ->
+    Language = couch_util:get_value(<<"language">>, Props, <<"javascript">>),
+    Options = couch_util:get_value(<<"options">>, Props, {[]}),
+    View0 = [{<<"map">>, couch_util:get_value(<<"map">>, Props)}],
+    View1 = View0 ++ case couch_util:get_value(<<"reduce">>, Props) of
+        RedSrc when is_binary(RedSrc) -> [{<<"reduce">>, RedSrc}];
+        _ -> []
+    end,
+    DDoc = {[
+        {<<"_id">>, couch_uuids:random()},
+        {<<"language">>, Language},
+        {<<"options">>, Options},
+        {<<"views">>, {[
+            {<<"temp">>, {View1}}
+        ]}}
+    ]},
+    couch_doc:from_json_obj(DDoc).
+
+
+get_row_count(#mrview{btree=Bt}) ->
+    {ok, {Count, _Reds}} = couch_btree:full_reduce(Bt),
+    {ok, Count}.
+
+
+all_docs_reduce_to_count(Reductions) ->
+    Reduce = fun couch_db_updater:btree_by_id_reduce/2,
+    {Count, _, _} = couch_btree:final_reduce(Reduce, Reductions),
+    Count.
+
+reduce_to_count(nil) ->
+    0;
+reduce_to_count(Reductions) ->
+    Reduce = fun
+        (reduce, KVs) ->
+            Counts = [
+                case V of {dups, Vals} -> length(Vals); _ -> 1 end
+                || {_,V} <- KVs
+            ],
+            {lists:sum(Counts), []};
+        (rereduce, Reds) ->
+            {lists:sum([Count0 || {Count0, _} <- Reds]), []}
+    end,
+    {Count, _} = couch_btree:final_reduce(Reduce, Reductions),
+    Count.
+
+
+fold(#mrview{btree=Bt}, Fun, Acc, Opts) ->
+    WrapperFun = fun(KV, Reds, Acc2) ->
+        fold_fun(Fun, expand_dups([KV], []), Reds, Acc2)
+    end,
+    {ok, _LastRed, _Acc} = couch_btree:fold(Bt, WrapperFun, Acc, Opts).
+
+
+fold_fun(_Fun, [], _, Acc) ->
+    {ok, Acc};
+fold_fun(Fun, [KV|Rest], {KVReds, Reds}, Acc) ->
+    case Fun(KV, {KVReds, Reds}, Acc) of
+        {ok, Acc2} ->
+            fold_fun(Fun, Rest, {[KV|KVReds], Reds}, Acc2);
+        {stop, Acc2} ->
+            {stop, Acc2}
+    end.
+
+
+fold_reduce({NthRed, Lang, View}, Fun,  Acc, Options) ->
+    #mrview{
+        btree=Bt,
+        reduce_funs=RedFuns
+    } = View,
+    LPad = lists:duplicate(NthRed - 1, []),
+    RPad = lists:duplicate(length(RedFuns) - NthRed, []),
+    {_Name, FunSrc} = lists:nth(NthRed,RedFuns),
+
+    ReduceFun = fun
+        (reduce, KVs0) ->
+            KVs1 = detuple_kvs(expand_dups(KVs0, []), []),
+            {ok, Red} = couch_query_servers:reduce(Lang, [FunSrc], KVs1),
+            {0, LPad ++ Red ++ RPad};
+        (rereduce, Reds) ->
+            ExtractRed = fun({_, UReds0}) -> [lists:nth(NthRed, UReds0)] end,
+            UReds = lists:map(ExtractRed, Reds),
+            {ok, Red} = couch_query_servers:rereduce(Lang, [FunSrc], UReds),
+            {0, LPad ++ Red ++ RPad}
+    end,
+
+    WrapperFun = fun({GroupedKey, _}, PartialReds, Acc0) ->
+        {_, Reds} = couch_btree:final_reduce(ReduceFun, PartialReds),
+        Fun(GroupedKey, lists:nth(NthRed, Reds), Acc0)
+    end,
+
+    couch_btree:fold_reduce(Bt, WrapperFun, Acc, Options).
+
+
+validate_args(Args) ->
+    Reduce = Args#mrargs.reduce,
+    case Reduce == undefined orelse is_boolean(Reduce) of
+        true -> ok;
+        _ -> mrverror(<<"Invalid `reduce` value.">>)
+    end,
+
+    case {Args#mrargs.view_type, Reduce} of
+        {map, true} -> mrverror(<<"Reduce is invalid for map-only views.">>);
+        _ -> ok
+    end,
+
+    case {Args#mrargs.view_type, Args#mrargs.group_level, Args#mrargs.keys} of
+        {red, exact, _} -> ok;
+        {red, _, KeyList} when is_list(KeyList) ->
+            Msg = <<"Multi-key fetchs for reduce views must use `group=true`">>,
+            mrverror(Msg);
+        _ -> ok
+    end,
+
+    case Args#mrargs.keys of
+        Keys when is_list(Keys) -> ok;
+        undefined -> ok;
+        _ -> mrverror(<<"`keys` must be an array of strings.">>)
+    end,
+
+    case {Args#mrargs.keys, Args#mrargs.start_key} of
+        {undefined, _} -> ok;
+        {[], _} -> ok;
+        {[_|_], undefined} -> ok;
+        _ -> mrverror(<<"`start_key` is incompatible with `keys`">>)
+    end,
+
+    case Args#mrargs.start_key_docid of
+        undefined -> ok;
+        SKDocId0 when is_binary(SKDocId0) -> ok;
+        _ -> mrverror(<<"`start_key_docid` must be a string.">>)
+    end,
+
+    case {Args#mrargs.keys, Args#mrargs.end_key} of
+        {undefined, _} -> ok;
+        {[], _} -> ok;
+        {[_|_], undefined} -> ok;
+        _ -> mrverror(<<"`end_key` is incompatible with `keys`">>)
+    end,
+
+    case Args#mrargs.end_key_docid of
+        undefined -> ok;
+        EKDocId0 when is_binary(EKDocId0) -> ok;
+        _ -> mrverror(<<"`end_key_docid` must be a string.">>)
+    end,
+
+    case Args#mrargs.direction of
+        fwd -> ok;
+        rev -> ok;
+        _ -> mrverror(<<"Invalid direction.">>)
+    end,
+
+    case {Args#mrargs.limit >= 0, Args#mrargs.limit == undefined} of
+        {true, _} -> ok;
+        {_, true} -> ok;
+        _ -> mrverror(<<"`limit` must be a positive integer.">>)
+    end,
+
+    case Args#mrargs.skip < 0 of
+        true -> mrverror(<<"`skip` must be >= 0">>);
+        _ -> ok
+    end,
+
+    case {Args#mrargs.view_type, Args#mrargs.group_level} of
+        {red, exact} -> ok;
+        {_, 0} -> ok;
+        {red, Int} when is_integer(Int), Int >= 0 -> ok;
+        {red, _} -> mrverror(<<"`group_level` must be >= 0">>);
+        {map, _} -> mrverror(<<"Invalid use of grouping on a map view.">>)
+    end,
+
+    case Args#mrargs.stale of
+        ok -> ok;
+        update_after -> ok;
+        false -> ok;
+        _ -> mrverror(<<"Invalid value for `stale`.">>)
+    end,
+
+    case is_boolean(Args#mrargs.inclusive_end) of
+        true -> ok;
+        _ -> mrverror(<<"Invalid value for `inclusive_end`.">>)
+    end,
+
+    case {Args#mrargs.view_type, Args#mrargs.include_docs} of
+        {red, true} -> mrverror(<<"`include_docs` is invalid for reduce">>);
+        {_, ID} when is_boolean(ID) -> ok;
+        _ -> mrverror(<<"Invalid value for `include_docs`">>)
+    end,
+
+    case {Args#mrargs.view_type, Args#mrargs.conflicts} of
+        {_, undefined} -> ok;
+        {map, V} when is_boolean(V) -> ok;
+        {red, undefined} -> ok;
+        {map, _} -> mrverror(<<"Invalid value for `conflicts`.">>);
+        {red, _} -> mrverror(<<"`conflicts` is invalid for reduce views.">>)
+    end,
+
+    SKDocId = case {Args#mrargs.direction, Args#mrargs.start_key_docid} of
+        {fwd, undefined} -> <<>>;
+        {rev, undefined} -> <<255>>;
+        {_, SKDocId1} -> SKDocId1
+    end,
+
+    EKDocId = case {Args#mrargs.direction, Args#mrargs.end_key_docid} of
+        {fwd, undefined} -> <<255>>;
+        {rev, undefined} -> <<>>;
+        {_, EKDocId1} -> EKDocId1
+    end,
+
+    Args#mrargs{
+        start_key_docid=SKDocId,
+        end_key_docid=EKDocId
+    }.
+
+
+check_range(#mrargs{start_key=undefined}, _Cmp) ->
+    ok;
+check_range(#mrargs{end_key=undefined}, _Cmp) ->
+    ok;
+check_range(#mrargs{start_key=K, end_key=K}, _Cmp) ->
+    ok;
+check_range(Args, Cmp) ->
+    #mrargs{
+        direction=Dir,
+        start_key=SK,
+        start_key_docid=SKD,
+        end_key=EK,
+        end_key_docid=EKD
+    } = Args,
+    case {Dir, Cmp({SK, SKD}, {EK, EKD})} of
+        {fwd, false} ->
+            throw({query_parse_error,
+                <<"No rows can match your key range, reverse your ",
+                    "start_key and end_key or set descending=true">>});
+        {rev, true} ->
+            throw({query_parse_error,
+                <<"No rows can match your key range, reverse your ",
+                    "start_key and end_key or set descending=false">>});
+        _ -> ok
+    end.
+
+
+view_cmp({_Nth, _Lang, View}) ->
+    view_cmp(View);
+view_cmp(View) ->
+    fun(A, B) -> couch_btree:less(View#mrview.btree, A, B) end.
+
+
+make_header(State) ->
+    #mrst{
+        update_seq=Seq,
+        purge_seq=PurgeSeq,
+        id_btree=IdBtree,
+        views=Views
+    } = State,
+    ViewStates = [
+        {
+            couch_btree:get_state(V#mrview.btree),
+            V#mrview.update_seq,
+            V#mrview.purge_seq
+        }
+        ||
+        V <- Views
+    ],
+    #mrheader{
+        seq=Seq,
+        purge_seq=PurgeSeq,
+        id_btree_state=couch_btree:get_state(IdBtree),
+        view_states=ViewStates
+    }.
+
+
+index_file(DbName, Sig) ->
+    FileName = couch_index_util:hexsig(Sig) ++ ".view",
+    couch_index_util:index_file(mrview, DbName, FileName).
+
+
+compaction_file(DbName, Sig) ->
+    FileName = couch_index_util:hexsig(Sig) ++ ".compact.view",
+    couch_index_util:index_file(mrview, DbName, FileName).
+
+
+open_file(FName) ->
+    case couch_file:open(FName, [nologifmissing]) of
+        {ok, Fd} -> {ok, Fd};
+        {error, enoent} -> couch_file:open(FName, [create]);
+        Error -> Error
+    end.
+
+
+delete_files(DbName, Sig) ->
+    delete_index_file(DbName, Sig),
+    delete_compaction_file(DbName, Sig).
+
+
+delete_index_file(DbName, Sig) ->
+    delete_file(index_file(DbName, Sig)).
+
+
+delete_compaction_file(DbName, Sig) ->
+    delete_file(compaction_file(DbName, Sig)).
+
+
+delete_file(FName) ->
+    case filelib:is_file(FName) of
+        true ->
+            RootDir = couch_index_util:root_dir(),
+            couch_file:delete(RootDir, FName);
+        _ ->
+            ok
+    end.
+
+
+reset_index(Db, Fd, #mrst{sig=Sig}=State) ->
+    ok = couch_file:truncate(Fd, 0),
+    ok = couch_file:write_header(Fd, {Sig, nil}),
+    init_state(Db, Fd, reset_state(State), nil).
+
+
+reset_state(State) ->
+    State#mrst{
+        fd=nil,
+        qserver=nil,
+        update_seq=0,
+        id_btree=nil,
+        views=[View#mrview{btree=nil} || View <- State#mrst.views]
+    }.
+
+
+all_docs_key_opts(Args) ->
+    all_docs_key_opts(Args, []).
+
+
+all_docs_key_opts(#mrargs{keys=undefined}=Args, Extra) ->
+    all_docs_key_opts(Args#mrargs{keys=[]}, Extra);
+all_docs_key_opts(#mrargs{keys=[], direction=Dir}=Args, Extra) ->
+    [[{dir, Dir}] ++ ad_skey_opts(Args) ++ ad_ekey_opts(Args) ++ Extra];
+all_docs_key_opts(#mrargs{keys=Keys, direction=Dir}=Args, Extra) ->
+    lists:map(fun(K) ->
+        [{dir, Dir}]
+        ++ ad_skey_opts(Args#mrargs{start_key=K})
+        ++ ad_ekey_opts(Args#mrargs{end_key=K})
+        ++ Extra
+    end, Keys).
+
+
+ad_skey_opts(#mrargs{start_key=SKey}) when is_binary(SKey) ->
+    [{start_key, SKey}];
+ad_skey_opts(#mrargs{start_key_docid=SKeyDocId}) ->
+    [{start_key, SKeyDocId}].
+
+
+ad_ekey_opts(#mrargs{end_key=EKey}=Args) when is_binary(EKey) ->
+    Type = if Args#mrargs.inclusive_end -> end_key; true -> end_key_gt end,
+    [{Type, EKey}];
+ad_ekey_opts(#mrargs{end_key_docid=EKeyDocId}=Args) ->
+    Type = if Args#mrargs.inclusive_end -> end_key; true -> end_key_gt end,
+    [{Type, EKeyDocId}].
+
+
+key_opts(Args) ->
+    key_opts(Args, []).
+
+key_opts(#mrargs{keys=undefined, direction=Dir}=Args, Extra) ->
+    [[{dir, Dir}] ++ skey_opts(Args) ++ ekey_opts(Args) ++ Extra];
+key_opts(#mrargs{keys=Keys, direction=Dir}=Args, Extra) ->
+    lists:map(fun(K) ->
+        [{dir, Dir}]
+        ++ skey_opts(Args#mrargs{start_key=K})
+        ++ ekey_opts(Args#mrargs{end_key=K})
+        ++ Extra
+    end, Keys).
+
+
+skey_opts(#mrargs{start_key=undefined}) ->
+    [];
+skey_opts(#mrargs{start_key=SKey, start_key_docid=SKeyDocId}) ->
+    [{start_key, {SKey, SKeyDocId}}].
+
+
+ekey_opts(#mrargs{end_key=undefined}) ->
+    [];
+ekey_opts(#mrargs{end_key=EKey, end_key_docid=EKeyDocId}=Args) ->
+    case Args#mrargs.inclusive_end of
+        true -> [{end_key, {EKey, EKeyDocId}}];
+        false -> [{end_key_gt, {EKey, reverse_key_default(EKeyDocId)}}]
+    end.
+
+
+reverse_key_default(<<>>) -> <<255>>;
+reverse_key_default(<<255>>) -> <<>>;
+reverse_key_default(Key) -> Key.
+
+
+calculate_data_size(IdBt, Views) ->
+    SumFun = fun(#mrview{btree=Bt}, Acc) ->
+        sum_btree_sizes(Acc, couch_btree:size(Bt))
+    end,
+    Size = lists:foldl(SumFun, couch_btree:size(IdBt), Views),
+    {ok, Size}.
+
+
+sum_btree_sizes(nil, _) ->
+    null;
+sum_btree_sizes(_, nil) ->
+    null;
+sum_btree_sizes(Size1, Size2) ->
+    Size1 + Size2.
+
+
+detuple_kvs([], Acc) ->
+    lists:reverse(Acc);
+detuple_kvs([KV | Rest], Acc) ->
+    {{Key,Id},Value} = KV,
+    NKV = [[Key, Id], Value],
+    detuple_kvs(Rest, [NKV | Acc]).
+
+
+expand_dups([], Acc) ->
+    lists:reverse(Acc);
+expand_dups([{Key, {dups, Vals}} | Rest], Acc) ->
+    Expanded = [{Key, Val} || Val <- Vals],
+    expand_dups(Rest, Expanded ++ Acc);
+expand_dups([KV | Rest], Acc) ->
+    expand_dups(Rest, [KV | Acc]).
+
+
+maybe_load_doc(_Db, _DI, #mrargs{include_docs=false}) ->
+    [];
+maybe_load_doc(Db, #doc_info{}=DI, #mrargs{conflicts=true, doc_options=Opts}) ->
+    doc_row(couch_index_util:load_doc(Db, DI, [conflicts]), Opts);
+maybe_load_doc(Db, #doc_info{}=DI, #mrargs{doc_options=Opts}) ->
+    doc_row(couch_index_util:load_doc(Db, DI, []), Opts).
+
+
+maybe_load_doc(_Db, _Id, _Val, #mrargs{include_docs=false}) ->
+    [];
+maybe_load_doc(Db, Id, Val, #mrargs{conflicts=true, doc_options=Opts}) ->
+    doc_row(couch_index_util:load_doc(Db, docid_rev(Id, Val), [conflicts]), Opts);
+maybe_load_doc(Db, Id, Val, #mrargs{doc_options=Opts}) ->
+    doc_row(couch_index_util:load_doc(Db, docid_rev(Id, Val), []), Opts).
+
+
+doc_row(null, _Opts) ->
+    [{doc, null}];
+doc_row(Doc, Opts) ->
+    [{doc, couch_doc:to_json_obj(Doc, Opts)}].
+
+
+docid_rev(Id, {Props}) ->
+    DocId = couch_util:get_value(<<"_id">>, Props, Id),
+    Rev = case couch_util:get_value(<<"_rev">>, Props, nil) of
+        nil -> nil;
+        Rev0 -> couch_doc:parse_rev(Rev0)
+    end,
+    {DocId, Rev};
+docid_rev(Id, _) ->
+    {Id, nil}.
+
+
+index_of(Key, List) ->
+    index_of(Key, List, 1).
+
+
+index_of(_, [], _) ->
+    throw({error, missing_named_view});
+index_of(Key, [Key | _], Idx) ->
+    Idx;
+index_of(Key, [_ | Rest], Idx) ->
+    index_of(Key, Rest, Idx+1).
+
+
+mrverror(Mesg) ->
+    throw({query_parse_error, Mesg}).
+
+
+%% Updates 1.2.x or earlier view files to 1.3.x or later view files
+%% transparently, the first time the 1.2.x view file is opened by
+%% 1.3.x or later.
+%%
+%% Here's how it works:
+%%
+%% Before opening a view index,
+%% If no matching index file is found in the new location:
+%%  calculate the <= 1.2.x view signature
+%%  if a file with that signature lives in the old location
+%%    rename it to the new location with the new signature in the name.
+%% Then proceed to open the view index as usual.
+%% After opening, read its header.
+%%
+%% If the header matches the <= 1.2.x style #index_header record:
+%%   upgrade the header to the new #mrheader record
+%% The next time the view is used, the new header is used.
+%%
+%% If we crash after the rename, but before the header upgrade,
+%%   the header upgrade is done on the next view opening.
+%%
+%% If we crash between upgrading to the new header and writing
+%%   that header to disk, we start with the old header again,
+%%   do the upgrade and write to disk.
+
+maybe_update_index_file(State) ->
+    DbName = State#mrst.db_name,
+    NewIndexFile = index_file(DbName, State#mrst.sig),
+    % open in read-only mode so we don't create
+    % the file if it doesn't exist.
+    case file:open(NewIndexFile, [read, raw]) of
+    {ok, Fd_Read} ->
+        % the new index file exists, there is nothing to do here.
+        file:close(Fd_Read);
+    _Error ->
+        update_index_file(State)
+    end.
+
+update_index_file(State) ->
+    Sig = sig_vsn_12x(State),
+    DbName = State#mrst.db_name,
+    FileName = couch_index_util:hexsig(Sig) ++ ".view",
+    IndexFile = couch_index_util:index_file("", DbName, FileName),
+
+    % If we have an old index, rename it to the new position.
+    case file:read_file_info(IndexFile) of
+    {ok, _FileInfo} ->
+        % Crash if the rename fails for any reason.
+        % If the target exists, e.g. the next request will find the
+        % new file and we are good. We might need to catch this
+        % further up to avoid a full server crash.
+        ?LOG_INFO("Attempting to update legacy view index file.", []),
+        NewIndexFile = index_file(DbName, State#mrst.sig),
+        ok = filelib:ensure_dir(NewIndexFile),
+        ok = file:rename(IndexFile, NewIndexFile),
+        ?LOG_INFO("Successfully updated legacy view index file.", []),
+        Sig;
+    _ ->
+        % Ignore missing index file
+        ok
+    end.
+
+sig_vsn_12x(State) ->
+    ViewInfo = [old_view_format(V) || V <- State#mrst.views],
+    SigData = case State#mrst.lib of
+    {[]} ->
+        {ViewInfo, State#mrst.language, State#mrst.design_opts};
+    _ ->
+        {ViewInfo, State#mrst.language, State#mrst.design_opts,
+            couch_index_util:sort_lib(State#mrst.lib)}
+    end,
+    couch_util:md5(term_to_binary(SigData)).
+
+old_view_format(View) ->
+{
+    view,
+    View#mrview.id_num,
+    View#mrview.map_names,
+    View#mrview.def,
+    View#mrview.btree,
+    View#mrview.reduce_funs,
+    View#mrview.options
+}.
+
+%% End of <= 1.2.x upgrade code.

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_mrview/test/01-load.t
----------------------------------------------------------------------
diff --git a/apps/couch_mrview/test/01-load.t b/apps/couch_mrview/test/01-load.t
new file mode 100644
index 0000000..a57c1a7
--- /dev/null
+++ b/apps/couch_mrview/test/01-load.t
@@ -0,0 +1,34 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% Test that we can load each module.
+
+main(_) ->
+    test_util:init_code_path(),
+    Modules = [
+        couch_mrview,
+        couch_mrview_compactor,
+        couch_mrview_http,
+        couch_mrview_index,
+        couch_mrview_updater,
+        couch_mrview_util
+    ],
+
+    etap:plan(length(Modules)),
+    lists:foreach(
+        fun(Module) ->
+            etap:loaded_ok(Module, lists:concat(["Loaded: ", Module]))
+        end, Modules),
+    etap:end_tests().

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_mrview/test/02-map-views.t
----------------------------------------------------------------------
diff --git a/apps/couch_mrview/test/02-map-views.t b/apps/couch_mrview/test/02-map-views.t
new file mode 100644
index 0000000..7e1ca0c
--- /dev/null
+++ b/apps/couch_mrview/test/02-map-views.t
@@ -0,0 +1,131 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+    test_util:init_code_path(),
+
+    etap:plan(6),
+    case (catch test()) of
+        ok ->
+            etap:end_tests();
+        Other ->
+            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+            etap:bail(Other)
+    end,
+    timer:sleep(300),
+    ok.
+
+test() ->
+    couch_server_sup:start_link(test_util:config_files()),
+
+    {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, map),
+
+    test_basic(Db),
+    test_range(Db),
+    test_rev_range(Db),
+    test_limit_and_skip(Db),
+    test_include_docs(Db),
+    test_empty_view(Db),
+
+    ok.
+
+
+test_basic(Db) ->
+    Result = run_query(Db, []),
+    Expect = {ok, [
+        {meta, [{total, 10}, {offset, 0}]},
+        {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
+        {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
+        {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
+        {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
+        {row, [{id, <<"5">>}, {key, 5}, {value, 5}]},
+        {row, [{id, <<"6">>}, {key, 6}, {value, 6}]},
+        {row, [{id, <<"7">>}, {key, 7}, {value, 7}]},
+        {row, [{id, <<"8">>}, {key, 8}, {value, 8}]},
+        {row, [{id, <<"9">>}, {key, 9}, {value, 9}]},
+        {row, [{id, <<"10">>}, {key, 10}, {value, 10}]}
+    ]},
+    etap:is(Result, Expect, "Simple view query worked.").
+
+
+test_range(Db) ->
+    Result = run_query(Db, [{start_key, 3}, {end_key, 5}]),
+    Expect = {ok, [
+        {meta, [{total, 10}, {offset, 2}]},
+        {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
+        {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
+        {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
+    ]},
+    etap:is(Result, Expect, "Query with range works.").
+
+
+test_rev_range(Db) ->
+    Result = run_query(Db, [
+        {direction, rev},
+        {start_key, 5}, {end_key, 3},
+        {inclusive_end, true}
+    ]),
+    Expect = {ok, [
+        {meta, [{total, 10}, {offset, 5}]},
+        {row, [{id, <<"5">>}, {key, 5}, {value, 5}]},
+        {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
+        {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}
+    ]},
+    etap:is(Result, Expect, "Query with reversed range works.").
+
+
+test_limit_and_skip(Db) ->
+    Result = run_query(Db, [
+        {start_key, 2},
+        {limit, 3},
+        {skip, 3}
+    ]),
+    Expect = {ok, [
+        {meta, [{total, 10}, {offset, 4}]},
+        {row, [{id, <<"5">>}, {key, 5}, {value, 5}]},
+        {row, [{id, <<"6">>}, {key, 6}, {value, 6}]},
+        {row, [{id, <<"7">>}, {key, 7}, {value, 7}]}
+    ]},
+    etap:is(Result, Expect, "Query with limit and skip works.").
+
+
+test_include_docs(Db) ->
+    Result = run_query(Db, [
+        {start_key, 8},
+        {end_key, 8},
+        {include_docs, true}
+    ]),
+    Doc = {[
+        {<<"_id">>,<<"8">>},
+        {<<"_rev">>, <<"1-55b9a29311341e07ec0a7ca13bc1b59f">>},
+        {<<"val">>,8}
+    ]},
+    Expect = {ok, [
+        {meta, [{total, 10}, {offset, 7}]},
+        {row, [{id, <<"8">>}, {key, 8}, {value, 8}, {doc, Doc}]}
+    ]},
+    etap:is(Result, Expect, "Query with include docs works.").
+
+
+test_empty_view(Db) ->
+    Result = couch_mrview:query_view(Db, <<"_design/bar">>, <<"bing">>),
+    Expect = {ok, [
+        {meta, [{total, 0}, {offset, 0}]}
+    ]},
+    etap:is(Result, Expect, "Empty views are correct.").
+
+
+run_query(Db, Opts) ->
+    couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, Opts).

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_mrview/test/03-red-views.t
----------------------------------------------------------------------
diff --git a/apps/couch_mrview/test/03-red-views.t b/apps/couch_mrview/test/03-red-views.t
new file mode 100644
index 0000000..6ad341b
--- /dev/null
+++ b/apps/couch_mrview/test/03-red-views.t
@@ -0,0 +1,78 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+    test_util:run(4, fun() -> test() end).
+
+test() ->
+    couch_server_sup:start_link(test_util:config_files()),
+
+    {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, red),
+
+    test_basic(Db),
+    test_key_range(Db),
+    test_group_level(Db),
+    test_group_exact(Db),
+
+    ok.
+
+
+test_basic(Db) ->
+    Result = run_query(Db, []),
+    Expect = {ok, [
+        {meta, []},
+        {row, [{key, null}, {value, 55}]}
+    ]},
+    etap:is(Result, Expect, "Simple reduce view works.").
+
+
+test_key_range(Db) ->
+    Result = run_query(Db, [{start_key, [0, 2]}, {end_key, [0, 4]}]),
+    Expect = {ok, [
+        {meta, []},
+        {row, [{key, null}, {value, 6}]}
+    ]},
+    etap:is(Result, Expect, "Reduce with key range works.").
+
+
+test_group_level(Db) ->
+    Result = run_query(Db, [{group_level, 1}]),
+    Expect = {ok, [
+        {meta, []},
+        {row, [{key, [0]}, {value, 30}]},
+        {row, [{key, [1]}, {value, 25}]}
+    ]},
+    etap:is(Result, Expect, "Group level works.").
+
+test_group_exact(Db) ->
+    Result = run_query(Db, [{group_level, exact}]),
+    Expect = {ok, [
+        {meta, []},
+        {row, [{key, [0, 2]}, {value, 2}]},
+        {row, [{key, [0, 4]}, {value, 4}]},
+        {row, [{key, [0, 6]}, {value, 6}]},
+        {row, [{key, [0, 8]}, {value, 8}]},
+        {row, [{key, [0, 10]}, {value, 10}]},
+        {row, [{key, [1, 1]}, {value, 1}]},
+        {row, [{key, [1, 3]}, {value, 3}]},
+        {row, [{key, [1, 5]}, {value, 5}]},
+        {row, [{key, [1, 7]}, {value, 7}]},
+        {row, [{key, [1, 9]}, {value, 9}]}
+    ]},
+    etap:is(Result, Expect, "Group exact works.").
+
+
+run_query(Db, Opts) ->
+    couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, Opts).

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_mrview/test/04-index-info.t
----------------------------------------------------------------------
diff --git a/apps/couch_mrview/test/04-index-info.t b/apps/couch_mrview/test/04-index-info.t
new file mode 100644
index 0000000..6b67b56
--- /dev/null
+++ b/apps/couch_mrview/test/04-index-info.t
@@ -0,0 +1,54 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+    test_util:init_code_path(),
+
+    etap:plan(9),
+    case (catch test()) of
+        ok ->
+            etap:end_tests();
+        Other ->
+            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+            etap:bail(Other)
+    end,
+    timer:sleep(300),
+    ok.
+
+sig() -> <<"276df562b152b3c4e5d34024f62672ed">>.
+
+test() ->
+    couch_server_sup:start_link(test_util:config_files()),
+
+    {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, map),
+    couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>),
+
+    {ok, Info} = couch_mrview:get_info(Db, <<"_design/bar">>),
+
+    etap:is(getval(signature, Info), sig(), "Signature is ok."),
+    etap:is(getval(language, Info), <<"javascript">>, "Language is ok."),
+    etap:is_greater(getval(disk_size, Info), 0, "Disk size is ok."),
+    etap:is_greater(getval(data_size, Info), 0, "Data size is ok."),
+    etap:is(getval(update_seq, Info), 11, "Update seq is ok."),
+    etap:is(getval(purge_seq, Info), 0, "Purge seq is ok."),
+    etap:is(getval(updater_running, Info), false, "No updater running."),
+    etap:is(getval(compact_running, Info), false, "No compaction running."),
+    etap:is(getval(waiting_clients, Info), 0, "No waiting clients."),
+
+    ok.
+
+getval(Key, PL) ->
+    {value, {Key, Val}} = lists:keysearch(Key, 1, PL),
+    Val.

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_mrview/test/05-collation.t
----------------------------------------------------------------------
diff --git a/apps/couch_mrview/test/05-collation.t b/apps/couch_mrview/test/05-collation.t
new file mode 100644
index 0000000..ac8f8bc
--- /dev/null
+++ b/apps/couch_mrview/test/05-collation.t
@@ -0,0 +1,163 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+    test_util:run(9, fun() -> test() end).
+
+
+test() ->
+    couch_server_sup:start_link(test_util:config_files()),
+    {ok, Db0} = couch_mrview_test_util:new_db(<<"foo">>, map),
+    {ok, Db1} = couch_mrview_test_util:save_docs(Db0, docs()),
+
+    test_collated_fwd(Db1),
+    test_collated_rev(Db1),
+    test_range_collation(Db1),
+    test_inclusive_end(Db1),
+    test_uninclusive_end(Db1),
+    test_with_endkey_docid(Db1),
+
+    ok.
+
+test_collated_fwd(Db) ->
+    {ok, Results} = run_query(Db, []),
+    Expect = [{meta, [{total, 26}, {offset, 0}]}] ++ rows(),
+    etap:is(Results, Expect, "Values were collated correctly.").
+
+
+test_collated_rev(Db) ->
+    {ok, Results} = run_query(Db, [{direction, rev}]),
+    Expect = [{meta, [{total, 26}, {offset, 0}]}] ++ lists:reverse(rows()),
+    etap:is(Results, Expect, "Values were collated correctly descending.").
+
+
+test_range_collation(Db) ->
+    {_, Error} = lists:foldl(fun(V, {Count, Error}) ->
+        {ok, Results} = run_query(Db, [{start_key, V}, {end_key, V}]),
+        Id = list_to_binary(integer_to_list(Count)),
+        Expect = [
+            {meta, [{total, 26}, {offset, Count}]},
+            {row, [{id, Id}, {key, V}, {value, 0}]}
+        ],
+        case Results == Expect of
+            true -> {Count+1, Error};
+            _ -> {Count+1, true}
+        end
+    end, {0, false}, vals()),
+    etap:is(Error, false, "Found each individual key correctly.").
+
+
+test_inclusive_end(Db) ->
+    Opts = [{end_key, <<"b">>}, {inclusive_end, true}],
+    {ok, Rows0} = run_query(Db, Opts),
+    LastRow0 = lists:last(Rows0),
+    Expect0 = {row, [{id,<<"10">>}, {key,<<"b">>}, {value,0}]},
+    etap:is(LastRow0, Expect0, "Inclusive end is correct."),
+
+    {ok, Rows1} = run_query(Db, Opts ++ [{direction, rev}]),
+    LastRow1 = lists:last(Rows1),
+    Expect1 = {row, [{id,<<"10">>}, {key,<<"b">>}, {value,0}]},
+    etap:is(LastRow1, Expect1,
+            "Inclusive end is correct with descending=true").
+
+test_uninclusive_end(Db) ->
+    Opts = [{end_key, <<"b">>}, {inclusive_end, false}],
+    {ok, Rows0} = run_query(Db, Opts),
+    LastRow0 = lists:last(Rows0),
+    Expect0 = {row, [{id,<<"9">>}, {key,<<"aa">>}, {value,0}]},
+    etap:is(LastRow0, Expect0, "Uninclusive end is correct."),
+
+    {ok, Rows1} = run_query(Db, Opts ++ [{direction, rev}]),
+    LastRow1 = lists:last(Rows1),
+    Expect1 = {row, [{id,<<"11">>}, {key,<<"B">>}, {value,0}]},
+    etap:is(LastRow1, Expect1,
+            "Uninclusive end is correct with descending=true").
+
+
+test_with_endkey_docid(Db) ->
+    {ok, Rows0} = run_query(Db, [
+        {end_key, <<"b">>}, {end_key_docid, <<"10">>},
+        {inclusive_end, false}
+    ]),
+    Result0 = lists:last(Rows0),
+    Expect0 = {row, [{id,<<"9">>}, {key,<<"aa">>}, {value,0}]},
+    etap:is(Result0, Expect0, "Uninclsuive end with endkey_docid set is ok."),
+
+    {ok, Rows1} = run_query(Db, [
+        {end_key, <<"b">>}, {end_key_docid, <<"11">>},
+        {inclusive_end, false}
+    ]),
+    Result1 = lists:last(Rows1),
+    Expect1 = {row, [{id,<<"10">>}, {key,<<"b">>}, {value,0}]},
+    etap:is(Result1, Expect1, "Uninclsuive end with endkey_docid set is ok.").
+
+
+run_query(Db, Opts) ->
+    couch_mrview:query_view(Db, <<"_design/bar">>, <<"zing">>, Opts).
+
+
+docs() ->
+    {Docs, _} = lists:foldl(fun(V, {Docs0, Count}) ->
+        Doc = couch_doc:from_json_obj({[
+            {<<"_id">>, list_to_binary(integer_to_list(Count))},
+            {<<"foo">>, V}
+        ]}),
+        {[Doc | Docs0], Count+1}
+    end, {[], 0}, vals()),
+    Docs.
+
+
+rows() ->
+    {Rows, _} = lists:foldl(fun(V, {Rows0, Count}) ->
+        Id = list_to_binary(integer_to_list(Count)),
+        Row = {row, [{id, Id}, {key, V}, {value, 0}]},
+        {[Row | Rows0], Count+1}
+    end, {[], 0}, vals()),
+    lists:reverse(Rows).
+
+
+vals() ->
+    [
+        null,
+        false,
+        true,
+
+        1,
+        2,
+        3.0,
+        4,
+
+        <<"a">>,
+        <<"A">>,
+        <<"aa">>,
+        <<"b">>,
+        <<"B">>,
+        <<"ba">>,
+        <<"bb">>,
+
+        [<<"a">>],
+        [<<"b">>],
+        [<<"b">>, <<"c">>],
+        [<<"b">>, <<"c">>, <<"a">>],
+        [<<"b">>, <<"d">>],
+        [<<"b">>, <<"d">>, <<"e">>],
+
+        {[{<<"a">>, 1}]},
+        {[{<<"a">>, 2}]},
+        {[{<<"b">>, 1}]},
+        {[{<<"b">>, 2}]},
+        {[{<<"b">>, 2}, {<<"a">>, 1}]},
+        {[{<<"b">>, 2}, {<<"c">>, 2}]}
+    ].

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_mrview/test/06-all-docs.t
----------------------------------------------------------------------
diff --git a/apps/couch_mrview/test/06-all-docs.t b/apps/couch_mrview/test/06-all-docs.t
new file mode 100644
index 0000000..4501aa5
--- /dev/null
+++ b/apps/couch_mrview/test/06-all-docs.t
@@ -0,0 +1,127 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+    test_util:run(6, fun() -> test() end).
+
+
+test() ->
+    couch_server_sup:start_link(test_util:config_files()),
+
+    {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, map),
+
+    test_basic(Db),
+    test_range(Db),
+    test_rev_range(Db),
+    test_limit_and_skip(Db),
+    test_include_docs(Db),
+    test_empty_view(Db),
+
+    ok.
+
+
+test_basic(Db) ->
+    Result = run_query(Db, []),
+    Expect = {ok, [
+        {meta, [{total, 11}, {offset, 0}]},
+        mk_row(<<"1">>, <<"1-08d53a5760b95fce6df2e2c5b008be39">>),
+        mk_row(<<"10">>, <<"1-a05b6ea2bc0243949f103d5b4f15f71e">>),
+        mk_row(<<"2">>, <<"1-b57c77a9e6f7574ca6469f0d6dcd78bb">>),
+        mk_row(<<"3">>, <<"1-7fbf84d56f8017880974402d60f5acd6">>),
+        mk_row(<<"4">>, <<"1-fcaf5852c08ffb239ac8ce16c409f253">>),
+        mk_row(<<"5">>, <<"1-aaac5d460fd40f9286e57b9bf12e23d2">>),
+        mk_row(<<"6">>, <<"1-aca21c2e7bc5f8951424fcfc5d1209d8">>),
+        mk_row(<<"7">>, <<"1-4374aeec17590d82f16e70f318116ad9">>),
+        mk_row(<<"8">>, <<"1-55b9a29311341e07ec0a7ca13bc1b59f">>),
+        mk_row(<<"9">>, <<"1-558c8487d9aee25399a91b5d31d90fe2">>),
+        mk_row(<<"_design/bar">>, <<"1-a44e1dd1994a7717bf89c894ebd1f081">>)
+    ]},
+    etap:is(Result, Expect, "Simple view query worked.").
+
+
+test_range(Db) ->
+    Result = run_query(Db, [{start_key, <<"3">>}, {end_key, <<"5">>}]),
+    Expect = {ok, [
+        {meta, [{total, 11}, {offset, 3}]},
+        mk_row(<<"3">>, <<"1-7fbf84d56f8017880974402d60f5acd6">>),
+        mk_row(<<"4">>, <<"1-fcaf5852c08ffb239ac8ce16c409f253">>),
+        mk_row(<<"5">>, <<"1-aaac5d460fd40f9286e57b9bf12e23d2">>)
+    ]},
+    etap:is(Result, Expect, "Query with range works.").
+
+
+test_rev_range(Db) ->
+    Result = run_query(Db, [
+        {direction, rev},
+        {start_key, <<"5">>}, {end_key, <<"3">>},
+        {inclusive_end, true}
+    ]),
+    Expect = {ok, [
+        {meta, [{total, 11}, {offset, 5}]},
+        mk_row(<<"5">>, <<"1-aaac5d460fd40f9286e57b9bf12e23d2">>),
+        mk_row(<<"4">>, <<"1-fcaf5852c08ffb239ac8ce16c409f253">>),
+        mk_row(<<"3">>, <<"1-7fbf84d56f8017880974402d60f5acd6">>)
+    ]},
+    etap:is(Result, Expect, "Query with reversed range works.").
+
+
+test_limit_and_skip(Db) ->
+    Result = run_query(Db, [
+        {start_key, <<"2">>},
+        {limit, 3},
+        {skip, 3}
+    ]),
+    Expect = {ok, [
+        {meta, [{total, 11}, {offset, 5}]},
+        mk_row(<<"5">>, <<"1-aaac5d460fd40f9286e57b9bf12e23d2">>),
+        mk_row(<<"6">>, <<"1-aca21c2e7bc5f8951424fcfc5d1209d8">>),
+        mk_row(<<"7">>, <<"1-4374aeec17590d82f16e70f318116ad9">>)
+    ]},
+    etap:is(Result, Expect, "Query with limit and skip works.").
+
+
+test_include_docs(Db) ->
+    Result = run_query(Db, [
+        {start_key, <<"8">>},
+        {end_key, <<"8">>},
+        {include_docs, true}
+    ]),
+    Doc = {[
+        {<<"_id">>,<<"8">>},
+        {<<"_rev">>, <<"1-55b9a29311341e07ec0a7ca13bc1b59f">>},
+        {<<"val">>, 8}
+    ]},
+    Val = {[{rev, <<"1-55b9a29311341e07ec0a7ca13bc1b59f">>}]},
+    Expect = {ok, [
+        {meta, [{total, 11}, {offset, 8}]},
+        {row, [{id, <<"8">>}, {key, <<"8">>}, {value, Val}, {doc, Doc}]}
+    ]},
+    etap:is(Result, Expect, "Query with include docs works.").
+
+
+test_empty_view(Db) ->
+    Result = couch_mrview:query_view(Db, <<"_design/bar">>, <<"bing">>),
+    Expect = {ok, [
+        {meta, [{total, 0}, {offset, 0}]}
+    ]},
+    etap:is(Result, Expect, "Empty views are correct.").
+
+
+mk_row(Id, Rev) ->
+    {row, [{id, Id}, {key, Id}, {value, {[{rev, Rev}]}}]}.
+
+
+run_query(Db, Opts) ->
+    couch_mrview:query_all_docs(Db, Opts).

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_mrview/test/07-compact-swap.t
----------------------------------------------------------------------
diff --git a/apps/couch_mrview/test/07-compact-swap.t b/apps/couch_mrview/test/07-compact-swap.t
new file mode 100644
index 0000000..4bfe124
--- /dev/null
+++ b/apps/couch_mrview/test/07-compact-swap.t
@@ -0,0 +1,57 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+    test_util:run(1, fun() -> test() end).
+
+
+test() ->
+    couch_server_sup:start_link(test_util:config_files()),
+    {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, map, 1000),
+    couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>),
+    test_swap(Db),
+    ok.
+
+
+test_swap(Db) ->
+    {ok, QPid} = start_query(Db),    
+    {ok, MonRef} = couch_mrview:compact(Db, <<"_design/bar">>, [monitor]),
+    receive
+        {'DOWN', MonRef, process, _, _} -> ok
+    after 1000 ->
+        throw(compaction_failed)
+    end,
+    QPid ! {self(), continue},
+    receive
+        {QPid, Count} ->
+            etap:is(Count, 1000, "View finished successfully.")
+    after 1000 ->
+        throw("query failed")
+    end.
+
+
+start_query(Db) ->
+    Self = self(),
+    Pid = spawn(fun() ->
+        CB = fun
+            (_, wait) -> receive {Self, continue} -> {ok, 0} end;
+            ({row, _}, Count) -> {ok, Count+1};
+            (_, Count) -> {ok, Count}
+        end,
+        {ok, Result} = 
+        couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, [], CB, wait),
+        Self ! {self(), Result}
+    end),
+    {ok, Pid}.

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_plugins/Makefile.am
----------------------------------------------------------------------
diff --git a/apps/couch_plugins/Makefile.am b/apps/couch_plugins/Makefile.am
new file mode 100644
index 0000000..37cd9d5
--- /dev/null
+++ b/apps/couch_plugins/Makefile.am
@@ -0,0 +1,40 @@
+## Licensed under the Apache License, Version 2.0 (the "License"); you may not
+## use this file except in compliance with the License. You may obtain a copy of
+## the License at
+##
+##   http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+## License for the specific language governing permissions and limitations under
+## the License.
+
+couch_pluginslibdir = $(localerlanglibdir)/couch_plugins-0.1
+couch_pluginsebindir = $(couch_pluginslibdir)/ebin
+
+couch_pluginsebin_DATA = $(compiled_files)
+
+
+source_files = \
+    src/couch_plugins.app.src \
+    src/couch_plugins.erl \
+    src/couch_plugins_httpd.erl
+
+compiled_files = \
+    ebin/couch_plugins.app \
+    ebin/couch_plugins.beam \
+    ebin/couch_plugins_httpd.beam
+
+EXTRA_DIST = $(source_files) README.md
+CLEANFILES = $(compiled_files)
+
+ebin/%.app: src/%.app.src
+	@mkdir -p ebin/
+	sed -e "s|%version%|@version@|g" \
+	< $< > $@
+
+ebin/%.beam: src/%.erl $(include_files)
+	@mkdir -p ebin/
+	$(ERLC) -Wall -I$(top_srcdir)/src -I$(top_srcdir)/src/couchdb \
+		-o ebin/ $(ERLC_FLAGS) ${TEST} $<;

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_plugins/README.md
----------------------------------------------------------------------
diff --git a/apps/couch_plugins/README.md b/apps/couch_plugins/README.md
new file mode 100644
index 0000000..b00a080
--- /dev/null
+++ b/apps/couch_plugins/README.md
@@ -0,0 +1,159 @@
+Heya,
+
+I couldn’t help myself thinking about plugin stuff and ended up
+whipping up a proof of concept.
+
+Here’s a <1 minute demo video:
+
+  https://dl.dropboxusercontent.com/u/82149/couchdb-plugins-demo.mov
+
+Alternative encoding:
+
+  https://dl.dropboxusercontent.com/u/82149/couchdb-plugins-demo.m4v)
+
+
+In my head the whole plugin idea is a very wide area, but I was so
+intrigued by the idea of getting something running with a click on a
+button in Futon. So I looked for a minimally viable plugin system.
+
+
+## Design principles
+
+It took me a day to put this all together and this was only possible
+because I took a lot of shortcuts. I believe they are all viable for a
+first iteration of a plugins system:
+
+1. Install with one click on a button in Futon (or HTTP call)
+2. Only pure Erlang plugins are allowed.
+3. The plugin author must provide a binary package for each Erlang (and,
+   later, each CouchDB version).
+4. Complete trust-based system. You trust me to not do any nasty things
+   when you click on the install button. No crypto, no nothing. Only
+   people who can commit to Futon can release new versions of plugins.
+5. Minimal user-friendlyness: won’t install plugins that don’t match
+   the current Erlang version, gives semi-sensible error messages
+   (wrapped in a HTTP 500 response :)
+6. Require a pretty strict format for binary releases.
+
+
+## Roadmap
+
+Here’s a list of things this first iterations does and doesn’t do:
+
+- Pure Erlang plugins only. No C-dependencies, no JavaScript, no nothing.
+- No C-dependencies.
+- Install a plugin via Futon (or HTTP call). Admin only.
+- A hardcoded list of plugins in Futon.
+- Loads a pre-packaged, pre-compiled .tar.gz file from a URL.
+- Only installs if Erlang version matches.
+- No security checking of binaries.
+- No identity checking of binaries.
+- Register installed plugins in the config system.
+- Make sure plugins start with the next restart of CouchDB.
+- Uninstall a plugin via Futon (or HTTP call). Admin only.
+- Show when a particular plugin is installed.
+- Only installs if CouchDB version matches.
+- Serve static web assets (for Futon/Fauxton) from `/_plugins/<name>/`.
+
+I hope you agree we can ship this with a few warnings so people can get a
+hang of it.
+
+
+A roadmap, progress and issues can be found here:
+
+https://issues.apache.org/jira/issues/?jql=component+%3D+Plugins+AND+project+%3D+COUCHDB+AND+resolution+%3D+Unresolved+ORDER+BY+priority+DESC
+
+
+
+## How it works
+
+This plugin system lives in `src/couch_plugins` and is a tiny CouchDB
+module.
+
+It exposes one new API endpoint `/_plugins` that an admin user can
+POST to.
+
+The additional Futon page lives at `/_utils/plugins.html` it is
+hardcoded.
+
+Futon (or you) post an object to `/_plugins` with four properties:
+
+    {
+      "name": "geocouch", // name of the plugin, must be unique
+      "url": "http://people.apache.org/~jan", // “base URL” for plugin releases (see below)
+      "version": "couchdb1.2.x_v0.3.0-11-g4ea0bea", // whatever version internal to the plugin
+      "checksums": {
+        "R15B03": "ZetgdHj2bY2w37buulWVf3USOZs=" // base64’d sha hash over the binary
+      }
+    }
+
+`couch_plugins` then attempts to download a .tar.gz from this
+location:
+
+    http://people.apache.org/~jan/geocouch-couchdb1.2.x_v0.3.0-12-g4ea0bea-R15B03.tar.gz
+
+It should be obvious how the URL is constructed from the POST data.
+(This url is live, feel free to play around with this tarball).
+
+Next it calculates the sha hash for the downloaded .tar.gz file and
+matches it against the correct version in the `checksums` parameter.
+
+If that succeeds, we unpack the .tar.gz file (currently in `/tmp`,
+need to find a better place for this) and adds the extracted directory
+to the Erlang code path
+(`code:add_path("/tmp/couchdb_plugins/geocouch-couchdb1.2.x_v0.3.0-12-g4ea0bea-R15B03/ebin")`)
+and loads the included application (`application:load(geocouch)`).
+
+Then it looks into the `./priv/default.d` directory that lives next to
+`ebin/` in the plugin directory for configuration `.ini` files and loads them.
+On next startup these configuration files are loaded after global defaults,
+and before any local configuration.
+
+If that all goes to plan, we report success back to the HTTP caller.
+
+That’s it! :)
+
+It’s deceptively simple, probably does a few things very wrong and
+leaves a few things open (see above).
+
+One open question I’d like an answer for is finding a good location to
+unpack & install the plugin files that isn’t `tmp`. If the answer is
+different for a pre-BigCouch/rcouch-merge and post-BigCouch/rcouch-
+merge world, I’d love to know :)
+
+
+## Code
+
+The main branch for this is 1867-feature-plugins:
+
+  ASF: https://git-wip-us.apache.org/repos/asf?p=couchdb.git;a=log;h=refs/heads/1867-feature-plugins
+  GitHub: https://github.com/janl/couchdb/compare/apache:master...1867-feature-plugins
+
+I created a branch on GeoCouch that adds a few lines to its `Makefile`
+that shows how a binary package is built:
+
+    https://github.com/janl/geocouch/compare/couchbase:couchdb1.3.x...couchdb1.3.x-plugins
+
+
+## Build
+
+Build CouchDB as usual:
+
+    ./bootstrap
+    ./configure
+    make
+    make dev
+    ./utils/run
+
+* * *
+
+I hope you like this :) Please comment and improve heavily!
+
+Let me know if you have any questions :)
+
+If you have any criticism, please phrase it in a way that we can use
+to improve this, thanks!
+
+Best,
+Jan
+--

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_plugins/src/couch_plugins.app.src
----------------------------------------------------------------------
diff --git a/apps/couch_plugins/src/couch_plugins.app.src b/apps/couch_plugins/src/couch_plugins.app.src
new file mode 100644
index 0000000..d961289
--- /dev/null
+++ b/apps/couch_plugins/src/couch_plugins.app.src
@@ -0,0 +1,23 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+{application, couch_plugins,
+ [
+  {description, "A CouchDB Plugin Installer"},
+  {vsn, "1"},
+  {registered, []},
+  {applications, [
+                  kernel,
+                  stdlib
+                 ]},
+  {mod, { couch_plugins_app, []}},
+  {env, []}
+ ]}.

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_plugins/src/couch_plugins.erl
----------------------------------------------------------------------
diff --git a/apps/couch_plugins/src/couch_plugins.erl b/apps/couch_plugins/src/couch_plugins.erl
new file mode 100644
index 0000000..dcbd2d3
--- /dev/null
+++ b/apps/couch_plugins/src/couch_plugins.erl
@@ -0,0 +1,300 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+-module(couch_plugins).
+-include_lib("couch/include/couch_db.hrl").
+-export([install/1, uninstall/1]).
+
+% couch_plugins:install({"geocouch", "http://127.0.0.1:8000", "1.0.0", [{"R15B03", "+XOJP6GSzmuO2qKdnjO+mWckXVs="}]}).
+% couch_plugins:install({"geocouch", "http://people.apache.org/~jan/", "couchdb1.2.x_v0.3.0-11-gd83ba22", [{"R15B03", "ZetgdHj2bY2w37buulWVf3USOZs="}]}).
+
+plugin_dir() ->
+  couch_config:get("couchdb", "plugin_dir").
+
+log(T) ->
+  ?LOG_DEBUG("[couch_plugins] ~p ~n", [T]).
+
+%% "geocouch", "http://localhost:8000/dist", "1.0.0"
+-type plugin() :: {string(), string(), string(), list()}.
+-spec install(plugin()) -> ok | {error, string()}.
+install({Name, _BaseUrl, Version, Checksums}=Plugin) ->
+  log("Installing " ++ Name),
+
+  {ok, LocalFilename} = download(Plugin),
+  log("downloaded to " ++ LocalFilename),
+
+  ok = verify_checksum(LocalFilename, Checksums),
+  log("checksum verified"),
+
+  ok = untargz(LocalFilename),
+  log("extraction done"),
+
+  ok = add_code_path(Name, Version),
+  log("added code path"),
+
+  ok = register_plugin(Name, Version),
+  log("registered plugin"),
+
+  load_config(Name, Version),
+  log("loaded config"),
+
+  ok.
+
+% Idempotent uninstall, if you uninstall a non-existant
+% plugin, you get an `ok`.
+-spec uninstall(plugin()) -> ok | {error, string()}.
+uninstall({Name, _BaseUrl, Version, _Checksums}) ->
+  % unload config
+  ok = unload_config(Name, Version),
+  log("config unloaded"),
+
+  % delete files
+  ok = delete_files(Name, Version),
+  log("files deleted"),
+
+  % delete code path
+  ok = del_code_path(Name, Version),
+  log("deleted code path"),
+
+  % unregister plugin
+  ok = unregister_plugin(Name),
+  log("unregistered plugin"),
+
+  % done
+  ok.
+
+%% * * *
+
+
+%% Plugin Registration
+%% On uninstall:
+%%  - add plugins/name = version to config
+%% On uninstall:
+%%  - remove plugins/name from config
+
+-spec register_plugin(string(), string()) -> ok.
+register_plugin(Name, Version) ->
+  couch_config:set("plugins", Name, Version).
+
+-spec unregister_plugin(string()) -> ok.
+unregister_plugin(Name) ->
+  couch_config:delete("plugins", Name).
+
+%% * * *
+
+
+%% Load Config
+%% Parses <plugindir>/priv/default.d/<pluginname.ini> and applies
+%% the contents to the config system, or removes them on uninstall
+
+-spec load_config(string(), string()) -> ok.
+load_config(Name, Version) ->
+    loop_config(Name, Version, fun set_config/1).
+
+-spec unload_config(string(), string()) -> ok.
+unload_config(Name, Version) ->
+    loop_config(Name, Version, fun delete_config/1).
+
+-spec loop_config(string(), string(), function()) -> ok.
+loop_config(Name, Version, Fun) ->
+    lists:foreach(fun(File) -> load_config_file(File, Fun) end,
+      filelib:wildcard(file_names(Name, Version))).
+
+-spec load_config_file(string(), function()) -> ok.
+load_config_file(File, Fun) ->
+    {ok, Config} = couch_config:parse_ini_file(File),
+    lists:foreach(Fun, Config).
+
+-spec set_config({{string(), string()}, string()}) -> ok.
+set_config({{Section, Key}, Value}) ->
+    ok = couch_config:set(Section, Key, Value).
+
+-spec delete_config({{string(), string()}, _Value}) -> ok.
+delete_config({{Section, Key}, _Value}) ->
+    ok = couch_config:delete(Section, Key).
+
+-spec file_names(string(), string()) -> string().
+file_names(Name, Version) ->
+  filename:join(
+    [plugin_dir(), get_file_slug(Name, Version),
+     "priv", "default.d", "*.ini"]).
+
+%% * * *
+
+
+%% Code Path Management
+%% The Erlang code path is where the Erlang runtime looks for `.beam`
+%% files to load on, say, `application:load()`. Since plugin directories
+%% are created on demand and named after CouchDB and Erlang versions,
+%% we manage the Erlang code path semi-automatically here.
+
+-spec add_code_path(string(), string()) -> ok | {error, bad_directory}.
+add_code_path(Name, Version) ->
+  PluginPath = plugin_dir() ++ "/" ++ get_file_slug(Name, Version) ++ "/ebin",
+  case code:add_path(PluginPath) of
+    true -> ok;
+    Else ->
+      ?LOG_ERROR("Failed to add PluginPath: '~s'", [PluginPath]),
+      Else
+  end.
+
+-spec del_code_path(string(), string()) -> ok | {error, atom()}.
+del_code_path(Name, Version) ->
+  PluginPath = plugin_dir() ++ "/" ++ get_file_slug(Name, Version) ++ "/ebin",
+  case code:del_path(PluginPath) of
+    true -> ok;
+    _Else ->
+      ?LOG_DEBUG("Failed to delete PluginPath: '~s', ignoring", [PluginPath]),
+      ok
+  end.
+
+%% * * *
+
+
+-spec untargz(string()) -> {ok, string()} | {error, string()}.
+untargz(Filename) ->
+  % read .gz file
+  {ok, GzData} = file:read_file(Filename),
+  % gunzip
+  log("unzipped"),
+  TarData = zlib:gunzip(GzData),
+  ok = filelib:ensure_dir(plugin_dir()),
+  % untar
+  erl_tar:extract({binary, TarData}, [{cwd, plugin_dir()}, keep_old_files]).
+
+-spec delete_files(string(), string()) -> ok | {error, atom()}.
+delete_files(Name, Version) ->
+  PluginPath = plugin_dir() ++ "/" ++ get_file_slug(Name, Version),
+  mochitemp:rmtempdir(PluginPath).
+
+
+% downloads a pluygin .tar.gz into a local plugins directory
+-spec download(string()) -> ok | {error, string()}.
+download({Name, _BaseUrl, Version, _Checksums}=Plugin) ->
+  TargetFile = "/tmp/" ++ get_filename(Name, Version),
+  case file_exists(TargetFile) of
+    %% wipe and redownload
+    true -> file:delete(TargetFile);
+    _Else -> ok
+  end,
+  Url = get_url(Plugin),
+  HTTPOptions = [
+    {connect_timeout, 30*1000}, % 30 seconds
+    {timeout, 30*1000} % 30 seconds
+  ],
+  % todo: windows
+  Options = [
+    {stream, TargetFile}, % /tmp/something
+    {body_format, binary},
+    {full_result, false}
+  ],
+  % todo: reduce to just httpc:request()
+  case httpc:request(get, {Url, []}, HTTPOptions, Options) of
+    {ok, _Result} ->
+      log("downloading " ++ Url),
+      {ok, TargetFile};
+    Error -> Error
+  end.
+
+-spec verify_checksum(string(), list()) -> ok | {error, string()}.
+verify_checksum(Filename, Checksums) ->
+
+  CouchDBVersion = couchdb_version(),
+  case proplists:get_value(CouchDBVersion, Checksums) of
+  undefined ->
+    ?LOG_ERROR("[couch_plugins] Can't find checksum for CouchDB Version '~s'", [CouchDBVersion]),
+    {error, no_couchdb_checksum};
+  OTPChecksum ->
+    OTPRelease = erlang:system_info(otp_release),
+    case proplists:get_value(OTPRelease, OTPChecksum) of
+    undefined ->
+      ?LOG_ERROR("[couch_plugins] Can't find checksum for Erlang Version '~s'", [OTPRelease]),
+      {error, no_erlang_checksum};
+    Checksum ->
+      do_verify_checksum(Filename, Checksum)
+    end
+  end.
+
+-spec do_verify_checksum(string(), string()) -> ok | {error, string()}.
+do_verify_checksum(Filename, Checksum) ->
+  ?LOG_DEBUG("Checking Filename: ~s", [Filename]),
+  case file:read_file(Filename) of
+  {ok, Data} ->
+    ComputedChecksum = binary_to_list(base64:encode(crypto:sha(Data))),
+    case ComputedChecksum of
+    Checksum -> ok;
+    _Else ->
+      ?LOG_ERROR("Checksum mismatch. Wanted: '~p'. Got '~p'", [Checksum, ComputedChecksum]),
+      {error, checksum_mismatch}
+    end;
+  Error -> Error
+  end.
+
+
+%% utils
+
+-spec get_url(plugin()) -> string().
+get_url({Name, BaseUrl, Version, _Checksums}) ->
+  BaseUrl ++ "/" ++ get_filename(Name, Version).
+
+-spec get_filename(string(), string()) -> string().
+get_filename(Name, Version) ->
+  get_file_slug(Name, Version) ++ ".tar.gz".
+
+-spec get_file_slug(string(), string()) -> string().
+get_file_slug(Name, Version) ->
+  % OtpRelease does not include patch levels like the -1 in R15B03-1
+  OTPRelease = erlang:system_info(otp_release),
+  CouchDBVersion = couchdb_version(),
+  string:join([Name, Version, OTPRelease, CouchDBVersion], "-").
+
+-spec file_exists(string()) -> boolean().
+file_exists(Filename) ->
+  does_file_exist(file:read_file_info(Filename)).
+-spec does_file_exist(term()) -> boolean().
+does_file_exist({error, enoent}) -> false;
+does_file_exist(_Else) -> true.
+
+couchdb_version() ->
+  couch_server:get_version(short).
+
+% installing a plugin:
+%  - POST /_plugins -d {plugin-def}
+%  - get plugin definition
+%  - get download URL (matching erlang version)
+%  - download archive
+%  - match checksum
+%  - untar-gz archive into a plugins dir
+%  - code:add_path(“geocouch-{geocouch_version}-{erlang_version}/ebin”)
+%  - [cp geocouch-{geocouch_version}-{erlang_version}/etc/ ]
+%  - application:start(geocouch)
+%  - register plugin in plugin registry
+
+% Plugin registry impl:
+%  - _plugins database
+%   - pro: known db ops
+%   - con: no need for replication, needs to be system db etc.
+%  - _config/plugins namespace in config
+%   - pro: lightweight, fits rarely-changing nature better
+%   - con: potentially not flexible enough
+
+
+
+% /geocouch
+% /geocouch/dist/
+% /geocouch/dist/geocouch-{geocouch_version}-{erlang_version}.tar.gz
+
+% tar.gz includes:
+% geocouch-{geocouch_version}-{erlang_version}/
+% geocouch-{geocouch_version}-{erlang_version}/ebin
+% [geocouch-{geocouch_version}-{erlang_version}/config/config.erlt]
+% [geocouch-{geocouch_version}-{erlang_version}/share/]
+

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_plugins/src/couch_plugins_httpd.erl
----------------------------------------------------------------------
diff --git a/apps/couch_plugins/src/couch_plugins_httpd.erl b/apps/couch_plugins/src/couch_plugins_httpd.erl
new file mode 100644
index 0000000..4dabbb4
--- /dev/null
+++ b/apps/couch_plugins/src/couch_plugins_httpd.erl
@@ -0,0 +1,65 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+-module(couch_plugins_httpd).
+
+-export([handle_req/1]).
+
+-include_lib("couch/include/couch_db.hrl").
+
+handle_req(#httpd{method='POST'}=Req) ->
+    ok = couch_httpd:verify_is_server_admin(Req),
+    couch_httpd:validate_ctype(Req, "application/json"),
+
+    {PluginSpec} = couch_httpd:json_body_obj(Req),
+    Url = binary_to_list(couch_util:get_value(<<"url">>, PluginSpec)),
+    Name = binary_to_list(couch_util:get_value(<<"name">>, PluginSpec)),
+    Version = binary_to_list(couch_util:get_value(<<"version">>, PluginSpec)),
+    Delete = couch_util:get_value(<<"delete">>, PluginSpec),
+    {Checksums0} = couch_util:get_value(<<"checksums">>, PluginSpec),
+    Checksums = parse_checksums(Checksums0),
+
+    Plugin = {Name, Url, Version, Checksums},
+    case do_install(Delete, Plugin) of
+    ok ->
+        couch_httpd:send_json(Req, 202, {[{ok, true}]});
+    Error ->
+        ?LOG_DEBUG("Plugin Spec: ~p", [PluginSpec]),
+        couch_httpd:send_error(Req, {bad_request, Error})
+    end;
+% handles /_plugins/<pluginname>/<file>
+% serves <plugin_dir>/<pluginname>-<pluginversion>-<otpversion>-<couchdbversion>/<file>
+handle_req(#httpd{method='GET',path_parts=[_, Name0 | Path0]}=Req) ->
+    Name = ?b2l(Name0),
+    Path = lists:map(fun binary_to_list/1, Path0),
+    OTPRelease = erlang:system_info(otp_release),
+    PluginVersion = couch_config:get("plugins", Name),
+    CouchDBVersion = couch_server:get_version(short),
+    FullName = string:join([Name, PluginVersion, OTPRelease, CouchDBVersion], "-"),
+    FullPath = filename:join([FullName, "priv", "www", string:join(Path, "/")]) ++ "/",
+    ?LOG_DEBUG("Serving ~p from ~p", [FullPath, plugin_dir()]),
+    couch_httpd:serve_file(Req, FullPath, plugin_dir());
+handle_req(Req) ->
+    couch_httpd:send_method_not_allowed(Req, "POST").
+
+plugin_dir() ->
+  couch_config:get("couchdb", "plugin_dir").
+do_install(false, Plugin) ->
+    couch_plugins:install(Plugin);
+do_install(true, Plugin) ->
+    couch_plugins:uninstall(Plugin).
+
+parse_checksums(Checksums) ->
+    lists:map(fun({K, {V}}) ->
+        {binary_to_list(K), parse_checksums(V)};
+      ({K, V}) ->
+         {binary_to_list(K), binary_to_list(V)}
+    end, Checksums).

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_replicator/Makefile.am
----------------------------------------------------------------------
diff --git a/apps/couch_replicator/Makefile.am b/apps/couch_replicator/Makefile.am
new file mode 100644
index 0000000..2dcd47d
--- /dev/null
+++ b/apps/couch_replicator/Makefile.am
@@ -0,0 +1,78 @@
+## Licensed under the Apache License, Version 2.0 (the "License"); you may not
+## use this file except in compliance with the License. You may obtain a copy of
+## the License at
+##
+##   http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+## License for the specific language governing permissions and limitations under
+## the License.
+
+couch_replicatorlibdir = $(localerlanglibdir)/couch_replicator-0.1
+couch_replicatorincludedir = $(couch_replicatorlibdir)/include
+couch_replicatorebindir = $(couch_replicatorlibdir)/ebin
+
+couch_replicatorinclude_DATA = $(include_files)
+couch_replicatorebin_DATA = $(compiled_files)
+
+include_files = \
+	src/couch_replicator_api_wrap.hrl \
+	src/couch_replicator.hrl \
+	src/couch_replicator_js_functions.hrl
+
+source_files = \
+	src/couch_replicator_api_wrap.erl \
+	src/couch_replicator_httpc_pool.erl \
+	src/couch_replicator_httpc.erl \
+	src/couch_replicator_httpd.erl \
+	src/couch_replicator_job_sup.erl \
+	src/couch_replicator_notifier.erl \
+	src/couch_replicator_manager.erl \
+	src/couch_replicator_utils.erl \
+	src/couch_replicator_worker.erl \
+	src/couch_replicator.app.src \
+	src/couch_replicator.erl
+
+test_files = \
+	test/01-load.t \
+	test/02-httpc-pool.t \
+	test/03-replication-compact.t \
+	test/04-replication-large-atts.t \
+	test/05-replication-many-leaves.t \
+	test/06-doc-missing-stubs.t \
+	test/07-use-checkpoints.t
+
+compiled_files = \
+	ebin/couch_replicator_api_wrap.beam \
+	ebin/couch_replicator_httpc_pool.beam \
+	ebin/couch_replicator_httpc.beam \
+	ebin/couch_replicator_httpd.beam \
+	ebin/couch_replicator_job_sup.beam \
+	ebin/couch_replicator_notifier.beam \
+	ebin/couch_replicator_manager.beam \
+	ebin/couch_replicator_utils.beam \
+	ebin/couch_replicator_worker.beam \
+	ebin/couch_replicator.app \
+	ebin/couch_replicator.beam
+
+EXTRA_DIST = $(include_files) $(source_files) $(test_files)
+CLEANFILES = $(compiled_files)
+
+check:
+if TESTS
+	$(abs_top_builddir)/test/etap/run $(abs_top_srcdir)/src/couch_replicator/test
+endif
+
+ebin/%.app: src/%.app.src
+	@mkdir -p ebin/
+	sed -e "s|%version%|@version@|g" \
+	< $< > $@
+
+ebin/%.beam: src/%.erl $(include_files)
+	@mkdir -p ebin/
+	$(ERLC) -Wall -I$(top_srcdir)/src -I$(top_srcdir)/src/couchdb \
+        -o ebin/ $(ERLC_FLAGS) ${TEST} $<;
+
+

http://git-wip-us.apache.org/repos/asf/couchdb/blob/add91738/apps/couch_replicator/src/couch_replicator.app.src
----------------------------------------------------------------------
diff --git a/apps/couch_replicator/src/couch_replicator.app.src b/apps/couch_replicator/src/couch_replicator.app.src
new file mode 100644
index 0000000..750eaad
--- /dev/null
+++ b/apps/couch_replicator/src/couch_replicator.app.src
@@ -0,0 +1,33 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+{application, couch_replicator, [
+    {description, "CouchDB replicator"},
+    {vsn, "%version%"},
+    {modules, [  
+        couch_replicator_api_wrap,
+        couch_replicator_httpc,
+        couch_replicator_httpd,
+        couch_replicator_job_sup,
+        couch_replicator_notifier,
+        couch_replicator_manager,
+        couch_replicator_httpc_pool,
+        couch_replicator_utils,
+        couch_replicator_worker,
+        couch_replicator
+    ]},
+    {registered, [
+        couch_replicator_job_sup
+    ]},
+    {applications, [kernel, stdlib]}
+]}.
+


Mime
View raw message