Return-Path: X-Original-To: apmail-couchdb-commits-archive@www.apache.org Delivered-To: apmail-couchdb-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D0838175E4 for ; Fri, 31 Oct 2014 19:53:16 +0000 (UTC) Received: (qmail 2095 invoked by uid 500); 31 Oct 2014 19:53:16 -0000 Delivered-To: apmail-couchdb-commits-archive@couchdb.apache.org Received: (qmail 1925 invoked by uid 500); 31 Oct 2014 19:53:16 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 1660 invoked by uid 99); 31 Oct 2014 19:53:16 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 31 Oct 2014 19:53:16 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 54139927A7E; Fri, 31 Oct 2014 19:53:16 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: bbastian@apache.org To: commits@couchdb.apache.org Date: Fri, 31 Oct 2014 19:53:20 -0000 Message-Id: In-Reply-To: <695c7c23941a44a0938af77efbf99d8b@git.apache.org> References: <695c7c23941a44a0938af77efbf99d8b@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [05/41] couch-mrview commit: updated refs/heads/master to 28e51f3 couch_index: add background indexing facility This change add the possibility to trigger a view indexation in background. The indexation can only work in background if at least one process acquired it using the `couch_index_server:acquire_index/3` function. If all the process that acquired it are down or released it using `couch_index_server:release_indexer/3` then the background task is stopped. By default the background indexation will happen every 1s or when 200 docs has been saved in the database. These parameters can be changed using the options `threshold` and `refresh_interval` in the couch_index section. To use it with couch_mrview a new option {refresh, true} has been added to couch_mrview_changes:handle_changes Also the query parameter refresh=true is passsed in t the HTTP changes API. Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/5691328f Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/5691328f Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/5691328f Branch: refs/heads/master Commit: 5691328fbd219d70760ddc5d579d0874c9ff2272 Parents: dfe991f Author: benoitc Authored: Sat Feb 8 19:55:40 2014 +0100 Committer: Benjamin Bastian Committed: Thu Oct 30 13:38:33 2014 -0700 ---------------------------------------------------------------------- src/couch_mrview_changes.erl | 56 ++++++++++++++++++++++++++++----------- test/10-index-changes.t | 17 +++++++++++- 2 files changed, 56 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/5691328f/src/couch_mrview_changes.erl ---------------------------------------------------------------------- diff --git a/src/couch_mrview_changes.erl b/src/couch_mrview_changes.erl index a0e5281..c31624e 100644 --- a/src/couch_mrview_changes.erl +++ b/src/couch_mrview_changes.erl @@ -28,14 +28,16 @@ heartbeat, timeout_acc=0, notifier, - stream}). + stream, + refresh}). -type changes_stream() :: true | false | once. -type changes_options() :: [{stream, changes_stream()} | {since, integer()} | {view_options, list()} | {timeout, integer()} | - {heartbeat, true | integer()}]. + {heartbeat, true | integer()} | + {refresh, true | false}]. -export_type([changes_stream/0]). -export_type([changes_options/0]). @@ -47,6 +49,7 @@ handle_changes(DbName, DDocId, View, Fun, Acc, Options) -> Since = proplists:get_value(since, Options, 0), Stream = proplists:get_value(stream, Options, false), ViewOptions = proplists:get_value(view_options, Options, []), + Refresh = proplists:get_value(refresh, Options, false), State0 = #vst{dbname=DbName, ddoc=DDocId, @@ -56,20 +59,25 @@ handle_changes(DbName, DDocId, View, Fun, Acc, Options) -> callback=Fun, acc=Acc}, - case view_changes_since(State0) of - {ok, #vst{since=LastSeq, acc=Acc2}=State} -> - case Stream of - true -> - start_loop(State#vst{stream=true}, Options); - once when LastSeq =:= Since -> - start_loop(State#vst{stream=once}, Options); - _ -> - Fun(stop, {LastSeq, Acc2}) - end; - {stop, #vst{since=LastSeq, acc=Acc2}} -> - Fun(stop, {LastSeq, Acc2}); - Error -> - Error + maybe_acquire_indexer(Refresh, DbName, DDocId), + try + case view_changes_since(State0) of + {ok, #vst{since=LastSeq, acc=Acc2}=State} -> + case Stream of + true -> + start_loop(State#vst{stream=true}, Options); + once when LastSeq =:= Since -> + start_loop(State#vst{stream=once}, Options); + _ -> + Fun(stop, {LastSeq, Acc2}) + end; + {stop, #vst{since=LastSeq, acc=Acc2}} -> + Fun(stop, {LastSeq, Acc2}); + Error -> + Error + end + after + maybe_release_indexer(Refresh, DbName, DDocId) end. start_loop(#vst{dbname=DbName, ddoc=DDocId}=State, Options) -> @@ -169,3 +177,19 @@ index_update_notifier(DbName, DDocId) -> ok end), NotifierPid. + +%% acquire the background indexing task so it can eventually be started +%% if the process close the background task will be automatically +%% released. +maybe_acquire_indexer(false, _, _) -> + ok; +maybe_acquire_indexer(true, DbName, DDocId) -> + couch_index_server:acquire_indexer(couch_mrview_index, DbName, + DDocId). + +%% release the background indexing task so it can eventually be stopped +maybe_release_indexer(false, _, _) -> + ok; +maybe_release_indexer(true, DbName, DDocId) -> + couch_index_server:release_indexer(couch_mrview_index, DbName, + DDocId). http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/5691328f/test/10-index-changes.t ---------------------------------------------------------------------- diff --git a/test/10-index-changes.t b/test/10-index-changes.t index 627376f..f53e9ed 100644 --- a/test/10-index-changes.t +++ b/test/10-index-changes.t @@ -15,7 +15,7 @@ % the License. main(_) -> - etap:plan(6), + etap:plan(8), case (catch test()) of ok -> etap:end_tests(); @@ -35,6 +35,7 @@ test() -> test_stream_once_timeout(Db), test_stream_once_heartbeat(Db), test_stream(Db), + test_indexer(Db), test_util:stop_couch(), ok. @@ -173,6 +174,20 @@ test_stream(Db) -> end. +test_indexer(Db) -> + Result = run_query(Db, [{since, 14}]), + Expect = {ok, 15, [{{15,14,<<"14">>},14}]}, + etap:is(Result, Expect, "refresh index by hand OK."), + + {ok, Db1} = save_doc(Db, 15), + timer:sleep(1000), + Result1 = run_query(Db, [{since, 14}]), + Expect1 = {ok, 16, [{{15,14,<<"14">>},14}, + {{16,15,<<"15">>},15}]}, + etap:is(Result1, Expect1, "changes indexed in background OK."), + ok. + + save_doc(Db, Id) -> Doc = couch_mrview_test_util:doc(Id), {ok, _Rev} = couch_db:update_doc(Db, Doc, []),