couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fdman...@apache.org
Subject svn commit: r1159922 - in /couchdb/trunk: bin/ etc/couchdb/ src/couchdb/ test/etap/
Date Sat, 20 Aug 2011 23:00:25 GMT
Author: fdmanana
Date: Sat Aug 20 23:00:24 2011
New Revision: 1159922

URL: http://svn.apache.org/viewvc?rev=1159922&view=rev
Log:
Add configurable automatic compaction

A set of rules can now be defined in order to automatically
trigger the compaction of databases and their views. This
configuration can be global or overrided for specific
databases. By default it's disabled. Enabling it can be
done via the .ini configuration by simply adding global
or database specific compaction rules.

Closes COUCHDB-1153.
Thanks everyone involved who gave feedback and suggestions
for improvements.


Added:
    couchdb/trunk/src/couchdb/couch_compaction_daemon.erl
    couchdb/trunk/test/etap/220-compaction-daemon.t   (with props)
Modified:
    couchdb/trunk/bin/couchdb.tpl.in
    couchdb/trunk/etc/couchdb/default.ini.tpl.in
    couchdb/trunk/src/couchdb/Makefile.am
    couchdb/trunk/src/couchdb/couch.app.tpl.in
    couchdb/trunk/src/couchdb/couch_app.erl
    couchdb/trunk/src/couchdb/couch_db.erl
    couchdb/trunk/src/couchdb/couch_db_updater.erl
    couchdb/trunk/src/couchdb/couch_httpd_db.erl
    couchdb/trunk/src/couchdb/couch_server.erl
    couchdb/trunk/src/couchdb/couch_view.erl
    couchdb/trunk/src/couchdb/couch_view_compactor.erl
    couchdb/trunk/src/couchdb/couch_view_group.erl
    couchdb/trunk/test/etap/200-view-group-no-db-leaks.t
    couchdb/trunk/test/etap/Makefile.am

Modified: couchdb/trunk/bin/couchdb.tpl.in
URL: http://svn.apache.org/viewvc/couchdb/trunk/bin/couchdb.tpl.in?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/bin/couchdb.tpl.in (original)
+++ couchdb/trunk/bin/couchdb.tpl.in Sat Aug 20 23:00:24 2011
@@ -15,7 +15,12 @@
 BACKGROUND=false
 DEFAULT_CONFIG_DIR=%localconfdir%/default.d
 DEFAULT_CONFIG_FILE=%localconfdir%/%defaultini%
-ERL_START_OPTIONS="-sasl errlog_type error +K true +A 4"
+ERL_OS_MON_OPTIONS="-os_mon \
+    start_memsup false \
+    start_cpu_sup false \
+    disk_space_check_interval 1 \
+    disk_almost_full_threshold 1"
+ERL_START_OPTIONS="$ERL_OS_MON_OPTIONS -sasl errlog_type error +K true +A 4"
 HEART_BEAT_TIMEOUT=11
 HEART_COMMAND="%bindir%/%couchdb_command_name% -k"
 INTERACTIVE=false

Modified: couchdb/trunk/etc/couchdb/default.ini.tpl.in
URL: http://svn.apache.org/viewvc/couchdb/trunk/etc/couchdb/default.ini.tpl.in?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/etc/couchdb/default.ini.tpl.in (original)
+++ couchdb/trunk/etc/couchdb/default.ini.tpl.in Sat Aug 20 23:00:24 2011
@@ -82,6 +82,7 @@ uuids={couch_uuids, start, []}
 auth_cache={couch_auth_cache, start_link, []}
 replication_manager={couch_replication_manager, start_link, []}
 os_daemons={couch_os_daemons, start_link, []}
+compaction_daemon={couch_compaction_daemon, start_link, []}
 
 [httpd_global_handlers]
 / = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome">>}
@@ -190,3 +191,91 @@ verify_ssl_certificates = false
 ; ssl_trusted_certificates_file = /etc/ssl/certs/ca-certificates.crt
 ; maximum peer certificate depth (must be set even if certificate validation is off)
 ssl_certificate_max_depth = 3
+
+[compaction_daemon]
+; The delay, in seconds, between each check for which database and view indexes
+; need to be compacted.
+check_interval = 300
+; If a database or view index file is smaller then this value (in bytes),
+; compaction will not happen. Very small files always have a very high
+; fragmentation therefore it's not worth to compact them.
+min_file_size = 131072
+
+[compactions]
+; List of compaction rules for the compaction daemon.
+; The daemon compacts databases and their respective view groups when all the
+; condition parameters are satisfied. Configuration can be per database or
+; global, and it has the following format:
+;
+; database_name = [ {ParamName, ParamValue}, {ParamName, ParamValue}, ... ]
+; _default = [ {ParamName, ParamValue}, {ParamName, ParamValue}, ... ]
+;
+; Possible parameters:
+;
+; * db_fragmentation - If the ratio (as an integer percentage), of the amount
+;                      of old data (and its supporting metadata) over the database
+;                      file size is equal to or greater then this value, this
+;                      database compaction condition is satisfied.
+;                      This value is computed as:
+;
+;                           (file_size - data_size) / file_size * 100
+;
+;                      The data_size and file_size values can be obtained when
+;                      querying a database's information URI (GET /dbname/).
+;
+; * view_fragmentation - If the ratio (as an integer percentage), of the amount
+;                        of old data (and its supporting metadata) over the view
+;                        index (view group) file size is equal to or greater then
+;                        this value, then this view index compaction condition is
+;                        satisfied. This value is computed as:
+;
+;                            (file_size - data_size) / file_size * 100
+;
+;                        The data_size and file_size values can be obtained when
+;                        querying a view group's information URI
+;                        (GET /dbname/_design/groupname/_info).
+;
+; * from _and_ to - The period for which a database (and its view groups) compaction
+;                   is allowed. The value for these parameters must obey the format:
+;
+;                   HH:MM - HH:MM  (HH in [0..23], MM in [0..59])
+;
+; * strict_window - If a compaction is still running after the end of the allowed
+;                   period, it will be canceled if this parameter is set to 'true'.
+;                   It defaults to 'false' and it's meaningful only if the *period*
+;                   parameter is also specified.
+;
+; * parallel_view_compaction - If set to 'true', the database and its views are
+;                              compacted in parallel. This is only useful on
+;                              certain setups, like for example when the database
+;                              and view index directories point to different
+;                              disks. It defaults to 'false'.
+;
+; Before a compaction is triggered, an estimation of how much free disk space is
+; needed is computed. This estimation corresponds to 2 times the data size of
+; the database or view index. When there's not enough free disk space to compact
+; a particular database or view index, a warning message is logged.
+;
+; Examples:
+;
+; 1) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}]
+;    The `foo` database is compacted if its fragmentation is 70% or more.
+;    Any view index of this database is compacted only if its fragmentation
+;    is 60% or more.
+;
+; 2) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}]
+;    Similar to the preceding example but a compaction (database or view index)
+;    is only triggered if the current time is between midnight and 4 AM.
+;
+; 3) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}, {strict_window, true}]
+;    Similar to the preceding example - a compaction (database or view index)
+;    is only triggered if the current time is between midnight and 4 AM. If at
+;    4 AM the database or one of its views is still compacting, the compaction
+;    process will be canceled.
+;
+; 4) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}, {strict_window, true}, {parallel_view_compaction, true}]
+;    Similar to the preceding example, but a database and its views can be
+;    compacted in parallel.
+;
+;_default = [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "23:00"}, {to, "04:00"}]
+

Modified: couchdb/trunk/src/couchdb/Makefile.am
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/Makefile.am?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/Makefile.am (original)
+++ couchdb/trunk/src/couchdb/Makefile.am Sat Aug 20 23:00:24 2011
@@ -34,6 +34,7 @@ source_files = \
     couch_auth_cache.erl \
     couch_btree.erl \
     couch_changes.erl \
+    couch_compaction_daemon.erl \
     couch_compress.erl \
     couch_config.erl \
     couch_config_writer.erl \
@@ -103,6 +104,7 @@ compiled_files = \
     couch_auth_cache.beam \
     couch_btree.beam \
     couch_changes.beam \
+    couch_compaction_daemon.beam \
     couch_compress.beam \
     couch_config.beam \
     couch_config_writer.beam \

Modified: couchdb/trunk/src/couchdb/couch.app.tpl.in
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch.app.tpl.in?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch.app.tpl.in (original)
+++ couchdb/trunk/src/couchdb/couch.app.tpl.in Sat Aug 20 23:00:24 2011
@@ -25,5 +25,5 @@
         "%localconfdir%/@localini@"
     ]}},
     {applications, [kernel, stdlib]},
-    {included_applications, [crypto, sasl, inets, oauth, ibrowse, mochiweb]}
+    {included_applications, [crypto, sasl, inets, oauth, ibrowse, mochiweb, os_mon]}
 ]}.

Modified: couchdb/trunk/src/couchdb/couch_app.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_app.erl?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_app.erl (original)
+++ couchdb/trunk/src/couchdb/couch_app.erl Sat Aug 20 23:00:24 2011
@@ -20,7 +20,7 @@
 
 start(_Type, DefaultIniFiles) ->
     IniFiles = get_ini_files(DefaultIniFiles),
-    case start_apps([crypto, public_key, sasl, inets, oauth, ssl, ibrowse, mochiweb]) of
+    case start_apps([crypto, public_key, sasl, inets, oauth, ssl, ibrowse, mochiweb, os_mon]) of
     ok ->
         couch_server_sup:start_link(IniFiles);
     {error, Reason} ->

Added: couchdb/trunk/src/couchdb/couch_compaction_daemon.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_compaction_daemon.erl?rev=1159922&view=auto
==============================================================================
--- couchdb/trunk/src/couchdb/couch_compaction_daemon.erl (added)
+++ couchdb/trunk/src/couchdb/couch_compaction_daemon.erl Sat Aug 20 23:00:24 2011
@@ -0,0 +1,487 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_compaction_daemon).
+-behaviour(gen_server).
+
+% public API
+-export([start_link/0, config_change/3]).
+
+% gen_server callbacks
+-export([init/1, handle_call/3, handle_info/2, handle_cast/2]).
+-export([code_change/3, terminate/2]).
+
+-include("couch_db.hrl").
+
+-define(CONFIG_ETS, couch_compaction_daemon_config).
+
+-record(state, {
+    loop_pid
+}).
+
+-record(config, {
+    db_frag = nil,
+    view_frag = nil,
+    period = nil,
+    cancel = false,
+    parallel_view_compact = false
+}).
+
+-record(period, {
+    from = nil,
+    to = nil
+}).
+
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+
+init(_) ->
+    process_flag(trap_exit, true),
+    ?CONFIG_ETS = ets:new(?CONFIG_ETS, [named_table, set, protected]),
+    ok = couch_config:register(fun ?MODULE:config_change/3),
+    load_config(),
+    Server = self(),
+    Loop = spawn_link(fun() -> compact_loop(Server) end),
+    {ok, #state{loop_pid = Loop}}.
+
+
+config_change("compactions", DbName, NewValue) ->
+    ok = gen_server:cast(?MODULE, {config_update, DbName, NewValue}).
+
+
+handle_cast({config_update, DbName, deleted}, State) ->
+    true = ets:delete(?CONFIG_ETS, ?l2b(DbName)),
+    {noreply, State};
+
+handle_cast({config_update, DbName, Config}, #state{loop_pid = Loop} = State) ->
+    case parse_config(DbName, Config) of
+    {ok, NewConfig} ->
+        WasEmpty = (ets:info(?CONFIG_ETS, size) =:= 0),
+        true = ets:insert(?CONFIG_ETS, {?l2b(DbName), NewConfig}),
+        case WasEmpty of
+        true ->
+            Loop ! {self(), have_config};
+        false ->
+            ok
+        end;
+    error ->
+        ok
+    end,
+    {noreply, State}.
+
+
+handle_call(Msg, _From, State) ->
+    {stop, {unexpected_call, Msg}, State}.
+
+
+handle_info({'EXIT', Pid, Reason}, #state{loop_pid = Pid} = State) ->
+    {stop, {compaction_loop_died, Reason}, State}.
+
+
+terminate(_Reason, _State) ->
+    true = ets:delete(?CONFIG_ETS).
+
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+
+compact_loop(Parent) ->
+    {ok, _} = couch_server:all_databases(
+        fun(DbName, Acc) ->
+            case ets:info(?CONFIG_ETS, size) =:= 0 of
+            true ->
+                {stop, Acc};
+            false ->
+                case get_db_config(DbName) of
+                nil ->
+                    ok;
+                {ok, Config} ->
+                    maybe_compact_db(DbName, Config)
+                end,
+                {ok, Acc}
+            end
+        end, ok),
+    case ets:info(?CONFIG_ETS, size) =:= 0 of
+    true ->
+        receive {Parent, have_config} -> ok end;
+    false ->
+        PausePeriod = list_to_integer(
+            couch_config:get("compaction_daemon", "check_interval", "300")),
+        ok = timer:sleep(PausePeriod * 1000)
+    end,
+    compact_loop(Parent).
+
+
+maybe_compact_db(DbName, Config) ->
+    case (catch couch_db:open_int(DbName, [])) of
+    {ok, Db} ->
+        DDocNames = db_ddoc_names(Db),
+        case can_db_compact(Config, Db) of
+        true ->
+            {ok, DbCompactPid} = couch_db:start_compact(Db),
+            TimeLeft = compact_time_left(Config),
+            case Config#config.parallel_view_compact of
+            true ->
+                ViewsCompactPid = spawn_link(fun() ->
+                    maybe_compact_views(DbName, DDocNames, Config)
+                end),
+                ViewsMonRef = erlang:monitor(process, ViewsCompactPid);
+            false ->
+                ViewsMonRef = nil
+            end,
+            DbMonRef = erlang:monitor(process, DbCompactPid),
+            receive
+            {'DOWN', DbMonRef, process, _, normal} ->
+                couch_db:close(Db),
+                case Config#config.parallel_view_compact of
+                true ->
+                    ok;
+                false ->
+                    maybe_compact_views(DbName, DDocNames, Config)
+                end;
+            {'DOWN', DbMonRef, process, _, Reason} ->
+                couch_db:close(Db),
+                ?LOG_ERROR("Compaction daemon - an error ocurred while"
+                    " compacting the database `~s`: ~p", [DbName, Reason])
+            after TimeLeft ->
+                ?LOG_INFO("Compaction daemon - canceling compaction for database"
+                    " `~s` because it's exceeding the allowed period.",
+                    [DbName]),
+                erlang:demonitor(DbMonRef, [flush]),
+                ok = couch_db:cancel_compact(Db),
+                couch_db:close(Db)
+            end,
+            case ViewsMonRef of
+            nil ->
+                ok;
+            _ ->
+                receive
+                {'DOWN', ViewsMonRef, process, _, _Reason} ->
+                    ok
+                end
+            end;
+        false ->
+            maybe_compact_views(DbName, DDocNames, Config)
+        end;
+    _ ->
+        ok
+    end.
+
+
+maybe_compact_views(_DbName, [], _Config) ->
+    ok;
+maybe_compact_views(DbName, [DDocName | Rest], Config) ->
+    case maybe_compact_view(DbName, DDocName, Config) of
+    ok ->
+        maybe_compact_views(DbName, Rest, Config);
+    timeout ->
+        ok
+    end.
+
+
+db_ddoc_names(Db) ->
+    {ok, _, DDocNames} = couch_db:enum_docs(
+        Db,
+        fun(#full_doc_info{id = <<"_design/", _/binary>>, deleted = true}, _, Acc) ->
+            {ok, Acc};
+        (#full_doc_info{id = <<"_design/", Id/binary>>}, _, Acc) ->
+            {ok, [Id | Acc]};
+        (_, _, Acc) ->
+            {stop, Acc}
+        end, [], [{start_key, <<"_design/">>}, {end_key_gt, <<"_design0">>}]),
+    DDocNames.
+
+
+maybe_compact_view(DbName, GroupId, Config) ->
+    DDocId = <<"_design/", GroupId/binary>>,
+    case (catch couch_view:get_group_info(DbName, DDocId)) of
+    {ok, GroupInfo} ->
+        case can_view_compact(Config, DbName, GroupId, GroupInfo) of
+        true ->
+            {ok, CompactPid} = couch_view_compactor:start_compact(DbName, GroupId),
+            TimeLeft = compact_time_left(Config),
+            MonRef = erlang:monitor(process, CompactPid),
+            receive
+            {'DOWN', MonRef, process, CompactPid, normal} ->
+                ok;
+            {'DOWN', MonRef, process, CompactPid, Reason} ->
+                ?LOG_ERROR("Compaction daemon - an error ocurred while compacting"
+                    " the view group `~s` from database `~s`: ~p",
+                    [GroupId, DbName, Reason]),
+                ok
+            after TimeLeft ->
+                ?LOG_INFO("Compaction daemon - canceling the compaction for the "
+                    "view group `~s` of the database `~s` because it's exceeding"
+                    " the allowed period.", [GroupId, DbName]),
+                erlang:demonitor(MonRef, [flush]),
+                ok = couch_view_compactor:cancel_compact(DbName, GroupId),
+                timeout
+            end;
+        false ->
+            ok
+        end;
+    Error ->
+        ?LOG_ERROR("Error opening view group `~s` from database `~s`: ~p",
+            [GroupId, DbName, Error]),
+        ok
+    end.
+
+
+compact_time_left(#config{cancel = false}) ->
+    infinity;
+compact_time_left(#config{period = nil}) ->
+    infinity;
+compact_time_left(#config{period = #period{to = {ToH, ToM} = To}}) ->
+    {H, M, _} = time(),
+    case To > {H, M} of
+    true ->
+        ((ToH - H) * 60 * 60 * 1000) + (abs(ToM - M) * 60 * 1000);
+    false ->
+        ((24 - H + ToH) * 60 * 60 * 1000) + (abs(ToM - M) * 60 * 1000)
+    end.
+
+
+get_db_config(DbName) ->
+    case ets:lookup(?CONFIG_ETS, DbName) of
+    [] ->
+        case ets:lookup(?CONFIG_ETS, <<"_default">>) of
+        [] ->
+            nil;
+        [{<<"_default">>, Config}] ->
+            {ok, Config}
+        end;
+    [{DbName, Config}] ->
+        {ok, Config}
+    end.
+
+
+can_db_compact(#config{db_frag = Threshold} = Config, Db) ->
+    case check_period(Config) of
+    false ->
+        false;
+    true ->
+        {ok, DbInfo} = couch_db:get_db_info(Db),
+        {Frag, SpaceRequired} = frag(DbInfo),
+        ?LOG_DEBUG("Fragmentation for database `~s` is ~p%, estimated space for"
+           " compaction is ~p bytes.", [Db#db.name, Frag, SpaceRequired]),
+        case check_frag(Threshold, Frag) of
+        false ->
+            false;
+        true ->
+            Free = free_space(couch_config:get("couchdb", "database_dir")),
+            case Free >= SpaceRequired of
+            true ->
+                true;
+            false ->
+                ?LOG_INFO("Compaction daemon - skipping database `~s` "
+                    "compaction: the estimated necessary disk space is about ~p"
+                    " bytes but the currently available disk space is ~p bytes.",
+                   [Db#db.name, SpaceRequired, Free]),
+                false
+            end
+        end
+    end.
+
+can_view_compact(Config, DbName, GroupId, GroupInfo) ->
+    case check_period(Config) of
+    false ->
+        false;
+    true ->
+        case couch_util:get_value(updater_running, GroupInfo) of
+        true ->
+            false;
+        false ->
+            {Frag, SpaceRequired} = frag(GroupInfo),
+            ?LOG_DEBUG("Fragmentation for view group `~s` (database `~s`) is "
+                "~p%, estimated space for compaction is ~p bytes.",
+                [GroupId, DbName, Frag, SpaceRequired]),
+            case check_frag(Config#config.view_frag, Frag) of
+            false ->
+                false;
+            true ->
+                Free = free_space(couch_config:get("couchdb", "view_index_dir")),
+                case Free >= SpaceRequired of
+                true ->
+                    true;
+                false ->
+                    ?LOG_INFO("Compaction daemon - skipping view group `~s` "
+                        "compaction (database `~s`): the estimated necessary "
+                        "disk space is about ~p bytes but the currently available"
+                        " disk space is ~p bytes.",
+                        [GroupId, DbName, SpaceRequired, Free]),
+                    false
+                end
+            end
+        end
+    end.
+
+
+check_period(#config{period = nil}) ->
+    true;
+check_period(#config{period = #period{from = From, to = To}}) ->
+    {HH, MM, _} = erlang:time(),
+    case From < To of
+    true ->
+        ({HH, MM} >= From) andalso ({HH, MM} < To);
+    false ->
+        ({HH, MM} >= From) orelse ({HH, MM} < To)
+    end.
+
+
+check_frag(nil, _) ->
+    true;
+check_frag(Threshold, Frag) ->
+    Frag >= Threshold.
+
+
+frag(Props) ->
+    FileSize = couch_util:get_value(disk_size, Props),
+    MinFileSize = list_to_integer(
+        couch_config:get("compaction_daemon", "min_file_size", "131072")),
+    case FileSize < MinFileSize of
+    true ->
+        {0, FileSize};
+    false ->
+        case couch_util:get_value(data_size, Props) of
+        null ->
+            {100, FileSize};
+        0 ->
+            {0, FileSize};
+        DataSize ->
+            Frag = round(((FileSize - DataSize) / FileSize * 100)),
+            {Frag, space_required(DataSize)}
+        end
+    end.
+
+% Rough, and pessimistic, estimation of necessary disk space to compact a
+% database or view index.
+space_required(DataSize) ->
+    round(DataSize * 2.0).
+
+
+load_config() ->
+    lists:foreach(
+        fun({DbName, ConfigString}) ->
+            case parse_config(DbName, ConfigString) of
+            {ok, Config} ->
+                true = ets:insert(?CONFIG_ETS, {?l2b(DbName), Config});
+            error ->
+                ok
+            end
+        end,
+        couch_config:get("compactions")).
+
+parse_config(DbName, ConfigString) ->
+    case (catch do_parse_config(ConfigString)) of
+    {ok, Conf} ->
+        {ok, Conf};
+    incomplete_period ->
+        ?LOG_ERROR("Incomplete period ('to' or 'from' missing) in the compaction"
+            " configuration for database `~s`", [DbName]),
+        error;
+    _ ->
+        ?LOG_ERROR("Invalid compaction configuration for database "
+            "`~s`: `~s`", [DbName, ConfigString]),
+        error
+    end.
+
+do_parse_config(ConfigString) ->
+    {ok, ConfProps} = couch_util:parse_term(ConfigString),
+    {ok, #config{period = Period} = Conf} = config_record(ConfProps, #config{}),
+    case Period of
+    nil ->
+        {ok, Conf};
+    #period{from = From, to = To} when From =/= nil, To =/= nil ->
+        {ok, Conf};
+    #period{} ->
+        incomplete_period
+    end.
+
+config_record([], Config) ->
+    {ok, Config};
+
+config_record([{db_fragmentation, V} | Rest], Config) ->
+    [Frag] = string:tokens(V, "%"),
+    config_record(Rest, Config#config{db_frag = list_to_integer(Frag)});
+
+config_record([{view_fragmentation, V} | Rest], Config) ->
+    [Frag] = string:tokens(V, "%"),
+    config_record(Rest, Config#config{view_frag = list_to_integer(Frag)});
+
+config_record([{from, V} | Rest], #config{period = Period0} = Config) ->
+    Time = parse_time(V),
+    Period = case Period0 of
+    nil ->
+        #period{from = Time};
+    #period{} ->
+        Period0#period{from = Time}
+    end,
+    config_record(Rest, Config#config{period = Period});
+
+config_record([{to, V} | Rest], #config{period = Period0} = Config) ->
+    Time = parse_time(V),
+    Period = case Period0 of
+    nil ->
+        #period{to = Time};
+    #period{} ->
+        Period0#period{to = Time}
+    end,
+    config_record(Rest, Config#config{period = Period});
+
+config_record([{strict_window, true} | Rest], Config) ->
+    config_record(Rest, Config#config{cancel = true});
+
+config_record([{strict_window, false} | Rest], Config) ->
+    config_record(Rest, Config#config{cancel = false});
+
+config_record([{parallel_view_compaction, true} | Rest], Config) ->
+    config_record(Rest, Config#config{parallel_view_compact = true});
+
+config_record([{parallel_view_compaction, false} | Rest], Config) ->
+    config_record(Rest, Config#config{parallel_view_compact = false}).
+
+
+parse_time(String) ->
+    [HH, MM] = string:tokens(String, ":"),
+    {list_to_integer(HH), list_to_integer(MM)}.
+
+
+free_space(Path) ->
+    DiskData = lists:sort(
+        fun({PathA, _, _}, {PathB, _, _}) ->
+            length(filename:split(PathA)) > length(filename:split(PathB))
+        end,
+        disksup:get_disk_data()),
+    free_space_rec(abs_path(Path), DiskData).
+
+free_space_rec(_Path, []) ->
+    undefined;
+free_space_rec(Path, [{MountPoint0, Total, Usage} | Rest]) ->
+    MountPoint = abs_path(MountPoint0),
+    case MountPoint =:= string:substr(Path, 1, length(MountPoint)) of
+    false ->
+        free_space_rec(Path, Rest);
+    true ->
+        trunc(Total - (Total * (Usage / 100))) * 1024
+    end.
+
+abs_path(Path0) ->
+    Path = filename:absname(Path0),
+    case lists:last(Path) of
+    $/ ->
+        Path;
+    _ ->
+        Path ++ "/"
+    end.

Modified: couchdb/trunk/src/couchdb/couch_db.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.erl?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db.erl (original)
+++ couchdb/trunk/src/couchdb/couch_db.erl Sat Aug 20 23:00:24 2011
@@ -13,7 +13,8 @@
 -module(couch_db).
 -behaviour(gen_server).
 
--export([open/2,open_int/2,close/1,create/2,start_compact/1,get_db_info/1,get_design_docs/1]).
+-export([open/2,open_int/2,close/1,create/2,get_db_info/1,get_design_docs/1]).
+-export([start_compact/1, cancel_compact/1]).
 -export([open_ref_counted/2,is_idle/1,monitor/1,count_changes_since/2]).
 -export([update_doc/3,update_doc/4,update_docs/4,update_docs/2,update_docs/3,delete_doc/3]).
 -export([get_doc_info/2,open_doc/2,open_doc/3,open_doc_revs/4]).
@@ -118,6 +119,9 @@ monitor(#db{main_pid=MainPid}) ->
 start_compact(#db{update_pid=Pid}) ->
     gen_server:call(Pid, start_compact).
 
+cancel_compact(#db{update_pid=Pid}) ->
+    gen_server:call(Pid, cancel_compact).
+
 delete_doc(Db, Id, Revisions) ->
     DeletedDocs = [#doc{id=Id, revs=[Rev], deleted=true} || Rev <- Revisions],
     {ok, [Result]} = update_docs(Db, DeletedDocs, []),

Modified: couchdb/trunk/src/couchdb/couch_db_updater.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db_updater.erl?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db_updater.erl (original)
+++ couchdb/trunk/src/couchdb/couch_db_updater.erl Sat Aug 20 23:00:24 2011
@@ -158,15 +158,22 @@ handle_call(start_compact, _From, Db) ->
         Pid = spawn_link(fun() -> start_copy_compact(Db) end),
         Db2 = Db#db{compactor_pid=Pid},
         ok = gen_server:call(Db#db.main_pid, {db_updated, Db2}),
-        {reply, ok, Db2};
+        {reply, {ok, Pid}, Db2};
     _ ->
         % compact currently running, this is a no-op
-        {reply, ok, Db}
-    end.
-
+        {reply, {ok, Db#db.compactor_pid}, Db}
+    end;
+handle_call(cancel_compact, _From, #db{compactor_pid = nil} = Db) ->
+    {reply, ok, Db};
+handle_call(cancel_compact, _From, #db{compactor_pid = Pid} = Db) ->
+    unlink(Pid),
+    exit(Pid, kill),
+    RootDir = couch_config:get("couchdb", "database_dir", "."),
+    ok = couch_file:delete(RootDir, Db#db.filepath ++ ".compact"),
+    {reply, ok, Db#db{compactor_pid = nil}};
 
 
-handle_cast({compact_done, CompactFilepath}, #db{filepath=Filepath}=Db) ->
+handle_call({compact_done, CompactFilepath}, _From, #db{filepath=Filepath}=Db) ->
     {ok, NewFd} = couch_file:open(CompactFilepath),
     ReaderFd = open_reader_fd(CompactFilepath, Db#db.options),
     {ok, NewHeader} = couch_file:read_header(NewFd),
@@ -198,18 +205,21 @@ handle_cast({compact_done, CompactFilepa
         ok = gen_server:call(Db#db.main_pid, {db_updated, NewDb3}, infinity),
         couch_db_update_notifier:notify({compacted, NewDb3#db.name}),
         ?LOG_INFO("Compaction for db \"~s\" completed.", [Db#db.name]),
-        {noreply, NewDb3#db{compactor_pid=nil}};
+        {reply, ok, NewDb3#db{compactor_pid=nil}};
     false ->
         ?LOG_INFO("Compaction file still behind main file "
             "(update seq=~p. compact update seq=~p). Retrying.",
             [Db#db.update_seq, NewSeq]),
         close_db(NewDb),
-        Pid = spawn_link(fun() -> start_copy_compact(Db) end),
-        Db2 = Db#db{compactor_pid=Pid},
-        {noreply, Db2}
+        {reply, {retry, Db}, Db}
     end.
 
 
+handle_cast(Msg, #db{name = Name} = Db) ->
+    ?LOG_ERROR("Database `~s` updater received unexpected cast: ~p", [Name, Msg]),
+    {stop, Msg, Db}.
+
+
 handle_info({update_docs, Client, GroupedDocs, NonRepDocs, MergeConflicts,
         FullCommit}, Db) ->
     GroupedDocs2 = [[{Client, D} || D <- DocGroup] || DocGroup <- GroupedDocs],
@@ -966,7 +976,13 @@ start_copy_compact(#db{name=Name,filepat
 
     NewDb3 = copy_compact(Db, NewDb2, Retry),
     close_db(NewDb3),
-    gen_server:cast(Db#db.update_pid, {compact_done, CompactFile}).
+    case gen_server:call(
+        Db#db.update_pid, {compact_done, CompactFile}, infinity) of
+    ok ->
+        ok;
+    {retry, CurrentDb} ->
+        start_copy_compact(CurrentDb)
+    end.
 
 make_doc_summary(#db{compression = Comp}, {Body0, Atts0}) ->
     Body = case couch_compress:is_compressed(Body0) of

Modified: couchdb/trunk/src/couchdb/couch_httpd_db.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_db.erl?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_httpd_db.erl (original)
+++ couchdb/trunk/src/couchdb/couch_httpd_db.erl Sat Aug 20 23:00:24 2011
@@ -128,13 +128,13 @@ handle_changes_req1(Req, Db) ->
 handle_compact_req(#httpd{method='POST',path_parts=[DbName,_,Id|_]}=Req, Db) ->
     ok = couch_db:check_is_admin(Db),
     couch_httpd:validate_ctype(Req, "application/json"),
-    ok = couch_view_compactor:start_compact(DbName, Id),
+    {ok, _} = couch_view_compactor:start_compact(DbName, Id),
     send_json(Req, 202, {[{ok, true}]});
 
 handle_compact_req(#httpd{method='POST'}=Req, Db) ->
     ok = couch_db:check_is_admin(Db),
     couch_httpd:validate_ctype(Req, "application/json"),
-    ok = couch_db:start_compact(Db),
+    {ok, _} = couch_db:start_compact(Db),
     send_json(Req, 202, {[{ok, true}]});
 
 handle_compact_req(Req, _Db) ->

Modified: couchdb/trunk/src/couchdb/couch_server.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_server.erl?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_server.erl (original)
+++ couchdb/trunk/src/couchdb/couch_server.erl Sat Aug 20 23:00:24 2011
@@ -13,7 +13,8 @@
 -module(couch_server).
 -behaviour(gen_server).
 
--export([open/2,create/2,delete/2,all_databases/0,get_version/0]).
+-export([open/2,create/2,delete/2,get_version/0]).
+-export([all_databases/0, all_databases/2]).
 -export([init/1, handle_call/3,sup_start_link/0]).
 -export([handle_cast/2,code_change/3,handle_info/2,terminate/2]).
 -export([dev_start/0,is_admin/2,has_admins/0,get_stats/0]).
@@ -158,19 +159,30 @@ terminate(_Reason, _Srv) ->
         ets:tab2list(couch_dbs_by_name)).
 
 all_databases() ->
+    {ok, DbList} = all_databases(
+        fun(DbName, Acc) -> {ok, [DbName | Acc]} end, []),
+    {ok, lists:usort(DbList)}.
+
+all_databases(Fun, Acc0) ->
     {ok, #server{root_dir=Root}} = gen_server:call(couch_server, get_server),
     NormRoot = couch_util:normpath(Root),
-    Filenames =
-    filelib:fold_files(Root, "^[a-z0-9\\_\\$()\\+\\-]*[\\.]couch$", true,
-        fun(Filename, AccIn) ->
-            NormFilename = couch_util:normpath(Filename),
-            case NormFilename -- NormRoot of
-            [$/ | RelativeFilename] -> ok;
-            RelativeFilename -> ok
-            end,
-            [list_to_binary(filename:rootname(RelativeFilename, ".couch")) | AccIn]
-        end, []),
-    {ok, lists:usort(Filenames)}.
+    FinalAcc = try
+        filelib:fold_files(Root, "^[a-z0-9\\_\\$()\\+\\-]*[\\.]couch$", true,
+            fun(Filename, AccIn) ->
+                NormFilename = couch_util:normpath(Filename),
+                case NormFilename -- NormRoot of
+                [$/ | RelativeFilename] -> ok;
+                RelativeFilename -> ok
+                end,
+                case Fun(?l2b(filename:rootname(RelativeFilename, ".couch")), AccIn) of
+                {ok, NewAcc} -> NewAcc;
+                {stop, NewAcc} -> throw({stop, Fun, NewAcc})
+                end
+            end, Acc0)
+    catch throw:{stop, Fun, Acc1} ->
+         Acc1
+    end,
+    {ok, FinalAcc}.
 
 
 maybe_close_lru_db(#server{dbs_open=NumOpen, max_dbs_open=MaxOpen}=Server)

Modified: couchdb/trunk/src/couchdb/couch_view.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view.erl?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_view.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view.erl Sat Aug 20 23:00:24 2011
@@ -76,9 +76,10 @@ get_temp_group(Db, Language, DesignOptio
         get_temp_updater(couch_db:name(Db), Language, DesignOptions, MapSrc, RedSrc),
         couch_db:get_update_seq(Db)).
 
-get_group_info(Db, GroupId) ->
-    couch_view_group:request_group_info(
-        get_group_server(couch_db:name(Db), GroupId)).
+get_group_info(#db{name = DbName}, GroupId) ->
+    get_group_info(DbName, GroupId);
+get_group_info(DbName, GroupId) ->
+    couch_view_group:request_group_info(get_group_server(DbName, GroupId)).
 
 cleanup_index_files(Db) ->
     % load all ddocs

Modified: couchdb/trunk/src/couchdb/couch_view_compactor.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_compactor.erl?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_view_compactor.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view_compactor.erl Sat Aug 20 23:00:24 2011
@@ -14,13 +14,17 @@
 
 -include ("couch_db.hrl").
 
--export([start_compact/2]).
+-export([start_compact/2, cancel_compact/2]).
 
 %% @spec start_compact(DbName::binary(), GroupId:binary()) -> ok
 %% @doc Compacts the views.  GroupId must not include the _design/ prefix
 start_compact(DbName, GroupId) ->
     Pid = couch_view:get_group_server(DbName, <<"_design/",GroupId/binary>>),
-    gen_server:cast(Pid, {start_compact, fun compact_group/3}).
+    gen_server:call(Pid, {start_compact, fun compact_group/3}).
+
+cancel_compact(DbName, GroupId) ->
+    Pid = couch_view:get_group_server(DbName, <<"_design/", GroupId/binary>>),
+    gen_server:call(Pid, cancel_compact).
 
 %%=============================================================================
 %% internal functions
@@ -82,9 +86,22 @@ compact_group(Group, EmptyGroup, DbName)
         views=NewViews,
         current_seq=Seq
     },
+    maybe_retry_compact(DbName, GroupId, NewGroup).
 
+maybe_retry_compact(DbName, GroupId, NewGroup) ->
     Pid = couch_view:get_group_server(DbName, GroupId),
-    gen_server:cast(Pid, {compact_done, NewGroup}).
+    case gen_server:call(Pid, {compact_done, NewGroup}) of
+    ok ->
+        ok;
+    update ->
+        {_, Ref} = erlang:spawn_monitor(fun() ->
+            couch_view_updater:update(nil, NewGroup, DbName)
+        end),
+        receive
+        {'DOWN', Ref, _, _, {new_group, NewGroup2}} ->
+            maybe_retry_compact(DbName, GroupId, NewGroup2)
+        end
+    end.
 
 %% @spec compact_view(View, EmptyView, Retry) -> CompactView
 compact_view(View, EmptyView, BufferSize) ->

Modified: couchdb/trunk/src/couchdb/couch_view_group.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_group.erl?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_view_group.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view_group.erl Sat Aug 20 23:00:24 2011
@@ -151,9 +151,9 @@ handle_call({request_group, RequestSeq},
 
 handle_call(request_group_info, _From, State) ->
     GroupInfo = get_group_info(State),
-    {reply, {ok, GroupInfo}, State}.
+    {reply, {ok, GroupInfo}, State};
 
-handle_cast({start_compact, CompactFun}, #group_state{compactor_pid=nil}
+handle_call({start_compact, CompactFun}, _From, #group_state{compactor_pid=nil}
         = State) ->
     #group_state{
         group = #group{name = GroupId, sig = GroupSig} = Group,
@@ -165,12 +165,12 @@ handle_cast({start_compact, CompactFun},
     NewGroup = reset_file(Db, Fd, DbName, Group),
     couch_db:close(Db),
     Pid = spawn_link(fun() -> CompactFun(Group, NewGroup, DbName) end),
-    {noreply, State#group_state{compactor_pid = Pid}};
-handle_cast({start_compact, _}, State) ->
+    {reply, {ok, Pid}, State#group_state{compactor_pid = Pid}};
+handle_call({start_compact, _}, _From, State) ->
     %% compact already running, this is a no-op
-    {noreply, State};
+    {reply, {ok, State#group_state.compactor_pid}, State};
 
-handle_cast({compact_done, #group{current_seq=NewSeq} = NewGroup},
+handle_call({compact_done, #group{current_seq=NewSeq} = NewGroup}, _From,
         #group_state{group = #group{current_seq=OldSeq}} = State)
         when NewSeq >= OldSeq ->
     #group_state{
@@ -206,31 +206,34 @@ handle_cast({compact_done, #group{curren
     {ok, NewRefCounter} = couch_ref_counter:start([NewGroup#group.fd]),
 
     self() ! delayed_commit,
-    {noreply, State#group_state{
+    {reply, ok, State#group_state{
         group=NewGroup,
         ref_counter=NewRefCounter,
         compactor_pid=nil,
         updater_pid=NewUpdaterPid
     }};
-handle_cast({compact_done, NewGroup}, State) ->
+handle_call({compact_done, NewGroup}, _From, State) ->
     #group_state{
         group = #group{name = GroupId, current_seq = CurrentSeq},
         init_args={_RootDir, DbName, _}
     } = State,
     ?LOG_INFO("View index compaction still behind for ~s ~s -- current: ~p " ++
         "compact: ~p", [DbName, GroupId, CurrentSeq, NewGroup#group.current_seq]),
-    Pid = spawn_link(fun() ->
-        {_,Ref} = erlang:spawn_monitor(fun() ->
-            couch_view_updater:update(nil, NewGroup, DbName)
-        end),
-        receive
-            {'DOWN', Ref, _, _, {new_group, NewGroup2}} ->
-                #group{name=GroupId} = NewGroup2,
-                Pid2 = couch_view:get_group_server(DbName, GroupId),
-                gen_server:cast(Pid2, {compact_done, NewGroup2})
-        end
-    end),
-    {noreply, State#group_state{compactor_pid = Pid}};
+    {reply, update, State};
+
+handle_call(cancel_compact, _From, #group_state{compactor_pid = nil} = State) ->
+    {reply, ok, State};
+handle_call(cancel_compact, _From, #group_state{compactor_pid = Pid} = State) ->
+    unlink(Pid),
+    exit(Pid, kill),
+    #group_state{
+        group = #group{sig=GroupSig},
+        init_args = {RootDir, DbName, _}
+    } = State,
+    CompactFile = index_file_name(compact, RootDir, DbName, GroupSig),
+    ok = couch_file:delete(RootDir, CompactFile),
+    {reply, ok, State#group_state{compactor_pid = nil}}.
+
 
 handle_cast({partial_update, Pid, NewGroup}, #group_state{updater_pid=Pid}
         = State) ->

Modified: couchdb/trunk/test/etap/200-view-group-no-db-leaks.t
URL: http://svn.apache.org/viewvc/couchdb/trunk/test/etap/200-view-group-no-db-leaks.t?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/test/etap/200-view-group-no-db-leaks.t (original)
+++ couchdb/trunk/test/etap/200-view-group-no-db-leaks.t Sat Aug 20 23:00:24 2011
@@ -151,7 +151,7 @@ delete_db() ->
 
 compact_db() ->
     {ok, Db} = couch_db:open_int(test_db_name(), []),
-    ok = couch_db:start_compact(Db),
+    {ok, _} = couch_db:start_compact(Db),
     ok = couch_db:close(Db),
     wait_db_compact_done(10).
 
@@ -169,7 +169,7 @@ wait_db_compact_done(N) ->
     end.
 
 compact_view_group() ->
-    ok = couch_view_compactor:start_compact(test_db_name(), ddoc_name()),
+    {ok, _} = couch_view_compactor:start_compact(test_db_name(), ddoc_name()),
     wait_view_compact_done(10).
 
 wait_view_compact_done(0) ->

Added: couchdb/trunk/test/etap/220-compaction-daemon.t
URL: http://svn.apache.org/viewvc/couchdb/trunk/test/etap/220-compaction-daemon.t?rev=1159922&view=auto
==============================================================================
--- couchdb/trunk/test/etap/220-compaction-daemon.t (added)
+++ couchdb/trunk/test/etap/220-compaction-daemon.t Sat Aug 20 23:00:24 2011
@@ -0,0 +1,227 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-record(user_ctx, {
+    name = null,
+    roles = [],
+    handler
+}).
+
+test_db_name() ->
+    <<"couch_test_compaction_daemon">>.
+
+main(_) ->
+    test_util:init_code_path(),
+
+    etap:plan(8),
+    case (catch test()) of
+        ok ->
+            etap:end_tests();
+        Other ->
+            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+            etap:bail(Other)
+    end,
+    ok.
+
+test() ->
+    couch_server_sup:start_link(test_util:config_files()),
+    timer:sleep(1000),
+    put(addr, couch_config:get("httpd", "bind_address", "127.0.0.1")),
+    put(port, integer_to_list(mochiweb_socket_server:get(couch_httpd, port))),
+    application:start(inets),
+
+    disable_compact_daemon(),
+
+    delete_db(),
+    {ok, Db} = create_db(),
+
+    add_design_doc(Db),
+    couch_db:close(Db),
+    populate(70, 70, 200 * 1024),
+
+    {_, DbFileSize} = get_db_frag(),
+    {_, ViewFileSize} = get_view_frag(),
+
+    % enable automatic compaction
+    ok = couch_config:set("compaction_daemon", "check_interval", "3", false),
+    ok = couch_config:set("compaction_daemon", "min_file_size", "100000", false),
+    ok = couch_config:set(
+        "compactions",
+        binary_to_list(test_db_name()),
+        "[{db_fragmentation, \"70%\"}, {view_fragmentation, \"70%\"}]",
+        false),
+
+    ok = timer:sleep(4000), % something >= check_interval
+    wait_compaction_finished(),
+
+    {DbFrag2, DbFileSize2} = get_db_frag(),
+    {ViewFrag2, ViewFileSize2} = get_view_frag(),
+
+    etap:is(true, (DbFrag2 < 70), "Database fragmentation is < 70% after compaction"),
+    etap:is(true, (ViewFrag2 < 70), "View fragmentation is < 70% after compaction"),
+    etap:is(true, (DbFileSize2 < DbFileSize), "Database file size decreased"),
+    etap:is(true, (ViewFileSize2 < ViewFileSize), "View file size decreased"),
+
+    disable_compact_daemon(),
+    ok = timer:sleep(6000), % 2 times check_interval
+    populate(70, 70, 200 * 1024),
+    {_, DbFileSize3} = get_db_frag(),
+    {_, ViewFileSize3} = get_view_frag(),
+
+    % enable automatic compaction
+    ok = couch_config:set(
+        "compactions",
+        "_default",
+        "[{db_fragmentation, \"70%\"}, {view_fragmentation, \"70%\"}]",
+        false),
+
+    ok = timer:sleep(4000), % something >= check_interval
+    wait_compaction_finished(),
+
+    {DbFrag4, DbFileSize4} = get_db_frag(),
+    {ViewFrag4, ViewFileSize4} = get_view_frag(),
+
+    etap:is(true, (DbFrag4 < 70), "Database fragmentation is < 70% after compaction"),
+    etap:is(true, (ViewFrag4 < 70), "View fragmentation is < 70% after compaction"),
+    etap:is(true, (DbFileSize4 < DbFileSize3), "Database file size decreased again"),
+    etap:is(true, (ViewFileSize4 < ViewFileSize3), "View file size decreased again"),
+
+    delete_db(),
+    couch_server_sup:stop(),
+    ok.
+
+disable_compact_daemon() ->
+    Configs = couch_config:get("compactions"),
+    lists:foreach(
+        fun({DbName, _}) ->
+            ok = couch_config:delete("compactions", DbName, false)
+        end,
+        Configs).
+
+admin_user_ctx() ->
+    {user_ctx, #user_ctx{roles = [<<"_admin">>]}}.
+
+create_db() ->
+    {ok, _} = couch_db:create(test_db_name(), [admin_user_ctx()]).
+
+delete_db() ->
+    couch_server:delete(test_db_name(), [admin_user_ctx()]).
+
+add_design_doc(Db) ->
+    DDoc = couch_doc:from_json_obj({[
+        {<<"_id">>, <<"_design/foo">>},
+        {<<"language">>, <<"javascript">>},
+        {<<"views">>, {[
+            {<<"foo">>, {[
+                {<<"map">>, <<"function(doc) { emit(doc._id, doc); }">>}
+            ]}},
+            {<<"foo2">>, {[
+                {<<"map">>, <<"function(doc) { emit(doc._id, doc); }">>}
+            ]}},
+            {<<"foo3">>, {[
+                {<<"map">>, <<"function(doc) { emit(doc._id, doc); }">>}
+            ]}}
+        ]}}
+    ]}),
+    {ok, _} = couch_db:update_docs(Db, [DDoc]),
+    {ok, _} = couch_db:ensure_full_commit(Db),
+    ok.
+
+populate(DbFrag, ViewFrag, MinFileSize) ->
+    {CurDbFrag, DbFileSize} = get_db_frag(),
+    {CurViewFrag, ViewFileSize} = get_view_frag(),
+    populate(
+        DbFrag, ViewFrag, MinFileSize, CurDbFrag, CurViewFrag,
+        lists:min([DbFileSize, ViewFileSize])).
+
+populate(DbFrag, ViewFrag, MinFileSize, CurDbFrag, CurViewFrag, FileSize)
+    when CurDbFrag >= DbFrag, CurViewFrag >= ViewFrag, FileSize >= MinFileSize ->
+    ok;
+populate(DbFrag, ViewFrag, MinFileSize, _, _, _) ->
+    update(),
+    {CurDbFrag, DbFileSize} = get_db_frag(),
+    {CurViewFrag, ViewFileSize} = get_view_frag(),
+    populate(
+        DbFrag, ViewFrag, MinFileSize, CurDbFrag, CurViewFrag,
+        lists:min([DbFileSize, ViewFileSize])).
+
+update() ->
+    {ok, Db} = couch_db:open_int(test_db_name(), []),
+    Docs = lists:map(
+        fun(_) ->
+            Doc = couch_doc:from_json_obj({[{<<"_id">>, couch_uuids:new()}]}),
+            {ok, _} = couch_db:update_docs(Db, [Doc])
+        end,
+        lists:seq(1, 100)),
+    couch_db:close(Db),
+    query_view().
+
+db_url() ->
+    "http://" ++ get(addr) ++ ":" ++ get(port) ++ "/" ++
+        binary_to_list(test_db_name()).
+
+query_view() ->
+    {ok, {{_, Code, _}, _Headers, _Body}} = http:request(
+        get,
+        {db_url() ++ "/_design/foo/_view/foo", []},
+        [],
+        [{sync, true}]),
+    case Code of
+    200 ->
+        ok;
+    _ ->
+        etap:bail("error querying view")
+    end.
+
+get_db_frag() ->
+    {ok, Db} = couch_db:open_int(test_db_name(), []),
+    {ok, Info} = couch_db:get_db_info(Db),
+    couch_db:close(Db),
+    FileSize = couch_util:get_value(disk_size, Info),
+    DataSize = couch_util:get_value(data_size, Info),
+    {round((FileSize - DataSize) / FileSize * 100), FileSize}.
+
+get_view_frag() ->
+    {ok, Db} = couch_db:open_int(test_db_name(), []),
+    {ok, Info} = couch_view:get_group_info(Db, <<"_design/foo">>),
+    couch_db:close(Db),
+    FileSize = couch_util:get_value(disk_size, Info),
+    DataSize = couch_util:get_value(data_size, Info),
+    {round((FileSize - DataSize) / FileSize * 100), FileSize}.
+
+
+wait_compaction_finished() ->
+    Parent = self(),
+    Loop = spawn_link(fun() -> wait_loop(Parent) end),
+    receive
+    {done, Loop} ->
+        etap:diag("Database and view compaction have finished")
+    after 60000 ->
+        etap:bail("Compaction not triggered")
+    end.
+
+wait_loop(Parent) ->
+    {ok, Db} = couch_db:open_int(test_db_name(), []),
+    {ok, DbInfo} = couch_db:get_db_info(Db),
+    {ok, ViewInfo} = couch_view:get_group_info(Db, <<"_design/foo">>),
+    couch_db:close(Db),
+    case (couch_util:get_value(compact_running, ViewInfo) =:= true) orelse
+        (couch_util:get_value(compact_running, DbInfo) =:= true) of
+    false ->
+        Parent ! {done, self()};
+    true ->
+        ok = timer:sleep(500),
+        wait_loop(Parent)
+    end.

Propchange: couchdb/trunk/test/etap/220-compaction-daemon.t
------------------------------------------------------------------------------
    svn:executable = *

Modified: couchdb/trunk/test/etap/Makefile.am
URL: http://svn.apache.org/viewvc/couchdb/trunk/test/etap/Makefile.am?rev=1159922&r1=1159921&r2=1159922&view=diff
==============================================================================
--- couchdb/trunk/test/etap/Makefile.am (original)
+++ couchdb/trunk/test/etap/Makefile.am Sat Aug 20 23:00:24 2011
@@ -84,4 +84,5 @@ EXTRA_DIST = \
     180-http-proxy.t \
     190-json-stream-parse.t \
     200-view-group-no-db-leaks.t \
-    210-os-proc-pool.t
+    210-os-proc-pool.t \
+    220-compaction-daemon.t



Mime
View raw message