couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject [1/3] git commit: Ensure compaction respects compression setting
Date Mon, 06 Feb 2012 18:48:40 GMT
Updated Branches:
  refs/heads/1.2.x 506deab47 -> ee2b9218c


Ensure compaction respects compression setting

For a database with file format version 6 (CouchDB 1.2+), if
the file_compression setting was changed, compaction ignored
it, making it impossible to change the compression method of
a database and its indexes.

Closes COUCHDB-1394


Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/71a0f583
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/71a0f583
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/71a0f583

Branch: refs/heads/1.2.x
Commit: 71a0f583b586914c2009cceafb0102aa6fb52fb0
Parents: 506deab
Author: Filipe David Borba Manana <fdmanana@apache.org>
Authored: Thu Jan 26 14:40:39 2012 +0000
Committer: Jan Lehnardt <jan@apache.org>
Committed: Mon Feb 6 19:47:54 2012 +0100

----------------------------------------------------------------------
 src/couchdb/couch_compress.erl   |   25 +++--
 src/couchdb/couch_db_updater.erl |    7 +-
 test/etap/076-file-compression.t |  185 +++++++++++++++++++++++++++++++++
 test/etap/Makefile.am            |    1 +
 4 files changed, 208 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb/blob/71a0f583/src/couchdb/couch_compress.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_compress.erl b/src/couchdb/couch_compress.erl
index 523108f..ac386fd 100644
--- a/src/couchdb/couch_compress.erl
+++ b/src/couchdb/couch_compress.erl
@@ -12,16 +12,17 @@
 
 -module(couch_compress).
 
--export([compress/2, decompress/1, is_compressed/1]).
+-export([compress/2, decompress/1, is_compressed/2]).
 -export([get_compression_method/0]).
 
 -include("couch_db.hrl").
 
 % binaries compressed with snappy have their first byte set to this value
 -define(SNAPPY_PREFIX, 1).
-% binaries that are a result of an erlang:term_to_binary/1,2 call have this
-% value as their first byte
+% Term prefixes documented at:
+%      http://www.erlang.org/doc/apps/erts/erl_ext_dist.html
 -define(TERM_PREFIX, 131).
+-define(COMPRESSED_TERM_PREFIX, 131, 80).
 
 
 get_compression_method() ->
@@ -38,6 +39,12 @@ get_compression_method() ->
     end.
 
 
+compress(<<?SNAPPY_PREFIX, _/binary>> = Bin, snappy) ->
+    Bin;
+compress(<<?SNAPPY_PREFIX, _/binary>> = Bin, Method) ->
+    compress(decompress(Bin), Method);
+compress(<<?TERM_PREFIX, _/binary>> = Bin, Method) ->
+    compress(decompress(Bin), Method);
 compress(Term, none) ->
     ?term_to_bin(Term);
 compress(Term, {deflate, Level}) ->
@@ -64,10 +71,14 @@ decompress(<<?TERM_PREFIX, _/binary>> = Bin) ->
     binary_to_term(Bin).
 
 
-is_compressed(<<?SNAPPY_PREFIX, _/binary>>) ->
+is_compressed(<<?SNAPPY_PREFIX, _/binary>>, Method) ->
+    Method =:= snappy;
+is_compressed(<<?COMPRESSED_TERM_PREFIX, _/binary>>, {deflate, _Level}) ->
     true;
-is_compressed(<<?TERM_PREFIX, _/binary>>) ->
-    true;
-is_compressed(Term) when not is_binary(Term) ->
+is_compressed(<<?COMPRESSED_TERM_PREFIX, _/binary>>, _Method) ->
+    false;
+is_compressed(<<?TERM_PREFIX, _/binary>>, Method) ->
+    Method =:= none;
+is_compressed(Term, _Method) when not is_binary(Term) ->
     false.
 

http://git-wip-us.apache.org/repos/asf/couchdb/blob/71a0f583/src/couchdb/couch_db_updater.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_db_updater.erl b/src/couchdb/couch_db_updater.erl
index b87ffcd..eaf1bd3 100644
--- a/src/couchdb/couch_db_updater.erl
+++ b/src/couchdb/couch_db_updater.erl
@@ -897,7 +897,8 @@ copy_docs(Db, #db{updater_fd = DestFd} = NewDb, InfoBySeq0, Retry) ->
 
 copy_compact(Db, NewDb0, Retry) ->
     FsyncOptions = [Op || Op <- NewDb0#db.fsync_options, Op == before_header],
-    NewDb = NewDb0#db{fsync_options=FsyncOptions},
+    Compression = couch_compress:get_compression_method(),
+    NewDb = NewDb0#db{fsync_options=FsyncOptions, compression=Compression},
     TotalChanges = couch_db:count_changes_since(Db, NewDb#db.update_seq),
     BufferSize = list_to_integer(
         couch_config:get("database_compaction", "doc_buffer_size", "524288")),
@@ -1016,14 +1017,14 @@ update_compact_task(NumChanges) ->
     couch_task_status:update([{changes_done, Changes2}, {progress, Progress}]).
 
 make_doc_summary(#db{compression = Comp}, {Body0, Atts0}) ->
-    Body = case couch_compress:is_compressed(Body0) of
+    Body = case couch_compress:is_compressed(Body0, Comp) of
     true ->
         Body0;
     false ->
         % pre 1.2 database file format
         couch_compress:compress(Body0, Comp)
     end,
-    Atts = case couch_compress:is_compressed(Atts0) of
+    Atts = case couch_compress:is_compressed(Atts0, Comp) of
     true ->
         Atts0;
     false ->

http://git-wip-us.apache.org/repos/asf/couchdb/blob/71a0f583/test/etap/076-file-compression.t
----------------------------------------------------------------------
diff --git a/test/etap/076-file-compression.t b/test/etap/076-file-compression.t
new file mode 100755
index 0000000..d316d96
--- /dev/null
+++ b/test/etap/076-file-compression.t
@@ -0,0 +1,185 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-record(user_ctx, {
+    name = null,
+    roles = [],
+    handler
+}).
+
+test_db_name() -> <<"couch_test_file_compression">>.
+ddoc_id() -> <<"_design/test">>.
+num_docs() -> 5000.
+
+
+main(_) ->
+    test_util:init_code_path(),
+
+    etap:plan(10),
+    case (catch test()) of
+        ok ->
+            etap:end_tests();
+        Other ->
+            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+            etap:bail(Other)
+    end,
+    ok.
+
+
+test() ->
+    couch_server_sup:start_link(test_util:config_files()),
+    couch_config:set("couchdb", "file_compression", "none", false),
+
+    create_database(),
+    compact_db(),
+    compact_view(),
+    DbDiskSize1 = db_disk_size(),
+    ViewDiskSize1 = view_disk_size(),
+
+    couch_config:set("couchdb", "file_compression", "snappy", false),
+    compact_db(),
+    compact_view(),
+    DbDiskSize2 = db_disk_size(),
+    ViewDiskSize2 = view_disk_size(),
+
+    etap:is(DbDiskSize2 < DbDiskSize1, true, "Database disk size decreased"),
+    etap:is(ViewDiskSize2 < ViewDiskSize1, true, "Index disk size decreased"),
+
+    couch_config:set("couchdb", "file_compression", "deflate_9", false),
+    compact_db(),
+    compact_view(),
+    DbDiskSize3 = db_disk_size(),
+    ViewDiskSize3 = view_disk_size(),
+
+    etap:is(DbDiskSize3 < DbDiskSize2, true, "Database disk size decreased again"),
+    etap:is(ViewDiskSize3 < ViewDiskSize2, true, "Index disk size decreased again"),
+
+    couch_config:set("couchdb", "file_compression", "deflate_1", false),
+    compact_db(),
+    compact_view(),
+    DbDiskSize4 = db_disk_size(),
+    ViewDiskSize4 = view_disk_size(),
+
+    etap:is(DbDiskSize4 > DbDiskSize3, true, "Database disk size increased"),
+    etap:is(ViewDiskSize4 > ViewDiskSize3, true, "Index disk size increased"),
+
+    couch_config:set("couchdb", "file_compression", "snappy", false),
+    compact_db(),
+    compact_view(),
+    DbDiskSize5 = db_disk_size(),
+    ViewDiskSize5 = view_disk_size(),
+
+    etap:is(DbDiskSize5 > DbDiskSize4, true, "Database disk size increased again"),
+    etap:is(ViewDiskSize5 > ViewDiskSize4, true, "Index disk size increased again"),
+
+    couch_config:set("couchdb", "file_compression", "none", false),
+    compact_db(),
+    compact_view(),
+    DbDiskSize6 = db_disk_size(),
+    ViewDiskSize6 = view_disk_size(),
+
+    etap:is(DbDiskSize6 > DbDiskSize5, true, "Database disk size increased again"),
+    etap:is(ViewDiskSize6 > ViewDiskSize5, true, "Index disk size increased again"),
+
+    delete_db(),
+    couch_server_sup:stop(),
+    ok.
+
+
+create_database() ->
+    {ok, Db} = couch_db:create(
+        test_db_name(),
+        [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}, overwrite]),
+    ok = populate_db(Db, num_docs()),
+    DDoc = couch_doc:from_json_obj({[
+        {<<"_id">>, ddoc_id()},
+        {<<"language">>, <<"javascript">>},
+        {<<"views">>, {[
+                {<<"view1">>, {[
+                    {<<"map">>, <<"function(doc) { emit(doc._id, doc.string);
}">>}
+                ]}}
+            ]}
+        }
+    ]}),
+    {ok, _} = couch_db:update_doc(Db, DDoc, []),
+    refresh_index(),
+    ok = couch_db:close(Db).
+
+
+populate_db(_Db, NumDocs) when NumDocs =< 0 ->
+    ok;
+populate_db(Db, NumDocs) ->
+    Docs = lists:map(
+        fun(_) ->
+            couch_doc:from_json_obj({[
+                {<<"_id">>, couch_uuids:random()},
+                {<<"string">>, list_to_binary(lists:duplicate(1000, $X))}
+            ]})
+        end,
+        lists:seq(1, 500)),
+    {ok, _} = couch_db:update_docs(Db, Docs, []),
+    populate_db(Db, NumDocs - 500).
+
+
+refresh_index() ->
+    {ok, Db} = couch_db:open_int(test_db_name(), []),
+    {ok, _, _} = couch_view:get_map_view(Db, ddoc_id(), <<"view1">>, false),
+    ok = couch_db:close(Db).
+
+
+compact_db() ->
+    {ok, Db} = couch_db:open_int(test_db_name(), []),
+    {ok, CompactPid} = couch_db:start_compact(Db),
+    MonRef = erlang:monitor(process, CompactPid),
+    receive
+    {'DOWN', MonRef, process, CompactPid, normal} ->
+        ok;
+    {'DOWN', MonRef, process, CompactPid, Reason} ->
+        etap:bail("Error compacting database: " ++ couch_util:to_list(Reason))
+    after 120000 ->
+        etap:bail("Timeout waiting for database compaction")
+    end,
+    ok = couch_db:close(Db).
+
+
+compact_view() ->
+    {ok, CompactPid} = couch_view_compactor:start_compact(test_db_name(), <<"test">>),
+    MonRef = erlang:monitor(process, CompactPid),
+    receive
+    {'DOWN', MonRef, process, CompactPid, normal} ->
+        ok;
+    {'DOWN', MonRef, process, CompactPid, Reason} ->
+        etap:bail("Error compacting view group: " ++ couch_util:to_list(Reason))
+    after 120000 ->
+        etap:bail("Timeout waiting for view group compaction")
+    end.
+
+
+db_disk_size() ->
+    {ok, Db} = couch_db:open_int(test_db_name(), []),
+    {ok, Info} = couch_db:get_db_info(Db),
+    ok = couch_db:close(Db),
+    couch_util:get_value(disk_size, Info).
+
+
+view_disk_size() ->
+    {ok, Db} = couch_db:open_int(test_db_name(), []),
+    {ok, Info} = couch_view:get_group_info(Db, ddoc_id()),
+    ok = couch_db:close(Db),
+    couch_util:get_value(disk_size, Info).
+
+
+delete_db() ->
+    ok = couch_server:delete(
+        test_db_name(), [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}]).

http://git-wip-us.apache.org/repos/asf/couchdb/blob/71a0f583/test/etap/Makefile.am
----------------------------------------------------------------------
diff --git a/test/etap/Makefile.am b/test/etap/Makefile.am
index 47f5626..277eedb 100644
--- a/test/etap/Makefile.am
+++ b/test/etap/Makefile.am
@@ -57,6 +57,7 @@ EXTRA_DIST = \
     072-cleanup.t \
     073-changes.t \
     075-auth-cache.t \
+    076-file-compression.t \
     080-config-get-set.t \
     081-config-override.1.ini \
     081-config-override.2.ini \


Mime
View raw message