Return-Path: X-Original-To: apmail-couchdb-commits-archive@www.apache.org Delivered-To: apmail-couchdb-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B3E8C9648 for ; Thu, 26 Jan 2012 15:23:37 +0000 (UTC) Received: (qmail 74489 invoked by uid 500); 26 Jan 2012 15:23:37 -0000 Delivered-To: apmail-couchdb-commits-archive@couchdb.apache.org Received: (qmail 74450 invoked by uid 500); 26 Jan 2012 15:23:37 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 74440 invoked by uid 99); 26 Jan 2012 15:23:36 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 26 Jan 2012 15:23:36 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.114] (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 26 Jan 2012 15:23:35 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 52318319C10; Thu, 26 Jan 2012 15:23:15 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: fdmanana@apache.org To: commits@couchdb.apache.org X-Mailer: ASF-Git Admin Mailer Subject: git commit: Ensure compaction respects compression setting Message-Id: <20120126152315.52318319C10@tyr.zones.apache.org> Date: Thu, 26 Jan 2012 15:23:15 +0000 (UTC) Updated Branches: refs/heads/1.2.x c7765f657 -> 450f4e2bb Ensure compaction respects compression setting For a database with file format version 6 (CouchDB 1.2+), if the file_compression setting was changed, compaction ignored it, making it impossible to change the compression method of a database and its indexes. Closes COUCHDB-1394 Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/450f4e2b Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/450f4e2b Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/450f4e2b Branch: refs/heads/1.2.x Commit: 450f4e2bbf97d9de96998cd93de10ce1e224c0a2 Parents: c7765f6 Author: Filipe David Borba Manana Authored: Thu Jan 26 14:40:39 2012 +0000 Committer: Filipe David Borba Manana Committed: Thu Jan 26 15:22:55 2012 +0000 ---------------------------------------------------------------------- src/couchdb/couch_compress.erl | 25 +++-- src/couchdb/couch_db_updater.erl | 7 +- test/etap/076-file-compression.t | 185 +++++++++++++++++++++++++++++++++ test/etap/Makefile.am | 1 + 4 files changed, 208 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb/blob/450f4e2b/src/couchdb/couch_compress.erl ---------------------------------------------------------------------- diff --git a/src/couchdb/couch_compress.erl b/src/couchdb/couch_compress.erl index 523108f..ac386fd 100644 --- a/src/couchdb/couch_compress.erl +++ b/src/couchdb/couch_compress.erl @@ -12,16 +12,17 @@ -module(couch_compress). --export([compress/2, decompress/1, is_compressed/1]). +-export([compress/2, decompress/1, is_compressed/2]). -export([get_compression_method/0]). -include("couch_db.hrl"). % binaries compressed with snappy have their first byte set to this value -define(SNAPPY_PREFIX, 1). -% binaries that are a result of an erlang:term_to_binary/1,2 call have this -% value as their first byte +% Term prefixes documented at: +% http://www.erlang.org/doc/apps/erts/erl_ext_dist.html -define(TERM_PREFIX, 131). +-define(COMPRESSED_TERM_PREFIX, 131, 80). get_compression_method() -> @@ -38,6 +39,12 @@ get_compression_method() -> end. +compress(<> = Bin, snappy) -> + Bin; +compress(<> = Bin, Method) -> + compress(decompress(Bin), Method); +compress(<> = Bin, Method) -> + compress(decompress(Bin), Method); compress(Term, none) -> ?term_to_bin(Term); compress(Term, {deflate, Level}) -> @@ -64,10 +71,14 @@ decompress(<> = Bin) -> binary_to_term(Bin). -is_compressed(<>) -> +is_compressed(<>, Method) -> + Method =:= snappy; +is_compressed(<>, {deflate, _Level}) -> true; -is_compressed(<>) -> - true; -is_compressed(Term) when not is_binary(Term) -> +is_compressed(<>, _Method) -> + false; +is_compressed(<>, Method) -> + Method =:= none; +is_compressed(Term, _Method) when not is_binary(Term) -> false. http://git-wip-us.apache.org/repos/asf/couchdb/blob/450f4e2b/src/couchdb/couch_db_updater.erl ---------------------------------------------------------------------- diff --git a/src/couchdb/couch_db_updater.erl b/src/couchdb/couch_db_updater.erl index b87ffcd..eaf1bd3 100644 --- a/src/couchdb/couch_db_updater.erl +++ b/src/couchdb/couch_db_updater.erl @@ -897,7 +897,8 @@ copy_docs(Db, #db{updater_fd = DestFd} = NewDb, InfoBySeq0, Retry) -> copy_compact(Db, NewDb0, Retry) -> FsyncOptions = [Op || Op <- NewDb0#db.fsync_options, Op == before_header], - NewDb = NewDb0#db{fsync_options=FsyncOptions}, + Compression = couch_compress:get_compression_method(), + NewDb = NewDb0#db{fsync_options=FsyncOptions, compression=Compression}, TotalChanges = couch_db:count_changes_since(Db, NewDb#db.update_seq), BufferSize = list_to_integer( couch_config:get("database_compaction", "doc_buffer_size", "524288")), @@ -1016,14 +1017,14 @@ update_compact_task(NumChanges) -> couch_task_status:update([{changes_done, Changes2}, {progress, Progress}]). make_doc_summary(#db{compression = Comp}, {Body0, Atts0}) -> - Body = case couch_compress:is_compressed(Body0) of + Body = case couch_compress:is_compressed(Body0, Comp) of true -> Body0; false -> % pre 1.2 database file format couch_compress:compress(Body0, Comp) end, - Atts = case couch_compress:is_compressed(Atts0) of + Atts = case couch_compress:is_compressed(Atts0, Comp) of true -> Atts0; false -> http://git-wip-us.apache.org/repos/asf/couchdb/blob/450f4e2b/test/etap/076-file-compression.t ---------------------------------------------------------------------- diff --git a/test/etap/076-file-compression.t b/test/etap/076-file-compression.t new file mode 100755 index 0000000..d316d96 --- /dev/null +++ b/test/etap/076-file-compression.t @@ -0,0 +1,185 @@ +#!/usr/bin/env escript +%% -*- erlang -*- +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-record(user_ctx, { + name = null, + roles = [], + handler +}). + +test_db_name() -> <<"couch_test_file_compression">>. +ddoc_id() -> <<"_design/test">>. +num_docs() -> 5000. + + +main(_) -> + test_util:init_code_path(), + + etap:plan(10), + case (catch test()) of + ok -> + etap:end_tests(); + Other -> + etap:diag(io_lib:format("Test died abnormally: ~p", [Other])), + etap:bail(Other) + end, + ok. + + +test() -> + couch_server_sup:start_link(test_util:config_files()), + couch_config:set("couchdb", "file_compression", "none", false), + + create_database(), + compact_db(), + compact_view(), + DbDiskSize1 = db_disk_size(), + ViewDiskSize1 = view_disk_size(), + + couch_config:set("couchdb", "file_compression", "snappy", false), + compact_db(), + compact_view(), + DbDiskSize2 = db_disk_size(), + ViewDiskSize2 = view_disk_size(), + + etap:is(DbDiskSize2 < DbDiskSize1, true, "Database disk size decreased"), + etap:is(ViewDiskSize2 < ViewDiskSize1, true, "Index disk size decreased"), + + couch_config:set("couchdb", "file_compression", "deflate_9", false), + compact_db(), + compact_view(), + DbDiskSize3 = db_disk_size(), + ViewDiskSize3 = view_disk_size(), + + etap:is(DbDiskSize3 < DbDiskSize2, true, "Database disk size decreased again"), + etap:is(ViewDiskSize3 < ViewDiskSize2, true, "Index disk size decreased again"), + + couch_config:set("couchdb", "file_compression", "deflate_1", false), + compact_db(), + compact_view(), + DbDiskSize4 = db_disk_size(), + ViewDiskSize4 = view_disk_size(), + + etap:is(DbDiskSize4 > DbDiskSize3, true, "Database disk size increased"), + etap:is(ViewDiskSize4 > ViewDiskSize3, true, "Index disk size increased"), + + couch_config:set("couchdb", "file_compression", "snappy", false), + compact_db(), + compact_view(), + DbDiskSize5 = db_disk_size(), + ViewDiskSize5 = view_disk_size(), + + etap:is(DbDiskSize5 > DbDiskSize4, true, "Database disk size increased again"), + etap:is(ViewDiskSize5 > ViewDiskSize4, true, "Index disk size increased again"), + + couch_config:set("couchdb", "file_compression", "none", false), + compact_db(), + compact_view(), + DbDiskSize6 = db_disk_size(), + ViewDiskSize6 = view_disk_size(), + + etap:is(DbDiskSize6 > DbDiskSize5, true, "Database disk size increased again"), + etap:is(ViewDiskSize6 > ViewDiskSize5, true, "Index disk size increased again"), + + delete_db(), + couch_server_sup:stop(), + ok. + + +create_database() -> + {ok, Db} = couch_db:create( + test_db_name(), + [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}, overwrite]), + ok = populate_db(Db, num_docs()), + DDoc = couch_doc:from_json_obj({[ + {<<"_id">>, ddoc_id()}, + {<<"language">>, <<"javascript">>}, + {<<"views">>, {[ + {<<"view1">>, {[ + {<<"map">>, <<"function(doc) { emit(doc._id, doc.string); }">>} + ]}} + ]} + } + ]}), + {ok, _} = couch_db:update_doc(Db, DDoc, []), + refresh_index(), + ok = couch_db:close(Db). + + +populate_db(_Db, NumDocs) when NumDocs =< 0 -> + ok; +populate_db(Db, NumDocs) -> + Docs = lists:map( + fun(_) -> + couch_doc:from_json_obj({[ + {<<"_id">>, couch_uuids:random()}, + {<<"string">>, list_to_binary(lists:duplicate(1000, $X))} + ]}) + end, + lists:seq(1, 500)), + {ok, _} = couch_db:update_docs(Db, Docs, []), + populate_db(Db, NumDocs - 500). + + +refresh_index() -> + {ok, Db} = couch_db:open_int(test_db_name(), []), + {ok, _, _} = couch_view:get_map_view(Db, ddoc_id(), <<"view1">>, false), + ok = couch_db:close(Db). + + +compact_db() -> + {ok, Db} = couch_db:open_int(test_db_name(), []), + {ok, CompactPid} = couch_db:start_compact(Db), + MonRef = erlang:monitor(process, CompactPid), + receive + {'DOWN', MonRef, process, CompactPid, normal} -> + ok; + {'DOWN', MonRef, process, CompactPid, Reason} -> + etap:bail("Error compacting database: " ++ couch_util:to_list(Reason)) + after 120000 -> + etap:bail("Timeout waiting for database compaction") + end, + ok = couch_db:close(Db). + + +compact_view() -> + {ok, CompactPid} = couch_view_compactor:start_compact(test_db_name(), <<"test">>), + MonRef = erlang:monitor(process, CompactPid), + receive + {'DOWN', MonRef, process, CompactPid, normal} -> + ok; + {'DOWN', MonRef, process, CompactPid, Reason} -> + etap:bail("Error compacting view group: " ++ couch_util:to_list(Reason)) + after 120000 -> + etap:bail("Timeout waiting for view group compaction") + end. + + +db_disk_size() -> + {ok, Db} = couch_db:open_int(test_db_name(), []), + {ok, Info} = couch_db:get_db_info(Db), + ok = couch_db:close(Db), + couch_util:get_value(disk_size, Info). + + +view_disk_size() -> + {ok, Db} = couch_db:open_int(test_db_name(), []), + {ok, Info} = couch_view:get_group_info(Db, ddoc_id()), + ok = couch_db:close(Db), + couch_util:get_value(disk_size, Info). + + +delete_db() -> + ok = couch_server:delete( + test_db_name(), [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}]). http://git-wip-us.apache.org/repos/asf/couchdb/blob/450f4e2b/test/etap/Makefile.am ---------------------------------------------------------------------- diff --git a/test/etap/Makefile.am b/test/etap/Makefile.am index 47f5626..277eedb 100644 --- a/test/etap/Makefile.am +++ b/test/etap/Makefile.am @@ -57,6 +57,7 @@ EXTRA_DIST = \ 072-cleanup.t \ 073-changes.t \ 075-auth-cache.t \ + 076-file-compression.t \ 080-config-get-set.t \ 081-config-override.1.ini \ 081-config-override.2.ini \