Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 95372200C22 for ; Tue, 21 Feb 2017 18:46:22 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 93DCB160B4F; Tue, 21 Feb 2017 17:46:22 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 959E1160B77 for ; Tue, 21 Feb 2017 18:46:20 +0100 (CET) Received: (qmail 33311 invoked by uid 500); 21 Feb 2017 17:46:19 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 31829 invoked by uid 99); 21 Feb 2017 17:46:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Feb 2017 17:46:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1010AE110F; Tue, 21 Feb 2017 17:46:18 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: davisp@apache.org To: commits@couchdb.apache.org Date: Tue, 21 Feb 2017 17:46:36 -0000 Message-Id: <5989b4c2b6e542998f4a70f89acea7b4@git.apache.org> In-Reply-To: <8b83471d06d34057a5c381b7f39623a2@git.apache.org> References: <8b83471d06d34057a5c381b7f39623a2@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [20/22] couch commit: updated refs/heads/COUCHDB-3287-pluggable-storage-engines to 327c053 archived-at: Tue, 21 Feb 2017 17:46:22 -0000 Add storage engine test suite This allows other storage engine implementations to reuse the same exact test suite without having to resort to shenanigans like keeping vendored copies up to date. COUCHDB-3287 Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/1e1702f9 Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/1e1702f9 Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/1e1702f9 Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines Commit: 1e1702f967677e2be866b6a212624a6a2bb1cfb5 Parents: c05c128 Author: Paul J. Davis Authored: Fri Feb 5 12:21:39 2016 -0600 Committer: Paul J. Davis Committed: Tue Feb 21 11:44:58 2017 -0600 ---------------------------------------------------------------------- .gitignore | 5 + src/test_engine_attachments.erl | 85 ++++ src/test_engine_compaction.erl | 181 +++++++++ src/test_engine_fold_changes.erl | 190 +++++++++ src/test_engine_fold_docs.erl | 390 +++++++++++++++++++ src/test_engine_get_set_props.erl | 70 ++++ src/test_engine_open_close_delete.erl | 81 ++++ src/test_engine_purge_docs.erl | 158 ++++++++ src/test_engine_read_write_docs.erl | 317 +++++++++++++++ src/test_engine_ref_counting.erl | 112 ++++++ src/test_engine_util.erl | 602 +++++++++++++++++++++++++++++ test/couch_bt_engine_tests.erl | 20 + 12 files changed, 2211 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/1e1702f9/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 30aa173..73fb0b6 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,10 @@ priv/*.dll priv/*.exe vc120.pdb +test/engines/coverage/ +test/engines/data/ +test/engines/etc/ +test/engines/log/ + .rebar/ .eunit http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/1e1702f9/src/test_engine_attachments.erl ---------------------------------------------------------------------- diff --git a/src/test_engine_attachments.erl b/src/test_engine_attachments.erl new file mode 100644 index 0000000..a19322d --- /dev/null +++ b/src/test_engine_attachments.erl @@ -0,0 +1,85 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_attachments). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +cet_write_attachment() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + AttBin = crypto:rand_bytes(32768), + + [Att0] = test_engine_util:prep_atts(Engine, St1, [ + {<<"ohai.txt">>, AttBin} + ]), + + {stream, Stream} = couch_att:fetch(data, Att0), + ?assertEqual(true, Engine:is_active_stream(St1, Stream)), + + Actions = [{create, {<<"first">>, [], [Att0]}}], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + + {ok, St4} = Engine:init(DbPath, []), + [FDI] = Engine:open_docs(St4, [<<"first">>]), + + #rev_info{ + rev = {RevPos, PrevRevId}, + deleted = Deleted, + body_sp = DocPtr + } = test_engine_util:prev_rev(FDI), + + Doc0 = #doc{ + id = <<"foo">>, + revs = {RevPos, [PrevRevId]}, + deleted = Deleted, + body = DocPtr + }, + + Doc1 = Engine:read_doc_body(St4, Doc0), + Atts1 = if not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; true -> + couch_compress:decompress(Doc1#doc.atts) + end, + + StreamSrc = fun(Sp) -> Engine:open_read_stream(St4, Sp) end, + [Att1] = [couch_att:from_disk_term(StreamSrc, T) || T <- Atts1], + ReadBin = couch_att:to_binary(Att1), + ?assertEqual(AttBin, ReadBin). + + +% N.B. This test may be overly specific for some theoretical +% storage engines that don't re-initialize their +% attachments streams when restarting (for instance if +% we ever have something that stores attachemnts in +% an external object store) +cet_inactive_stream() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + AttBin = crypto:rand_bytes(32768), + + [Att0] = test_engine_util:prep_atts(Engine, St1, [ + {<<"ohai.txt">>, AttBin} + ]), + + {stream, Stream} = couch_att:fetch(data, Att0), + ?assertEqual(true, Engine:is_active_stream(St1, Stream)), + + Engine:terminate(normal, St1), + {ok, St2} = Engine:init(DbPath, []), + + ?assertEqual(false, Engine:is_active_stream(St2, Stream)). http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/1e1702f9/src/test_engine_compaction.erl ---------------------------------------------------------------------- diff --git a/src/test_engine_compaction.erl b/src/test_engine_compaction.erl new file mode 100644 index 0000000..b178bae --- /dev/null +++ b/src/test_engine_compaction.erl @@ -0,0 +1,181 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_compaction). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +cet_compact_empty() -> + {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + Db1 = test_engine_util:db_as_term(Engine, St1), + {ok, St2, DbName, _, Term} = test_engine_util:compact(Engine, St1, Path), + {ok, St3, undefined} = Engine:finish_compaction(St2, DbName, [], Term), + Db2 = test_engine_util:db_as_term(Engine, St3), + Diff = test_engine_util:term_diff(Db1, Db2), + ?assertEqual(nodiff, Diff). + + +cet_compact_doc() -> + {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + Actions = [{create, {<<"foo">>, []}}], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + Db1 = test_engine_util:db_as_term(Engine, St2), + {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), + {ok, St4, undefined} = Engine:finish_compaction(St3, DbName, [], Term), + Db2 = test_engine_util:db_as_term(Engine, St4), + Diff = test_engine_util:term_diff(Db1, Db2), + ?assertEqual(nodiff, Diff). + + +cet_compact_local_doc() -> + {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + Actions = [{create, {<<"_local/foo">>, []}}], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + Db1 = test_engine_util:db_as_term(Engine, St2), + {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), + {ok, St4, undefined} = Engine:finish_compaction(St3, DbName, [], Term), + Db2 = test_engine_util:db_as_term(Engine, St4), + Diff = test_engine_util:term_diff(Db1, Db2), + ?assertEqual(nodiff, Diff). + + +cet_compact_with_everything() -> + {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + + % Add a whole bunch of docs + DocActions = lists:map(fun(Seq) -> + {create, {docid(Seq), [{<<"int">>, Seq}]}} + end, lists:seq(1, 1000)), + + LocalActions = lists:map(fun(I) -> + {create, {local_docid(I), [{<<"int">>, I}]}} + end, lists:seq(1, 25)), + + Actions1 = DocActions ++ LocalActions, + + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + {ok, St3} = Engine:set_security(St2, [{<<"readers">>, <<"ohai">>}]), + {ok, St4} = Engine:set_revs_limit(St3, 500), + + Actions2 = [ + {create, {<<"foo">>, []}}, + {create, {<<"bar">>, [{<<"hooray">>, <<"purple">>}]}}, + {conflict, {<<"bar">>, [{<<"booo">>, false}]}} + ], + + {ok, St5} = test_engine_util:apply_actions(Engine, St4, Actions2), + + [FooFDI, BarFDI] = Engine:open_docs(St5, [<<"foo">>, <<"bar">>]), + + FooRev = test_engine_util:prev_rev(FooFDI), + BarRev = test_engine_util:prev_rev(BarFDI), + + Actions3 = [ + {batch, [ + {purge, {<<"foo">>, FooRev#rev_info.rev}}, + {purge, {<<"bar">>, BarRev#rev_info.rev}} + ]} + ], + + {ok, St6} = test_engine_util:apply_actions(Engine, St5, Actions3), + + PurgedIdRevs = [ + {<<"bar">>, [BarRev#rev_info.rev]}, + {<<"foo">>, [FooRev#rev_info.rev]} + ], + + ?assertEqual(PurgedIdRevs, lists:sort(Engine:get_last_purged(St6))), + + [Att0, Att1, Att2, Att3, Att4] = test_engine_util:prep_atts(Engine, St6, [ + {<<"ohai.txt">>, crypto:rand_bytes(2048)}, + {<<"stuff.py">>, crypto:rand_bytes(32768)}, + {<<"a.erl">>, crypto:rand_bytes(29)}, + {<<"a.hrl">>, crypto:rand_bytes(5000)}, + {<<"a.app">>, crypto:rand_bytes(400)} + ]), + + Actions4 = [ + {create, {<<"small_att">>, [], [Att0]}}, + {create, {<<"large_att">>, [], [Att1]}}, + {create, {<<"multi_att">>, [], [Att2, Att3, Att4]}} + ], + {ok, St7} = test_engine_util:apply_actions(Engine, St6, Actions4), + {ok, St8} = Engine:commit_data(St7), + + Db1 = test_engine_util:db_as_term(Engine, St8), + + Config = [ + {"database_compaction", "doc_buffer_size", "1024"}, + {"database_compaction", "checkpoint_after", "2048"} + ], + + {ok, St9, DbName, _, Term} = test_engine_util:with_config(Config, fun() -> + test_engine_util:compact(Engine, St8, Path) + end), + + {ok, St10, undefined} = Engine:finish_compaction(St9, DbName, [], Term), + Db2 = test_engine_util:db_as_term(Engine, St10), + Diff = test_engine_util:term_diff(Db1, Db2), + ?assertEqual(nodiff, Diff). + + +cet_recompact_updates() -> + {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + + Actions1 = [ + {create, {<<"foo">>, []}}, + {create, {<<"bar">>, []}} + ], + + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), + + Actions2 = [ + {update, {<<"foo">>, [{<<"updated">>, true}]}}, + {create, {<<"baz">>, []}} + ], + + {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2), + Db1 = test_engine_util:db_as_term(Engine, St4), + + {ok, St5, NewPid} = Engine:finish_compaction(St4, DbName, [], Term), + + ?assertEqual(true, is_pid(NewPid)), + Ref = erlang:monitor(process, NewPid), + + NewTerm = receive + {'$gen_cast', {compact_done, Engine, Term0}} -> + Term0; + {'DOWN', Ref, _, _, Reason} -> + erlang:error({compactor_died, Reason}) + after 10000 -> + erlang:error(compactor_timed_out) + end, + + {ok, St6, undefined} = Engine:finish_compaction(St5, DbName, [], NewTerm), + Db2 = test_engine_util:db_as_term(Engine, St6), + Diff = test_engine_util:term_diff(Db1, Db2), + ?assertEqual(nodiff, Diff). + + +docid(I) -> + Str = io_lib:format("~4..0b", [I]), + iolist_to_binary(Str). + + +local_docid(I) -> + Str = io_lib:format("_local/~4..0b", [I]), + iolist_to_binary(Str). http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/1e1702f9/src/test_engine_fold_changes.erl ---------------------------------------------------------------------- diff --git a/src/test_engine_fold_changes.erl b/src/test_engine_fold_changes.erl new file mode 100644 index 0000000..6e97fda --- /dev/null +++ b/src/test_engine_fold_changes.erl @@ -0,0 +1,190 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_fold_changes). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +-define(NUM_DOCS, 100). + + +cet_empty_changes() -> + {ok, Engine, St} = test_engine_util:init_engine(), + + ?assertEqual(0, Engine:count_changes_since(St, 0)), + ?assertEqual({ok, []}, Engine:fold_changes(St, 0, fun fold_fun/2, [], [])). + + +cet_single_change() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions = [{create, {<<"a">>, []}}], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + + ?assertEqual(1, Engine:count_changes_since(St2, 0)), + ?assertEqual({ok, [{<<"a">>, 1}]}, + Engine:fold_changes(St2, 0, fun fold_fun/2, [], [])). + + +cet_two_changes() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions = [ + {create, {<<"a">>, []}}, + {create, {<<"b">>, []}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + + ?assertEqual(2, Engine:count_changes_since(St2, 0)), + {ok, Changes} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes)). + + +cet_two_changes_batch() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions1 = [ + {batch, [ + {create, {<<"a">>, []}}, + {create, {<<"b">>, []}} + ]} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + + ?assertEqual(2, Engine:count_changes_since(St2, 0)), + {ok, Changes1} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes1)), + + {ok, Engine, St3} = test_engine_util:init_engine(), + Actions2 = [ + {batch, [ + {create, {<<"b">>, []}}, + {create, {<<"a">>, []}} + ]} + ], + {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2), + + ?assertEqual(2, Engine:count_changes_since(St4, 0)), + {ok, Changes2} = Engine:fold_changes(St4, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"b">>, 1}, {<<"a">>, 2}], lists:reverse(Changes2)). + + +cet_update_one() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions = [ + {create, {<<"a">>, []}}, + {update, {<<"a">>, []}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + + ?assertEqual(1, Engine:count_changes_since(St2, 0)), + ?assertEqual({ok, [{<<"a">>, 2}]}, + Engine:fold_changes(St2, 0, fun fold_fun/2, [], [])). + + +cet_update_first_of_two() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions = [ + {create, {<<"a">>, []}}, + {create, {<<"b">>, []}}, + {update, {<<"a">>, []}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + + ?assertEqual(2, Engine:count_changes_since(St2, 0)), + {ok, Changes} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"b">>, 2}, {<<"a">>, 3}], lists:reverse(Changes)). + + +cet_update_second_of_two() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions = [ + {create, {<<"a">>, []}}, + {create, {<<"b">>, []}}, + {update, {<<"b">>, []}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + + ?assertEqual(2, Engine:count_changes_since(St2, 0)), + {ok, Changes} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"a">>, 1}, {<<"b">>, 3}], lists:reverse(Changes)). + + +cet_check_mutation_ordering() -> + Actions = shuffle(lists:map(fun(Seq) -> + {create, {docid(Seq), []}} + end, lists:seq(1, ?NUM_DOCS))), + + DocIdOrder = [DocId || {_, {DocId, _}} <- Actions], + DocSeqs = lists:zip(DocIdOrder, lists:seq(1, ?NUM_DOCS)), + + {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + + % First lets see that we can get the correct + % suffix/prefix starting at every update sequence + lists:foreach(fun(Seq) -> + {ok, Suffix} = Engine:fold_changes(St2, Seq, fun fold_fun/2, [], []), + ?assertEqual(lists:nthtail(Seq, DocSeqs), lists:reverse(Suffix)), + + {ok, Prefix} = Engine:fold_changes(St2, Seq, fun fold_fun/2, [], [ + {dir, rev} + ]), + ?assertEqual(lists:sublist(DocSeqs, Seq + 1), Prefix) + end, lists:seq(0, ?NUM_DOCS)), + + ok = do_mutation_ordering(Engine, St2, ?NUM_DOCS + 1, DocSeqs, []). + + +do_mutation_ordering(Engine, St, _Seq, [], FinalDocSeqs) -> + {ok, RevOrder} = Engine:fold_changes(St, 0, fun fold_fun/2, [], []), + ?assertEqual(FinalDocSeqs, lists:reverse(RevOrder)), + ok; + +do_mutation_ordering(Engine, St, Seq, [{DocId, _OldSeq} | Rest], DocSeqAcc) -> + Actions = [{update, {DocId, []}}], + {ok, NewSt} = test_engine_util:apply_actions(Engine, St, Actions), + NewAcc = DocSeqAcc ++ [{DocId, Seq}], + Expected = Rest ++ NewAcc, + {ok, RevOrder} = Engine:fold_changes(NewSt, 0, fun fold_fun/2, [], []), + ?assertEqual(Expected, lists:reverse(RevOrder)), + do_mutation_ordering(Engine, NewSt, Seq + 1, Rest, NewAcc). + + +shuffle(List) -> + random:seed(os:timestamp()), + Paired = [{random:uniform(), I} || I <- List], + Sorted = lists:sort(Paired), + [I || {_, I} <- Sorted]. + + +remove_random(List) -> + Pos = random:uniform(length(List)), + remove_random(Pos, List). + + +remove_random(1, [Item | Rest]) -> + {Item, Rest}; + +remove_random(N, [Skip | Rest]) when N > 1 -> + {Item, Tail} = remove_random(N - 1, Rest), + {Item, [Skip | Tail]}. + + +fold_fun(#full_doc_info{id=Id, update_seq=Seq}, Acc) -> + {ok, [{Id, Seq} | Acc]}. + + +docid(I) -> + Str = io_lib:format("~4..0b", [I]), + iolist_to_binary(Str). http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/1e1702f9/src/test_engine_fold_docs.erl ---------------------------------------------------------------------- diff --git a/src/test_engine_fold_docs.erl b/src/test_engine_fold_docs.erl new file mode 100644 index 0000000..34d7f3e --- /dev/null +++ b/src/test_engine_fold_docs.erl @@ -0,0 +1,390 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_fold_docs). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +-define(NUM_DOCS, 100). + + +cet_fold_all() -> + fold_all(fold_docs, fun docid/1). + + +cet_fold_all_local() -> + fold_all(fold_local_docs, fun local_docid/1). + + +cet_fold_start_key() -> + fold_start_key(fold_docs, fun docid/1). + + +cet_fold_start_key_local() -> + fold_start_key(fold_local_docs, fun local_docid/1). + + +cet_fold_end_key() -> + fold_end_key(fold_docs, fun docid/1). + + +cet_fold_end_key_local() -> + fold_end_key(fold_local_docs, fun local_docid/1). + + +cet_fold_end_key_gt() -> + fold_end_key_gt(fold_docs, fun docid/1). + + +cet_fold_end_key_gt_local() -> + fold_end_key_gt(fold_local_docs, fun local_docid/1). + + +cet_fold_range() -> + fold_range(fold_docs, fun docid/1). + + +cet_fold_range_local() -> + fold_range(fold_local_docs, fun local_docid/1). + + +cet_fold_stop() -> + fold_stop(fold_docs, fun docid/1). + + +cet_fold_stop_local() -> + fold_stop(fold_local_docs, fun local_docid/1). + + +% This is a loose test but we have to have this until +% I figure out what to do about the total_rows/offset +% meta data included in _all_docs +cet_fold_include_reductions() -> + {ok, Engine, St} = init_st(fun docid/1), + FoldFun = fun(_, _, nil) -> {ok, nil} end, + {ok, Count, nil} = Engine:fold_docs(St, FoldFun, nil, [include_reductions]), + ?assert(is_integer(Count)), + ?assert(Count >= 0). + + +fold_all(FoldFun, DocIdFun) -> + DocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], + {ok, Engine, St} = init_st(DocIdFun), + + {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], []), + ?assertEqual(?NUM_DOCS, length(DocIdAccFwd)), + ?assertEqual(DocIds, lists:reverse(DocIdAccFwd)), + + {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [{dir, rev}]), + ?assertEqual(?NUM_DOCS, length(DocIdAccRev)), + ?assertEqual(DocIds, DocIdAccRev). + + +fold_start_key(FoldFun, DocIdFun) -> + {ok, Engine, St} = init_st(DocIdFun), + + StartKeyNum = ?NUM_DOCS div 4, + StartKey = DocIdFun(StartKeyNum), + + AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], + DocIdsFwd = [DocIdFun(I) || I <- lists:seq(StartKeyNum, ?NUM_DOCS)], + DocIdsRev = [DocIdFun(I) || I <- lists:seq(1, StartKeyNum)], + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {start_key, <<255>>} + ])), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, <<"">>} + ])), + + {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {start_key, <<"">>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), + + {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, <<255>>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, AllDocIdAccRev), + + {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {start_key, StartKey} + ]), + ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), + ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), + + {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, StartKey} + ]), + ?assertEqual(length(DocIdsRev), length(DocIdAccRev)), + ?assertEqual(DocIdsRev, DocIdAccRev). + + +fold_end_key(FoldFun, DocIdFun) -> + {ok, Engine, St} = init_st(DocIdFun), + + EndKeyNum = ?NUM_DOCS div 4, + EndKey = DocIdFun(EndKeyNum), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {end_key, <<"">>} + ])), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {end_key, <<255>>} + ])), + + AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], + + {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {end_key, <<255>>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), + + {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {end_key, <<"">>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, AllDocIdAccRev), + + DocIdsFwd = [DocIdFun(I) || I <- lists:seq(1, EndKeyNum)], + + {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {end_key, EndKey} + ]), + ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), + ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), + + DocIdsRev = [DocIdFun(I) || I <- lists:seq(EndKeyNum, ?NUM_DOCS)], + + {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {end_key, EndKey} + ]), + ?assertEqual(length(DocIdsRev), length(DocIdAccRev)), + ?assertEqual(DocIdsRev, DocIdAccRev). + + +fold_end_key_gt(FoldFun, DocIdFun) -> + {ok, Engine, St} = init_st(DocIdFun), + + EndKeyNum = ?NUM_DOCS div 4, + EndKey = DocIdFun(EndKeyNum), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {end_key_gt, <<"">>} + ])), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {end_key_gt, <<255>>} + ])), + + AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], + + {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {end_key_gt, <<255>>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), + + {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {end_key_gt, <<"">>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, AllDocIdAccRev), + + DocIdsFwd = [DocIdFun(I) || I <- lists:seq(1, EndKeyNum - 1)], + + {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {end_key_gt, EndKey} + ]), + ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), + ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), + + DocIdsRev = [DocIdFun(I) || I <- lists:seq(EndKeyNum + 1, ?NUM_DOCS)], + + {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {end_key_gt, EndKey} + ]), + ?assertEqual(length(DocIdsRev), length(DocIdAccRev)), + ?assertEqual(DocIdsRev, DocIdAccRev). + + +fold_range(FoldFun, DocIdFun) -> + {ok, Engine, St} = init_st(DocIdFun), + + StartKeyNum = ?NUM_DOCS div 4, + EndKeyNum = StartKeyNum * 3, + + StartKey = DocIdFun(StartKeyNum), + EndKey = DocIdFun(EndKeyNum), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {start_key, <<"">>}, + {end_key, <<"">>} + ])), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, <<"">>}, + {end_key, <<255>>} + ])), + + AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], + + {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {start_key, <<"">>}, + {end_key, <<255>>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), + + {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, <<255>>}, + {end_key_gt, <<"">>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, AllDocIdAccRev), + + DocIdsFwd = [DocIdFun(I) || I <- lists:seq(StartKeyNum, EndKeyNum)], + + {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {start_key, StartKey}, + {end_key, EndKey} + ]), + ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), + ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), + + DocIdsRev = [DocIdFun(I) || I <- lists:seq(StartKeyNum, EndKeyNum)], + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, StartKey}, + {end_key, EndKey} + ])), + + {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, EndKey}, + {end_key, StartKey} + ]), + ?assertEqual(length(DocIdsRev), length(DocIdAccRev)), + ?assertEqual(DocIdsRev, DocIdAccRev). + + +fold_stop(FoldFun, DocIdFun) -> + {ok, Engine, St} = init_st(DocIdFun), + + StartKeyNum = ?NUM_DOCS div 4, + StartKey = DocIdFun(StartKeyNum), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {start_key, <<255>>} + ])), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {dir, rev}, + {start_key, <<"">>} + ])), + + SuffixDocIds = [DocIdFun(I) || I <- lists:seq(?NUM_DOCS - 3, ?NUM_DOCS)], + + {ok, SuffixDocIdAcc} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {start_key, DocIdFun(?NUM_DOCS - 3)} + ]), + ?assertEqual(length(SuffixDocIds), length(SuffixDocIdAcc)), + ?assertEqual(SuffixDocIds, lists:reverse(SuffixDocIdAcc)), + + PrefixDocIds = [DocIdFun(I) || I <- lists:seq(1, 3)], + + {ok, PrefixDocIdAcc} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {dir, rev}, + {start_key, DocIdFun(3)} + ]), + ?assertEqual(3, length(PrefixDocIdAcc)), + ?assertEqual(PrefixDocIds, PrefixDocIdAcc), + + FiveDocIdsFwd = [DocIdFun(I) + || I <- lists:seq(StartKeyNum, StartKeyNum + 5)], + + {ok, FiveDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {start_key, StartKey} + ]), + ?assertEqual(length(FiveDocIdsFwd), length(FiveDocIdAccFwd)), + ?assertEqual(FiveDocIdsFwd, lists:reverse(FiveDocIdAccFwd)), + + FiveDocIdsRev = [DocIdFun(I) + || I <- lists:seq(StartKeyNum - 5, StartKeyNum)], + + {ok, FiveDocIdAccRev} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {dir, rev}, + {start_key, StartKey} + ]), + ?assertEqual(length(FiveDocIdsRev), length(FiveDocIdAccRev)), + ?assertEqual(FiveDocIdsRev, FiveDocIdAccRev). + + +init_st(DocIdFun) -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions = lists:map(fun(Id) -> + {create, {DocIdFun(Id), [{<<"int">>, Id}]}} + end, lists:seq(1, ?NUM_DOCS)), + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, Engine, St2}. + + +fold_fun(Doc, Acc) -> + Id = case Doc of + #doc{id = Id0} -> Id0; + #full_doc_info{id = Id0} -> Id0 + end, + {ok, [Id | Acc]}. + + +fold_fun_stop(Doc, Acc) -> + Id = case Doc of + #doc{id = Id0} -> Id0; + #full_doc_info{id = Id0} -> Id0 + end, + case length(Acc) of + N when N =< 4 -> + {ok, [Id | Acc]}; + _ -> + {stop, [Id | Acc]} + end. + + +docid(I) -> + Str = io_lib:format("~4..0b", [I]), + iolist_to_binary(Str). + + +local_docid(I) -> + Str = io_lib:format("_local/~4..0b", [I]), + iolist_to_binary(Str). http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/1e1702f9/src/test_engine_get_set_props.erl ---------------------------------------------------------------------- diff --git a/src/test_engine_get_set_props.erl b/src/test_engine_get_set_props.erl new file mode 100644 index 0000000..6d2a447 --- /dev/null +++ b/src/test_engine_get_set_props.erl @@ -0,0 +1,70 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_get_set_props). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). + + +cet_default_props() -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + {ok, St} = Engine:init(DbPath, [ + create, + {default_security_object, dso} + ]), + + Node = node(), + + ?assertEqual(0, Engine:get_doc_count(St)), + ?assertEqual(0, Engine:get_del_doc_count(St)), + ?assertEqual(true, is_list(Engine:get_size_info(St))), + ?assertEqual(true, is_integer(Engine:get_disk_version(St))), + ?assertEqual(0, Engine:get_update_seq(St)), + ?assertEqual(0, Engine:get_purge_seq(St)), + ?assertEqual([], Engine:get_last_purged(St)), + ?assertEqual(dso, Engine:get_security(St)), + ?assertEqual(1000, Engine:get_revs_limit(St)), + ?assertMatch(<<_:32/binary>>, Engine:get_uuid(St)), + ?assertEqual([{Node, 0}], Engine:get_epochs(St)), + ?assertEqual(0, Engine:get_compacted_seq(St)). + + +cet_set_security() -> + check_prop_set(get_security, set_security, dso, [{<<"readers">>, []}]). + + +cet_set_revs_limit() -> + check_prop_set(get_revs_limit, set_revs_limit, 1000, 50). + + +check_prop_set(GetFun, SetFun, Default, Value) -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + {ok, St0} = Engine:init(DbPath, [ + create, + {default_security_object, dso} + ]), + ?assertEqual(Default, Engine:GetFun(St0)), + + {ok, St1} = Engine:SetFun(St0, Value), + ?assertEqual(Value, Engine:GetFun(St1)), + + {ok, St2} = Engine:commit_data(St1), + Engine:terminate(normal, St2), + + {ok, St3} = Engine:init(DbPath, []), + ?assertEqual(Value, Engine:GetFun(St3)). http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/1e1702f9/src/test_engine_open_close_delete.erl ---------------------------------------------------------------------- diff --git a/src/test_engine_open_close_delete.erl b/src/test_engine_open_close_delete.erl new file mode 100644 index 0000000..b099d9f --- /dev/null +++ b/src/test_engine_open_close_delete.erl @@ -0,0 +1,81 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_open_close_delete). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). + + +cet_open_non_existent() -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + ?assertEqual(false, Engine:exists(DbPath)), + ?assertThrow({not_found, no_db_file}, Engine:init(DbPath, [])), + ?assertEqual(false, Engine:exists(DbPath)). + + +cet_open_create() -> + process_flag(trap_exit, true), + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + ?assertEqual(false, Engine:exists(DbPath)), + ?assertMatch({ok, _}, Engine:init(DbPath, [create])), + ?assertEqual(true, Engine:exists(DbPath)). + + +cet_open_when_exists() -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + ?assertEqual(false, Engine:exists(DbPath)), + ?assertMatch({ok, _}, Engine:init(DbPath, [create])), + ?assertThrow({error, eexist}, Engine:init(DbPath, [create])). + + +cet_terminate() -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + ?assertEqual(false, Engine:exists(DbPath)), + {ok, St} = Engine:init(DbPath, [create]), + Engine:terminate(normal, St), + ?assertEqual(true, Engine:exists(DbPath)). + + +cet_rapid_recycle() -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + {ok, St0} = Engine:init(DbPath, [create]), + Engine:terminate(normal, St0), + + lists:foreach(fun(_) -> + {ok, St1} = Engine:init(DbPath, []), + Engine:terminate(normal, St1) + end, lists:seq(1, 100)). + + +cet_delete() -> + Engine = test_engine_util:get_engine(), + RootDir = test_engine_util:rootdir(), + DbPath = test_engine_util:dbpath(), + + ?assertEqual(false, Engine:exists(DbPath)), + {ok, St} = Engine:init(DbPath, [create]), + Engine:terminate(normal, St), + ?assertEqual(true, Engine:exists(DbPath)), + ?assertEqual(ok, Engine:delete(RootDir, DbPath, [async])), + ?assertEqual(false, Engine:exists(DbPath)). http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/1e1702f9/src/test_engine_purge_docs.erl ---------------------------------------------------------------------- diff --git a/src/test_engine_purge_docs.erl b/src/test_engine_purge_docs.erl new file mode 100644 index 0000000..e5bf249 --- /dev/null +++ b/src/test_engine_purge_docs.erl @@ -0,0 +1,158 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_purge_docs). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +cet_purge_simple() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + + Actions1 = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + + ?assertEqual(1, Engine:get_doc_count(St2)), + ?assertEqual(0, Engine:get_del_doc_count(St2)), + ?assertEqual(1, Engine:get_update_seq(St2)), + ?assertEqual(0, Engine:get_purge_seq(St2)), + ?assertEqual([], Engine:get_last_purged(St2)), + + [FDI] = Engine:open_docs(St2, [<<"foo">>]), + PrevRev = test_engine_util:prev_rev(FDI), + Rev = PrevRev#rev_info.rev, + + Actions2 = [ + {purge, {<<"foo">>, Rev}} + ], + {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), + + ?assertEqual(0, Engine:get_doc_count(St3)), + ?assertEqual(0, Engine:get_del_doc_count(St3)), + ?assertEqual(2, Engine:get_update_seq(St3)), + ?assertEqual(1, Engine:get_purge_seq(St3)), + ?assertEqual([{<<"foo">>, [Rev]}], Engine:get_last_purged(St3)). + + +cet_purge_conflicts() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + + Actions1 = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {conflict, {<<"foo">>, [{<<"vsn">>, 2}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + + ?assertEqual(1, Engine:get_doc_count(St2)), + ?assertEqual(0, Engine:get_del_doc_count(St2)), + ?assertEqual(2, Engine:get_update_seq(St2)), + ?assertEqual(0, Engine:get_purge_seq(St2)), + ?assertEqual([], Engine:get_last_purged(St2)), + + [FDI1] = Engine:open_docs(St2, [<<"foo">>]), + PrevRev1 = test_engine_util:prev_rev(FDI1), + Rev1 = PrevRev1#rev_info.rev, + + Actions2 = [ + {purge, {<<"foo">>, Rev1}} + ], + {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), + + ?assertEqual(1, Engine:get_doc_count(St3)), + ?assertEqual(0, Engine:get_del_doc_count(St3)), + ?assertEqual(4, Engine:get_update_seq(St3)), + ?assertEqual(1, Engine:get_purge_seq(St3)), + ?assertEqual([{<<"foo">>, [Rev1]}], Engine:get_last_purged(St3)), + + [FDI2] = Engine:open_docs(St3, [<<"foo">>]), + PrevRev2 = test_engine_util:prev_rev(FDI2), + Rev2 = PrevRev2#rev_info.rev, + + Actions3 = [ + {purge, {<<"foo">>, Rev2}} + ], + {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions3), + + ?assertEqual(0, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(5, Engine:get_update_seq(St4)), + ?assertEqual(2, Engine:get_purge_seq(St4)), + ?assertEqual([{<<"foo">>, [Rev2]}], Engine:get_last_purged(St4)). + + +cet_add_delete_purge() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + + Actions1 = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {delete, {<<"foo">>, [{<<"vsn">>, 2}]}} + ], + + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + + ?assertEqual(0, Engine:get_doc_count(St2)), + ?assertEqual(1, Engine:get_del_doc_count(St2)), + ?assertEqual(2, Engine:get_update_seq(St2)), + ?assertEqual(0, Engine:get_purge_seq(St2)), + ?assertEqual([], Engine:get_last_purged(St2)), + + [FDI] = Engine:open_docs(St2, [<<"foo">>]), + PrevRev = test_engine_util:prev_rev(FDI), + Rev = PrevRev#rev_info.rev, + + Actions2 = [ + {purge, {<<"foo">>, Rev}} + ], + {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), + + ?assertEqual(0, Engine:get_doc_count(St3)), + ?assertEqual(0, Engine:get_del_doc_count(St3)), + ?assertEqual(3, Engine:get_update_seq(St3)), + ?assertEqual(1, Engine:get_purge_seq(St3)), + ?assertEqual([{<<"foo">>, [Rev]}], Engine:get_last_purged(St3)). + + +cet_add_two_purge_one() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + + Actions1 = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {create, {<<"bar">>, []}} + ], + + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + + ?assertEqual(2, Engine:get_doc_count(St2)), + ?assertEqual(0, Engine:get_del_doc_count(St2)), + ?assertEqual(2, Engine:get_update_seq(St2)), + ?assertEqual(0, Engine:get_purge_seq(St2)), + ?assertEqual([], Engine:get_last_purged(St2)), + + [FDI] = Engine:open_docs(St2, [<<"foo">>]), + PrevRev = test_engine_util:prev_rev(FDI), + Rev = PrevRev#rev_info.rev, + + Actions2 = [ + {purge, {<<"foo">>, Rev}} + ], + {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), + + ?assertEqual(1, Engine:get_doc_count(St3)), + ?assertEqual(0, Engine:get_del_doc_count(St3)), + ?assertEqual(3, Engine:get_update_seq(St3)), + ?assertEqual(1, Engine:get_purge_seq(St3)), + ?assertEqual([{<<"foo">>, [Rev]}], Engine:get_last_purged(St3)). http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/1e1702f9/src/test_engine_read_write_docs.erl ---------------------------------------------------------------------- diff --git a/src/test_engine_read_write_docs.erl b/src/test_engine_read_write_docs.erl new file mode 100644 index 0000000..4307702 --- /dev/null +++ b/src/test_engine_read_write_docs.erl @@ -0,0 +1,317 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_read_write_docs). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +cet_read_empty_docs() -> + {ok, Engine, St} = test_engine_util:init_engine(), + + ?assertEqual([not_found], Engine:open_docs(St, [<<"foo">>])), + ?assertEqual( + [not_found, not_found], + Engine:open_docs(St, [<<"a">>, <<"b">>]) + ). + + +cet_read_empty_local_docs() -> + {ok, Engine, St} = test_engine_util:init_engine(), + + ?assertEqual([not_found], Engine:open_local_docs(St, [<<"_local/foo">>])), + ?assertEqual( + [not_found, not_found], + Engine:open_local_docs(St, [<<"_local/a">>, <<"_local/b">>]) + ). + + +cet_write_one_doc() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(1, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(1, Engine:get_update_seq(St4)), + + [FDI] = Engine:open_docs(St4, [<<"foo">>]), + #rev_info{ + rev = {RevPos, PrevRevId}, + deleted = Deleted, + body_sp = DocPtr + } = test_engine_util:prev_rev(FDI), + + Doc0 = #doc{ + id = <<"foo">>, + revs = {RevPos, [PrevRevId]}, + deleted = Deleted, + body = DocPtr + }, + + Doc1 = Engine:read_doc_body(St4, Doc0), + Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> + couch_compress:decompress(Doc1#doc.body) + end, + ?assertEqual([{<<"vsn">>, 1}], Body1). + + +cet_write_two_docs() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {create, {<<"bar">>, [{<<"stuff">>, true}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(2, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(2, Engine:get_update_seq(St4)), + + Resps = Engine:open_docs(St4, [<<"foo">>, <<"bar">>]), + ?assertEqual(false, lists:member(not_found, Resps)). + + +cet_write_three_doc_batch() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {batch, [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {create, {<<"bar">>, [{<<"stuff">>, true}]}}, + {create, {<<"baz">>, []}} + ]} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(3, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(3, Engine:get_update_seq(St4)), + + Resps = Engine:open_docs(St4, [<<"foo">>, <<"bar">>, <<"baz">>]), + ?assertEqual(false, lists:member(not_found, Resps)). + + +cet_update_doc() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {update, {<<"foo">>, [{<<"vsn">>, 2}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(1, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(2, Engine:get_update_seq(St4)), + + [FDI] = Engine:open_docs(St4, [<<"foo">>]), + + #rev_info{ + rev = {RevPos, PrevRevId}, + deleted = Deleted, + body_sp = DocPtr + } = test_engine_util:prev_rev(FDI), + + Doc0 = #doc{ + id = <<"foo">>, + revs = {RevPos, [PrevRevId]}, + deleted = Deleted, + body = DocPtr + }, + + Doc1 = Engine:read_doc_body(St4, Doc0), + Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> + couch_compress:decompress(Doc1#doc.body) + end, + + ?assertEqual([{<<"vsn">>, 2}], Body1). + + +cet_delete_doc() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {delete, {<<"foo">>, []}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(0, Engine:get_doc_count(St4)), + ?assertEqual(1, Engine:get_del_doc_count(St4)), + ?assertEqual(2, Engine:get_update_seq(St4)), + + [FDI] = Engine:open_docs(St4, [<<"foo">>]), + + #rev_info{ + rev = {RevPos, PrevRevId}, + deleted = Deleted, + body_sp = DocPtr + } = test_engine_util:prev_rev(FDI), + + Doc0 = #doc{ + id = <<"foo">>, + revs = {RevPos, [PrevRevId]}, + deleted = Deleted, + body = DocPtr + }, + + Doc1 = Engine:read_doc_body(St4, Doc0), + Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> + couch_compress:decompress(Doc1#doc.body) + end, + + ?assertEqual([], Body1). + + +cet_write_local_doc() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"_local/foo">>, [{<<"yay">>, false}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(0, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(0, Engine:get_update_seq(St4)), + + [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), + [#doc{} = Doc] = Engine:open_local_docs(St4, [<<"_local/foo">>]), + ?assertEqual([{<<"yay">>, false}], Doc#doc.body). + + +cet_write_mixed_batch() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {batch, [ + {create, {<<"bar">>, []}}, + {create, {<<"_local/foo">>, [{<<"yay">>, false}]}} + ]} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(1, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(1, Engine:get_update_seq(St4)), + + [#full_doc_info{}] = Engine:open_docs(St4, [<<"bar">>]), + [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), + + [not_found] = Engine:open_local_docs(St4, [<<"bar">>]), + [#doc{}] = Engine:open_local_docs(St4, [<<"_local/foo">>]). + + +cet_update_local_doc() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"_local/foo">>, []}}, + {update, {<<"_local/foo">>, [{<<"stuff">>, null}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(0, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(0, Engine:get_update_seq(St4)), + + [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), + [#doc{} = Doc] = Engine:open_local_docs(St4, [<<"_local/foo">>]), + ?assertEqual([{<<"stuff">>, null}], Doc#doc.body). + + +cet_delete_local_doc() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"_local/foo">>, []}}, + {delete, {<<"_local/foo">>, []}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(0, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(0, Engine:get_update_seq(St4)), + + [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), + ?assertEqual([not_found], Engine:open_local_docs(St4, [<<"_local/foo">>])). http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/1e1702f9/src/test_engine_ref_counting.erl ---------------------------------------------------------------------- diff --git a/src/test_engine_ref_counting.erl b/src/test_engine_ref_counting.erl new file mode 100644 index 0000000..5e60276 --- /dev/null +++ b/src/test_engine_ref_counting.erl @@ -0,0 +1,112 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_ref_counting). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +-define(NUM_CLIENTS, 1000). + + +cet_empty_monitors() -> + {ok, Engine, St} = test_engine_util:init_engine(), + Pids = Engine:monitored_by(St), + ?assert(is_list(Pids)), + ?assertEqual([], Pids -- [self(), whereis(couch_stats_process_tracker)]). + + +cet_test_system_db() -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + {ok, St} = Engine:init(DbPath, [create, sys_db]), + Pids = Engine:monitored_by(St), + ?assertEqual(1, length(Pids)). + + +cet_incref_decref() -> + {ok, Engine, St} = test_engine_util:init_engine(), + + {Pid, _} = Client = start_client(Engine, St), + wait_client(Client), + + Pids1 = Engine:monitored_by(St), + ?assert(lists:member(Pid, Pids1)), + + close_client(Client), + + Pids2 = Engine:monitored_by(St), + ?assert(not lists:member(Pid, Pids2)). + + +cet_incref_decref_many() -> + {ok, Engine, St} = test_engine_util:init_engine(), + Clients = lists:map(fun(_) -> + start_client(Engine, St) + end, lists:seq(1, ?NUM_CLIENTS)), + + lists:foreach(fun(C) -> wait_client(C) end, Clients), + + Pids1 = Engine:monitored_by(St), + % +2 for db pid and process tracker + ?assertEqual(?NUM_CLIENTS + 2, length(Pids1)), + + lists:foreach(fun(C) -> close_client(C) end, Clients), + + Pids2 = Engine:monitored_by(St), + ?assertEqual(2, length(Pids2)). + + +start_client(Engine, St1) -> + spawn_monitor(fun() -> + {ok, St2} = Engine:incref(St1), + + receive + {waiting, Pid} -> + Pid ! go + after 1000 -> + erlang:error(timeout) + end, + + receive + close -> + ok + after 1000 -> + erlang:error(timeout) + end, + + Engine:decref(St2) + end). + + +wait_client({Pid, _Ref}) -> + Pid ! {waiting, self()}, + receive + go -> ok + after 1000 -> + erlang:error(timeout) + end. + + +close_client({Pid, Ref}) -> + Pid ! close, + receive + {'DOWN', Ref, _, _, _} -> + ok + after 1000 -> + erlang:error(timeout) + end. + http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/1e1702f9/src/test_engine_util.erl ---------------------------------------------------------------------- diff --git a/src/test_engine_util.erl b/src/test_engine_util.erl new file mode 100644 index 0000000..33048d3 --- /dev/null +++ b/src/test_engine_util.erl @@ -0,0 +1,602 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_util). +-compile(export_all). + + +-include_lib("couch/include/couch_db.hrl"). + + +-define(TEST_MODULES, [ + test_engine_open_close_delete, + test_engine_get_set_props, + test_engine_read_write_docs, + test_engine_attachments, + test_engine_fold_docs, + test_engine_fold_changes, + test_engine_purge_docs, + test_engine_compaction, + test_engine_ref_counting +]). + + +create_tests(EngineApp) -> + create_tests(EngineApp, EngineApp). + + +create_tests(EngineApp, EngineModule) -> + application:set_env(couch, test_engine, {EngineApp, EngineModule}), + Tests = lists:map(fun(TestMod) -> + {atom_to_list(TestMod), gather(TestMod)} + end, ?TEST_MODULES), + Setup = fun() -> + Ctx = test_util:start_couch(), + config:set("log", "include_sasl", "false", false), + Ctx + end, + { + setup, + Setup, + fun test_util:stop_couch/1, + fun(_) -> Tests end + }. + + +gather(Module) -> + Exports = Module:module_info(exports), + Tests = lists:foldl(fun({Fun, Arity}, Acc) -> + case {atom_to_list(Fun), Arity} of + {[$c, $e, $t, $_ | _], 0} -> + TestFun = make_test_fun(Module, Fun), + [{spawn, TestFun} | Acc]; + _ -> + Acc + end + end, [], Exports), + lists:reverse(Tests). + + +make_test_fun(Module, Fun) -> + Name = lists:flatten(io_lib:format("~s:~s", [Module, Fun])), + Wrapper = fun() -> + process_flag(trap_exit, true), + Module:Fun() + end, + {Name, Wrapper}. + +rootdir() -> + config:get("couchdb", "database_dir", "."). + + +dbpath() -> + binary_to_list(filename:join(rootdir(), couch_uuids:random())). + + +get_engine() -> + case application:get_env(couch, test_engine) of + {ok, {_, Engine}} -> + Engine; + _ -> + couch_bt_engine + end. + + +init_engine() -> + init_engine(default). + + +init_engine(default) -> + Engine = get_engine(), + DbPath = dbpath(), + {ok, St} = Engine:init(DbPath, [ + create, + {default_security_object, []} + ]), + {ok, Engine, St}; + +init_engine(dbpath) -> + Engine = get_engine(), + DbPath = dbpath(), + {ok, St} = Engine:init(DbPath, [ + create, + {default_security_object, []} + ]), + {ok, Engine, DbPath, St}. + + +apply_actions(_Engine, St, []) -> + {ok, St}; + +apply_actions(Engine, St, [Action | Rest]) -> + NewSt = apply_action(Engine, St, Action), + apply_actions(Engine, NewSt, Rest). + + +apply_action(Engine, St, {batch, BatchActions}) -> + apply_batch(Engine, St, BatchActions); + +apply_action(Engine, St, Action) -> + apply_batch(Engine, St, [Action]). + + +apply_batch(Engine, St, Actions) -> + UpdateSeq = Engine:get_update_seq(St) + 1, + AccIn = {UpdateSeq, [], [], []}, + AccOut = lists:foldl(fun(Action, Acc) -> + {SeqAcc, DocAcc, LDocAcc, PurgeAcc} = Acc, + case Action of + {_, {<<"_local/", _/binary>>, _}} -> + LDoc = gen_local_write(Engine, St, Action), + {SeqAcc, DocAcc, [LDoc | LDocAcc], PurgeAcc}; + _ -> + case gen_write(Engine, St, Action, SeqAcc) of + {_OldFDI, _NewFDI} = Pair -> + {SeqAcc + 1, [Pair | DocAcc], LDocAcc, PurgeAcc}; + {Pair, NewSeqAcc, NewPurgeInfo} -> + NewPurgeAcc = [NewPurgeInfo | PurgeAcc], + {NewSeqAcc, [Pair | DocAcc], LDocAcc, NewPurgeAcc} + end + end + end, AccIn, Actions), + {_, Docs0, LDocs, PurgeIdRevs} = AccOut, + Docs = lists:reverse(Docs0), + {ok, NewSt} = Engine:write_doc_infos(St, Docs, LDocs, PurgeIdRevs), + NewSt. + + +gen_local_write(Engine, St, {Action, {DocId, Body}}) -> + PrevRev = case Engine:open_local_docs(St, [DocId]) of + [not_found] -> + 0; + [#doc{revs = {0, []}}] -> + 0; + [#doc{revs = {0, [RevStr | _]}}] -> + list_to_integer(binary_to_list(RevStr)) + end, + {RevId, Deleted} = case Action of + Action when Action == create; Action == update -> + {list_to_binary(integer_to_list(PrevRev + 1)), false}; + delete -> + {<<"0">>, true} + end, + #doc{ + id = DocId, + revs = {0, [RevId]}, + body = Body, + deleted = Deleted + }. + +gen_write(Engine, St, {Action, {DocId, Body}}, UpdateSeq) -> + gen_write(Engine, St, {Action, {DocId, Body, []}}, UpdateSeq); + +gen_write(Engine, St, {create, {DocId, Body, Atts0}}, UpdateSeq) -> + [not_found] = Engine:open_docs(St, [DocId]), + Atts = [couch_att:to_disk_term(Att) || Att <- Atts0], + + Rev = crypto:hash(md5, term_to_binary({DocId, Body, Atts})), + + Doc0 = #doc{ + id = DocId, + revs = {0, [Rev]}, + deleted = false, + body = Body, + atts = Atts + }, + + Doc1 = make_doc_summary(Engine, St, Doc0), + {ok, Doc2, Len} = Engine:write_doc_body(St, Doc1), + + Sizes = #size_info{ + active = Len, + external = erlang:external_size(Doc1#doc.body) + }, + + Leaf = #leaf{ + deleted = false, + ptr = Doc2#doc.body, + seq = UpdateSeq, + sizes = Sizes, + atts = Atts + }, + + {not_found, #full_doc_info{ + id = DocId, + deleted = false, + update_seq = UpdateSeq, + rev_tree = [{0, {Rev, Leaf, []}}], + sizes = Sizes + }}; + +gen_write(Engine, St, {purge, {DocId, PrevRevs0, _}}, UpdateSeq) -> + [#full_doc_info{} = PrevFDI] = Engine:open_docs(St, [DocId]), + PrevRevs = if is_list(PrevRevs0) -> PrevRevs0; true -> [PrevRevs0] end, + + #full_doc_info{ + rev_tree = PrevTree + } = PrevFDI, + + {NewTree, RemRevs} = couch_key_tree:remove_leafs(PrevTree, PrevRevs), + RemovedAll = lists:sort(RemRevs) == lists:sort(PrevRevs), + if RemovedAll -> ok; true -> + % If we didn't purge all the requested revisions + % then its a bug in the test. + erlang:error({invalid_purge_test_revs, PrevRevs}) + end, + + case NewTree of + [] -> + % We've completely purged the document + {{PrevFDI, not_found}, UpdateSeq, {DocId, RemRevs}}; + _ -> + % We have to relabel the update_seq of all + % leaves. See couch_db_updater for details. + {NewNewTree, NewUpdateSeq} = couch_key_tree:mapfold(fun + (_RevId, Leaf, leaf, InnerSeqAcc) -> + {Leaf#leaf{seq = InnerSeqAcc}, InnerSeqAcc + 1}; + (_RevId, Value, _Type, InnerSeqAcc) -> + {Value, InnerSeqAcc} + end, UpdateSeq, NewTree), + NewFDI = PrevFDI#full_doc_info{ + update_seq = NewUpdateSeq - 1, + rev_tree = NewNewTree + }, + {{PrevFDI, NewFDI}, NewUpdateSeq, {DocId, RemRevs}} + end; + +gen_write(Engine, St, {Action, {DocId, Body, Atts0}}, UpdateSeq) -> + [#full_doc_info{} = PrevFDI] = Engine:open_docs(St, [DocId]), + Atts = [couch_att:to_disk_term(Att) || Att <- Atts0], + + #full_doc_info{ + id = DocId, + rev_tree = PrevRevTree + } = PrevFDI, + + #rev_info{ + rev = PrevRev + } = prev_rev(PrevFDI), + + {RevPos, PrevRevId} = PrevRev, + + Rev = gen_revision(Action, DocId, PrevRev, Body, Atts), + + Doc0 = #doc{ + id = DocId, + revs = {RevPos + 1, [Rev, PrevRevId]}, + deleted = false, + body = Body, + atts = Atts + }, + + Doc1 = make_doc_summary(Engine, St, Doc0), + {ok, Doc2, Len} = Engine:write_doc_body(St, Doc1), + + Deleted = case Action of + update -> false; + conflict -> false; + delete -> true + end, + + Sizes = #size_info{ + active = Len, + external = erlang:external_size(Doc1#doc.body) + }, + + Leaf = #leaf{ + deleted = Deleted, + ptr = Doc2#doc.body, + seq = UpdateSeq, + sizes = Sizes, + atts = Atts + }, + + Path = gen_path(Action, RevPos, PrevRevId, Rev, Leaf), + RevsLimit = Engine:get_revs_limit(St), + NodeType = case Action of + conflict -> new_branch; + _ -> new_leaf + end, + {NewTree, NodeType} = couch_key_tree:merge(PrevRevTree, Path, RevsLimit), + + NewFDI = PrevFDI#full_doc_info{ + deleted = couch_doc:is_deleted(NewTree), + update_seq = UpdateSeq, + rev_tree = NewTree, + sizes = Sizes + }, + + {PrevFDI, NewFDI}. + + +gen_revision(conflict, DocId, _PrevRev, Body, Atts) -> + crypto:hash(md5, term_to_binary({DocId, Body, Atts})); +gen_revision(delete, DocId, PrevRev, Body, Atts) -> + gen_revision(update, DocId, PrevRev, Body, Atts); +gen_revision(update, DocId, PrevRev, Body, Atts) -> + crypto:hash(md5, term_to_binary({DocId, PrevRev, Body, Atts})). + + +gen_path(conflict, _RevPos, _PrevRevId, Rev, Leaf) -> + {0, {Rev, Leaf, []}}; +gen_path(delete, RevPos, PrevRevId, Rev, Leaf) -> + gen_path(update, RevPos, PrevRevId, Rev, Leaf); +gen_path(update, RevPos, PrevRevId, Rev, Leaf) -> + {RevPos, {PrevRevId, ?REV_MISSING, [{Rev, Leaf, []}]}}. + + +make_doc_summary(Engine, St, DocData) -> + {_, Ref} = spawn_monitor(fun() -> + exit({result, Engine:serialize_doc(St, DocData)}) + end), + receive + {'DOWN', Ref, _, _, {result, Summary}} -> + Summary; + {'DOWN', Ref, _, _, Error} -> + erlang:error({make_doc_summary_error, Error}) + after 1000 -> + erlang:error(make_doc_summary_timeout) + end. + + +prep_atts(_Engine, _St, []) -> + []; + +prep_atts(Engine, St, [{FileName, Data} | Rest]) -> + {_, Ref} = spawn_monitor(fun() -> + {ok, Stream} = Engine:open_write_stream(St, []), + exit(write_att(Stream, FileName, Data, Data)) + end), + Att = receive + {'DOWN', Ref, _, _, Resp} -> + Resp + after 5000 -> + erlang:error(attachment_write_timeout) + end, + [Att | prep_atts(Engine, St, Rest)]. + + +write_att(Stream, FileName, OrigData, <<>>) -> + {StreamEngine, Len, Len, Md5, Md5} = couch_stream:close(Stream), + couch_util:check_md5(Md5, crypto:hash(md5, OrigData)), + Len = size(OrigData), + couch_att:new([ + {name, FileName}, + {type, <<"application/octet-stream">>}, + {data, {stream, StreamEngine}}, + {att_len, Len}, + {disk_len, Len}, + {md5, Md5}, + {encoding, identity} + ]); + +write_att(Stream, FileName, OrigData, Data) -> + {Chunk, Rest} = case size(Data) > 4096 of + true -> + <> = Data, + {Head, Tail}; + false -> + {Data, <<>>} + end, + ok = couch_stream:write(Stream, Chunk), + write_att(Stream, FileName, OrigData, Rest). + + +prev_rev(#full_doc_info{} = FDI) -> + #doc_info{ + revs = [#rev_info{} = PrevRev | _] + } = couch_doc:to_doc_info(FDI), + PrevRev. + + +db_as_term(Engine, St) -> + [ + {props, db_props_as_term(Engine, St)}, + {docs, db_docs_as_term(Engine, St)}, + {local_docs, db_local_docs_as_term(Engine, St)}, + {changes, db_changes_as_term(Engine, St)} + ]. + + +db_props_as_term(Engine, St) -> + Props = [ + get_doc_count, + get_del_doc_count, + get_disk_version, + get_update_seq, + get_purge_seq, + get_last_purged, + get_security, + get_revs_limit, + get_uuid, + get_epochs + ], + lists:map(fun(Fun) -> + {Fun, Engine:Fun(St)} + end, Props). + + +db_docs_as_term(Engine, St) -> + FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, + {ok, FDIs} = Engine:fold_docs(St, FoldFun, [], []), + lists:reverse(lists:map(fun(FDI) -> + fdi_to_term(Engine, St, FDI) + end, FDIs)). + + +db_local_docs_as_term(Engine, St) -> + FoldFun = fun(Doc, Acc) -> {ok, [Doc | Acc]} end, + {ok, LDocs} = Engine:fold_local_docs(St, FoldFun, [], []), + lists:reverse(LDocs). + + +db_changes_as_term(Engine, St) -> + FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, + {ok, Changes} = Engine:fold_changes(St, 0, FoldFun, [], []), + lists:reverse(lists:map(fun(FDI) -> + fdi_to_term(Engine, St, FDI) + end, Changes)). + + +fdi_to_term(Engine, St, FDI) -> + #full_doc_info{ + id = DocId, + rev_tree = OldTree + } = FDI, + {NewRevTree, _} = couch_key_tree:mapfold(fun(Rev, Node, Type, Acc) -> + tree_to_term(Rev, Node, Type, Acc, DocId) + end, {Engine, St}, OldTree), + FDI#full_doc_info{ + rev_tree = NewRevTree, + % Blank out sizes because we allow storage + % engines to handle this with their own + % definition until further notice. + sizes = #size_info{ + active = -1, + external = -1 + } + }. + + +tree_to_term(_Rev, _Leaf, branch, Acc, _DocId) -> + {?REV_MISSING, Acc}; + +tree_to_term({Pos, RevId}, #leaf{} = Leaf, leaf, {Engine, St}, DocId) -> + #leaf{ + deleted = Deleted, + ptr = Ptr + } = Leaf, + + Doc0 = #doc{ + id = DocId, + revs = {Pos, [RevId]}, + deleted = Deleted, + body = Ptr + }, + + Doc1 = Engine:read_doc_body(St, Doc0), + + Body = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> + couch_compress:decompress(Doc1#doc.body) + end, + + Atts1 = if not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; true -> + couch_compress:decompress(Doc1#doc.atts) + end, + + StreamSrc = fun(Sp) -> Engine:open_read_stream(St, Sp) end, + Atts2 = [couch_att:from_disk_term(StreamSrc, Att) || Att <- Atts1], + Atts = [att_to_term(Att) || Att <- Atts2], + + NewLeaf = Leaf#leaf{ + ptr = Body, + sizes = #size_info{active = -1, external = -1}, + atts = Atts + }, + {NewLeaf, {Engine, St}}. + + +att_to_term(Att) -> + Bin = couch_att:to_binary(Att), + couch_att:store(data, Bin, Att). + + +term_diff(T1, T2) when is_tuple(T1), is_tuple(T2) -> + tuple_diff(tuple_to_list(T1), tuple_to_list(T2)); + +term_diff(L1, L2) when is_list(L1), is_list(L2) -> + list_diff(L1, L2); + +term_diff(V1, V2) when V1 == V2 -> + nodiff; + +term_diff(V1, V2) -> + {V1, V2}. + + +tuple_diff([], []) -> + nodiff; + +tuple_diff([T1 | _], []) -> + {longer, T1}; + +tuple_diff([], [T2 | _]) -> + {shorter, T2}; + +tuple_diff([T1 | R1], [T2 | R2]) -> + case term_diff(T1, T2) of + nodiff -> + tuple_diff(R1, R2); + Else -> + {T1, Else} + end. + + +list_diff([], []) -> + nodiff; + +list_diff([T1 | _], []) -> + {longer, T1}; + +list_diff([], [T2 | _]) -> + {shorter, T2}; + +list_diff([T1 | R1], [T2 | R2]) -> + case term_diff(T1, T2) of + nodiff -> + list_diff(R1, R2); + Else -> + {T1, Else} + end. + + +compact(Engine, St1, DbPath) -> + DbName = filename:basename(DbPath), + {ok, St2, Pid} = Engine:start_compaction(St1, DbName, [], self()), + Ref = erlang:monitor(process, Pid), + + % Ideally I'd assert that Pid is linked to us + % at this point but its technically possible + % that it could have finished compacting by + % the time we check... Quite the quandry. + + Term = receive + {'$gen_cast', {compact_done, Engine, Term0}} -> + Term0; + {'DOWN', Ref, _, _, Reason} -> + erlang:error({compactor_died, Reason}) + after 10000 -> + erlang:error(compactor_timed_out) + end, + + {ok, St2, DbName, Pid, Term}. + + +with_config(Config, Fun) -> + OldConfig = apply_config(Config), + try + Fun() + after + apply_config(OldConfig) + end. + + +apply_config([]) -> + []; + +apply_config([{Section, Key, Value} | Rest]) -> + Orig = config:get(Section, Key), + case Value of + undefined -> config:delete(Section, Key); + _ -> config:set(Section, Key, Value) + end, + [{Section, Key, Orig} | apply_config(Rest)]. http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/1e1702f9/test/couch_bt_engine_tests.erl ---------------------------------------------------------------------- diff --git a/test/couch_bt_engine_tests.erl b/test/couch_bt_engine_tests.erl new file mode 100644 index 0000000..df200df --- /dev/null +++ b/test/couch_bt_engine_tests.erl @@ -0,0 +1,20 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_bt_engine_tests). + + +-include_lib("eunit/include/eunit.hrl"). + + +couch_bt_engine_test_()-> + test_engine_util:create_tests(couch, couch_bt_engine).