From commits-return-29428-apmail-couchdb-commits-archive=couchdb.apache.org@couchdb.apache.org Sat May 13 03:27:11 2017 Return-Path: X-Original-To: apmail-couchdb-commits-archive@www.apache.org Delivered-To: apmail-couchdb-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2218118577 for ; Sat, 13 May 2017 03:27:11 +0000 (UTC) Received: (qmail 5277 invoked by uid 500); 13 May 2017 03:27:11 -0000 Delivered-To: apmail-couchdb-commits-archive@couchdb.apache.org Received: (qmail 5218 invoked by uid 500); 13 May 2017 03:27:10 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 5207 invoked by uid 99); 13 May 2017 03:27:10 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 13 May 2017 03:27:10 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 23E54849A4; Sat, 13 May 2017 03:27:08 +0000 (UTC) Date: Sat, 13 May 2017 03:27:12 +0000 To: "commits@couchdb.apache.org" Subject: [couchdb] 04/10: Add storage engine test suite MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: davisp@apache.org Reply-To: "commits@couchdb.apache.org" In-Reply-To: <149464602842.13623.10675101182473878611@gitbox.apache.org> References: <149464602842.13623.10675101182473878611@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: couchdb X-Git-Refname: refs/heads/COUCHDB-3287-pluggable-storage-engines X-Git-Reftype: branch X-Git-Rev: 73452d89a7babaeb0afc430dea6e0d80f78ec1ef X-Git-NotificationType: diff X-Git-Multimail-Version: 1.3.dev Auto-Submitted: auto-generated Message-Id: <20170513032709.23E54849A4@gitbox.apache.org> This is an automated email from the ASF dual-hosted git repository. davisp pushed a commit to branch COUCHDB-3287-pluggable-storage-engines in repository https://gitbox.apache.org/repos/asf/couchdb.git commit 73452d89a7babaeb0afc430dea6e0d80f78ec1ef Author: Paul J. Davis AuthorDate: Fri Feb 5 12:21:39 2016 -0600 Add storage engine test suite This allows other storage engine implementations to reuse the same exact test suite without having to resort to shenanigans like keeping vendored copies up to date. COUCHDB-3287 --- src/couch/.gitignore | 5 + src/couch/src/test_engine_attachments.erl | 93 ++++ src/couch/src/test_engine_compaction.erl | 185 ++++++++ src/couch/src/test_engine_fold_changes.erl | 190 ++++++++ src/couch/src/test_engine_fold_docs.erl | 390 +++++++++++++++ src/couch/src/test_engine_get_set_props.erl | 70 +++ src/couch/src/test_engine_open_close_delete.erl | 81 ++++ src/couch/src/test_engine_purge_docs.erl | 158 +++++++ src/couch/src/test_engine_read_write_docs.erl | 317 +++++++++++++ src/couch/src/test_engine_ref_counting.erl | 103 ++++ src/couch/src/test_engine_util.erl | 604 ++++++++++++++++++++++++ src/couch/test/couch_bt_engine_tests.erl | 20 + 12 files changed, 2216 insertions(+) diff --git a/src/couch/.gitignore b/src/couch/.gitignore index 30aa173..73fb0b6 100644 --- a/src/couch/.gitignore +++ b/src/couch/.gitignore @@ -11,5 +11,10 @@ priv/*.dll priv/*.exe vc120.pdb +test/engines/coverage/ +test/engines/data/ +test/engines/etc/ +test/engines/log/ + .rebar/ .eunit diff --git a/src/couch/src/test_engine_attachments.erl b/src/couch/src/test_engine_attachments.erl new file mode 100644 index 0000000..b0b3472 --- /dev/null +++ b/src/couch/src/test_engine_attachments.erl @@ -0,0 +1,93 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_attachments). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +cet_write_attachment() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + AttBin = crypto:rand_bytes(32768), + + try + [Att0] = test_engine_util:prep_atts(Engine, St1, [ + {<<"ohai.txt">>, AttBin} + ]), + + {stream, Stream} = couch_att:fetch(data, Att0), + ?assertEqual(true, Engine:is_active_stream(St1, Stream)), + + Actions = [{create, {<<"first">>, [], [Att0]}}], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + + {ok, St4} = Engine:init(DbPath, []), + [FDI] = Engine:open_docs(St4, [<<"first">>]), + + #rev_info{ + rev = {RevPos, PrevRevId}, + deleted = Deleted, + body_sp = DocPtr + } = test_engine_util:prev_rev(FDI), + + Doc0 = #doc{ + id = <<"foo">>, + revs = {RevPos, [PrevRevId]}, + deleted = Deleted, + body = DocPtr + }, + + Doc1 = Engine:read_doc_body(St4, Doc0), + Atts1 = if not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; true -> + couch_compress:decompress(Doc1#doc.atts) + end, + + StreamSrc = fun(Sp) -> Engine:open_read_stream(St4, Sp) end, + [Att1] = [couch_att:from_disk_term(StreamSrc, T) || T <- Atts1], + ReadBin = couch_att:to_binary(Att1), + ?assertEqual(AttBin, ReadBin) + catch throw:not_supported -> + ok + end. + + +% N.B. This test may be overly specific for some theoretical +% storage engines that don't re-initialize their +% attachments streams when restarting (for instance if +% we ever have something that stores attachemnts in +% an external object store) +cet_inactive_stream() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + AttBin = crypto:rand_bytes(32768), + + try + [Att0] = test_engine_util:prep_atts(Engine, St1, [ + {<<"ohai.txt">>, AttBin} + ]), + + {stream, Stream} = couch_att:fetch(data, Att0), + ?assertEqual(true, Engine:is_active_stream(St1, Stream)), + + Engine:terminate(normal, St1), + {ok, St2} = Engine:init(DbPath, []), + + ?assertEqual(false, Engine:is_active_stream(St2, Stream)) + catch throw:not_supported -> + ok + end. diff --git a/src/couch/src/test_engine_compaction.erl b/src/couch/src/test_engine_compaction.erl new file mode 100644 index 0000000..619edd7 --- /dev/null +++ b/src/couch/src/test_engine_compaction.erl @@ -0,0 +1,185 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_compaction). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +cet_compact_empty() -> + {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + Db1 = test_engine_util:db_as_term(Engine, St1), + {ok, St2, DbName, _, Term} = test_engine_util:compact(Engine, St1, Path), + {ok, St3, undefined} = Engine:finish_compaction(St2, DbName, [], Term), + Db2 = test_engine_util:db_as_term(Engine, St3), + Diff = test_engine_util:term_diff(Db1, Db2), + ?assertEqual(nodiff, Diff). + + +cet_compact_doc() -> + {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + Actions = [{create, {<<"foo">>, []}}], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + Db1 = test_engine_util:db_as_term(Engine, St2), + {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), + {ok, St4, undefined} = Engine:finish_compaction(St3, DbName, [], Term), + Db2 = test_engine_util:db_as_term(Engine, St4), + Diff = test_engine_util:term_diff(Db1, Db2), + ?assertEqual(nodiff, Diff). + + +cet_compact_local_doc() -> + {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + Actions = [{create, {<<"_local/foo">>, []}}], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + Db1 = test_engine_util:db_as_term(Engine, St2), + {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), + {ok, St4, undefined} = Engine:finish_compaction(St3, DbName, [], Term), + Db2 = test_engine_util:db_as_term(Engine, St4), + Diff = test_engine_util:term_diff(Db1, Db2), + ?assertEqual(nodiff, Diff). + + +cet_compact_with_everything() -> + {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + + % Add a whole bunch of docs + DocActions = lists:map(fun(Seq) -> + {create, {docid(Seq), [{<<"int">>, Seq}]}} + end, lists:seq(1, 1000)), + + LocalActions = lists:map(fun(I) -> + {create, {local_docid(I), [{<<"int">>, I}]}} + end, lists:seq(1, 25)), + + Actions1 = DocActions ++ LocalActions, + + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + {ok, St3} = Engine:set_security(St2, [{<<"readers">>, <<"ohai">>}]), + {ok, St4} = Engine:set_revs_limit(St3, 500), + + Actions2 = [ + {create, {<<"foo">>, []}}, + {create, {<<"bar">>, [{<<"hooray">>, <<"purple">>}]}}, + {conflict, {<<"bar">>, [{<<"booo">>, false}]}} + ], + + {ok, St5} = test_engine_util:apply_actions(Engine, St4, Actions2), + + [FooFDI, BarFDI] = Engine:open_docs(St5, [<<"foo">>, <<"bar">>]), + + FooRev = test_engine_util:prev_rev(FooFDI), + BarRev = test_engine_util:prev_rev(BarFDI), + + Actions3 = [ + {batch, [ + {purge, {<<"foo">>, FooRev#rev_info.rev}}, + {purge, {<<"bar">>, BarRev#rev_info.rev}} + ]} + ], + + {ok, St6} = test_engine_util:apply_actions(Engine, St5, Actions3), + + PurgedIdRevs = [ + {<<"bar">>, [BarRev#rev_info.rev]}, + {<<"foo">>, [FooRev#rev_info.rev]} + ], + + ?assertEqual(PurgedIdRevs, lists:sort(Engine:get_last_purged(St6))), + + {ok, St7} = try + [Att0, Att1, Att2, Att3, Att4] = test_engine_util:prep_atts(Engine, St6, [ + {<<"ohai.txt">>, crypto:rand_bytes(2048)}, + {<<"stuff.py">>, crypto:rand_bytes(32768)}, + {<<"a.erl">>, crypto:rand_bytes(29)}, + {<<"a.hrl">>, crypto:rand_bytes(5000)}, + {<<"a.app">>, crypto:rand_bytes(400)} + ]), + + Actions4 = [ + {create, {<<"small_att">>, [], [Att0]}}, + {create, {<<"large_att">>, [], [Att1]}}, + {create, {<<"multi_att">>, [], [Att2, Att3, Att4]}} + ], + test_engine_util:apply_actions(Engine, St6, Actions4) + catch throw:not_supported -> + {ok, St6} + end, + {ok, St8} = Engine:commit_data(St7), + + Db1 = test_engine_util:db_as_term(Engine, St8), + + Config = [ + {"database_compaction", "doc_buffer_size", "1024"}, + {"database_compaction", "checkpoint_after", "2048"} + ], + + {ok, St9, DbName, _, Term} = test_engine_util:with_config(Config, fun() -> + test_engine_util:compact(Engine, St8, Path) + end), + + {ok, St10, undefined} = Engine:finish_compaction(St9, DbName, [], Term), + Db2 = test_engine_util:db_as_term(Engine, St10), + Diff = test_engine_util:term_diff(Db1, Db2), + ?assertEqual(nodiff, Diff). + + +cet_recompact_updates() -> + {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + + Actions1 = [ + {create, {<<"foo">>, []}}, + {create, {<<"bar">>, []}} + ], + + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), + + Actions2 = [ + {update, {<<"foo">>, [{<<"updated">>, true}]}}, + {create, {<<"baz">>, []}} + ], + + {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2), + Db1 = test_engine_util:db_as_term(Engine, St4), + + {ok, St5, NewPid} = Engine:finish_compaction(St4, DbName, [], Term), + + ?assertEqual(true, is_pid(NewPid)), + Ref = erlang:monitor(process, NewPid), + + NewTerm = receive + {'$gen_cast', {compact_done, Engine, Term0}} -> + Term0; + {'DOWN', Ref, _, _, Reason} -> + erlang:error({compactor_died, Reason}) + after 10000 -> + erlang:error(compactor_timed_out) + end, + + {ok, St6, undefined} = Engine:finish_compaction(St5, DbName, [], NewTerm), + Db2 = test_engine_util:db_as_term(Engine, St6), + Diff = test_engine_util:term_diff(Db1, Db2), + ?assertEqual(nodiff, Diff). + + +docid(I) -> + Str = io_lib:format("~4..0b", [I]), + iolist_to_binary(Str). + + +local_docid(I) -> + Str = io_lib:format("_local/~4..0b", [I]), + iolist_to_binary(Str). diff --git a/src/couch/src/test_engine_fold_changes.erl b/src/couch/src/test_engine_fold_changes.erl new file mode 100644 index 0000000..6e97fda --- /dev/null +++ b/src/couch/src/test_engine_fold_changes.erl @@ -0,0 +1,190 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_fold_changes). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +-define(NUM_DOCS, 100). + + +cet_empty_changes() -> + {ok, Engine, St} = test_engine_util:init_engine(), + + ?assertEqual(0, Engine:count_changes_since(St, 0)), + ?assertEqual({ok, []}, Engine:fold_changes(St, 0, fun fold_fun/2, [], [])). + + +cet_single_change() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions = [{create, {<<"a">>, []}}], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + + ?assertEqual(1, Engine:count_changes_since(St2, 0)), + ?assertEqual({ok, [{<<"a">>, 1}]}, + Engine:fold_changes(St2, 0, fun fold_fun/2, [], [])). + + +cet_two_changes() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions = [ + {create, {<<"a">>, []}}, + {create, {<<"b">>, []}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + + ?assertEqual(2, Engine:count_changes_since(St2, 0)), + {ok, Changes} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes)). + + +cet_two_changes_batch() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions1 = [ + {batch, [ + {create, {<<"a">>, []}}, + {create, {<<"b">>, []}} + ]} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + + ?assertEqual(2, Engine:count_changes_since(St2, 0)), + {ok, Changes1} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes1)), + + {ok, Engine, St3} = test_engine_util:init_engine(), + Actions2 = [ + {batch, [ + {create, {<<"b">>, []}}, + {create, {<<"a">>, []}} + ]} + ], + {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2), + + ?assertEqual(2, Engine:count_changes_since(St4, 0)), + {ok, Changes2} = Engine:fold_changes(St4, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"b">>, 1}, {<<"a">>, 2}], lists:reverse(Changes2)). + + +cet_update_one() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions = [ + {create, {<<"a">>, []}}, + {update, {<<"a">>, []}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + + ?assertEqual(1, Engine:count_changes_since(St2, 0)), + ?assertEqual({ok, [{<<"a">>, 2}]}, + Engine:fold_changes(St2, 0, fun fold_fun/2, [], [])). + + +cet_update_first_of_two() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions = [ + {create, {<<"a">>, []}}, + {create, {<<"b">>, []}}, + {update, {<<"a">>, []}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + + ?assertEqual(2, Engine:count_changes_since(St2, 0)), + {ok, Changes} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"b">>, 2}, {<<"a">>, 3}], lists:reverse(Changes)). + + +cet_update_second_of_two() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions = [ + {create, {<<"a">>, []}}, + {create, {<<"b">>, []}}, + {update, {<<"b">>, []}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + + ?assertEqual(2, Engine:count_changes_since(St2, 0)), + {ok, Changes} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"a">>, 1}, {<<"b">>, 3}], lists:reverse(Changes)). + + +cet_check_mutation_ordering() -> + Actions = shuffle(lists:map(fun(Seq) -> + {create, {docid(Seq), []}} + end, lists:seq(1, ?NUM_DOCS))), + + DocIdOrder = [DocId || {_, {DocId, _}} <- Actions], + DocSeqs = lists:zip(DocIdOrder, lists:seq(1, ?NUM_DOCS)), + + {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + + % First lets see that we can get the correct + % suffix/prefix starting at every update sequence + lists:foreach(fun(Seq) -> + {ok, Suffix} = Engine:fold_changes(St2, Seq, fun fold_fun/2, [], []), + ?assertEqual(lists:nthtail(Seq, DocSeqs), lists:reverse(Suffix)), + + {ok, Prefix} = Engine:fold_changes(St2, Seq, fun fold_fun/2, [], [ + {dir, rev} + ]), + ?assertEqual(lists:sublist(DocSeqs, Seq + 1), Prefix) + end, lists:seq(0, ?NUM_DOCS)), + + ok = do_mutation_ordering(Engine, St2, ?NUM_DOCS + 1, DocSeqs, []). + + +do_mutation_ordering(Engine, St, _Seq, [], FinalDocSeqs) -> + {ok, RevOrder} = Engine:fold_changes(St, 0, fun fold_fun/2, [], []), + ?assertEqual(FinalDocSeqs, lists:reverse(RevOrder)), + ok; + +do_mutation_ordering(Engine, St, Seq, [{DocId, _OldSeq} | Rest], DocSeqAcc) -> + Actions = [{update, {DocId, []}}], + {ok, NewSt} = test_engine_util:apply_actions(Engine, St, Actions), + NewAcc = DocSeqAcc ++ [{DocId, Seq}], + Expected = Rest ++ NewAcc, + {ok, RevOrder} = Engine:fold_changes(NewSt, 0, fun fold_fun/2, [], []), + ?assertEqual(Expected, lists:reverse(RevOrder)), + do_mutation_ordering(Engine, NewSt, Seq + 1, Rest, NewAcc). + + +shuffle(List) -> + random:seed(os:timestamp()), + Paired = [{random:uniform(), I} || I <- List], + Sorted = lists:sort(Paired), + [I || {_, I} <- Sorted]. + + +remove_random(List) -> + Pos = random:uniform(length(List)), + remove_random(Pos, List). + + +remove_random(1, [Item | Rest]) -> + {Item, Rest}; + +remove_random(N, [Skip | Rest]) when N > 1 -> + {Item, Tail} = remove_random(N - 1, Rest), + {Item, [Skip | Tail]}. + + +fold_fun(#full_doc_info{id=Id, update_seq=Seq}, Acc) -> + {ok, [{Id, Seq} | Acc]}. + + +docid(I) -> + Str = io_lib:format("~4..0b", [I]), + iolist_to_binary(Str). diff --git a/src/couch/src/test_engine_fold_docs.erl b/src/couch/src/test_engine_fold_docs.erl new file mode 100644 index 0000000..34d7f3e --- /dev/null +++ b/src/couch/src/test_engine_fold_docs.erl @@ -0,0 +1,390 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_fold_docs). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +-define(NUM_DOCS, 100). + + +cet_fold_all() -> + fold_all(fold_docs, fun docid/1). + + +cet_fold_all_local() -> + fold_all(fold_local_docs, fun local_docid/1). + + +cet_fold_start_key() -> + fold_start_key(fold_docs, fun docid/1). + + +cet_fold_start_key_local() -> + fold_start_key(fold_local_docs, fun local_docid/1). + + +cet_fold_end_key() -> + fold_end_key(fold_docs, fun docid/1). + + +cet_fold_end_key_local() -> + fold_end_key(fold_local_docs, fun local_docid/1). + + +cet_fold_end_key_gt() -> + fold_end_key_gt(fold_docs, fun docid/1). + + +cet_fold_end_key_gt_local() -> + fold_end_key_gt(fold_local_docs, fun local_docid/1). + + +cet_fold_range() -> + fold_range(fold_docs, fun docid/1). + + +cet_fold_range_local() -> + fold_range(fold_local_docs, fun local_docid/1). + + +cet_fold_stop() -> + fold_stop(fold_docs, fun docid/1). + + +cet_fold_stop_local() -> + fold_stop(fold_local_docs, fun local_docid/1). + + +% This is a loose test but we have to have this until +% I figure out what to do about the total_rows/offset +% meta data included in _all_docs +cet_fold_include_reductions() -> + {ok, Engine, St} = init_st(fun docid/1), + FoldFun = fun(_, _, nil) -> {ok, nil} end, + {ok, Count, nil} = Engine:fold_docs(St, FoldFun, nil, [include_reductions]), + ?assert(is_integer(Count)), + ?assert(Count >= 0). + + +fold_all(FoldFun, DocIdFun) -> + DocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], + {ok, Engine, St} = init_st(DocIdFun), + + {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], []), + ?assertEqual(?NUM_DOCS, length(DocIdAccFwd)), + ?assertEqual(DocIds, lists:reverse(DocIdAccFwd)), + + {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [{dir, rev}]), + ?assertEqual(?NUM_DOCS, length(DocIdAccRev)), + ?assertEqual(DocIds, DocIdAccRev). + + +fold_start_key(FoldFun, DocIdFun) -> + {ok, Engine, St} = init_st(DocIdFun), + + StartKeyNum = ?NUM_DOCS div 4, + StartKey = DocIdFun(StartKeyNum), + + AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], + DocIdsFwd = [DocIdFun(I) || I <- lists:seq(StartKeyNum, ?NUM_DOCS)], + DocIdsRev = [DocIdFun(I) || I <- lists:seq(1, StartKeyNum)], + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {start_key, <<255>>} + ])), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, <<"">>} + ])), + + {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {start_key, <<"">>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), + + {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, <<255>>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, AllDocIdAccRev), + + {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {start_key, StartKey} + ]), + ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), + ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), + + {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, StartKey} + ]), + ?assertEqual(length(DocIdsRev), length(DocIdAccRev)), + ?assertEqual(DocIdsRev, DocIdAccRev). + + +fold_end_key(FoldFun, DocIdFun) -> + {ok, Engine, St} = init_st(DocIdFun), + + EndKeyNum = ?NUM_DOCS div 4, + EndKey = DocIdFun(EndKeyNum), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {end_key, <<"">>} + ])), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {end_key, <<255>>} + ])), + + AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], + + {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {end_key, <<255>>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), + + {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {end_key, <<"">>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, AllDocIdAccRev), + + DocIdsFwd = [DocIdFun(I) || I <- lists:seq(1, EndKeyNum)], + + {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {end_key, EndKey} + ]), + ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), + ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), + + DocIdsRev = [DocIdFun(I) || I <- lists:seq(EndKeyNum, ?NUM_DOCS)], + + {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {end_key, EndKey} + ]), + ?assertEqual(length(DocIdsRev), length(DocIdAccRev)), + ?assertEqual(DocIdsRev, DocIdAccRev). + + +fold_end_key_gt(FoldFun, DocIdFun) -> + {ok, Engine, St} = init_st(DocIdFun), + + EndKeyNum = ?NUM_DOCS div 4, + EndKey = DocIdFun(EndKeyNum), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {end_key_gt, <<"">>} + ])), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {end_key_gt, <<255>>} + ])), + + AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], + + {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {end_key_gt, <<255>>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), + + {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {end_key_gt, <<"">>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, AllDocIdAccRev), + + DocIdsFwd = [DocIdFun(I) || I <- lists:seq(1, EndKeyNum - 1)], + + {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {end_key_gt, EndKey} + ]), + ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), + ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), + + DocIdsRev = [DocIdFun(I) || I <- lists:seq(EndKeyNum + 1, ?NUM_DOCS)], + + {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {end_key_gt, EndKey} + ]), + ?assertEqual(length(DocIdsRev), length(DocIdAccRev)), + ?assertEqual(DocIdsRev, DocIdAccRev). + + +fold_range(FoldFun, DocIdFun) -> + {ok, Engine, St} = init_st(DocIdFun), + + StartKeyNum = ?NUM_DOCS div 4, + EndKeyNum = StartKeyNum * 3, + + StartKey = DocIdFun(StartKeyNum), + EndKey = DocIdFun(EndKeyNum), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {start_key, <<"">>}, + {end_key, <<"">>} + ])), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, <<"">>}, + {end_key, <<255>>} + ])), + + AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], + + {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {start_key, <<"">>}, + {end_key, <<255>>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), + + {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, <<255>>}, + {end_key_gt, <<"">>} + ]), + ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), + ?assertEqual(AllDocIds, AllDocIdAccRev), + + DocIdsFwd = [DocIdFun(I) || I <- lists:seq(StartKeyNum, EndKeyNum)], + + {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {start_key, StartKey}, + {end_key, EndKey} + ]), + ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), + ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), + + DocIdsRev = [DocIdFun(I) || I <- lists:seq(StartKeyNum, EndKeyNum)], + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, StartKey}, + {end_key, EndKey} + ])), + + {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {dir, rev}, + {start_key, EndKey}, + {end_key, StartKey} + ]), + ?assertEqual(length(DocIdsRev), length(DocIdAccRev)), + ?assertEqual(DocIdsRev, DocIdAccRev). + + +fold_stop(FoldFun, DocIdFun) -> + {ok, Engine, St} = init_st(DocIdFun), + + StartKeyNum = ?NUM_DOCS div 4, + StartKey = DocIdFun(StartKeyNum), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {start_key, <<255>>} + ])), + + ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {dir, rev}, + {start_key, <<"">>} + ])), + + SuffixDocIds = [DocIdFun(I) || I <- lists:seq(?NUM_DOCS - 3, ?NUM_DOCS)], + + {ok, SuffixDocIdAcc} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {start_key, DocIdFun(?NUM_DOCS - 3)} + ]), + ?assertEqual(length(SuffixDocIds), length(SuffixDocIdAcc)), + ?assertEqual(SuffixDocIds, lists:reverse(SuffixDocIdAcc)), + + PrefixDocIds = [DocIdFun(I) || I <- lists:seq(1, 3)], + + {ok, PrefixDocIdAcc} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {dir, rev}, + {start_key, DocIdFun(3)} + ]), + ?assertEqual(3, length(PrefixDocIdAcc)), + ?assertEqual(PrefixDocIds, PrefixDocIdAcc), + + FiveDocIdsFwd = [DocIdFun(I) + || I <- lists:seq(StartKeyNum, StartKeyNum + 5)], + + {ok, FiveDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {start_key, StartKey} + ]), + ?assertEqual(length(FiveDocIdsFwd), length(FiveDocIdAccFwd)), + ?assertEqual(FiveDocIdsFwd, lists:reverse(FiveDocIdAccFwd)), + + FiveDocIdsRev = [DocIdFun(I) + || I <- lists:seq(StartKeyNum - 5, StartKeyNum)], + + {ok, FiveDocIdAccRev} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {dir, rev}, + {start_key, StartKey} + ]), + ?assertEqual(length(FiveDocIdsRev), length(FiveDocIdAccRev)), + ?assertEqual(FiveDocIdsRev, FiveDocIdAccRev). + + +init_st(DocIdFun) -> + {ok, Engine, St1} = test_engine_util:init_engine(), + Actions = lists:map(fun(Id) -> + {create, {DocIdFun(Id), [{<<"int">>, Id}]}} + end, lists:seq(1, ?NUM_DOCS)), + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, Engine, St2}. + + +fold_fun(Doc, Acc) -> + Id = case Doc of + #doc{id = Id0} -> Id0; + #full_doc_info{id = Id0} -> Id0 + end, + {ok, [Id | Acc]}. + + +fold_fun_stop(Doc, Acc) -> + Id = case Doc of + #doc{id = Id0} -> Id0; + #full_doc_info{id = Id0} -> Id0 + end, + case length(Acc) of + N when N =< 4 -> + {ok, [Id | Acc]}; + _ -> + {stop, [Id | Acc]} + end. + + +docid(I) -> + Str = io_lib:format("~4..0b", [I]), + iolist_to_binary(Str). + + +local_docid(I) -> + Str = io_lib:format("_local/~4..0b", [I]), + iolist_to_binary(Str). diff --git a/src/couch/src/test_engine_get_set_props.erl b/src/couch/src/test_engine_get_set_props.erl new file mode 100644 index 0000000..6d2a447 --- /dev/null +++ b/src/couch/src/test_engine_get_set_props.erl @@ -0,0 +1,70 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_get_set_props). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). + + +cet_default_props() -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + {ok, St} = Engine:init(DbPath, [ + create, + {default_security_object, dso} + ]), + + Node = node(), + + ?assertEqual(0, Engine:get_doc_count(St)), + ?assertEqual(0, Engine:get_del_doc_count(St)), + ?assertEqual(true, is_list(Engine:get_size_info(St))), + ?assertEqual(true, is_integer(Engine:get_disk_version(St))), + ?assertEqual(0, Engine:get_update_seq(St)), + ?assertEqual(0, Engine:get_purge_seq(St)), + ?assertEqual([], Engine:get_last_purged(St)), + ?assertEqual(dso, Engine:get_security(St)), + ?assertEqual(1000, Engine:get_revs_limit(St)), + ?assertMatch(<<_:32/binary>>, Engine:get_uuid(St)), + ?assertEqual([{Node, 0}], Engine:get_epochs(St)), + ?assertEqual(0, Engine:get_compacted_seq(St)). + + +cet_set_security() -> + check_prop_set(get_security, set_security, dso, [{<<"readers">>, []}]). + + +cet_set_revs_limit() -> + check_prop_set(get_revs_limit, set_revs_limit, 1000, 50). + + +check_prop_set(GetFun, SetFun, Default, Value) -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + {ok, St0} = Engine:init(DbPath, [ + create, + {default_security_object, dso} + ]), + ?assertEqual(Default, Engine:GetFun(St0)), + + {ok, St1} = Engine:SetFun(St0, Value), + ?assertEqual(Value, Engine:GetFun(St1)), + + {ok, St2} = Engine:commit_data(St1), + Engine:terminate(normal, St2), + + {ok, St3} = Engine:init(DbPath, []), + ?assertEqual(Value, Engine:GetFun(St3)). diff --git a/src/couch/src/test_engine_open_close_delete.erl b/src/couch/src/test_engine_open_close_delete.erl new file mode 100644 index 0000000..b099d9f --- /dev/null +++ b/src/couch/src/test_engine_open_close_delete.erl @@ -0,0 +1,81 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_open_close_delete). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). + + +cet_open_non_existent() -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + ?assertEqual(false, Engine:exists(DbPath)), + ?assertThrow({not_found, no_db_file}, Engine:init(DbPath, [])), + ?assertEqual(false, Engine:exists(DbPath)). + + +cet_open_create() -> + process_flag(trap_exit, true), + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + ?assertEqual(false, Engine:exists(DbPath)), + ?assertMatch({ok, _}, Engine:init(DbPath, [create])), + ?assertEqual(true, Engine:exists(DbPath)). + + +cet_open_when_exists() -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + ?assertEqual(false, Engine:exists(DbPath)), + ?assertMatch({ok, _}, Engine:init(DbPath, [create])), + ?assertThrow({error, eexist}, Engine:init(DbPath, [create])). + + +cet_terminate() -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + ?assertEqual(false, Engine:exists(DbPath)), + {ok, St} = Engine:init(DbPath, [create]), + Engine:terminate(normal, St), + ?assertEqual(true, Engine:exists(DbPath)). + + +cet_rapid_recycle() -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + {ok, St0} = Engine:init(DbPath, [create]), + Engine:terminate(normal, St0), + + lists:foreach(fun(_) -> + {ok, St1} = Engine:init(DbPath, []), + Engine:terminate(normal, St1) + end, lists:seq(1, 100)). + + +cet_delete() -> + Engine = test_engine_util:get_engine(), + RootDir = test_engine_util:rootdir(), + DbPath = test_engine_util:dbpath(), + + ?assertEqual(false, Engine:exists(DbPath)), + {ok, St} = Engine:init(DbPath, [create]), + Engine:terminate(normal, St), + ?assertEqual(true, Engine:exists(DbPath)), + ?assertEqual(ok, Engine:delete(RootDir, DbPath, [async])), + ?assertEqual(false, Engine:exists(DbPath)). diff --git a/src/couch/src/test_engine_purge_docs.erl b/src/couch/src/test_engine_purge_docs.erl new file mode 100644 index 0000000..e5bf249 --- /dev/null +++ b/src/couch/src/test_engine_purge_docs.erl @@ -0,0 +1,158 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_purge_docs). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +cet_purge_simple() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + + Actions1 = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + + ?assertEqual(1, Engine:get_doc_count(St2)), + ?assertEqual(0, Engine:get_del_doc_count(St2)), + ?assertEqual(1, Engine:get_update_seq(St2)), + ?assertEqual(0, Engine:get_purge_seq(St2)), + ?assertEqual([], Engine:get_last_purged(St2)), + + [FDI] = Engine:open_docs(St2, [<<"foo">>]), + PrevRev = test_engine_util:prev_rev(FDI), + Rev = PrevRev#rev_info.rev, + + Actions2 = [ + {purge, {<<"foo">>, Rev}} + ], + {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), + + ?assertEqual(0, Engine:get_doc_count(St3)), + ?assertEqual(0, Engine:get_del_doc_count(St3)), + ?assertEqual(2, Engine:get_update_seq(St3)), + ?assertEqual(1, Engine:get_purge_seq(St3)), + ?assertEqual([{<<"foo">>, [Rev]}], Engine:get_last_purged(St3)). + + +cet_purge_conflicts() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + + Actions1 = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {conflict, {<<"foo">>, [{<<"vsn">>, 2}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + + ?assertEqual(1, Engine:get_doc_count(St2)), + ?assertEqual(0, Engine:get_del_doc_count(St2)), + ?assertEqual(2, Engine:get_update_seq(St2)), + ?assertEqual(0, Engine:get_purge_seq(St2)), + ?assertEqual([], Engine:get_last_purged(St2)), + + [FDI1] = Engine:open_docs(St2, [<<"foo">>]), + PrevRev1 = test_engine_util:prev_rev(FDI1), + Rev1 = PrevRev1#rev_info.rev, + + Actions2 = [ + {purge, {<<"foo">>, Rev1}} + ], + {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), + + ?assertEqual(1, Engine:get_doc_count(St3)), + ?assertEqual(0, Engine:get_del_doc_count(St3)), + ?assertEqual(4, Engine:get_update_seq(St3)), + ?assertEqual(1, Engine:get_purge_seq(St3)), + ?assertEqual([{<<"foo">>, [Rev1]}], Engine:get_last_purged(St3)), + + [FDI2] = Engine:open_docs(St3, [<<"foo">>]), + PrevRev2 = test_engine_util:prev_rev(FDI2), + Rev2 = PrevRev2#rev_info.rev, + + Actions3 = [ + {purge, {<<"foo">>, Rev2}} + ], + {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions3), + + ?assertEqual(0, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(5, Engine:get_update_seq(St4)), + ?assertEqual(2, Engine:get_purge_seq(St4)), + ?assertEqual([{<<"foo">>, [Rev2]}], Engine:get_last_purged(St4)). + + +cet_add_delete_purge() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + + Actions1 = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {delete, {<<"foo">>, [{<<"vsn">>, 2}]}} + ], + + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + + ?assertEqual(0, Engine:get_doc_count(St2)), + ?assertEqual(1, Engine:get_del_doc_count(St2)), + ?assertEqual(2, Engine:get_update_seq(St2)), + ?assertEqual(0, Engine:get_purge_seq(St2)), + ?assertEqual([], Engine:get_last_purged(St2)), + + [FDI] = Engine:open_docs(St2, [<<"foo">>]), + PrevRev = test_engine_util:prev_rev(FDI), + Rev = PrevRev#rev_info.rev, + + Actions2 = [ + {purge, {<<"foo">>, Rev}} + ], + {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), + + ?assertEqual(0, Engine:get_doc_count(St3)), + ?assertEqual(0, Engine:get_del_doc_count(St3)), + ?assertEqual(3, Engine:get_update_seq(St3)), + ?assertEqual(1, Engine:get_purge_seq(St3)), + ?assertEqual([{<<"foo">>, [Rev]}], Engine:get_last_purged(St3)). + + +cet_add_two_purge_one() -> + {ok, Engine, St1} = test_engine_util:init_engine(), + + Actions1 = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {create, {<<"bar">>, []}} + ], + + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + + ?assertEqual(2, Engine:get_doc_count(St2)), + ?assertEqual(0, Engine:get_del_doc_count(St2)), + ?assertEqual(2, Engine:get_update_seq(St2)), + ?assertEqual(0, Engine:get_purge_seq(St2)), + ?assertEqual([], Engine:get_last_purged(St2)), + + [FDI] = Engine:open_docs(St2, [<<"foo">>]), + PrevRev = test_engine_util:prev_rev(FDI), + Rev = PrevRev#rev_info.rev, + + Actions2 = [ + {purge, {<<"foo">>, Rev}} + ], + {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), + + ?assertEqual(1, Engine:get_doc_count(St3)), + ?assertEqual(0, Engine:get_del_doc_count(St3)), + ?assertEqual(3, Engine:get_update_seq(St3)), + ?assertEqual(1, Engine:get_purge_seq(St3)), + ?assertEqual([{<<"foo">>, [Rev]}], Engine:get_last_purged(St3)). diff --git a/src/couch/src/test_engine_read_write_docs.erl b/src/couch/src/test_engine_read_write_docs.erl new file mode 100644 index 0000000..4307702 --- /dev/null +++ b/src/couch/src/test_engine_read_write_docs.erl @@ -0,0 +1,317 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_read_write_docs). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +cet_read_empty_docs() -> + {ok, Engine, St} = test_engine_util:init_engine(), + + ?assertEqual([not_found], Engine:open_docs(St, [<<"foo">>])), + ?assertEqual( + [not_found, not_found], + Engine:open_docs(St, [<<"a">>, <<"b">>]) + ). + + +cet_read_empty_local_docs() -> + {ok, Engine, St} = test_engine_util:init_engine(), + + ?assertEqual([not_found], Engine:open_local_docs(St, [<<"_local/foo">>])), + ?assertEqual( + [not_found, not_found], + Engine:open_local_docs(St, [<<"_local/a">>, <<"_local/b">>]) + ). + + +cet_write_one_doc() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(1, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(1, Engine:get_update_seq(St4)), + + [FDI] = Engine:open_docs(St4, [<<"foo">>]), + #rev_info{ + rev = {RevPos, PrevRevId}, + deleted = Deleted, + body_sp = DocPtr + } = test_engine_util:prev_rev(FDI), + + Doc0 = #doc{ + id = <<"foo">>, + revs = {RevPos, [PrevRevId]}, + deleted = Deleted, + body = DocPtr + }, + + Doc1 = Engine:read_doc_body(St4, Doc0), + Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> + couch_compress:decompress(Doc1#doc.body) + end, + ?assertEqual([{<<"vsn">>, 1}], Body1). + + +cet_write_two_docs() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {create, {<<"bar">>, [{<<"stuff">>, true}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(2, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(2, Engine:get_update_seq(St4)), + + Resps = Engine:open_docs(St4, [<<"foo">>, <<"bar">>]), + ?assertEqual(false, lists:member(not_found, Resps)). + + +cet_write_three_doc_batch() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {batch, [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {create, {<<"bar">>, [{<<"stuff">>, true}]}}, + {create, {<<"baz">>, []}} + ]} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(3, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(3, Engine:get_update_seq(St4)), + + Resps = Engine:open_docs(St4, [<<"foo">>, <<"bar">>, <<"baz">>]), + ?assertEqual(false, lists:member(not_found, Resps)). + + +cet_update_doc() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {update, {<<"foo">>, [{<<"vsn">>, 2}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(1, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(2, Engine:get_update_seq(St4)), + + [FDI] = Engine:open_docs(St4, [<<"foo">>]), + + #rev_info{ + rev = {RevPos, PrevRevId}, + deleted = Deleted, + body_sp = DocPtr + } = test_engine_util:prev_rev(FDI), + + Doc0 = #doc{ + id = <<"foo">>, + revs = {RevPos, [PrevRevId]}, + deleted = Deleted, + body = DocPtr + }, + + Doc1 = Engine:read_doc_body(St4, Doc0), + Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> + couch_compress:decompress(Doc1#doc.body) + end, + + ?assertEqual([{<<"vsn">>, 2}], Body1). + + +cet_delete_doc() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, + {delete, {<<"foo">>, []}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(0, Engine:get_doc_count(St4)), + ?assertEqual(1, Engine:get_del_doc_count(St4)), + ?assertEqual(2, Engine:get_update_seq(St4)), + + [FDI] = Engine:open_docs(St4, [<<"foo">>]), + + #rev_info{ + rev = {RevPos, PrevRevId}, + deleted = Deleted, + body_sp = DocPtr + } = test_engine_util:prev_rev(FDI), + + Doc0 = #doc{ + id = <<"foo">>, + revs = {RevPos, [PrevRevId]}, + deleted = Deleted, + body = DocPtr + }, + + Doc1 = Engine:read_doc_body(St4, Doc0), + Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> + couch_compress:decompress(Doc1#doc.body) + end, + + ?assertEqual([], Body1). + + +cet_write_local_doc() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"_local/foo">>, [{<<"yay">>, false}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(0, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(0, Engine:get_update_seq(St4)), + + [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), + [#doc{} = Doc] = Engine:open_local_docs(St4, [<<"_local/foo">>]), + ?assertEqual([{<<"yay">>, false}], Doc#doc.body). + + +cet_write_mixed_batch() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {batch, [ + {create, {<<"bar">>, []}}, + {create, {<<"_local/foo">>, [{<<"yay">>, false}]}} + ]} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(1, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(1, Engine:get_update_seq(St4)), + + [#full_doc_info{}] = Engine:open_docs(St4, [<<"bar">>]), + [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), + + [not_found] = Engine:open_local_docs(St4, [<<"bar">>]), + [#doc{}] = Engine:open_local_docs(St4, [<<"_local/foo">>]). + + +cet_update_local_doc() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"_local/foo">>, []}}, + {update, {<<"_local/foo">>, [{<<"stuff">>, null}]}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(0, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(0, Engine:get_update_seq(St4)), + + [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), + [#doc{} = Doc] = Engine:open_local_docs(St4, [<<"_local/foo">>]), + ?assertEqual([{<<"stuff">>, null}], Doc#doc.body). + + +cet_delete_local_doc() -> + {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + + ?assertEqual(0, Engine:get_doc_count(St1)), + ?assertEqual(0, Engine:get_del_doc_count(St1)), + ?assertEqual(0, Engine:get_update_seq(St1)), + + Actions = [ + {create, {<<"_local/foo">>, []}}, + {delete, {<<"_local/foo">>, []}} + ], + {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, St3} = Engine:commit_data(St2), + Engine:terminate(normal, St3), + {ok, St4} = Engine:init(DbPath, []), + + ?assertEqual(0, Engine:get_doc_count(St4)), + ?assertEqual(0, Engine:get_del_doc_count(St4)), + ?assertEqual(0, Engine:get_update_seq(St4)), + + [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), + ?assertEqual([not_found], Engine:open_local_docs(St4, [<<"_local/foo">>])). diff --git a/src/couch/src/test_engine_ref_counting.erl b/src/couch/src/test_engine_ref_counting.erl new file mode 100644 index 0000000..18e75fb --- /dev/null +++ b/src/couch/src/test_engine_ref_counting.erl @@ -0,0 +1,103 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_ref_counting). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +-define(NUM_CLIENTS, 1000). + + +cet_empty_monitors() -> + {ok, Engine, St} = test_engine_util:init_engine(), + Pids = Engine:monitored_by(St), + ?assert(is_list(Pids)), + ?assertEqual([], Pids -- [self(), whereis(couch_stats_process_tracker)]). + + +cet_incref_decref() -> + {ok, Engine, St} = test_engine_util:init_engine(), + + {Pid, _} = Client = start_client(Engine, St), + wait_client(Client), + + Pids1 = Engine:monitored_by(St), + ?assert(lists:member(Pid, Pids1)), + + close_client(Client), + + Pids2 = Engine:monitored_by(St), + ?assert(not lists:member(Pid, Pids2)). + + +cet_incref_decref_many() -> + {ok, Engine, St} = test_engine_util:init_engine(), + Clients = lists:map(fun(_) -> + start_client(Engine, St) + end, lists:seq(1, ?NUM_CLIENTS)), + + lists:foreach(fun(C) -> wait_client(C) end, Clients), + + Pids1 = Engine:monitored_by(St), + % +2 for db pid and process tracker + ?assertEqual(?NUM_CLIENTS + 2, length(Pids1)), + + lists:foreach(fun(C) -> close_client(C) end, Clients), + + Pids2 = Engine:monitored_by(St), + ?assertEqual(2, length(Pids2)). + + +start_client(Engine, St1) -> + spawn_monitor(fun() -> + {ok, St2} = Engine:incref(St1), + + receive + {waiting, Pid} -> + Pid ! go + after 1000 -> + erlang:error(timeout) + end, + + receive + close -> + ok + after 1000 -> + erlang:error(timeout) + end, + + Engine:decref(St2) + end). + + +wait_client({Pid, _Ref}) -> + Pid ! {waiting, self()}, + receive + go -> ok + after 1000 -> + erlang:error(timeout) + end. + + +close_client({Pid, Ref}) -> + Pid ! close, + receive + {'DOWN', Ref, _, _, _} -> + ok + after 1000 -> + erlang:error(timeout) + end. + diff --git a/src/couch/src/test_engine_util.erl b/src/couch/src/test_engine_util.erl new file mode 100644 index 0000000..d19b7f1 --- /dev/null +++ b/src/couch/src/test_engine_util.erl @@ -0,0 +1,604 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(test_engine_util). +-compile(export_all). + + +-include_lib("couch/include/couch_db.hrl"). + + +-define(TEST_MODULES, [ + test_engine_open_close_delete, + test_engine_get_set_props, + test_engine_read_write_docs, + test_engine_attachments, + test_engine_fold_docs, + test_engine_fold_changes, + test_engine_purge_docs, + test_engine_compaction, + test_engine_ref_counting +]). + + +create_tests(EngineApp) -> + create_tests(EngineApp, EngineApp). + + +create_tests(EngineApp, EngineModule) -> + application:set_env(couch, test_engine, {EngineApp, EngineModule}), + Tests = lists:map(fun(TestMod) -> + {atom_to_list(TestMod), gather(TestMod)} + end, ?TEST_MODULES), + Setup = fun() -> + Ctx = test_util:start_couch(), + config:set("log", "include_sasl", "false", false), + Ctx + end, + { + setup, + Setup, + fun test_util:stop_couch/1, + fun(_) -> Tests end + }. + + +gather(Module) -> + Exports = Module:module_info(exports), + Tests = lists:foldl(fun({Fun, Arity}, Acc) -> + case {atom_to_list(Fun), Arity} of + {[$c, $e, $t, $_ | _], 0} -> + TestFun = make_test_fun(Module, Fun), + [{spawn, TestFun} | Acc]; + _ -> + Acc + end + end, [], Exports), + lists:reverse(Tests). + + +make_test_fun(Module, Fun) -> + Name = lists:flatten(io_lib:format("~s:~s", [Module, Fun])), + Wrapper = fun() -> + process_flag(trap_exit, true), + Module:Fun() + end, + {Name, Wrapper}. + +rootdir() -> + config:get("couchdb", "database_dir", "."). + + +dbpath() -> + binary_to_list(filename:join(rootdir(), couch_uuids:random())). + + +get_engine() -> + case application:get_env(couch, test_engine) of + {ok, {_, Engine}} -> + Engine; + _ -> + couch_bt_engine + end. + + +init_engine() -> + init_engine(default). + + +init_engine(default) -> + Engine = get_engine(), + DbPath = dbpath(), + {ok, St} = Engine:init(DbPath, [ + create, + {default_security_object, []} + ]), + {ok, Engine, St}; + +init_engine(dbpath) -> + Engine = get_engine(), + DbPath = dbpath(), + {ok, St} = Engine:init(DbPath, [ + create, + {default_security_object, []} + ]), + {ok, Engine, DbPath, St}. + + +apply_actions(_Engine, St, []) -> + {ok, St}; + +apply_actions(Engine, St, [Action | Rest]) -> + NewSt = apply_action(Engine, St, Action), + apply_actions(Engine, NewSt, Rest). + + +apply_action(Engine, St, {batch, BatchActions}) -> + apply_batch(Engine, St, BatchActions); + +apply_action(Engine, St, Action) -> + apply_batch(Engine, St, [Action]). + + +apply_batch(Engine, St, Actions) -> + UpdateSeq = Engine:get_update_seq(St) + 1, + AccIn = {UpdateSeq, [], [], []}, + AccOut = lists:foldl(fun(Action, Acc) -> + {SeqAcc, DocAcc, LDocAcc, PurgeAcc} = Acc, + case Action of + {_, {<<"_local/", _/binary>>, _}} -> + LDoc = gen_local_write(Engine, St, Action), + {SeqAcc, DocAcc, [LDoc | LDocAcc], PurgeAcc}; + _ -> + case gen_write(Engine, St, Action, SeqAcc) of + {_OldFDI, _NewFDI} = Pair -> + {SeqAcc + 1, [Pair | DocAcc], LDocAcc, PurgeAcc}; + {Pair, NewSeqAcc, NewPurgeInfo} -> + NewPurgeAcc = [NewPurgeInfo | PurgeAcc], + {NewSeqAcc, [Pair | DocAcc], LDocAcc, NewPurgeAcc} + end + end + end, AccIn, Actions), + {_, Docs0, LDocs, PurgeIdRevs} = AccOut, + Docs = lists:reverse(Docs0), + {ok, NewSt} = Engine:write_doc_infos(St, Docs, LDocs, PurgeIdRevs), + NewSt. + + +gen_local_write(Engine, St, {Action, {DocId, Body}}) -> + PrevRev = case Engine:open_local_docs(St, [DocId]) of + [not_found] -> + 0; + [#doc{revs = {0, []}}] -> + 0; + [#doc{revs = {0, [RevStr | _]}}] -> + list_to_integer(binary_to_list(RevStr)) + end, + {RevId, Deleted} = case Action of + Action when Action == create; Action == update -> + {list_to_binary(integer_to_list(PrevRev + 1)), false}; + delete -> + {<<"0">>, true} + end, + #doc{ + id = DocId, + revs = {0, [RevId]}, + body = Body, + deleted = Deleted + }. + +gen_write(Engine, St, {Action, {DocId, Body}}, UpdateSeq) -> + gen_write(Engine, St, {Action, {DocId, Body, []}}, UpdateSeq); + +gen_write(Engine, St, {create, {DocId, Body, Atts0}}, UpdateSeq) -> + [not_found] = Engine:open_docs(St, [DocId]), + Atts = [couch_att:to_disk_term(Att) || Att <- Atts0], + + Rev = crypto:hash(md5, term_to_binary({DocId, Body, Atts})), + + Doc0 = #doc{ + id = DocId, + revs = {0, [Rev]}, + deleted = false, + body = Body, + atts = Atts + }, + + Doc1 = make_doc_summary(Engine, St, Doc0), + {ok, Doc2, Len} = Engine:write_doc_body(St, Doc1), + + Sizes = #size_info{ + active = Len, + external = erlang:external_size(Doc1#doc.body) + }, + + Leaf = #leaf{ + deleted = false, + ptr = Doc2#doc.body, + seq = UpdateSeq, + sizes = Sizes, + atts = Atts + }, + + {not_found, #full_doc_info{ + id = DocId, + deleted = false, + update_seq = UpdateSeq, + rev_tree = [{0, {Rev, Leaf, []}}], + sizes = Sizes + }}; + +gen_write(Engine, St, {purge, {DocId, PrevRevs0, _}}, UpdateSeq) -> + [#full_doc_info{} = PrevFDI] = Engine:open_docs(St, [DocId]), + PrevRevs = if is_list(PrevRevs0) -> PrevRevs0; true -> [PrevRevs0] end, + + #full_doc_info{ + rev_tree = PrevTree + } = PrevFDI, + + {NewTree, RemRevs} = couch_key_tree:remove_leafs(PrevTree, PrevRevs), + RemovedAll = lists:sort(RemRevs) == lists:sort(PrevRevs), + if RemovedAll -> ok; true -> + % If we didn't purge all the requested revisions + % then its a bug in the test. + erlang:error({invalid_purge_test_revs, PrevRevs}) + end, + + case NewTree of + [] -> + % We've completely purged the document + {{PrevFDI, not_found}, UpdateSeq, {DocId, RemRevs}}; + _ -> + % We have to relabel the update_seq of all + % leaves. See couch_db_updater for details. + {NewNewTree, NewUpdateSeq} = couch_key_tree:mapfold(fun + (_RevId, Leaf, leaf, InnerSeqAcc) -> + {Leaf#leaf{seq = InnerSeqAcc}, InnerSeqAcc + 1}; + (_RevId, Value, _Type, InnerSeqAcc) -> + {Value, InnerSeqAcc} + end, UpdateSeq, NewTree), + NewFDI = PrevFDI#full_doc_info{ + update_seq = NewUpdateSeq - 1, + rev_tree = NewNewTree + }, + {{PrevFDI, NewFDI}, NewUpdateSeq, {DocId, RemRevs}} + end; + +gen_write(Engine, St, {Action, {DocId, Body, Atts0}}, UpdateSeq) -> + [#full_doc_info{} = PrevFDI] = Engine:open_docs(St, [DocId]), + Atts = [couch_att:to_disk_term(Att) || Att <- Atts0], + + #full_doc_info{ + id = DocId, + rev_tree = PrevRevTree + } = PrevFDI, + + #rev_info{ + rev = PrevRev + } = prev_rev(PrevFDI), + + {RevPos, PrevRevId} = PrevRev, + + Rev = gen_revision(Action, DocId, PrevRev, Body, Atts), + + Doc0 = #doc{ + id = DocId, + revs = {RevPos + 1, [Rev, PrevRevId]}, + deleted = false, + body = Body, + atts = Atts + }, + + Doc1 = make_doc_summary(Engine, St, Doc0), + {ok, Doc2, Len} = Engine:write_doc_body(St, Doc1), + + Deleted = case Action of + update -> false; + conflict -> false; + delete -> true + end, + + Sizes = #size_info{ + active = Len, + external = erlang:external_size(Doc1#doc.body) + }, + + Leaf = #leaf{ + deleted = Deleted, + ptr = Doc2#doc.body, + seq = UpdateSeq, + sizes = Sizes, + atts = Atts + }, + + Path = gen_path(Action, RevPos, PrevRevId, Rev, Leaf), + RevsLimit = Engine:get_revs_limit(St), + NodeType = case Action of + conflict -> new_branch; + _ -> new_leaf + end, + {NewTree, NodeType} = couch_key_tree:merge(PrevRevTree, Path, RevsLimit), + + NewFDI = PrevFDI#full_doc_info{ + deleted = couch_doc:is_deleted(NewTree), + update_seq = UpdateSeq, + rev_tree = NewTree, + sizes = Sizes + }, + + {PrevFDI, NewFDI}. + + +gen_revision(conflict, DocId, _PrevRev, Body, Atts) -> + crypto:hash(md5, term_to_binary({DocId, Body, Atts})); +gen_revision(delete, DocId, PrevRev, Body, Atts) -> + gen_revision(update, DocId, PrevRev, Body, Atts); +gen_revision(update, DocId, PrevRev, Body, Atts) -> + crypto:hash(md5, term_to_binary({DocId, PrevRev, Body, Atts})). + + +gen_path(conflict, _RevPos, _PrevRevId, Rev, Leaf) -> + {0, {Rev, Leaf, []}}; +gen_path(delete, RevPos, PrevRevId, Rev, Leaf) -> + gen_path(update, RevPos, PrevRevId, Rev, Leaf); +gen_path(update, RevPos, PrevRevId, Rev, Leaf) -> + {RevPos, {PrevRevId, ?REV_MISSING, [{Rev, Leaf, []}]}}. + + +make_doc_summary(Engine, St, DocData) -> + {_, Ref} = spawn_monitor(fun() -> + exit({result, Engine:serialize_doc(St, DocData)}) + end), + receive + {'DOWN', Ref, _, _, {result, Summary}} -> + Summary; + {'DOWN', Ref, _, _, Error} -> + erlang:error({make_doc_summary_error, Error}) + after 1000 -> + erlang:error(make_doc_summary_timeout) + end. + + +prep_atts(_Engine, _St, []) -> + []; + +prep_atts(Engine, St, [{FileName, Data} | Rest]) -> + {_, Ref} = spawn_monitor(fun() -> + {ok, Stream} = Engine:open_write_stream(St, []), + exit(write_att(Stream, FileName, Data, Data)) + end), + Att = receive + {'DOWN', Ref, _, _, {{no_catch, not_supported}, _}} -> + throw(not_supported); + {'DOWN', Ref, _, _, Resp} -> + Resp + after 5000 -> + erlang:error(attachment_write_timeout) + end, + [Att | prep_atts(Engine, St, Rest)]. + + +write_att(Stream, FileName, OrigData, <<>>) -> + {StreamEngine, Len, Len, Md5, Md5} = couch_stream:close(Stream), + couch_util:check_md5(Md5, crypto:hash(md5, OrigData)), + Len = size(OrigData), + couch_att:new([ + {name, FileName}, + {type, <<"application/octet-stream">>}, + {data, {stream, StreamEngine}}, + {att_len, Len}, + {disk_len, Len}, + {md5, Md5}, + {encoding, identity} + ]); + +write_att(Stream, FileName, OrigData, Data) -> + {Chunk, Rest} = case size(Data) > 4096 of + true -> + <> = Data, + {Head, Tail}; + false -> + {Data, <<>>} + end, + ok = couch_stream:write(Stream, Chunk), + write_att(Stream, FileName, OrigData, Rest). + + +prev_rev(#full_doc_info{} = FDI) -> + #doc_info{ + revs = [#rev_info{} = PrevRev | _] + } = couch_doc:to_doc_info(FDI), + PrevRev. + + +db_as_term(Engine, St) -> + [ + {props, db_props_as_term(Engine, St)}, + {docs, db_docs_as_term(Engine, St)}, + {local_docs, db_local_docs_as_term(Engine, St)}, + {changes, db_changes_as_term(Engine, St)} + ]. + + +db_props_as_term(Engine, St) -> + Props = [ + get_doc_count, + get_del_doc_count, + get_disk_version, + get_update_seq, + get_purge_seq, + get_last_purged, + get_security, + get_revs_limit, + get_uuid, + get_epochs + ], + lists:map(fun(Fun) -> + {Fun, Engine:Fun(St)} + end, Props). + + +db_docs_as_term(Engine, St) -> + FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, + {ok, FDIs} = Engine:fold_docs(St, FoldFun, [], []), + lists:reverse(lists:map(fun(FDI) -> + fdi_to_term(Engine, St, FDI) + end, FDIs)). + + +db_local_docs_as_term(Engine, St) -> + FoldFun = fun(Doc, Acc) -> {ok, [Doc | Acc]} end, + {ok, LDocs} = Engine:fold_local_docs(St, FoldFun, [], []), + lists:reverse(LDocs). + + +db_changes_as_term(Engine, St) -> + FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, + {ok, Changes} = Engine:fold_changes(St, 0, FoldFun, [], []), + lists:reverse(lists:map(fun(FDI) -> + fdi_to_term(Engine, St, FDI) + end, Changes)). + + +fdi_to_term(Engine, St, FDI) -> + #full_doc_info{ + id = DocId, + rev_tree = OldTree + } = FDI, + {NewRevTree, _} = couch_key_tree:mapfold(fun(Rev, Node, Type, Acc) -> + tree_to_term(Rev, Node, Type, Acc, DocId) + end, {Engine, St}, OldTree), + FDI#full_doc_info{ + rev_tree = NewRevTree, + % Blank out sizes because we allow storage + % engines to handle this with their own + % definition until further notice. + sizes = #size_info{ + active = -1, + external = -1 + } + }. + + +tree_to_term(_Rev, _Leaf, branch, Acc, _DocId) -> + {?REV_MISSING, Acc}; + +tree_to_term({Pos, RevId}, #leaf{} = Leaf, leaf, {Engine, St}, DocId) -> + #leaf{ + deleted = Deleted, + ptr = Ptr + } = Leaf, + + Doc0 = #doc{ + id = DocId, + revs = {Pos, [RevId]}, + deleted = Deleted, + body = Ptr + }, + + Doc1 = Engine:read_doc_body(St, Doc0), + + Body = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> + couch_compress:decompress(Doc1#doc.body) + end, + + Atts1 = if not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; true -> + couch_compress:decompress(Doc1#doc.atts) + end, + + StreamSrc = fun(Sp) -> Engine:open_read_stream(St, Sp) end, + Atts2 = [couch_att:from_disk_term(StreamSrc, Att) || Att <- Atts1], + Atts = [att_to_term(Att) || Att <- Atts2], + + NewLeaf = Leaf#leaf{ + ptr = Body, + sizes = #size_info{active = -1, external = -1}, + atts = Atts + }, + {NewLeaf, {Engine, St}}. + + +att_to_term(Att) -> + Bin = couch_att:to_binary(Att), + couch_att:store(data, Bin, Att). + + +term_diff(T1, T2) when is_tuple(T1), is_tuple(T2) -> + tuple_diff(tuple_to_list(T1), tuple_to_list(T2)); + +term_diff(L1, L2) when is_list(L1), is_list(L2) -> + list_diff(L1, L2); + +term_diff(V1, V2) when V1 == V2 -> + nodiff; + +term_diff(V1, V2) -> + {V1, V2}. + + +tuple_diff([], []) -> + nodiff; + +tuple_diff([T1 | _], []) -> + {longer, T1}; + +tuple_diff([], [T2 | _]) -> + {shorter, T2}; + +tuple_diff([T1 | R1], [T2 | R2]) -> + case term_diff(T1, T2) of + nodiff -> + tuple_diff(R1, R2); + Else -> + {T1, Else} + end. + + +list_diff([], []) -> + nodiff; + +list_diff([T1 | _], []) -> + {longer, T1}; + +list_diff([], [T2 | _]) -> + {shorter, T2}; + +list_diff([T1 | R1], [T2 | R2]) -> + case term_diff(T1, T2) of + nodiff -> + list_diff(R1, R2); + Else -> + {T1, Else} + end. + + +compact(Engine, St1, DbPath) -> + DbName = filename:basename(DbPath), + {ok, St2, Pid} = Engine:start_compaction(St1, DbName, [], self()), + Ref = erlang:monitor(process, Pid), + + % Ideally I'd assert that Pid is linked to us + % at this point but its technically possible + % that it could have finished compacting by + % the time we check... Quite the quandry. + + Term = receive + {'$gen_cast', {compact_done, Engine, Term0}} -> + Term0; + {'DOWN', Ref, _, _, Reason} -> + erlang:error({compactor_died, Reason}) + after 10000 -> + erlang:error(compactor_timed_out) + end, + + {ok, St2, DbName, Pid, Term}. + + +with_config(Config, Fun) -> + OldConfig = apply_config(Config), + try + Fun() + after + apply_config(OldConfig) + end. + + +apply_config([]) -> + []; + +apply_config([{Section, Key, Value} | Rest]) -> + Orig = config:get(Section, Key), + case Value of + undefined -> config:delete(Section, Key); + _ -> config:set(Section, Key, Value) + end, + [{Section, Key, Orig} | apply_config(Rest)]. diff --git a/src/couch/test/couch_bt_engine_tests.erl b/src/couch/test/couch_bt_engine_tests.erl new file mode 100644 index 0000000..df200df --- /dev/null +++ b/src/couch/test/couch_bt_engine_tests.erl @@ -0,0 +1,20 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_bt_engine_tests). + + +-include_lib("eunit/include/eunit.hrl"). + + +couch_bt_engine_test_()-> + test_engine_util:create_tests(couch, couch_bt_engine). -- To stop receiving notification emails like this one, please contact "commits@couchdb.apache.org" .