From commits-return-36938-archive-asf-public=cust-asf.ponee.io@couchdb.apache.org Mon Apr 8 18:58:03 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id EC151180784 for ; Mon, 8 Apr 2019 20:58:02 +0200 (CEST) Received: (qmail 71025 invoked by uid 500); 8 Apr 2019 18:38:24 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 70927 invoked by uid 99); 8 Apr 2019 18:38:24 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 08 Apr 2019 18:38:24 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 47FB880A4B; Mon, 8 Apr 2019 18:58:01 +0000 (UTC) Date: Mon, 08 Apr 2019 18:58:02 +0000 To: "commits@couchdb.apache.org" Subject: [couchdb] 01/01: Prototype implementation of RFC 0001 MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: davisp@apache.org In-Reply-To: <155474988108.29837.13147983854974960922@gitbox.apache.org> References: <155474988108.29837.13147983854974960922@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: couchdb X-Git-Refname: refs/heads/prototype/rfc-001-revision-metadata-model X-Git-Reftype: branch X-Git-Rev: 38c0f69410a31507309aef5a86a249efbca6ba7a X-Git-NotificationType: diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated Message-Id: <20190408185801.47FB880A4B@gitbox.apache.org> This is an automated email from the ASF dual-hosted git repository. davisp pushed a commit to branch prototype/rfc-001-revision-metadata-model in repository https://gitbox.apache.org/repos/asf/couchdb.git commit 38c0f69410a31507309aef5a86a249efbca6ba7a Author: Paul J. Davis AuthorDate: Mon Apr 8 13:58:39 2019 -0500 Prototype implementation of RFC 0001 This makes the necessary changes to have partial support for RFC 001 Revision Metadata Model. This only covers interactive updates for the moment. --- RFC001_NOTES.md | 42 +++++ src/fabric/src/fabric2_db.erl | 407 ++++++++++++++++------------------------- src/fabric/src/fabric2_fdb.erl | 274 +++++++++++++++++---------- 3 files changed, 380 insertions(+), 343 deletions(-) diff --git a/RFC001_NOTES.md b/RFC001_NOTES.md new file mode 100644 index 0000000..a5affc7 --- /dev/null +++ b/RFC001_NOTES.md @@ -0,0 +1,42 @@ +Notes +=== + +Mark winning revs explicitly +--- + +Currently we rely on winners sorting last and denote that by the fact of +them having a different value structure. I think it'd be good to assert +that the winner is a winner or not in the key structure and then have +a consistent value structure (although Sequence and BranchCount can be +both null in non-winners). + +This allows us to assert that our tree stored in fdb is not corrupt in a +number of places rather than possibly allowing for confusion if we +have a bug when updating the revision tree. + + +Revision infos need to track their size +--- + +If we want to maintain a database size counter we'll want to store the +size of a given doc body for each revision so that we don't have to +read the old body when updating the tree. + +Need to add Incarnation +--- + +Currently ignoring this in favor of the rest of the RFC + + +Need to add Batch Id +--- + +Currently ignoring as well. + + +Incarnation +--- + +Defined as a single byte which seems a bit limiting. The tuple layer +has variable length integers which seem like a better solution. + diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl index 75259c3..d62e5dc 100644 --- a/src/fabric/src/fabric2_db.erl +++ b/src/fabric/src/fabric2_db.erl @@ -391,14 +391,13 @@ open_doc(#{} = Db, DocId) -> open_doc(#{} = Db, DocId, _Options) -> with_tx(Db, fun(TxDb) -> - case fabric2_fdb:get_full_doc_info(TxDb, DocId) of - not_found -> + case fabric2_fdb:get_winning_revs(TxDb, DocId, 1) of + [] -> {not_found, missing}; - #full_doc_info{} = FDI -> - {_, Path} = couch_doc:to_doc_info_path(FDI), - case fabric2_fdb:get_doc_body(TxDb, DocId, Path) of + [#{winner := true} = RevInfo] -> + case fabric2_fdb:get_doc_body(TxDb, DocId, RevInfo) of #doc{} = Doc -> {ok, Doc}; - Error -> Error + Else -> Else end end end). @@ -450,17 +449,17 @@ update_docs(Db, Docs) -> update_docs(Db, Docs, Options) -> - with_tx(Db, fun(TxDb) -> - {Resps, Status} = lists:mapfoldl(fun(Doc, Acc) -> + {Resps, Status} = lists:mapfoldl(fun(Doc, Acc) -> + with_tx(Db, fun(TxDb) -> case update_doc_int(TxDb, Doc, Options) of {ok, _} = Resp -> {Resp, Acc}; {error, _} = Resp -> {Resp, error} end - end, ok, Docs), - {Status, Resps} - end). + end) + end, ok, Docs), + {Status, Resps}. fold_docs(Db, UserFun, UserAcc) -> @@ -482,6 +481,7 @@ fold_changes(Db, SinceSeq, UserFun, UserAcc, Options) -> fabric2_fdb:fold_changes(TxDb, SinceSeq, UserFun, UserAcc, Options) end). + new_revid(Doc) -> #doc{ body = Body, @@ -607,246 +607,157 @@ get_members(SecProps) -> % TODO: Handle _local docs separately. -update_doc_int(#{} = Db, #doc{} = Doc0, Options) -> - UpdateType = case lists:member(replicated_changes, Options) of - true -> replicated_changes; - false -> interactive_edit - end, - +update_doc_int(#{} = Db, #doc{} = Doc, Options) -> try - FDI1 = fabric2_fdb:get_full_doc_info(Db, Doc0#doc.id), - Doc1 = prep_and_validate(Db, FDI1, Doc0, UpdateType), - Doc2 = case UpdateType of - interactive_edit -> new_revid(Doc1); - replicated_changes -> Doc1 - end, - FDI2 = if FDI1 /= not_found -> FDI1; true -> - #full_doc_info{id = Doc2#doc.id} - end, - {FDI3, Doc3} = merge_rev_tree(FDI2, Doc2, UpdateType), - - OldExists = case FDI1 of - not_found -> false; - #full_doc_info{deleted = true} -> false; - _ -> true - end, - NewExists = not FDI3#full_doc_info.deleted, - - ok = fabric2_fdb:store_doc(Db, FDI3, Doc3), - - {_, {WinPos, [WinRev | _]}} = couch_doc:to_doc_info_path(FDI3), - - case {OldExists, NewExists} of - {false, true} -> - fabric2_fdb:add_to_all_docs(Db, Doc3#doc.id, {WinPos, WinRev}), - fabric2_fdb:incr_stat(Db, <<"doc_count">>, 1); - {true, false} -> - fabric2_fdb:rem_from_all_docs(Db, Doc3#doc.id), - fabric2_fdb:incr_stat(Db, <<"doc_count">>, -1), - fabric2_fdb:incr_stat(Db, <<"doc_del_count">>, 1); - {Exists, Exists} -> - % No change - ok - end, - - % Need to count design documents - % Need to track db size changes - % Need to update VDUs on ddoc change - #doc{ - revs = {RevStart, [Rev | _]} - } = Doc3, - {ok, {RevStart, Rev}} + case lists:member(replicated_changes, Options) of + false -> update_doc_interactive(Db, Doc, Options); + true -> update_doc_replicated(Db, Doc, Options) + end catch throw:{?MODULE, Return} -> Return end. -prep_and_validate(Db, not_found, Doc, UpdateType) -> - case Doc#doc.revs of - {0, []} -> - ok; - _ when UpdateType == replicated_changes -> - ok; - _ -> - ?RETURN({error, conflict}) - end, - prep_and_validate(Db, Doc, fun() -> nil end); +update_doc_interactive(Db, Doc0, _Options) -> + DocRevId = doc_to_revid(Doc0), -prep_and_validate(Db, FDI, Doc, interactive_edit) -> - #doc{ - revs = {Start, Revs} - } = Doc, - - Leafs = couch_key_tree:get_all_leafs(FDI#full_doc_info.rev_tree), - LeafRevs = lists:map(fun({_Leaf, {LeafStart, [LeafRev | _] = Path}}) -> - {{LeafStart, LeafRev}, Path} - end, Leafs), - - GetDocFun = case Revs of - [PrevRev | _] -> - case lists:keyfind({Start, PrevRev}, 1, LeafRevs) of - {{Start, PrevRev}, Path} -> - fun() -> - fabric2_fdb:get_doc_body(Db, Doc#doc.id, {Start, Path}) - end; - false -> - ?RETURN({error, conflict}) + {Winner, MaybeNewWinner} = case Doc0#doc.deleted of + true -> + case fabric2_fdb:get_winning_revs(Db, Doc0, 2) of + [] -> + {not_found, not_found}; + [Winner0] -> + {Winner0, not_found}; + [Winner0, MaybeNewWinner0] -> + {Winner0, MaybeNewWinner0} end; - [] -> - case FDI#full_doc_info.deleted of - true -> - fun() -> nil end; - false -> + false -> + case fabric2_fdb:get_winning_revs(Db, Doc0, 1) of + [] -> + {not_found, not_found}; + [Winner0] -> + {Winner0, not_found} + end + end, + + {Doc1, ExtendedRevInfo} = case Winner of + not_found when DocRevId == {0, <<>>} -> + {Doc0, not_found}; + #{winner := true, deleted := true} when DocRevId == {0, <<>>} -> + {WPos, WRev} = maps:get(rev_id, Winner), + {Doc0#doc{revs = {WPos, [WRev]}}, Winner}; + #{winner := true, deleted := false} when DocRevId == {0, <<>>} -> + ?RETURN({error, conflict}); + #{winner := true, deleted := false, rev_id := DocRevId} -> + {Doc0, Winner}; + #{winner := true, rev_id := WRevId} when WRevId /= DocRevId -> + case fabric2_fdb:get_non_deleted_rev(Db, Doc0) of + not_found -> + ?RETURN({error, conflict}); + #{deleted := false, rev_id := DocRevId} = PrevRevInfo-> + {Doc0, PrevRevInfo}; + #{deleted := true} -> ?RETURN({error, conflict}) end end, - prep_and_validate(Db, Doc, GetDocFun); -prep_and_validate(Db, FDI, Doc, replicated_changes) -> - #full_doc_info{ - rev_tree = RevTree - } = FDI, - OldLeafs = couch_key_tree:get_all_leafs_full(RevTree), - OldLeafsLU = [{Start, RevId} || {Start, [{RevId, _} | _]} <- OldLeafs], + #doc{ + deleted = NewDeleted, + revs = {NewRevPos, [NewRev | NewRevPath]} + } = Doc2 = prep_and_validate_interactive(Db, Doc1, ExtendedRevInfo), + + NewRevInfo = #{ + winner => undefined, + deleted => NewDeleted, + rev_id => {NewRevPos, NewRev}, + rev_path => NewRevPath, + sequence => undefined, + branch_count => undefined + }, - NewPath = couch_doc:to_path(Doc), - NewRevTree = couch_key_tree:merge(RevTree, NewPath), + AllRevInfos = [NewRevInfo, Winner, MaybeNewWinner], + {NewWinner, ToUpdate} = interactive_winner(AllRevInfos, ExtendedRevInfo), - Leafs = couch_key_tree:get_all_leafs_full(NewRevTree), - LeafRevsFull = lists:map(fun({Start, [{RevId, _} | _]} = FullPath) -> - [{{Start, RevId}, FullPath}] - end, Leafs), - LeafRevsFullDict = dict:from_list(LeafRevsFull), + ok = fabric2_fdb:write_doc_interactive( + Db, + Doc2, + NewWinner, + Winner, + ToUpdate, + [ExtendedRevInfo] + ), - #doc{ - revs = {DocStart, [DocRev | _]} - } = Doc, - DocRevId = {DocStart, DocRev}, - - IsOldLeaf = lists:member(DocRevId, OldLeafsLU), - GetDocFun = case dict:find(DocRevId, LeafRevsFullDict) of - {ok, {DocStart, RevPath}} when not IsOldLeaf -> - % An incoming replicated edit only sends us leaf - % nodes which may have included multiple updates - % we haven't seen locally. Thus we have to search - % back through the tree to find the first edit - % we do know about. - case find_prev_known_rev(DocStart, RevPath) of - not_found -> fun() -> nil end; - PrevRevs -> fun() -> - fabric2_fdb:get_doc_body(Db, Doc#doc.id, PrevRevs) - end - end; - _ -> - % The update merged to an internal node that we - % already know about which means we're done with - % this update. - ?RETURN({ok, []}) - end, + {ok, {NewRevPos, NewRev}}. - prep_and_validate(Db, Doc, GetDocFun). +prep_and_validate_interactive(Db, Doc, ExtendedRevInfo) -> + HasStubs = couch_doc:has_stubs(Doc), + HasVDUs = [] /= maps:get(validate_doc_update_funs, Db), + IsDDoc = case Doc#doc.id of + <> -> true; + _ -> false + end, -prep_and_validate(Db, Doc, GetDocBody) -> - NewDoc = case couch_doc:has_stubs(Doc) of + PrevDoc = case HasStubs orelse (HasVDUs and not IsDDoc) of true -> - case GetDocBody() of - #doc{} = PrevDoc -> - couch_doc:merge_stubs(Doc, PrevDoc); - _ -> - % Force a missing stubs error - couch_doc:mege_stubs(Doc, #doc{}) + case fabric2_fdb:get_doc_body(Db, Doc, ExtendedRevInfo) of + #doc{} = Doc -> Doc; + {not_found, _} -> #doc{} end; false -> - Doc + nil end, - validate_doc_update(Db, NewDoc, GetDocBody), - NewDoc. - - -merge_rev_tree(FDI, Doc, interactive_edit) when FDI#full_doc_info.deleted -> - % We're recreating a document that was previously - % deleted. To check that this is a recreation from - % the root we assert that the new document has a - % revision depth of 1 (this is to avoid recreating a - % doc from a previous internal revision) and is also - % not deleted. To avoid expanding the revision tree - % unnecessarily we create a new revision based on - % the winning deleted revision. - - {RevDepth, _} = Doc#doc.revs, - case RevDepth == 1 andalso not Doc#doc.deleted of - true -> - % Update the new doc based on revisions in OldInfo - #doc_info{revs=[WinningRev | _]} = couch_doc:to_doc_info(FDI), - #rev_info{rev={OldPos, OldRev}} = WinningRev, - Body = case fabric2_util:get_value(comp_body, Doc#doc.meta) of - CompBody when is_binary(CompBody) -> - couch_compress:decompress(CompBody); - _ -> - Doc#doc.body - end, - NewDoc = new_revid(Doc#doc{ - revs = {OldPos, [OldRev]}, - body = Body - }), - - % Merge our modified new doc into the tree - #full_doc_info{rev_tree = RevTree} = FDI, - case couch_key_tree:merge(RevTree, couch_doc:to_path(NewDoc)) of - {NewRevTree, new_leaf} -> - % We changed the revision id so inform the caller - NewFDI = FDI#full_doc_info{ - rev_tree = NewRevTree, - deleted = false - }, - {NewFDI, NewDoc}; - _ -> - throw(doc_recreation_failed) - end; - _ -> - ?RETURN({error, conflict}) - end; -merge_rev_tree(FDI, Doc, interactive_edit) -> - % We're attempting to merge a new revision into an - % undeleted document. To not be a conflict we require - % that the merge results in extending a branch. - - RevTree = FDI#full_doc_info.rev_tree, - case couch_key_tree:merge(RevTree, couch_doc:to_path(Doc)) of - {NewRevTree, new_leaf} when not Doc#doc.deleted -> - NewFDI = FDI#full_doc_info{ - rev_tree = NewRevTree, - deleted = false - }, - {NewFDI, Doc}; - {NewRevTree, new_leaf} when Doc#doc.deleted -> - % We have to check if we just deleted this - % document completely or if it was a conflict - % resolution. - NewFDI = FDI#full_doc_info{ - rev_tree = NewRevTree, - deleted = couch_doc:is_deleted(NewRevTree) - }, - {NewFDI, Doc}; - _ -> - ?RETURN({error, conflict}) - end; -merge_rev_tree(FDI, Doc, replicated_changes) -> - % We're merging in revisions without caring about - % conflicts. Most likely this is a replication update. - RevTree = FDI#full_doc_info.rev_tree, - {NewRevTree, _} = couch_key_tree:merge(RevTree, couch_doc:to_path(Doc)), - NewFDI = FDI#full_doc_info{ - rev_tree = NewRevTree, - deleted = couch_doc:is_deleted(NewRevTree) - }, - % If a replicated change did not modify the revision - % tree then we've got nothing else to do. - if NewFDI /= FDI -> ok; true -> - ?RETURN({ok, []}) + + MergedDoc = if not HasStubs -> Doc; true -> + couch_doc:merge_stubs(Doc, PrevDoc) + end, + + validate_doc_update(Db, MergedDoc, PrevDoc), + new_revid(MergedDoc). + + +interactive_winner(RevInfos0, ExtendedRevInfo) -> + % Fetch the current winner so we can copy + % the current branch count + BranchCount = case [W || #{winner := true} = W <- RevInfos0] of + [] -> + % Creating a doc, the branch count is + % now 1 by definition + 1; + [#{winner := true, branch_count := Count}] -> + % Interactive edits can never create new + % branches so we just copy the current Count + Count + end, + + % Remove the previous winner if it was updated + RevInfos1 = RevInfos0 -- [ExtendedRevInfo], + + % Remove not_found if we didn't need or have a + % MaybeNewWinner + RevInfos2 = RevInfos1 -- [not_found], + + SortFun = fun(A, B) -> + #{ + deleted := DeletedA, + rev_id := RevIdA + } = A, + #{ + deleted := DeletedB, + rev_id := RevIdB + } = B, + {not DeletedA, RevIdA} > {not DeletedB, RevIdB} end, - {NewFDI, Doc}. + [Winner0 | Rest0] = lists:sort(SortFun, RevInfos2), + Winner = Winner0#{ + winner := true, + branch_count := BranchCount + }, + {Winner, [R#{winner := false} || R <- Rest0]}. + + +update_doc_replicated(_Db, _Doc, _Options) -> + erlang:error(not_implemented). validate_doc_update(Db, #doc{id = <<"_design/", _/binary>>} = Doc, _) -> @@ -856,18 +767,17 @@ validate_doc_update(Db, #doc{id = <<"_design/", _/binary>>} = Doc, _) -> end; validate_doc_update(_Db, #doc{id = <<"_local/", _/binary>>}, _) -> ok; -validate_doc_update(Db, Doc, GetDiskDocFun) -> +validate_doc_update(Db, Doc, PrevDoc) -> #{ security_doc := Security, user_ctx := UserCtx, validate_doc_update_funs := VDUs } = Db, Fun = fun() -> - DiskDoc = GetDiskDocFun(), JsonCtx = fabric2_util:user_ctx_to_json(UserCtx), try lists:map(fun(VDU) -> - case VDU(Doc, DiskDoc, JsonCtx, Security) of + case VDU(Doc, PrevDoc, JsonCtx, Security) of ok -> ok; Error -> ?RETURN(Error) end @@ -897,20 +807,20 @@ validate_ddoc(Db, DDoc) -> end. -find_prev_known_rev(_Pos, []) -> - not_found; -find_prev_known_rev(Pos, [{_Rev, #doc{}} | RestPath]) -> - % doc records are skipped because these are the result - % of replication sending us an update. We're only interested - % in what we have locally since we're comparing attachment - % stubs. The replicator should never do this because it - % should only ever send leaves but the possibility exists - % so we account for it. - find_prev_known_rev(Pos - 1, RestPath); -find_prev_known_rev(Pos, [{_Rev, ?REV_MISSING} | RestPath]) -> - find_prev_known_rev(Pos - 1, RestPath); -find_prev_known_rev(Pos, [{_Rev, #leaf{}} | _] = DocPath) -> - {Pos, [Rev || {Rev, _Val} <- DocPath]}. +%% find_prev_known_rev(_Pos, []) -> +%% not_found; +%% find_prev_known_rev(Pos, [{_Rev, #doc{}} | RestPath]) -> +%% % doc records are skipped because these are the result +%% % of replication sending us an update. We're only interested +%% % in what we have locally since we're comparing attachment +%% % stubs. The replicator should never do this because it +%% % should only ever send leaves but the possibility exists +%% % so we account for it. +%% find_prev_known_rev(Pos - 1, RestPath); +%% find_prev_known_rev(Pos, [{_Rev, ?REV_MISSING} | RestPath]) -> +%% find_prev_known_rev(Pos - 1, RestPath); +%% find_prev_known_rev(Pos, [{_Rev, #leaf{}} | _] = DocPath) -> +%% {Pos, [Rev || {Rev, _Val} <- DocPath]}. transactional(DbName, Options, Fun) -> @@ -926,3 +836,10 @@ with_tx(#{tx := undefined} = Db, Fun) -> with_tx(#{tx := {erlfdb_transaction, _}} = Db, Fun) -> Fun(Db). + + +doc_to_revid(#doc{revs = Revs}) -> + case Revs of + {0, []} -> {0, <<>>}; + {RevPos, [Rev | _]} -> {RevPos, Rev} + end. diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl index f5f2ba0..94d1ef4 100644 --- a/src/fabric/src/fabric2_fdb.erl +++ b/src/fabric/src/fabric2_fdb.erl @@ -31,13 +31,12 @@ get_stat/2, incr_stat/3, - get_full_doc_info/2, - get_doc_body/3, + get_winning_revs/3, + get_non_deleted_rev/2, - store_doc/3, + get_doc_body/3, - add_to_all_docs/3, - rem_from_all_docs/2, + write_doc_interactive/6, fold_docs/4, fold_changes/5, @@ -77,6 +76,11 @@ -define(DB_LOCAL_DOCS, 22). +% Versions + +-define(CURR_REV_FORMAT, 0). + + % Various utility macros -define(REQUIRE_TX(Db), {erlfdb_transaction, _} = maps:get(tx, Db)). @@ -351,86 +355,162 @@ incr_stat(#{} = Db, StatKey, Increment) when is_integer(Increment) -> erlfdb:add(Tx, Key, Increment). -get_full_doc_info(#{} = Db, DocId) -> +get_winning_revs(Db, #doc{} = Doc, NumRevs) -> + get_winning_revs(Db, Doc#doc.id, NumRevs); + +get_winning_revs(#{} = Db, DocId, NumRevs) -> ?REQUIRE_CURRENT(Db), #{ tx := Tx, db_prefix := DbPrefix } = Db, - Key = erlfdb_tuple:pack({?DB_DOCS, DocId}, DbPrefix), - Val = erlfdb:wait(erlfdb:get(Tx, Key)), - fdb_to_fdi(Db, DocId, Val). + Prefix = erlfdb_tuple:pack({?DB_REVS, DocId}, DbPrefix), + Options = [{reverse, true}, {limit, NumRevs}], + Future = erlfdb:get_range_startswith(Tx, Prefix, Options), + lists:map(fun({K, V}) -> + Key = erlfdb_tuple:unpack(K, DbPrefix), + Val = erlfdb_tuple:unpack(V), + fdb_to_revinfo(Key, Val) + end, erlfdb:wait(Future)). -get_doc_body(#{} = Db, DocId, {Pos, [Rev | _] = Path}) -> +get_non_deleted_rev(#{} = Db, #doc{} = Doc) -> ?REQUIRE_CURRENT(Db), #{ tx := Tx, db_prefix := DbPrefix } = Db, - Key = erlfdb_tuple:pack({?DB_REVS, DocId, Pos, Rev}, DbPrefix), - Val = erlfdb:wait(erlfdb:get(Tx, Key)), - fdb_to_doc(Db, DocId, Pos, Path, Val). + #doc{ + revs = {RevPos, [Rev | _]} + } = Doc, + BaseKey = {?DB_REVS, Doc#doc.id, true, RevPos, Rev}, + Key = erlfdb_tuple:pack(BaseKey, DbPrefix), + case erlfdb:wait(erlfdb:get(Tx, Key)) of + not_found -> + not_found; + Val -> + fdb_to_revinfo(Key, erlfdb_tuple:unpack(Val)) + end. -store_doc(#{} = Db, #full_doc_info{} = FDI, #doc{} = Doc) -> + +get_doc_body(#{} = Db, DocId, RevInfo) -> ?REQUIRE_CURRENT(Db), #{ tx := Tx, db_prefix := DbPrefix } = Db, - #full_doc_info{ - id = DocId, - update_seq = OldUpdateSeq - } = FDI, + #{ + rev_id := {RevPos, Rev}, + rev_path := RevPath + } = RevInfo, + + Key = erlfdb_tuple:pack({?DB_DOCS, DocId, RevPos, Rev}, DbPrefix), + Val = erlfdb:wait(erlfdb:get(Tx, Key)), + fdb_to_doc(Db, DocId, RevPos, [Rev | RevPath], Val). + + +write_doc_interactive(Db, Doc, NewWinner, OldWinner, UpdateInfos, RemInfos) -> + ?REQUIRE_CURRENT(Db), + #{ + tx := Tx, + db_prefix := DbPrefix + } = Db, #doc{ - revs = {Pos, [Rev | _]}, + id = DocId, deleted = Deleted } = Doc, - % Delete old entry in changes feed - OldSeqKey = erlfdb_tuple:pack({?DB_CHANGES, OldUpdateSeq}, DbPrefix), - erlfdb:clear(Tx, OldSeqKey), + NewRevId = maps:get(rev_id, NewWinner), + + % Update the revision tree + + {WKey, WVal} = revinfo_to_fdb(DbPrefix, DocId, NewWinner), + ok = erlfdb:set_versionstamped_value(Tx, WKey, WVal), + + lists:foreach(fun(RI) -> + {K, V} = revinfo_to_fdb(DbPrefix, DocId, RI), + ok = erlfdb:set(Tx, K, V) + end, UpdateInfos), + + lists:foreach(fun(RI) -> + if RI == not_found -> ok; true -> + {K, _} = revinfo_to_fdb(DbPrefix, DocId, RI), + ok = erlfdb:clear(Tx, K) + end + end, RemInfos), + + % Update all_docs index + + UpdateStatus = case {OldWinner, NewWinner} of + {not_found, #{deleted := false}} -> + created; + {#{deleted := true}, #{deleted := false}} -> + recreated; + {#{deleted := false}, #{deleted := false}} -> + updated; + {#{deleted := false}, #{deleted := true}} -> + deleted + end, + + case UpdateStatus of + Status when Status == created orelse Status == recreated -> + ADKey = erlfdb_tuple:pack({?DB_ALL_DOCS, DocId}, DbPrefix), + ADVal = erlfdb_tuple:pack(NewRevId), + ok = erlfdb:set(Tx, ADKey, ADVal); + deleted -> + ADKey = erlfdb_tuple:pack({?DB_ALL_DOCS, DocId}, DbPrefix), + ok = erlfdb:clear(Tx, ADKey); + updated -> + ok + end, + + % Update the changes index + + if OldWinner == not_found -> ok; true -> + OldSeq = maps:get(sequence, OldWinner), + OldSeqKey = erlfdb_tuple:pack({?DB_CHANGES, OldSeq}, DbPrefix), + erlfdb:clear(Tx, OldSeqKey) + end, - % Add new entry to changes feed NewSeqKey = erlfdb_tuple:pack_vs({?DB_CHANGES, ?UNSET_VS}, DbPrefix), - NewSeqVal = erlfdb_tuple:pack({DocId, Deleted, {Pos, Rev}}), + NewSeqVal = erlfdb_tuple:pack({DocId, Deleted, NewRevId}), erlfdb:set_versionstamped_key(Tx, NewSeqKey, NewSeqVal), - % Write document data - {NewDocKey, NewDocVal} = doc_to_fdb(Db, Doc), - erlfdb:set(Tx, NewDocKey, NewDocVal), + % Store document metadata and body - % Update revision tree entry - {NewFDIKey, NewFDIVal} = fdi_to_fdb(Db, FDI), - erlfdb:set_versionstamped_value(Tx, NewFDIKey, NewFDIVal). + ok = write_doc_body(Db, Doc), + % Update doc counts -add_to_all_docs(#{} = Db, DocId, {Pos, Rev}) -> - ?REQUIRE_CURRENT(Db), - #{ - tx := Tx, - db_prefix := DbPrefix - } = Db, + case UpdateStatus of + created -> + incr_stat(Db, <<"doc_count">>, 1); + recreated -> + incr_stat(Db, <<"doc_count">>, 1), + incr_stat(Db, <<"doc_del_count">>, -1); + deleted -> + incr_stat(Db, <<"doc_count">>, -1), + incr_stat(Db, <<"doc_del_count">>, 1); + updated -> + ok + end, - Key = erlfdb_tuple:pack({?DB_ALL_DOCS, DocId}, DbPrefix), - Val = erlfdb_tuple:pack({Pos, Rev}), - erlfdb:set(Tx, Key, Val). + ok. -rem_from_all_docs(#{} = Db, DocId) -> +write_doc_body(#{} = Db, #doc{} = Doc) -> ?REQUIRE_CURRENT(Db), #{ - tx := Tx, - db_prefix := DbPrefix + tx := Tx } = Db, - Key = erlfdb_tuple:pack({?DB_ALL_DOCS, DocId}, DbPrefix), - ok = erlfdb:clear(Tx, Key). + {NewDocKey, NewDocVal} = doc_to_fdb(Db, Doc), + erlfdb:set(Tx, NewDocKey, NewDocVal). fold_docs(#{} = Db, UserFun, UserAcc0, _Options) -> @@ -530,6 +610,57 @@ bump_metadata_version(Tx) -> erlfdb:set_versionstamped_value(Tx, ?METADATA_VERSION_KEY, <<0:112>>). +revinfo_to_fdb(DbPrefix, DocId, #{winner := true} = RevId) -> + #{ + deleted := Deleted, + rev_id := {RevPos, Rev}, + rev_path := RevPath, + branch_count := BranchCount + } = RevId, + Key = {?DB_REVS, DocId, not Deleted, RevPos, Rev}, + Val = {?CURR_REV_FORMAT, ?UNSET_VS, BranchCount, list_to_tuple(RevPath)}, + KBin = erlfdb_tuple:pack(Key, DbPrefix), + VBin = erlfdb_tuple:pack_vs(Val), + {KBin, VBin}; + +revinfo_to_fdb(DbPrefix, DocId, #{} = RevId) -> + #{ + deleted := Deleted, + rev_id := {RevPos, Rev}, + rev_path := RevPath + } = RevId, + Key = {?DB_REVS, DocId, not Deleted, RevPos, Rev}, + Val = {?CURR_REV_FORMAT, list_to_tuple(RevPath)}, + KBin = erlfdb_tuple:pack(Key, DbPrefix), + VBin = erlfdb_tuple:pack(Val), + {KBin, VBin}. + + +fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _} = Val) -> + {?DB_REVS, _DocId, NotDeleted, RevPos, Rev} = Key, + {_RevFormat, Sequence, BranchCount, RevPath} = Val, + #{ + winner => true, + deleted => not NotDeleted, + rev_id => {RevPos, Rev}, + rev_path => tuple_to_list(RevPath), + sequence => Sequence, + branch_count => BranchCount + }; + +fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _} = Val) -> + {?DB_REVS, _DocId, NotDeleted, RevPos, Rev} = Key, + {_RevFormat, RevPath} = Val, + #{ + winner => false, + deleted => not NotDeleted, + rev_id => {RevPos, Rev}, + rev_path => tuple_to_list(RevPath), + sequence => undefined, + branch_count => undefined + }. + + doc_to_fdb(Db, #doc{} = Doc) -> #{ db_prefix := DbPrefix @@ -543,7 +674,7 @@ doc_to_fdb(Db, #doc{} = Doc) -> deleted = Deleted } = Doc, - Key = erlfdb_tuple:pack({?DB_REVS, Id, Start, Rev}, DbPrefix), + Key = erlfdb_tuple:pack({?DB_DOCS, Id, Start, Rev}, DbPrefix), Val = {Body, Atts, Deleted}, {Key, term_to_binary(Val, [{minor_version, 1}])}. @@ -559,56 +690,3 @@ fdb_to_doc(_Db, DocId, Pos, Path, Bin) when is_binary(Bin) -> }; fdb_to_doc(_Db, _DocId, _Pos, _Path, not_found) -> {not_found, missing}. - - -fdi_to_fdb(Db, #full_doc_info{} = FDI) -> - #{ - db_prefix := DbPrefix - } = Db, - - #full_doc_info{ - id = Id, - deleted = Deleted, - rev_tree = RevTree - } = flush_tree(FDI), - - Key = erlfdb_tuple:pack({?DB_DOCS, Id}, DbPrefix), - RevTreeBin = term_to_binary(RevTree, [{minor_version, 1}]), - ValTuple = { - Deleted, - RevTreeBin, - ?UNSET_VS - }, - Val = erlfdb_tuple:pack_vs(ValTuple), - {Key, Val}. - - -flush_tree(FDI) -> - #full_doc_info{ - rev_tree = Unflushed - } = FDI, - - Flushed = couch_key_tree:map(fun(_Rev, Value) -> - case Value of - #doc{deleted = Del} -> #leaf{deleted = Del}; - _ -> Value - end - end, Unflushed), - - FDI#full_doc_info{ - rev_tree = Flushed - }. - - -fdb_to_fdi(_Db, Id, Bin) when is_binary(Bin) -> - {Deleted, RevTreeBin, {versionstamp, V, B}} = erlfdb_tuple:unpack(Bin), - RevTree = binary_to_term(RevTreeBin, [safe]), - UpdateSeq = <>, - #full_doc_info{ - id = Id, - deleted = Deleted, - rev_tree = RevTree, - update_seq = UpdateSeq - }; -fdb_to_fdi(_Db, _Id, not_found) -> - not_found.