couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dav...@apache.org
Subject [couchdb] 24/31: Add total row count support
Date Tue, 23 Jul 2019 21:55:09 GMT
This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch prototype/views
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 81521e1e621dd5ca17a27d9c7d3893e9eb4224e7
Author: Paul J. Davis <paul.joseph.davis@gmail.com>
AuthorDate: Tue Jul 23 15:12:31 2019 -0500

    Add total row count support
---
 src/couch_views/include/couch_views.hrl           | 16 ++++----
 src/couch_views/src/couch_views_fdb.erl           | 50 ++++++++++++++++++++---
 src/couch_views/src/couch_views_reader.erl        |  9 ++--
 src/couch_views/test/couch_views_indexer_test.erl |  4 +-
 4 files changed, 59 insertions(+), 20 deletions(-)

diff --git a/src/couch_views/include/couch_views.hrl b/src/couch_views/include/couch_views.hrl
index 4fcc57e..525f62f 100644
--- a/src/couch_views/include/couch_views.hrl
+++ b/src/couch_views/include/couch_views.hrl
@@ -11,15 +11,15 @@
 % the License.
 
 % indexing
--define(VIEW_UPDATE_SEQ, 1).
--define(VIEW_ID_RANGE,   2).
--define(VIEW_MAP_RANGE,  3).
--define(VIEW_BUILDS,     4).
--define(VIEW_STATUS,     5).
--define(VIEW_WATCH,      6).
+-define(VIEW_UPDATE_SEQ, 0).
+-define(VIEW_ID_INFO, 1).
+-define(VIEW_ID_RANGE, 2).
+-define(VIEW_MAP_RANGE, 3).
 
--define(VIEW_ROW_KEY,    0).
--define(VIEW_ROW_VALUE,  1).
+-define(VIEW_ROW_COUNT, 0).
+
+-define(VIEW_ROW_KEY, 0).
+-define(VIEW_ROW_VALUE, 1).
 
 % jobs api
 -define(INDEX_JOB_TYPE, <<"views">>).
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index 09a9802..a0e4bd1 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -16,6 +16,8 @@
     get_update_seq/2,
     set_update_seq/3,
 
+    get_row_count/3,
+
     fold_map_idx/6,
 
     write_doc/4
@@ -59,6 +61,18 @@ set_update_seq(TxDb, Sig, Seq) ->
     ok = erlfdb:set(Tx, seq_key(DbPrefix, Sig), Seq).
 
 
+get_row_count(TxDb, #mrst{sig = Sig}, ViewId) ->
+    #{
+        tx := Tx,
+        db_prefix := DbPrefix
+    } = TxDb,
+
+    case erlfdb:wait(erlfdb:get(Tx, row_count_key(DbPrefix, Sig, ViewId))) of
+        not_found -> 0; % Can this happen?
+        CountBin -> ?bin2uint(CountBin)
+    end.
+
+
 fold_map_idx(TxDb, Sig, ViewId, Options, Callback, Acc0) ->
     #{
         db_prefix := DbPrefix
@@ -107,8 +121,9 @@ write_doc(TxDb, Sig, _ViewIds, #{deleted := true} = Doc) ->
     ExistingViewKeys = get_view_keys(TxDb, Sig, DocId),
 
     clear_id_idx(TxDb, Sig, DocId),
-    lists:foreach(fun({ViewId, ViewKeys}) ->
-        clear_map_idx(TxDb, Sig, ViewId, DocId, ViewKeys)
+    lists:foreach(fun({ViewId, TotalKeys, UniqueKeys}) ->
+        clear_map_idx(TxDb, Sig, ViewId, DocId, UniqueKeys),
+        update_row_count(TxDb, Sig, ViewId, -TotalKeys)
     end, ExistingViewKeys);
 
 write_doc(TxDb, Sig, ViewIds, Doc) ->
@@ -122,8 +137,17 @@ write_doc(TxDb, Sig, ViewIds, Doc) ->
     clear_id_idx(TxDb, Sig, DocId),
 
     lists:foreach(fun({ViewId, NewRows}) ->
-        ExistingKeys = fabric2_util:get_value(ViewId, ExistingViewKeys, []),
         update_id_idx(TxDb, Sig, ViewId, DocId, NewRows),
+
+        ExistingKeys = case lists:keyfind(ViewId, 1, ExistingViewKeys) of
+            {ViewId, TotalRows, EKeys} ->
+                Change = length(NewRows) - TotalRows,
+                update_row_count(TxDb, Sig, ViewId, Change),
+                EKeys;
+            false ->
+                update_row_count(TxDb, Sig, ViewId, length(NewRows)),
+                []
+        end,
         update_map_idx(TxDb, Sig, ViewId, DocId, ExistingKeys, NewRows)
     end, lists:zip(ViewIds, Results)).
 
@@ -251,7 +275,7 @@ update_id_idx(TxDb, Sig, ViewId, DocId, NewRows) ->
     couch_log:error("Updating ID index: ~p ~p ~p ~p", [ViewId, DocId, NewRows, Unique]),
 
     Key = id_idx_key(DbPrefix, Sig, DocId, ViewId),
-    Val = couch_views_encoding:encode(Unique),
+    Val = couch_views_encoding:encode([length(NewRows), Unique]),
     ok = erlfdb:set(Tx, Key, Val).
 
 
@@ -289,16 +313,30 @@ get_view_keys(TxDb, Sig, DocId) ->
     lists:map(fun({K, V}) ->
         {?DB_VIEWS, Sig, ?VIEW_ID_RANGE, DocId, ViewId} =
                 erlfdb_tuple:unpack(K, DbPrefix),
-        ViewKeys = couch_views_encoding:decode(V),
-        {ViewId, ViewKeys}
+        [TotalKeys, UniqueKeys] = couch_views_encoding:decode(V),
+        {ViewId, TotalKeys, UniqueKeys}
     end, erlfdb:get_range(Tx, Start, End, [])).
 
 
+update_row_count(TxDb, Sig, ViewId, Increment) ->
+    #{
+        tx := Tx,
+        db_prefix := DbPrefix
+    } = TxDb,
+    Key = row_count_key(DbPrefix, Sig, ViewId),
+    erlfdb:add(Tx, Key, Increment).
+
+
 seq_key(DbPrefix, Sig) ->
     Key = {?DB_VIEWS, Sig, ?VIEW_UPDATE_SEQ},
     erlfdb_tuple:pack(Key, DbPrefix).
 
 
+row_count_key(DbPrefix, Sig, ViewId) ->
+    Key = {?DB_VIEWS, Sig, ?VIEW_ID_INFO, ViewId, ?VIEW_ROW_COUNT},
+    erlfdb_tuple:pack(Key, DbPrefix).
+
+
 id_idx_key(DbPrefix, Sig, DocId, ViewId) ->
     Key = {?DB_VIEWS, Sig, ?VIEW_ID_RANGE, DocId, ViewId},
     erlfdb_tuple:pack(Key, DbPrefix).
diff --git a/src/couch_views/src/couch_views_reader.erl b/src/couch_views/src/couch_views_reader.erl
index ce5097ba..8bbe1fc 100644
--- a/src/couch_views/src/couch_views_reader.erl
+++ b/src/couch_views/src/couch_views_reader.erl
@@ -34,11 +34,12 @@ read(Db, Mrst, ViewName, UserCallback, UserAcc0, Args) ->
     Fun = fun handle_row/4,
 
     try
-        % Need to add total_rows support
-        Meta = {meta, [{total_rows, null}, {offset, null}]},
-        UserAcc1 = maybe_stop(UserCallback(Meta, UserAcc0)),
-
         fabric2_fdb:transactional(Db, fun(TxDb) ->
+            TotalRows = couch_views_fdb:get_row_count(TxDb, Mrst, ViewId),
+
+            Meta = {meta, [{total, TotalRows}, {offset, null}]},
+            UserAcc1 = maybe_stop(UserCallback(Meta, UserAcc0)),
+
             Acc0 = #{
                 db => TxDb,
                 skip => Args#mrargs.skip,
diff --git a/src/couch_views/test/couch_views_indexer_test.erl b/src/couch_views/test/couch_views_indexer_test.erl
index e6dfdc4..fa0d99e 100644
--- a/src/couch_views/test/couch_views_indexer_test.erl
+++ b/src/couch_views/test/couch_views_indexer_test.erl
@@ -157,7 +157,7 @@ updated_docs_are_reindexed(Db) ->
     DbName = fabric2_db:name(Db),
     {ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
     Sig = Mrst#mrst.sig,
-    Expect = [{0, [1]}, {1, []}],
+    Expect = [{0, 1, [1]}, {1, 0, []}],
     fabric2_fdb:transactional(Db, fun(TxDb) ->
         ?assertEqual(
                 Expect,
@@ -214,7 +214,7 @@ updated_docs_without_changes_are_reindexed(Db) ->
     DbName = fabric2_db:name(Db),
     {ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
     Sig = Mrst#mrst.sig,
-    Expect = [{0, [0]}, {1, []}],
+    Expect = [{0, 1, [0]}, {1, 0, []}],
     fabric2_fdb:transactional(Db, fun(TxDb) ->
         ?assertEqual(
                 Expect,


Mime
View raw message