Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@couchdb.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: davisp@apache.org
To: commits@couchdb.apache.org
Date: Wed, 01 Feb 2017 16:11:58 -0000
Message-Id: <1361af5b93ec42f9808a419592f8566a@git.apache.org>
In-Reply-To: <441831948ccc450b9b9ee5d84e0f6875@git.apache.org>
References: <441831948ccc450b9b9ee5d84e0f6875@git.apache.org>
Subject: [3/6] couch commit: updated
 refs/heads/45918-pluggable-storage-engines to 4990951
archived-at: Wed, 01 Feb 2017 16:11:59 -0000

http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/3aee939f/src/couch_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_util.erl b/src/couch_util.erl
index 6001ae2..29e720e 100644
--- a/src/couch_util.erl
+++ b/src/couch_util.erl
@@ -12,7 +12,7 @@
 
 -module(couch_util).
 
--export([priv_dir/0, normpath/1]).
+-export([priv_dir/0, normpath/1, fold_files/5]).
 -export([should_flush/0, should_flush/1, to_existing_atom/1]).
 -export([rand32/0, implode/2, collate/2, collate/3]).
 -export([abs_pathname/1,abs_pathname/2, trim/1, drop_dot_couch_ext/1]).
@@ -33,6 +33,7 @@
 -export([find_in_binary/2]).
 -export([callback_exists/3, validate_callback_exists/3]).
 -export([with_proc/4]).
+-export([check_md5/2]).
 
 -include_lib("couch/include/couch_db.hrl").
 
@@ -62,6 +63,37 @@ normparts(["." | RestParts], Acc) ->
 normparts([Part | RestParts], Acc) ->
     normparts(RestParts, [Part | Acc]).
 
+
+fold_files(Dir, RegExp, Recursive, Fun, Acc) ->
+    {ok, Re} = re:compile(RegExp, [unicode]),
+    fold_files1(Dir, Re, Recursive, Fun, Acc).
+
+fold_files1(Dir, RegExp, Recursive, Fun, Acc) ->
+    case file:list_dir(Dir) of
+        {ok, Files} ->
+            fold_files2(Files, Dir, RegExp, Recursive, Fun, Acc);
+        {error, _} ->
+            Acc
+    end.
+
+fold_files2([], _Dir, _RegExp, _Recursive, _Fun, Acc) ->
+    Acc;
+fold_files2([File | Rest], Dir, RegExp, Recursive, Fun, Acc0) ->
+    FullName = filename:join(Dir, File),
+    case (catch re:run(File, RegExp, [{capture, none}])) of
+        match ->
+            Acc1 = Fun(FullName, Acc0),
+            fold_files2(Rest, Dir, RegExp, Recursive, Fun, Acc1);
+        _ ->
+            case Recursive andalso filelib:is_dir(FullName) of
+                true ->
+                    Acc1 = fold_files1(FullName, RegExp, Recursive, Fun, Acc0),
+                    fold_files2(Rest, Dir, RegExp, Recursive, Fun, Acc1);
+                false ->
+                    fold_files2(Rest, Dir, RegExp, Recursive, Fun, Acc0)
+            end
+    end.
+
 % works like list_to_existing_atom, except can be list or binary and it
 % gives you the original value instead of an error if no existing atom.
 to_existing_atom(V) when is_list(V) ->
@@ -571,6 +603,12 @@ validate_callback_exists(Module, Function, Arity) ->
             {undefined_callback, CallbackStr, {Module, Function, Arity}}})
     end.
 
+
+check_md5(_NewSig, <<>>) -> ok;
+check_md5(Sig, Sig) -> ok;
+check_md5(_, _) -> throw(md5_mismatch).
+
+
 ensure_loaded(Module) when is_atom(Module) ->
     case code:ensure_loaded(Module) of
     {module, Module} ->

http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/3aee939f/test/couch_stream_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_stream_tests.erl b/test/couch_stream_tests.erl
index 3d7bf09..64ba014 100644
--- a/test/couch_stream_tests.erl
+++ b/test/couch_stream_tests.erl
@@ -14,10 +14,11 @@
 
 -include_lib("couch/include/couch_eunit.hrl").
 
+-define(ENGINE, {couch_bt_engine_stream, {Fd, []}}).
 
 setup() ->
     {ok, Fd} = couch_file:open(?tempfile(), [create, overwrite]),
-    {ok, Stream} = couch_stream:open(Fd),
+    {ok, Stream} = couch_stream:open(?ENGINE, []),
     {Fd, Stream}.
 
 teardown({Fd, _}) ->
@@ -61,7 +62,8 @@ should_write_empty_binary({_, Stream}) ->
 
 should_return_file_pointers_on_close({_, Stream}) ->
     couch_stream:write(Stream, <<"foodfoob">>),
-    {Ptrs, _, _, _, _} = couch_stream:close(Stream),
+    {NewEngine, _, _, _, _} = couch_stream:close(Stream),
+    {ok, Ptrs} = couch_stream:to_disk_term(NewEngine),
     ?_assertEqual([{0, 8}], Ptrs).
 
 should_return_stream_size_on_close({_, Stream}) ->
@@ -71,39 +73,41 @@ should_return_stream_size_on_close({_, Stream}) ->
 
 should_return_valid_pointers({Fd, Stream}) ->
     couch_stream:write(Stream, <<"foodfoob">>),
-    {Ptrs, _, _, _, _} = couch_stream:close(Stream),
-    ?_assertEqual(<<"foodfoob">>, read_all(Fd, Ptrs)).
+    {NewEngine, _, _, _, _} = couch_stream:close(Stream),
+    ?_assertEqual(<<"foodfoob">>, read_all(NewEngine)).
 
 should_recall_last_pointer_position({Fd, Stream}) ->
     couch_stream:write(Stream, <<"foodfoob">>),
     {_, _, _, _, _} = couch_stream:close(Stream),
     {ok, ExpPtr} = couch_file:bytes(Fd),
-    {ok, Stream2} = couch_stream:open(Fd),
+    {ok, Stream2} = couch_stream:open(?ENGINE),
     ZeroBits = <<0:(8 * 10)>>,
     OneBits = <<1:(8 * 10)>>,
     ok = couch_stream:write(Stream2, OneBits),
     ok = couch_stream:write(Stream2, ZeroBits),
-    {Ptrs, 20, _, _, _} = couch_stream:close(Stream2),
+    {NewEngine, 20, _, _, _} = couch_stream:close(Stream2),
+    {ok, Ptrs} = couch_stream:to_disk_term(NewEngine),
     [{ExpPtr, 20}] = Ptrs,
     AllBits = iolist_to_binary([OneBits, ZeroBits]),
-    ?_assertEqual(AllBits, read_all(Fd, Ptrs)).
+    ?_assertEqual(AllBits, read_all(NewEngine)).
 
 should_stream_more_with_4K_chunk_size({Fd, _}) ->
-    {ok, Stream} = couch_stream:open(Fd, [{buffer_size, 4096}]),
+    {ok, Stream} = couch_stream:open(?ENGINE, [{buffer_size, 4096}]),
     lists:foldl(
         fun(_, Acc) ->
             Data = <<"a1b2c">>,
             couch_stream:write(Stream, Data),
             [Data | Acc]
         end, [], lists:seq(1, 1024)),
-    ?_assertMatch({[{0, 4100}, {4106, 1020}], 5120, _, _, _},
-                  couch_stream:close(Stream)).
+    {NewEngine, Length, _, _, _} = couch_stream:close(Stream),
+    {ok, Ptrs} = couch_stream:to_disk_term(NewEngine),
+    ?_assertMatch({[{0, 4100}, {4106, 1020}], 5120}, {Ptrs, Length}).
 
 should_stop_on_normal_exit_of_stream_opener({Fd, _}) ->
     RunnerPid = self(),
     OpenerPid = spawn(
         fun() ->
-            {ok, StreamPid} = couch_stream:open(Fd),
+            {ok, StreamPid} = couch_stream:open(?ENGINE),
             RunnerPid ! {pid, StreamPid}
         end),
     StreamPid = receive
@@ -115,6 +119,6 @@ should_stop_on_normal_exit_of_stream_opener({Fd, _}) ->
     ?_assertNot(is_process_alive(StreamPid)).
 
 
-read_all(Fd, PosList) ->
-    Data = couch_stream:foldl(Fd, PosList, fun(Bin, Acc) -> [Bin, Acc] end, []),
+read_all(Engine) ->
+    Data = couch_stream:foldl(Engine, fun(Bin, Acc) -> [Bin, Acc] end, []),
     iolist_to_binary(Data).

http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/3aee939f/test/couchdb_compaction_daemon_tests.erl
----------------------------------------------------------------------
diff --git a/test/couchdb_compaction_daemon_tests.erl b/test/couchdb_compaction_daemon_tests.erl
index 4c35fb7..9ece8fd 100644
--- a/test/couchdb_compaction_daemon_tests.erl
+++ b/test/couchdb_compaction_daemon_tests.erl
@@ -190,7 +190,7 @@ get_db_frag(DbName) ->
     {ok, Info} = couch_db:get_db_info(Db),
     couch_db:close(Db),
     FileSize = get_size(file, Info),
-    DataSize = get_size(external, Info),
+    DataSize = get_size(active, Info),
     {round((FileSize - DataSize) / FileSize * 100), FileSize}.
 
 get_view_frag(DbName) ->
@@ -198,7 +198,7 @@ get_view_frag(DbName) ->
     {ok, Info} = couch_mrview:get_info(Db, <<"_design/foo">>),
     couch_db:close(Db),
     FileSize = get_size(file, Info),
-    DataSize = get_size(external, Info),
+    DataSize = get_size(active, Info),
     {round((FileSize - DataSize) / FileSize * 100), FileSize}.
 
 get_size(Kind, Info) ->

http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/3aee939f/test/couchdb_views_tests.erl
----------------------------------------------------------------------
diff --git a/test/couchdb_views_tests.erl b/test/couchdb_views_tests.erl
index 7b04e85..f966645 100644
--- a/test/couchdb_views_tests.erl
+++ b/test/couchdb_views_tests.erl
@@ -544,22 +544,24 @@ has_doc(DocId1, Rows) ->
     lists:any(fun({R}) -> lists:member({<<"id">>, DocId}, R) end, Rows).
 
 backup_db_file(DbName) ->
-    DbDir = config:get("couchdb", "database_dir"),
-    DbFile = filename:join([DbDir, ?b2l(DbName) ++ ".couch"]),
-    {ok, _} = file:copy(DbFile, DbFile ++ ".backup"),
-    ok.
+    {ok, Db} = couch_db:open_int(DbName, []),
+    try
+        SrcPath = couch_db:get_path(Db),
+        Src = if
+            is_list(SrcPath) -> SrcPath;
+            true -> binary_to_list(SrcPath)
+        end,
+        ok = copy_tree(Src, Src ++ ".backup")
+    after
+        couch_db:close(Db)
+    end.
 
 restore_backup_db_file(DbName) ->
-    DbDir = config:get("couchdb", "database_dir"),
-
     {ok, Db} = couch_db:open_int(DbName, []),
+    Src = couch_db:get_path(Db),
     ok = couch_db:close(Db),
-    exit(Db#db.main_pid, shutdown),
-
-    DbFile = filename:join([DbDir, ?b2l(DbName) ++ ".couch"]),
-    ok = file:delete(DbFile),
-    ok = file:rename(DbFile ++ ".backup", DbFile),
-    ok.
+    exit(couch_db:pid(Db), shutdown),
+    ok = copy_tree(Src ++ ".backup", Src).
 
 compact_db(DbName) ->
     {ok, Db} = couch_db:open_int(DbName, []),
@@ -706,3 +708,22 @@ wait_indexer(IndexerPid) ->
                 ok
         end
     end).
+
+copy_tree(Src, Dst) ->
+    case filelib:is_dir(Src) of
+        true ->
+            {ok, Files} = file:list_dir(Src),
+            copy_tree(Files, Src, Dst);
+        false ->
+            ok = filelib:ensure_dir(Dst),
+            {ok, _} = file:copy(Src, Dst),
+            ok
+    end.
+
+copy_tree([], _Src, _Dst) ->
+    ok;
+copy_tree([File | Rest], Src, Dst) ->
+    FullSrc = filename:join(Src, File),
+    FullDst = filename:join(Dst, File),
+    ok = copy_tree(FullSrc, FullDst),
+    copy_tree(Rest, Src, Dst).