couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fdman...@apache.org
Subject [1/3] git commit: Make couch_file:flush/1 calls automatic
Date Tue, 29 Nov 2011 14:13:28 GMT
Updated Branches:
  refs/heads/COUCHDB-1342 62dde7e9f -> 4f7336ed3


Make couch_file:flush/1 calls automatic


Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/4f7336ed
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/4f7336ed
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/4f7336ed

Branch: refs/heads/COUCHDB-1342
Commit: 4f7336ed30c0a34823cb332190b57a608725a21c
Parents: 0b89f5a
Author: Filipe David Borba Manana <fdmanana@apache.org>
Authored: Tue Nov 29 00:12:10 2011 +0000
Committer: Filipe David Borba Manana <fdmanana@apache.org>
Committed: Tue Nov 29 13:58:43 2011 +0000

----------------------------------------------------------------------
 src/couch_mrview/src/couch_mrview_index.erl   |    3 +-
 src/couch_mrview/src/couch_mrview_updater.erl |    8 +---
 src/couch_mrview/src/couch_mrview_util.erl    |    1 -
 src/couchdb/couch_db_updater.erl              |    9 +----
 src/couchdb/couch_file.erl                    |   37 +++++++++++++++----
 test/etap/010-file-basics.t                   |    6 ---
 test/etap/011-file-headers.t                  |    7 ----
 test/etap/020-btree-basics.t                  |   10 -----
 test/etap/021-btree-reductions.t              |    1 -
 test/etap/050-stream.t                        |    1 -
 10 files changed, 33 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couch_mrview/src/couch_mrview_index.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl
index 21d38d1..1e7b7a3 100644
--- a/src/couch_mrview/src/couch_mrview_index.erl
+++ b/src/couch_mrview/src/couch_mrview_index.erl
@@ -128,8 +128,7 @@ finish_update(State) ->
 
 commit(State) ->
     Header = {State#mrst.sig, couch_mrview_util:make_header(State)},
-    ok = couch_file:write_header(State#mrst.fd, Header),
-    ok = couch_file:flush(State#mrst.fd).
+    ok = couch_file:write_header(State#mrst.fd, Header).
 
 
 compact(Db, State, Opts) ->

http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couch_mrview/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl
index ef6f2e7..3014664 100644
--- a/src/couch_mrview/src/couch_mrview_updater.erl
+++ b/src/couch_mrview/src/couch_mrview_updater.erl
@@ -55,8 +55,7 @@ start_update(Partial, State, NumChanges) ->
 purge(_Db, PurgeSeq, PurgedIdRevs, State) ->
     #mrst{
         id_btree=IdBtree,
-        views=Views,
-        fd=Fd
+        views=Views
     } = State,
 
     Ids = [Id || {Id, _Revs} <- PurgedIdRevs],
@@ -88,7 +87,6 @@ purge(_Db, PurgeSeq, PurgedIdRevs, State) ->
     end,
 
     Views2 = lists:map(RemKeysFun, Views),
-    ok = couch_file:flush(Fd),
     {ok, State#mrst{
         id_btree=IdBtree2,
         views=Views2,
@@ -231,8 +229,7 @@ insert_results(DocId, [KVs | RKVs], [{Id, VKVs} | RVKVs], VKVAcc, VIdKeys)
->
 write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys) ->
     #mrst{
         id_btree=IdBtree,
-        first_build=FirstBuild,
-        fd=Fd
+        first_build=FirstBuild
     } = State,
 
     {ok, ToRemove, IdBtree2} = update_id_btree(IdBtree, DocIdKeys, FirstBuild),
@@ -248,7 +245,6 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys) ->
         View#mrview{btree=VBtree2, update_seq=NewUpdateSeq}
     end,
 
-    ok = couch_file:flush(Fd),
     State#mrst{
         views=lists:zipwith(UpdateView, State#mrst.views, ViewKVs),
         update_seq=UpdateSeq,

http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couch_mrview/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 065a2cd..a13b154 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -546,7 +546,6 @@ delete_file(FName) ->
 reset_index(Db, Fd, #mrst{sig=Sig}=State) ->
     ok = couch_file:truncate(Fd, 0),
     ok = couch_file:write_header(Fd, {Sig, nil}),
-    ok = couch_file:flush(Fd),
     init_state(Db, Fd, reset_state(State), nil).
 
 

http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couchdb/couch_db_updater.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_db_updater.erl b/src/couchdb/couch_db_updater.erl
index 03a04aa..a4d4202 100644
--- a/src/couchdb/couch_db_updater.erl
+++ b/src/couchdb/couch_db_updater.erl
@@ -659,8 +659,7 @@ update_docs_int(Db, DocsList, NonRepDocs, MergeConflicts, FullCommit)
->
         fulldocinfo_by_id_btree = DocInfoByIdBTree,
         docinfo_by_seq_btree = DocInfoBySeqBTree,
         update_seq = LastSeq,
-        revs_limit = RevsLimit,
-        fd = Fd
+        revs_limit = RevsLimit
         } = Db,
     Ids = [Id || [{_Client, #doc{id=Id}}|_] <- DocsList],
     % lookup up the old documents, if they exist.
@@ -695,7 +694,6 @@ update_docs_int(Db, DocsList, NonRepDocs, MergeConflicts, FullCommit)
->
         fulldocinfo_by_id_btree = DocInfoByIdBTree2,
         docinfo_by_seq_btree = DocInfoBySeqBTree2,
         update_seq = NewSeq},
-    couch_file:flush(Fd),
 
     % Check if we just updated any design documents, and update the validation
     % funs if we did.
@@ -787,7 +785,6 @@ commit_data(Db, _) ->
         end,
 
         ok = couch_file:write_header(Fd, Header),
-        ok = couch_file:flush(Fd),
 
         case lists:member(after_header, FsyncOptions) of
         true -> ok = couch_file:sync(Fd);
@@ -838,7 +835,6 @@ copy_docs(Db, #db{fd = DestFd} = NewDb, InfoBySeq0, Retry) ->
     InfoBySeq = lists:usort(fun(#doc_info{id=A}, #doc_info{id=B}) -> A =< B end,
         InfoBySeq0),
     Ids = [Id || #doc_info{id=Id} <- InfoBySeq],
-    ok = couch_file:flush(DestFd),
     LookupResults = couch_btree:lookup(Db#db.fulldocinfo_by_id_btree, Ids),
 
     NewFullDocInfos1 = lists:map(
@@ -888,8 +884,6 @@ copy_docs(Db, #db{fd = DestFd} = NewDb, InfoBySeq0, Retry) ->
 copy_compact(Db, NewDb0, Retry) ->
     FsyncOptions = [Op || Op <- NewDb0#db.fsync_options, Op == before_header],
     NewDb = NewDb0#db{fsync_options=FsyncOptions},
-    ok = couch_file:flush(Db#db.fd),
-    ok = couch_file:flush(NewDb#db.fd),
     TotalChanges = couch_db:count_changes_since(Db, NewDb#db.update_seq),
     BufferSize = list_to_integer(
         couch_config:get("database_compaction", "doc_buffer_size", "524288")),
@@ -955,7 +949,6 @@ copy_compact(Db, NewDb0, Retry) ->
         NewDb4 = NewDb3
     end,
 
-    ok = couch_file:flush(NewDb4#db.fd),
     commit_data(NewDb4#db{update_seq=Db#db.update_seq}).
 
 start_copy_compact(#db{name=Name,filepath=Filepath,header=#db_header{purge_seq=PurgeSeq}}=Db)
->

http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couchdb/couch_file.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_file.erl b/src/couchdb/couch_file.erl
index 3979eb7..bb69004 100644
--- a/src/couchdb/couch_file.erl
+++ b/src/couchdb/couch_file.erl
@@ -132,6 +132,16 @@ pread_binary(Fd, Pos) ->
 
 
 pread_iolist(Fd, Pos) ->
+    case do_read_iolist(Fd, Pos) of
+    eof ->
+        flush(Fd),
+        do_read_iolist(Fd, Pos);
+    Else ->
+        Else
+    end.
+
+
+do_read_iolist(Fd, Pos) ->
     case gen_server:call(Fd, {pread_iolist, Pos}, infinity) of
     {ok, _IoList} = Ok ->
         Ok;
@@ -260,7 +270,8 @@ write_header(Fd, Data) ->
     Md5 = couch_util:md5(Bin),
     % now we assemble the final header binary and write to disk
     FinalBin = <<Md5/binary, Bin/binary>>,
-    ok = gen_server:call(Fd, {write_header, FinalBin}, infinity).
+    ok = gen_server:call(Fd, {write_header, FinalBin}, infinity),
+    ok = flush(Fd).
 
 
 % server functions
@@ -472,8 +483,12 @@ read_raw_iolist_int(ReadFd, {Pos, _Size}, Len) -> % 0110 UPGRADE CODE
 read_raw_iolist_int(ReadFd, Pos, Len) ->
     BlockOffset = Pos rem ?SIZE_BLOCK,
     TotalBytes = calculate_total_read_len(BlockOffset, Len),
-    {ok, <<RawBin:TotalBytes/binary>>} = file:pread(ReadFd, Pos, TotalBytes),
-    {remove_block_prefixes(BlockOffset, RawBin), Pos + TotalBytes}.
+    case file:pread(ReadFd, Pos, TotalBytes) of
+    {ok, <<RawBin:TotalBytes/binary>>} ->
+        {remove_block_prefixes(BlockOffset, RawBin), Pos + TotalBytes};
+    eof ->
+        throw(eof)
+    end.
 
 -spec extract_md5(iolist()) -> {binary(), iolist()}.
 extract_md5(FullIoList) ->
@@ -649,7 +664,13 @@ write_header_blocks(Fd, Eof, Header) ->
 reader_loop(Fd) ->
     receive
     {read, Pos, From} ->
-        read_iolist(Fd, Pos, From),
+        Result = try
+            read_iolist(Fd, Pos)
+        catch
+        throw:eof ->
+            eof
+        end,
+        gen_server:reply(From, Result),
         reader_loop(Fd);
     {find_header, Eof, From} ->
         gen_server:reply(From, find_header(Fd, Eof div ?SIZE_BLOCK)),
@@ -660,9 +681,9 @@ reader_loop(Fd) ->
     end.
 
 
--compile({inline, [read_iolist/3]}).
+-compile({inline, [read_iolist/2]}).
 
-read_iolist(Fd, Pos, From) ->
+read_iolist(Fd, Pos) ->
     {RawData, NextPos} = try
         % up to 8Kbs of read ahead
         read_raw_iolist_int(Fd, Pos, 2 * ?SIZE_BLOCK - (Pos rem ?SIZE_BLOCK))
@@ -676,8 +697,8 @@ read_iolist(Fd, Pos, From) ->
     1 ->
         {Md5, IoList} = extract_md5(
             maybe_read_more_iolist(RestRawData, 16 + Len, NextPos, Fd)),
-        gen_server:reply(From, {ok, IoList, Md5});
+        {ok, IoList, Md5};
     0 ->
         IoList = maybe_read_more_iolist(RestRawData, Len, NextPos, Fd),
-        gen_server:reply(From, {ok, IoList})
+        {ok, IoList}
     end.

http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/010-file-basics.t
----------------------------------------------------------------------
diff --git a/test/etap/010-file-basics.t b/test/etap/010-file-basics.t
index a1b9a98..fb1b29e 100755
--- a/test/etap/010-file-basics.t
+++ b/test/etap/010-file-basics.t
@@ -57,7 +57,6 @@ test() ->
 
     ?etap_match(couch_file:append_binary(Fd, <<"fancy!">>), {ok, Size, _},
         "Appending a binary returns the current file size."),
-    ok = couch_file:flush(Fd),
 
     etap:is({ok, foo}, couch_file:pread_term(Fd, 0),
         "Reading the first term returns what we wrote: foo"),
@@ -71,30 +70,25 @@ test() ->
     ),
 
     {ok, BinPos, _} = couch_file:append_binary(Fd, <<131,100,0,3,102,111,111>>),
-    ok = couch_file:flush(Fd),
     etap:is({ok, foo}, couch_file:pread_term(Fd, BinPos),
         "Reading a term from a written binary term representation succeeds."),
         
     BigBin = list_to_binary(lists:duplicate(100000, 0)),
     {ok, BigBinPos, _} = couch_file:append_binary(Fd, BigBin),
-    ok = couch_file:flush(Fd),
     etap:is({ok, BigBin}, couch_file:pread_binary(Fd, BigBinPos),
         "Reading a large term from a written representation succeeds."),
     
     ok = couch_file:write_header(Fd, hello),
-    ok = couch_file:flush(Fd),
     etap:is({ok, hello}, couch_file:read_header(Fd),
         "Reading a header succeeds."),
         
     {ok, BigBinPos2, _} = couch_file:append_binary(Fd, BigBin),
-    ok = couch_file:flush(Fd),
     etap:is({ok, BigBin}, couch_file:pread_binary(Fd, BigBinPos2),
         "Reading a large term from a written representation succeeds 2."),
 
     % append_binary == append_iolist?
     % Possible bug in pread_iolist or iolist() -> append_binary
     {ok, IOLPos, _} = couch_file:append_binary(Fd, ["foo", $m, <<"bam">>]),
-    ok = couch_file:flush(Fd),
     {ok, IoList} = couch_file:pread_iolist(Fd, IOLPos),
     etap:is(<<"foombam">>, iolist_to_binary(IoList),
         "Reading an results in a binary form of the written iolist()"),

http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/011-file-headers.t
----------------------------------------------------------------------
diff --git a/test/etap/011-file-headers.t b/test/etap/011-file-headers.t
index 82c028f..a26b032 100755
--- a/test/etap/011-file-headers.t
+++ b/test/etap/011-file-headers.t
@@ -45,7 +45,6 @@ test() ->
     etap:is_greater(Size1, 0,
         "Writing a header allocates space in the file."),
 
-    ok = couch_file:flush(Fd),
     etap:is({ok, {<<"some_data">>, 32}}, couch_file:read_header(Fd),
         "Reading the header returns what we wrote."),
 
@@ -56,7 +55,6 @@ test() ->
     etap:is_greater(Size2, Size1,
         "Writing a second header allocates more space."),
 
-    ok = couch_file:flush(Fd),
     etap:is({ok, [foo, <<"more">>]}, couch_file:read_header(Fd),
         "Reading the second header does not return the first header."),
 
@@ -71,7 +69,6 @@ test() ->
         "Rewriting the same second header returns the same second size."),
 
     couch_file:write_header(Fd, erlang:make_tuple(5000, <<"CouchDB">>)),
-    ok = couch_file:flush(Fd),
     etap:is(
         couch_file:read_header(Fd),
         {ok, erlang:make_tuple(5000, <<"CouchDB">>)},
@@ -85,7 +82,6 @@ test() ->
 
     % Destroy the 0x1 byte that marks a header
     check_header_recovery(fun(CouchFd, RawFd, Expect, HeaderPos) ->
-        ok = couch_file:flush(CouchFd),
         etap:isnt(Expect, couch_file:read_header(CouchFd),
             "Should return a different header before corruption."),
         file:pwrite(RawFd, HeaderPos, <<0>>),
@@ -95,7 +91,6 @@ test() ->
 
     % Corrupt the size.
     check_header_recovery(fun(CouchFd, RawFd, Expect, HeaderPos) ->
-        ok = couch_file:flush(CouchFd),
         etap:isnt(Expect, couch_file:read_header(CouchFd),
             "Should return a different header before corruption."),
         % +1 for 0x1 byte marker
@@ -106,7 +101,6 @@ test() ->
 
     % Corrupt the MD5 signature
     check_header_recovery(fun(CouchFd, RawFd, Expect, HeaderPos) ->
-        ok = couch_file:flush(CouchFd),
         etap:isnt(Expect, couch_file:read_header(CouchFd),
             "Should return a different header before corruption."),
         % +5 = +1 for 0x1 byte and +4 for term size.
@@ -117,7 +111,6 @@ test() ->
 
     % Corrupt the data
     check_header_recovery(fun(CouchFd, RawFd, Expect, HeaderPos) ->
-        ok = couch_file:flush(CouchFd),
         etap:isnt(Expect, couch_file:read_header(CouchFd),
             "Should return a different header before corruption."),
         % +21 = +1 for 0x1 byte, +4 for term size and +16 for MD5 sig

http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/020-btree-basics.t
----------------------------------------------------------------------
diff --git a/test/etap/020-btree-basics.t b/test/etap/020-btree-basics.t
index 372b494..e65aeb2 100755
--- a/test/etap/020-btree-basics.t
+++ b/test/etap/020-btree-basics.t
@@ -97,7 +97,6 @@ test_kvs(KeyValues) ->
             "After removing all keys btree size is 0."),
 
     {Btree4, _} = lists:foldl(fun(KV, {BtAcc, PrevSize}) ->
-        ok = couch_file:flush(Fd),
         {ok, BtAcc2} = couch_btree:add_remove(BtAcc, [KV], []),
         case couch_btree:size(BtAcc2) > PrevSize of
         true ->
@@ -107,7 +106,6 @@ test_kvs(KeyValues) ->
         end,
         {BtAcc2, couch_btree:size(BtAcc2)}
     end, {Btree3, couch_btree:size(Btree3)}, KeyValues),
-    ok = couch_file:flush(Fd),
 
     etap:ok(test_btree(Btree4, KeyValues),
         "Adding all keys one at a time returns a complete btree."),
@@ -115,7 +113,6 @@ test_kvs(KeyValues) ->
             "Non empty btrees have a size > 0."),
 
     {Btree5, _} = lists:foldl(fun({K, _}, {BtAcc, PrevSize}) ->
-        ok = couch_file:flush(Fd),
         {ok, BtAcc2} = couch_btree:add_remove(BtAcc, [], [K]),
         case couch_btree:size(BtAcc2) < PrevSize of
         true ->
@@ -125,7 +122,6 @@ test_kvs(KeyValues) ->
         end,
         {BtAcc2, couch_btree:size(BtAcc2)}
     end, {Btree4, couch_btree:size(Btree4)}, KeyValues),
-    ok = couch_file:flush(Fd),
 
     etap:ok(test_btree(Btree5, []),
         "Removing all keys one at a time returns an empty btree."),
@@ -134,7 +130,6 @@ test_kvs(KeyValues) ->
 
     KeyValuesRev = lists:reverse(KeyValues),
     {Btree6, _} = lists:foldl(fun(KV, {BtAcc, PrevSize}) ->
-        ok = couch_file:flush(Fd),
         {ok, BtAcc2} = couch_btree:add_remove(BtAcc, [KV], []),
         case couch_btree:size(BtAcc2) > PrevSize of
         true ->
@@ -177,7 +172,6 @@ test_kvs(KeyValues) ->
     true.
 
 test_btree(#btree{fd = Fd} = Btree, KeyValues) ->
-    ok = couch_file:flush(Fd),
     ok = test_key_access(Btree, KeyValues),
     ok = test_lookup_access(Btree, KeyValues),
     ok = test_final_reductions(Btree, KeyValues),
@@ -185,7 +179,6 @@ test_btree(#btree{fd = Fd} = Btree, KeyValues) ->
     true.
 
 test_add_remove(#btree{fd = Fd} = Btree, OutKeyValues, RemainingKeyValues) ->
-    ok = couch_file:flush(Fd),
     Btree2 = lists:foldl(fun({K, _}, BtAcc) ->
         {ok, BtAcc2} = couch_btree:add_remove(BtAcc, [], [K]),
         BtAcc2
@@ -207,13 +200,11 @@ test_key_access(#btree{fd = Fd} = Btree, List) ->
     end,
     Length = length(List),
     Sorted = lists:sort(List),
-    ok = couch_file:flush(Fd),
     {ok, _, {[], Length}} = couch_btree:foldl(Btree, FoldFun, {Sorted, 0}),
     {ok, _, {[], Length}} = couch_btree:fold(Btree, FoldFun, {Sorted, 0}, [{dir, rev}]),
     ok.
 
 test_lookup_access(#btree{fd = Fd} = Btree, KeyValues) ->
-    ok = couch_file:flush(Fd),
     FoldFun = fun({Key, Value}, {Key, Value}) -> {stop, true} end,
     lists:foreach(fun({Key, Value}) ->
         [{ok, {Key, Value}}] = couch_btree:lookup(Btree, [Key]),
@@ -221,7 +212,6 @@ test_lookup_access(#btree{fd = Fd} = Btree, KeyValues) ->
     end, KeyValues).
 
 test_final_reductions(#btree{fd = Fd} = Btree, KeyValues) ->
-    ok = couch_file:flush(Fd),
     KVLen = length(KeyValues),
     FoldLFun = fun(_X, LeadingReds, Acc) ->
         CountToStart = KVLen div 3 + Acc,

http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/021-btree-reductions.t
----------------------------------------------------------------------
diff --git a/test/etap/021-btree-reductions.t b/test/etap/021-btree-reductions.t
index e597772..331e49a 100755
--- a/test/etap/021-btree-reductions.t
+++ b/test/etap/021-btree-reductions.t
@@ -47,7 +47,6 @@ test()->
     end, {"odd", []}, lists:seq(1, rows())),
 
     {ok, Btree2} = couch_btree:add_remove(Btree, EvenOddKVs, []),
-    ok = couch_file:flush(Fd),
 
     GroupFun = fun({K1, _}, {K2, _}) -> K1 == K2 end,
     FoldFun = fun(GroupedKey, Unreduced, Acc) ->

http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/050-stream.t
----------------------------------------------------------------------
diff --git a/test/etap/050-stream.t b/test/etap/050-stream.t
index 622079a..de0dfad 100755
--- a/test/etap/050-stream.t
+++ b/test/etap/050-stream.t
@@ -26,7 +26,6 @@ main(_) ->
     ok.
 
 read_all(Fd, PosList) ->
-    ok = couch_file:flush(Fd),
     Data = couch_stream:foldl(Fd, PosList, fun(Bin, Acc) -> [Bin, Acc] end, []),
     iolist_to_binary(Data).
 


Mime
View raw message