couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dav...@apache.org
Subject [1/2] couch commit: updated refs/heads/master to 6997c3c
Date Tue, 04 Oct 2016 18:19:13 GMT
Repository: couchdb-couch
Updated Branches:
  refs/heads/master 2a66f4142 -> 6997c3c23


Use vectored reads to search for buried headers

Current behavior attempts to read a header at each block, starting at
the eof and working backwards one block at a time. Deeply buried headers
can take a long time to find, depending on the depth of the header,
server load, etc.

This commit changes the behavior so that if the last block in the file
does not contain a header, it switches to using a "vectored" approach
where multiple candidate header prefixes (of 5 bytes) are read in a
single operation, greatly speeding up the search. On a modern linux
system with SSD, we see improvements up to 15x.

COUCHDB-3061


Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/d5fbb6dd
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/d5fbb6dd
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/d5fbb6dd

Branch: refs/heads/master
Commit: d5fbb6dd0d770861222b6a5de3124f10d9eafd4f
Parents: 2a66f41
Author: Jay Doane <jay.s.doane@gmail.com>
Authored: Tue Oct 4 10:58:51 2016 -0700
Committer: Jay Doane <jay.s.doane@gmail.com>
Committed: Tue Oct 4 11:12:06 2016 -0700

----------------------------------------------------------------------
 src/couch_file.erl | 80 +++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 67 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/d5fbb6dd/src/couch_file.erl
----------------------------------------------------------------------
diff --git a/src/couch_file.erl b/src/couch_file.erl
index 8346b02..49a8456 100644
--- a/src/couch_file.erl
+++ b/src/couch_file.erl
@@ -22,7 +22,12 @@
 -define(SIZE_BLOCK, 16#1000). % 4 KiB
 -define(READ_AHEAD, 2 * ?SIZE_BLOCK).
 -define(IS_OLD_STATE(S), tuple_size(S) /= tuple_size(#file{})).
+-define(PREFIX_SIZE, 5).
+-define(DEFAULT_READ_COUNT, 1024).
 
+-type block_id() :: non_neg_integer().
+-type location() :: non_neg_integer().
+-type header_size() :: non_neg_integer().
 
 -record(file, {
     fd,
@@ -524,34 +529,83 @@ handle_info({'EXIT', _, Reason}, Fd) ->
     {stop, Reason, Fd}.
 
 
-find_header(_Fd, -1) ->
-    no_valid_header;
 find_header(Fd, Block) ->
     case (catch load_header(Fd, Block)) of
     {ok, Bin} ->
         {ok, Bin};
     _Error ->
-        find_header(Fd, Block -1)
+        ReadCount = config:get_integer(
+            "couchdb", "find_header_read_count", ?DEFAULT_READ_COUNT),
+        find_header(Fd, Block -1, ReadCount)
     end.
 
 load_header(Fd, Block) ->
     {ok, <<1, HeaderLen:32/integer, RestBlock/binary>>} =
         file:pread(Fd, Block * ?SIZE_BLOCK, ?SIZE_BLOCK),
-    TotalBytes = calculate_total_read_len(5, HeaderLen),
-    case TotalBytes > byte_size(RestBlock) of
-    false ->
-        <<RawBin:TotalBytes/binary, _/binary>> = RestBlock;
-    true ->
-        {ok, Missing} = file:pread(
-            Fd, (Block * ?SIZE_BLOCK) + 5 + byte_size(RestBlock),
-            TotalBytes - byte_size(RestBlock)),
-        RawBin = <<RestBlock/binary, Missing/binary>>
+    load_header(Fd, Block * ?SIZE_BLOCK, HeaderLen, RestBlock).
+
+load_header(Fd, Pos, HeaderLen) ->
+    load_header(Fd, Pos, HeaderLen, <<>>).
+
+load_header(Fd, Pos, HeaderLen, RestBlock) ->
+    TotalBytes = calculate_total_read_len(?PREFIX_SIZE, HeaderLen),
+    RawBin = case TotalBytes =< byte_size(RestBlock) of
+        true ->
+            <<RawBin0:TotalBytes/binary, _/binary>> = RestBlock,
+            RawBin0;
+        false ->
+            ReadStart = Pos + ?PREFIX_SIZE + byte_size(RestBlock),
+            ReadLen = TotalBytes - byte_size(RestBlock),
+            {ok, Missing} = file:pread(Fd, ReadStart, ReadLen),
+            <<RestBlock/binary, Missing/binary>>
     end,
     <<Md5Sig:16/binary, HeaderBin/binary>> =
-        iolist_to_binary(remove_block_prefixes(5, RawBin)),
+        iolist_to_binary(remove_block_prefixes(?PREFIX_SIZE, RawBin)),
     Md5Sig = couch_crypto:hash(md5, HeaderBin),
     {ok, HeaderBin}.
 
+
+%% Read multiple block locations using a single file:pread/2.
+-spec find_header(file:fd(), block_id(), non_neg_integer()) ->
+    {ok, binary()} | no_valid_header.
+find_header(_Fd, Block, _ReadCount) when Block < 0 ->
+    no_valid_header;
+find_header(Fd, Block, ReadCount) ->
+    FirstBlock = max(0, Block - ReadCount + 1),
+    BlockLocations = [?SIZE_BLOCK*B || B <- lists:seq(FirstBlock, Block)],
+    {ok, DataL} = file:pread(Fd, [{L, ?PREFIX_SIZE} || L <- BlockLocations]),
+    %% Since BlockLocations are ordered from oldest to newest, we rely
+    %% on lists:foldl/3 to reverse the order, making HeaderLocations
+    %% correctly ordered from newest to oldest.
+    HeaderLocations = lists:foldl(fun
+        ({Loc, <<1, HeaderSize:32/integer>>}, Acc) ->
+            [{Loc, HeaderSize} | Acc];
+        (_, Acc) ->
+            Acc
+    end, [], lists:zip(BlockLocations, DataL)),
+    case find_newest_header(Fd, HeaderLocations) of
+        {ok, _Location, HeaderBin} ->
+            {ok, HeaderBin};
+        _ ->
+            ok = file:advise(
+                Fd, hd(BlockLocations), ReadCount * ?SIZE_BLOCK, dont_need),
+            NextBlock = hd(BlockLocations) div ?SIZE_BLOCK - 1,
+            find_header(Fd, NextBlock, ReadCount)
+    end.
+
+-spec find_newest_header(file:fd(), [{location(), header_size()}]) ->
+    {ok, binary()} | not_found.
+find_newest_header(_Fd, []) ->
+    not_found;
+find_newest_header(Fd, [{Location, Size} | LocationSizes]) ->
+    case (catch load_header(Fd, Location, Size)) of
+        {ok, HeaderBin} ->
+            {ok, Location, HeaderBin};
+        _Error ->
+            find_newest_header(Fd, LocationSizes)
+    end.
+
+
 maybe_read_more_iolist(Buffer, DataSize, _, _)
     when DataSize =< byte_size(Buffer) ->
     <<Data:DataSize/binary, _/binary>> = Buffer,


Mime
View raw message