couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kocol...@apache.org
Subject git commit: Fix retrieval of headers larger than 4k
Date Sun, 30 Oct 2011 18:30:46 GMT
Updated Branches:
  refs/heads/1.1.x aa7650589 -> 7e1f0b54e


Fix retrieval of headers larger than 4k

Our headers start with a <<1>> and then four bytes indicating the length
of the header and its checksum. When the header is larger than 4090
bytes it will be split across multiple blocks in the file and will need
to be reassembled on read. The reassembly consists of stripping out
<<0>> from the beginning of each subsequent block in the
remove_block_prefixes/2 function. The bug here is that we tell
remove_block_prefixes that we're starting 1 byte into the current block
instead of 5, so it ends up removing one good byte from the header and
injecting one or more random <<0>>s.

Headers larger than 4k are very rare and generally require a view group
with a huge number of indexes or indexes with fairly large reductions,
which explains why this bug has gone undetected until now.

Closes COUCHDB-1319.


Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/7e1f0b54
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/7e1f0b54
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/7e1f0b54

Branch: refs/heads/1.1.x
Commit: 7e1f0b54e6ab26fb6ee870c2dd786045f14ef606
Parents: aa76505
Author: Adam Kocoloski <kocolosk@apache.org>
Authored: Wed Oct 26 14:04:54 2011 -0400
Committer: Adam Kocoloski <kocolosk@apache.org>
Committed: Sun Oct 30 14:30:36 2011 -0400

----------------------------------------------------------------------
 src/couchdb/couch_file.erl   |    2 +-
 test/etap/011-file-headers.t |    9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb/blob/7e1f0b54/src/couchdb/couch_file.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_file.erl b/src/couchdb/couch_file.erl
index 07ec18d..5c5d51b 100644
--- a/src/couchdb/couch_file.erl
+++ b/src/couchdb/couch_file.erl
@@ -517,7 +517,7 @@ load_header(Fd, Block) ->
         RawBin = <<RestBlock/binary, Missing/binary>>
     end,
     <<Md5Sig:16/binary, HeaderBin/binary>> =
-        iolist_to_binary(remove_block_prefixes(1, RawBin)),
+        iolist_to_binary(remove_block_prefixes(5, RawBin)),
     Md5Sig = couch_util:md5(HeaderBin),
     {ok, HeaderBin}.
 

http://git-wip-us.apache.org/repos/asf/couchdb/blob/7e1f0b54/test/etap/011-file-headers.t
----------------------------------------------------------------------
diff --git a/test/etap/011-file-headers.t b/test/etap/011-file-headers.t
index 4705f62..764b10d 100755
--- a/test/etap/011-file-headers.t
+++ b/test/etap/011-file-headers.t
@@ -22,7 +22,7 @@ main(_) ->
     {S1, S2, S3} = now(),
     random:seed(S1, S2, S3),
 
-    etap:plan(17),
+    etap:plan(18),
     case (catch test()) of
         ok ->
             etap:end_tests();
@@ -68,6 +68,13 @@ test() ->
     etap:is({ok, Size2}, couch_file:bytes(Fd),
         "Rewriting the same second header returns the same second size."),
 
+    couch_file:write_header(Fd, erlang:make_tuple(5000, <<"CouchDB">>)),
+    etap:is(
+        couch_file:read_header(Fd),
+        {ok, erlang:make_tuple(5000, <<"CouchDB">>)},
+        "Headers larger than the block size can be saved (COUCHDB-1319)"
+    ),
+
     ok = couch_file:close(Fd),
 
     % Now for the fun stuff. Try corrupting the second header and see


Mime
View raw message