couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chewbra...@apache.org
Subject [03/50] [abbrv] couch commit: updated refs/heads/1963-eunit-bigcouch to 08c6f0b
Date Fri, 29 Aug 2014 20:43:05 GMT
Port 140-attachments-comp.t etap test suite to eunit

- Merge into couchdb_attachments_tests suite;
- Add PUT requests to test_request util;
- Remove dependency from files outside fixtures directory;
- Group test cases to reduce amount of duplicate code;
- Fix hidden issue with gzip encoding: for encoding_length stub info
  check using zlib:gzip on 2KiB+ files leads to mismatch by 2-4 bytes
  and this difference grows with file size. Using gzip fun code from
  couch_stream solves the issue.


Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/754b099e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/754b099e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/754b099e

Branch: refs/heads/1963-eunit-bigcouch
Commit: 754b099e481943d7405ef43c96fbb9f6d0003ec7
Parents: 2f2a177
Author: Alexander Shorin <kxepal@apache.org>
Authored: Tue Jun 3 01:51:46 2014 +0400
Committer: Russell Branca <chewbranca@apache.org>
Committed: Fri Aug 29 13:42:23 2014 -0700

----------------------------------------------------------------------
 test/couchdb/couchdb_attachments_tests.erl | 380 +++++++++++++++++++++++-
 test/couchdb/fixtures/logo.png             | Bin 0 -> 3010 bytes
 test/couchdb/test_request.erl              |   9 +-
 3 files changed, 385 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/754b099e/test/couchdb/couchdb_attachments_tests.erl
----------------------------------------------------------------------
diff --git a/test/couchdb/couchdb_attachments_tests.erl b/test/couchdb/couchdb_attachments_tests.erl
index aef1873..cf59785 100644
--- a/test/couchdb/couchdb_attachments_tests.erl
+++ b/test/couchdb/couchdb_attachments_tests.erl
@@ -15,13 +15,23 @@
 -include("couch_eunit.hrl").
 -include_lib("couchdb/couch_db.hrl").
 
+-define(COMPRESSION_LEVEL, 8).
+-define(ATT_BIN_NAME, <<"logo.png">>).
+-define(ATT_TXT_NAME, <<"file.erl">>).
+-define(FIXTURE_PNG, filename:join([?FIXTURESDIR, "logo.png"])).
+-define(FIXTURE_TXT, ?FILE).
 -define(TIMEOUT, 1000).
+-define(TIMEOUT_EUNIT, 10).
 -define(TIMEWAIT, 100).
 -define(i2l(I), integer_to_list(I)).
 
 
 start() ->
     {ok, Pid} = couch_server_sup:start_link(?CONFIG_CHAIN),
+    % ensure in default compression settings for attachments_compression_tests
+    couch_config:set("attachments", "compression_level",
+                     ?i2l(?COMPRESSION_LEVEL), false),
+    couch_config:set("attachments", "compressible_types", "text/*", false),
     Pid.
 
 stop(Pid) ->
@@ -43,7 +53,35 @@ setup() ->
     Host = Addr ++ ":" ++ ?i2l(Port),
     {Host, ?b2l(DbName)}.
 
+setup({binary, standalone}) ->
+    {Host, DbName} = setup(),
+        setup_att(fun create_standalone_png_att/2, Host, DbName, ?FIXTURE_PNG);
+setup({text, standalone}) ->
+    {Host, DbName} = setup(),
+    setup_att(fun create_standalone_text_att/2, Host, DbName, ?FIXTURE_TXT);
+setup({binary, inline}) ->
+    {Host, DbName} = setup(),
+    setup_att(fun create_inline_png_att/2, Host, DbName, ?FIXTURE_PNG);
+setup({text, inline}) ->
+    {Host, DbName} = setup(),
+    setup_att(fun create_inline_text_att/2, Host, DbName, ?FIXTURE_TXT);
+setup(compressed) ->
+    {Host, DbName} = setup(),
+    setup_att(fun create_already_compressed_att/2, Host, DbName, ?FIXTURE_TXT).
+setup_att(Fun, Host, DbName, File) ->
+    HttpHost = "http://" ++ Host,
+    AttUrl = Fun(HttpHost, DbName),
+    {ok, Data} = file:read_file(File),
+    DocUrl = string:join([HttpHost, DbName, "doc"], "/"),
+    Helpers = {DbName, DocUrl, AttUrl},
+    {Data, Helpers}.
+
+teardown(_, {_, {DbName, _, _}}) ->
+    teardown(DbName).
+
 teardown({_, DbName}) ->
+    teardown(DbName);
+teardown(DbName) ->
     ok = couch_server:delete(?l2b(DbName), []),
     ok.
 
@@ -55,7 +93,8 @@ attachments_test_() ->
             setup,
             fun start/0, fun stop/1,
             [
-                attachments_md5_tests()
+                attachments_md5_tests(),
+                attachments_compression_tests()
             ]
         }
     }.
@@ -79,6 +118,67 @@ attachments_md5_tests() ->
         }
     }.
 
+attachments_compression_tests() ->
+    Funs = [
+         fun should_get_att_without_accept_gzip_encoding/2,
+         fun should_get_att_with_accept_gzip_encoding/2,
+         fun should_get_att_with_accept_deflate_encoding/2,
+         fun should_return_406_response_on_unsupported_encoding/2,
+         fun should_get_doc_with_att_data/2,
+         fun should_get_doc_with_att_data_stub/2
+    ],
+    {
+        "Attachments compression tests",
+        [
+            {
+                "Created via Attachments API",
+                created_attachments_compression_tests(standalone, Funs)
+            },
+            {
+                "Created inline via Document API",
+                created_attachments_compression_tests(inline, Funs)
+            },
+            {
+                "Created already been compressed via Attachments API",
+                {
+                    foreachx,
+                    fun setup/1, fun teardown/2,
+                    [{compressed, Fun} || Fun <- Funs]
+                }
+            },
+            {
+                foreach,
+                fun setup/0, fun teardown/1,
+                [
+                    fun should_not_create_compressed_att_with_deflate_encoding/1,
+                    fun should_not_create_compressed_att_with_compress_encoding/1,
+                    fun should_create_compressible_att_with_ctype_params/1
+                ]
+            }
+        ]
+    }.
+
+created_attachments_compression_tests(Mod, Funs) ->
+    [
+        {
+            "Compressiable attachments",
+            {
+                foreachx,
+                fun setup/1, fun teardown/2,
+                [{{text, Mod}, Fun} || Fun <- Funs]
+            }
+        },
+        {
+            "Uncompressiable attachments",
+            {
+                foreachx,
+                fun setup/1, fun teardown/2,
+                [{{binary, Mod}, Fun} || Fun <- Funs]
+            }
+        }
+    ].
+
+
 
 should_upload_attachment_without_md5({Host, DbName}) ->
     ?_test(begin
@@ -212,10 +312,218 @@ should_reject_chunked_attachment_with_invalid_md5_trailer({Host, DbName})
->
         ],
         {ok, Code, Json} = request("PUT", AttUrl, Headers, Body),
         ?assertEqual(400, Code),
-        ?assertEqual(<<"content_md5_mismatch">>,
-                     get_json(Json, [<<"error">>]))
+        ?assertEqual(<<"content_md5_mismatch">>, get_json(Json, [<<"error">>]))
+    end).
+
+should_get_att_without_accept_gzip_encoding(_, {Data, {_, _, AttUrl}}) ->
+    ?_test(begin
+        {ok, Code, Headers, Body} = test_request:get(AttUrl),
+        ?assertEqual(200, Code),
+        ?assertNot(lists:member({"Content-Encoding", "gzip"}, Headers)),
+        ?assertEqual(Data, iolist_to_binary(Body))
+    end).
+
+should_get_att_with_accept_gzip_encoding(compressed, {Data, {_, _, AttUrl}}) ->
+    ?_test(begin
+        {ok, Code, Headers, Body} = test_request:get(
+            AttUrl, [{"Accept-Encoding", "gzip"}]),
+        ?assertEqual(200, Code),
+        ?assert(lists:member({"Content-Encoding", "gzip"}, Headers)),
+        ?assertEqual(Data, zlib:gunzip(iolist_to_binary(Body)))
+    end);
+should_get_att_with_accept_gzip_encoding({text, _}, {Data, {_, _, AttUrl}}) ->
+    ?_test(begin
+        {ok, Code, Headers, Body} = test_request:get(
+            AttUrl, [{"Accept-Encoding", "gzip"}]),
+        ?assertEqual(200, Code),
+        ?assert(lists:member({"Content-Encoding", "gzip"}, Headers)),
+        ?assertEqual(Data, zlib:gunzip(iolist_to_binary(Body)))
+    end);
+should_get_att_with_accept_gzip_encoding({binary, _}, {Data, {_, _, AttUrl}}) ->
+    ?_test(begin
+        {ok, Code, Headers, Body} = test_request:get(
+            AttUrl, [{"Accept-Encoding", "gzip"}]),
+        ?assertEqual(200, Code),
+        ?assertEqual(undefined,
+                     couch_util:get_value("Content-Encoding", Headers)),
+        ?assertEqual(Data, iolist_to_binary(Body))
+    end).
+
+should_get_att_with_accept_deflate_encoding(_, {Data, {_, _, AttUrl}}) ->
+    ?_test(begin
+        {ok, Code, Headers, Body} = test_request:get(
+            AttUrl, [{"Accept-Encoding", "deflate"}]),
+        ?assertEqual(200, Code),
+        ?assertEqual(undefined,
+                     couch_util:get_value("Content-Encoding", Headers)),
+        ?assertEqual(Data, iolist_to_binary(Body))
+    end).
+
+should_return_406_response_on_unsupported_encoding(_, {_, {_, _, AttUrl}}) ->
+    ?_assertEqual(406,
+        begin
+            {ok, Code, _, _} = test_request:get(
+                AttUrl, [{"Accept-Encoding", "deflate, *;q=0"}]),
+            Code
+        end).
+
+should_get_doc_with_att_data(compressed, {Data, {_, DocUrl, _}}) ->
+    ?_test(begin
+        Url = DocUrl ++ "?attachments=true",
+        {ok, Code, _, Body} = test_request:get(
+            Url, [{"Accept", "application/json"}]),
+        ?assertEqual(200, Code),
+        Json = ejson:decode(Body),
+        AttJson = couch_util:get_nested_json_value(
+            Json, [<<"_attachments">>, ?ATT_TXT_NAME]),
+        AttData = couch_util:get_nested_json_value(
+            AttJson, [<<"data">>]),
+        ?assertEqual(
+            <<"text/plain">>,
+            couch_util:get_nested_json_value(AttJson,[<<"content_type">>])),
+        ?assertEqual(Data, base64:decode(AttData))
+    end);
+should_get_doc_with_att_data({text, _}, {Data, {_, DocUrl, _}}) ->
+    ?_test(begin
+        Url = DocUrl ++ "?attachments=true",
+        {ok, Code, _, Body} = test_request:get(
+            Url, [{"Accept", "application/json"}]),
+        ?assertEqual(200, Code),
+        Json = ejson:decode(Body),
+        AttJson = couch_util:get_nested_json_value(
+            Json, [<<"_attachments">>, ?ATT_TXT_NAME]),
+        AttData = couch_util:get_nested_json_value(
+            AttJson, [<<"data">>]),
+        ?assertEqual(
+            <<"text/plain">>,
+            couch_util:get_nested_json_value(AttJson,[<<"content_type">>])),
+        ?assertEqual(Data, base64:decode(AttData))
+    end);
+should_get_doc_with_att_data({binary, _}, {Data, {_, DocUrl, _}}) ->
+    ?_test(begin
+        Url = DocUrl ++ "?attachments=true",
+        {ok, Code, _, Body} = test_request:get(
+            Url, [{"Accept", "application/json"}]),
+        ?assertEqual(200, Code),
+        Json = ejson:decode(Body),
+        AttJson = couch_util:get_nested_json_value(
+            Json, [<<"_attachments">>, ?ATT_BIN_NAME]),
+        AttData = couch_util:get_nested_json_value(
+            AttJson, [<<"data">>]),
+        ?assertEqual(
+            <<"image/png">>,
+            couch_util:get_nested_json_value(AttJson,[<<"content_type">>])),
+        ?assertEqual(Data, base64:decode(AttData))
+    end).
+
+should_get_doc_with_att_data_stub(compressed, {Data, {_, DocUrl, _}}) ->
+    ?_test(begin
+        Url = DocUrl ++ "?att_encoding_info=true",
+        {ok, Code, _, Body} = test_request:get(
+            Url, [{"Accept", "application/json"}]),
+        ?assertEqual(200, Code),
+        Json = ejson:decode(Body),
+        {AttJson} = couch_util:get_nested_json_value(
+            Json, [<<"_attachments">>, ?ATT_TXT_NAME]),
+        ?assertEqual(<<"gzip">>,
+                     couch_util:get_value(<<"encoding">>, AttJson)),
+        AttLength = couch_util:get_value(<<"length">>, AttJson),
+        EncLength = couch_util:get_value(<<"encoded_length">>, AttJson),
+        ?assertEqual(AttLength, EncLength),
+        ?assertEqual(iolist_size(zlib:gzip(Data)), AttLength)
+    end);
+should_get_doc_with_att_data_stub({text, _}, {Data, {_, DocUrl, _}}) ->
+    ?_test(begin
+        Url = DocUrl ++ "?att_encoding_info=true",
+        {ok, Code, _, Body} = test_request:get(
+            Url, [{"Accept", "application/json"}]),
+        ?assertEqual(200, Code),
+        Json = ejson:decode(Body),
+        {AttJson} = couch_util:get_nested_json_value(
+            Json, [<<"_attachments">>, ?ATT_TXT_NAME]),
+        ?assertEqual(<<"gzip">>,
+                     couch_util:get_value(<<"encoding">>, AttJson)),
+        AttEncLength = iolist_size(gzip(Data)),
+        ?assertEqual(AttEncLength,
+                     couch_util:get_value(<<"encoded_length">>, AttJson)),
+        ?assertEqual(byte_size(Data),
+                     couch_util:get_value(<<"length">>, AttJson))
+    end);
+should_get_doc_with_att_data_stub({binary, _}, {Data, {_, DocUrl, _}}) ->
+    ?_test(begin
+        Url = DocUrl ++ "?att_encoding_info=true",
+        {ok, Code, _, Body} = test_request:get(
+            Url, [{"Accept", "application/json"}]),
+        ?assertEqual(200, Code),
+        Json = ejson:decode(Body),
+        {AttJson} = couch_util:get_nested_json_value(
+            Json, [<<"_attachments">>, ?ATT_BIN_NAME]),
+        ?assertEqual(undefined,
+                     couch_util:get_value(<<"encoding">>, AttJson)),
+        ?assertEqual(undefined,
+                     couch_util:get_value(<<"encoded_length">>, AttJson)),
+        ?assertEqual(byte_size(Data),
+                     couch_util:get_value(<<"length">>, AttJson))
     end).
 
+should_not_create_compressed_att_with_deflate_encoding({Host, DbName}) ->
+    ?_assertEqual(415,
+        begin
+            HttpHost = "http://" ++ Host,
+            AttUrl = string:join([HttpHost, DbName, ?docid(), "file.txt"], "/"),
+            {ok, Data} = file:read_file(?FIXTURE_TXT),
+            Body = zlib:compress(Data),
+            Headers = [
+                {"Content-Encoding", "deflate"},
+                {"Content-Type", "text/plain"}
+            ],
+            {ok, Code, _, _} = test_request:put(AttUrl, Headers, Body),
+            Code
+        end).
+
+should_not_create_compressed_att_with_compress_encoding({Host, DbName}) ->
+    % Note: As of OTP R13B04, it seems there's no LZW compression
+    % (i.e. UNIX compress utility implementation) lib in OTP.
+    % However there's a simple working Erlang implementation at:
+    % http://scienceblogs.com/goodmath/2008/01/simple_lempelziv_compression_i.php
+    ?_assertEqual(415,
+        begin
+            HttpHost = "http://" ++ Host,
+            AttUrl = string:join([HttpHost, DbName, ?docid(), "file.txt"], "/"),
+            {ok, Data} = file:read_file(?FIXTURE_TXT),
+            Headers = [
+                {"Content-Encoding", "compress"},
+                {"Content-Type", "text/plain"}
+            ],
+            {ok, Code, _, _} = test_request:put(AttUrl, Headers, Data),
+            Code
+        end).
+
+should_create_compressible_att_with_ctype_params({Host, DbName}) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(begin
+        HttpHost = "http://" ++ Host,
+        DocUrl = string:join([HttpHost, DbName, ?docid()], "/"),
+        AttUrl = string:join([DocUrl, ?b2l(?ATT_TXT_NAME)], "/"),
+        {ok, Data} = file:read_file(?FIXTURE_TXT),
+        Headers = [{"Content-Type", "text/plain; charset=UTF-8"}],
+        {ok, Code0, _, _} = test_request:put(AttUrl, Headers, Data),
+        ?assertEqual(201, Code0),
+
+        {ok, Code1, _, Body} = test_request:get(
+            DocUrl ++ "?att_encoding_info=true"),
+        ?assertEqual(200, Code1),
+        Json = ejson:decode(Body),
+        {AttJson} = couch_util:get_nested_json_value(
+            Json, [<<"_attachments">>, ?ATT_TXT_NAME]),
+        ?assertEqual(<<"gzip">>,
+                     couch_util:get_value(<<"encoding">>, AttJson)),
+        AttEncLength = iolist_size(gzip(Data)),
+        ?assertEqual(AttEncLength,
+                     couch_util:get_value(<<"encoded_length">>, AttJson)),
+        ?assertEqual(byte_size(Data),
+                     couch_util:get_value(<<"length">>, AttJson))
+    end)}.
+
 
 get_json(Json, Path) ->
     couch_util:get_nested_json_value(Json, Path).
@@ -262,3 +570,69 @@ request(Method, Url, Headers, Body) ->
         erlang:decode_packet(http, Header, []),
     Json = ejson:decode(Body1),
     {ok, Code, Json}.
+
+create_standalone_text_att(Host, DbName) ->
+    {ok, Data} = file:read_file(?FIXTURE_TXT),
+    Url = string:join([Host, DbName, "doc", ?b2l(?ATT_TXT_NAME)], "/"),
+    {ok, Code, _Headers, _Body} = test_request:put(
+        Url, [{"Content-Type", "text/plain"}], Data),
+    ?assertEqual(201, Code),
+    Url.
+
+create_standalone_png_att(Host, DbName) ->
+    {ok, Data} = file:read_file(?FIXTURE_PNG),
+    Url = string:join([Host, DbName, "doc", ?b2l(?ATT_BIN_NAME)], "/"),
+    {ok, Code, _Headers, _Body} = test_request:put(
+        Url, [{"Content-Type", "image/png"}], Data),
+    ?assertEqual(201, Code),
+    Url.
+
+create_inline_text_att(Host, DbName) ->
+    {ok, Data} = file:read_file(?FIXTURE_TXT),
+    Url = string:join([Host, DbName, "doc"], "/"),
+    Doc = {[
+        {<<"_attachments">>, {[
+            {?ATT_TXT_NAME, {[
+                {<<"content_type">>, <<"text/plain">>},
+                {<<"data">>, base64:encode(Data)}
+            ]}
+        }]}}
+    ]},
+    {ok, Code, _Headers, _Body} = test_request:put(
+        Url, [{"Content-Type", "application/json"}], ejson:encode(Doc)),
+    ?assertEqual(201, Code),
+    string:join([Url, ?b2l(?ATT_TXT_NAME)], "/").
+
+create_inline_png_att(Host, DbName) ->
+    {ok, Data} = file:read_file(?FIXTURE_PNG),
+    Url = string:join([Host, DbName, "doc"], "/"),
+    Doc = {[
+        {<<"_attachments">>, {[
+            {?ATT_BIN_NAME, {[
+                {<<"content_type">>, <<"image/png">>},
+                {<<"data">>, base64:encode(Data)}
+            ]}
+        }]}}
+    ]},
+    {ok, Code, _Headers, _Body} = test_request:put(
+        Url, [{"Content-Type", "application/json"}], ejson:encode(Doc)),
+    ?assertEqual(201, Code),
+    string:join([Url, ?b2l(?ATT_BIN_NAME)], "/").
+
+create_already_compressed_att(Host, DbName) ->
+    {ok, Data} = file:read_file(?FIXTURE_TXT),
+    Url = string:join([Host, DbName, "doc", ?b2l(?ATT_TXT_NAME)], "/"),
+    {ok, Code, _Headers, _Body} = test_request:put(
+        Url, [{"Content-Type", "text/plain"}, {"Content-Encoding", "gzip"}],
+        zlib:gzip(Data)),
+    ?assertEqual(201, Code),
+    Url.
+
+gzip(Data) ->
+    Z = zlib:open(),
+    ok = zlib:deflateInit(Z, ?COMPRESSION_LEVEL, deflated, 16 + 15, 8, default),
+    zlib:deflate(Z, Data),
+    Last = zlib:deflate(Z, [], finish),
+    ok = zlib:deflateEnd(Z),
+    ok = zlib:close(Z),
+    Last.

http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/754b099e/test/couchdb/fixtures/logo.png
----------------------------------------------------------------------
diff --git a/test/couchdb/fixtures/logo.png b/test/couchdb/fixtures/logo.png
new file mode 100644
index 0000000..d21ac02
Binary files /dev/null and b/test/couchdb/fixtures/logo.png differ

http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/754b099e/test/couchdb/test_request.erl
----------------------------------------------------------------------
diff --git a/test/couchdb/test_request.erl b/test/couchdb/test_request.erl
index 7abb92f..d6725ea 100644
--- a/test/couchdb/test_request.erl
+++ b/test/couchdb/test_request.erl
@@ -12,7 +12,7 @@
 
 -module(test_request).
 
--export([get/1, get/2]).
+-export([get/1, get/2, put/2, put/3]).
 -export([request/3, request/4]).
 
 get(Url) ->
@@ -22,6 +22,13 @@ get(Url, Headers) ->
     request(get, Url, Headers).
 
 
+put(Url, Body) ->
+    request(put, Url, [], Body).
+
+put(Url, Headers, Body) ->
+    request(put, Url, Headers, Body).
+
+
 request(Method, Url, Headers) ->
     request(Method, Url, Headers, []).
 


Mime
View raw message