Return-Path: X-Original-To: apmail-couchdb-commits-archive@www.apache.org Delivered-To: apmail-couchdb-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1239911B79 for ; Tue, 26 Aug 2014 20:59:56 +0000 (UTC) Received: (qmail 29734 invoked by uid 500); 26 Aug 2014 20:59:54 -0000 Delivered-To: apmail-couchdb-commits-archive@couchdb.apache.org Received: (qmail 29313 invoked by uid 500); 26 Aug 2014 20:59:53 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 28762 invoked by uid 99); 26 Aug 2014 20:59:53 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 26 Aug 2014 20:59:53 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 752D09CEEDD; Tue, 26 Aug 2014 20:59:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: chewbranca@apache.org To: commits@couchdb.apache.org Date: Tue, 26 Aug 2014 21:00:14 -0000 Message-Id: In-Reply-To: <2f010890076e4986acd86778ee57ec89@git.apache.org> References: <2f010890076e4986acd86778ee57ec89@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [23/50] [abbrv] couch commit: updated refs/heads/1963-eunit-bigcouch to 0efdc71 Port 200-view-group-no-db-leaks.t etap test suite to eunit Merged into couchdb_views_tests suite. Apply minor refactor changes. Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/3d8cc0f0 Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/3d8cc0f0 Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/3d8cc0f0 Branch: refs/heads/1963-eunit-bigcouch Commit: 3d8cc0f077a4f87ea1666a1bce3bb94e9c7395f4 Parents: 4655ed6 Author: Alexander Shorin Authored: Thu Jun 5 23:03:31 2014 +0400 Committer: Russell Branca Committed: Tue Aug 26 13:59:31 2014 -0700 ---------------------------------------------------------------------- test/couchdb/couchdb_views_tests.erl | 240 +++++++++++++++++++++++++++--- 1 file changed, 223 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/3d8cc0f0/test/couchdb/couchdb_views_tests.erl ---------------------------------------------------------------------- diff --git a/test/couchdb/couchdb_views_tests.erl b/test/couchdb/couchdb_views_tests.erl index d10b567..77ee4f5 100644 --- a/test/couchdb/couchdb_views_tests.erl +++ b/test/couchdb/couchdb_views_tests.erl @@ -16,8 +16,10 @@ -include_lib("couchdb/couch_db.hrl"). -define(ADMIN_USER, {user_ctx, #user_ctx{roles=[<<"_admin">>]}}). +-define(DELAY, 100). -define(TIMEOUT, 1000). + start() -> {ok, Pid} = couch_server_sup:start_link(?CONFIG_CHAIN), Pid. @@ -34,7 +36,8 @@ stop(Pid) -> setup() -> DbName = ?tempdb(), - {ok, _} = couch_db:create(DbName, [?ADMIN_USER]), + {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]), + ok = couch_db:close(Db), FooRev = create_design_doc(DbName, <<"_design/foo">>, <<"bar">>), query_view(DbName, "foo", "bar"), BooRev = create_design_doc(DbName, <<"_design/boo">>, <<"baz">>), @@ -43,7 +46,8 @@ setup() -> setup_with_docs() -> DbName = ?tempdb(), - {ok, _} = couch_db:create(DbName, [?ADMIN_USER]), + {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]), + ok = couch_db:close(Db), create_docs(DbName), create_design_doc(DbName, <<"_design/foo">>, <<"bar">>), DbName. @@ -51,7 +55,7 @@ setup_with_docs() -> teardown({DbName, _}) -> teardown(DbName); teardown(DbName) when is_binary(DbName) -> - ok = couch_server:delete(DbName, [?ADMIN_USER]), + couch_server:delete(DbName, [?ADMIN_USER]), ok. @@ -73,17 +77,33 @@ view_indexes_cleanup_test_() -> } }. +view_group_db_leaks_test_() -> + { + "View group db leaks", + { + setup, + fun start/0, fun stop/1, + { + foreach, + fun setup_with_docs/0, fun teardown/1, + [ + fun couchdb_1138/1, + fun couchdb_1309/1 + ] + } + } + }. + + should_not_remember_docs_in_index_after_backup_restore_test() -> %% COUCHDB-640 start(), DbName = setup_with_docs(), ok = backup_db_file(DbName), - create_doc(DbName), + create_doc(DbName, "doc666"), - Body0 = query_view(DbName, "foo", "bar"), - ViewJson0 = ejson:decode(Body0), - Rows0 = couch_util:get_nested_json_value(ViewJson0, [<<"rows">>]), + Rows0 = query_view(DbName, "foo", "bar"), ?assert(has_doc("doc1", Rows0)), ?assert(has_doc("doc2", Rows0)), ?assert(has_doc("doc3", Rows0)), @@ -91,9 +111,7 @@ should_not_remember_docs_in_index_after_backup_restore_test() -> restore_backup_db_file(DbName), - Body1 = query_view(DbName, "foo", "bar"), - ViewJson1 = ejson:decode(Body1), - Rows1 = couch_util:get_nested_json_value(ViewJson1, [<<"rows">>]), + Rows1 = query_view(DbName, "foo", "bar"), ?assert(has_doc("doc1", Rows1)), ?assert(has_doc("doc2", Rows1)), ?assert(has_doc("doc3", Rows1)), @@ -118,13 +136,106 @@ should_cleanup_all_index_files({DbName, {FooRev, BooRev}})-> view_cleanup(DbName), ?_assertEqual(0, count_index_files(DbName)). - -create_doc(DbName) -> +couchdb_1138(DbName) -> + ?_test(begin + {ok, IndexerPid} = couch_index_server:get_index( + couch_mrview_index, DbName, <<"_design/foo">>), + ?assert(is_pid(IndexerPid)), + ?assert(is_process_alive(IndexerPid)), + ?assertEqual(2, count_db_refs(DbName)), + + Rows0 = query_view(DbName, "foo", "bar"), + ?assertEqual(3, length(Rows0)), + ?assertEqual(2, count_db_refs(DbName)), + ?assert(is_process_alive(IndexerPid)), + + create_doc(DbName, "doc1000"), + Rows1 = query_view(DbName, "foo", "bar"), + ?assertEqual(4, length(Rows1)), + ?assertEqual(2, count_db_refs(DbName)), + ?assert(is_process_alive(IndexerPid)), + + Ref1 = get_db_ref_counter(DbName), + compact_db(DbName), + Ref2 = get_db_ref_counter(DbName), + ?assertEqual(2, couch_ref_counter:count(Ref2)), + ?assertNotEqual(Ref2, Ref1), + ?assertNot(is_process_alive(Ref1)), + ?assert(is_process_alive(IndexerPid)), + + compact_view_group(DbName, "foo"), + ?assertEqual(2, count_db_refs(DbName)), + Ref3 = get_db_ref_counter(DbName), + ?assertEqual(Ref3, Ref2), + ?assert(is_process_alive(IndexerPid)), + + create_doc(DbName, "doc1001"), + Rows2 = query_view(DbName, "foo", "bar"), + ?assertEqual(5, length(Rows2)), + ?assertEqual(2, count_db_refs(DbName)), + ?assert(is_process_alive(IndexerPid)) + end). + +couchdb_1309(DbName) -> + ?_test(begin + {ok, IndexerPid} = couch_index_server:get_index( + couch_mrview_index, DbName, <<"_design/foo">>), + ?assert(is_pid(IndexerPid)), + ?assert(is_process_alive(IndexerPid)), + ?assertEqual(2, count_db_refs(DbName)), + + create_doc(DbName, "doc1001"), + Rows0 = query_view(DbName, "foo", "bar"), + check_rows_value(Rows0, null), + ?assertEqual(4, length(Rows0)), + ?assertEqual(2, count_db_refs(DbName)), + ?assert(is_process_alive(IndexerPid)), + + update_design_doc(DbName, <<"_design/foo">>, <<"bar">>), + {ok, NewIndexerPid} = couch_index_server:get_index( + couch_mrview_index, DbName, <<"_design/foo">>), + ?assert(is_pid(NewIndexerPid)), + ?assert(is_process_alive(NewIndexerPid)), + ?assertNotEqual(IndexerPid, NewIndexerPid), + ?assertEqual(2, count_db_refs(DbName)), + + Rows1 = query_view(DbName, "foo", "bar", ok), + ?assertEqual(0, length(Rows1)), + Rows2 = query_view(DbName, "foo", "bar"), + check_rows_value(Rows2, 1), + ?assertEqual(4, length(Rows2)), + + MonRef0 = erlang:monitor(process, IndexerPid), + receive + {'DOWN', MonRef0, _, _, _} -> + ok + after ?TIMEOUT -> + erlang:error( + {assertion_failed, + [{module, ?MODULE}, {line, ?LINE}, + {reason, "old view group is not dead after ddoc update"}]}) + end, + + MonRef1 = erlang:monitor(process, NewIndexerPid), + ok = couch_server:delete(DbName, [?ADMIN_USER]), + receive + {'DOWN', MonRef1, _, _, _} -> + ok + after ?TIMEOUT -> + erlang:error( + {assertion_failed, + [{module, ?MODULE}, {line, ?LINE}, + {reason, "new view group did not die after DB deletion"}]}) + end + end). + +create_doc(DbName, DocId) when is_list(DocId) -> + create_doc(DbName, ?l2b(DocId)); +create_doc(DbName, DocId) when is_binary(DocId) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_USER]), Doc666 = couch_doc:from_json_obj({[ - {<<"_id">>, <<"doc666">>}, + {<<"_id">>, DocId}, {<<"value">>, 999} - ]}), {ok, _} = couch_db:update_docs(Db, [Doc666]), couch_db:ensure_full_commit(Db), @@ -158,7 +269,7 @@ create_design_doc(DbName, DDName, ViewName) -> {<<"language">>, <<"javascript">>}, {<<"views">>, {[ {ViewName, {[ - {<<"map">>, <<"function(doc) { emit(doc.value, 1); }">>} + {<<"map">>, <<"function(doc) { emit(doc.value, null); }">>} ]}} ]}} ]}), @@ -167,6 +278,26 @@ create_design_doc(DbName, DDName, ViewName) -> couch_db:close(Db), Rev. +update_design_doc(DbName, DDName, ViewName) -> + {ok, Db} = couch_db:open(DbName, [?ADMIN_USER]), + {ok, Doc} = couch_db:open_doc(Db, DDName, [?ADMIN_USER]), + {Props} = couch_doc:to_json_obj(Doc, []), + Rev = couch_util:get_value(<<"_rev">>, Props), + DDoc = couch_doc:from_json_obj({[ + {<<"_id">>, DDName}, + {<<"_rev">>, Rev}, + {<<"language">>, <<"javascript">>}, + {<<"views">>, {[ + {ViewName, {[ + {<<"map">>, <<"function(doc) { emit(doc.value, 1); }">>} + ]}} + ]}} + ]}), + {ok, NewRev} = couch_db:update_doc(Db, DDoc, [?ADMIN_USER]), + couch_db:ensure_full_commit(Db), + couch_db:close(Db), + NewRev. + delete_design_doc(DbName, DDName, Rev) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_USER]), DDoc = couch_doc:from_json_obj({[ @@ -183,16 +314,43 @@ db_url(DbName) -> "http://" ++ Addr ++ ":" ++ Port ++ "/" ++ ?b2l(DbName). query_view(DbName, DDoc, View) -> + query_view(DbName, DDoc, View, false). + +query_view(DbName, DDoc, View, Stale) -> {ok, Code, _Headers, Body} = test_request:get( - db_url(DbName) ++ "/_design/" ++ DDoc ++ "/_view/" ++ View), + db_url(DbName) ++ "/_design/" ++ DDoc ++ "/_view/" ++ View + ++ case Stale of + false -> []; + _ -> "?stale=" ++ atom_to_list(Stale) + end), ?assertEqual(200, Code), - Body. + {Props} = ejson:decode(Body), + couch_util:get_value(<<"rows">>, Props, []). + +check_rows_value(Rows, Value) -> + lists:foreach( + fun({Row}) -> + ?assertEqual(Value, couch_util:get_value(<<"value">>, Row)) + end, Rows). view_cleanup(DbName) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_USER]), couch_mrview:cleanup(Db), couch_db:close(Db). +get_db_ref_counter(DbName) -> + {ok, #db{fd_ref_counter = Ref} = Db} = couch_db:open_int(DbName, []), + ok = couch_db:close(Db), + Ref. + +count_db_refs(DbName) -> + Ref = get_db_ref_counter(DbName), + % have to sleep a bit to let couchdb cleanup all refs and leave only + % active ones. otherwise the related tests will randomly fail due to + % count number mismatch + timer:sleep(200), + couch_ref_counter:count(Ref). + count_index_files(DbName) -> % call server to fetch the index files RootDir = couch_config:get("couchdb", "view_index_dir"), @@ -217,3 +375,51 @@ restore_backup_db_file(DbName) -> ok = file:rename(DbFile ++ ".backup", DbFile), start(), ok. + +compact_db(DbName) -> + {ok, Db} = couch_db:open_int(DbName, []), + {ok, _} = couch_db:start_compact(Db), + ok = couch_db:close(Db), + wait_db_compact_done(DbName, 10). + +wait_db_compact_done(_DbName, 0) -> + erlang:error({assertion_failed, + [{module, ?MODULE}, + {line, ?LINE}, + {reason, "DB compaction failed to finish"}]}); +wait_db_compact_done(DbName, N) -> + {ok, Db} = couch_db:open_int(DbName, []), + ok = couch_db:close(Db), + case is_pid(Db#db.compactor_pid) of + false -> + ok; + true -> + ok = timer:sleep(?DELAY), + wait_db_compact_done(DbName, N - 1) + end. + +compact_view_group(DbName, DDocId) when is_list(DDocId) -> + compact_view_group(DbName, ?l2b("_design/" ++ DDocId)); +compact_view_group(DbName, DDocId) when is_binary(DDocId) -> + ok = couch_mrview:compact(DbName, DDocId), + wait_view_compact_done(DbName, DDocId, 10). + +wait_view_compact_done(_DbName, _DDocId, 0) -> + erlang:error({assertion_failed, + [{module, ?MODULE}, + {line, ?LINE}, + {reason, "DB compaction failed to finish"}]}); +wait_view_compact_done(DbName, DDocId, N) -> + {ok, Code, _Headers, Body} = test_request:get( + db_url(DbName) ++ "/" ++ ?b2l(DDocId) ++ "/_info"), + ?assertEqual(200, Code), + {Info} = ejson:decode(Body), + {IndexInfo} = couch_util:get_value(<<"view_index">>, Info), + CompactRunning = couch_util:get_value(<<"compact_running">>, IndexInfo), + case CompactRunning of + false -> + ok; + true -> + ok = timer:sleep(?DELAY), + wait_view_compact_done(DbName, DDocId, N - 1) + end.