From commits-return-9885-apmail-couchdb-commits-archive=couchdb.apache.org@couchdb.apache.org Wed Feb 6 21:58:57 2013 Return-Path: X-Original-To: apmail-couchdb-commits-archive@www.apache.org Delivered-To: apmail-couchdb-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6B326D921 for ; Wed, 6 Feb 2013 21:58:57 +0000 (UTC) Received: (qmail 45187 invoked by uid 500); 6 Feb 2013 21:58:57 -0000 Delivered-To: apmail-couchdb-commits-archive@couchdb.apache.org Received: (qmail 45035 invoked by uid 500); 6 Feb 2013 21:58:57 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 45018 invoked by uid 99); 6 Feb 2013 21:58:57 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 06 Feb 2013 21:58:57 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id E21372450E; Wed, 6 Feb 2013 21:58:56 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jan@apache.org To: commits@couchdb.apache.org X-Mailer: ASF-Git Admin Mailer Subject: [1/2] git commit: COUCHDB-1654: Transparently update view signatures from <= 1.2.x. Message-Id: <20130206215856.E21372450E@tyr.zones.apache.org> Date: Wed, 6 Feb 2013 21:58:56 +0000 (UTC) COUCHDB-1654: Transparently update view signatures from <= 1.2.x. Updates 1.2.x or earlier view files to 1.3.x or later view files transparently, the first time the 1.2.x view file is opened by 1.3.x or later. Here's how it works: Before opening a view index, If no matching index file is found in the new location: calculate the <= 1.2.x view signature if a file with that signature lives in the old location copy it to the new location with the new signature in the name. Then proceed to open the view index as usual. After opening, read its header. If the header matches the <= 1.2.x style #index_header record: upgrade the header to the new #mrheader record The next time the view is used, the new header is used. If we crash after the rename, but before the header upgrade, the header upgrade is done on the next view opening. If we crash between upgrading to the new header and writing that header to disk, we start with the old header again, do the upgrade and write to disk. Includes etap tests in 250*.t. Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/da35ed0f Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/da35ed0f Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/da35ed0f Branch: refs/heads/master Commit: da35ed0f012c265d35a71cc12e3683bdaad9e6d3 Parents: b267a29 Author: Jan Lehnardt Authored: Mon Feb 4 15:29:17 2013 +0100 Committer: Jan Lehnardt Committed: Wed Feb 6 22:52:09 2013 +0100 ---------------------------------------------------------------------- src/couch_mrview/src/couch_mrview_index.erl | 21 ++ src/couch_mrview/src/couch_mrview_util.erl | 101 +++++++++ test/etap/250-upgrade-legacy-view-files.t | 165 +++++++++++++++ .../fixtures/3b835456c235b1827e012e25666152f3.view | Bin 0 -> 4192 bytes test/etap/fixtures/test.couch | Bin 0 -> 16482 bytes 5 files changed, 287 insertions(+), 0 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb/blob/da35ed0f/src/couch_mrview/src/couch_mrview_index.erl ---------------------------------------------------------------------- diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl index 6bcb63f..7466dbc 100644 --- a/src/couch_mrview/src/couch_mrview_index.erl +++ b/src/couch_mrview/src/couch_mrview_index.erl @@ -75,9 +75,30 @@ open(Db, State) -> sig=Sig } = State, IndexFName = couch_mrview_util:index_file(DbName, Sig), + + % If we are upgrading from <=1.2.x, we upgrade the view + % index file on the fly, avoiding an index reset. + % + % OldSig is `ok` if no upgrade happened. + % + % To remove suppport for 1.2.x auto-upgrades in the + % future, just remove the next line and the code + % between "upgrade code for <= 1.2.x" and + % "end upgrade code for <= 1.2.x" and the corresponding + % code in couch_mrview_util + + OldSig = couch_mrview_util:maybe_update_index_file(State), + case couch_mrview_util:open_file(IndexFName) of {ok, Fd} -> case (catch couch_file:read_header(Fd)) of + % upgrade code for <= 1.2.x + {ok, {OldSig, Header}} -> + % Matching view signatures. + NewSt = couch_mrview_util:init_state(Db, Fd, State, Header), + {ok, RefCounter} = couch_ref_counter:start([Fd]), + {ok, NewSt#mrst{refc=RefCounter}}; + % end of upgrade code for <= 1.2.x {ok, {Sig, Header}} -> % Matching view signatures. NewSt = couch_mrview_util:init_state(Db, Fd, State, Header), http://git-wip-us.apache.org/repos/asf/couchdb/blob/da35ed0f/src/couch_mrview/src/couch_mrview_util.erl ---------------------------------------------------------------------- diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl index 8e6e4dc..092ae3d 100644 --- a/src/couch_mrview/src/couch_mrview_util.erl +++ b/src/couch_mrview/src/couch_mrview_util.erl @@ -24,6 +24,7 @@ -export([calculate_data_size/2]). -export([validate_args/1]). -export([maybe_load_doc/3, maybe_load_doc/4]). +-export([maybe_update_index_file/1]). -define(MOD, couch_mrview_index). @@ -168,6 +169,19 @@ init_state(Db, Fd, #mrst{views=Views}=State, nil) -> view_states=[{nil, 0, 0} || _ <- Views] }, init_state(Db, Fd, State, Header); +% read <= 1.2.x header record and transpile it to >=1.3.x +% header record +init_state(Db, Fd, State, #index_header{ + seq=Seq, + purge_seq=PurgeSeq, + id_btree_state=IdBtreeState, + view_states=ViewStates}) -> + init_state(Db, Fd, State, #mrheader{ + seq=Seq, + purge_seq=PurgeSeq, + id_btree_state=IdBtreeState, + view_states=ViewStates + }); init_state(Db, Fd, State, Header) -> #mrst{language=Lang, views=Views} = State, #mrheader{ @@ -704,3 +718,90 @@ index_of(Key, [_ | Rest], Idx) -> mrverror(Mesg) -> throw({query_parse_error, Mesg}). + + +%% Updates 1.2.x or earlier view files to 1.3.x or later view files +%% transparently, the first time the 1.2.x view file is opened by +%% 1.3.x or later. +%% +%% Here's how it works: +%% +%% Before opening a view index, +%% If no matching index file is found in the new location: +%% calculate the <= 1.2.x view signature +%% if a file with that signature lives in the old location +%% rename it to the new location with the new signature in the name. +%% Then proceed to open the view index as usual. +%% After opening, read its header. +%% +%% If the header matches the <= 1.2.x style #index_header record: +%% upgrade the header to the new #mrheader record +%% The next time the view is used, the new header is used. +%% +%% If we crash after the rename, but before the header upgrade, +%% the header upgrade is done on the next view opening. +%% +%% If we crash between upgrading to the new header and writing +%% that header to disk, we start with the old header again, +%% do the upgrade and write to disk. + +maybe_update_index_file(State) -> + DbName = State#mrst.db_name, + NewIndexFile = index_file(DbName, State#mrst.sig), + % open in read-only mode so we don't create + % the file if it doesn't exist. + case file:open(NewIndexFile, [read, raw]) of + {ok, Fd_Read} -> + % the new index file exists, there is nothing to do here. + file:close(Fd_Read); + _Error -> + update_index_file(State) + end. + +update_index_file(State) -> + Sig = sig_vsn_12x(State), + DbName = State#mrst.db_name, + FileName = couch_index_util:hexsig(Sig) ++ ".view", + IndexFile = couch_index_util:index_file("", DbName, FileName), + + % If we have an old index, rename it to the new position. + case file:read_file_info(IndexFile) of + {ok, _FileInfo} -> + % Crash if the rename fails for any reason. + % If the target exists, e.g. the next request will find the + % new file and we are good. We might need to catch this + % further up to avoid a full server crash. + ?LOG_INFO("Attempting to update legacy view index file.", []), + NewIndexFile = index_file(DbName, State#mrst.sig), + ok = filelib:ensure_dir(NewIndexFile), + ok = file:rename(IndexFile, NewIndexFile), + ?LOG_INFO("Successfully updated legacy view index file.", []), + Sig; + _ -> + % Ignore missing index file + ok + end. + +sig_vsn_12x(State) -> + ViewInfo = [old_view_format(V) || V <- State#mrst.views], + SigData = case State#mrst.lib of + {[]} -> + {ViewInfo, State#mrst.language, State#mrst.design_opts}; + _ -> + {ViewInfo, State#mrst.language, State#mrst.design_opts, + couch_index_util:sort_lib(State#mrst.lib)} + end, + couch_util:md5(term_to_binary(SigData)). + +old_view_format(View) -> +{ + view, + View#mrview.id_num, + View#mrview.map_names, + View#mrview.def, + View#mrview.btree, + View#mrview.reduce_funs, + View#mrview.options +}. + +%% End of <= 1.2.x upgrade code. http://git-wip-us.apache.org/repos/asf/couchdb/blob/da35ed0f/test/etap/250-upgrade-legacy-view-files.t ---------------------------------------------------------------------- diff --git a/test/etap/250-upgrade-legacy-view-files.t b/test/etap/250-upgrade-legacy-view-files.t new file mode 100644 index 0000000..998264d --- /dev/null +++ b/test/etap/250-upgrade-legacy-view-files.t @@ -0,0 +1,165 @@ +#!/usr/bin/env escript +%% -*- erlang -*- +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +main(_) -> + test_util:init_code_path(), + + etap:plan(8), + case (catch test()) of + ok -> + etap:end_tests(); + Other -> + etap:diag(io_lib:format("Test died abnormally: ~p", [Other])), + etap:bail(Other) + end, + ok. + + +test() -> + couch_server_sup:start_link(test_util:config_files()), + + % commit sofort + ok = couch_config:set("query_server_config", "commit_freq", "0"), + + test_upgrade(), + + couch_server_sup:stop(), + ok. + +fixture_path() -> + test_util:srcdir() ++ "/test/etap/fixtures". + +old_db() -> + fixture_path() ++ "/" ++ old_db_name(). + +old_db_name() -> + "test.couch". + +old_view() -> + fixture_path() ++ "/" ++ old_view_name(). + +old_view_name() -> + "3b835456c235b1827e012e25666152f3.view". +new_view_name() -> + "a1c5929f912aca32f13446122cc6ce50.view". + +couch_url() -> + "http://" ++ addr() ++ ":" ++ port(). + +addr() -> + couch_config:get("httpd", "bind_address", "127.0.0.1"). + +port() -> + integer_to_list(mochiweb_socket_server:get(couch_httpd, port)). + + +% <= 1.2.x +-record(index_header, + {seq=0, + purge_seq=0, + id_btree_state=nil, + view_states=nil + }). + +% >= 1.3.x +-record(mrheader, { + seq=0, + purge_seq=0, + id_btree_state=nil, + view_states=nil +}). + +ensure_header(File, MatchFun, Msg) -> + {ok, Fd} = couch_file:open(File), + {ok, {_Sig, Header}} = couch_file:read_header(Fd), + couch_file:close(Fd), + etap:fun_is(MatchFun, Header, "ensure " ++ Msg ++ " header for file: " ++ File). + +file_exists(File) -> + % open without creating + case file:open(File, [read, raw]) of + {ok, Fd_Read} -> + file:close(Fd_Read), + true; + _Error -> + false + end. + +cleanup() -> + DbDir = couch_config:get("couchdb", "database_dir"), + Files = [ + DbDir ++ "/test.couch", + DbDir ++ "/.test_design/" ++ old_view_name(), + DbDir ++ "/.test_design/mrview/" ++ new_view_name() + ], + lists:foreach(fun(File) -> file:delete(File) end, Files), + etap:ok(true, "cleanup"). + +test_upgrade() -> + + cleanup(), + + % copy old db file into db dir + DbDir = couch_config:get("couchdb", "database_dir"), + DbTarget = DbDir ++ "/" ++ old_db_name(), + filelib:ensure_dir(DbTarget), + file:copy(old_db(), DbTarget), + + % copy old view file into view dir + ViewDir = couch_config:get("couchdb", "index_dir"), + ViewTarget = ViewDir ++ "/.test_design/" ++ old_view_name(), + filelib:ensure_dir(ViewTarget), + file:copy(old_view(), ViewTarget), + + % ensure old header + ensure_header(ViewTarget, fun(#index_header{}) -> true; (_) -> false end, "old"), + + % query view + ViewUrl = couch_url() ++ "/test/_design/test/_view/test", + {ok, Code, _Headers, Body} = test_util:request(ViewUrl, [], get), + + % expect results + etap:is(Code, 200, "valid view result http status code"), + ExpectBody = <<"{\"total_rows\":2,\"offset\":0,\"rows\":[\r\n{\"id\":\"193f2f9c596ddc7ad326f7da470009ec\",\"key\":1,\"value\":null},\r\n{\"id\":\"193f2f9c596ddc7ad326f7da470012b6\",\"key\":2,\"value\":null}\r\n]}\n">>, + etap:is(Body, ExpectBody, "valid view result"), + + % ensure old file gone. + etap:is(file_exists(ViewTarget), false, "ensure old file is gone"), + + % ensure new header + NewViewFile = ViewDir ++ "/.test_design/mrview/" ++ new_view_name(), + + % add doc(s) + test_util:request( + couch_url() ++ "/test/boo", + [{"Content-Type", "application/json"}], + put, + <<"{\"a\":3}">>), + + % query again + {ok, Code2, _Headers2, Body2} = test_util:request(ViewUrl, [], get), + + % expect results + etap:is(Code2, 200, "valid view result http status code"), + ExpectBody2 = <<"{\"total_rows\":3,\"offset\":0,\"rows\":[\r\n{\"id\":\"193f2f9c596ddc7ad326f7da470009ec\",\"key\":1,\"value\":null},\r\n{\"id\":\"193f2f9c596ddc7ad326f7da470012b6\",\"key\":2,\"value\":null},\r\n{\"id\":\"boo\",\"key\":3,\"value\":null}\r\n]}\n">>, + etap:is(Body2, ExpectBody2, "valid view result after doc add"), + + % ensure no rebuild + % TBD no idea how to actually test this. + + % ensure new header. + timer:sleep(1000), + ensure_header(NewViewFile, fun(#mrheader{}) -> true; (_) -> false end, "new"), + + ok. http://git-wip-us.apache.org/repos/asf/couchdb/blob/da35ed0f/test/etap/fixtures/3b835456c235b1827e012e25666152f3.view ---------------------------------------------------------------------- diff --git a/test/etap/fixtures/3b835456c235b1827e012e25666152f3.view b/test/etap/fixtures/3b835456c235b1827e012e25666152f3.view new file mode 100644 index 0000000..9c67648 Binary files /dev/null and b/test/etap/fixtures/3b835456c235b1827e012e25666152f3.view differ http://git-wip-us.apache.org/repos/asf/couchdb/blob/da35ed0f/test/etap/fixtures/test.couch ---------------------------------------------------------------------- diff --git a/test/etap/fixtures/test.couch b/test/etap/fixtures/test.couch new file mode 100644 index 0000000..32c79af Binary files /dev/null and b/test/etap/fixtures/test.couch differ