couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dav...@apache.org
Subject svn commit: r1006339 - in /couchdb/trunk: ./ share/www/script/test/ src/couchdb/
Date Sun, 10 Oct 2010 19:07:10 GMT
Author: davisp
Date: Sun Oct 10 19:07:10 2010
New Revision: 1006339

URL: http://svn.apache.org/viewvc?rev=1006339&view=rev
Log:
Fixes COUCHDB-799 - More granular ETags for views.

ETags for views now only change when their underlying view index
changes due to indexing or purges. ETags are also specific to each
view.

Thanks to Klaus Trainer for the patch.


Modified:
    couchdb/trunk/THANKS
    couchdb/trunk/share/www/script/test/design_docs.js
    couchdb/trunk/share/www/script/test/etags_views.js
    couchdb/trunk/src/couchdb/couch_db.hrl
    couchdb/trunk/src/couchdb/couch_httpd_show.erl
    couchdb/trunk/src/couchdb/couch_httpd_view.erl
    couchdb/trunk/src/couchdb/couch_view_group.erl
    couchdb/trunk/src/couchdb/couch_view_updater.erl

Modified: couchdb/trunk/THANKS
URL: http://svn.apache.org/viewvc/couchdb/trunk/THANKS?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/THANKS (original)
+++ couchdb/trunk/THANKS Sun Oct 10 19:07:10 2010
@@ -68,5 +68,6 @@ suggesting improvements or submitting ch
  * David Rose <doppler@gmail.com>
  * Lim Yue Chuan <shasderias@gmail.com>
  * David Davis <xantus@xantus.org>
+ * Klaus Trainer <klaus.trainer@web.de>
 
 For a list of authors see the `AUTHORS` file.

Modified: couchdb/trunk/share/www/script/test/design_docs.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/design_docs.js?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/share/www/script/test/design_docs.js (original)
+++ couchdb/trunk/share/www/script/test/design_docs.js Sun Oct 10 19:07:10 2010
@@ -110,7 +110,7 @@ function() {
   var vinfo = dinfo.view_index;
   TEquals(51, vinfo.disk_size);
   TEquals(false, vinfo.compact_running);
-  TEquals("dc3264b45b74cc6d94666e3043e07154", vinfo.signature, 'ddoc sig');
+  TEquals("86e9b34892b4df35cd2f7c27da30c94d", vinfo.signature, 'ddoc sig');
 
   db.bulkSave(makeDocs(1, numDocs + 1));
 

Modified: couchdb/trunk/share/www/script/test/etags_views.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/etags_views.js?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/share/www/script/test/etags_views.js (original)
+++ couchdb/trunk/share/www/script/test/etags_views.js Sun Oct 10 19:07:10 2010
@@ -11,23 +11,34 @@
 // the License.
 
 couchTests.etags_views = function(debug) {
-  var db = new CouchDB("test_suite_db", {"X-Couch-Full-Commit":"false"});
+  var db = new CouchDB("test_suite_db", {"X-Couch-Full-Commit":"true"});
   db.deleteDb();
   db.createDb();
   if (debug) debugger;
 
   var designDoc = {
-    _id:"_design/etags",
+    _id: "_design/etags",
     language: "javascript",
     views : {
+      fooView: {
+        map: stringFun(function(doc) {
+          if (doc.foo) {
+            emit("bar", 1);
+          }
+        }),
+      },
       basicView : {
         map : stringFun(function(doc) {
-          emit(doc.integer, doc.string);
+          if(doc.integer && doc.string) {
+            emit(doc.integer, doc.string);
+          }
         })
       },
       withReduce : {
         map : stringFun(function(doc) {
-          emit(doc.integer, doc.string);
+          if(doc.integer && doc.string) {
+            emit(doc.integer, doc.string);
+          }
         }),
         reduce : stringFun(function(keys, values, rereduce) {
           if (rereduce) {
@@ -40,9 +51,9 @@ couchTests.etags_views = function(debug)
     }
   };
   T(db.save(designDoc).ok);
+  db.bulkSave(makeDocs(0, 10));
+
   var xhr;
-  var docs = makeDocs(0, 10);
-  db.bulkSave(docs);
 
   // verify get w/Etag on map view
   xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
@@ -52,17 +63,92 @@ couchTests.etags_views = function(debug)
     headers: {"if-none-match": etag}
   });
   T(xhr.status == 304);
-  // TODO GET with keys (when that is available)
+
+  // verify ETag doesn't change when an update
+  // doesn't change the view group's index
+  T(db.save({"_id":"doc1", "foo":"bar"}).ok);
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 == etag);
+ 
+  // Verify that purges affect etags
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+  var foo_etag = xhr.getResponseHeader("etag");
+  var doc1 = db.open("doc1");
+  xhr = CouchDB.request("POST", "/test_suite_db/_purge", {
+    body: JSON.stringify({"doc1":[doc1._rev]})
+  });
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 != foo_etag);
+
+  // Test that _purge didn't affect the other view etags.
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 == etag);
+
+  // verify different views in the same view group may have different ETags
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+  var etag1 = xhr.getResponseHeader("etag");
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+  var etag2 = xhr.getResponseHeader("etag");
+  T(etag1 != etag2);
+
+  // verify ETag changes when an update changes the view group's index.
+  db.bulkSave(makeDocs(10, 20));
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 != etag);
+
+  // verify ETag is the same after a restart
+  restartServer();
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+  var etag2 = xhr.getResponseHeader("etag");
+  T(etag1 == etag2);
 
   // reduce view
   xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
   T(xhr.status == 200);
   var etag = xhr.getResponseHeader("etag");
-  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce", {
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce",{
     headers: {"if-none-match": etag}
   });
   T(xhr.status == 304);
 
+  // verify ETag doesn't change when an update
+  // doesn't change the view group's index
+  T(db.save({"_id":"doc3", "foo":"bar"}).ok);
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 == etag);
+  // purge
+  var doc3 = db.open("doc3");
+  xhr = CouchDB.request("POST", "/test_suite_db/_purge", {
+    body: JSON.stringify({"doc3":[doc3._rev]})
+  });
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 == etag);
+
+  // verify different views in the same view group may have different ETags
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+  var etag1 = xhr.getResponseHeader("etag");
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+  var etag2 = xhr.getResponseHeader("etag");
+  T(etag1 != etag2);
+
+  // verify ETag changes when an update changes the view group's index
+  db.bulkSave(makeDocs(20, 30));
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+  var etag1 = xhr.getResponseHeader("etag");
+  T(etag1 != etag);
+
+  // verify ETag is the same after a restart
+  restartServer();
+  xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+  var etag2 = xhr.getResponseHeader("etag");
+  T(etag1 == etag2);
+
   // confirm ETag changes with different POST bodies
   xhr = CouchDB.request("POST", "/test_suite_db/_design/etags/_view/basicView",
     {body: JSON.stringify({keys:[1]})}

Modified: couchdb/trunk/src/couchdb/couch_db.hrl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.hrl?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db.hrl (original)
+++ couchdb/trunk/src/couchdb/couch_db.hrl Sun Oct 10 19:07:10 2010
@@ -238,6 +238,8 @@
 
 -record(view,
     {id_num,
+    update_seq=0,
+    purge_seq=0,
     map_names=[],
     def,
     btree=nil,

Modified: couchdb/trunk/src/couchdb/couch_httpd_show.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_show.erl?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_httpd_show.erl (original)
+++ couchdb/trunk/src/couchdb/couch_httpd_show.erl Sun Oct 10 19:07:10 2010
@@ -188,14 +188,14 @@ handle_view_list_req(Req, _Db, _DDoc) ->
 handle_view_list(Req, Db, DDoc, LName, {ViewDesignName, ViewName}, Keys) ->
     ViewDesignId = <<"_design/", ViewDesignName/binary>>,
     {ViewType, View, Group, QueryArgs} = couch_httpd_view:load_view(Req, Db, {ViewDesignId,
ViewName}, Keys),
-    Etag = list_etag(Req, Db, Group, {couch_httpd:doc_etag(DDoc), Keys}),
+    Etag = list_etag(Req, Db, Group, View, {couch_httpd:doc_etag(DDoc), Keys}),
     couch_httpd:etag_respond(Req, Etag, fun() ->
             output_list(ViewType, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group)
         end).
 
-list_etag(#httpd{user_ctx=UserCtx}=Req, Db, Group, More) ->
+list_etag(#httpd{user_ctx=UserCtx}=Req, Db, Group, View, More) ->
     Accept = couch_httpd:header_value(Req, "Accept"),
-    couch_httpd_view:view_group_etag(Group, Db, {More, Accept, UserCtx#user_ctx.roles}).
+    couch_httpd_view:view_etag(Db, Group, View, {More, Accept, UserCtx#user_ctx.roles}).
 
 output_list(map, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group) ->
     output_map_list(Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group);

Modified: couchdb/trunk/src/couchdb/couch_httpd_view.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_view.erl?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_httpd_view.erl (original)
+++ couchdb/trunk/src/couchdb/couch_httpd_view.erl Sun Oct 10 19:07:10 2010
@@ -17,7 +17,7 @@
 
 -export([parse_view_params/3]).
 -export([make_view_fold_fun/7, finish_view_fold/4, finish_view_fold/5, view_row_obj/3]).
--export([view_group_etag/2, view_group_etag/3, make_reduce_fold_funs/6]).
+-export([view_etag/3, view_etag/4, make_reduce_fold_funs/6]).
 -export([design_doc_view/5, parse_bool_param/1, doc_member/2]).
 -export([make_key_options/1, load_view/4]).
 
@@ -113,7 +113,7 @@ output_map_view(Req, View, Group, Db, Qu
         limit = Limit,
         skip = SkipCount
     } = QueryArgs,
-    CurrentEtag = view_group_etag(Group, Db),
+    CurrentEtag = view_etag(Db, Group, View),
     couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
         {ok, RowCount} = couch_view:get_row_count(View),
         FoldlFun = make_view_fold_fun(Req, QueryArgs, CurrentEtag, Db, Group#group.current_seq,
RowCount, #view_fold_helper_funs{reduce_count=fun couch_view:reduce_to_count/1}),
@@ -129,7 +129,7 @@ output_map_view(Req, View, Group, Db, Qu
         limit = Limit,
         skip = SkipCount
     } = QueryArgs,
-    CurrentEtag = view_group_etag(Group, Db, Keys),
+    CurrentEtag = view_etag(Db, Group, View, Keys),
     couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
         {ok, RowCount} = couch_view:get_row_count(View),
         FoldAccInit = {Limit, SkipCount, undefined, []},
@@ -154,7 +154,7 @@ output_reduce_view(Req, Db, View, Group,
         skip = Skip,
         group_level = GroupLevel
     } = QueryArgs,
-    CurrentEtag = view_group_etag(Group, Db),
+    CurrentEtag = view_etag(Db, Group, View),
     couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
         {ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel,
                 QueryArgs, CurrentEtag, Group#group.current_seq,
@@ -172,7 +172,7 @@ output_reduce_view(Req, Db, View, Group,
         skip = Skip,
         group_level = GroupLevel
     } = QueryArgs,
-    CurrentEtag = view_group_etag(Group, Db, Keys),
+    CurrentEtag = view_etag(Db, Group, View, Keys),
     couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
         {ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel,
                 QueryArgs, CurrentEtag, Group#group.current_seq,
@@ -607,16 +607,15 @@ send_json_reduce_row(Resp, {Key, Value},
     send_chunk(Resp, RowFront ++ ?JSON_ENCODE({[{key, Key}, {value, Value}]})),
     {ok, ",\r\n"}.
 
-view_group_etag(Group, Db) ->
-    view_group_etag(Group, Db, nil).
+view_etag(Db, Group, View) ->
+    view_etag(Db, Group, View, nil).
 
-view_group_etag(#group{sig=Sig,current_seq=CurrentSeq}, _Db, Extra) ->
-    % ?LOG_ERROR("Group ~p",[Group]),
-    % This is not as granular as it could be.
-    % If there are updates to the db that do not effect the view index,
-    % they will change the Etag. For more granular Etags we'd need to keep
-    % track of the last Db seq that caused an index change.
-    couch_httpd:make_etag({Sig, CurrentSeq, Extra}).
+view_etag(Db, Group, {reduce, _, _, View}, Extra) ->
+    view_etag(Db, Group, View, Extra);
+view_etag(Db, Group, {temp_reduce, View}, Extra) ->
+    view_etag(Db, Group, View, Extra);
+view_etag(_Db, #group{sig=Sig}, #view{update_seq=UpdateSeq, purge_seq=PurgeSeq}, Extra) ->
+    couch_httpd:make_etag({Sig, UpdateSeq, PurgeSeq, Extra}).
 
 % the view row has an error
 view_row_obj(_Db, {{Key, error}, Value}, _IncludeDocs) ->

Modified: couchdb/trunk/src/couchdb/couch_view_group.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_group.erl?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_view_group.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view_group.erl Sun Oct 10 19:07:10 2010
@@ -404,11 +404,15 @@ prepare_group({RootDir, DbName, #group{s
 
 get_index_header_data(#group{current_seq=Seq, purge_seq=PurgeSeq,
             id_btree=IdBtree,views=Views}) ->
-    ViewStates = [couch_btree:get_state(Btree) || #view{btree=Btree} <- Views],
-    #index_header{seq=Seq,
-            purge_seq=PurgeSeq,
-            id_btree_state=couch_btree:get_state(IdBtree),
-            view_states=ViewStates}.
+    ViewStates = [
+        {couch_btree:get_state(V#view.btree), V#view.update_seq, V#view.purge_seq} || V <-
Views
+    ],
+    #index_header{
+        seq=Seq,
+        purge_seq=PurgeSeq,
+        id_btree_state=couch_btree:get_state(IdBtree),
+        view_states=ViewStates
+    }.
 
 hex_sig(GroupSig) ->
     couch_util:to_hex(?b2l(GroupSig)).
@@ -460,13 +464,15 @@ set_view_sig(#group{
             lib={[]},
             def_lang=Language,
             design_options=DesignOptions}=G) ->
-    G#group{sig=couch_util:md5(term_to_binary({Views, Language, DesignOptions}))};
+    ViewInfo = [V#view{update_seq=0, purge_seq=0} || V <- Views],
+    G#group{sig=couch_util:md5(term_to_binary({ViewInfo, Language, DesignOptions}))};
 set_view_sig(#group{
             views=Views,
             lib=Lib,
             def_lang=Language,
             design_options=DesignOptions}=G) ->
-    G#group{sig=couch_util:md5(term_to_binary({Views, Language, DesignOptions, sort_lib(Lib)}))}.
+    ViewInfo = [V#view{update_seq=0, purge_seq=0} || V <- Views],
+    G#group{sig=couch_util:md5(term_to_binary({ViewInfo, Language, DesignOptions, sort_lib(Lib)}))}.
 
 sort_lib({Lib}) ->
     sort_lib(Lib, []).
@@ -574,14 +580,14 @@ delete_index_file(RootDir, DbName, Group
 init_group(Db, Fd, #group{views=Views}=Group, nil) ->
     init_group(Db, Fd, Group,
         #index_header{seq=0, purge_seq=couch_db:get_purge_seq(Db),
-            id_btree_state=nil, view_states=[nil || _ <- Views]});
+            id_btree_state=nil, view_states=[{nil, 0, 0} || _ <- Views]});
 init_group(Db, Fd, #group{def_lang=Lang,views=Views}=
             Group, IndexHeader) ->
      #index_header{seq=Seq, purge_seq=PurgeSeq,
             id_btree_state=IdBtreeState, view_states=ViewStates} = IndexHeader,
     {ok, IdBtree} = couch_btree:open(IdBtreeState, Fd),
     Views2 = lists:zipwith(
-        fun(BtreeState, #view{reduce_funs=RedFuns,options=Options}=View) ->
+        fun({BTState, USeq, PSeq}, #view{reduce_funs=RedFuns,options=Options}=View) ->
             FunSrcs = [FunSrc || {_Name, FunSrc} <- RedFuns],
             ReduceFun =
                 fun(reduce, KVs) ->
@@ -604,10 +610,10 @@ init_group(Db, Fd, #group{def_lang=Lang,
             <<"raw">> ->
                 Less = fun(A,B) -> A < B end
             end,
-            {ok, Btree} = couch_btree:open(BtreeState, Fd,
-                        [{less, Less},
-                            {reduce, ReduceFun}]),
-            View#view{btree=Btree}
+            {ok, Btree} = couch_btree:open(BTState, Fd,
+                    [{less, Less}, {reduce, ReduceFun}]
+            ),
+            View#view{btree=Btree, update_seq=USeq, purge_seq=PSeq}
         end,
         ViewStates, Views),
     Group#group{db=Db, fd=Fd, current_seq=Seq, purge_seq=PurgeSeq,

Modified: couchdb/trunk/src/couchdb/couch_view_updater.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_updater.erl?rev=1006339&r1=1006338&r2=1006339&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_view_updater.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view_updater.erl Sun Oct 10 19:07:10 2010
@@ -96,19 +96,25 @@ purge_index(#group{db=Db, views=Views, i
         end, dict:new(), Lookups),
 
     % Now remove the values from the btrees
+    PurgeSeq = couch_db:get_purge_seq(Db),
     Views2 = lists:map(
         fun(#view{id_num=Num,btree=Btree}=View) ->
             case dict:find(Num, ViewKeysToRemoveDict) of
             {ok, RemoveKeys} ->
-                {ok, Btree2} = couch_btree:add_remove(Btree, [], RemoveKeys),
-                View#view{btree=Btree2};
+                {ok, ViewBtree2} = couch_btree:add_remove(Btree, [], RemoveKeys),
+                case ViewBtree2 =/= Btree of
+                    true ->
+                        View#view{btree=ViewBtree2, purge_seq=PurgeSeq};
+                    _ ->
+                        View#view{btree=ViewBtree2}
+                end;
             error -> % no keys to remove in this view
                 View
             end
         end, Views),
     Group#group{id_btree=IdBtree2,
             views=Views2,
-            purge_seq=couch_db:get_purge_seq(Db)}.
+            purge_seq=PurgeSeq}.
 
 
 load_doc(Db, DocInfo, MapQueue, DocOpts, IncludeDesign) ->
@@ -247,7 +253,12 @@ write_changes(Group, ViewKeyValuesToAdd,
     Views2 = lists:zipwith(fun(View, {_View, AddKeyValues}) ->
             KeysToRemove = couch_util:dict_find(View#view.id_num, KeysToRemoveByView, []),
             {ok, ViewBtree2} = couch_btree:add_remove(View#view.btree, AddKeyValues, KeysToRemove),
-            View#view{btree = ViewBtree2}
+            case ViewBtree2 =/= View#view.btree of
+                true ->
+                    View#view{btree=ViewBtree2, update_seq=NewSeq};
+                _ ->
+                    View#view{btree=ViewBtree2}
+            end
         end,    Group#group.views, ViewKeyValuesToAdd),
     Group#group{views=Views2, current_seq=NewSeq, id_btree=IdBtree2}.
 



Mime
View raw message