couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dam...@apache.org
Subject svn commit: r751163 - in /couchdb/branches/rep_security: share/www/script/ share/www/script/test/ src/couchdb/
Date Sat, 07 Mar 2009 01:28:48 GMT
Author: damien
Date: Sat Mar  7 01:28:47 2009
New Revision: 751163

URL: http://svn.apache.org/viewvc?rev=751163&view=rev
Log:
Revision stemming implemented. Set default limit to 1000

Modified:
    couchdb/branches/rep_security/share/www/script/couch.js
    couchdb/branches/rep_security/share/www/script/couch_tests.js
    couchdb/branches/rep_security/share/www/script/test/compact.js
    couchdb/branches/rep_security/share/www/script/test/security_validation.js
    couchdb/branches/rep_security/src/couchdb/couch_db.erl
    couchdb/branches/rep_security/src/couchdb/couch_db.hrl
    couchdb/branches/rep_security/src/couchdb/couch_db_updater.erl
    couchdb/branches/rep_security/src/couchdb/couch_httpd_db.erl
    couchdb/branches/rep_security/src/couchdb/couch_key_tree.erl
    couchdb/branches/rep_security/src/couchdb/couch_rep.erl

Modified: couchdb/branches/rep_security/share/www/script/couch.js
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/share/www/script/couch.js?rev=751163&r1=751162&r2=751163&view=diff
==============================================================================
--- couchdb/branches/rep_security/share/www/script/couch.js [utf-8] (original)
+++ couchdb/branches/rep_security/share/www/script/couch.js [utf-8] Sat Mar  7 01:28:47 2009
@@ -202,6 +202,20 @@
     return JSON.parse(this.last_req.responseText);
   }
   
+  this.setDbProperty = function(propId, propValue) {
+    this.last_req = this.request("PUT", this.uri + propId,{
+      body:JSON.stringify(propValue)
+    });
+    CouchDB.maybeThrowError(this.last_req);
+    return JSON.parse(this.last_req.responseText);
+  }
+  
+  this.getDbProperty = function(propId) {
+    this.last_req = this.request("GET", this.uri + propId);
+    CouchDB.maybeThrowError(this.last_req);
+    return JSON.parse(this.last_req.responseText);
+  }
+  
   this.setAdmins = function(adminsArray) {
     this.last_req = this.request("PUT", this.uri + "_admins",{
       body:JSON.stringify(adminsArray)

Modified: couchdb/branches/rep_security/share/www/script/couch_tests.js
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/share/www/script/couch_tests.js?rev=751163&r1=751162&r2=751163&view=diff
==============================================================================
--- couchdb/branches/rep_security/share/www/script/couch_tests.js [utf-8] (original)
+++ couchdb/branches/rep_security/share/www/script/couch_tests.js [utf-8] Sat Mar  7 01:28:47
2009
@@ -67,6 +67,7 @@
 loadTest("config.js");
 loadTest("security_validation.js");
 loadTest("stats.js");
+loadTest("rev_stemming.js");
 
 function makeDocs(start, end, templateDoc) {
   var templateDocSrc = templateDoc ? JSON.stringify(templateDoc) : "{}"

Modified: couchdb/branches/rep_security/share/www/script/test/compact.js
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/share/www/script/test/compact.js?rev=751163&r1=751162&r2=751163&view=diff
==============================================================================
--- couchdb/branches/rep_security/share/www/script/test/compact.js (original)
+++ couchdb/branches/rep_security/share/www/script/test/compact.js Sat Mar  7 01:28:47 2009
@@ -35,12 +35,11 @@
   for(var i in docs) {
       db.deleteDoc(docs[i]);
   }
-  db.setAdmins(["Foo bar"]);
   var deletesize = db.info().disk_size;
   T(deletesize > originalsize);
 
-  var xhr = CouchDB.request("POST", "/test_suite_db/_compact");
-  T(xhr.status == 202);
+  T(db.compact().ok);
+  T(db.last_req.status == 202);
   // compaction isn't instantaneous, loop until done
   while (db.info().compact_running) {};
   

Modified: couchdb/branches/rep_security/share/www/script/test/security_validation.js
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/share/www/script/test/security_validation.js?rev=751163&r1=751162&r2=751163&view=diff
==============================================================================
--- couchdb/branches/rep_security/share/www/script/test/security_validation.js (original)
+++ couchdb/branches/rep_security/share/www/script/test/security_validation.js Sat Mar  7
01:28:47 2009
@@ -94,8 +94,8 @@
         T(userDb.last_req.status == 401);
       }
 
-      // add user as admin
-      db.setAdmins(["Damien Katz"]);
+      // set user as the admin
+      T(db.setDbProperty("_admins", ["Damien Katz"]).ok);
 
       T(userDb.save(designDoc).ok);
 

Modified: couchdb/branches/rep_security/src/couchdb/couch_db.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/src/couchdb/couch_db.erl?rev=751163&r1=751162&r2=751163&view=diff
==============================================================================
--- couchdb/branches/rep_security/src/couchdb/couch_db.erl (original)
+++ couchdb/branches/rep_security/src/couchdb/couch_db.erl Sat Mar  7 01:28:47 2009
@@ -17,6 +17,7 @@
 -export([open_ref_counted/2,is_idle/1,monitor/1,count_changes_since/2]).
 -export([update_doc/3,update_docs/4,update_docs/2,update_docs/3,delete_doc/3]).
 -export([get_doc_info/2,open_doc/2,open_doc/3,open_doc_revs/4]).
+-export([set_revs_limit/2,get_revs_limit/1]).
 -export([get_missing_revs/2,name/1,doc_to_tree/1,get_update_seq/1,get_committed_update_seq/1]).
 -export([enum_docs/4,enum_docs/5,enum_docs_since/4,enum_docs_since/5]).
 -export([enum_docs_since_reduce_to_count/1,enum_docs_reduce_to_count/1]).
@@ -184,15 +185,31 @@
         ],
     {ok, InfoList}.
 
+check_is_admin(#db{admins=Admins, user_ctx=#user_ctx{name=Name,roles=Roles}}) ->
+    DbAdmins = [<<"_admin">> | Admins],
+    case DbAdmins -- [Name | Roles] of
+    DbAdmins -> % same list, not an admin
+        throw({unauthorized, <<"You are not a db or server admin.">>});
+    _ ->
+        ok
+    end.
+
 get_admins(#db{admins=Admins}) ->
     Admins.
 
-set_admins(#db{update_pid=UpdatePid,user_ctx=Ctx}, 
-        Admins) when is_list(Admins) ->
-    case gen_server:call(UpdatePid, {set_admins, Admins, Ctx}, infinity) of
-    ok -> ok;
-    Error -> throw(Error)
-    end.
+set_admins(#db{update_pid=Pid}=Db, Admins) when is_list(Admins) ->
+    check_is_admin(Db),
+    gen_server:call(Pid, {set_admins, Admins}, infinity).
+
+
+get_revs_limit(#db{revs_limit=Limit}) ->
+    Limit.
+
+set_revs_limit(#db{update_pid=Pid}=Db, Limit) when Limit > 0 ->
+    check_is_admin(Db),
+    gen_server:call(Pid, {set_revs_limit, Limit}, infinity);
+set_revs_limit(_Db, _Limit) ->
+    throw(invalid_revs_limit).
 
 name(#db{name=Name}) ->
     Name.

Modified: couchdb/branches/rep_security/src/couchdb/couch_db.hrl
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/src/couchdb/couch_db.hrl?rev=751163&r1=751162&r2=751163&view=diff
==============================================================================
--- couchdb/branches/rep_security/src/couchdb/couch_db.hrl (original)
+++ couchdb/branches/rep_security/src/couchdb/couch_db.hrl Sat Mar  7 01:28:47 2009
@@ -102,7 +102,7 @@
 % if the disk revision is incremented, then new upgrade logic will need to be
 % added to couch_db_updater:init_db.
 
--define(LATEST_DISK_VERSION, 0).
+-define(LATEST_DISK_VERSION, 1).
 
 -record(db_header,
     {disk_version = ?LATEST_DISK_VERSION,  
@@ -113,13 +113,14 @@
      local_docs_btree_state = nil,
      purge_seq = 0,
      purged_docs = nil,
-     admins_ptr = nil 
+     admins_ptr = nil,
+     revs_limit = 1000
     }).
 
 -record(db,
-    {main_pid=nil,
-    update_pid=nil,
-    compactor_pid=nil,
+    {main_pid = nil,
+    update_pid = nil,
+    compactor_pid = nil,
     instance_start_time, % number of microsecs since jan 1 1970 as a binary string
     fd,
     fd_ref_counter,
@@ -131,11 +132,12 @@
     update_seq,
     name,
     filepath,
-    validate_doc_funs=[],
-    admins=[],
-    admins_ptr=nil,
-    user_ctx=#user_ctx{},
-    waiting_delayed_commit=nil
+    validate_doc_funs = [],
+    admins = [],
+    admins_ptr = nil,
+    user_ctx = #user_ctx{},
+    waiting_delayed_commit = nil,
+    revs_limit = 1000
     }).
 
 

Modified: couchdb/branches/rep_security/src/couchdb/couch_db_updater.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/src/couchdb/couch_db_updater.erl?rev=751163&r1=751162&r2=751163&view=diff
==============================================================================
--- couchdb/branches/rep_security/src/couchdb/couch_db_updater.erl (original)
+++ couchdb/branches/rep_security/src/couchdb/couch_db_updater.erl Sat Mar  7 01:28:47 2009
@@ -62,18 +62,18 @@
     couch_db_update_notifier:notify({updated, Db#db.name}),
     {reply, {ok, Db2#db.update_seq}, Db2};
 
-handle_call({set_admins, NewAdmins, #user_ctx{roles=Roles}}, _From, Db) ->
-    DbAdmins = [<<"_admin">> | Db#db.admins],
-    case length(DbAdmins -- Roles) == length(DbAdmins) of
-    true ->
-        {reply, {unauthorized, <<"You are not a db or server admin.">>}, Db};
-    false ->
-        {ok, Ptr} = couch_file:append_term(Db#db.fd, NewAdmins),
-        Db2 = commit_data(Db#db{admins=NewAdmins, admins_ptr=Ptr,
-                update_seq=Db#db.update_seq+1}),
-        ok = gen_server:call(Db2#db.main_pid, {db_updated, Db2}),
-        {reply, ok, Db2}
-    end;
+handle_call({set_admins, NewAdmins}, _From, Db) ->
+    {ok, Ptr} = couch_file:append_term(Db#db.fd, NewAdmins),
+    Db2 = commit_data(Db#db{admins=NewAdmins, admins_ptr=Ptr,
+            update_seq=Db#db.update_seq+1}),
+    ok = gen_server:call(Db2#db.main_pid, {db_updated, Db2}),
+    {reply, ok, Db2};
+
+handle_call({set_revs_limit, Limit}, _From, Db) ->
+    Db2 = commit_data(Db#db{revs_limit=Limit,
+            update_seq=Db#db.update_seq+1}),
+    ok = gen_server:call(Db2#db.main_pid, {db_updated, Db2}),
+    {reply, ok, Db2};
 
 handle_call({purge_docs, _IdRevs}, _From,
         #db{compactor_pid=Pid}=Db) when Pid /= nil ->
@@ -296,7 +296,8 @@
         filepath = Filepath,
         admins = Admins,
         admins_ptr = AdminsPtr,
-        instance_start_time = StartTime
+        instance_start_time = StartTime,
+        revs_limit = Header#db_header.revs_limit
         }.
 
 
@@ -391,6 +392,12 @@
         [FullDocInfo#full_doc_info{deleted=Deleted}|AccById],
         [DocInfo|AccBySeq]).
 
+
+stem_full_doc_infos(#db{revs_limit=Limit}, DocInfos) ->
+    [Info#full_doc_info{rev_tree=couch_key_tree:stem(Tree, Limit)} ||
+            #full_doc_info{rev_tree=Tree}=Info <- DocInfos].
+        
+
 update_docs_int(Db, DocsList, Options) ->
     #db{
         fulldocinfo_by_id_btree = DocInfoByIdBTree,
@@ -421,9 +428,12 @@
         Ids, OldDocLookups),
     
     % Merge the new docs into the revision trees.
-    {ok, NewDocInfos, Conflicts, NewSeq} = merge_rev_trees(
+    {ok, NewDocInfos0, Conflicts, NewSeq} = merge_rev_trees(
             lists:member(merge_conflicts, Options),
             DocsList2, OldDocInfos, [], [], LastSeq),
+    
+    NewDocInfos = stem_full_doc_infos(Db, NewDocInfos0),
+    
     RemoveSeqs =
         [OldSeq || {ok, #full_doc_info{update_seq=OldSeq}} <- OldDocLookups],
     
@@ -502,7 +512,8 @@
         docinfo_by_seq_btree_state = couch_btree:get_state(Db#db.docinfo_by_seq_btree),
         fulldocinfo_by_id_btree_state = couch_btree:get_state(Db#db.fulldocinfo_by_id_btree),
         local_docs_btree_state = couch_btree:get_state(Db#db.local_docs_btree),
-        admins_ptr = Db#db.admins_ptr
+        admins_ptr = Db#db.admins_ptr,
+        revs_limit = Db#db.revs_limit
         },
     if Header == Header2 ->
         Db;
@@ -551,10 +562,11 @@
 copy_docs(#db{fd=SrcFd}=Db, #db{fd=DestFd,summary_stream=DestStream}=NewDb, InfoBySeq, Retry)
->
     Ids = [Id || #doc_info{id=Id} <- InfoBySeq],
     LookupResults = couch_btree:lookup(Db#db.fulldocinfo_by_id_btree, Ids),
-    NewFullDocInfos = lists:map(
+    NewFullDocInfos0 = lists:map(
         fun({ok, #full_doc_info{rev_tree=RevTree}=Info}) ->
             Info#full_doc_info{rev_tree=copy_rev_tree(SrcFd, DestFd, DestStream, RevTree)}
         end, LookupResults),
+    NewFullDocInfos = stem_full_doc_infos(Db, NewFullDocInfos0),
     NewDocInfos = [couch_doc:to_doc_info(Info) || Info <- NewFullDocInfos],
     RemoveSeqs =
     case Retry of

Modified: couchdb/branches/rep_security/src/couchdb/couch_httpd_db.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/src/couchdb/couch_httpd_db.erl?rev=751163&r1=751162&r2=751163&view=diff
==============================================================================
--- couchdb/branches/rep_security/src/couchdb/couch_httpd_db.erl (original)
+++ couchdb/branches/rep_security/src/couchdb/couch_httpd_db.erl Sat Mar  7 01:28:47 2009
@@ -273,6 +273,18 @@
 db_req(#httpd{path_parts=[_,<<"_admins">>]}=Req, _Db) ->
     send_method_not_allowed(Req, "PUT,GET");
 
+db_req(#httpd{method='PUT',path_parts=[_,<<"_revs_limit">>]}=Req,
+        Db) ->
+    Limit = couch_httpd:json_body(Req),
+    ok = couch_db:set_revs_limit(Db, Limit),
+    send_json(Req, {[{<<"ok">>, true}]});
+
+db_req(#httpd{method='GET',path_parts=[_,<<"_revs_limit">>]}=Req, Db) ->
+    send_json(Req, couch_db:get_revs_limit(Db));
+
+db_req(#httpd{path_parts=[_,<<"_revs_limit">>]}=Req, _Db) ->
+    send_method_not_allowed(Req, "PUT,GET");
+
 % Special case to enable using an unencoded slash in the URL of design docs, 
 % as slashes in document IDs must otherwise be URL encoded.
 db_req(#httpd{method='GET',mochi_req=MochiReq, path_parts=[DbName,<<"_design/",_/binary>>|_]}=Req,
_Db) ->

Modified: couchdb/branches/rep_security/src/couchdb/couch_key_tree.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/src/couchdb/couch_key_tree.erl?rev=751163&r1=751162&r2=751163&view=diff
==============================================================================
--- couchdb/branches/rep_security/src/couchdb/couch_key_tree.erl (original)
+++ couchdb/branches/rep_security/src/couchdb/couch_key_tree.erl Sat Mar  7 01:28:47 2009
@@ -14,7 +14,7 @@
 
 -export([merge/2, find_missing/2, get_key_leafs/2, get_full_key_paths/2, get/2]).
 -export([map/2, get_all_leafs/1, count_leafs/1, remove_leafs/2,
-    get_all_leafs_full/1,test/0]).
+    get_all_leafs_full/1,stem/2,test/0]).
 
 % a key tree looks like this:
 % Tree -> [] or [{Key, Value, ChildTree} | SiblingTree]
@@ -25,10 +25,7 @@
 
 % partial trees arranged by how much they are cut off.
 
-merge(A0, B0) ->
-    % convert to canonical
-    A = make_canonical(A0),
-    B = make_canonical(B0),
+merge(A, B) ->
     {Merged, HasConflicts} = 
     lists:foldl(
         fun(InsertTree, {AccTrees, AccConflicts}) ->
@@ -48,11 +45,6 @@
     end,
     {lists:sort(Merged), Conflicts}.
 
-make_canonical([{_Start, _Tree} | _] = Can) ->
-    Can; % already canonical
-make_canonical(Trees) ->
-    [{0, Tree} || Tree <- Trees].
-
 merge_one([], Insert, OutAcc, ConflictsAcc) ->
     {ok, [Insert | OutAcc], ConflictsAcc};
 merge_one([{Start, Tree}|Rest], {StartInsert, TreeInsert}, OutAcc, ConflictsAcc) ->
@@ -146,24 +138,6 @@
     ImpossibleKeys ++ find_missing_simple(Pos, RestTree, SrcKeys3).
 
 
-get_all_key_paths_reverse_simple(Pos, [], KeyPathAcc) ->
-    [{Pos - 1, KeyPathAcc}];
-get_all_key_paths_reverse_simple(Pos, [{Key, Value, SubTree} | RestTree], KeyPathAcc) ->
-    SubLists = get_all_key_paths_reverse_simple(Pos + 1, SubTree, [{Key, Value} | KeyPathAcc]),
-    SiblingLists =
-        if RestTree /= [] ->
-            get_all_key_paths_reverse_simple(Pos, RestTree, KeyPathAcc);
-        true ->
-            []
-        end,
-    SiblingLists ++ SubLists.
-
-get_all_key_paths_reverse([], PathsAcc) ->
-    PathsAcc;
-get_all_key_paths_reverse([{Start, Tree}|Rest], PathsAcc) ->
-    get_all_key_paths_reverse(Rest, get_all_key_paths_reverse_simple(Start, [Tree], []) ++
PathsAcc).
-
-
 filter_leafs([], _Keys, FilteredAcc, RemovedKeysAcc) ->
     {FilteredAcc, RemovedKeysAcc};
 filter_leafs([{Pos, [{LeafKey, _}|_]} = Path |Rest], Keys, FilteredAcc, RemovedKeysAcc) ->
@@ -179,7 +153,7 @@
 % Removes any branches from the tree whose leaf node(s) are in the Keys
 remove_leafs(Trees, Keys) ->
     % flatten each branch in a tree into a tree path
-    Paths = get_all_key_paths_reverse(make_canonical(Trees), []),
+    Paths = get_all_leafs_full(Trees),
     
     % filter out any that are in the keys list.
     {FilteredPaths, RemovedKeys} = filter_leafs(Paths, Keys, [], []),
@@ -318,6 +292,20 @@
     [{Key, Value2, map_simple(Fun, Pos + 1, SubTree)} | map_simple(Fun, Pos, RestTree)].
 
 
+stem(Trees, Limit) ->
+    % flatten each branch in a tree into a tree path
+    Paths = get_all_leafs_full(Trees),
+    
+    Paths2 = [{Pos, lists:sublist(Path, Limit)} || {Pos, Path} <- Paths],
+    
+    % convert paths back to trees
+    lists:foldl(
+        fun({PathPos, Path},TreeAcc) ->
+            [SingleTree] = lists:foldl(
+                fun({K,V},NewTreeAcc) -> [{K,V,NewTreeAcc}] end, [], Path),
+            {NewTrees, _} = merge(TreeAcc, [{PathPos + 1 - length(Path), SingleTree}]),
+            NewTrees
+        end, [], Paths2).
 
 test() ->
     EmptyTree = [],
@@ -353,7 +341,7 @@
     {TwoChildSibs, []} = remove_leafs(TwoChildSibs, []),
     {TwoChildSibs, []} = remove_leafs(TwoChildSibs, [{0, "1"}]),
     {OneChild, [{1, "1b"}]} = remove_leafs(TwoChildSibs, [{1, "1b"}]),
-    {[], [{1, "1a"},{1, "1b"}]} = remove_leafs(TwoChildSibs, [{1, "1a"}, {1, "1b"}]),
+    {[], [{1, "1b"},{1, "1a"}]} = remove_leafs(TwoChildSibs, [{1, "1a"}, {1, "1b"}]),
     {Stemmed1a, []} = remove_leafs(Stemmed1a, [{1, "1a"}]),
     {[], [{2, "1aa"}]} = remove_leafs(Stemmed1a, [{2, "1aa"}]),
     {TwoChildSibs, []} = remove_leafs(TwoChildSibs, []),
@@ -380,7 +368,9 @@
     2 = count_leafs(TwoChildSibs),
     1 = count_leafs(Stemmed1a),
     
-    
+    TwoChild = stem(TwoChild, 3),
+    Stemmed1a = stem(TwoChild, 2),
+    Stemmed1aa = stem(TwoChild, 1),
     ok.
     
     

Modified: couchdb/branches/rep_security/src/couchdb/couch_rep.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/src/couchdb/couch_rep.erl?rev=751163&r1=751162&r2=751163&view=diff
==============================================================================
--- couchdb/branches/rep_security/src/couchdb/couch_rep.erl (original)
+++ couchdb/branches/rep_security/src/couchdb/couch_rep.erl Sat Mar  7 01:28:47 2009
@@ -139,7 +139,7 @@
         % commit changes to both src and tgt. The src because if changes
         % we replicated are lost, we'll record the a seq number ahead 
         % of what was committed. If those changes are lost and the seq number
-        % reverts to a previous committed value, we will lose future changes
+        % reverts to a previous committed value, we will skip future changes
         % when new doc updates are given our already replicated seq nums.
         
         % commit the src async



Mime
View raw message