couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dam...@apache.org
Subject svn commit: r750996 - in /couchdb/branches/rep_security: etc/couchdb/ share/www/script/ share/www/script/test/ src/couchdb/
Date Fri, 06 Mar 2009 17:35:00 GMT
Author: damien
Date: Fri Mar  6 17:35:00 2009
New Revision: 750996

URL: http://svn.apache.org/viewvc?rev=750996&view=rev
Log:
More replication security work. Fixed authentication for replicating remote databases.

Modified:
    couchdb/branches/rep_security/etc/couchdb/local_dev.ini
    couchdb/branches/rep_security/share/www/script/couch.js
    couchdb/branches/rep_security/share/www/script/test/replication.js
    couchdb/branches/rep_security/share/www/script/test/security_validation.js
    couchdb/branches/rep_security/src/couchdb/couch_db.erl
    couchdb/branches/rep_security/src/couchdb/couch_httpd.erl
    couchdb/branches/rep_security/src/couchdb/couch_httpd_misc_handlers.erl
    couchdb/branches/rep_security/src/couchdb/couch_rep.erl

Modified: couchdb/branches/rep_security/etc/couchdb/local_dev.ini
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/etc/couchdb/local_dev.ini?rev=750996&r1=750995&r2=750996&view=diff
==============================================================================
--- couchdb/branches/rep_security/etc/couchdb/local_dev.ini (original)
+++ couchdb/branches/rep_security/etc/couchdb/local_dev.ini Fri Mar  6 17:35:00 2009
@@ -12,7 +12,7 @@
 ;bind_address = 127.0.0.1
 
 [log]
-level = info
+level = error
 
 [update_notification]
 ;unique notifier name=/full/path/to/exe -with "cmd line arg"

Modified: couchdb/branches/rep_security/share/www/script/couch.js
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/share/www/script/couch.js?rev=750996&r1=750995&r2=750996&view=diff
==============================================================================
--- couchdb/branches/rep_security/share/www/script/couch.js [utf-8] (original)
+++ couchdb/branches/rep_security/share/www/script/couch.js [utf-8] Fri Mar  6 17:35:00 2009
@@ -282,10 +282,13 @@
   return JSON.parse(CouchDB.last_req.responseText).version;
 }
 
-CouchDB.replicate = function(source, target, options) {
-  options = options || {};
+CouchDB.replicate = function(source, target, rep_options) {
+  rep_options = rep_options || {};
+  var headers = rep_options.headers || {};
+  var options = rep_options.options || {};
   CouchDB.last_req = CouchDB.request("POST", "/_replicate", {
-    body: JSON.stringify({source: source, target: target, "options": options})
+    headers: headers,
+    body: JSON.stringify({source: source, target: target, options: options})
   });
   CouchDB.maybeThrowError(CouchDB.last_req);
   return JSON.parse(CouchDB.last_req.responseText);

Modified: couchdb/branches/rep_security/share/www/script/test/replication.js
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/share/www/script/test/replication.js?rev=750996&r1=750995&r2=750996&view=diff
==============================================================================
--- couchdb/branches/rep_security/share/www/script/test/replication.js (original)
+++ couchdb/branches/rep_security/share/www/script/test/replication.js Fri Mar  6 17:35:00
2009
@@ -179,25 +179,37 @@
       }
     }
 
-    T(CouchDB.replicate(A, B).ok);
+    var result = CouchDB.replicate(A, B);
+    
+    var seqA = result.source_last_seq;
 
     for(test in repTests) {
       if(repTests[test].afterAB1) repTests[test].afterAB1(dbA, dbB);
     }
 
-    T(CouchDB.replicate(B, A).ok);
+    result = CouchDB.replicate(B, A);
+    
+    var seqB = result.source_last_seq;
 
     for(test in repTests) {
       if(repTests[test].afterBA1) repTests[test].afterBA1(dbA, dbB);
     }
 
-    T(CouchDB.replicate(A, B).ok);
+    result = CouchDB.replicate(A, B)
+    
+    T(seqA < result.source_last_seq);
+    
+    seqA = result.source_last_seq;
 
     for(test in repTests) {
       if(repTests[test].afterAB2) repTests[test].afterAB2(dbA, dbB);
     }
 
-    T(CouchDB.replicate(B, A).ok);
+    result = CouchDB.replicate(B, A)
+    
+    T(seqB < result.source_last_seq);
+    
+    seqB = result.source_last_seq;
 
     for(test in repTests) {
       if(repTests[test].afterBA2) repTests[test].afterBA2(dbA, dbB);

Modified: couchdb/branches/rep_security/share/www/script/test/security_validation.js
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/share/www/script/test/security_validation.js?rev=750996&r1=750995&r2=750996&view=diff
==============================================================================
--- couchdb/branches/rep_security/share/www/script/test/security_validation.js (original)
+++ couchdb/branches/rep_security/share/www/script/test/security_validation.js Fri Mar  6
17:35:00 2009
@@ -156,20 +156,20 @@
       var host = CouchDB.host;
       var dbPairs = [
         {source:"test_suite_db_a",
-          target:"test_suite_db_b",
-          options:{}},
+          target:"test_suite_db_b"},
     
         {source:"test_suite_db_a",
-          target:"http://" + host + "/test_suite_db_b",
-          options: {target_headers: AuthHeaders}},
+          target:{url: "http://" + host + "/test_suite_db_b",
+                  headers: AuthHeaders}},
         
-        {source:"http://" + host + "/test_suite_db_a",
-          target:"test_suite_db_b",
-          options: {source_headers: AuthHeaders}},
+        {source:{url:"http://" + host + "/test_suite_db_a",
+                 headers: AuthHeaders},
+          target:"test_suite_db_b"},
         
-        {source:"http://" + host + "/test_suite_db_a",
-          target:"http://" + host + "/test_suite_db_b",
-          options:{source_headers: AuthHeaders, target_headers: AuthHeaders}},
+        {source:{url:"http://" + host + "/test_suite_db_a",
+                 headers: AuthHeaders},
+         target:{url:"http://" + host + "/test_suite_db_b",
+                 headers: AuthHeaders}},
       ]
       var adminDbA = new CouchDB("test_suite_db_a");
       var adminDbB = new CouchDB("test_suite_db_b");
@@ -181,7 +181,6 @@
       for (var testPair = 0; testPair < dbPairs.length; testPair++) {
         var A = dbPairs[testPair].source
         var B = dbPairs[testPair].target
-        var Options = dbPairs[testPair].options
 
         adminDbA.deleteDb();
         adminDbA.createDb();
@@ -194,8 +193,8 @@
         dbA.save({_id:"foo2",value:"a",author:"Christopher Lenz"});
         dbA.save({_id:"bad1",value:"a"});
 
-        T(CouchDB.replicate(A, B).ok);
-        T(CouchDB.replicate(B, A).ok);
+        T(CouchDB.replicate(A, B, {headers:AuthHeaders}).ok);
+        T(CouchDB.replicate(B, A, {headers:AuthHeaders}).ok);
 
         T(dbA.open("foo1"));
         T(dbB.open("foo1"));
@@ -226,12 +225,12 @@
         foo2.value = "b";
         dbB.save(foo2);
   
-        var results = CouchDB.replicate(B, A);
+        var results = CouchDB.replicate(B, A, {headers:AuthHeaders});
   
         T(results.ok);
   
-        T(results.history[0].docs_written == 2);
-        T(results.history[0].doc_write_failures == 1);
+        T(results.history[0].docs_written == 1);
+        T(results.history[0].doc_write_failures == 2);
   
         // bad2 should not be on dbA
         T(dbA.open("bad2") == null);
@@ -240,7 +239,7 @@
         T(dbA.open("foo1").value == "a");
   
         // The edit to foo2 should have replicated.
-        T(dbA.open("foo2").value == "a");
+        T(dbA.open("foo2").value == "b");
       }
     });
 };

Modified: couchdb/branches/rep_security/src/couchdb/couch_db.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/src/couchdb/couch_db.erl?rev=750996&r1=750995&r2=750996&view=diff
==============================================================================
--- couchdb/branches/rep_security/src/couchdb/couch_db.erl (original)
+++ couchdb/branches/rep_security/src/couchdb/couch_db.erl Fri Mar  6 17:35:00 2009
@@ -338,17 +338,7 @@
 
 update_docs(#db{update_pid=UpdatePid}=Db, Docs, Options) ->
     update_docs(#db{update_pid=UpdatePid}=Db, Docs, Options, interactive_edit).
-    
-should_validate(Db, Docs) ->
-    % true if our db has validation funs, we have design docs,
-    % or we have attachments.
-    (Db#db.validate_doc_funs /= []) orelse
-        lists:any(
-            fun(#doc{id= <<?DESIGN_DOC_PREFIX, _/binary>>}) ->
-                true;
-            (#doc{attachments=Atts}) ->
-                Atts /= []
-            end, Docs).
+
 
 validate_replicated_updates(_Db, [], [], AccPrepped, AccErrors) ->
     Errors2 = [{{Id, {Pos, Rev}}, Error} || 
@@ -399,7 +389,7 @@
                     {AccValidated, AccErrors2}
                 end
             end,
-            {[], []}, Bucket),
+            {[], AccErrors}, Bucket),
         validate_replicated_updates(Db, RestBuckets, RestOldInfo, [ValidatedBucket | AccPrepped],
AccErrors3)
     end.
 
@@ -407,7 +397,11 @@
     couch_stats_collector:increment({couchdb, database_writes}),
     DocBuckets = group_alike_docs(Docs),
     
-    case should_validate(Db, Docs) of
+    case (Db#db.validate_doc_funs /= []) orelse
+        lists:any(
+            fun(#doc{id= <<?DESIGN_DOC_PREFIX, _/binary>>}) -> true;
+            (_) -> false
+            end, Docs) of
     true ->
         Ids = [Id || [#doc{id=Id}|_] <- DocBuckets],
         ExistingDocs = get_full_doc_infos(Db, Ids),
@@ -437,9 +431,13 @@
         end, Docs),
     DocBuckets = group_alike_docs(Docs2),
     
-    case ((Db#db.validate_doc_funs /= []) orelse
-            lists:any(fun(#doc{id= <<?DESIGN_DOC_PREFIX, _/binary>>}) -> true;
-                            (Doc) -> couch_doc:has_stubs(Doc) end, Docs)) of
+    case (Db#db.validate_doc_funs /= []) orelse
+        lists:any(
+            fun(#doc{id= <<?DESIGN_DOC_PREFIX, _/binary>>}) ->
+                true;
+            (#doc{attachments=Atts}) ->
+                Atts /= []
+            end, Docs) of
     true ->
         % lookup the doc by id and get the most recent
         Ids = [Id || [#doc{id=Id}|_] <- DocBuckets],

Modified: couchdb/branches/rep_security/src/couchdb/couch_httpd.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/src/couchdb/couch_httpd.erl?rev=750996&r1=750995&r2=750996&view=diff
==============================================================================
--- couchdb/branches/rep_security/src/couchdb/couch_httpd.erl (original)
+++ couchdb/branches/rep_security/src/couchdb/couch_httpd.erl Fri Mar  6 17:35:00 2009
@@ -158,7 +158,7 @@
     catch
         throw:Error ->
             send_error(HttpReq, Error);
-        Tag:Error ->
+        Tag:Error when Error ==foo ->
             ?LOG_ERROR("Uncaught error in HTTP request: ~p",[{Tag, Error}]),
             ?LOG_DEBUG("Stacktrace: ~p",[erlang:get_stacktrace()]),
             send_error(HttpReq, Error)

Modified: couchdb/branches/rep_security/src/couchdb/couch_httpd_misc_handlers.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/src/couchdb/couch_httpd_misc_handlers.erl?rev=750996&r1=750995&r2=750996&view=diff
==============================================================================
--- couchdb/branches/rep_security/src/couchdb/couch_httpd_misc_handlers.erl (original)
+++ couchdb/branches/rep_security/src/couchdb/couch_httpd_misc_handlers.erl Fri Mar  6 17:35:00
2009
@@ -70,26 +70,32 @@
 handle_task_status_req(Req) ->
     send_method_not_allowed(Req, "GET,HEAD").
 
+% convert to list and add trailing slash if missing
+fix_db_url(UrlBin) ->
+    case lists:last(Url = ?b2l(UrlBin)) of
+    $/ -> Url;
+    _  -> Url ++ "/"
+    end.
+    
+
+get_rep_endpoint(_Req, {Props}) ->
+    Url = proplists:get_value(<<"url">>, Props),
+    {BinHeaders} = proplists:get_value(<<"headers">>, Props, {[]}),
+    {remote, fix_db_url(Url), [{?b2l(K),?b2l(V)} || {K,V} <- BinHeaders]};
+get_rep_endpoint(_Req, <<"http://",_/binary>>=Url) ->
+    {remote, fix_db_url(Url), []};
+get_rep_endpoint(_Req, <<"https://",_/binary>>=Url) ->
+    {remote, fix_db_url(Url), []};
+get_rep_endpoint(#httpd{user_ctx=UserCtx}, <<DbName/binary>>) ->
+    {local, DbName, UserCtx}.
 
-handle_replicate_req(#httpd{user_ctx=UserCtx,method='POST'}=Req) ->
+handle_replicate_req(#httpd{method='POST'}=Req) ->
     {Props} = couch_httpd:json_body(Req),
-    Source = proplists:get_value(<<"source">>, Props),
-    Target = proplists:get_value(<<"target">>, Props),
-    
-    {Options} = proplists:get_value(<<"options">>, Props, {[]}),
-    {SrcHeadersJson} = proplists:get_value(<<"source_headers">>, Options, {[]}),
-    SrcHeaders = [{?b2l(K),(V)} || {K,V} <- SrcHeadersJson],
-    
-    {TgtHeadersJson} = proplists:get_value(<<"target_headers">>, Options, {[]}),
-    TgtHeaders = [{?b2l(K),(V)} || {K,V} <- TgtHeadersJson],
-    
-    {ok, {JsonResults}} = couch_rep:replicate(Source, Target,
-            [{source_options,   % Only one of following is used by api
-                [{headers, SrcHeaders}, % Headers used when src DB is URI
-                {user_ctx, UserCtx}]},  % Ctx used when src DB is local.
-            {target_options,
-                [{headers, TgtHeaders},
-                {user_ctx, UserCtx}]}]),
+    Src = get_rep_endpoint(Req, proplists:get_value(<<"source">>, Props)),
+    Tgt = get_rep_endpoint(Req, proplists:get_value(<<"target">>, Props)),
+    {OptionsBin} = proplists:get_value(<<"options">>, Props, {[]}),
+    Options = [{couch_util:to_existing_atom(K), V} || {K,V} <- OptionsBin],
+    {ok, {JsonResults}} = couch_rep:replicate(Src, Tgt, Options),
     send_json(Req, {[{ok, true} | JsonResults]});
 handle_replicate_req(Req) ->
     send_method_not_allowed(Req, "POST").

Modified: couchdb/branches/rep_security/src/couchdb/couch_rep.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/rep_security/src/couchdb/couch_rep.erl?rev=750996&r1=750995&r2=750996&view=diff
==============================================================================
--- couchdb/branches/rep_security/src/couchdb/couch_rep.erl (original)
+++ couchdb/branches/rep_security/src/couchdb/couch_rep.erl Fri Mar  6 17:35:00 2009
@@ -58,11 +58,9 @@
     replicate(DbNameA, DbNameB, []).
 
 replicate(Source, Target, Options) ->
-    {ok, DbSrc} = open_db(Source,
-            proplists:get_value(source_options, Options, [])),
+    {ok, DbSrc} = open_db(Source),
     try
-        {ok, DbTgt} = open_db(Target,
-                proplists:get_value(target_options, Options, [])),
+        {ok, DbTgt} = open_db(Target),
         try
             replicate2(Source, DbSrc, Target, DbTgt, Options)
         after
@@ -74,9 +72,9 @@
     
 replicate2(Source, DbSrc, Target, DbTgt, Options) ->
     {ok, HostName} = inet:gethostname(),
-    HostNameBin = list_to_binary(HostName),
-    RepRecKey = <<?LOCAL_DOC_PREFIX, HostNameBin/binary, 
-            ":", Source/binary, ":", Target/binary>>,
+    RepRecordNameDigest = couch_util:to_hex(
+            erlang:md5(term_to_binary([HostName, Source, Target]))),
+    RepRecKey = ?l2b(?LOCAL_DOC_PREFIX ++ RepRecordNameDigest),
     
     ReplicationStartTime = httpd_util:rfc1123_date(),
     
@@ -86,8 +84,7 @@
     SrcInstanceStartTime = proplists:get_value(instance_start_time, InfoSrc),
     TgtInstanceStartTime = proplists:get_value(instance_start_time, InfoTgt),
     
-    case proplists:get_value(full, Options, false)
-        orelse proplists:get_value("full", Options, false) of
+    case proplists:get_value(full, Options, false) of
     true ->
          RepRecSrc = RepRecTgt = #doc{id=RepRecKey};
     false ->
@@ -157,6 +154,7 @@
                 "replication is redone and documents reexamined.", []),
             SeqNum
         end,
+        % convert the stats record into a proplist
         [rep_stats | StatsList] = tuple_to_list(Stats),
         StatFieldNames =
                 [?l2b(atom_to_list(T)) || T <- record_info(fields, rep_stats)],
@@ -268,7 +266,9 @@
         docs_written=Stats#rep_stats.docs_written+length(Docs)-length(Errors),
         doc_write_failures=Stats#rep_stats.doc_write_failures+length(Errors)}.
 
-
+% we should probably write these to a special replication log
+% or have a callback where the caller decides what to do with replication
+% errors.
 dump_update_errors([]) -> ok;
 dump_update_errors([{{Id, Rev}, Error}|Rest]) ->
     ?LOG_INFO("error replicating document \"~s\" rev \"~s\":~p",
@@ -327,34 +327,20 @@
 
 
 sum_rep_stats(StatsA, StatsB) ->
-    % Quick and dirty sum matchng members of the records
+    % Quick and dirty way to sum matchng fields of the records.
     % convert to lists
-    [rep_stats | MembersA] = tuple_to_list(StatsA),
-    [rep_stats | MembersB] = tuple_to_list(StatsB),
-    % pairwise add the members and convert back to the record
+    [rep_stats | FieldsA] = tuple_to_list(StatsA),
+    [rep_stats | FieldsB] = tuple_to_list(StatsB),
+    % pairwise add the fields and convert back to the record
     list_to_tuple([rep_stats |
-            lists:zipwith(fun(A,B) -> A + B end, MembersA, MembersB)]).
-
+            lists:zipwith(fun(A,B) -> A + B end, FieldsA, FieldsB)]).
 
-fix_url(UrlBin) ->
-    Url = binary_to_list(UrlBin),
-    case lists:last(Url) of
-    $/ ->
-        Url;
-    _ ->
-        Url ++ "/"
-    end.
 
-open_http_db(UrlBin, Options) ->
-    Headers = proplists:get_value(headers, Options, {[]}),
-    {ok, #http_db{uri=fix_url(UrlBin), headers=Headers}}.
             
-open_db(<<"http://", _/binary>>=Url, Options)->
-    open_http_db(Url, Options);
-open_db(<<"https://", _/binary>>=Url, Options)->
-    open_http_db(Url, Options);
-open_db(DbName, Options)->
-    couch_db:open(DbName, Options).
+open_db({remote, Url, Headers})->
+    {ok, #http_db{uri=Url, headers=Headers}};
+open_db({local, DbName, UserCtx})->
+    couch_db:open(DbName, [{user_ctx, UserCtx}]).
 
 close_db(#http_db{})->
     ok;



Mime
View raw message