couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vatam...@apache.org
Subject [29/50] couch-replicator commit: updated refs/heads/63012-scheduler to 27a5eae
Date Tue, 14 Mar 2017 19:26:24 GMT
Share replicator connections by proxy

Prior to this commit, proxied and unproxied replication connections
would be shared based on their source or target. This caused a bug where
a proxied connection could function as an unproxied connection if an
idle unproxied connection was shared with it.


Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/58ddc265
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/58ddc265
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/58ddc265

Branch: refs/heads/63012-scheduler
Commit: 58ddc26531789320e00d497c4fa3b95dea618d36
Parents: 1413a6f
Author: Benjamin Bastian <benjamin.bastian@gmail.com>
Authored: Mon Nov 21 09:57:30 2016 -0800
Committer: Benjamin Bastian <benjamin.bastian@gmail.com>
Committed: Wed Nov 23 13:49:52 2016 -0800

----------------------------------------------------------------------
 src/couch_replicator_api_wrap.hrl   |  3 ++-
 src/couch_replicator_connection.erl |  7 ++++++-
 src/couch_replicator_docs.erl       | 32 ++++++++++++++++++--------------
 src/couch_replicator_httpc.erl      | 14 ++++++++++++--
 src/couch_replicator_scheduler.erl  |  9 ++++++++-
 5 files changed, 46 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/58ddc265/src/couch_replicator_api_wrap.hrl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.hrl b/src/couch_replicator_api_wrap.hrl
index d15d214..fc94054 100644
--- a/src/couch_replicator_api_wrap.hrl
+++ b/src/couch_replicator_api_wrap.hrl
@@ -25,7 +25,8 @@
     wait = 250,         % milliseconds
     httpc_pool = nil,
     http_connections,
-    first_error_timestamp = nil
+    first_error_timestamp = nil,
+    proxy_url
 }).
 
 -record(oauth, {

http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/58ddc265/src/couch_replicator_connection.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_connection.erl b/src/couch_replicator_connection.erl
index b776624..e580663 100644
--- a/src/couch_replicator_connection.erl
+++ b/src/couch_replicator_connection.erl
@@ -55,6 +55,9 @@ init([]) ->
     {ok, #state{close_interval=Interval, timer=Timer}}.
 
 
+acquire(URL) when is_binary(URL) ->
+    acquire(binary_to_list(URL));
+
 acquire(URL) ->
     case gen_server:call(?MODULE, {acquire, URL}) of
         {ok, Worker} ->
@@ -85,7 +88,9 @@ handle_call({acquire, URL}, From, State) ->
                     couch_stats:increment_counter([couch_replicator, connection, acquires]),
                     ets:insert(?MODULE, Worker#connection{mref=monitor(process, Pid)}),
                     {reply, {ok, Worker#connection.worker}, State}
-            end
+            end;
+        {error, invalid_uri} ->
+            {reply, {error, invalid_uri}, State}
     end;
 
 handle_call({create, URL, Worker}, From, State) ->

http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/58ddc265/src/couch_replicator_docs.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_docs.erl b/src/couch_replicator_docs.erl
index 07344bb..204cb39 100644
--- a/src/couch_replicator_docs.erl
+++ b/src/couch_replicator_docs.erl
@@ -259,17 +259,15 @@ parse_rep_doc(Doc, UserCtx) ->
 
 -spec parse_rep_doc_without_id({[_]}, #user_ctx{}) -> {ok, #rep{}}.
 parse_rep_doc_without_id({Props}, UserCtx) ->
-    ProxyParams = parse_proxy_params(get_value(<<"proxy">>, Props, <<>>)),
+    Proxy = get_value(<<"proxy">>, Props, <<>>),
     Opts = make_options(Props),
     case get_value(cancel, Opts, false) andalso
         (get_value(id, Opts, nil) =/= nil) of
     true ->
         {ok, #rep{options = Opts, user_ctx = UserCtx}};
     false ->
-        Source = parse_rep_db(get_value(<<"source">>, Props),
-                              ProxyParams, Opts),
-        Target = parse_rep_db(get_value(<<"target">>, Props),
-                              ProxyParams, Opts),
+        Source = parse_rep_db(get_value(<<"source">>, Props), Proxy, Opts),
+        Target = parse_rep_db(get_value(<<"target">>, Props), Proxy, Opts),
         {Type, View} = case couch_replicator_filters:view_type(Props, Opts) of
         {error, Error} ->
             throw({bad_request, Error});
@@ -380,8 +378,13 @@ rep_user_ctx({RepDoc}) ->
         }
     end.
 
--spec parse_rep_db({[_]} | binary(), [_], [_]) -> #httpd{} | binary().
-parse_rep_db({Props}, ProxyParams, Options) ->
+-spec parse_rep_db({[_]} | binary(), binary(), [_]) -> #httpd{} | binary().
+parse_rep_db({Props}, Proxy, Options) ->
+    ProxyParams = parse_proxy_params(Proxy),
+    ProxyURL = case ProxyParams of
+        [] -> undefined;
+        _ -> binary_to_list(Proxy)
+    end,
     Url = maybe_add_trailing_slash(get_value(<<"url">>, Props)),
     {AuthProps} = get_value(<<"auth">>, Props, {[]}),
     {BinHeaders} = get_value(<<"headers">>, Props, {[]}),
@@ -414,15 +417,16 @@ parse_rep_db({Props}, ProxyParams, Options) ->
                 ProxyParams ++ ssl_params(Url)]),
         timeout = get_value(connection_timeout, Options),
         http_connections = get_value(http_connections, Options),
-        retries = get_value(retries, Options)
+        retries = get_value(retries, Options),
+        proxy_url = ProxyURL
     };
-parse_rep_db(<<"http://", _/binary>> = Url, ProxyParams, Options) ->
-    parse_rep_db({[{<<"url">>, Url}]}, ProxyParams, Options);
-parse_rep_db(<<"https://", _/binary>> = Url, ProxyParams, Options) ->
-    parse_rep_db({[{<<"url">>, Url}]}, ProxyParams, Options);
-parse_rep_db(<<DbName/binary>>, _ProxyParams, _Options) ->
+parse_rep_db(<<"http://", _/binary>> = Url, Proxy, Options) ->
+    parse_rep_db({[{<<"url">>, Url}]}, Proxy, Options);
+parse_rep_db(<<"https://", _/binary>> = Url, Proxy, Options) ->
+    parse_rep_db({[{<<"url">>, Url}]}, Proxy, Options);
+parse_rep_db(<<DbName/binary>>, _Proxy, _Options) ->
     DbName;
-parse_rep_db(undefined, _ProxyParams, _Options) ->
+parse_rep_db(undefined, _Proxy, _Options) ->
     throw({error, <<"Missing replicator database">>}).
 
 

http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/58ddc265/src/couch_replicator_httpc.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc.erl b/src/couch_replicator_httpc.erl
index 5fb7842..58fb0e1 100644
--- a/src/couch_replicator_httpc.erl
+++ b/src/couch_replicator_httpc.erl
@@ -40,8 +40,18 @@
 -define(MAX_DISCARDED_MESSAGES, 16).
 
 
-setup(#httpdb{httpc_pool = nil, url = Url, http_connections = MaxConns} = Db) ->
-    {ok, Pid} = couch_replicator_httpc_pool:start_link(Url, [{max_connections, MaxConns}]),
+setup(Db) ->
+    #httpdb{
+        httpc_pool = nil,
+        url = Url,
+        http_connections = MaxConns,
+        proxy_url = ProxyURL
+    } = Db,
+    HttpcURL = case ProxyURL of
+        undefined -> Url;
+        _ when is_list(ProxyURL) -> ProxyURL
+    end,
+    {ok, Pid} = couch_replicator_httpc_pool:start_link(HttpcURL, [{max_connections, MaxConns}]),
     {ok, Db#httpdb{httpc_pool = Pid}}.
 
 

http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/58ddc265/src/couch_replicator_scheduler.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_scheduler.erl b/src/couch_replicator_scheduler.erl
index 440458e..f08e829 100644
--- a/src/couch_replicator_scheduler.erl
+++ b/src/couch_replicator_scheduler.erl
@@ -119,6 +119,12 @@ job_summary(JobId, HealthThreshold) ->
                 {Pid, ErrorCount} when is_pid(Pid) ->
                      {running, null}
             end,
+            StrippedProxyURL = case (Rep#rep.source)#httpdb.proxy_url of
+                undefined ->
+                    null;
+                ProxyURL when is_list(ProxyURL) ->
+                    list_to_binary(couch_util:url_strip_password(ProxyURL))
+            end,
             [
                 {source, iolist_to_binary(ejson_url(Rep#rep.source))},
                 {target, iolist_to_binary(ejson_url(Rep#rep.target))},
@@ -126,7 +132,8 @@ job_summary(JobId, HealthThreshold) ->
                 {info, Info},
                 {error_count, ErrorCount},
                 {last_updated, last_updated(History)},
-                {start_time, couch_replicator_utils:iso8601(Rep#rep.start_time)}
+                {start_time, couch_replicator_utils:iso8601(Rep#rep.start_time)},
+                {proxy, StrippedProxyURL}
             ];
         {error, not_found} ->
             nil  % Job might have just completed


Mime
View raw message