couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kocol...@apache.org
Subject [18/50] git commit: Replicator: configurable # of retries per request
Date Wed, 26 Oct 2011 18:05:33 GMT
Replicator: configurable # of retries per request

Instead of using an hardcoded value of 10, this parameter is
now configurable via .ini configuration or per replication
in the replication document/object (field "retries_per_request").


Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/37afbce7
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/37afbce7
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/37afbce7

Branch: refs/heads/1319-large-headers-are-corrupted
Commit: 37afbce7cf80289783b2eb813d023fbde4ad69fd
Parents: fb5d36f
Author: Filipe David Manana <fdmanana@apache.org>
Authored: Tue Oct 4 15:39:28 2011 -0700
Committer: Filipe David Manana <fdmanana@apache.org>
Committed: Tue Oct 4 15:39:28 2011 -0700

----------------------------------------------------------------------
 etc/couchdb/default.ini.tpl.in         |    2 ++
 src/couchdb/couch_api_wrap_httpc.erl   |    4 +++-
 src/couchdb/couch_replicator.erl       |    2 ++
 src/couchdb/couch_replicator_utils.erl |   11 ++++++++---
 4 files changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb/blob/37afbce7/etc/couchdb/default.ini.tpl.in
----------------------------------------------------------------------
diff --git a/etc/couchdb/default.ini.tpl.in b/etc/couchdb/default.ini.tpl.in
index 1e8c88a..12bc20f 100644
--- a/etc/couchdb/default.ini.tpl.in
+++ b/etc/couchdb/default.ini.tpl.in
@@ -173,6 +173,8 @@ http_connections = 20
 ; Even for very fast/reliable networks it might need to be increased if a remote
 ; database is too busy.
 connection_timeout = 30000
+; If a request fails, the replicator will retry it up to N times.
+retries_per_request = 10
 ; Some socket options that might boost performance in some scenarios:
 ;       {nodelay, boolean()}
 ;       {sndbuf, integer()}

http://git-wip-us.apache.org/repos/asf/couchdb/blob/37afbce7/src/couchdb/couch_api_wrap_httpc.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_api_wrap_httpc.erl b/src/couchdb/couch_api_wrap_httpc.erl
index bff581f..f1ef7d3 100644
--- a/src/couchdb/couch_api_wrap_httpc.erl
+++ b/src/couchdb/couch_api_wrap_httpc.erl
@@ -26,6 +26,7 @@
 ]).
 
 -define(replace(L, K, V), lists:keystore(K, 1, L, {K, V})).
+-define(MAX_WAIT, 5 * 60 * 1000).
 
 
 setup(#httpdb{httpc_pool = nil, url = Url, http_connections = MaxConns} = Db) ->
@@ -156,7 +157,8 @@ maybe_retry(Error, Worker, #httpdb{retries = Retries, wait = Wait} = HttpDb,
     ?LOG_INFO("Retrying ~s request to ~s in ~p seconds due to error ~s",
         [Method, Url, Wait / 1000, error_cause(Error)]),
     ok = timer:sleep(Wait),
-    send_req(HttpDb#httpdb{retries = Retries - 1, wait = Wait * 2}, Params, Cb).
+    Wait2 = erlang:min(Wait * 2, ?MAX_WAIT),
+    send_req(HttpDb#httpdb{retries = Retries - 1, wait = Wait2}, Params, Cb).
 
 
 report_error(Worker, HttpDb, Params, Error) ->

http://git-wip-us.apache.org/repos/asf/couchdb/blob/37afbce7/src/couchdb/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_replicator.erl b/src/couchdb/couch_replicator.erl
index efea373..40cb9a4 100644
--- a/src/couchdb/couch_replicator.erl
+++ b/src/couchdb/couch_replicator.erl
@@ -306,9 +306,11 @@ do_init(#rep{options = Options, id = {BaseId, Ext}} = Rep) ->
         "~ca worker batch size of ~p~n"
         "~c~p HTTP connections~n"
         "~ca connection timeout of ~p milliseconds~n"
+        "~c~p retries per request~n"
         "~csocket options are: ~s~s",
         [BaseId ++ Ext, $\t, NumWorkers, $\t, BatchSize, $\t,
             MaxConns, $\t, get_value(connection_timeout, Options),
+            $\t, get_value(retries, Options),
             $\t, io_lib:format("~p", [get_value(socket_options, Options)]),
             case StartSeq of
             ?LOWEST_SEQ ->

http://git-wip-us.apache.org/repos/asf/couchdb/blob/37afbce7/src/couchdb/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_replicator_utils.erl b/src/couchdb/couch_replicator_utils.erl
index 3c61400..07630e7 100644
--- a/src/couchdb/couch_replicator_utils.erl
+++ b/src/couchdb/couch_replicator_utils.erl
@@ -193,7 +193,8 @@ parse_rep_db({Props}, ProxyParams, Options) ->
             [{socket_options, get_value(socket_options, Options)} |
                 ProxyParams ++ ssl_params(Url)]),
         timeout = get_value(connection_timeout, Options),
-        http_connections = get_value(http_connections, Options)
+        http_connections = get_value(http_connections, Options),
+        retries = get_value(retries, Options)
     };
 parse_rep_db(<<"http://", _/binary>> = Url, ProxyParams, Options) ->
     parse_rep_db({[{<<"url">>, Url}]}, ProxyParams, Options);
@@ -220,16 +221,18 @@ make_options(Props) ->
     DefBatchSize = couch_config:get("replicator", "worker_batch_size", "500"),
     DefConns = couch_config:get("replicator", "http_connections", "20"),
     DefTimeout = couch_config:get("replicator", "connection_timeout", "30000"),
+    DefRetries = couch_config:get("replicator", "retries_per_request", "10"),
     {ok, DefSocketOptions} = couch_util:parse_term(
         couch_config:get("replicator", "socket_options",
             "[{keepalive, true}, {nodelay, false}]")),
-    lists:ukeymerge(1, Options, [
+    lists:ukeymerge(1, Options, lists:keysort(1, [
         {connection_timeout, list_to_integer(DefTimeout)},
+        {retries, list_to_integer(DefRetries)},
         {http_connections, list_to_integer(DefConns)},
         {socket_options, DefSocketOptions},
         {worker_batch_size, list_to_integer(DefBatchSize)},
         {worker_processes, list_to_integer(DefWorkers)}
-    ]).
+    ])).
 
 
 convert_options([])->
@@ -261,6 +264,8 @@ convert_options([{<<"http_connections">>, V} | R]) ->
     [{http_connections, couch_util:to_integer(V)} | convert_options(R)];
 convert_options([{<<"connection_timeout">>, V} | R]) ->
     [{connection_timeout, couch_util:to_integer(V)} | convert_options(R)];
+convert_options([{<<"retries_per_request">>, V} | R]) ->
+    [{retries, couch_util:to_integer(V)} | convert_options(R)];
 convert_options([{<<"socket_options">>, V} | R]) ->
     {ok, SocketOptions} = couch_util:parse_term(V),
     [{socket_options, SocketOptions} | convert_options(R)];


Mime
View raw message