Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: common-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1379203 - in /hadoop/common/branches/branch-1:
 CHANGES.txt
 src/core/org/apache/hadoop/io/retry/RetryUtils.java
Date: Thu, 30 Aug 2012 22:46:46 -0000
To: common-commits@hadoop.apache.org
From: acmurthy@apache.org
Message-Id: <20120830224646.A1AE323888FD@eris.apache.org>

Author: acmurthy
Date: Thu Aug 30 22:46:46 2012
New Revision: 1379203

URL: http://svn.apache.org/viewvc?rev=1379203&view=rev
Log:
HADOOP-8748. Refactor DFSClient retry utility methods to a new class in org.apache.hadoop.io.retry.  Contributed by Arun C Murthy.

Added:
    hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/retry/RetryUtils.java
Modified:
    hadoop/common/branches/branch-1/CHANGES.txt

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1379203&r1=1379202&r2=1379203&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Thu Aug 30 22:46:46 2012
@@ -106,6 +106,9 @@ Release 1.2.0 - unreleased
     MAPREDUCE-4499. Looking for speculative tasks is very expensive in 1.x
     (Koji Noguchi via tgraves)
 
+    HADOOP-8748. Refactor DFSClient retry utility methods to a new class in
+    org.apache.hadoop.io.retry.  Contributed by Arun C Murthy.
+
   OPTIMIZATIONS
 
     HDFS-2533. Backport: Remove needless synchronization on some FSDataSet

Added: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/retry/RetryUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/retry/RetryUtils.java?rev=1379203&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/retry/RetryUtils.java (added)
+++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/retry/RetryUtils.java Thu Aug 30 22:46:46 2012
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.retry;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RemoteException;
+
+public class RetryUtils {
+  public static final Log LOG = LogFactory.getLog(RetryUtils.class);
+  
+  /**
+   * Return the default retry policy set in conf.
+   * 
+   * If the value retryPolicyEnabledKey is set to false in conf,
+   * use TRY_ONCE_THEN_FAIL.
+   * 
+   * Otherwise, get the MultipleLinearRandomRetry policy specified in the conf
+   * and then
+   * (1) use multipleLinearRandomRetry for
+   *     - remoteExceptionToRetry, or
+   *     - IOException other than RemoteException; and
+   * (2) use TRY_ONCE_THEN_FAIL for
+   *     - RemoteException other than remoteExceptionToRetry, or
+   *     - non-IOException.
+   *
+   * @param conf
+   * @param retryPolicyEnabledKey     conf property key for enabling retry
+   * @param defaultRetryPolicyEnabled default retryPolicyEnabledKey conf value 
+   * @param retryPolicySpecKey        conf property key for retry policy spec
+   * @param defaultRetryPolicySpec    default retryPolicySpecKey conf value
+   * @param remoteExceptionToRetry    The particular RemoteException to retry
+   * @return the default retry policy.
+   */
+  public static RetryPolicy getDefaultRetryPolicy(
+      Configuration conf,
+      String retryPolicyEnabledKey,
+      boolean defaultRetryPolicyEnabled,
+      String retryPolicySpecKey,
+      String defaultRetryPolicySpec,
+      final Class<? extends Exception> remoteExceptionToRetry
+      ) {
+    
+    final RetryPolicy multipleLinearRandomRetry = 
+        getMultipleLinearRandomRetry(
+            conf, 
+            retryPolicyEnabledKey, defaultRetryPolicyEnabled, 
+            retryPolicySpecKey, defaultRetryPolicySpec
+            );
+    
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("multipleLinearRandomRetry = " + multipleLinearRandomRetry);
+    }
+
+    if (multipleLinearRandomRetry == null) {
+      //no retry
+      return RetryPolicies.TRY_ONCE_THEN_FAIL;
+    } else {
+      return new RetryPolicy() {
+        @Override
+        public boolean shouldRetry(Exception e, int retries) throws Exception {
+          //see (1) and (2) in the javadoc of this method.
+          final RetryPolicy p;
+          if (e instanceof RemoteException) {
+            final RemoteException re = (RemoteException)e;
+            p = remoteExceptionToRetry.getName().equals(re.getClassName())?
+                multipleLinearRandomRetry: RetryPolicies.TRY_ONCE_THEN_FAIL;
+          } else if (e instanceof IOException) {
+            p = multipleLinearRandomRetry;
+          } else { //non-IOException
+            p = RetryPolicies.TRY_ONCE_THEN_FAIL;
+          }
+
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("RETRY " + retries + ") policy="
+                + p.getClass().getSimpleName() + ", exception=" + e);
+          }
+          return p.shouldRetry(e, retries);
+        }
+        
+        @Override
+        public String toString() {
+          return "RetryPolicy[" + multipleLinearRandomRetry + ", "
+              + RetryPolicies.TRY_ONCE_THEN_FAIL.getClass().getSimpleName()
+              + "]";
+        }
+      };
+    }
+  }
+
+  /**
+   * Return the MultipleLinearRandomRetry policy specified in the conf,
+   * or null if the feature is disabled.
+   * If the policy is specified in the conf but the policy cannot be parsed,
+   * the default policy is returned.
+   * 
+   * Retry policy spec:
+   *   N pairs of sleep-time and number-of-retries "s1,n1,s2,n2,..."
+   * 
+   * @param conf
+   * @param retryPolicyEnabledKey     conf property key for enabling retry
+   * @param defaultRetryPolicyEnabled default retryPolicyEnabledKey conf value 
+   * @param retryPolicySpecKey        conf property key for retry policy spec
+   * @param defaultRetryPolicySpec    default retryPolicySpecKey conf value
+   * @return the MultipleLinearRandomRetry policy specified in the conf,
+   *         or null if the feature is disabled.
+   */
+  public static RetryPolicy getMultipleLinearRandomRetry(
+      Configuration conf,
+      String retryPolicyEnabledKey,
+      boolean defaultRetryPolicyEnabled,
+      String retryPolicySpecKey,
+      String defaultRetryPolicySpec
+      ) {
+    final boolean enabled = 
+        conf.getBoolean(retryPolicyEnabledKey, defaultRetryPolicyEnabled);
+    if (!enabled) {
+      return null;
+    }
+
+    final String policy = conf.get(retryPolicySpecKey, defaultRetryPolicySpec);
+
+    final RetryPolicy r = 
+        RetryPolicies.MultipleLinearRandomRetry.parseCommaSeparatedString(
+            policy);
+    return (r != null) ? 
+        r : 
+        RetryPolicies.MultipleLinearRandomRetry.parseCommaSeparatedString(
+        defaultRetryPolicySpec);
+  }
+}