ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mpapirkovs...@apache.org
Subject [2/2] ambari git commit: AMBARI-13744. Implement ability to retry operations after database exception for blueprint deploy. (mpapirkovskyy)
Date Thu, 05 Nov 2015 17:36:08 GMT
AMBARI-13744. Implement ability to retry operations after database exception for blueprint
deploy. (mpapirkovskyy)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b3a562c6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b3a562c6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b3a562c6

Branch: refs/heads/trunk
Commit: b3a562c6356a63d597bc1105886e7dcfc2d03607
Parents: 4db9a1e
Author: Myroslav Papirkovskyi <mpapyrkovskyy@hortonworks.com>
Authored: Thu Nov 5 11:06:40 2015 +0200
Committer: Myroslav Papirkovskyi <mpapyrkovskyy@hortonworks.com>
Committed: Thu Nov 5 19:34:57 2015 +0200

----------------------------------------------------------------------
 .../server/configuration/Configuration.java     | 40 ++++---------
 .../ambari/server/controller/AmbariServer.java  |  2 +-
 .../internal/AbstractResourceProvider.java      |  2 +-
 .../ambari/server/topology/AmbariContext.java   | 53 +++++++++++++----
 .../ambari/server/topology/TopologyManager.java | 41 ++++++++++---
 .../apache/ambari/server/utils/RetryHelper.java | 51 ++++++++++++----
 .../internal/ClusterResourceProviderTest.java   | 61 ++++++++++++++++++++
 7 files changed, 191 insertions(+), 59 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/b3a562c6/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index b4d5de8..e5e2c90 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -202,10 +202,8 @@ public class Configuration {
   public static final String SERVER_JDBC_CONNECTION_POOL_ACQUISITION_RETRY_ATTEMPTS = "server.jdbc.connection-pool.acquisition-retry-attempts";
   public static final String SERVER_JDBC_CONNECTION_POOL_ACQUISITION_RETRY_DELAY = "server.jdbc.connection-pool.acquisition-retry-delay";
 
-  public static final String API_OPERATIONS_RETRY_ATTEMPTS_KEY = "api.operations.retry-attempts";
-  public static final String BLUEPRINTS_OPERATIONS_RETRY_ATTEMPTS_KEY = "blueprints.operations.retry-attempts";
-  public static final String API_OPERATIONS_RETRY_ATTEMPTS_DEFAULT = "0";
-  public static final String BLUEPRINTS_OPERATIONS_RETRY_ATTEMPTS_DEFAULT = "0";
+  public static final String OPERATIONS_RETRY_ATTEMPTS_KEY = "server.operations.retry-attempts";
+  public static final String OPERATIONS_RETRY_ATTEMPTS_DEFAULT = "0";
   public static final int RETRY_ATTEMPTS_LIMIT = 10;
 
   public static final String SERVER_JDBC_RCA_USER_NAME_KEY = "server.jdbc.rca.user.name";
@@ -2359,38 +2357,22 @@ public class Configuration {
   }
 
   /**
-   * @return number of retry attempts for API update requests
+   * @return number of retry attempts for api and blueprint operations
    */
-  public int getApiOperationsRetryAttempts() {
-    String property = properties.getProperty(API_OPERATIONS_RETRY_ATTEMPTS_KEY, API_OPERATIONS_RETRY_ATTEMPTS_DEFAULT);
+  public int getOperationsRetryAttempts() {
+    String property = properties.getProperty(OPERATIONS_RETRY_ATTEMPTS_KEY, OPERATIONS_RETRY_ATTEMPTS_DEFAULT);
     Integer attempts = Integer.valueOf(property);
     if (attempts < 0) {
-      LOG.warn("Invalid API retry attempts number ({}), should be [0,{}]. Value reset to
default {}",
-          attempts, RETRY_ATTEMPTS_LIMIT, API_OPERATIONS_RETRY_ATTEMPTS_DEFAULT);
-      attempts = Integer.valueOf(API_OPERATIONS_RETRY_ATTEMPTS_DEFAULT);
+      LOG.warn("Invalid operations retry attempts number ({}), should be [0,{}]. Value reset
to default {}",
+          attempts, RETRY_ATTEMPTS_LIMIT, OPERATIONS_RETRY_ATTEMPTS_DEFAULT);
+      attempts = Integer.valueOf(OPERATIONS_RETRY_ATTEMPTS_DEFAULT);
     } else if (attempts > RETRY_ATTEMPTS_LIMIT) {
-      LOG.warn("Invalid API retry attempts number ({}), should be [0,{}]. Value set to {}",
+      LOG.warn("Invalid operations retry attempts number ({}), should be [0,{}]. Value set
to {}",
           attempts, RETRY_ATTEMPTS_LIMIT, RETRY_ATTEMPTS_LIMIT);
       attempts = RETRY_ATTEMPTS_LIMIT;
     }
-    return attempts;
-  }
-
-  /**
-   * @return number of retry attempts for blueprints operations
-   */
-  public int getBlueprintsOperationsRetryAttempts() {
-    String property = properties.getProperty(BLUEPRINTS_OPERATIONS_RETRY_ATTEMPTS_KEY,
-            BLUEPRINTS_OPERATIONS_RETRY_ATTEMPTS_DEFAULT);
-    Integer attempts = Integer.valueOf(property);
-    if (attempts < 0) {
-      LOG.warn("Invalid blueprint operations retry attempts number ({}), should be [0,{}].
Value reset to default {}",
-          attempts, RETRY_ATTEMPTS_LIMIT, BLUEPRINTS_OPERATIONS_RETRY_ATTEMPTS_DEFAULT);
-      attempts = Integer.valueOf(BLUEPRINTS_OPERATIONS_RETRY_ATTEMPTS_DEFAULT);
-    } else if (attempts > RETRY_ATTEMPTS_LIMIT) {
-      LOG.warn("Invalid blueprint operations retry attempts number ({}), should be [0,{}].
Value set to {}",
-          attempts, RETRY_ATTEMPTS_LIMIT, RETRY_ATTEMPTS_LIMIT);
-      attempts = RETRY_ATTEMPTS_LIMIT;
+    if (attempts > 0) {
+      LOG.info("Operations retry enabled. Number of retry attempts: {}", attempts);
     }
     return attempts;
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3a562c6/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index ea178b5..15cfb90 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -710,7 +710,7 @@ public class AmbariServer {
     TopologyManager.init(injector.getInstance(StackAdvisorBlueprintProcessor.class));
     StackAdvisorBlueprintProcessor.init(injector.getInstance(StackAdvisorHelper.class));
 
-    RetryHelper.init(configs.getApiOperationsRetryAttempts(), configs.getBlueprintsOperationsRetryAttempts());
+    RetryHelper.init(configs.getOperationsRetryAttempts());
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3a562c6/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractResourceProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractResourceProvider.java
index 2bf8fe3..3464c19 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractResourceProvider.java
@@ -443,7 +443,7 @@ public abstract class AbstractResourceProvider extends BaseProvider implements
R
   //invoke command with retry support in case of database fail
   private <T> T invokeWithRetry(Command<T> command) throws AmbariException {
     RetryHelper.clearAffectedClusters();
-    int retryAttempts = RetryHelper.getApiOperationsRetryAttempts();
+    int retryAttempts = RetryHelper.getOperationsRetryAttempts();
     do {
 
       try {

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3a562c6/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index 5e93aeb..4b0a1d1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -56,6 +56,7 @@ import org.apache.ambari.server.state.Host;
 import org.apache.ambari.server.state.SecurityType;
 import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.configgroup.ConfigGroup;
+import org.apache.ambari.server.utils.RetryHelper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -66,6 +67,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.Callable;
 import java.util.concurrent.atomic.AtomicLong;
 
 /**
@@ -144,9 +146,16 @@ public class AmbariContext {
 
   public void createAmbariClusterResource(String clusterName, String stackName, String stackVersion,
SecurityType securityType) {
     String stackInfo = String.format("%s-%s", stackName, stackVersion);
-    ClusterRequest clusterRequest = new ClusterRequest(null, clusterName, null, securityType,
stackInfo, null);
+    final ClusterRequest clusterRequest = new ClusterRequest(null, clusterName, null, securityType,
stackInfo, null);
     try {
-      getController().createCluster(clusterRequest);
+      RetryHelper.executeWithRetry(new Callable<Object>() {
+        @Override
+        public Object call() throws Exception {
+          getController().createCluster(clusterRequest);
+          return null;
+        }
+      });
+
     } catch (AmbariException e) {
       e.printStackTrace();
       throw new RuntimeException("Failed to create Cluster resource: " + e, e);
@@ -229,7 +238,7 @@ public class AmbariContext {
           hostName, e.toString()), e);
     }
 
-    Set<ServiceComponentHostRequest> requests = new HashSet<ServiceComponentHostRequest>();
+    final Set<ServiceComponentHostRequest> requests = new HashSet<ServiceComponentHostRequest>();
     for (Map.Entry<String, Collection<String>> entry : components.entrySet())
{
       String service = entry.getKey();
       for (String component : entry.getValue()) {
@@ -240,7 +249,13 @@ public class AmbariContext {
       }
     }
     try {
-      getController().createHostComponents(requests);
+      RetryHelper.executeWithRetry(new Callable<Object>() {
+        @Override
+        public Object call() throws Exception {
+          getController().createHostComponents(requests);
+          return null;
+        }
+      });
     } catch (AmbariException e) {
       e.printStackTrace();
       throw new RuntimeException(String.format("Unable to create host component resource
for host '%s': %s",
@@ -286,9 +301,15 @@ public class AmbariContext {
     hostRoleCommandFactory = factory;
   }
 
-  public void registerHostWithConfigGroup(String hostName, ClusterTopology topology, String
groupName) {
+  public void registerHostWithConfigGroup(final String hostName, final ClusterTopology topology,
final String groupName) {
     try {
-      if (!addHostToExistingConfigGroups(hostName, topology, groupName)) {
+      boolean hostAdded = RetryHelper.executeWithRetry(new Callable<Boolean>() {
+        @Override
+        public Boolean call() throws Exception {
+          return addHostToExistingConfigGroups(hostName, topology, groupName);
+        }
+      });
+      if (!hostAdded) {
         createConfigGroupsAndRegisterHost(topology, groupName);
       }
     } catch (Exception e) {
@@ -325,10 +346,16 @@ public class AmbariContext {
    */
   public void persistInstallStateForUI(String clusterName, String stackName, String stackVersion)
{
     String stackInfo = String.format("%s-%s", stackName, stackVersion);
-    ClusterRequest clusterRequest = new ClusterRequest(null, clusterName, "INSTALLED", null,
stackInfo, null);
+    final ClusterRequest clusterRequest = new ClusterRequest(null, clusterName, "INSTALLED",
null, stackInfo, null);
 
     try {
-      getController().updateClusters(Collections.singleton(clusterRequest), null);
+      RetryHelper.executeWithRetry(new Callable<Object>() {
+        @Override
+        public Object call() throws Exception {
+          getController().updateClusters(Collections.singleton(clusterRequest), null);
+          return null;
+        }
+      });
     } catch (AmbariException e) {
       LOG.error("Unable to set install state for UI", e);
     }
@@ -339,9 +366,15 @@ public class AmbariContext {
     return AbstractResourceProvider.getConfigurationRequests("Clusters", clusterProperties);
   }
 
-  public void setConfigurationOnCluster(ClusterRequest clusterRequest) {
+  public void setConfigurationOnCluster(final ClusterRequest clusterRequest) {
     try {
-      getController().updateClusters(Collections.singleton(clusterRequest), null);
+      RetryHelper.executeWithRetry(new Callable<Object>() {
+        @Override
+        public Object call() throws Exception {
+          getController().updateClusters(Collections.singleton(clusterRequest), null);
+          return null;
+        }
+      });
     } catch (AmbariException e) {
       e.printStackTrace();
       throw new RuntimeException("Failed to set configurations on cluster: " + e, e);

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3a562c6/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
index 480c2ba..7f571ce 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
@@ -42,6 +42,7 @@ import org.apache.ambari.server.orm.entities.StageEntity;
 import org.apache.ambari.server.security.encryption.CredentialStoreService;
 import org.apache.ambari.server.state.SecurityType;
 import org.apache.ambari.server.state.host.HostImpl;
+import org.apache.ambari.server.utils.RetryHelper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,6 +56,7 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -121,7 +123,7 @@ public class TopologyManager {
     }
   }
 
-  public RequestStatusResponse provisionCluster(ProvisionClusterRequest request) throws InvalidTopologyException,
AmbariException {
+  public RequestStatusResponse provisionCluster(final ProvisionClusterRequest request) throws
InvalidTopologyException, AmbariException {
     ensureInitialized();
     ClusterTopology topology = new ClusterTopologyImpl(ambariContext, request);
     final String clusterName = request.getClusterName();
@@ -153,7 +155,12 @@ public class TopologyManager {
     // set recommendation strategy
     topology.setConfigRecommendationStrategy(request.getConfigRecommendationStrategy());
     // persist request after it has successfully validated
-    PersistedTopologyRequest persistedRequest = persistedState.persistTopologyRequest(request);
+    PersistedTopologyRequest persistedRequest = RetryHelper.executeWithRetry(new Callable<PersistedTopologyRequest>()
{
+      @Override
+      public PersistedTopologyRequest call() throws Exception {
+        return persistedState.persistTopologyRequest(request);
+      }
+    });
 
     clusterTopologyMap.put(clusterId, topology);
 
@@ -522,13 +529,19 @@ public class TopologyManager {
     return logicalRequest;
   }
 
-  private LogicalRequest createLogicalRequest(PersistedTopologyRequest request, ClusterTopology
topology, Long requestId)
+  private LogicalRequest createLogicalRequest(final PersistedTopologyRequest request, ClusterTopology
topology, Long requestId)
       throws AmbariException {
 
-    LogicalRequest logicalRequest = logicalRequestFactory.createRequest(
+    final LogicalRequest logicalRequest = logicalRequestFactory.createRequest(
         requestId, request.getRequest(), topology);
 
-    persistedState.persistLogicalRequest(logicalRequest, request.getId());
+    RetryHelper.executeWithRetry(new Callable<Object>() {
+      @Override
+      public Object call() throws Exception {
+        persistedState.persistLogicalRequest(logicalRequest, request.getId());
+        return null;
+      }
+    });
 
     allRequests.put(logicalRequest.getRequestId(), logicalRequest);
     LOG.info("TopologyManager.createLogicalRequest: created LogicalRequest with ID = {} and
completed persistence of this request.",
@@ -541,8 +554,8 @@ public class TopologyManager {
     return logicalRequest;
   }
 
-  private void processAcceptedHostOffer(ClusterTopology topology, HostOfferResponse response,
HostImpl host) {
-    String hostName = host.getHostName();
+  private void processAcceptedHostOffer(ClusterTopology topology, final HostOfferResponse
response, HostImpl host) {
+    final String hostName = host.getHostName();
     try {
       topology.addHostToTopology(response.getHostGroupName(), hostName);
     } catch (InvalidTopologyException e) {
@@ -554,7 +567,19 @@ public class TopologyManager {
     }
 
     // persist the host request -> hostName association
-    persistedState.registerHostName(response.getHostRequestId(), hostName);
+    try {
+      RetryHelper.executeWithRetry(new Callable<Object>() {
+        @Override
+        public Object call() throws Exception {
+          persistedState.registerHostName(response.getHostRequestId(), hostName);
+          return null;
+        }
+      });
+    } catch (AmbariException e) {
+      LOG.error("Exception ocurred while registering host name", e);
+      throw new RuntimeException(e);
+    }
+
 
     LOG.info("TopologyManager.processAcceptedHostOffer: about to execute tasks for host =
{}",
         hostName);

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3a562c6/ambari-server/src/main/java/org/apache/ambari/server/utils/RetryHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/RetryHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/RetryHelper.java
index 9d2fe9e..877e84d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/RetryHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/RetryHelper.java
@@ -17,19 +17,24 @@
  */
 package org.apache.ambari.server.utils;
 
-import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.state.Cluster;
 import org.eclipse.persistence.exceptions.DatabaseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.concurrent.Callable;
 
 /**
  * Provides utility methods to support operations retry
  * TODO injection as Guice singleon, static for now to avoid major modifications
  */
 public class RetryHelper {
+  protected final static Logger LOG = LoggerFactory.getLogger(RetryHelper.class);
+
   private static ThreadLocal<Set<Cluster>> affectedClusters = new ThreadLocal<Set<Cluster>>(){
     @Override
     protected Set<Cluster> initialValue() {
@@ -37,16 +42,14 @@ public class RetryHelper {
     }
   };
 
-  private static int apiRetryAttempts = 0;
-  private static int blueprintsRetryAttempts = 0;
+  private static int operationsRetryAttempts = 0;
 
-  public static void init(int apiOperationsRetryAttempts, int blueprintOperationsRetryAttempts)
{
-    apiRetryAttempts = apiOperationsRetryAttempts;
-    blueprintsRetryAttempts = blueprintOperationsRetryAttempts;
+  public static void init(int operationsRetryAttempts) {
+    RetryHelper.operationsRetryAttempts = operationsRetryAttempts;
   }
 
   public static void addAffectedCluster(Cluster cluster) {
-    if (apiRetryAttempts > 0 || blueprintsRetryAttempts > 0) {
+    if (operationsRetryAttempts > 0 ) {
       affectedClusters.get().add(cluster);
     }
   }
@@ -56,13 +59,13 @@ public class RetryHelper {
   }
 
   public static void clearAffectedClusters() {
-    if (apiRetryAttempts > 0 || blueprintsRetryAttempts > 0) {
+    if (operationsRetryAttempts > 0) {
       affectedClusters.get().clear();
     }
   }
 
-  public static int getApiOperationsRetryAttempts() {
-    return apiRetryAttempts;
+  public static int getOperationsRetryAttempts() {
+    return operationsRetryAttempts;
   }
 
   public static boolean isDatabaseException(Throwable ex) {
@@ -82,4 +85,32 @@ public class RetryHelper {
       cluster.invalidateData();
     }
   }
+
+  public static <T> T executeWithRetry(Callable<T> command) throws AmbariException
{
+    RetryHelper.clearAffectedClusters();
+    int retryAttempts = RetryHelper.getOperationsRetryAttempts();
+    do {
+
+      try {
+        return command.call();
+      } catch (Exception e) {
+        if (RetryHelper.isDatabaseException(e)) {
+
+          RetryHelper.invalidateAffectedClusters();
+
+          if (retryAttempts > 0) {
+            LOG.error("Ignoring database exception to perform operation retry, attempts remaining:
" + retryAttempts, e);
+            retryAttempts--;
+          } else {
+            RetryHelper.clearAffectedClusters();
+            throw new AmbariException(e.getMessage(), e);
+          }
+        } else {
+          RetryHelper.clearAffectedClusters();
+          throw new AmbariException(e.getMessage(), e);
+        }
+      }
+
+    } while (true);
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3a562c6/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
index f885a5b..84de604 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
@@ -40,8 +40,10 @@ import org.apache.ambari.server.topology.SecurityConfiguration;
 import org.apache.ambari.server.topology.SecurityConfigurationFactory;
 import org.apache.ambari.server.topology.TopologyManager;
 import org.apache.ambari.server.topology.TopologyRequestFactory;
+import org.apache.ambari.server.utils.RetryHelper;
 import org.easymock.Capture;
 import org.easymock.EasyMock;
+import org.eclipse.persistence.exceptions.DatabaseException;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -65,6 +67,7 @@ import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.createStrictMock;
 import static org.easymock.EasyMock.eq;
 import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
 import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.reset;
 import static org.easymock.EasyMock.verify;
@@ -280,6 +283,64 @@ public class ClusterResourceProviderTest {
   }
 
   @Test
+  public void testCreateResourcesWithRetry() throws Exception {
+    RetryHelper.init(3);
+    Resource.Type type = Resource.Type.Cluster;
+
+    AmbariManagementController managementController = createMock(AmbariManagementController.class);
+    RequestStatusResponse response = createNiceMock(RequestStatusResponse.class);
+
+    managementController.createCluster(
+        AbstractResourceProviderTest.Matcher.getClusterRequest(null, "Cluster100", "HDP-0.1",
null));
+    expectLastCall().andThrow(new DatabaseException("test"){}).once().andVoid().atLeastOnce();
+
+    // replay
+    replay(managementController, response);
+
+    ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider(
+        type,
+        PropertyHelper.getPropertyIds(type),
+        PropertyHelper.getKeyPropertyIds(type),
+        managementController);
+
+    AbstractResourceProviderTest.TestObserver observer = new AbstractResourceProviderTest.TestObserver();
+
+    ((ObservableResourceProvider)provider).addObserver(observer);
+
+    // add the property map to a set for the request.  add more maps for multiple creates
+    Set<Map<String, Object>> propertySet = new LinkedHashSet<Map<String,
Object>>();
+
+    // Cluster 1: create a map of properties for the request
+    Map<String, Object> properties = new LinkedHashMap<String, Object>();
+
+    // add the cluster name to the properties map
+    properties.put(ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID, "Cluster100");
+
+    // add the version to the properties map
+    properties.put(ClusterResourceProvider.CLUSTER_VERSION_PROPERTY_ID, "HDP-0.1");
+
+    propertySet.add(properties);
+
+    // create the request
+    Request request = PropertyHelper.getCreateRequest(propertySet, null);
+
+    provider.createResources(request);
+
+    ResourceProviderEvent lastEvent = observer.getLastEvent();
+    Assert.assertNotNull(lastEvent);
+    Assert.assertEquals(Resource.Type.Cluster, lastEvent.getResourceType());
+    Assert.assertEquals(ResourceProviderEvent.Type.Create, lastEvent.getType());
+    Assert.assertEquals(request, lastEvent.getRequest());
+    Assert.assertNull(lastEvent.getPredicate());
+
+    // verify
+    verify(managementController, response);
+
+    RetryHelper.init(0);
+
+  }
+
+  @Test
   public void testGetResources() throws Exception{
     Resource.Type type = Resource.Type.Cluster;
 


Mime
View raw message