ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [02/10] ambari git commit: AMBARI-21175 - Host Versions Remain on CURRENT During STANDARD (Full) Upgrades (jonathanhurley)
Date Mon, 12 Jun 2017 17:19:19 GMT
AMBARI-21175 - Host Versions Remain on CURRENT During STANDARD (Full) Upgrades (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7847e4b6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7847e4b6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7847e4b6

Branch: refs/heads/trunk
Commit: 7847e4b6cf65b777e516a4806cbf5f45cdd207ee
Parents: 4767389
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Fri Jun 2 08:28:51 2017 -0400
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Fri Jun 2 10:30:06 2017 -0400

----------------------------------------------------------------------
 .../ambari/server/orm/dao/HostVersionDAO.java   |  16 +++
 .../server/orm/entities/HostVersionEntity.java  |   5 +
 .../upgrades/FinalizeUpgradeAction.java         |  57 ++++++++--
 .../upgrades/UpgradeActionTest.java             | 109 +++++++++++++++++--
 4 files changed, 164 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/7847e4b6/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java
b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java
index ad34ec7..7be7691 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java
@@ -130,6 +130,22 @@ public class HostVersionDAO extends CrudDAO<HostVersionEntity, Long>
{
   }
 
   /**
+   * Retrieve all of the host versions for the given cluster name, and state. <br/>
+   * @param clusterName Cluster name
+   * @param state repository version state
+   * @return Return all of the host versions that match the criteria.
+   */
+  @RequiresSession
+  public List<HostVersionEntity> findByClusterAndState(String clusterName, RepositoryVersionState
state) {
+    final TypedQuery<HostVersionEntity> query = entityManagerProvider.get()
+        .createNamedQuery("findByClusterAndState", HostVersionEntity.class);
+    query.setParameter("clusterName", clusterName);
+    query.setParameter("state", state);
+
+    return daoUtils.selectList(query);  
+  }
+  
+  /**
    * Retrieve all of the host versions for the given cluster name, host name, and state.
<br/>
    * @param clusterName Cluster name
    * @param hostName FQDN of host

http://git-wip-us.apache.org/repos/asf/ambari/blob/7847e4b6/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java
b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java
index dbe4fe0..4a030af 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java
@@ -61,6 +61,11 @@ import org.apache.ambari.server.state.RepositoryVersionState;
         "SELECT hostVersion FROM HostVersionEntity hostVersion JOIN hostVersion.hostEntity
host " +
             "WHERE hostVersion.hostEntity.hostName=:hostName"),
 
+    @NamedQuery(
+        name = "findByClusterAndState",
+        query = "SELECT hostVersion FROM HostVersionEntity hostVersion JOIN hostVersion.hostEntity
host JOIN host.clusterEntities clusters "
+            + "WHERE clusters.clusterName=:clusterName AND hostVersion.state=:state"),
+
     @NamedQuery(name = "hostVersionByClusterHostnameAndState", query =
         "SELECT hostVersion FROM HostVersionEntity hostVersion JOIN hostVersion.hostEntity
host JOIN host.clusterEntities clusters " +
             "WHERE clusters.clusterName=:clusterName AND hostVersion.hostEntity.hostName=:hostName
AND hostVersion.state=:state"),

http://git-wip-us.apache.org/repos/asf/ambari/blob/7847e4b6/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java
index 475a8c9..8f206bf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java
@@ -140,8 +140,8 @@ public class FinalizeUpgradeAction extends AbstractUpgradeServerAction
{
         throw new AmbariException(messageBuff.toString());
       }
 
-      // for all hosts participating in this upgrade, update thei repository
-      // versions and upgrade state
+      // find every host version for this upgrade and ensure it has transitioned
+      // to CURRENT if required
       List<HostVersionEntity> hostVersions = hostVersionDAO.findHostVersionByClusterAndRepository(
           cluster.getClusterId(), repositoryVersion);
 
@@ -165,7 +165,8 @@ public class FinalizeUpgradeAction extends AbstractUpgradeServerAction
{
         }
       }
 
-      // throw an exception if there are hosts which are not not fully upgraded
+      // throw an exception if there are hosts which did not transition the
+      // repository to CURRENT
       if (hostsWithoutCorrectVersionState.size() > 0) {
         message = String.format(
             "The following %d host(s) have not been upgraded to version %s. "
@@ -178,10 +179,11 @@ public class FinalizeUpgradeAction extends AbstractUpgradeServerAction
{
       }
 
       outSB.append(
-          String.format("Finalizing the upgrade state of %d host(s).",
+          String.format("Finalizing the upgrade state and repository version for %d host(s).",
               hostVersionsAllowed.size())).append(System.lineSeparator());
 
-      // Reset the upgrade state
+      // at this point, all host versions are correct - do some cleanup like
+      // resetting the upgrade state
       for (HostVersionEntity hostVersion : hostVersionsAllowed) {
         Collection<HostComponentStateEntity> hostComponentStates = hostComponentStateDAO.findByHost(hostVersion.getHostName());
         for (HostComponentStateEntity hostComponentStateEntity: hostComponentStates) {
@@ -190,16 +192,16 @@ public class FinalizeUpgradeAction extends AbstractUpgradeServerAction
{
         }
       }
 
-      // Impacts all hosts that have a version
-      outSB.append(
-          String.format("Finalizing the version for %d host(s).",
-              hostVersionsAllowed.size())).append(System.lineSeparator());
-
-      versionEventPublisher.publish(new StackUpgradeFinishEvent(cluster));
+      // move host versions from CURRENT to INSTALLED if their repos are no
+      // longer used
+      finalizeHostRepositoryVersions(cluster);
 
       // Reset upgrade state
       cluster.setUpgradeEntity(null);
 
+      // the upgrade is done!
+      versionEventPublisher.publish(new StackUpgradeFinishEvent(cluster));
+
       message = String.format("The upgrade to %s has completed.", version);
       outSB.append(message).append(System.lineSeparator());
       return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", outSB.toString(), errSB.toString());
@@ -372,6 +374,39 @@ public class FinalizeUpgradeAction extends AbstractUpgradeServerAction
{
     return errors;
   }
 
+  /**
+   * Moves any {@link HostVersionEntity}s which are
+   * {@link RepositoryVersionState#CURRENT} to either
+   * {@link RepositoryVersionState#INSTALLED} or
+   * {@link RepositoryVersionState#NOT_REQUIRED} if their assocaited
+   * repositories are no longer in use.
+   *
+   * @param cluster
+   * @throws AmbariException
+   */
+  private void finalizeHostRepositoryVersions(Cluster cluster) throws AmbariException {
+    // create a set of all of the repos that the services are on
+    Set<RepositoryVersionEntity> desiredRepoVersions = new HashSet<>();
+    Set<String> serviceNames = cluster.getServices().keySet();
+    for (String serviceName : serviceNames) {
+      Service service = cluster.getService(serviceName);
+      desiredRepoVersions.add(service.getDesiredRepositoryVersion());
+    }
+
+    // if any CURRENT host version is for a repo which is no longer desired by
+    // ANY service, move it to INSTALLED
+    List<HostVersionEntity> currentHostVersions = hostVersionDAO.findByClusterAndState(
+        cluster.getClusterName(), RepositoryVersionState.CURRENT);
+
+    for (HostVersionEntity hostVersion : currentHostVersions) {
+      RepositoryVersionEntity hostRepoVersion = hostVersion.getRepositoryVersion();
+      if (!desiredRepoVersions.contains(hostRepoVersion)) {
+        hostVersion.setState(RepositoryVersionState.INSTALLED);
+        hostVersion = hostVersionDAO.merge(hostVersion);
+      }
+    }
+  }
+
   protected static class InfoTuple implements Comparable<InfoTuple> {
     protected final String serviceName;
     protected final String componentName;

http://git-wip-us.apache.org/repos/asf/ambari/blob/7847e4b6/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
index 35fffda..e3ffae0 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
@@ -100,7 +100,6 @@ public class UpgradeActionTest {
   private static final String HDP_2_1_1_1 = "2.1.1.1-2";
 
   private static final String HDP_2_2_0_1 = "2.2.0.1-3";
-  private static final String HDP_2_2_0_2 = "2.2.0.2-4";
 
   private static final StackId HDP_21_STACK = new StackId("HDP-2.1.1");
   private static final StackId HDP_22_STACK = new StackId("HDP-2.2.0");
@@ -147,7 +146,6 @@ public class UpgradeActionTest {
   private RepositoryVersionEntity repositoryVersion2110;
   private RepositoryVersionEntity repositoryVersion2111;
   private RepositoryVersionEntity repositoryVersion2201;
-  private RepositoryVersionEntity repositoryVersion2202;
 
   @Before
   public void setup() throws Exception {
@@ -167,7 +165,6 @@ public class UpgradeActionTest {
     repositoryVersion2110 = m_helper.getOrCreateRepositoryVersion(HDP_21_STACK, HDP_2_1_1_0);
     repositoryVersion2111 = m_helper.getOrCreateRepositoryVersion(HDP_21_STACK, HDP_2_1_1_1);
     repositoryVersion2201 = m_helper.getOrCreateRepositoryVersion(HDP_22_STACK, HDP_2_2_0_1);
-    repositoryVersion2202 = m_helper.getOrCreateRepositoryVersion(HDP_22_STACK, HDP_2_2_0_2);
   }
 
   @After
@@ -199,11 +196,11 @@ public class UpgradeActionTest {
     hostVersionDAO.create(entity);
   }
 
-  private void createUpgradeCluster(
+  private Cluster createUpgradeCluster(
       RepositoryVersionEntity sourceRepoVersion, String hostName) throws Exception {
 
     clusters.addCluster(clusterName, sourceRepoVersion.getStackId());
-    Cluster c = clusters.getCluster(clusterName);
+    Cluster cluster = clusters.getCluster(clusterName);
 
     // add a host component
     clusters.addHost(hostName);
@@ -222,6 +219,8 @@ public class UpgradeActionTest {
         sourceRepoVersion, RepositoryVersionState.INSTALLED);
 
     hostVersionDAO.create(entity);
+
+    return cluster;
   }
 
   private void createHostVersions(RepositoryVersionEntity targetRepoVersion,
@@ -313,13 +312,13 @@ public class UpgradeActionTest {
     Cluster cluster = clusters.getCluster(clusterName);
 
     // Install ZK and HDFS with some components
-    Service zk = installService(cluster, "ZOOKEEPER");
+    Service zk = installService(cluster, "ZOOKEEPER", repositoryVersion2110);
     addServiceComponent(cluster, zk, "ZOOKEEPER_SERVER");
     addServiceComponent(cluster, zk, "ZOOKEEPER_CLIENT");
     createNewServiceComponentHost(cluster, "ZOOKEEPER", "ZOOKEEPER_SERVER", "h1");
     createNewServiceComponentHost(cluster, "ZOOKEEPER", "ZOOKEEPER_CLIENT", "h1");
 
-    Service hdfs = installService(cluster, "HDFS");
+    Service hdfs = installService(cluster, "HDFS", repositoryVersion2110);
     addServiceComponent(cluster, hdfs, "NAMENODE");
     addServiceComponent(cluster, hdfs, "DATANODE");
     createNewServiceComponentHost(cluster, "HDFS", "NAMENODE", "h1");
@@ -393,7 +392,7 @@ public class UpgradeActionTest {
     for (HostVersionEntity entity : hostVersionDAO.findByClusterAndHost(clusterName, "h1"))
{
       if (StringUtils.equals(entity.getRepositoryVersion().getVersion(), repositoryVersion2110.getVersion()))
{
         assertEquals(RepositoryVersionState.CURRENT, entity.getState());
-      } else if (StringUtils.equals(entity.getRepositoryVersion().getVersion(), repositoryVersion2111.getVersion()))
{
+      } else {
         assertEquals(RepositoryVersionState.INSTALLED, entity.getState());
       }
     }
@@ -497,10 +496,95 @@ public class UpgradeActionTest {
     assertEquals(HostRoleStatus.COMPLETED.name(), report.getStatus());
   }
 
+  /**
+   * Tests that all host versions are correct after upgrade. This test will
+   * ensure that the prior CURRENT versions are moved to INSTALLED while not
+   * touching any others.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testHostVersionsAfterUpgrade() throws Exception {
+    String hostName = "h1";
+    Cluster cluster = createUpgradeCluster(repositoryVersion2110, hostName);
+    createHostVersions(repositoryVersion2111, hostName);
+    createHostVersions(repositoryVersion2201, hostName);
+
+    // Install ZK with some components
+    Service zk = installService(cluster, "ZOOKEEPER", repositoryVersion2110);
+    addServiceComponent(cluster, zk, "ZOOKEEPER_SERVER");
+    addServiceComponent(cluster, zk, "ZOOKEEPER_CLIENT");
+    createNewServiceComponentHost(cluster, "ZOOKEEPER", "ZOOKEEPER_SERVER", hostName);
+    createNewServiceComponentHost(cluster, "ZOOKEEPER", "ZOOKEEPER_CLIENT", hostName);
+
+    List<HostVersionEntity> hostVersions = hostVersionDAO.findAll();
+    assertEquals(3, hostVersions.size());
+
+    // repo 2110 - CURRENT (upgrading from)
+    // repo 2111 - CURRENT (all hosts reported in during upgrade)
+    // repo 2201 - NOT_REQUIRED (different stack)
+    for (HostVersionEntity hostVersion : hostVersions) {
+      RepositoryVersionEntity hostRepoVersion = hostVersion.getRepositoryVersion();
+      if (repositoryVersion2110.equals(hostRepoVersion)) {
+        hostVersion.setState(RepositoryVersionState.CURRENT);
+      } else if (repositoryVersion2111.equals(hostRepoVersion)) {
+        hostVersion.setState(RepositoryVersionState.CURRENT);
+      } else {
+        hostVersion.setState(RepositoryVersionState.NOT_REQUIRED);
+      }
+
+      hostVersionDAO.merge(hostVersion);
+    }
+
+    // upgrade to 2111
+    createUpgrade(cluster, repositoryVersion2111);
+
+    // push all services to the correct repo version for finalize
+    Map<String, Service> services = cluster.getServices();
+    assertTrue(services.size() > 0);
+    for (Service service : services.values()) {
+      service.setDesiredRepositoryVersion(repositoryVersion2111);
+    }
+
+    // push all components to the correct version
+    List<HostComponentStateEntity> hostComponentStates = hostComponentStateDAO.findByHost(hostName);
+    for (HostComponentStateEntity hostComponentState : hostComponentStates) {
+      hostComponentState.setVersion(repositoryVersion2111.getVersion());
+      hostComponentStateDAO.merge(hostComponentState);
+    }
+
+    Map<String, String> commandParams = new HashMap<>();
+    ExecutionCommand executionCommand = new ExecutionCommand();
+    executionCommand.setCommandParams(commandParams);
+    executionCommand.setClusterName(clusterName);
+
+    HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null);
+    hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand));
+
+    finalizeUpgradeAction.setExecutionCommand(executionCommand);
+    finalizeUpgradeAction.setHostRoleCommand(hostRoleCommand);
+
+    // finalize
+    CommandReport report = finalizeUpgradeAction.execute(null);
+    assertNotNull(report);
+    assertEquals(HostRoleStatus.COMPLETED.name(), report.getStatus());
+
+    for (HostVersionEntity hostVersion : hostVersions) {
+      RepositoryVersionEntity hostRepoVersion = hostVersion.getRepositoryVersion();
+      if (repositoryVersion2110.equals(hostRepoVersion)) {
+        assertEquals(RepositoryVersionState.INSTALLED, hostVersion.getState());
+      } else if (repositoryVersion2111.equals(hostRepoVersion)) {
+        assertEquals(RepositoryVersionState.CURRENT, hostVersion.getState());
+      } else {
+        assertEquals(RepositoryVersionState.NOT_REQUIRED, hostVersion.getState());
+      }
+    }
+  }
+
   private ServiceComponentHost createNewServiceComponentHost(Cluster cluster, String svc,
                                                              String svcComponent, String
hostName) throws AmbariException {
     Assert.assertNotNull(cluster.getConfigGroups());
-    Service s = installService(cluster, svc);
+    Service s = installService(cluster, svc, sourceRepositoryVersion);
     ServiceComponent sc = addServiceComponent(cluster, s, svcComponent);
 
     ServiceComponentHost sch = serviceComponentHostFactory.createNew(sc, hostName);
@@ -511,13 +595,14 @@ public class UpgradeActionTest {
     return sch;
   }
 
-  private Service installService(Cluster cluster, String serviceName) throws AmbariException
{
+  private Service installService(Cluster cluster, String serviceName,
+      RepositoryVersionEntity repositoryVersionEntity) throws AmbariException {
     Service service = null;
 
     try {
       service = cluster.getService(serviceName);
     } catch (ServiceNotFoundException e) {
-      service = serviceFactory.createNew(cluster, serviceName, sourceRepositoryVersion);
+      service = serviceFactory.createNew(cluster, serviceName, repositoryVersionEntity);
       cluster.addService(service);
     }
 
@@ -525,7 +610,7 @@ public class UpgradeActionTest {
   }
 
   private ServiceComponent addServiceComponent(Cluster cluster, Service service,
-                                               String componentName) throws AmbariException
{
+      String componentName) throws AmbariException {
     ServiceComponent serviceComponent = null;
     try {
       serviceComponent = service.getServiceComponent(componentName);


Mime
View raw message