ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alejan...@apache.org
Subject ambari git commit: AMBARI-21481. Upgrading IOP cluster with Spark2 to Ambari 2.5.2 fails on start because config mapping spark2-javaopts-properties is never selected (alejandro)
Date Mon, 17 Jul 2017 21:28:25 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 13bcea0b0 -> 0ed09cd53


AMBARI-21481. Upgrading IOP cluster with Spark2 to Ambari 2.5.2 fails on start because config
mapping spark2-javaopts-properties is never selected (alejandro)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0ed09cd5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0ed09cd5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0ed09cd5

Branch: refs/heads/branch-2.5
Commit: 0ed09cd5342cfc4cac0d6061a7b7b9a3cef127c1
Parents: 13bcea0
Author: Alejandro Fernandez <afernandez@hortonworks.com>
Authored: Fri Jul 14 16:15:07 2017 -0700
Committer: Alejandro Fernandez <afernandez@hortonworks.com>
Committed: Mon Jul 17 14:29:36 2017 -0700

----------------------------------------------------------------------
 .../server/upgrade/UpgradeCatalog252.java       | 99 ++++++++++++++++++++
 .../configuration/spark-javaopts-properties.xml |  3 +
 .../spark2-javaopts-properties.xml              |  5 +-
 .../4.2.5/services/SPARK2/metainfo.xml          |  2 +-
 .../configuration/spark-javaopts-properties.xml |  3 +
 5 files changed, 110 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed09cd5/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
index 3c8686c..ea1b034 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
@@ -18,12 +18,19 @@
 package org.apache.ambari.server.upgrade;
 
 import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo;
+import org.apache.ambari.server.orm.dao.ClusterDAO;
+import org.apache.ambari.server.orm.entities.ClusterConfigMappingEntity;
+import org.apache.ambari.server.orm.entities.ClusterEntity;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
@@ -34,6 +41,8 @@ import org.apache.commons.lang.StringUtils;
 import com.google.common.collect.Sets;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * The {@link org.apache.ambari.server.upgrade.UpgradeCatalog252} upgrades Ambari from 2.5.1
to 2.5.2.
@@ -54,6 +63,13 @@ public class UpgradeCatalog252 extends AbstractUpgradeCatalog {
 
   private static final String CLUSTER_ENV = "cluster-env";
 
+  private static final List<String> configTypesToEnsureSelected = Arrays.asList("spark2-javaopts-properties");
+  
+  /**
+   * Logger.
+   */
+  private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog252.class);
+
   /**
    * Constructor.
    *
@@ -102,6 +118,7 @@ public class UpgradeCatalog252 extends AbstractUpgradeCatalog {
   @Override
   protected void executeDMLUpdates() throws AmbariException, SQLException {
     resetStackToolsAndFeatures();
+    ensureConfigTypesHaveAtLeastOneVersionSelected();
   }
 
   /**
@@ -197,4 +214,86 @@ public class UpgradeCatalog252 extends AbstractUpgradeCatalog {
       updateConfigurationPropertiesForCluster(cluster, CLUSTER_ENV, newStackProperties, true,
false);
     }
   }
+
+  /**
+   * When doing a cross-stack upgrade, we found that one config type (spark2-javaopts-properties)
+   * did not have any mappings that were selected, so it caused Ambari Server start to fail
on the DB Consistency Checker.
+   * To fix this, iterate over all config types and ensure that at least one is selected.
+   * If none are selected, then pick the one with the greatest time stamp; this should be
safe since we are only adding
+   * more data to use as opposed to removing.
+   */
+  private void ensureConfigTypesHaveAtLeastOneVersionSelected() {
+    ClusterDAO clusterDAO = injector.getInstance(ClusterDAO.class);
+    List<ClusterEntity> clusters = clusterDAO.findAll();
+
+    if (null == clusters) {
+      return;
+    }
+
+    for (ClusterEntity clusterEntity : clusters) {
+      LOG.info("Ensuring all config types have at least one selected config for cluster {}",
clusterEntity.getClusterName());
+
+      boolean atLeastOneChanged = false;
+      Collection<ClusterConfigMappingEntity> configMappingEntities = clusterEntity.getConfigMappingEntities();
+
+      if (configMappingEntities != null) {
+        Set<String> configTypesNotSelected = new HashSet<>();
+        Set<String> configTypesWithAtLeastOneSelected = new HashSet<>();
+
+        for (ClusterConfigMappingEntity clusterConfigMappingEntity : configMappingEntities)
{
+          String typeName = clusterConfigMappingEntity.getType();
+
+          if (clusterConfigMappingEntity.isSelected() == 1) {
+            configTypesWithAtLeastOneSelected.add(typeName);
+          } else {
+            configTypesNotSelected.add(typeName);
+          }
+        }
+
+        // Due to the ordering, eliminate any configs with at least one selected.
+        configTypesNotSelected.removeAll(configTypesWithAtLeastOneSelected);
+        if (!configTypesNotSelected.isEmpty()) {
+          LOG.info("The following config types have config mappings which don't have at least
one as selected. {}", StringUtils.join(configTypesNotSelected, ", "));
+
+          LOG.info("Filtering only config types these config types: {}", StringUtils.join(configTypesToEnsureSelected,
", "));
+          // Get the intersection with a subset of configs that are allowed to be selected
during the migration.
+          configTypesNotSelected.retainAll(configTypesToEnsureSelected);
+        }
+
+        if (!configTypesNotSelected.isEmpty()) {
+          LOG.info("The following config types have config mappings which don't have at least
one as selected. {}", StringUtils.join(configTypesNotSelected, ", "));
+
+          for (String typeName : configTypesNotSelected) {
+            ClusterConfigMappingEntity clusterConfigMappingWithGreatestTimeStamp = null;
+
+            for (ClusterConfigMappingEntity clusterConfigMappingEntity : configMappingEntities)
{
+              if (typeName.equals(clusterConfigMappingEntity.getType())) {
+
+                if (null == clusterConfigMappingWithGreatestTimeStamp) {
+                  clusterConfigMappingWithGreatestTimeStamp = clusterConfigMappingEntity;
+                } else {
+                  if (clusterConfigMappingEntity.getCreateTimestamp() >= clusterConfigMappingWithGreatestTimeStamp.getCreateTimestamp())
{
+                    clusterConfigMappingWithGreatestTimeStamp = clusterConfigMappingEntity;
+                  }
+                }
+              }
+            }
+
+            if (null != clusterConfigMappingWithGreatestTimeStamp) {
+              LOG.info("Saving. Config type {} has a mapping with tag {} and greatest timestamp
{} that is not selected, so will mark it selected.",
+                  typeName, clusterConfigMappingWithGreatestTimeStamp.getTag(), clusterConfigMappingWithGreatestTimeStamp.getCreateTimestamp());
+              atLeastOneChanged = true;
+              clusterConfigMappingWithGreatestTimeStamp.setSelected(1);
+            }
+          }
+        } else {
+          LOG.info("All config types have at least one mapping that is selected. Nothing
to do.");
+        }
+      }
+
+      if (atLeastOneChanged) {
+        clusterDAO.mergeConfigMappings(configMappingEntities);
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed09cd5/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/SPARK/configuration/spark-javaopts-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/SPARK/configuration/spark-javaopts-properties.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/SPARK/configuration/spark-javaopts-properties.xml
index a197e34..c8fe152 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/SPARK/configuration/spark-javaopts-properties.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/SPARK/configuration/spark-javaopts-properties.xml
@@ -23,6 +23,9 @@
     <name>content</name>
     <description>Spark-javaopts-properties</description>
     <value> </value>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
     <on-ambari-upgrade add="true"/>
   </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed09cd5/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/configuration/spark2-javaopts-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/configuration/spark2-javaopts-properties.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/configuration/spark2-javaopts-properties.xml
index f8d50fc..e8c3f5a 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/configuration/spark2-javaopts-properties.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/configuration/spark2-javaopts-properties.xml
@@ -23,7 +23,10 @@
     <name>content</name>
     <description>Spark2-javaopts-properties</description>
     <value> </value>
-    <on-ambari-upgrade add="true"/>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
   </property>
 </configuration>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed09cd5/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/metainfo.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/metainfo.xml
index bf75f47..1692890 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/metainfo.xml
@@ -79,7 +79,7 @@
        
       <configuration-dependencies>
         <config-type>spark2-defaults</config-type>
-	<config-type>spark2-javaopts-properties</config-type>
+        <config-type>spark2-javaopts-properties</config-type>
         <config-type>spark2-thrift-sparkconf</config-type>
       </configuration-dependencies>
       

http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed09cd5/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SPARK/configuration/spark-javaopts-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SPARK/configuration/spark-javaopts-properties.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SPARK/configuration/spark-javaopts-properties.xml
index 77a7282..f7c8c7e 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SPARK/configuration/spark-javaopts-properties.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SPARK/configuration/spark-javaopts-properties.xml
@@ -23,6 +23,9 @@
     <name>content</name>
     <description>Spark-javaopts-properties</description>
     <value> </value>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
     <on-ambari-upgrade add="true"/>
   </property>
 </configuration>


Mime
View raw message