ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject ambari git commit: AMBARI-14385. Robust Handling of Ambari properites and DB state. (aonishuk)
Date Tue, 29 Dec 2015 11:33:42 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk f697f4aff -> 65752c9bd


AMBARI-14385. Robust Handling of Ambari properites and DB state. (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/65752c9b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/65752c9b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/65752c9b

Branch: refs/heads/trunk
Commit: 65752c9bd95af7f64002e1b1da6b734b3bd99d8f
Parents: f697f4a
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Tue Dec 29 13:33:42 2015 +0200
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Tue Dec 29 13:33:42 2015 +0200

----------------------------------------------------------------------
 .../ambari/server/controller/AmbariServer.java  |   7 +-
 .../controller/utilities/DatabaseChecker.java   | 115 ++++++++++++++++++-
 ambari-server/src/main/python/ambari-server.py  |   4 +
 .../python/ambari_server/serverConfiguration.py |  14 +++
 .../src/main/python/ambari_server_main.py       |  13 ++-
 5 files changed, 148 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index c6f90c0..cc69134 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -260,8 +260,11 @@ public class AmbariServer {
     server.setSessionIdManager(sessionIdManager);
     Server serverForAgent = new Server();
 
-    DatabaseChecker.checkDBVersion();
-    DatabaseChecker.checkDBConsistency();
+    if (System.getProperty("skipDatabaseConsistencyValidation") == null) {
+      DatabaseChecker.checkDBVersion();
+      DatabaseChecker.checkDBConsistency();
+      DatabaseChecker.checkDBConfigsConsistency();
+    }
 
     try {
       ClassPathXmlApplicationContext parentSpringAppContext =

http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java
index c4a4e4c..4839d21 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java
@@ -19,22 +19,31 @@
 package org.apache.ambari.server.controller.utilities;
 
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.orm.dao.ClusterDAO;
 import org.apache.ambari.server.orm.dao.MetainfoDAO;
+import org.apache.ambari.server.orm.entities.ClusterConfigEntity;
+import org.apache.ambari.server.orm.entities.ClusterConfigMappingEntity;
 import org.apache.ambari.server.orm.entities.ClusterEntity;
 import org.apache.ambari.server.orm.entities.ClusterServiceEntity;
+import org.apache.ambari.server.orm.entities.ClusterStateEntity;
 import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity;
 import org.apache.ambari.server.orm.entities.HostComponentStateEntity;
 import org.apache.ambari.server.orm.entities.MetainfoEntity;
 import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity;
 import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntity;
+import org.apache.ambari.server.orm.entities.StackEntity;
 import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.ServiceInfo;
 import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.StackInfo;
+import org.apache.ambari.server.state.State;
 import org.apache.ambari.server.utils.VersionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -142,7 +151,108 @@ public class DatabaseChecker {
     if (checkPassed) {
       LOG.info("DB consistency check passed.");
     } else {
-      LOG.error("DB consistency check failed.");
+      String errorMessage = "DB consistency check failed. Run \"ambari-server start --skip-database-validation\"
to skip validation.";
+      LOG.error(errorMessage);
+      throw new AmbariException(errorMessage);
+    }
+  }
+
+  private static boolean clusterConfigsContainTypeAndTag(Collection<ClusterConfigEntity>
clusterConfigEntities,
+                                                         String typeName, String tag) {
+    for (ClusterConfigEntity clusterConfigEntity : clusterConfigEntities) {
+      if (typeName.equals(clusterConfigEntity.getType()) && tag.equals(clusterConfigEntity.getTag()))
{
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Validates configuration consistency. Checks that every config type from stack is presented
in
+   * ClusterConfigMapping. Checks that for each config type exactly one is selected. Checks
that ClusterConfig
+   * contains type_names and tags from ClusterConfigMapping.
+   *
+   * @throws AmbariException if check failed
+   */
+  public static void checkDBConfigsConsistency() throws AmbariException {
+    LOG.info("Checking DB configs consistency");
+
+    boolean checkPassed = true;
+
+    if (ambariMetaInfo == null) {
+      ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class);
+    }
+
+    ClusterDAO clusterDAO = injector.getInstance(ClusterDAO.class);
+    List<ClusterEntity> clusters = clusterDAO.findAll();
+    if (clusters != null) {
+      for (ClusterEntity clusterEntity : clusters) {
+        Collection<ClusterConfigMappingEntity> configMappingEntities = clusterEntity.getConfigMappingEntities();
+        Collection<ClusterConfigEntity> clusterConfigEntities = clusterEntity.getClusterConfigEntities();
+
+        if (configMappingEntities != null) {
+          Map<String, Integer> selectedCountForType = new HashMap<>();
+          for (ClusterConfigMappingEntity clusterConfigMappingEntity : configMappingEntities)
{
+            String typeName = clusterConfigMappingEntity.getType();
+            if (clusterConfigMappingEntity.isSelected() > 0) {
+              int selectedCount = selectedCountForType.get(typeName) != null ? selectedCountForType.get(typeName)
: 0;
+              selectedCountForType.put(typeName, selectedCount + 1);
+
+              // Check that ClusterConfig contains type_name and tag from ClusterConfigMapping
+              if (!clusterConfigsContainTypeAndTag(clusterConfigEntities, typeName, clusterConfigMappingEntity.getTag()))
{
+                checkPassed = false;
+                LOG.error("ClusterConfig does not contain mapping for type_name=" + typeName
+ " tag="
+                    + clusterConfigMappingEntity.getTag());
+              }
+            } else {
+              if (!selectedCountForType.containsKey(typeName)) {
+                selectedCountForType.put(typeName, 0);
+              }
+            }
+          }
+
+          // Check that every config type from stack is presented in ClusterConfigMapping
+          Collection<ClusterServiceEntity> clusterServiceEntities = clusterEntity.getClusterServiceEntities();
+          ClusterStateEntity clusterStateEntity = clusterEntity.getClusterStateEntity();
+          if (clusterStateEntity != null) {
+            StackEntity currentStack = clusterStateEntity.getCurrentStack();
+            StackInfo stack = ambariMetaInfo.getStack(currentStack.getStackName(), currentStack.getStackVersion());
+
+            for (ClusterServiceEntity clusterServiceEntity : clusterServiceEntities) {
+              if (!State.INIT.equals(clusterServiceEntity.getServiceDesiredStateEntity().getDesiredState()))
{
+                String serviceName = clusterServiceEntity.getServiceName();
+                ServiceInfo serviceInfo = ambariMetaInfo.getService(stack.getName(), stack.getVersion(),
serviceName);
+                for (String configTypeName : serviceInfo.getConfigTypeAttributes().keySet())
{
+                  if (selectedCountForType.get(configTypeName) == null) {
+                    checkPassed = false;
+                    LOG.error("ClusterConfigMapping does not contain mapping for service="
+ serviceName + " type_name="
+                        + configTypeName);
+                  } else {
+                    // Check that for each config type exactly one is selected
+                    if (selectedCountForType.get(configTypeName) == 0) {
+                      checkPassed = false;
+                      LOG.error("ClusterConfigMapping selected count is 0 for service=" +
serviceName + " type_name="
+                          + configTypeName);
+                    } else if (selectedCountForType.get(configTypeName) > 1) {
+                      checkPassed = false;
+                      LOG.error("ClusterConfigMapping selected count is more than 1 for service="
+ serviceName
+                          + " type_name=" + configTypeName);
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+
+    if (checkPassed) {
+      LOG.info("DB configs consistency check passed.");
+    } else {
+      String errorMessage = "DB configs consistency check failed. Run \"ambari-server start
--skip-database-validation\" to skip validation.";
+      LOG.error(errorMessage);
+      throw new AmbariException(errorMessage);
     }
   }
 
@@ -169,7 +279,8 @@ public class DatabaseChecker {
       String error = "Current database store version is not compatible with " +
         "current server version"
         + ", serverVersion=" + serverVersion
-        + ", schemaVersion=" + schemaVersion;
+        + ", schemaVersion=" + schemaVersion
+        + " Run \"ambari-server start --skip-database-validation\" to skip validation";
       LOG.warn(error);
       throw new AmbariException(error);
     }

http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/python/ambari-server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py
index 41fdfb6..9f13ac5 100755
--- a/ambari-server/src/main/python/ambari-server.py
+++ b/ambari-server/src/main/python/ambari-server.py
@@ -316,6 +316,8 @@ def init_parser_options(parser):
                     help="Database user password")
   parser.add_option('--jdbc-driver', default=None, dest="jdbc_driver",
                     help="Specifies the path to the JDBC driver JAR file")
+  parser.add_option('--skip-properties-validation', action="store_true", default=False, help="Skip
properties file validation", dest="skip_properties_validation")
+  parser.add_option('--skip-database-validation', action="store_true", default=False, help="Skip
database consistency validation", dest="skip_database_validation")
   # -b and -i the remaining available short options
   # -h reserved for help
 
@@ -380,6 +382,8 @@ def init_parser_options(parser):
                     dest="jdbc_db")
   parser.add_option('--cluster-name', default=None, help="Cluster name", dest="cluster_name")
   parser.add_option('--version-display-name', default=None, help="Display name of desired
repo version", dest="desired_repo_version")
+  parser.add_option('--skip-properties-validation', action="store_true", default=False, help="Skip
properties file validation", dest="skip_properties_validation")
+  parser.add_option('--skip-database-validation', action="store_true", default=False, help="Skip
database consistency validation", dest="skip_database_validation")
   parser.add_option('--force-version', action="store_true", default=False, help="Force version
to current", dest="force_repo_version")
   parser.add_option('--version', dest="stack_versions", default=None, action="append", type="string",
                     help="Specify stack version that needs to be enabled. All other stacks
versions will be disabled")

http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
index 223c608..aeb4d5b 100644
--- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
@@ -177,6 +177,11 @@ SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server
s
 
 DEFAULT_DB_NAME = "ambari"
 
+COMMON_SERVICES_PATH_PROPERTY = 'common.services.path'
+WEBAPP_DIR_PROPERTY = 'webapp.dir'
+REQUIRED_PROPERTIES = [OS_FAMILY_PROPERTY, OS_TYPE_PROPERTY, COMMON_SERVICES_PATH_PROPERTY,
SERVER_VERSION_FILE_PATH, \
+                       WEBAPP_DIR_PROPERTY, STACK_LOCATION_KEY]
+
 class ServerDatabaseType(object):
   internal = 0
   remote = 1
@@ -1303,3 +1308,12 @@ def get_stack_location(properties):
   if stack_location is None:
     stack_location = configDefaults.STACK_LOCATION_DEFAULT
   return stack_location
+
+def get_missing_properties(properties):
+  missing_propertiers = []
+  for property in REQUIRED_PROPERTIES:
+    value = properties[property]
+    if not value:
+      missing_propertiers.append(property)
+
+  return missing_propertiers
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/python/ambari_server_main.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server_main.py b/ambari-server/src/main/python/ambari_server_main.py
index cf8e8c0..5c98bf4 100644
--- a/ambari-server/src/main/python/ambari_server_main.py
+++ b/ambari-server/src/main/python/ambari_server_main.py
@@ -32,7 +32,7 @@ from ambari_server.serverConfiguration import configDefaults, find_jdk,
get_amba
   get_conf_dir, get_is_persisted, get_is_secure, get_java_exe_path, get_original_master_key,
read_ambari_user, \
   get_is_active_instance, \
   PID_NAME, SECURITY_KEY_ENV_VAR_NAME, SECURITY_MASTER_KEY_LOCATION, \
-  SETUP_OR_UPGRADE_MSG, check_database_name_property, parse_properties_file
+  SETUP_OR_UPGRADE_MSG, check_database_name_property, parse_properties_file, get_missing_properties
 from ambari_server.serverUtils import refresh_stack_hash
 from ambari_server.setupHttps import get_fqdn
 from ambari_server.setupSecurity import generate_env, \
@@ -237,6 +237,13 @@ def server_process_main(options, scmStatus=None):
 
   properties = get_ambari_properties()
 
+  if not options.skip_properties_validation:
+    missing_properties = get_missing_properties(properties)
+    if missing_properties:
+      err = "Required properties are not found: " + str(missing_properties) + ". To skip
properties validation " \
+            "use \"--skip-properties-validation\""
+      raise FatalException(1, err)
+
   # Preparations
   if is_root():
     print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
@@ -267,6 +274,10 @@ def server_process_main(options, scmStatus=None):
   suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
   suspend_mode = 'y' if suspend_start else 'n'
 
+  if options.skip_database_validation:
+    global jvm_args
+    jvm_args += " -DskipDatabaseConsistencyValidation"
+
   param_list = generate_child_process_param_list(ambari_user, java_exe,
                                                  serverClassPath.get_full_ambari_classpath_escaped_for_shell(),
debug_start,
                                                  suspend_mode)


Mime
View raw message