ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [05/24] ambari git commit: AMBARI-22454. ambari-server upgrade to 2.6.1 should surface the GPL agreement (aonishuk)
Date Tue, 28 Nov 2017 21:20:24 GMT
AMBARI-22454. ambari-server upgrade to 2.6.1 should surface the GPL agreement (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3df5ae74
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3df5ae74
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3df5ae74

Branch: refs/heads/branch-2.6
Commit: 3df5ae74a5917923d254a47ba116816e899dfbce
Parents: 4b01b65
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Fri Nov 17 13:44:58 2017 +0200
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Fri Nov 17 13:44:58 2017 +0200

----------------------------------------------------------------------
 .../server/upgrade/AbstractUpgradeCatalog.java  |   8 ++
 .../server/upgrade/SchemaUpgradeHelper.java     |  27 +++++
 .../ambari/server/upgrade/UpgradeCatalog.java   |   7 ++
 .../server/upgrade/UpgradeCatalog261.java       | 118 +++++++++++++++++++
 .../python/ambari_server/serverConfiguration.py |  22 +++-
 .../main/python/ambari_server/serverSetup.py    |  20 +---
 .../main/python/ambari_server/serverUpgrade.py  |  18 ++-
 .../src/test/python/TestAmbariServer.py         |   2 +-
 8 files changed, 200 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
index dac871e..5ecb638 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
@@ -140,6 +140,7 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog
{
     new HashMap<>();
 
   protected String ambariUpgradeConfigUpdatesFileName;
+  private Map<String,String> upgradeJsonOutput = new HashMap<>();
 
   @Inject
   public AbstractUpgradeCatalog(Injector injector) {
@@ -258,6 +259,13 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog
{
   }
 
   /**
+   * {@inheritDoc}
+   */
+  public Map<String,String> getUpgradeJsonOutput() {
+    return upgradeJsonOutput;
+  }
+
+  /**
    * Update metainfo to new version.
    */
   @Transactional

http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
index a2dea40..963fd26 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
@@ -29,7 +29,11 @@ import java.util.Date;
 import java.util.List;
 import java.util.Properties;
 import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
 
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.audit.AuditLoggerModule;
 import org.apache.ambari.server.configuration.Configuration;
@@ -56,6 +60,7 @@ public class SchemaUpgradeHelper {
   private DBAccessor dbAccessor;
   private Configuration configuration;
   private static final String[] rcaTableNames = {"workflow", "job", "task", "taskAttempt",
"hdfsEvent", "mapreduceEvent", "clusterEvent"};
+  static final Gson gson = new GsonBuilder().create();
 
   @Inject
   public SchemaUpgradeHelper(Set<UpgradeCatalog> allUpgradeCatalogs,
@@ -184,6 +189,7 @@ public class SchemaUpgradeHelper {
       catalogBinder.addBinding().to(UpgradeCatalog251.class);
       catalogBinder.addBinding().to(UpgradeCatalog252.class);
       catalogBinder.addBinding().to(UpgradeCatalog260.class);
+      catalogBinder.addBinding().to(UpgradeCatalog261.class);
       catalogBinder.addBinding().to(UpdateAlertScriptPaths.class);
       catalogBinder.addBinding().to(FinalUpgradeCatalog.class);
 
@@ -254,6 +260,26 @@ public class SchemaUpgradeHelper {
     }
   }
 
+  public void outputUpgradeJsonOutput(List<UpgradeCatalog> upgradeCatalogs)
+      throws AmbariException {
+    LOG.info("Combining upgrade json output.");
+    Map<String,String> combinedUpgradeJsonOutput = new HashMap<>();
+
+    if (upgradeCatalogs != null && !upgradeCatalogs.isEmpty()) {
+      for (UpgradeCatalog upgradeCatalog : upgradeCatalogs) {
+        try {
+          combinedUpgradeJsonOutput.putAll(upgradeCatalog.getUpgradeJsonOutput());
+
+        } catch (Exception e) {
+          LOG.error("Upgrade failed. ", e);
+          throw new AmbariException(e.getMessage(), e);
+        }
+      }
+    }
+    String content = gson.toJson(combinedUpgradeJsonOutput);
+    System.out.println(content);
+  }
+
   public void resetUIState() throws AmbariException {
     LOG.info("Resetting UI state.");
     try {
@@ -424,6 +450,7 @@ public class SchemaUpgradeHelper {
       schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs, ambariUpgradeConfigUpdatesFileName);
 
       schemaUpgradeHelper.executeOnPostUpgrade(upgradeCatalogs);
+      schemaUpgradeHelper.outputUpgradeJsonOutput(upgradeCatalogs);
 
       schemaUpgradeHelper.resetUIState();
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
index 21273fd..a005951 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
@@ -18,6 +18,8 @@
 package org.apache.ambari.server.upgrade;
 
 import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.ambari.server.AmbariException;
 
@@ -88,4 +90,9 @@ public interface UpgradeCatalog {
    * Update schema version in the database to the Target one
    */
   void updateDatabaseSchemaVersion();
+
+  /*
+  Get upgrade json output, which is sent to python executing process.
+   */
+  Map<String,String> getUpgradeJsonOutput();
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java
new file mode 100644
index 0000000..7f463aa
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.upgrade;
+
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.SQLException;
+import java.util.Map;
+
+/**
+ * The {@link UpgradeCatalog261} upgrades Ambari from 2.6.0 to 2.6.1.
+ */
+public class UpgradeCatalog261 extends AbstractUpgradeCatalog {
+  private static final String CORE_SITE = "core-site";
+  private static final String COMPRESSION_CODECS_PROPERTY = "io.compression.codecs";
+  private static final String COMPRESSION_CODECS_LZO = "com.hadoop.compression.lzo";
+  private static final String LZO_ENABLED_JSON_KEY = "lzo_enabled";
+
+  private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog261.class);
+
+  /**
+   * Constructor.
+   *
+   * @param injector
+   */
+  @Inject
+  public UpgradeCatalog261(Injector injector) {
+    super(injector);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public String getSourceVersion() {
+      return "2.6.0";
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public String getTargetVersion() {
+      return "2.6.1";
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  protected void executeDDLUpdates() throws AmbariException, SQLException {
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  protected void executePreDMLUpdates() throws AmbariException, SQLException {
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  protected void executeDMLUpdates() throws AmbariException, SQLException {
+    // TODO: make this map with clusterids as keys
+    this.getUpgradeJsonOutput().put(LZO_ENABLED_JSON_KEY, isLzoEnabled().toString());
+  }
+
+  protected Boolean isLzoEnabled() throws AmbariException {
+    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+    Clusters clusters = ambariManagementController.getClusters();
+    if (clusters != null) {
+      Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
+      if (clusterMap != null && !clusterMap.isEmpty()) {
+        for (final Cluster cluster : clusterMap.values()) {
+
+          Config coreSite = cluster.getDesiredConfigByType(CORE_SITE);
+          if (coreSite != null) {
+            Map<String, String> coreSiteProperties = coreSite.getProperties();
+
+            if (coreSiteProperties.containsKey(COMPRESSION_CODECS_PROPERTY)) {
+              String compressionCodecs = coreSiteProperties.get(COMPRESSION_CODECS_PROPERTY);
+
+              if(compressionCodecs.contains(COMPRESSION_CODECS_LZO)) {
+                return true;
+              }
+            }
+          }
+        }
+      }
+    }
+    return false;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/python/ambari_server/serverConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
index 262e944..6e19b48 100644
--- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
@@ -1149,9 +1149,25 @@ def update_ambari_env():
 
   return 0
 
-def prompt_gpl_agreement():
-  result = get_YN_input(GPL_LICENSE_PROMPT_TEXT, True)
-  return str(result).lower()
+def write_gpl_license_accepted(is_silent=False):
+  properties = get_ambari_properties()
+  if properties == -1:
+    err = "Error getting ambari properties"
+    raise FatalException(-1, err)
+
+
+  if GPL_LICENSE_ACCEPTED_PROPERTY in properties.keys() and properties.get_property(GPL_LICENSE_ACCEPTED_PROPERTY).lower()
== "true":
+    return True
+
+  if is_silent:
+    result = options.accept_gpl
+  else:
+    result = get_YN_input(GPL_LICENSE_PROMPT_TEXT, True)
+
+  properties.process_pair(GPL_LICENSE_ACCEPTED_PROPERTY, str(result).lower())
+  update_properties(properties)
+
+  return result
 
 def update_ambari_properties():
   prev_conf_file = search_file(configDefaults.AMBARI_PROPERTIES_BACKUP_FILE, get_conf_dir())

http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/python/ambari_server/serverSetup.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverSetup.py b/ambari-server/src/main/python/ambari_server/serverSetup.py
index 8a14066..4ffe0bc 100644
--- a/ambari-server/src/main/python/ambari_server/serverSetup.py
+++ b/ambari-server/src/main/python/ambari_server/serverSetup.py
@@ -38,7 +38,7 @@ from ambari_commons.str_utils import compress_backslashes
 from ambari_server.dbConfiguration import DBMSConfigFactory, TAR_GZ_ARCHIVE_TYPE, default_connectors_map,
check_jdbc_drivers
 from ambari_server.serverConfiguration import configDefaults, JDKRelease, \
   get_ambari_properties, get_is_secure, get_is_persisted, get_java_exe_path, get_JAVA_HOME,
get_missing_properties, \
-  get_resources_location, get_value_from_properties, read_ambari_user, update_properties,
validate_jdk, prompt_gpl_agreement, write_property, \
+  get_resources_location, get_value_from_properties, read_ambari_user, update_properties,
validate_jdk, write_gpl_license_accepted, write_property, \
   JAVA_HOME, JAVA_HOME_PROPERTY, JCE_NAME_PROPERTY, JDBC_RCA_URL_PROPERTY, JDBC_URL_PROPERTY,
\
   JDK_NAME_PROPERTY, JDK_RELEASES, NR_USER_PROPERTY, OS_FAMILY, OS_FAMILY_PROPERTY, OS_TYPE,
OS_TYPE_PROPERTY, OS_VERSION, \
   VIEWS_DIR_PROPERTY, JDBC_DATABASE_PROPERTY, JDK_DOWNLOAD_SUPPORTED_PROPERTY, JCE_DOWNLOAD_SUPPORTED_PROPERTY,
SETUP_DONE_PROPERTIES, GPL_LICENSE_ACCEPTED_PROPERTY
@@ -1081,20 +1081,6 @@ def check_setup_already_done():
 
   return not bool(get_missing_properties(properties, property_set=SETUP_DONE_PROPERTIES))
 
-def write_gpl_license_accepted(options):
-  properties = get_ambari_properties()
-  if properties == -1:
-    err = "Error getting ambari properties"
-    raise FatalException(-1, err)
-
-  if get_silent():
-    result = str(options.accept_gpl).lower()
-  else:
-    result = prompt_gpl_agreement()
-
-  properties.process_pair(GPL_LICENSE_ACCEPTED_PROPERTY, result)
-  update_properties(properties)
-
 #
 # Setup the Ambari Server.
 #
@@ -1142,8 +1128,8 @@ def setup(options):
     err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(e)
     raise FatalException(e.code, err)
 
-  print 'Prompting GPL software agreement...'
-  write_gpl_license_accepted(options)
+  print 'Checking GPL software agreement...'
+  write_gpl_license_accepted(is_silent=get_silent())
 
   print 'Completing setup...'
   retcode = configure_os_settings()

http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/python/ambari_server/serverUpgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverUpgrade.py b/ambari-server/src/main/python/ambari_server/serverUpgrade.py
index 421adfc..a864609 100644
--- a/ambari-server/src/main/python/ambari_server/serverUpgrade.py
+++ b/ambari-server/src/main/python/ambari_server/serverUpgrade.py
@@ -28,6 +28,7 @@ import re
 import glob
 import optparse
 import logging
+import ambari_simplejson as json
 
 from ambari_commons.exceptions import FatalException
 from ambari_commons.logging_utils import print_info_msg, print_warning_msg, print_error_msg,
get_verbose
@@ -41,7 +42,7 @@ from ambari_server.serverConfiguration import configDefaults, get_resources_loca
   update_database_name_property, get_admin_views_dir, get_views_dir, get_views_jars, \
   AMBARI_PROPERTIES_FILE, IS_LDAP_CONFIGURED, LDAP_PRIMARY_URL_PROPERTY, RESOURCES_DIR_PROPERTY,
\
   SETUP_OR_UPGRADE_MSG, update_krb_jaas_login_properties, AMBARI_KRB_JAAS_LOGIN_FILE, get_db_type,
update_ambari_env, \
-  AMBARI_ENV_FILE, JDBC_DATABASE_PROPERTY, get_default_views_dir
+  AMBARI_ENV_FILE, JDBC_DATABASE_PROPERTY, get_default_views_dir, write_gpl_license_accepted
 from ambari_server.setupSecurity import adjust_directory_permissions, \
   generate_env, ensure_can_start_under_current_user
 from ambari_server.utils import compare_versions
@@ -73,6 +74,11 @@ SCHEMA_UPGRADE_DEBUG = False
 
 SUSPEND_START_MODE = False
 
+INSALLED_LZO_WITHOUT_GPL_TEXT = "By saying no, Ambari will not automatically install LZO
on any  new host in the cluster." + \
+"It is up to you to ensure LZO is installed and configured appropriately." + \
+"Without LZO being installed and configured data compressed with LZO will not be readable.
" + \
+"Are you sure you want to proceed? [y/n] (n)?"
+
 def load_stack_values(version, filename):
   import xml.etree.ElementTree as ET
   values = {}
@@ -146,6 +152,10 @@ def run_schema_upgrade(args):
   environ = generate_env(args, ambari_user, current_user)
 
   (retcode, stdout, stderr) = run_os_command(command, env=environ)
+  upgrade_response = json.loads(stdout)
+
+  check_gpl_license_approved(upgrade_response)
+
   print_info_msg("Return code from schema upgrade command, retcode = {0}".format(str(retcode)),
True)
   if stdout:
     print_info_msg("Console output from schema upgrade command:", True)
@@ -161,6 +171,12 @@ def run_schema_upgrade(args):
     print_info_msg('Schema upgrade completed', True)
   return retcode
 
+def check_gpl_license_approved(upgrade_response):
+  if 'lzo_enabled' not in upgrade_response or upgrade_response['lzo_enabled'].lower() !=
"true":
+    return
+
+  while not write_gpl_license_accepted() and not get_YN_input(INSALLED_LZO_WITHOUT_GPL_TEXT,
False):
+    pass
 
 #
 # Upgrades the Ambari Server.

http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index 840b9a6..a53d274 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -4938,7 +4938,7 @@ class TestAmbariServer(TestCase):
                               ensure_can_start_under_current_user_mock, get_jdbc_mock,
                               ensure_jdbc_driver_is_installed_mock):
     java_exe_path_mock.return_value = "/usr/lib/java/bin/java"
-    run_os_command_mock.return_value = (0, None, None)
+    run_os_command_mock.return_value = (0, '{"lzo_enabled":"false"}', None)
     get_conf_dir_mock.return_value = '/etc/conf'
     command = '/usr/lib/java/bin/java -cp /etc/conf' + os.pathsep + 'test' + os.pathsep +
'path12' + \
               os.pathsep +'/path/to/jdbc.jar ' \


Mime
View raw message