Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 4B2B3200CCC for ; Thu, 6 Jul 2017 15:09:58 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 49C431664F7; Thu, 6 Jul 2017 13:09:58 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 971D61664D2 for ; Thu, 6 Jul 2017 15:09:55 +0200 (CEST) Received: (qmail 43499 invoked by uid 500); 6 Jul 2017 13:09:54 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 43483 invoked by uid 99); 6 Jul 2017 13:09:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Jul 2017 13:09:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 76770E9471; Thu, 6 Jul 2017 13:09:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ncole@apache.org To: commits@ambari.apache.org Date: Thu, 06 Jul 2017 13:09:55 -0000 Message-Id: <9551d09d038646418ae10a7e9567de80@git.apache.org> In-Reply-To: <83333e5129324cde92a6636c0838209d@git.apache.org> References: <83333e5129324cde92a6636c0838209d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [02/50] [abbrv] ambari git commit: AMBARI-21099. Drop JDK 7 support for Ambari Server and Ambari managed services (AMS, LogSearch, Infra) (oleewere) archived-at: Thu, 06 Jul 2017 13:09:58 -0000 AMBARI-21099. Drop JDK 7 support for Ambari Server and Ambari managed services (AMS, LogSearch, Infra) (oleewere) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2f0de691 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2f0de691 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2f0de691 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 2f0de6919ba8bb43ae156d66b39335f9860d26b9 Parents: f4fb174 Author: oleewere Authored: Wed Jun 28 21:15:48 2017 +0200 Committer: oleewere Committed: Wed Jun 28 21:16:11 2017 +0200 ---------------------------------------------------------------------- ambari-server/docs/configuration/index.md | 13 ++- .../ambari/server/agent/ExecutionCommand.java | 4 + .../server/configuration/Configuration.java | 65 ++++++++++-- .../controller/AmbariActionExecutionHelper.java | 2 + .../AmbariCustomCommandExecutionHelper.java | 12 +-- .../AmbariManagementControllerImpl.java | 2 + .../internal/ClientConfigResourceProvider.java | 9 +- .../apache/ambari/server/utils/StageUtils.java | 54 ++++++++++ ambari-server/src/main/python/ambari-server.py | 2 + .../python/ambari_server/serverConfiguration.py | 6 ++ .../main/python/ambari_server/serverSetup.py | 105 +++++++++++++++++-- .../0.1.0/package/scripts/params.py | 4 +- .../0.1.0/package/scripts/params.py | 10 +- .../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 8 +- .../ATLAS/0.1.0.2.3/package/scripts/params.py | 1 + .../LOGSEARCH/0.5.0/package/scripts/params.py | 4 +- .../RANGER/0.4.0/package/scripts/params.py | 1 + .../0.4.0/package/scripts/setup_ranger_xml.py | 10 +- .../2.0.6/hooks/before-ANY/scripts/params.py | 2 + .../before-ANY/scripts/shared_initialization.py | 30 ++++-- .../2.0.6/hooks/before-START/scripts/params.py | 4 + .../scripts/shared_initialization.py | 22 ++-- .../HDP/3.0/hooks/before-ANY/scripts/params.py | 3 + .../before-ANY/scripts/shared_initialization.py | 31 ++++-- .../3.0/hooks/before-START/scripts/params.py | 4 + .../scripts/shared_initialization.py | 22 ++-- .../main/resources/stacks/HDP/3.0/metainfo.xml | 2 +- .../PERF/1.0/hooks/before-ANY/scripts/params.py | 3 + .../before-ANY/scripts/shared_initialization.py | 23 +++- .../ClientConfigResourceProviderTest.java | 8 ++ .../ambari/server/utils/StageUtilsTest.java | 99 +++++++++++++++++ .../src/test/python/TestAmbariServer.py | 49 ++++++++- 32 files changed, 524 insertions(+), 90 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/docs/configuration/index.md ---------------------------------------------------------------------- diff --git a/ambari-server/docs/configuration/index.md b/ambari-server/docs/configuration/index.md index 2394264..9dbe9c4 100644 --- a/ambari-server/docs/configuration/index.md +++ b/ambari-server/docs/configuration/index.md @@ -142,9 +142,9 @@ The following are the properties which can be used to configure Ambari. | http.x-content-type-options | The value that will be used to set the `X-CONTENT-TYPE` HTTP response header. |`nosniff` | | http.x-frame-options | The value that will be used to set the `X-Frame-Options` HTTP response header. |`DENY` | | http.x-xss-protection | The value that will be used to set the `X-XSS-Protection` HTTP response header. |`1; mode=block` | -| java.home | The location of the JDK on the Ambari Agent hosts.

The following are examples of valid values:
  • `/usr/jdk64/jdk1.7.0_45`
| | -| jce.name | The name of the JCE policy ZIP file.

The following are examples of valid values:
  • `UnlimitedJCEPolicyJDK7.zip`
| | -| jdk.name | The name of the JDK installation binary.

The following are examples of valid values:
  • `jdk-7u45-linux-x64.tar.gz`
| | +| java.home | The location of the JDK on the Ambari Agent hosts. If stack.java.home exists, that is only used by Ambari Server (or you can find that as ambari_java_home in the commandParams on the agent side)

The following are examples of valid values:
  • `/usr/jdk64/jdk1.8.0_112`
| | +| jce.name | The name of the JCE policy ZIP file. If stack.jce.name exists, that is only used by Ambari Server (or you can find that as ambari_jce_name in the commandParams on the agent side)

The following are examples of valid values:
  • `UnlimitedJCEPolicyJDK8.zip`
| | +| jdk.name | The name of the JDK installation binary. If stack.jdk.name exists, that is only used by Ambari Server (or you can find that as ambari_jdk_name in the commandParams on the agent side)

The following are examples of valid values:
  • `jdk-8u112-linux-x64.tar.gz`
| | | kdcserver.connection.check.timeout | The timeout, in milliseconds, to wait when communicating with a Kerberos Key Distribution Center. |`10000` | | kerberos.check.jaas.configuration | Determines whether Kerberos-enabled Ambari deployments should use JAAS to validate login credentials. |`false` | | kerberos.keytab.cache.dir | The location on the Ambari Server where Kerberos keytabs are cached. |`/var/lib/ambari-server/data/cache` | @@ -161,6 +161,7 @@ The following are the properties which can be used to configure Ambari. | metrics.retrieval-service.request.ttl | The number of seconds to wait between issuing JMX or REST metric requests to the same endpoint. This property is used to throttle requests to the same URL being made too close together

This property is related to `metrics.retrieval-service.request.ttl.enabled`. |`5` | | metrics.retrieval-service.request.ttl.enabled | Enables throttling requests to the same endpoint within a fixed amount of time. This property will prevent Ambari from making new metric requests to update the cache for URLs which have been recently retrieved.

This property is related to `metrics.retrieval-service.request.ttl`. |`true` | | mpacks.staging.path | The Ambari Management Pack staging directory on the Ambari Server.

The following are examples of valid values:
  • `/var/lib/ambari-server/resources/mpacks`
| | +| notification.dispatch.alert.script.directory | The directory for scripts which are used by the alert notification dispatcher. |`/var/lib/ambari-server/resources/scripts` | | packages.pre.installed | Determines whether Ambari Agent instances have already have the necessary stack software installed |`false` | | pam.configuration | The PAM configuration file. | | | property.mask.file | The path of the file which lists the properties that should be masked from the api that returns ambari.properties | | @@ -209,6 +210,7 @@ The following are the properties which can be used to configure Ambari. | server.ecCacheSize | The size of the cache which is used to hold current operations in memory until they complete. |`10000` | | server.execution.scheduler.isClustered | Determines whether Quartz will use a clustered job scheduled when performing scheduled actions like rolling restarts. |`false` | | server.execution.scheduler.maxDbConnections | The number of concurrent database connections that the Quartz job scheduler can use. |`5` | +| server.execution.scheduler.maxStatementsPerConnection | The maximum number of prepared statements cached per database connection. |`120` | | server.execution.scheduler.maxThreads | The number of threads that the Quartz job scheduler will use when executing scheduled jobs. |`5` | | server.execution.scheduler.misfire.toleration.minutes | The time, in minutes, that a scheduled job can be run after its missed scheduled execution time. |`480` | | server.execution.scheduler.start.delay.seconds | The delay, in seconds, that a Quartz job must wait before it starts. |`120` | @@ -280,6 +282,10 @@ The following are the properties which can be used to configure Ambari. | ssl.trustStore.password | The password to use when setting the `javax.net.ssl.trustStorePassword` property | | | ssl.trustStore.path | The location of the truststore to use when setting the `javax.net.ssl.trustStore` property. | | | ssl.trustStore.type | The type of truststore used by the `javax.net.ssl.trustStoreType` property. | | +| stack.java.home | The location of the JDK on the Ambari Agent hosts for stack services.

The following are examples of valid values:
  • `/usr/jdk64/jdk1.7.0_45`
| | +| stack.java.version | JDK version of the stack, use in case of it differs from Ambari JDK version.

The following are examples of valid values:
  • `1.7`
| | +| stack.jce.name | The name of the JCE policy ZIP file for stack services.

The following are examples of valid values:
  • `UnlimitedJCEPolicyJDK7.zip`
| | +| stack.jdk.name | The name of the JDK installation binary for stack services.

The following are examples of valid values:
  • `jdk-7u45-linux-x64.tar.gz`
| | | stack.upgrade.auto.retry.check.interval.secs | The amount of time to wait, in seconds, between checking for upgrade tasks to be retried. This value is only applicable if `stack.upgrade.auto.retry.timeout.mins` is positive.

This property is related to `stack.upgrade.auto.retry.timeout.mins`. |`20` | | stack.upgrade.auto.retry.command.details.to.ignore | A comma-separate list of upgrade tasks details to skip when retrying failed commands automatically. |`"Execute HDFS Finalize"` | | stack.upgrade.auto.retry.command.names.to.ignore | A comma-separate list of upgrade tasks names to skip when retrying failed commands automatically. |`"ComponentVersionCheckAction","FinalizeUpgradeAction"` | @@ -298,6 +304,7 @@ The following are the properties which can be used to configure Ambari. | views.ambari.request.connect.timeout.millis | The amount of time, in milliseconds, that a view will wait when trying to connect on HTTP(S) operations to the Ambari REST API. |`30000` | | views.ambari.request.read.timeout.millis | The amount of time, in milliseconds, that a view will wait before terminating an HTTP(S) read request to the Ambari REST API. |`45000` | | views.dir | The directory on the Ambari Server file system used for expanding Views and storing webapp work. |`/var/lib/ambari-server/resources/views` | +| views.directory.watcher.disable | Determines whether the view directory watcher service should be disabled. |`false` | | views.http.cache-control | The value that will be used to set the `Cache-Control` HTTP response header for Ambari View requests. |`no-store` | | views.http.charset | The value that will be used to set the Character encoding to HTTP response header for Ambari View requests. |`utf-8` | | views.http.pragma | The value that will be used to set the `PRAGMA` HTTP response header for Ambari View requests. |`no-cache` | http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index 60df8cf..29d28da 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -428,6 +428,10 @@ public class ExecutionCommand extends AgentCommand { String PACKAGE_LIST = "package_list"; String JDK_LOCATION = "jdk_location"; String JAVA_HOME = "java_home"; + String AMBARI_JAVA_HOME = "ambari_java_home"; + String AMBARI_JDK_NAME = "ambari_jdk_name"; + String AMBARI_JCE_NAME = "ambari_jce_name"; + String AMBARI_JAVA_VERSION = "ambari_java_version"; String JAVA_VERSION = "java_version"; String JDK_NAME = "jdk_name"; String JCE_NAME = "jce_name"; http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index fb06e6d..28f9d64 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -733,8 +733,8 @@ public class Configuration { * The location of the JDK on the Ambari Agent hosts. */ @Markdown( - description = "The location of the JDK on the Ambari Agent hosts.", - examples = { "/usr/jdk64/jdk1.7.0_45" }) + description = "The location of the JDK on the Ambari Agent hosts. If stack.java.home exists, that is only used by Ambari Server (or you can find that as ambari_java_home in the commandParams on the agent side)", + examples = { "/usr/jdk64/jdk1.8.0_112" }) public static final ConfigurationProperty JAVA_HOME = new ConfigurationProperty<>( "java.home", null); @@ -742,8 +742,8 @@ public class Configuration { * The name of the JDK installation binary. */ @Markdown( - description = "The name of the JDK installation binary.", - examples = { "jdk-7u45-linux-x64.tar.gz" }) + description = "The name of the JDK installation binary. If stack.jdk.name exists, that is only used by Ambari Server (or you can find that as ambari_jdk_name in the commandParams on the agent side)", + examples = { "jdk-8u112-linux-x64.tar.gz" }) public static final ConfigurationProperty JDK_NAME = new ConfigurationProperty<>( "jdk.name", null); @@ -751,12 +751,48 @@ public class Configuration { * The name of the JCE policy ZIP file. */ @Markdown( - description = "The name of the JCE policy ZIP file. ", - examples = {"UnlimitedJCEPolicyJDK7.zip"}) + description = "The name of the JCE policy ZIP file. If stack.jce.name exists, that is only used by Ambari Server (or you can find that as ambari_jce_name in the commandParams on the agent side)", + examples = {"UnlimitedJCEPolicyJDK8.zip"}) public static final ConfigurationProperty JCE_NAME = new ConfigurationProperty<>( "jce.name", null); /** + * The location of the JDK on the Ambari Agent hosts. + */ + @Markdown( + description = "The location of the JDK on the Ambari Agent hosts for stack services.", + examples = { "/usr/jdk64/jdk1.7.0_45" }) + public static final ConfigurationProperty STACK_JAVA_HOME = new ConfigurationProperty<>( + "stack.java.home", null); + + /** + * The name of the JDK installation binary. + */ + @Markdown( + description = "The name of the JDK installation binary for stack services.", + examples = { "jdk-7u45-linux-x64.tar.gz" }) + public static final ConfigurationProperty STACK_JDK_NAME = new ConfigurationProperty<>( + "stack.jdk.name", null); + + /** + * The name of the JCE policy ZIP file. + */ + @Markdown( + description = "The name of the JCE policy ZIP file for stack services.", + examples = {"UnlimitedJCEPolicyJDK7.zip"}) + public static final ConfigurationProperty STACK_JCE_NAME = new ConfigurationProperty<>( + "stack.jce.name", null); + + /** + * Java version of the stack + */ + @Markdown( + description = "JDK version of the stack, use in case of it differs from Ambari JDK version.", + examples = {"1.7"}) + public static final ConfigurationProperty STACK_JAVA_VERSION = new ConfigurationProperty<>( + "stack.java.version", null); + + /** * The auto group creation by Ambari. */ @Markdown( @@ -4128,6 +4164,23 @@ public class Configuration { public String getJCEName() { return getProperty(JCE_NAME); } + + public String getStackJavaHome() { + return getProperty(STACK_JAVA_HOME); + } + + public String getStackJDKName() { + return getProperty(STACK_JDK_NAME); + } + + public String getStackJCEName() { + return getProperty(STACK_JCE_NAME); + } + + public String getStackJavaVersion() { + return getProperty(STACK_JAVA_VERSION); + } + public String getAmbariBlacklistFile() { return getProperty(PROPERTY_MASK_FILE); } http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java index 1b0e0e0..8f522b0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java @@ -60,6 +60,7 @@ import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpInProgressEvent; import org.apache.ambari.server.utils.SecretReference; +import org.apache.ambari.server.utils.StageUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -418,6 +419,7 @@ public class AmbariActionExecutionHelper { commandParams.put(SCRIPT, actionName + ".py"); commandParams.put(SCRIPT_TYPE, TYPE_PYTHON); + StageUtils.useAmbariJdkInCommandParams(commandParams, configs); ExecutionCommand execCmd = stage.getExecutionCommandWrapper(hostName, actionContext.getActionName()).getExecutionCommand(); http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 0473690..28aa4e4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -29,11 +29,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL; @@ -508,7 +504,7 @@ public class AmbariCustomCommandExecutionHelper { if (isUpgradeSuspended) { cluster.addSuspendedUpgradeParameters(commandParams, roleParams); } - + StageUtils.useAmbariJdkInCommandParams(commandParams, configs); roleParams.put(COMPONENT_CATEGORY, componentInfo.getCategory()); execCmd.setCommandParams(commandParams); @@ -815,6 +811,7 @@ public class AmbariCustomCommandExecutionHelper { } commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder()); commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder()); + StageUtils.useAmbariJdkInCommandParams(commandParams, configs); execCmd.setCommandParams(commandParams); @@ -1481,11 +1478,8 @@ public class AmbariCustomCommandExecutionHelper { Map createDefaultHostParams(Cluster cluster, StackId stackId) throws AmbariException { TreeMap hostLevelParams = new TreeMap<>(); + StageUtils.useStackJdkIfExists(hostLevelParams, configs); hostLevelParams.put(JDK_LOCATION, managementController.getJdkResourceUrl()); - hostLevelParams.put(JAVA_HOME, managementController.getJavaHome()); - hostLevelParams.put(JAVA_VERSION, String.valueOf(configs.getJavaVersion())); - hostLevelParams.put(JDK_NAME, managementController.getJDKName()); - hostLevelParams.put(JCE_NAME, managementController.getJCEName()); hostLevelParams.put(STACK_NAME, stackId.getStackName()); hostLevelParams.put(STACK_VERSION, stackId.getStackVersion()); hostLevelParams.put(DB_NAME, managementController.getServerDB()); http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 77883e3..5639dc1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -245,6 +245,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle private static final String BASE_LOG_DIR = "/tmp/ambari"; private static final String PASSWORD = "password"; + public static final String SKIP_INSTALL_FOR_COMPONENTS = "skipInstallForComponents"; public static final String DONT_SKIP_INSTALL_FOR_COMPONENTS = "dontSkipInstallForComponents"; @@ -2473,6 +2474,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle if (customCommandExecutionHelper.isTopologyRefreshRequired(roleCommand.name(), clusterName, serviceName)) { commandParams.put(ExecutionCommand.KeyNames.REFRESH_TOPOLOGY, "True"); } + StageUtils.useAmbariJdkInCommandParams(commandParams, configs); String repoInfo = customCommandExecutionHelper.getRepoInfo(cluster, component, host); if (LOG.isDebugEnabled()) { http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java index 21cf16c..bd445eb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java @@ -23,11 +23,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STA import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL; @@ -363,11 +359,8 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv osFamily = clusters.getHost(hostName).getOsFamily(); TreeMap hostLevelParams = new TreeMap<>(); + StageUtils.useStackJdkIfExists(hostLevelParams, configs); hostLevelParams.put(JDK_LOCATION, managementController.getJdkResourceUrl()); - hostLevelParams.put(JAVA_HOME, managementController.getJavaHome()); - hostLevelParams.put(JAVA_VERSION, String.valueOf(configs.getJavaVersion())); - hostLevelParams.put(JDK_NAME, managementController.getJDKName()); - hostLevelParams.put(JCE_NAME, managementController.getJCEName()); hostLevelParams.put(STACK_NAME, stackId.getStackName()); hostLevelParams.put(STACK_VERSION, stackId.getStackVersion()); hostLevelParams.put(DB_NAME, managementController.getServerDB()); http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java index e7a94d4..9409f70 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java @@ -17,6 +17,15 @@ */ package org.apache.ambari.server.utils; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_JAVA_HOME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_JAVA_VERSION; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_JCE_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_JDK_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -82,6 +91,7 @@ public class StageUtils { protected static final String PORTS = "all_ping_ports"; protected static final String RACKS = "all_racks"; protected static final String IPV4_ADDRESSES = "all_ipv4_ips"; + private static Map componentToClusterInfoKeyMap = new HashMap<>(); private static Map decommissionedToClusterInfoKeyMap = @@ -598,4 +608,48 @@ public class StageUtils { startOfRange + separator + endOfRange; return rangeItem; } + + /** + * Add ambari specific JDK details to command parameters. + */ + public static void useAmbariJdkInCommandParams(Map commandParams, Configuration configuration) { + if (StringUtils.isNotEmpty(configuration.getJavaHome()) && !configuration.getJavaHome().equals(configuration.getStackJavaHome())) { + commandParams.put(AMBARI_JAVA_HOME, configuration.getJavaHome()); + commandParams.put(AMBARI_JAVA_VERSION, String.valueOf(configuration.getJavaVersion())); + if (StringUtils.isNotEmpty(configuration.getJDKName())) { // if not custom jdk + commandParams.put(AMBARI_JDK_NAME, configuration.getJDKName()); + } + if (StringUtils.isNotEmpty(configuration.getJCEName())) { // if not custom jdk + commandParams.put(AMBARI_JCE_NAME, configuration.getJCEName()); + } + } + } + + /** + * Fill hots level parameters with Jdk details, override them with the stack JDK data, in case of stack JAVA_HOME exists + */ + public static void useStackJdkIfExists(Map hostLevelParams, Configuration configuration) { + // set defaults first + hostLevelParams.put(JAVA_HOME, configuration.getJavaHome()); + hostLevelParams.put(JDK_NAME, configuration.getJDKName()); + hostLevelParams.put(JCE_NAME, configuration.getJCEName()); + hostLevelParams.put(JAVA_VERSION, String.valueOf(configuration.getJavaVersion())); + if (StringUtils.isNotEmpty(configuration.getStackJavaHome()) + && !configuration.getStackJavaHome().equals(configuration.getJavaHome())) { + hostLevelParams.put(JAVA_HOME, configuration.getStackJavaHome()); + if (StringUtils.isNotEmpty(configuration.getStackJavaVersion())) { + hostLevelParams.put(JAVA_VERSION, configuration.getStackJavaVersion()); + } + if (StringUtils.isNotEmpty(configuration.getStackJDKName())) { + hostLevelParams.put(JDK_NAME, configuration.getStackJDKName()); + } else { + hostLevelParams.put(JDK_NAME, null); // custom jdk for stack + } + if (StringUtils.isNotEmpty(configuration.getStackJCEName())) { + hostLevelParams.put(JCE_NAME, configuration.getStackJCEName()); + } else { + hostLevelParams.put(JCE_NAME, null); // custom jdk for stack + } + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index c7bdcf9..5adcb04 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -482,6 +482,8 @@ def init_setup_parser_options(parser): other_group.add_option('-j', '--java-home', default=None, help="Use specified java_home. Must be valid on all hosts") + other_group.add_option('--stack-java-home', dest="stack_java_home", default=None, + help="Use specified java_home for stack services. Must be valid on all hosts") other_group.add_option('--skip-view-extraction', action="store_true", default=False, help="Skip extraction of system views", dest="skip_view_extraction") other_group.add_option('--postgresschema', default=None, help="Postgres database schema name", dest="postgres_schema") http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/python/ambari_server/serverConfiguration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py index 4780338..631c9a7 100644 --- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py +++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py @@ -87,6 +87,12 @@ JCE_NAME_PROPERTY = "jce.name" JDK_DOWNLOAD_SUPPORTED_PROPERTY = "jdk.download.supported" JCE_DOWNLOAD_SUPPORTED_PROPERTY = "jce.download.supported" +# Stack JDK +STACK_JAVA_HOME_PROPERTY = "stack.java.home" +STACK_JDK_NAME_PROPERTY = "stack.jdk.name" +STACK_JCE_NAME_PROPERTY = "stack.jce.name" +STACK_JAVA_VERSION = "stack.java.version" + #TODO property used incorrectly in local case, it was meant to be dbms name, not postgres database name, # has workaround for now, as we don't need dbms name if persistence_type=local http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/python/ambari_server/serverSetup.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverSetup.py b/ambari-server/src/main/python/ambari_server/serverSetup.py index c6de088..5c016c5 100644 --- a/ambari-server/src/main/python/ambari_server/serverSetup.py +++ b/ambari-server/src/main/python/ambari_server/serverSetup.py @@ -41,7 +41,8 @@ from ambari_server.serverConfiguration import configDefaults, JDKRelease, \ get_resources_location, get_value_from_properties, read_ambari_user, update_properties, validate_jdk, write_property, \ JAVA_HOME, JAVA_HOME_PROPERTY, JCE_NAME_PROPERTY, JDBC_RCA_URL_PROPERTY, JDBC_URL_PROPERTY, \ JDK_NAME_PROPERTY, JDK_RELEASES, NR_USER_PROPERTY, OS_FAMILY, OS_FAMILY_PROPERTY, OS_TYPE, OS_TYPE_PROPERTY, OS_VERSION, \ - VIEWS_DIR_PROPERTY, JDBC_DATABASE_PROPERTY, JDK_DOWNLOAD_SUPPORTED_PROPERTY, JCE_DOWNLOAD_SUPPORTED_PROPERTY, SETUP_DONE_PROPERTIES + VIEWS_DIR_PROPERTY, JDBC_DATABASE_PROPERTY, JDK_DOWNLOAD_SUPPORTED_PROPERTY, JCE_DOWNLOAD_SUPPORTED_PROPERTY, SETUP_DONE_PROPERTIES, \ + STACK_JAVA_HOME_PROPERTY, STACK_JDK_NAME_PROPERTY, STACK_JCE_NAME_PROPERTY, STACK_JAVA_VERSION from ambari_server.serverUtils import is_server_runing from ambari_server.setupSecurity import adjust_directory_permissions from ambari_server.userInput import get_YN_input, get_validated_string_input @@ -79,7 +80,7 @@ UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}" JDK_PROMPT = "[{0}] {1}\n" JDK_VALID_CHOICES = "^[{0}{1:d}]$" - +JDK_VERSION_CHECK_CMD = """{0} -version 2>&1 | grep -i version | sed 's/.*version ".*\.\(.*\)\..*"/\\1/; 1q' 2>&1""" def get_supported_jdbc_drivers(): factory = DBMSConfigFactory() @@ -409,7 +410,7 @@ class JDKSetup(object): # # Downloads and installs the JDK and the JCE policy archive # - def download_and_install_jdk(self, args, properties): + def download_and_install_jdk(self, args, properties, ambariOnly = False): conf_file = properties.fileName jcePolicyWarn = "JCE Policy files are required for configuring Kerberos security. If you plan to use Kerberos," \ @@ -429,8 +430,22 @@ class JDKSetup(object): properties.removeOldProp(JDK_NAME_PROPERTY) properties.removeOldProp(JCE_NAME_PROPERTY) + if not ambariOnly: + properties.process_pair(STACK_JAVA_HOME_PROPERTY, args.java_home) + properties.removeOldProp(STACK_JDK_NAME_PROPERTY) + properties.removeOldProp(STACK_JCE_NAME_PROPERTY) + self._ensure_java_home_env_var_is_set(args.java_home) self.jdk_index = self.custom_jdk_number + + if args.stack_java_home: # reset stack specific jdk properties if stack_java_home exists + print 'Setting JAVA_HOME for stack services...' + print_warning_msg("JAVA_HOME " + args.stack_java_home + " (Stack) must be valid on ALL hosts") + print_warning_msg(jcePolicyWarn) + properties.process_pair(STACK_JAVA_HOME_PROPERTY, args.stack_java_home) + properties.removeOldProp(STACK_JDK_NAME_PROPERTY) + properties.removeOldProp(STACK_JCE_NAME_PROPERTY) + return java_home_var = get_JAVA_HOME() @@ -440,7 +455,10 @@ class JDKSetup(object): progress_func = download_progress if java_home_var: - change_jdk = get_YN_input("Do you want to change Oracle JDK [y/n] (n)? ", False) + message = "Do you want to change Oracle JDK [y/n] (n)? " + if ambariOnly: + message = "Do you want to change Oracle JDK for Ambari Server [y/n] (n)? " + change_jdk = get_YN_input(message, False) if not change_jdk: self._ensure_java_home_env_var_is_set(java_home_var) self.jdk_index = self.custom_jdk_number @@ -448,7 +466,7 @@ class JDKSetup(object): #Continue with the normal setup, taking the first listed JDK version as the default option jdk_num = str(self.jdk_index + 1) - (self.jdks, jdk_choice_prompt, jdk_valid_choices, self.custom_jdk_number) = self._populate_jdk_configs(properties, jdk_num) + (self.jdks, jdk_choice_prompt, jdk_valid_choices, self.custom_jdk_number) = self._populate_jdk_configs(properties, jdk_num, ambariOnly) jdk_num = get_validated_string_input( jdk_choice_prompt, @@ -478,10 +496,18 @@ class JDKSetup(object): properties.removeOldProp(JDK_NAME_PROPERTY) properties.removeOldProp(JCE_NAME_PROPERTY) + if not ambariOnly: + properties.process_pair(STACK_JAVA_HOME_PROPERTY, args.java_home) + properties.removeOldProp(STACK_JDK_NAME_PROPERTY) + properties.removeOldProp(STACK_JCE_NAME_PROPERTY) + # Make sure any previously existing JDK and JCE name properties are removed. These will # confuse things in a Custom JDK scenario properties.removeProp(JDK_NAME_PROPERTY) properties.removeProp(JCE_NAME_PROPERTY) + if not ambariOnly: + properties.removeOldProp(STACK_JDK_NAME_PROPERTY) + properties.removeOldProp(STACK_JCE_NAME_PROPERTY) self._ensure_java_home_env_var_is_set(args.java_home) return @@ -551,10 +577,13 @@ class JDKSetup(object): properties.process_pair(JDK_NAME_PROPERTY, jdk_cfg.dest_file) properties.process_pair(JAVA_HOME_PROPERTY, java_home_dir) + if not ambariOnly: + properties.process_pair(STACK_JDK_NAME_PROPERTY, jdk_cfg.dest_file) + properties.process_pair(STACK_JAVA_HOME_PROPERTY, java_home_dir) self._ensure_java_home_env_var_is_set(java_home_dir) - def download_and_unpack_jce_policy(self, properties): + def download_and_unpack_jce_policy(self, properties, ambariOnly = False): err_msg_stdout = "JCE Policy files are required for secure HDP setup. Please ensure " \ " all hosts have the JCE unlimited strength policy 6, files." @@ -563,7 +592,7 @@ class JDKSetup(object): jdk_cfg = self.jdks[self.jdk_index] try: - JDKSetup._download_jce_policy(jdk_cfg.jcpol_url, jdk_cfg.dest_jcpol_file, resources_dir, properties) + JDKSetup._download_jce_policy(jdk_cfg.jcpol_url, jdk_cfg.dest_jcpol_file, resources_dir, properties, ambariOnly) except FatalException, e: print err_msg_stdout print_error_msg("Failed to download JCE policy files:") @@ -590,10 +619,22 @@ class JDKSetup(object): jce_zip_path = os.path.abspath(os.path.join(resources_dir, jce_packed_file)) expand_jce_zip_file(jce_zip_path, jdk_security_path) - def _populate_jdk_configs(self, properties, jdk_num): + def _populate_jdk_configs(self, properties, jdk_num, ambariOnly = False): + def remove_jdk_condition(name): + """ + Removes jdk1.7 from the default choices. + This method can be removed if JDK 7 support (for stack services) will be dropped. + """ + if name != "jdk1.7": + return True + else: + print "JDK 7 detected. Removed from choices." + return False if properties.has_key(JDK_RELEASES): jdk_names = properties[JDK_RELEASES].split(',') jdk_names = filter(None, jdk_names) + if ambariOnly: + jdk_names = filter(lambda x : remove_jdk_condition(x), jdk_names) jdks = [] for jdk_name in jdk_names: jdkR = JDKRelease.from_properties(properties, jdk_name) @@ -630,7 +671,7 @@ class JDKSetup(object): raise FatalException(1, err) @staticmethod - def _download_jce_policy(jcpol_url, dest_jcpol_file, resources_dir, properties): + def _download_jce_policy(jcpol_url, dest_jcpol_file, resources_dir, properties, ambariOnly = False): dest_file = os.path.abspath(os.path.join(resources_dir, dest_jcpol_file)) if not os.path.exists(dest_file): @@ -653,6 +694,8 @@ class JDKSetup(object): print "JCE Policy archive already exists, using " + dest_file properties.process_pair(JCE_NAME_PROPERTY, dest_jcpol_file) + if not ambariOnly: + properties.process_pair(STACK_JCE_NAME_PROPERTY, dest_jcpol_file) # Base implementation, overriden in the subclasses def _install_jdk(self, java_inst_file, java_home_dir): @@ -828,6 +871,14 @@ def download_and_install_jdk(options): update_properties(properties) + ambari_java_version_valid = check_ambari_java_version_is_valid(get_JAVA_HOME(), jdkSetup.JAVA_BIN, 8, properties) + if not ambari_java_version_valid: + jdkSetup = JDKSetup() # recreate object + jdkSetup.download_and_install_jdk(options, properties, True) + if jdkSetup.jdk_index != jdkSetup.custom_jdk_number: + jdkSetup.download_and_unpack_jce_policy(properties, True) + update_properties(properties) + return 0 @@ -1200,7 +1251,43 @@ def setup_jce_policy(args): print 'NOTE: Restart Ambari Server to apply changes' + \ ' ("ambari-server restart|stop|start")' +def check_ambari_java_version_is_valid(java_home, java_bin, min_version, properties): + """ + Check that ambari uses the proper (minimum) JDK with a shell command. + Returns true, if Ambari meets with the minimal JDK version requirement. + """ + result = True + print 'Check JDK version for Ambari Server...' + try: + command = JDK_VERSION_CHECK_CMD.format(os.path.join(java_home, 'bin', java_bin)) + process = subprocess.Popen(command, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True + ) + (out, err) = process.communicate() + if process.returncode != 0: + err = "Checking JDK version command returned with exit code %s" % process.returncode + raise FatalException(process.returncode, err) + else: + actual_jdk_version = int(out) + print 'JDK version found: {0}'.format(actual_jdk_version) + if actual_jdk_version < min_version: + print 'Minimum JDK version is {0} for Ambari. Setup JDK again only for Ambari Server.'.format(min_version) + properties.process_pair(STACK_JAVA_VERSION, out) + result = False + else: + print 'Minimum JDK version is {0} for Ambari. Skipping to setup different JDK for Ambari Server.'.format(min_version) + + except FatalException as e: + err = 'Running java version check command failed: {0}. Exiting.'.format(e) + raise FatalException(e.code, err) + except Exception as e: + err = 'Running java version check command failed: {0}. Exiting.'.format(e) + raise FatalException(1, err) + return result # # Resets the Ambari Server. # http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/params.py index 6eb3ba8..5f547f3 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/params.py @@ -60,7 +60,9 @@ user_group = config['configurations']['cluster-env']['user_group'] fetch_nonlocal_groups = config['configurations']['cluster-env']["fetch_nonlocal_groups"] # shared configs -java64_home = config['hostLevelParams']['java_home'] +java_home = config['hostLevelParams']['java_home'] +ambari_java_home = default("/commandParams/ambari_java_home", None) +java64_home = ambari_java_home if ambari_java_home is not None else java_home java_exec = format("{java64_home}/bin/java") zookeeper_hosts_list = config['clusterHostInfo']['zookeeper_hosts'] zookeeper_hosts_list.sort() http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index b8c14f4..486f568 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -203,9 +203,15 @@ security_enabled = False if not is_hbase_distributed else config['configurations # this is "hadoop-metrics.properties" for 1.x stacks metric_prop_file_name = "hadoop-metrics2-hbase.properties" +java_home = config['hostLevelParams']['java_home'] +ambari_java_home = default("/commandParams/ambari_java_home", None) # not supporting 32 bit jdk. -java64_home = config['hostLevelParams']['java_home'] -java_version = expect("/hostLevelParams/java_version", int) +java64_home = ambari_java_home if ambari_java_home is not None else java_home +ambari_java_version = default("/commandParams/ambari_java_version", None) +if ambari_java_version: + java_version = expect("/commandParams/ambari_java_version", int) +else : + java_version = expect("/hostLevelParams/java_version", int) metrics_collector_heapsize = default('/configurations/ams-env/metrics_collector_heapsize', "512") metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60) http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py index 36c4598..d424f5b 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py @@ -207,7 +207,7 @@ def upload_conf_set(config_set, jaasFile): config_set_dir=format("{conf_dir}/solr"), config_set=config_set, tmp_dir=params.tmp_dir, - java64_home=params.java64_home, + java64_home=params.ambari_java_home, solrconfig_content=InlineTemplate(params.metadata_solrconfig_content), jaas_file=jaasFile, retry=30, interval=5) @@ -220,7 +220,7 @@ def create_collection(collection, config_set, jaasFile): solr_znode=params.infra_solr_znode, collection = collection, config_set=config_set, - java64_home=params.java64_home, + java64_home=params.ambari_java_home, jaas_file=jaasFile, shards=params.atlas_solr_shards, replication_factor = params.infra_solr_replication_factor) @@ -230,7 +230,7 @@ def secure_znode(znode, jaasFile): solr_cloud_util.secure_znode(config=params.config, zookeeper_quorum=params.zookeeper_quorum, solr_znode=znode, jaas_file=jaasFile, - java64_home=params.java64_home, sasl_users=[params.atlas_jaas_principal]) + java64_home=params.ambari_java_home, sasl_users=[params.atlas_jaas_principal]) @@ -240,4 +240,4 @@ def check_znode(): solr_cloud_util.check_znode( zookeeper_quorum=params.zookeeper_quorum, solr_znode=params.infra_solr_znode, - java64_home=params.java64_home) + java64_home=params.ambari_java_home) http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index d26df33..111a248 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -126,6 +126,7 @@ user_group = config['configurations']['cluster-env']['user_group'] # metadata env java64_home = config['hostLevelParams']['java_home'] +ambari_java_home = default("/commandParams/ambari_java_home", java64_home) java_exec = format("{java64_home}/bin/java") env_sh_template = config['configurations']['atlas-env']['content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index 662f49e..1b77999 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -74,7 +74,9 @@ logfeeder_pid_file = status_params.logfeeder_pid_file user_group = config['configurations']['cluster-env']['user_group'] # shared configs -java64_home = config['hostLevelParams']['java_home'] +java_home = config['hostLevelParams']['java_home'] +ambari_java_home = default("/commandParams/ambari_java_home", None) +java64_home = ambari_java_home if ambari_java_home is not None else java_home cluster_name = str(config['clusterName']) configurations = config['configurations'] # need reference inside logfeeder jinja templates http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py index 3789358..6d5581d 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py @@ -123,6 +123,7 @@ if stack_supports_ranger_tagsync: usersync_services_file = format('{stack_root}/current/ranger-usersync/ranger-usersync-services.sh') java_home = config['hostLevelParams']['java_home'] +ambari_java_home = default("/commandParams/ambari_java_home", java_home) unix_user = config['configurations']['ranger-env']['ranger_user'] unix_group = config['configurations']['ranger-env']['ranger_group'] ranger_pid_dir = default("/configurations/ranger-env/ranger_pid_dir", "/var/run/ranger") http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py index ba21494..4bcf9b0 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py @@ -711,7 +711,7 @@ def setup_ranger_audit_solr(): config_set = params.ranger_solr_config_set, config_set_dir = params.ranger_solr_conf, tmp_dir = params.tmp_dir, - java64_home = params.java_home, + java64_home = params.ambari_java_home, solrconfig_content = InlineTemplate(params.ranger_solr_config_content), jaas_file=params.solr_jaas_file, retry=30, interval=5 @@ -725,7 +725,7 @@ def setup_ranger_audit_solr(): config_set = params.ranger_solr_config_set, config_set_dir = params.ranger_solr_conf, tmp_dir = params.tmp_dir, - java64_home = params.java_home, + java64_home = params.ambari_java_home, jaas_file=params.solr_jaas_file, retry=30, interval=5) @@ -748,7 +748,7 @@ def setup_ranger_audit_solr(): solr_znode = params.solr_znode, collection = params.ranger_solr_collection_name, config_set = params.ranger_solr_config_set, - java64_home = params.java_home, + java64_home = params.ambari_java_home, shards = params.ranger_solr_shards, replication_factor = int(params.replication_factor), jaas_file = params.solr_jaas_file) @@ -774,14 +774,14 @@ def check_znode(): solr_cloud_util.check_znode( zookeeper_quorum=params.zookeeper_quorum, solr_znode=params.solr_znode, - java64_home=params.java_home) + java64_home=params.ambari_java_home) def secure_znode(znode, jaasFile): import params solr_cloud_util.secure_znode(config=params.config, zookeeper_quorum=params.zookeeper_quorum, solr_znode=znode, jaas_file=jaasFile, - java64_home=params.java_home, sasl_users=[params.ranger_admin_jaas_principal]) + java64_home=params.ambari_java_home, sasl_users=[params.ranger_admin_jaas_principal]) def get_ranger_plugin_principals(services_defaults_tuple_list): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py index 4052d1d..1d79efb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py @@ -65,6 +65,8 @@ version = default("/commandParams/version", None) # Handle upgrade and downgrade if (upgrade_type is not None) and version: stack_version_formatted = format_stack_version(version) +ambari_java_home = default("/commandParams/ambari_java_home", None) +ambari_jdk_name = default("/commandParams/ambari_jdk_name", None) security_enabled = config['configurations']['cluster-env']['security_enabled'] hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py index 1f17cd1..39f5a47 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py @@ -176,17 +176,26 @@ def setup_hadoop_env(): def setup_java(): """ - Installs jdk using specific params, that comes from ambari-server + Install jdk using specific params. + Install ambari jdk as well if the stack and ambari jdk are different. """ import params + __setup_java(custom_java_home=params.java_home, custom_jdk_name=params.jdk_name) + if params.ambari_java_home and params.ambari_java_home != params.java_home: + __setup_java(custom_java_home=params.ambari_java_home, custom_jdk_name=params.ambari_jdk_name) - java_exec = format("{java_home}/bin/java") +def __setup_java(custom_java_home, custom_jdk_name): + """ + Installs jdk using specific params, that comes from ambari-server + """ + import params + java_exec = format("{custom_java_home}/bin/java") if not os.path.isfile(java_exec): if not params.jdk_name: # if custom jdk is used. raise Fail(format("Unable to access {java_exec}. Confirm you have copied jdk to this host.")) - jdk_curl_target = format("{tmp_dir}/{jdk_name}") + jdk_curl_target = format("{tmp_dir}/{custom_jdk_name}") java_dir = os.path.dirname(params.java_home) Directory(params.artifact_dir, @@ -194,13 +203,13 @@ def setup_java(): ) File(jdk_curl_target, - content = DownloadSource(format("{jdk_location}/{jdk_name}")), + content = DownloadSource(format("{jdk_location}/{custom_jdk_name}")), not_if = format("test -f {jdk_curl_target}") - ) + ) File(jdk_curl_target, mode = 0755, - ) + ) tmp_java_dir = tempfile.mkdtemp(prefix="jdk_tmp_", dir=params.tmp_dir) @@ -213,7 +222,7 @@ def setup_java(): install_cmd = format("cd {tmp_java_dir} && tar -xf {jdk_curl_target} && {sudo} cp -rp {tmp_java_dir}/* {java_dir}") Directory(java_dir - ) + ) Execute(chmod_cmd, sudo = True, @@ -225,10 +234,11 @@ def setup_java(): finally: Directory(tmp_java_dir, action="delete") - File(format("{java_home}/bin/java"), + File(format("{custom_java_home}/bin/java"), mode=0755, cd_access="a", ) Execute(('chmod', '-R', '755', params.java_home), - sudo = True, - ) + sudo = True, + ) + http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py index 3488e75..49b0063 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py @@ -259,6 +259,10 @@ refresh_topology = False command_params = config["commandParams"] if "commandParams" in config else None if command_params is not None: refresh_topology = bool(command_params["refresh_topology"]) if "refresh_topology" in command_params else False + +ambari_java_home = default("/commandParams/ambari_java_home", None) +ambari_jdk_name = default("/commandParams/ambari_jdk_name", None) +ambari_jce_name = default("/commandParams/ambari_jce_name", None) ambari_libs_dir = "/var/lib/ambari-agent/lib" is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'] http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py index 148d235..42785ba 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py @@ -197,9 +197,17 @@ def create_microsoft_r_dir(): except Exception as exception: Logger.warning("Could not check the existence of {0} on DFS while starting {1}, exception: {2}".format(directory, params.current_service, str(exception))) - def setup_unlimited_key_jce_policy(): """ + Sets up the unlimited key JCE policy if needed. (sets up ambari JCE as well if ambari and the stack use different JDK) + """ + import params + __setup_unlimited_key_jce_policy(custom_java_home=params.java_home, custom_jdk_name=params.jdk_name, custom_jce_name = params.jce_policy_zip) + if params.ambari_jce_name and params.ambari_jce_name != params.jce_policy_zip: + __setup_unlimited_key_jce_policy(custom_java_home=params.ambari_java_home, custom_jdk_name=params.ambari_jdk_name, custom_jce_name = params.ambari_jce_name) + +def __setup_unlimited_key_jce_policy(custom_java_home, custom_jdk_name, custom_jce_name): + """ Sets up the unlimited key JCE policy if needed. The following criteria must be met: @@ -223,27 +231,27 @@ def setup_unlimited_key_jce_policy(): if params.sysprep_skip_setup_jce: Logger.info("Skipping unlimited key JCE policy check and setup since the host is sys prepped") - elif not params.jdk_name: + elif not custom_jdk_name: Logger.debug("Skipping unlimited key JCE policy check and setup since the Java VM is not managed by Ambari") elif not params.unlimited_key_jce_required: Logger.debug("Skipping unlimited key JCE policy check and setup since it is not required") else: - jcePolicyInfo = JcePolicyInfo(params.java_home) + jcePolicyInfo = JcePolicyInfo(custom_java_home) if jcePolicyInfo.is_unlimited_key_jce_policy(): Logger.info("The unlimited key JCE policy is required, and appears to have been installed.") - elif params.jce_policy_zip is None: + elif custom_jce_name is None: raise Fail("The unlimited key JCE policy needs to be installed; however the JCE policy zip is not specified.") else: Logger.info("The unlimited key JCE policy is required, and needs to be installed.") - jce_zip_target = format("{artifact_dir}/{jce_policy_zip}") - jce_zip_source = format("{ambari_server_resources_url}/{jce_policy_zip}") - java_security_dir = format("{java_home}/jre/lib/security") + jce_zip_target = format("{artifact_dir}/{custom_jce_name}") + jce_zip_source = format("{ambari_server_resources_url}/{custom_jce_name}") + java_security_dir = format("{custom_java_home}/jre/lib/security") Logger.debug("Downloading the unlimited key JCE policy files from {0} to {1}.".format(jce_zip_source, jce_zip_target)) Directory(params.artifact_dir, create_parents=True) http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py index 9be9101..eb5feae 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py @@ -65,6 +65,9 @@ version = default("/commandParams/version", None) if (upgrade_type is not None) and version: stack_version_formatted = format_stack_version(version) +ambari_java_home = default("/commandParams/ambari_java_home", None) +ambari_jdk_name = default("/commandParams/ambari_jdk_name", None) + security_enabled = config['configurations']['cluster-env']['security_enabled'] hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/shared_initialization.py index 5d79084..dbd1727 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/shared_initialization.py @@ -172,17 +172,26 @@ def setup_hadoop_env(): def setup_java(): """ - Installs jdk using specific params, that comes from ambari-server + Install jdk using specific params. + Install ambari jdk as well if the stack and ambari jdk are different. """ import params + __setup_java(custom_java_home=params.java_home, custom_jdk_name=params.jdk_name) + if params.ambari_java_home and params.ambari_java_home != params.java_home: + __setup_java(custom_java_home=params.ambari_java_home, custom_jdk_name=params.ambari_jdk_name) - java_exec = format("{java_home}/bin/java") +def __setup_java(custom_java_home, custom_jdk_name): + """ + Installs jdk using specific params, that comes from ambari-server + """ + import params + java_exec = format("{custom_java_home}/bin/java") if not os.path.isfile(java_exec): if not params.jdk_name: # if custom jdk is used. raise Fail(format("Unable to access {java_exec}. Confirm you have copied jdk to this host.")) - jdk_curl_target = format("{tmp_dir}/{jdk_name}") + jdk_curl_target = format("{tmp_dir}/{custom_jdk_name}") java_dir = os.path.dirname(params.java_home) Directory(params.artifact_dir, @@ -190,9 +199,13 @@ def setup_java(): ) File(jdk_curl_target, - content = DownloadSource(format("{jdk_location}/{jdk_name}")), + content = DownloadSource(format("{jdk_location}/{custom_jdk_name}")), not_if = format("test -f {jdk_curl_target}") - ) + ) + + File(jdk_curl_target, + mode = 0755, + ) tmp_java_dir = tempfile.mkdtemp(prefix="jdk_tmp_", dir=params.tmp_dir) @@ -205,7 +218,7 @@ def setup_java(): install_cmd = format("cd {tmp_java_dir} && tar -xf {jdk_curl_target} && {sudo} cp -rp {tmp_java_dir}/* {java_dir}") Directory(java_dir - ) + ) Execute(chmod_cmd, sudo = True, @@ -217,10 +230,10 @@ def setup_java(): finally: Directory(tmp_java_dir, action="delete") - File(format("{java_home}/bin/java"), + File(format("{custom_java_home}/bin/java"), mode=0755, cd_access="a", ) Execute(('chmod', '-R', '755', params.java_home), - sudo = True, - ) + sudo = True, + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py index a3830f7..a0259af 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py @@ -243,6 +243,10 @@ refresh_topology = False command_params = config["commandParams"] if "commandParams" in config else None if command_params is not None: refresh_topology = bool(command_params["refresh_topology"]) if "refresh_topology" in command_params else False + +ambari_java_home = default("/commandParams/ambari_java_home", None) +ambari_jdk_name = default("/commandParams/ambari_jdk_name", None) +ambari_jce_name = default("/commandParams/ambari_jce_name", None) ambari_libs_dir = "/var/lib/ambari-agent/lib" is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'] http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/shared_initialization.py index aed1124..5156dd4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/shared_initialization.py @@ -190,9 +190,17 @@ def create_microsoft_r_dir(): except Exception as exception: Logger.warning("Could not check the existence of {0} on DFS while starting {1}, exception: {2}".format(directory, params.current_service, str(exception))) - def setup_unlimited_key_jce_policy(): """ + Sets up the unlimited key JCE policy if needed. (sets up ambari JCE as well if ambari and the stack use different JDK) + """ + import params + __setup_unlimited_key_jce_policy(custom_java_home=params.java_home, custom_jdk_name=params.jdk_name, custom_jce_name = params.jce_policy_zip) + if params.ambari_jce_name and params.ambari_jce_name != params.jce_policy_zip: + __setup_unlimited_key_jce_policy(custom_java_home=params.ambari_java_home, custom_jdk_name=params.ambari_jdk_name, custom_jce_name = params.ambari_jce_name) + +def __setup_unlimited_key_jce_policy(custom_java_home, custom_jdk_name, custom_jce_name): + """ Sets up the unlimited key JCE policy if needed. The following criteria must be met: @@ -216,27 +224,27 @@ def setup_unlimited_key_jce_policy(): if params.sysprep_skip_setup_jce: Logger.info("Skipping unlimited key JCE policy check and setup since the host is sys prepped") - elif not params.jdk_name: + elif not custom_jdk_name: Logger.debug("Skipping unlimited key JCE policy check and setup since the Java VM is not managed by Ambari") elif not params.unlimited_key_jce_required: Logger.debug("Skipping unlimited key JCE policy check and setup since it is not required") else: - jcePolicyInfo = JcePolicyInfo(params.java_home) + jcePolicyInfo = JcePolicyInfo(custom_java_home) if jcePolicyInfo.is_unlimited_key_jce_policy(): Logger.info("The unlimited key JCE policy is required, and appears to have been installed.") - elif params.jce_policy_zip is None: + elif custom_jce_name is None: raise Fail("The unlimited key JCE policy needs to be installed; however the JCE policy zip is not specified.") else: Logger.info("The unlimited key JCE policy is required, and needs to be installed.") - jce_zip_target = format("{artifact_dir}/{jce_policy_zip}") - jce_zip_source = format("{ambari_server_resources_url}/{jce_policy_zip}") - java_security_dir = format("{java_home}/jre/lib/security") + jce_zip_target = format("{artifact_dir}/{custom_jce_name}") + jce_zip_source = format("{ambari_server_resources_url}/{custom_jce_name}") + java_security_dir = format("{custom_java_home}/jre/lib/security") Logger.debug("Downloading the unlimited key JCE policy files from {0} to {1}.".format(jce_zip_source, jce_zip_target)) Directory(params.artifact_dir, create_parents=True) http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/stacks/HDP/3.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/metainfo.xml index 0364d41..9af03c0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/metainfo.xml @@ -19,6 +19,6 @@ true - 1.7 + 1.8 1.8 http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-ANY/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-ANY/scripts/params.py index 2c2c901..e0e78b9 100644 --- a/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-ANY/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-ANY/scripts/params.py @@ -39,6 +39,9 @@ artifact_dir = format("{tmp_dir}/AMBARI-artifacts/") jdk_location = config['hostLevelParams']['jdk_location'] java_version = expect("/hostLevelParams/java_version", int) +ambari_java_home = default("/commandParams/ambari_java_home", None) +ambari_jdk_name = default("/commandParams/ambari_jdk_name", None) + service_name = config["serviceName"] component_name = config["role"] sudo = AMBARI_SUDO_BINARY \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-ANY/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-ANY/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-ANY/scripts/shared_initialization.py index 7dc1a48..0aae910 100644 --- a/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-ANY/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-ANY/scripts/shared_initialization.py @@ -40,17 +40,26 @@ def setup_users(): def setup_java(): """ - Installs jdk using specific params, that comes from ambari-server + Install jdk using specific params. + Install ambari jdk as well if the stack and ambari jdk are different. """ import params + __setup_java(custom_java_home=params.java_home, custom_jdk_name=params.jdk_name) + if params.ambari_java_home and params.ambari_java_home != params.java_home: + __setup_java(custom_java_home=params.ambari_java_home, custom_jdk_name=params.ambari_jdk_name) - java_exec = format("{java_home}/bin/java") +def __setup_java(custom_java_home, custom_jdk_name): + """ + Installs jdk using specific params, that comes from ambari-server + """ + import params + java_exec = format("{custom_java_home}/bin/java") if not os.path.isfile(java_exec): if not params.jdk_name: # if custom jdk is used. raise Fail(format("Unable to access {java_exec}. Confirm you have copied jdk to this host.")) - jdk_curl_target = format("{tmp_dir}/{jdk_name}") + jdk_curl_target = format("{tmp_dir}/{custom_jdk_name}") java_dir = os.path.dirname(params.java_home) Directory(params.artifact_dir, @@ -58,10 +67,14 @@ def setup_java(): ) File(jdk_curl_target, - content = DownloadSource(format("{jdk_location}/{jdk_name}")), + content = DownloadSource(format("{jdk_location}/{custom_jdk_name}")), not_if = format("test -f {jdk_curl_target}") ) + File(jdk_curl_target, + mode = 0755, + ) + tmp_java_dir = tempfile.mkdtemp(prefix="jdk_tmp_", dir=params.tmp_dir) try: @@ -85,7 +98,7 @@ def setup_java(): finally: Directory(tmp_java_dir, action="delete") - File(format("{java_home}/bin/java"), + File(format("{custom_java_home}/bin/java"), mode=0755, cd_access="a", ) http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java index 29b0476..1f906ad 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java @@ -271,7 +271,11 @@ public class ClientConfigResourceProviderTest { expect(configMap.get(Configuration.AMBARI_PYTHON_WRAP.getKey())).andReturn(Configuration.AMBARI_PYTHON_WRAP.getDefaultValue()); expect(configuration.getConfigsMap()).andReturn(returnConfigMap); expect(configuration.getResourceDirPath()).andReturn(stackRoot); + expect(configuration.getJavaHome()).andReturn("dummy_java_home"); + expect(configuration.getJDKName()).andReturn(null); + expect(configuration.getJCEName()).andReturn(null); expect(configuration.getJavaVersion()).andReturn(8); + expect(configuration.getStackJavaHome()).andReturn(null); expect(configuration.areHostsSysPrepped()).andReturn("false"); expect(configuration.isAgentStackRetryOnInstallEnabled()).andReturn("false"); expect(configuration.getAgentStackRetryOnInstallCount()).andReturn("5"); @@ -524,7 +528,11 @@ public class ClientConfigResourceProviderTest { expect(configMap.get(Configuration.AMBARI_PYTHON_WRAP.getKey())).andReturn(Configuration.AMBARI_PYTHON_WRAP.getDefaultValue()); expect(configuration.getConfigsMap()).andReturn(returnConfigMap); expect(configuration.getResourceDirPath()).andReturn("/var/lib/ambari-server/src/main/resources"); + expect(configuration.getJavaHome()).andReturn("dummy_java_home"); + expect(configuration.getJDKName()).andReturn(null); + expect(configuration.getJCEName()).andReturn(null); expect(configuration.getJavaVersion()).andReturn(8); + expect(configuration.getStackJavaHome()).andReturn(null); expect(configuration.areHostsSysPrepped()).andReturn("false"); expect(configuration.isAgentStackRetryOnInstallEnabled()).andReturn("false"); expect(configuration.getAgentStackRetryOnInstallCount()).andReturn("5"); http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java index b1cce55..c3b820b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java @@ -22,6 +22,7 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.getCurrentArguments; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.io.ByteArrayInputStream; @@ -615,6 +616,104 @@ public class StageUtilsTest extends EasyMockSupport { } } + @Test + public void testUseAmbariJdkWithoutavaHome() { + // GIVEN + Map commandParams = new HashMap<>(); + Configuration configuration = new Configuration(); + // WHEN + StageUtils.useAmbariJdkInCommandParams(commandParams, configuration); + // THEN + assertTrue(commandParams.isEmpty()); + } + + @Test + public void testUseAmbariJdkWithCustomJavaHome() { + // GIVEN + Map commandParams = new HashMap<>(); + Configuration configuration = new Configuration(); + configuration.setProperty("java.home", "myJavaHome"); + // WHEN + StageUtils.useAmbariJdkInCommandParams(commandParams, configuration); + // THEN + assertEquals("myJavaHome", commandParams.get("ambari_java_home")); + assertEquals(2, commandParams.size()); + } + + @Test + public void testUseAmbariJdk() { + // GIVEN + Map commandParams = new HashMap<>(); + Configuration configuration = new Configuration(); + configuration.setProperty("java.home", "myJavaHome"); + configuration.setProperty("jdk.name", "myJdkName"); + configuration.setProperty("jce.name", "myJceName"); + // WHEN + StageUtils.useAmbariJdkInCommandParams(commandParams, configuration); + // THEN + assertEquals("myJavaHome", commandParams.get("ambari_java_home")); + assertEquals("myJdkName", commandParams.get("ambari_jdk_name")); + assertEquals("myJceName", commandParams.get("ambari_jce_name")); + assertEquals(4, commandParams.size()); + } + + @Test + public void testUseStackJdkIfExistsWithCustomStackJdk() { + // GIVEN + Map hostLevelParams = new HashMap<>(); + Configuration configuration = new Configuration(); + configuration.setProperty("java.home", "myJavaHome"); + configuration.setProperty("jdk.name", "myJdkName"); + configuration.setProperty("jce.name", "myJceName"); + configuration.setProperty("stack.java.home", "myStackJavaHome"); + // WHEN + StageUtils.useStackJdkIfExists(hostLevelParams, configuration); + // THEN + assertEquals("myStackJavaHome", hostLevelParams.get("java_home")); + assertNull(hostLevelParams.get("jdk_name")); + assertNull(hostLevelParams.get("jce_name")); + assertEquals(4, hostLevelParams.size()); + } + + @Test + public void testUseStackJdkIfExists() { + // GIVEN + Map hostLevelParams = new HashMap<>(); + Configuration configuration = new Configuration(); + configuration.setProperty("java.home", "myJavaHome"); + configuration.setProperty("jdk.name", "myJdkName"); + configuration.setProperty("jce.name", "myJceName"); + configuration.setProperty("stack.java.home", "myStackJavaHome"); + configuration.setProperty("stack.jdk.name", "myStackJdkName"); + configuration.setProperty("stack.jce.name", "myStackJceName"); + configuration.setProperty("stack.java.version", "7"); + // WHEN + StageUtils.useStackJdkIfExists(hostLevelParams, configuration); + // THEN + assertEquals("myStackJavaHome", hostLevelParams.get("java_home")); + assertEquals("myStackJdkName", hostLevelParams.get("jdk_name")); + assertEquals("myStackJceName", hostLevelParams.get("jce_name")); + assertEquals("7", hostLevelParams.get("java_version")); + assertEquals(4, hostLevelParams.size()); + } + + @Test + public void testUseStackJdkIfExistsWithoutStackJdk() { + // GIVEN + Map hostLevelParams = new HashMap<>(); + Configuration configuration = new Configuration(); + configuration.setProperty("java.home", "myJavaHome"); + configuration.setProperty("jdk.name", "myJdkName"); + configuration.setProperty("jce.name", "myJceName"); + // WHEN + StageUtils.useStackJdkIfExists(hostLevelParams, configuration); + // THEN + assertEquals("myJavaHome", hostLevelParams.get("java_home")); + assertEquals("myJdkName", hostLevelParams.get("jdk_name")); + assertEquals("myJceName", hostLevelParams.get("jce_name")); + assertEquals(4, hostLevelParams.size()); + } + private void checkServiceHostIndexes(Map> info, String componentName, String mappedComponentName, Map> serviceTopology, List hostList) { Set expectedHostsList = new HashSet<>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/2f0de691/ambari-server/src/test/python/TestAmbariServer.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py index 8c135c3..c511237 100644 --- a/ambari-server/src/test/python/TestAmbariServer.py +++ b/ambari-server/src/test/python/TestAmbariServer.py @@ -108,11 +108,11 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur get_pass_file_path, GET_FQDN_SERVICE_URL, JDBC_USE_INTEGRATED_AUTH_PROPERTY, SECURITY_KEY_ENV_VAR_NAME, \ JAVA_HOME_PROPERTY, JDK_NAME_PROPERTY, JCE_NAME_PROPERTY, STACK_LOCATION_KEY, SERVER_VERSION_FILE_PATH, \ COMMON_SERVICES_PATH_PROPERTY, WEBAPP_DIR_PROPERTY, SHARED_RESOURCES_DIR, BOOTSTRAP_SCRIPT, \ - CUSTOM_ACTION_DEFINITIONS, BOOTSTRAP_SETUP_AGENT_SCRIPT, STACKADVISOR_SCRIPT, BOOTSTRAP_DIR_PROPERTY, MPACKS_STAGING_PATH_PROPERTY + CUSTOM_ACTION_DEFINITIONS, BOOTSTRAP_SETUP_AGENT_SCRIPT, STACKADVISOR_SCRIPT, BOOTSTRAP_DIR_PROPERTY, MPACKS_STAGING_PATH_PROPERTY, STACK_JAVA_VERSION from ambari_server.serverUtils import is_server_runing, refresh_stack_hash from ambari_server.serverSetup import check_selinux, check_ambari_user, proceedJDBCProperties, SE_STATUS_DISABLED, SE_MODE_ENFORCING, configure_os_settings, \ download_and_install_jdk, prompt_db_properties, setup, \ - AmbariUserChecks, AmbariUserChecksLinux, AmbariUserChecksWindows, JDKSetup, reset, setup_jce_policy, expand_jce_zip_file + AmbariUserChecks, AmbariUserChecksLinux, AmbariUserChecksWindows, JDKSetup, reset, setup_jce_policy, expand_jce_zip_file, check_ambari_java_version_is_valid from ambari_server.serverUpgrade import upgrade, change_objects_owner, \ run_schema_upgrade, move_user_custom_actions, find_and_copy_custom_services from ambari_server.setupHttps import is_valid_https_port, setup_https, import_cert_and_key_action, get_fqdn, \ @@ -2811,9 +2811,10 @@ class TestAmbariServer(TestCase): @patch("ambari_server.serverSetup.get_JAVA_HOME") @patch("ambari_server.serverSetup.get_resources_location") @patch("ambari_server.serverSetup.get_ambari_properties") + @patch("ambari_server.serverSetup.check_ambari_java_version_is_valid") @patch("shutil.copyfile") @patch("sys.exit") - def test_download_jdk(self, exit_mock, copyfile_mock, get_ambari_properties_mock, get_resources_location_mock, get_JAVA_HOME_mock, \ + def test_download_jdk(self, exit_mock, copyfile_mock, check_ambari_java_version_is_valid_mock, get_ambari_properties_mock, get_resources_location_mock, get_JAVA_HOME_mock, \ validate_jdk_mock, print_info_msg_mock, get_validated_string_input_mock, update_properties_mock, \ run_os_command_mock, get_YN_input_mock, force_download_file_mock, expand_jce_zip_file_mock, adjust_jce_permissions_mock, os_makedirs_mock, @@ -2874,6 +2875,7 @@ class TestAmbariServer(TestCase): get_JAVA_HOME_mock.return_value = False read_ambari_user_mock.return_value = "ambari" get_ambari_properties_mock.return_value = p + check_ambari_java_version_is_valid_mock.return_value = True # Test case: ambari.properties not found try: download_and_install_jdk(args) @@ -3166,6 +3168,47 @@ class TestAmbariServer(TestCase): pass @not_for_platform(PLATFORM_WINDOWS) + @patch("subprocess.Popen") + def test_check_ambari_java_version_is_valid(self, popenMock): + # case 1: jdk7 is picked for stacks + properties = Properties() + p = MagicMock() + p.communicate.return_value = ('7', None) + p.returncode = 0 + popenMock.return_value = p + result = check_ambari_java_version_is_valid('/usr/jdk64/jdk_1.7.0/', 'java', 8, properties) + self.assertEqual(properties.get_property(STACK_JAVA_VERSION), "7") + self.assertFalse(result) + + # case 2: jdk8 is picked for stacks + properties = Properties() + p.communicate.return_value = ('8', None) + p.returncode = 0 + result = check_ambari_java_version_is_valid('/usr/jdk64/jdk_1.8.0/', 'java', 8, properties) + self.assertFalse(properties.get_property(STACK_JAVA_VERSION)) + self.assertTrue(result) + + # case 3: return code is not 0 + p.returncode = 1 + try: + check_ambari_java_version_is_valid('/usr/jdk64/jdk_1.8.0/', 'java', 8, properties) + self.fail("Should throw exception") + except FatalException: + # expected + pass + + # case 4: unparseable response - type error + p.communicate.return_value = ('something else', None) + p.returncode = 0 + try: + check_ambari_java_version_is_valid('/usr/jdk64/jdk_1.8.0/', 'java', 8, properties) + self.fail("Should throw exception") + except FatalException as e: + # expected + self.assertEqual(e.code, 1) + pass + + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) @patch.object(LinuxDBMSConfig, "_setup_remote_server") @patch("ambari_server.dbConfiguration_linux.print_info_msg")