Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2C01E18A0D for ; Tue, 18 Aug 2015 03:07:52 +0000 (UTC) Received: (qmail 67624 invoked by uid 500); 18 Aug 2015 03:07:51 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 67594 invoked by uid 500); 18 Aug 2015 03:07:51 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 67584 invoked by uid 99); 18 Aug 2015 03:07:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 18 Aug 2015 03:07:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 893C0E0169; Tue, 18 Aug 2015 03:07:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jluniya@apache.org To: commits@ambari.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-6965: JDK download URLs in ambari.properties need to be parameterized (jluniya) Date: Tue, 18 Aug 2015 03:07:51 +0000 (UTC) Repository: ambari Updated Branches: refs/heads/branch-2.1.1 199345942 -> a0a12912c AMBARI-6965: JDK download URLs in ambari.properties need to be parameterized (jluniya) Conflicts: ambari-server/src/test/python/TestAmbariServer.py Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a0a12912 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a0a12912 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a0a12912 Branch: refs/heads/branch-2.1.1 Commit: a0a12912c648b8de3c7e5eafca2e1018f7d8cdf4 Parents: 1993459 Author: Jayush Luniya Authored: Mon Aug 17 19:59:19 2015 -0700 Committer: Jayush Luniya Committed: Mon Aug 17 20:07:41 2015 -0700 ---------------------------------------------------------------------- .../GenerateStackDefinition.py | 28 ++- .../pluggable_stack_definition/configs/PHD.json | 8 + .../python/pluggable_stack_definition/test.json | 199 ------------------- ambari-server/conf/unix/ambari.properties | 2 + ambari-server/conf/windows/ambari.properties | 2 + ambari-server/pom.xml | 6 +- .../python/ambari_server/serverConfiguration.py | 2 + .../main/python/ambari_server/serverSetup.py | 13 +- .../src/test/python/TestAmbariServer.py | 2 + 9 files changed, 55 insertions(+), 207 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a0a12912/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py b/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py index 691c9b0..6e0e255 100644 --- a/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py +++ b/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py @@ -67,7 +67,6 @@ class _named_dict(dict): else: dict.__getattr__(self, item) - def copy_tree(src, dest, exclude=None, post_copy=None): """ Copy files form src to dest. @@ -140,7 +139,6 @@ def process_replacements(file_path, config_data, stack_version_changes): target.write(file_data.encode('utf-8')) return file_path - def process_metainfo(file_path, config_data, stack_version_changes, common_services = []): tree = ET.parse(file_path) root = tree.getroot() @@ -364,8 +362,6 @@ def process_py_files(file_path, config_data, stack_version_changes): def process_xml_files(file_path, config_data, stack_version_changes): return process_replacements(file_path, config_data, stack_version_changes) - - class GeneratorHelper(object): def __init__(self, config_data, resources_folder, output_folder): self.config_data = config_data @@ -460,7 +456,6 @@ class GeneratorHelper(object): self.copy_common_services(parent_services) pass - def copy_resource_management(self): source_folder = join(os.path.abspath(join(self.resources_folder, "..", "..", "..", "..")), 'ambari-common', 'src', 'main', 'python', 'resource_management') @@ -476,6 +471,28 @@ class GeneratorHelper(object): copy_tree(source_folder, target_folder, ignored_files, post_copy=post_copy) + def copy_ambari_properties(self): + source_ambari_properties = join(os.path.abspath(join(self.resources_folder, "..", "..", "..", "..")), + 'ambari-server', 'conf', 'unix', "ambari.properties") + target_ambari_properties = join(self.output_folder, 'conf', 'unix', 'ambari.properties') + target_dirname = os.path.dirname(target_ambari_properties) + + if not os.path.exists(target_dirname): + os.makedirs(target_dirname) + propertyMap = {} + if "ambariProperties" in self.config_data: + propertyMap = self.config_data.ambariProperties + + with open(source_ambari_properties, 'r') as in_file: + with open(target_ambari_properties, 'w') as out_file: + for line in in_file: + property = line.split('=')[0] + if property in propertyMap: + out_file.write('='.join([property, propertyMap[property]])) + out_file.write(os.linesep) + else: + out_file.write(line) + def generate_ui_mapping(self): stack_name = self.config_data.stackName records = [] @@ -523,6 +540,7 @@ def main(argv): gen_helper.copy_stacks() gen_helper.copy_resource_management() gen_helper.copy_common_services() + gen_helper.copy_ambari_properties() gen_helper.generate_ui_mapping() http://git-wip-us.apache.org/repos/asf/ambari/blob/a0a12912/ambari-common/src/main/python/pluggable_stack_definition/configs/PHD.json ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/pluggable_stack_definition/configs/PHD.json b/ambari-common/src/main/python/pluggable_stack_definition/configs/PHD.json index 98ebd76..ca6f49c 100644 --- a/ambari-common/src/main/python/pluggable_stack_definition/configs/PHD.json +++ b/ambari-common/src/main/python/pluggable_stack_definition/configs/PHD.json @@ -13,6 +13,14 @@ "baseStackFolder": "HDP2" } ], + "ambariProperties": { + "jdk.download.supported" : "false", + "jce.download.supported" : "false", + "jdk1.7.url" : "http://DOWNLOAD_NOT_SUPPORTED", + "jdk1.8.url" : "http://DOWNLOAD_NOT_SUPPORTED", + "jdk1.7.jcpol-url" : "http://DOWNLOAD_NOT_SUPPORTED", + "jdk1.8.jcpol-url" : "http://DOWNLOAD_NOT_SUPPORTED" + }, "versions": [ { "version": "2.0.6", http://git-wip-us.apache.org/repos/asf/ambari/blob/a0a12912/ambari-common/src/main/python/pluggable_stack_definition/test.json ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/pluggable_stack_definition/test.json b/ambari-common/src/main/python/pluggable_stack_definition/test.json deleted file mode 100644 index 92da79e..0000000 --- a/ambari-common/src/main/python/pluggable_stack_definition/test.json +++ /dev/null @@ -1,199 +0,0 @@ -{ - "stackName": "PHD", - "baseStackName": "HDP", - "performCommonReplacements": true, - "textReplacements": [ - ["hdp-select", "distro-select"], - ["conf-select", "phd-conf-select"] - ], - "preservedText": ["ext-2.2", "HDP-oozie", "hdp.version", "HDP_VERSION"], - "uiMapping": [ - { - "stackVersionNumber": "3.0", - "sign": "<", - "baseStackFolder": "HDP2" - } - ], - "versions": [ - { - "version": "2.0.6", - "baseVersion": "2.0.6", - "active": "false", - "services": [ - { - "name": "HDFS" - - }, - { - "name": "ZOOKEEPER" - }, - { - "name": "HBASE" - }, - { - "name": "YARN" - }, - { - "name": "MAPREDUCE2" - }, - { - "name": "HIVE" - }, - { - "name": "TEZ" - }, - { - "name": "OOZIE" - }, - { - "name": "KNOX" - }, - { - "name": "PIG" - } - ] - }, - { - "version": "2.1", - "baseVersion": "2.1", - "active": "false", - "services": [ - { - "name": "HDFS" - }, - { - "name": "ZOOKEEPER" - }, - { - "name": "HBASE" - }, - { - "name": "YARN" - }, - { - "name": "MAPREDUCE2" - }, - { - "name": "HIVE" - }, - { - "name": "TEZ" - }, - { - "name": "OOZIE" - }, - { - "name": "KNOX" - }, - { - "name": "PIG" - } - ] - }, - { - "version": "3.0", - "baseVersion": "2.2", - "active": "true", - "services": [ - { - "name": "HDFS" - }, - { - "name": "ZOOKEEPER" - }, - { - "name": "HBASE", - "packages":[ - { - "family": "redhat6,suse11", - "packages": [ - "hbase_3_0_*" - ] - } - ] - }, - { - "name": "YARN" - }, - { - "name": "MAPREDUCE2" - }, - { - "name": "HIVE" - }, - { - "name": "TEZ" - }, - { - "name": "OOZIE", - "packages":[ - { - "family": "redhat6,suse11", - "packages": [ - "oozie_3_0_*" - ] - }, - { - "family": "any", - "packages": [ - "zip", - "extjs" - ] - } - ] - }, - { - "name": "KNOX", - "packages":[ - { - "family": "redhat6,suse11", - "packages": [ - "knox_3_0_*" - ] - } - ] - }, - { - "name": "PIG" - } - ] - }, - { - "version": "3.1", - "baseVersion": "2.3", - "active": "true", - "services": [ - { - "name": "HDFS" - }, - { - "name": "ZOOKEEPER" - }, - { - "name": "HBASE" - }, - { - "name": "YARN" - }, - { - "name": "MAPREDUCE2" - }, - { - "name": "HIVE" - }, - { - "name": "TEZ" - }, - { - "name": "OOZIE" - }, - { - "name": "KNOX" - }, - { - "name": "PIG" - } - ] - } - ] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/a0a12912/ambari-server/conf/unix/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/unix/ambari.properties b/ambari-server/conf/unix/ambari.properties index 9795bc7..d409d77 100644 --- a/ambari-server/conf/unix/ambari.properties +++ b/ambari-server/conf/unix/ambari.properties @@ -38,6 +38,8 @@ jdk1.8.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-8.zip jdk1.8.jcpol-file=jce_policy-8.zip jdk1.8.home=/usr/jdk64/ jdk1.8.re=(jdk.*)/jre +jdk.download.supported=true +jce.download.supported=true metadata.path=/var/lib/ambari-server/resources/stacks common.services.path=/var/lib/ambari-server/resources/common-services http://git-wip-us.apache.org/repos/asf/ambari/blob/a0a12912/ambari-server/conf/windows/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/windows/ambari.properties b/ambari-server/conf/windows/ambari.properties index 58c8d9f..a6a5aac 100644 --- a/ambari-server/conf/windows/ambari.properties +++ b/ambari-server/conf/windows/ambari.properties @@ -31,6 +31,8 @@ jdk1.7.67.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEP jdk1.7.67.jcpol-file=UnlimitedJCEPolicyJDK7.zip jdk1.7.67.home=C:\\jdk1.7.0_67 jdk1.7.67.re=(jdk.*)/jre +jdk.download.supported=true +jce.download.supported=true metadata.path=resources\\stacks common.services.path=resources\\common-services http://git-wip-us.apache.org/repos/asf/ambari/blob/a0a12912/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index 5725446..329b169 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -43,6 +43,7 @@ 1.5 ${project.basedir}/../ambari-common/src/main/python/resource_management ${basedir}/../ambari-web/app/data/custom_stack_map.js + conf/unix/ambari.properties target/classes/common-services target/classes/stacks/${stack.distribution} src/main/resources @@ -402,7 +403,7 @@ true - conf/unix/ambari.properties + ${ambariProperties} conf/unix/log4j.properties @@ -788,7 +789,7 @@ - conf/unix/ambari.properties + ${ambariProperties} file perm @@ -1526,6 +1527,7 @@ target/pluggable-stack-definition/custom_stack_map.js + target/pluggable-stack-definition/conf/unix/ambari.properties target/pluggable-stack-definition/python/resource_management target/pluggable-stack-definition/common-services target/pluggable-stack-definition/stacks/${stack.distribution} http://git-wip-us.apache.org/repos/asf/ambari/blob/a0a12912/ambari-server/src/main/python/ambari_server/serverConfiguration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py index f2661d3..7122fc4 100644 --- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py +++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py @@ -79,6 +79,8 @@ AMBARI_VERSION_VAR = "AMBARI_VERSION_VAR" JAVA_HOME_PROPERTY = "java.home" JDK_NAME_PROPERTY = "jdk.name" JCE_NAME_PROPERTY = "jce.name" +JDK_DOWNLOAD_SUPPORTED_PROPERTY = "jdk.download.supported" +JCE_DOWNLOAD_SUPPORTED_PROPERTY = "jce.download.supported" # JDBC JDBC_PATTERNS = {"oracle": "*ojdbc*.jar", "mysql": "*mysql*.jar", "mssql": "*sqljdbc*.jar"} http://git-wip-us.apache.org/repos/asf/ambari/blob/a0a12912/ambari-server/src/main/python/ambari_server/serverSetup.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverSetup.py b/ambari-server/src/main/python/ambari_server/serverSetup.py index 123ae3b..045e969 100644 --- a/ambari-server/src/main/python/ambari_server/serverSetup.py +++ b/ambari-server/src/main/python/ambari_server/serverSetup.py @@ -39,7 +39,7 @@ from ambari_server.serverConfiguration import configDefaults, JDKRelease, \ get_resources_location, get_value_from_properties, read_ambari_user, update_properties, validate_jdk, write_property, \ JAVA_HOME, JAVA_HOME_PROPERTY, JCE_NAME_PROPERTY, JDBC_RCA_URL_PROPERTY, JDBC_URL_PROPERTY, \ JDK_NAME_PROPERTY, JDK_RELEASES, NR_USER_PROPERTY, OS_FAMILY, OS_FAMILY_PROPERTY, OS_TYPE, OS_TYPE_PROPERTY, OS_VERSION, \ - VIEWS_DIR_PROPERTY, JDBC_DATABASE_PROPERTY + VIEWS_DIR_PROPERTY, JDBC_DATABASE_PROPERTY, JDK_DOWNLOAD_SUPPORTED_PROPERTY, JCE_DOWNLOAD_SUPPORTED_PROPERTY from ambari_server.serverUtils import is_server_runing from ambari_server.setupSecurity import adjust_directory_permissions from ambari_server.userInput import get_YN_input, get_validated_string_input @@ -472,6 +472,12 @@ class JDKSetup(object): dest_file = os.path.abspath(os.path.join(resources_dir, jdk_cfg.dest_file)) if os.path.exists(dest_file): print "JDK already exists, using " + dest_file + elif properties[JDK_DOWNLOAD_SUPPORTED_PROPERTY].upper() == "FALSE": + print "ERROR: Oracle JDK is not found in {1}. JDK download is not supported in this distribution. Please download Oracle JDK " \ + "archive ({0}) manually from Oracle site, place it into {1} and re-run this script.".format(jdk_cfg.dest_file, dest_file) + print "NOTE: If you have already downloaded the file, please verify if the name is exactly same as {0}.".format(jdk_cfg.dest_file) + print 'Exiting...' + sys.exit(1) else: ok = get_YN_input("To download the Oracle JDK and the Java Cryptography Extension (JCE) " "Policy Files you must accept the " @@ -606,6 +612,11 @@ class JDKSetup(object): dest_file = os.path.abspath(os.path.join(resources_dir, dest_jcpol_file)) if not os.path.exists(dest_file): + if properties[JCE_DOWNLOAD_SUPPORTED_PROPERTY].upper() == "FALSE": + print "ERROR: JCE Policy archive is not found in {1}. JCE Policy archive download is not supported in this distribution. " \ + "Please download JCE Policy archive ({0}) from Oracle site, place it into {1} and re-run this script.".format(dest_jcpol_file, dest_file) + print 'Exiting...' + sys.exit(1) print 'Downloading JCE Policy archive from ' + jcpol_url + ' to ' + dest_file try: force_download_file(jcpol_url, dest_file) http://git-wip-us.apache.org/repos/asf/ambari/blob/a0a12912/ambari-server/src/test/python/TestAmbariServer.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py index db16927..c826e9e 100644 --- a/ambari-server/src/test/python/TestAmbariServer.py +++ b/ambari-server/src/test/python/TestAmbariServer.py @@ -2112,6 +2112,8 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV p.process_pair("jdk1.jcpol-file", "some-jcpol.tar.gz") p.process_pair("jdk1.home", "/jdk1") p.process_pair("jdk1.re", "(jdk.*)/jre") + p.process_pair("jdk.download.supported", "true") + p.process_pair("jce.download.supported", "true") validate_jdk_mock.return_value = False path_existsMock.return_value = False