Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0B1C618D0B for ; Tue, 12 Jan 2016 18:55:12 +0000 (UTC) Received: (qmail 19254 invoked by uid 500); 12 Jan 2016 18:55:08 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 19152 invoked by uid 500); 12 Jan 2016 18:55:08 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 18993 invoked by uid 99); 12 Jan 2016 18:55:08 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Jan 2016 18:55:08 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 84961E2C2A; Tue, 12 Jan 2016 18:55:08 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: yusaku@apache.org To: commits@ambari.apache.org Message-Id: <62d30f9b0ef4404698de01809a09eded@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-14550. Ranger KMS install from Ambari - Add ability to skip DB/DB user setup. (gautam via yusaku) Date: Tue, 12 Jan 2016 18:55:08 +0000 (UTC) Repository: ambari Updated Branches: refs/heads/branch-2.2 5da8c1bae -> 84af9c49d AMBARI-14550. Ranger KMS install from Ambari - Add ability to skip DB/DB user setup. (gautam via yusaku) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/84af9c49 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/84af9c49 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/84af9c49 Branch: refs/heads/branch-2.2 Commit: 84af9c49d0f3b77db5f06d176d9baccfcb6191cf Parents: 5da8c1b Author: Yusaku Sako Authored: Tue Jan 12 10:54:45 2016 -0800 Committer: Yusaku Sako Committed: Tue Jan 12 10:54:45 2016 -0800 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog221.java | 46 +++ .../0.5.0.2.3/configuration/dbks-site.xml | 38 ++- .../0.5.0.2.3/configuration/kms-env.xml | 22 ++ .../0.5.0.2.3/configuration/kms-properties.xml | 84 ++++- .../configuration/ranger-kms-audit.xml | 1 + .../configuration/ranger-kms-policymgr-ssl.xml | 2 + .../RANGER_KMS/0.5.0.2.3/package/scripts/kms.py | 31 +- .../0.5.0.2.3/package/scripts/params.py | 9 + .../HDP/2.3/services/RANGER_KMS/metainfo.xml | 8 + .../RANGER_KMS/themes/theme_version_1.json | 303 +++++++++++++++++++ .../stacks/HDP/2.3/services/stack_advisor.py | 36 +++ .../server/upgrade/UpgradeCatalog221Test.java | 44 +++ .../test/python/stacks/2.3/configs/default.json | 4 + .../models/configs/objects/service_config.js | 2 +- ambari-web/app/models/stack_service.js | 5 + 15 files changed, 621 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java index b41bee4..9fce3cf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java @@ -81,6 +81,13 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog { private static final String BLUEPRINT_HOSTGROUP_COMPONENT_TABLE_NAME = "hostgroup_component"; private static final String BLUEPRINT_PROVISION_ACTION_COLUMN_NAME = "provision_action"; + private static final String RANGER_KMS_DBKS_CONFIG = "dbks-site"; + private static final String RANGER_KMS_DB_FLAVOR = "DB_FLAVOR"; + private static final String RANGER_KMS_DB_HOST = "db_host"; + private static final String RANGER_KMS_DB_NAME = "db_name"; + private static final String RANGER_KMS_JDBC_URL = "ranger.ks.jpa.jdbc.url"; + private static final String RANGER_KMS_JDBC_DRIVER = "ranger.ks.jpa.jdbc.driver"; + private static final String RANGER_KMS_PROPERTIES = "kms-properties"; @@ -148,6 +155,7 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog { addNewConfigurationsFromXml(); updateAlerts(); updateOozieConfigs(); + updateRangerKmsDbksConfigs(); } protected void updateAlerts() { @@ -355,4 +363,42 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog { } } + protected void updateRangerKmsDbksConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + + for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { + Map newRangerKmsProps = new HashMap<>(); + Config rangerKmsDbConfigs = cluster.getDesiredConfigByType(RANGER_KMS_PROPERTIES); + if (rangerKmsDbConfigs != null) { + String dbFlavor = rangerKmsDbConfigs.getProperties().get(RANGER_KMS_DB_FLAVOR); + String dbHost = rangerKmsDbConfigs.getProperties().get(RANGER_KMS_DB_HOST); + String dbName = rangerKmsDbConfigs.getProperties().get(RANGER_KMS_DB_NAME); + String dbConnectionString = null; + String dbDriver = null; + + if (dbFlavor != null && dbHost != null && dbName != null) { + if ("MYSQL".equalsIgnoreCase(dbFlavor)) { + dbConnectionString = "jdbc:mysql://"+dbHost+"/"+dbName; + dbDriver = "com.mysql.jdbc.Driver"; + } else if ("ORACLE".equalsIgnoreCase(dbFlavor)) { + dbConnectionString = "jdbc:oracle:thin:@//"+dbHost; + dbDriver = "oracle.jdbc.driver.OracleDriver"; + } else if ("POSTGRES".equalsIgnoreCase(dbFlavor)) { + dbConnectionString = "jdbc:postgresql://"+dbHost+"/"+dbName; + dbDriver = "org.postgresql.Driver"; + } else if ("MSSQL".equalsIgnoreCase(dbFlavor)) { + dbConnectionString = "jdbc:sqlserver://"+dbHost+";databaseName="+dbName; + dbDriver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; + } else if ("SQLA".equalsIgnoreCase(dbFlavor)) { + dbConnectionString = "jdbc:sqlanywhere:database="+dbName+";host="+dbHost; + dbDriver = "sap.jdbc4.sqlanywhere.IDriver"; + } + newRangerKmsProps.put(RANGER_KMS_JDBC_URL, dbConnectionString); + newRangerKmsProps.put(RANGER_KMS_JDBC_DRIVER, dbDriver); + updateConfigurationPropertiesForCluster(cluster, RANGER_KMS_DBKS_CONFIG, newRangerKmsProps, true, false); + } + } + } + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/dbks-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/dbks-site.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/dbks-site.xml index 8291e02..45bbbd6 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/dbks-site.xml +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/dbks-site.xml @@ -30,13 +30,32 @@ ranger.db.encrypt.key.password _ + PASSWORD Password used for encrypting Master Key ranger.ks.jpa.jdbc.url - {{db_jdbc_url}} + JDBC connect string + jdbc:mysql://localhost URL for Database + + false + + + + kms-properties + DB_FLAVOR + + + kms-properties + db_host + + + kms-properties + db_name + + @@ -48,6 +67,7 @@ ranger.ks.jpa.jdbc.password _ + PASSWORD Database user's password @@ -77,13 +97,23 @@ ranger.ks.jpa.jdbc.driver - {{db_jdbc_driver}} - Driver used for database + Driver class name for a JDBC Ranger KMS database + com.mysql.jdbc.Driver + Driver used for database + + false + + + + kms-properties + DB_FLAVOR + + ranger.ks.jdbc.sqlconnectorjar - {{driver_curl_target}} + {{ews_lib_jar_path}} Driver used for database http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml index 10c14ac..c2c98c6 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml @@ -48,4 +48,26 @@ + + create_db_user + Setup Database and Database User + true + If set to Yes, Ambari will create and setup Ranger Database and Database User. This will require to specify Database Admin user and password + + false + value-list + + + true + + + + false + + + + 1 + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-properties.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-properties.xml index 6c37b94..c84615a 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-properties.xml +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-properties.xml @@ -22,12 +22,14 @@ REPOSITORY_CONFIG_USERNAME + Repository config username keyadmin REPOSITORY_CONFIG_PASSWORD + Repository config password keyadmin PASSWORD @@ -35,59 +37,125 @@ DB_FLAVOR + DB FLAVOR MYSQL - + The database type to be used + + false + value-list + + + MYSQL + + + + ORACLE + + + + POSTGRES + + + + MSSQL + + + + SQLA + + + + 1 + SQL_CONNECTOR_JAR + SQL connector jar /usr/share/java/mysql-connector-java.jar Location of DB client library (please check the location of the jar file) + + false + + + + kms-properties + DB_FLAVOR + + db_root_user + Database Administrator (DBA) username root - + Database admin user. This user should have DBA permission to create the Ranger Database and Ranger Database User + + false + db_root_password + Database Administrator (DBA) password PASSWORD - + Database password for the database admin username + + false + db_host - localhost - + Ranger KMS DB host + + Database host + + false + db_name + Ranger KMS DB name rangerkms - + Database name + + false + db_user + Ranger KMS DB username rangerkms - + Database username used for the Ranger KMS schema + + false + db_password + Ranger KMS DB password PASSWORD - + Database password for the Ranger KMS schema + + false + KMS_MASTER_KEY_PASSWD + KMS master key password PASSWORD + + false + http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-audit.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-audit.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-audit.xml index d5b0aa7..c155531 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-audit.xml +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-audit.xml @@ -51,6 +51,7 @@ xasecure.audit.destination.db.password crypted + PASSWORD Audit DB JDBC Password http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-policymgr-ssl.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-policymgr-ssl.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-policymgr-ssl.xml index fb4a4e6..9b38f7f 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-policymgr-ssl.xml +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-policymgr-ssl.xml @@ -29,6 +29,7 @@ xasecure.policymgr.clientssl.keystore.password myKeyFilePassword + PASSWORD password for keystore @@ -41,6 +42,7 @@ xasecure.policymgr.clientssl.truststore.password changeit + PASSWORD java truststore password http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py index 79353e6..7c7a7c8 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py @@ -37,6 +37,7 @@ from resource_management.core.utils import PasswordString from resource_management.core.shell import as_sudo import re import time +import socket def password_validation(password, key): import params @@ -114,7 +115,12 @@ def setup_kms_db(): dba_setup = format('python {kms_home}/dba_script.py -q') db_setup = format('python {kms_home}/db_setup.py') - Execute(dba_setup, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10) + if params.create_db_user: + Logger.info('Setting up Ranger KMS DB and DB User') + Execute(dba_setup, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10) + else: + Logger.info('Separate DBA property not set. Assuming Ranger KMS DB and DB User exists!') + Execute(db_setup, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10) def setup_java_patch(): @@ -174,6 +180,23 @@ def kms(): recursive = True ) + File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), + content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), + mode = 0644, + ) + + cp = format("{check_db_connection_jar}") + cp = cp + os.pathsep + format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}") + + db_connection_check_command = format( + "{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_kms_jdbc_connection_url}' {db_user} {db_password!p} {ranger_kms_jdbc_driver}") + + env_dict = {} + if params.db_flavor.lower() == 'sqla': + env_dict = {'LD_LIBRARY_PATH':params.ld_library_path} + + Execute(db_connection_check_command, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10, environment=env_dict) + if params.xa_audit_db_is_enabled: File(params.downloaded_connector_path, content = DownloadSource(params.driver_source), @@ -402,6 +425,9 @@ def create_repo(url, data, usernamepassword): else: Logger.error("Error creating service. Reason - {0}.".format(e.reason)) return False + except socket.timeout as e: + Logger.error("Error creating service. Reason - {0}".format(e)) + return False def get_repo(url, name, usernamepassword): try: @@ -432,3 +458,6 @@ def get_repo(url, name, usernamepassword): else: Logger.error("Error getting {0} service. Reason - {1}.".format(name, e.reason)) return False + except socket.timeout as e: + Logger.error("Error getting service. Reason - {0}".format(e)) + return False http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py index 2e2e52d..95f0896 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py @@ -52,6 +52,8 @@ has_ranger_admin = len(ranger_admin_hosts) > 0 kms_host = config['clusterHostInfo']['ranger_kms_server_hosts'][0] kms_port = config['configurations']['kms-env']['kms_port'] +create_db_user = config['configurations']['kms-env']['create_db_user'] + #kms properties db_flavor = (config['configurations']['kms-properties']['DB_FLAVOR']).lower() db_host = config['configurations']['kms-properties']['db_host'] @@ -123,6 +125,7 @@ downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}") +ews_lib_jar_path = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}") if db_flavor == 'sqla': downloaded_custom_connector = format("{tmp_dir}/sqla-client-jdbc.tar.gz") @@ -191,7 +194,13 @@ ssl_truststore_password = unicode(config['configurations']['ranger-kms-policymgr #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla': xa_audit_db_is_enabled = False + current_host = config['hostname'] ranger_kms_hosts = config['clusterHostInfo']['ranger_kms_server_hosts'] if current_host in ranger_kms_hosts: kms_host = current_host + +check_db_connection_jar_name = "DBConnectionVerification.jar" +check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}") +ranger_kms_jdbc_connection_url = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.url'] +ranger_kms_jdbc_driver = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.driver'] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/metainfo.xml index b1a5584..5606a26 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/metainfo.xml @@ -42,6 +42,14 @@ + + + + theme_version_1.json + true + + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/themes/theme_version_1.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/themes/theme_version_1.json b/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/themes/theme_version_1.json new file mode 100644 index 0000000..c08a56c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/themes/theme_version_1.json @@ -0,0 +1,303 @@ +{ + "name": "default", + "description": "Default theme for Ranger KMS service", + "configuration": { + "layouts": [ + { + "name": "default", + "tabs": [ + { + "name": "db_settings", + "display-name": "Settings", + "layout": { + "tab-columns": "2", + "tab-rows": "2", + "sections": [ + { + "name": "section-db-settings", + "display-name": "", + "row-index": "0", + "column-index": "0", + "row-span": "4", + "column-span": "2", + "section-columns": "2", + "section-rows": "4", + "subsections": [ + { + "name": "subsection-kms-db-row1-col1", + "display-name": "Ranger KMS DB", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + }, + { + "name": "subsection-kms-db-row1-col2", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1" + }, + { + "name": "subsection-kms-create-db-user-row2-col", + "display-name": "Setup Database and Database User", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "2" + }, + { + "name": "subsection-kms-db-root-user-row3-col1", + "display-name": "Ranger KMS Root DB", + "row-index": "2", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "depends-on": [ + { + "configs":[ + "kms-env/create_db_user" + ], + "if": "${kms-env/create_db_user}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "name": "subsection-kms-db-root-user-row3-col2", + "row-index": "2", + "column-index": "1", + "row-span": "1", + "column-span": "1", + "depends-on": [ + { + "configs":[ + "kms-env/create_db_user" + ], + "if": "${kms-env/create_db_user}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "name": "subsection-kms-master-row4-col", + "display-name": "KMS Master Secret Password", + "row-index": "3", + "column-index": "0", + "row-span": "1", + "column-span": "2" + } + ] + } + ] + } + } + ] + } + ], + "placement": { + "configuration-layout": "default", + "configs": [ + { + "config": "kms-properties/DB_FLAVOR", + "subsection-name": "subsection-kms-db-row1-col1" + }, + { + "config": "kms-properties/db_name", + "subsection-name": "subsection-kms-db-row1-col1" + }, + { + "config": "dbks-site/ranger.ks.jpa.jdbc.url", + "subsection-name": "subsection-kms-db-row1-col1" + }, + { + "config": "kms-properties/db_user", + "subsection-name": "subsection-kms-db-row1-col1" + }, + { + "config": "kms-properties/db_host", + "subsection-name": "subsection-kms-db-row1-col2" + }, + { + "config": "kms-properties/SQL_CONNECTOR_JAR", + "subsection-name": "subsection-kms-db-row1-col2", + "depends-on" : [ + { + "configs":[ + "kms-properties/DB_FLAVOR" + ], + "if": "${kms-properties/DB_FLAVOR} === SQLA", + "then": { + "property_value_attributes": { + "visible": false + } + }, + "else": { + "property_value_attributes": { + "visible": true + } + } + } + ] + }, + { + "config": "dbks-site/ranger.ks.jpa.jdbc.driver", + "subsection-name": "subsection-kms-db-row1-col2" + }, + { + "config": "kms-properties/db_password", + "subsection-name": "subsection-kms-db-row1-col2" + }, + { + "config": "kms-properties/db_root_user", + "subsection-name": "subsection-kms-db-root-user-row3-col1" + }, + { + "config": "kms-properties/db_root_password", + "subsection-name": "subsection-kms-db-root-user-row3-col2" + }, + { + "config": "kms-properties/KMS_MASTER_KEY_PASSWD", + "subsection-name": "subsection-kms-master-row4-col" + }, + { + "config" : "kms-env/create_db_user", + "subsection-name": "subsection-kms-create-db-user-row2-col" + }, + { + "config": "kms-env/test_db_kms_connection", + "subsection-name": "subsection-kms-create-db-user-row2-col", + "property_value_attributes": { + "ui_only_property": true + }, + "depends-on": [ + { + "configs":[ + "kms-env/create_db_user" + ], + "if": "${kms-env/create_db_user}", + "then": { + "property_value_attributes": { + "visible": false + } + }, + "else": { + "property_value_attributes": { + "visible": true + } + } + } + ] + } + ] + }, + "widgets": [ + { + "config": "kms-properties/DB_FLAVOR", + "widget": { + "type": "combo" + } + }, + { + "config": "kms-properties/db_user", + "widget": { + "type": "text-field" + } + }, + { + "config": "kms-properties/db_name", + "widget": { + "type": "text-field" + } + }, + { + "config": "kms-properties/SQL_CONNECTOR_JAR", + "widget": { + "type": "text-field" + } + }, + { + "config": "kms-properties/db_root_user", + "widget": { + "type": "text-field" + } + }, + { + "config": "kms-properties/db_host", + "widget": { + "type": "text-field" + } + }, + { + "config": "kms-properties/db_password", + "widget": { + "type": "password" + } + }, + { + "config": "kms-properties/db_root_password", + "widget": { + "type": "password" + } + }, + { + "config": "kms-properties/KMS_MASTER_KEY_PASSWD", + "widget": { + "type": "password" + } + }, + { + "config": "kms-env/create_db_user", + "widget": { + "type": "toggle" + } + }, + { + "config": "kms-env/test_db_kms_connection", + "widget": { + "type": "test-db-connection", + "display-name": "Test Connection", + "required-properties": { + "jdbc.driver.class": "dbks-site/ranger.ks.jpa.jdbc.driver", + "jdbc.driver.url": "dbks-site/ranger.ks.jpa.jdbc.url", + "db.connection.source.host": "ranger_kms-site/ranger_kms_server_hosts", + "db.type": "kms-properties/DB_FLAVOR", + "db.connection.destination.host": "kms-properties/db_host", + "db.connection.user": "kms-properties/db_user", + "db.connection.password": "kms-properties/db_password" + } + } + }, + { + "config": "dbks-site/ranger.ks.jpa.jdbc.driver", + "widget" : { + "type": "text-field" + } + }, + { + "config": "dbks-site/ranger.ks.jpa.jdbc.url", + "widget": { + "type": "text-field" + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 13cb817..46768e1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -135,6 +135,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor): "HBASE": self.recommendHBASEConfigurations, "KAFKA": self.recommendKAFKAConfigurations, "RANGER": self.recommendRangerConfigurations, + "RANGER_KMS": self.recommendRangerKMSConfigurations, "HAWQ": self.recommendHAWQConfigurations } parentRecommendConfDict.update(childRecommendConfDict) @@ -427,6 +428,41 @@ class HDP23StackAdvisor(HDP22StackAdvisor): elif not security_enabled: putKafkaBrokerAttributes('authorizer.class.name', 'delete', 'true') + def recommendRangerKMSConfigurations(self, configurations, clusterData, services, hosts): + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + putRangerKmsDbksProperty = self.putProperty(configurations, "dbks-site", services) + putRangerKmsProperty = self.putProperty(configurations, "kms-properties", services) + + if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']): + + rangerKmsDbFlavor = services['configurations']["kms-properties"]["properties"]["DB_FLAVOR"] + ranger_kms_sql_connector_dict = { + 'MYSQL': '/usr/share/java/mysql-connector-java.jar', + 'ORACLE': '/usr/share/java/ojdbc6.jar', + 'POSTGRES': '/usr/share/java/postgresql.jar', + 'MSSQL': '/usr/share/java/sqljdbc4.jar', + 'SQLA': '/path_to_driver/sqla-client-jdbc.tar.gz' + } + + rangerKmsSqlConnectorProperty = ranger_kms_sql_connector_dict.get(rangerKmsDbFlavor, ranger_kms_sql_connector_dict['MYSQL']) + putRangerKmsProperty('SQL_CONNECTOR_JAR', rangerKmsSqlConnectorProperty) + + if ('db_host' in services['configurations']['kms-properties']['properties']) and ('db_name' in services['configurations']['kms-properties']['properties']): + + rangerKmsDbHost = services['configurations']["kms-properties"]["properties"]["db_host"] + rangerKmsDbName = services['configurations']["kms-properties"]["properties"]["db_name"] + + ranger_kms_db_url_dict = { + 'MYSQL': {'ranger.ks.jpa.jdbc.driver': 'com.mysql.jdbc.Driver', 'ranger.ks.jpa.jdbc.url': 'jdbc:mysql://' + rangerKmsDbHost + '/' + rangerKmsDbName}, + 'ORACLE': {'ranger.ks.jpa.jdbc.driver': 'oracle.jdbc.driver.OracleDriver', 'ranger.ks.jpa.jdbc.url': 'jdbc:oracle:thin:@//' + rangerKmsDbHost}, + 'POSTGRES': {'ranger.ks.jpa.jdbc.driver': 'org.postgresql.Driver', 'ranger.ks.jpa.jdbc.url': 'jdbc:postgresql://' + rangerKmsDbHost + '/' + rangerKmsDbName}, + 'MSSQL': {'ranger.ks.jpa.jdbc.driver': 'com.microsoft.sqlserver.jdbc.SQLServerDriver', 'ranger.ks.jpa.jdbc.url': 'jdbc:sqlserver://' + rangerKmsDbHost + ';databaseName=' + rangerKmsDbName}, + 'SQLA': {'ranger.ks.jpa.jdbc.driver': 'sap.jdbc4.sqlanywhere.IDriver', 'ranger.ks.jpa.jdbc.url': 'jdbc:sqlanywhere:host=' + rangerKmsDbHost + ';database=' + rangerKmsDbName} + } + + rangerKmsDbProperties = ranger_kms_db_url_dict.get(rangerKmsDbFlavor, ranger_kms_db_url_dict['MYSQL']) + for key in rangerKmsDbProperties: + putRangerKmsDbksProperty(key, rangerKmsDbProperties.get(key)) def recommendRangerConfigurations(self, configurations, clusterData, services, hosts): super(HDP23StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData, services, hosts) http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java index eba3391..cc4f641 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java @@ -149,11 +149,13 @@ public class UpgradeCatalog221Test { Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml"); Method updateAlerts = UpgradeCatalog221.class.getDeclaredMethod("updateAlerts"); Method updateOozieConfigs = UpgradeCatalog221.class.getDeclaredMethod("updateOozieConfigs"); + Method updateRangerKmsDbksConfigs = UpgradeCatalog221.class.getDeclaredMethod("updateRangerKmsDbksConfigs"); UpgradeCatalog221 upgradeCatalog221 = createMockBuilder(UpgradeCatalog221.class) .addMockedMethod(addNewConfigurationsFromXml) .addMockedMethod(updateAlerts) .addMockedMethod(updateOozieConfigs) + .addMockedMethod(updateRangerKmsDbksConfigs) .createMock(); upgradeCatalog221.addNewConfigurationsFromXml(); @@ -162,6 +164,8 @@ public class UpgradeCatalog221Test { expectLastCall().once(); upgradeCatalog221.updateOozieConfigs(); expectLastCall().once(); + upgradeCatalog221.updateRangerKmsDbksConfigs(); + expectLastCall().once(); replay(upgradeCatalog221); @@ -253,6 +257,46 @@ public class UpgradeCatalog221Test { } @Test + public void testUpdateRangerKmsDbksConfigs() throws Exception { + EasyMockSupport easyMockSupport = new EasyMockSupport(); + final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class); + final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class); + final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class); + + final Map propertiesRangerKmsDbConfigs = new HashMap(); + propertiesRangerKmsDbConfigs.put("DB_FLAVOR", "MYSQL"); + propertiesRangerKmsDbConfigs.put("db_host", "localhost"); + propertiesRangerKmsDbConfigs.put("db_name", "testdb"); + + final Config mockrangerKmsDbConfigs = easyMockSupport.createNiceMock(Config.class); + + final Injector mockInjector = Guice.createInjector(new AbstractModule() { + @Override + protected void configure() { + bind(AmbariManagementController.class).toInstance(mockAmbariManagementController); + bind(Clusters.class).toInstance(mockClusters); + bind(EntityManager.class).toInstance(entityManager); + + bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class)); + bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class)); + } + }); + + expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once(); + expect(mockClusters.getClusters()).andReturn(new HashMap() {{ + put("normal", mockClusterExpected); + }}).atLeastOnce(); + + expect(mockClusterExpected.getDesiredConfigByType("kms-properties")).andReturn(mockrangerKmsDbConfigs).atLeastOnce(); + expect(mockrangerKmsDbConfigs.getProperties()).andReturn(propertiesRangerKmsDbConfigs).times(3); + + easyMockSupport.replayAll(); + mockInjector.getInstance(UpgradeCatalog221.class).updateRangerKmsDbksConfigs(); + easyMockSupport.verifyAll(); + + } + + @Test public void testUpdateAmsHbaseSiteConfigs() throws Exception { Map clusterEnvProperties = new HashMap(); http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-server/src/test/python/stacks/2.3/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json index 21bff13..986c282 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json @@ -204,6 +204,10 @@ }, "ranger-env": { "xml_configurations_supported" : "true" + }, + "dbks-site": { + "ranger.ks.jpa.jdbc.url": "{{db_jdbc_url}}", + "ranger.ks.jpa.jdbc.driver": "{{db_jdbc_driver}}" } }, "configuration_attributes": { http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-web/app/models/configs/objects/service_config.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/models/configs/objects/service_config.js b/ambari-web/app/models/configs/objects/service_config.js index 5d827b7..71b05e6 100644 --- a/ambari-web/app/models/configs/objects/service_config.js +++ b/ambari-web/app/models/configs/objects/service_config.js @@ -76,7 +76,7 @@ App.ServiceConfig = Ember.Object.extend({ } }); return masterErrors + slaveErrors + overrideErrors + enhancedConfigsErrors; - }.property('configs.@each.isValid', 'configs.@each.isVisible', 'configCategories.@each.slaveErrorCount', 'configs.@each.overrideErrorTrigger'), + }.property('configs.@each.isValid', 'configs.@each.isVisible', 'configs.@each.hiddenBySection', 'configCategories.@each.slaveErrorCount', 'configs.@each.overrideErrorTrigger'), /** * checks if for example for kdc_type, the value isn't just the pretty version of the saved value, for example mit-kdc http://git-wip-us.apache.org/repos/asf/ambari/blob/84af9c49/ambari-web/app/models/stack_service.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/models/stack_service.js b/ambari-web/app/models/stack_service.js index 957d7ae..0174530 100644 --- a/ambari-web/app/models/stack_service.js +++ b/ambari-web/app/models/stack_service.js @@ -354,6 +354,11 @@ App.StackService.configCategories = function () { App.ServiceConfigCategory.create({ name: 'KnoxSSOSettings', displayName: 'Knox SSO Settings'}) ]); break; + case 'RANGER_KMS': + serviceConfigCategories.pushObjects([ + App.ServiceConfigCategory.create({ name: 'RANGER_KMS_SERVER', displayName: 'Ranger KMS Server', showHost: true}) + ]); + break; case 'ACCUMULO': serviceConfigCategories.pushObjects([ App.ServiceConfigCategory.create({ name: 'General', displayName: 'General'})