ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vbrodets...@apache.org
Subject ambari git commit: AMBARI-12274. UpgradeHelper script for HDP 2.2 to HDP 2.3 manual upgrade.(vbrodetskyi)
Date Fri, 03 Jul 2015 10:33:46 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 89b0c0a4b -> 73e86cb6d


AMBARI-12274. UpgradeHelper script for HDP 2.2 to HDP 2.3 manual upgrade.(vbrodetskyi)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/73e86cb6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/73e86cb6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/73e86cb6

Branch: refs/heads/branch-2.1
Commit: 73e86cb6da04e1bc5f2e62ffc351ee85bf5ad3bc
Parents: 89b0c0a
Author: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Authored: Fri Jul 3 12:59:59 2015 +0300
Committer: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Committed: Fri Jul 3 12:59:59 2015 +0300

----------------------------------------------------------------------
 ambari-server/src/main/python/upgradeHelper.py  |  129 +++
 .../catalog/UpgradeCatalog_2.1_to_2.3.json      |    5 +-
 .../catalog/UpgradeCatalog_2.2_to_2.3.json      | 1086 +++++++++++++++++-
 .../src/test/python/TestUpgradeHelper.py        |   45 +
 4 files changed, 1260 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/73e86cb6/ambari-server/src/main/python/upgradeHelper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/upgradeHelper.py b/ambari-server/src/main/python/upgradeHelper.py
index a4d01da..2d5a589 100644
--- a/ambari-server/src/main/python/upgradeHelper.py
+++ b/ambari-server/src/main/python/upgradeHelper.py
@@ -186,10 +186,16 @@ class Options(Const):
   REPLACE_RM_HOST_NAME_TAG = "REPLACE_RM_HOST"
   REPLACE_WITH_TAG = "REPLACE_WITH_"
   ZK_OPTIONS = "zoo.cfg"
+  KAFKA_BROKER_CONF = "kafka-broker"
+  RANGER_ADMIN = "admin-properties"
+  KAFKA_PORT = "port"
+  RANGER_EXTERNAL_URL = "ranger.externalurl"
   ZK_CLIENTPORT = "clientPort"
   DELETE_OLD_TAG = "DELETE_OLD"
 
   ZOOKEEPER_SERVER = "ZOOKEEPER_SERVER"
+  KAFKA_BROKER = "KAFKA_BROKER"
+  NAMENODE = "NAMENODE"
 
   MR_MAPPING = None
   logger = None
@@ -1022,6 +1028,72 @@ def get_tez_history_url_base():
   url = '{0}://{1}:{2}/#/main/views/TEZ/{3}/TEZ_CLUSTER_INSTANCE'.format(Options.API_PROTOCOL, Options.HOST, Options.API_PORT, version)
   return url
 
+def get_kafka_listeners():
+  kafka_host="localhost"
+  kafka_port="6667"
+  if Options.server_config_factory is not None and Options.KAFKA_BROKER_CONF in Options.server_config_factory.items():
+    props = Options.server_config_factory.get_config(Options.KAFKA_BROKER_CONF)
+    if Options.KAFKA_PORT in props.properties:
+      kafka_port = props.properties[Options.KAFKA_PORT]
+
+  # Default kafka listeners string
+  kafka_listeners = ["PLAINTEXT://{0}:{1}".format(kafka_host, kafka_port)]
+
+  # Get hosts where kafka_broker is installed
+  kafka_cfg = curl(Options.COMPONENTS_FORMAT.format(Options.KAFKA_BROKER), validate=False, simulate=False, parse=True)
+  if "host_components" in kafka_cfg:
+    kafka_listeners = []
+    for item in kafka_cfg["host_components"]:
+      kafka_listeners.append("PLAINTEXT://{0}:{1}".format(item["HostRoles"]["host_name"], kafka_port))
+
+  return ",".join(kafka_listeners)
+
+def get_ranger_xaaudit_hdfs_destination_directory():
+  namenode_hostname="localhost"
+  namenode_cfg = curl(Options.COMPONENTS_FORMAT.format(Options.NAMENODE), validate=False, simulate=False, parse=True)
+  if "host_components" in namenode_cfg:
+    namenode_hostname = namenode_cfg["host_components"][0]["HostRoles"]["host_name"]
+
+  return "hdfs://{0}:8020/ranger/audit".format(namenode_hostname)
+
+def get_ranger_policymgr_external_url():
+  url = "{{ranger_external_url}}"
+  if Options.server_config_factory is not None and Options.RANGER_ADMIN in Options.server_config_factory.items():
+    props = Options.server_config_factory.get_config(Options.RANGER_ADMIN)
+    if Options.RANGER_EXTERNAL_URL in props.properties:
+      url = props.properties[Options.RANGER_EXTERNAL_URL]
+  return url
+
+def get_jdbc_driver(config_name):
+  driver = "{{jdbc_driver}}"
+  if Options.server_config_factory is not None and config_name in Options.server_config_factory.items():
+    props = Options.server_config_factory.get_config(config_name)
+    if "XAAUDIT.DB.FLAVOUR" in props.properties:
+      db = props.properties["XAAUDIT.DB.FLAVOUR"]
+
+  if db == "mysql":
+    driver = "com.mysql.jdbc.Driver"
+  elif db == "oracle":
+    driver = "oracle.jdbc.OracleDriver"
+  return driver
+
+def get_audit_jdbc_url(config_name):
+  audit_jdbc_url = "{{audit_jdbc_url}}"
+  if Options.server_config_factory is not None and config_name in Options.server_config_factory.items():
+    props = Options.server_config_factory.get_config(config_name)
+    if "XAAUDIT.DB.FLAVOUR" in props.properties:
+      xa_audit_db_flavor = props.properties["XAAUDIT.DB.FLAVOUR"]
+    if "XAAUDIT.DB.HOSTNAME" in props.properties:
+      xa_db_host =  props.properties["XAAUDIT.DB.HOSTNAME"]
+    if "XAAUDIT.DB.DATABASE_NAME" in props.properties:
+      xa_audit_db_name = props.properties["XAAUDIT.DB.DATABASE_NAME"]
+
+  if xa_audit_db_flavor == 'mysql':
+    audit_jdbc_url = "jdbc:mysql://{0}/{1}".format(xa_db_host, xa_audit_db_name)
+  elif xa_audit_db_flavor == 'oracle':
+    audit_jdbc_url = "jdbc:oracle:thin:\@//{0}".format(xa_db_host)
+  return audit_jdbc_url
+
 def get_jt_host(catalog):
   """
   :type catalog: UpgradeCatalog
@@ -1097,6 +1169,63 @@ def _substitute_handler(upgrade_catalog, tokens, value):
       value = value.replace(token, get_ranger_host())
     elif token == "{RANGER_JDBC_DIALECT}":
       value = value.replace(token, get_ranger_service_details()['RANGER_JDBC_DIALECT'])
+    elif token == "{KAFKA_LISTENERS}":
+      value = value.replace(token, get_kafka_listeners())
+    elif token == "{RANGER_PLUGIN_HBASE_POLICY_CACHE_DIR}":
+      value = value.replace(token, "/etc/ranger/{0}{1}/policycache".format(Options.CLUSTER_NAME, "_hbase"))
+    elif token == "{RANGER_PLUGIN_HDFS_POLICY_CACHE_DIR}":
+      value = value.replace(token, "/etc/ranger/{0}{1}/policycache".format(Options.CLUSTER_NAME, "_hdfs"))
+    elif token == "{RANGER_PLUGIN_HIVE_POLICY_CACHE_DIR}":
+      value = value.replace(token, "/etc/ranger/{0}{1}/policycache".format(Options.CLUSTER_NAME, "_hive"))
+    elif token == "{RANGER_PLUGIN_KNOX_POLICY_CACHE_DIR}":
+      value = value.replace(token, "/etc/ranger/{0}{1}/policycache".format(Options.CLUSTER_NAME, "_knox"))
+    elif token == "{RANGER_PLUGIN_STORM_POLICY_CACHE_DIR}":
+      value = value.replace(token, "/etc/ranger/{0}{1}/policycache".format(Options.CLUSTER_NAME, "_storm"))
+    elif token == "{RANGER_HBASE_KEYSTORE_CREDENTIAL_FILE}":
+      value = value.replace(token, "jceks://file/etc/ranger/{0}{1}/cred.jceks".format(Options.CLUSTER_NAME, "_hbase"))
+    elif token == "{RANGER_HDFS_KEYSTORE_CREDENTIAL_FILE}":
+      value = value.replace(token, "jceks://file/etc/ranger/{0}{1}/cred.jceks".format(Options.CLUSTER_NAME, "_hdfs"))
+    elif token == "{RANGER_HIVE_KEYSTORE_CREDENTIAL_FILE}":
+      value = value.replace(token, "jceks://file/etc/ranger/{0}{1}/cred.jceks".format(Options.CLUSTER_NAME, "_hive"))
+    elif token == "{RANGER_KNOX_KEYSTORE_CREDENTIAL_FILE}":
+      value = value.replace(token, "jceks://file/etc/ranger/{0}{1}/cred.jceks".format(Options.CLUSTER_NAME, "_knox"))
+    elif token == "{RANGER_STORM_KEYSTORE_CREDENTIAL_FILE}":
+      value = value.replace(token, "jceks://file/etc/ranger/{0}{1}/cred.jceks".format(Options.CLUSTER_NAME, "_storm"))
+
+    elif token == "{XAAUDIT_HDFS_DESTINATION_DIRECTORY}":
+      value = value.replace(token, get_ranger_xaaudit_hdfs_destination_directory())
+    elif token == "{HBASE_RANGER_REPO_NAME}":
+      value = value.replace(token, Options.CLUSTER_NAME+"_hbase")
+    elif token == "{HDFS_RANGER_REPO_NAME}":
+      value = value.replace(token, Options.CLUSTER_NAME+"_hdfs")
+    elif token == "{HIVE_RANGER_REPO_NAME}":
+      value = value.replace(token, Options.CLUSTER_NAME+"_hive")
+    elif token == "{HNOX_RANGER_REPO_NAME}":
+      value = value.replace(token, Options.CLUSTER_NAME+"_knox")
+    elif token == "{STORM_RANGER_REPO_NAME}":
+      value = value.replace(token, Options.CLUSTER_NAME+"_storm")
+    elif token == "{POLICYMGR_MGR_URL}":
+      value = value.replace(token, get_ranger_policymgr_external_url())
+    elif token == "{HDFS_JDBC_DRIVER}":
+      value = value.replace(token, get_jdbc_driver("ranger-hdfs-plugin-properties"))
+    elif token == "{HBASE_JDBC_DRIVER}":
+      value = value.replace(token, get_jdbc_driver("ranger-hbase-plugin-properties"))
+    elif token == "{HIVE_JDBC_DRIVER}":
+      value = value.replace(token, get_jdbc_driver("ranger-hive-plugin-properties"))
+    elif token == "{KNOX_JDBC_DRIVER}":
+      value = value.replace(token, get_jdbc_driver("ranger-knox-plugin-properties"))
+    elif token == "{STORM_JDBC_DRIVER}":
+      value = value.replace(token, get_jdbc_driver("ranger-storm-plugin-properties"))
+    elif token == "{HDFS_AUDIT_JDBC_URL}":
+      value = value.replace(token, get_audit_jdbc_url("ranger-hdfs-plugin-properties"))
+    elif token == "{HBASE_AUDIT_JDBC_URL}":
+      value = value.replace(token, get_audit_jdbc_url("ranger-hbase-plugin-properties"))
+    elif token == "{HIVE_AUDIT_JDBC_URL}":
+      value = value.replace(token, get_audit_jdbc_url("ranger-hive-plugin-properties"))
+    elif token == "{KNOX_AUDIT_JDBC_URL}":
+      value = value.replace(token, get_audit_jdbc_url("ranger-knox-plugin-properties"))
+    elif token == "{STORM_AUDIT_JDBC_URL}":
+      value = value.replace(token, get_audit_jdbc_url("ranger-storm-plugin-properties"))
 
   return value
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/73e86cb6/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
index 5e32aa8..daa291c 100644
--- a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
@@ -308,7 +308,10 @@
           "yarn.resourcemanager.work-preserving-recovery.enabled": "false",
           "yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms": "10000",
           "yarn.resourcemanager.zk-acl": "world:anyone:rwcda",
-          "yarn.resourcemanager.zk-address": "localhost:2181",
+          "yarn.resourcemanager.zk-address": {
+            "value": "{ZOOKEEPER_QUORUM}",
+            "template": "yes"
+          },
           "yarn.resourcemanager.zk-num-retries": "1000",
           "yarn.resourcemanager.zk-retry-interval-ms": "1000",
           "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore",

http://git-wip-us.apache.org/repos/asf/ambari/blob/73e86cb6/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
index 8025eea..75be829 100644
--- a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
@@ -7,6 +7,110 @@
             "target-version": "2.3",
             "options": {
                 "config-types": {
+                    "falcon-startup.properties": {
+                      "merged-copy": "yes"
+                    },
+                    "tez-site":{
+                      "merged-copy": "yes"
+                    },
+                    "oozie-site": {
+                      "merged-copy": "yes"
+                    },
+                    "hbase-site": {
+                      "merged-copy": "yes"
+                    },
+                    "mapred-site": {
+                      "merged-copy": "yes"
+                    },
+                    "hdfs-site": {
+                      "merged-copy": "yes"
+                    },
+                    "yarn-site": {
+                      "merged-copy": "yes"
+                    },
+                    "kafka-broker": {
+                      "merged-copy": "yes"
+                    },
+                  "ranger-hdfs-policymgr-ssl": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","HDFS"]
+                  },
+                  "ranger-hdfs-security": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","HDFS"]
+                  },
+                  "ranger-hdfs-audit": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","HDFS"]
+                  },
+                  "ranger-hdfs-plugin-properties": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","HDFS"]
+                  },
+                  "ranger-hbase-policymgr-ssl": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","HBASE"]
+                  },
+                  "ranger-hbase-security": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","HBASE"]
+                  },
+                  "ranger-hbase-audit": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","HBASE"]
+                  },
+                  "ranger-hbase-plugin-properties": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","HBASE"]
+                  },
+                  "ranger-hive-policymgr-ssl": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","HIVE"]
+                  },
+                  "ranger-hive-security": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","HIVE"]
+                  },
+                  "ranger-hive-audit": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","HIVE"]
+                  },
+                  "ranger-hive-plugin-properties": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","HIVE"]
+                  },
+                  "ranger-knox-policymgr-ssl": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","KNOX"]
+                  },
+                  "ranger-knox-security": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","KNOX"]
+                  },
+                  "ranger-knox-audit": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","KNOX"]
+                  },
+                  "ranger-knox-plugin-properties": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","KNOX"]
+                  },
+                  "ranger-storm-policymgr-ssl": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","STORM"]
+                  },
+                  "ranger-storm-security": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","STORM"]
+                  },
+                  "ranger-storm-audit": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","STORM"]
+                  },
+                  "ranger-storm-plugin-properties": {
+                    "merged-copy": "yes",
+                    "required-services": ["RANGER","STORM"]
+                  },
                     "admin-properties": {
                         "merged-copy": "yes",
                         "required-services": [
@@ -46,6 +150,491 @@
                 }
             },
             "properties": {
+                "falcon-startup.properties": {
+                  "*.shared.libs": "activemq-core,ant,geronimo-j2ee-management,jms,json-simple,oozie-client,spring-jms,commons-lang3,commons-el",
+                  "*.configstore.listeners": "org.apache.falcon.entity.v0.EntityGraph,\\\n      org.apache.falcon.entity.ColoClusterRelation,\\\n      org.apache.falcon.group.FeedGroupMap,\\\n      org.apache.falcon.service.SharedLibraryHostingService"
+                },
+                "tez-site":{
+                  "tez.am.view-acls": "*",
+                  "tez.tez-ui.history-url.base": {
+                    "value": "{TEZ_HISTORY_URL_BASE}",
+                    "template": "yes"
+                  }
+                },
+                "oozie-site": {
+                  "oozie.authentication.simple.anonymous.allowed": "false",
+                  "oozie.service.AuthorizationService.authorization.enabled": "true",
+                  "oozie.authentication.kerberos.name.rules": "RULE:[2:$1@$0]([jt]t@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-MAPREDUSER/\nRULE:[2:$1@$0]([nd]n@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HDFSUSER/\nRULE:[2:$1@$0](hm@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HBASE-USER/\nRULE:[2:$1@$0](rs@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HBASE-USER/\nDEFAULT",
+                  "oozie.credentials.credentialclasses": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials",
+                  "oozie.service.CallableQueueService.callable.concurrency": {"remove": "yes"},
+                  "oozie.service.CallableQueueService.queue.size": {"remove": "yes"},
+                  "oozie.service.CallableQueueService.threads": {"remove": "yes"},
+                  "oozie.service.JPAService.create.db.schema": {"remove": "yes"},
+                  "oozie.service.JPAService.pool.max.active.conn": {"remove": "yes"},
+                  "oozie.service.PurgeService.older.than": {"remove": "yes"},
+                  "oozie.service.PurgeService.purge.interval": {"remove": "yes"},
+                  "oozie.service.SchemaService.wf.ext.schemas": {"remove": "yes"},
+                  "oozie.service.WorkflowAppService.system.libpath": {"remove": "yes"},
+                  "oozie.service.coord.normal.default.timeout": {"remove": "yes"},
+                  "oozie.service.coord.push.check.requeue.interval": {"remove": "yes"},
+                  "oozie.services": {"remove": "yes"},
+                  "oozie.system.id": {"remove": "yes"},
+                  "oozie.systemmode": {"remove": "yes"},
+                  "use.system.libpath.for.mapreduce.and.pig.jobs": {"remove": "yes"}
+                },
+                "hbase-site": {
+                  "hbase.region.server.rpc.scheduler.factory.class": "org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory",
+                  "hbase.rpc.controllerfactory.class": "org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory",
+                  "hbase.regionserver.wal.codec": "org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec",
+                  "phoenix.functions.allowUserDefinedFunctions": "true",
+                  "fs.hdfs.impl": "org.apache.hadoop.hdfs.DistributedFileSystem",
+                  "hbase.bucketcache.percentage.in.combinedcache": {"remove": "yes"}
+                },
+                "mapred-site": {
+                  "mapreduce.fileoutputcommitter.algorithm.version": "1",
+                  "mapreduce.task.tmp.dir": {"remove": "yes"}
+                },
+                "hdfs-site": {
+                  "dfs.namenode.inode.attributes.provider.class": {
+                    "value": "org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer",
+                    "required-services": ["RANGER"]
+                  }
+                },
+                "yarn-site": {
+                  "yarn.node-labels.enabled": false,
+                  "yarn.timeline-service.recovery.enabled": "true",
+                  "yarn.timeline-service.state-store-class": "org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore",
+                  "yarn.timeline-service.leveldb-state-store.path": "/var/log/hadoop-yarn/timeline"
+                },
+                "kafka-broker": {
+                  "listeners": {
+                    "value": "{KAFKA_LISTENERS}",
+                    "template": "yes"
+                  },
+                  "controlled.shutdown.enable": "true"
+                },
+                "ranger-hdfs-policymgr-ssl": {
+                  "xasecure.policymgr.clientssl.keystore.credential.file": {
+                    "value": "{RANGER_HDFS_KEYSTORE_CREDENTIAL_FILE}",
+                    "template": "yes",
+                    "required-services": ["RANGER"]
+                  },
+                  "xasecure.policymgr.clientssl.truststore.credential.file": {
+                    "value": "{RANGER_HDFS_KEYSTORE_CREDENTIAL_FILE}",
+                    "template": "yes",
+                    "required-services": ["RANGER"]
+                  }
+                },
+                "ranger-hdfs-security": {
+                  "ranger.plugin.hdfs.policy.source.impl": {
+                    "value": "org.apache.ranger.admin.client.RangerAdminRESTClient",
+                    "required-services": ["RANGER"]
+                  },
+                  "ranger.plugin.hdfs.policy.rest.ssl.config.file": {
+                    "value": "/usr/hdp/current/hadoop-client/conf/ranger-policymgr-ssl.xml",
+                    "required-services": ["RANGER"]
+                  },
+                  "ranger.plugin.hdfs.policy.pollIntervalMs": {
+                    "value": "30000",
+                    "required-services": ["RANGER"]
+                  },
+                  "ranger.plugin.hdfs.policy.cache.dir": {
+                    "value": "{RANGER_PLUGIN_HDFS_POLICY_CACHE_DIR}",
+                    "template": "yes",
+                    "required-services": ["RANGER"]
+                  },
+                  "xasecure.add-hadoop-authorization": {
+                    "value": "TRUE",
+                    "required-services": ["RANGER"]
+                  }
+                },
+                "ranger-hdfs-audit": {
+                  "xasecure.audit.credential.provider.file": {
+                    "value": "{RANGER_HDFS_KEYSTORE_CREDENTIAL_FILE}",
+                    "template": "yes",
+                    "required-services": ["RANGER"]
+                  },
+                  "xasecure.audit.destination.solr": {
+                    "value": "FALSE",
+                    "required-services": ["RANGER"]
+                  },
+                  "xasecure.audit.destination.solr.urls": {
+                    "value": "{{ranger_audit_solr_urls}}",
+                    "required-services": ["RANGER"]
+                  },
+                  "xasecure.audit.destination.solr.zookeepers": {
+                    "value": "none",
+                    "required-services": ["RANGER"]
+                  },
+                  "xasecure.audit.destination.solr.batch.filespool.dir": {
+                    "value": "/var/log/hadoop/hdfs/audit/solr/spool",
+                    "required-services": ["RANGER"]
+                  },
+                  "xasecure.audit.destination.db.jdbc.driver": {
+                    "value": "{HDFS_JDBC_DRIVER}",
+                    "required-services": ["RANGER"]
+                  },
+                  "xasecure.audit.destination.db.jdbc.url": {
+                    "value": "{HDFS_AUDIT_JDBC_URL}",
+                    "required-services": ["RANGER"]
+                  },
+                  "xasecure.audit.provider.summary.enabled": {
+                    "value": "false",
+                    "required-services": ["RANGER"]
+                  },
+                  "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/hadoop/hdfs/audit/db/spool",
+                  "xasecure.audit.is.enabled": "true"
+                },
+              "ranger-hdfs-plugin-properties": {
+                "XAAUDIT.HDFS.DESTINTATION_FILE": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                }
+              },
+              "ranger-hbase-policymgr-ssl": {
+                "xasecure.policymgr.clientssl.keystore.credential.file": {
+                  "value": "{RANGER_HBASE_KEYSTORE_CREDENTIAL_FILE}",
+                  "template": "yes"
+                },
+                "xasecure.policymgr.clientssl.truststore.credential.file": {
+                  "value": "{RANGER_HBASE_KEYSTORE_CREDENTIAL_FILE}",
+                  "template": "yes"
+                }
+              },
+              "ranger-hbase-security": {
+                "ranger.plugin.hbase.policy.source.impl": "org.apache.ranger.admin.client.RangerAdminRESTClient",
+                "ranger.plugin.hbase.policy.rest.ssl.config.file": "/usr/hdp/current/hbase-client/conf/ranger-policymgr-ssl.xml",
+                "ranger.plugin.hbase.policy.pollIntervalMs": "30000",
+                "ranger.plugin.hbase.policy.cache.dir": {
+                  "value": "{RANGER_PLUGIN_HBASE_POLICY_CACHE_DIR}",
+                  "template": "yes"
+                }
+              },
+              "ranger-hbase-audit": {
+                "xasecure.audit.credential.provider.file": {
+                  "value": "{RANGER_HBASE_KEYSTORE_CREDENTIAL_FILE}",
+                  "template": "yes"
+                },
+                "xasecure.audit.destination.solr": {
+                  "value": "TRUE"
+                },
+                "xasecure.audit.destination.solr.urls": {
+                  "value": "{{ranger_audit_solr_urls}}"
+                },
+                "xasecure.audit.destination.solr.zookeepers": {
+                  "value": "none"
+                },
+                "xasecure.audit.destination.solr.batch.filespool.dir": {
+                  "value": "/var/log/hbase/audit/solr/spool"
+                },
+                "xasecure.audit.destination.db.jdbc.driver": {
+                  "value": "{HBASE_JDBC_DRIVER}"
+                },
+                "xasecure.audit.destination.db.jdbc.url": {
+                  "value": "{HBASE_AUDIT_JDBC_URL}"
+                },
+                "xasecure.audit.provider.summary.enabled": {
+                  "value": ""
+                },
+                "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/hbase/audit/db/spool",
+                "xasecure.audit.is.enabled": "true"
+              },
+              "ranger-hbase-plugin-properties": {
+                "XAAUDIT.HDFS.DESTINTATION_FILE": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "SQL_CONNECTOR_JAR": {
+                  "remove": "yes"
+                }
+              },
+              "ranger-hive-policymgr-ssl": {
+                "xasecure.policymgr.clientssl.keystore.credential.file": {
+                  "value": "{RANGER_HIVE_KEYSTORE_CREDENTIAL_FILE}",
+                  "template": "yes"
+                },
+                "xasecure.policymgr.clientssl.truststore.credential.file": {
+                  "value": "{RANGER_HIVE_KEYSTORE_CREDENTIAL_FILE}",
+                  "template": "yes"
+                }
+              },
+              "ranger-hive-security": {
+                "ranger.plugin.hive.policy.rest.url": {
+                  "value": "{POLICYMGR_MGR_URL}",
+                  "template": "yes"
+                },
+                "ranger.plugin.hive.policy.rest.ssl.config.file": "/etc/hive/conf/conf.server/ranger-policymgr-ssl.xml",
+                "ranger.plugin.hive.policy.pollIntervalMs": "30000",
+                "ranger.plugin.hive.policy.cache.dir": {
+                  "value": "{RANGER_PLUGIN_HIVE_POLICY_CACHE_DIR}",
+                  "template": "yes"
+                }
+              },
+              "ranger-hive-audit": {
+                "xasecure.audit.credential.provider.file": {
+                  "value": "{RANGER_HIVE_KEYSTORE_CREDENTIAL_FILE}",
+                  "template": "yes"
+                },
+                "xasecure.audit.destination.solr": {
+                  "value": "false"
+                },
+                "xasecure.audit.destination.solr.urls": {
+                  "value": "{{ranger_audit_solr_urls}}"
+                },
+                "xasecure.audit.destination.solr.zookeepers": {
+                  "value": "none"
+                },
+                "xasecure.audit.destination.solr.batch.filespool.dir": {
+                  "value": "/var/log/hive/audit/solr/spool"
+                },
+                "xasecure.audit.destination.db.jdbc.driver": {
+                  "value": "{HIVE_JDBC_DRIVER}"
+                },
+                "xasecure.audit.destination.db.jdbc.url": {
+                  "value": "{HIVE_AUDIT_JDBC_URL}"
+                },
+                "xasecure.audit.provider.summary.enabled": {
+                  "value": ""
+                },
+                "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/hive/audit/db/spool",
+                "xasecure.audit.is.enabled": "true"
+              },
+              "ranger-hive-plugin-properties": {
+                "XAAUDIT.HDFS.DESTINTATION_FILE": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                }
+              },
+              "ranger-knox-policymgr-ssl": {
+                "xasecure.policymgr.clientssl.keystore.credential.file": {
+                  "value": "{RANGER_KNOX_KEYSTORE_CREDENTIAL_FILE}",
+                  "template": "yes"
+                },
+                "xasecure.policymgr.clientssl.truststore.credential.file": {
+                  "value": "{RANGER_KNOX_KEYSTORE_CREDENTIAL_FILE}",
+                  "template": "yes"
+                }
+              },
+              "ranger-knox-security": {
+                "ranger.plugin.knox.policy.source.impl": {
+                  "value": "org.apache.ranger.admin.client.RangerAdminJersey2RESTClient"
+                },
+                "ranger.plugin.knox.policy.rest.ssl.config.file": "/usr/hdp/current/knox-server/conf/ranger-policymgr-ssl.xml",
+                "ranger.plugin.knox.policy.pollIntervalMs": "30000",
+                "ranger.plugin.knox.policy.cache.dir": {
+                  "value": "{RANGER_PLUGIN_KNOX_POLICY_CACHE_DIR}",
+                  "template": "yes"
+                }
+              },
+              "ranger-knox-audit": {
+                "xasecure.audit.credential.provider.file": {
+                  "value": "{RANGER_KNOX_KEYSTORE_CREDENTIAL_FILE}",
+                  "template": "yes"
+                },
+                "xasecure.audit.destination.solr": {
+                  "value": "false"
+                },
+                "xasecure.audit.destination.solr.urls": {
+                  "value": "{{ranger_audit_solr_urls}}"
+                },
+                "xasecure.audit.destination.solr.zookeepers": {
+                  "value": "none"
+                },
+                "xasecure.audit.destination.solr.batch.filespool.dir": {
+                  "value": "/var/log/knox/audit/solr/spool"
+                },
+                "xasecure.audit.destination.db.jdbc.driver": {
+                  "value": "{KNOX_JDBC_DRIVER}"
+                },
+                "xasecure.audit.destination.db.jdbc.url": {
+                  "value": "{KNOX_AUDIT_JDBC_URL}"
+                },
+                "xasecure.audit.provider.summary.enabled": {
+                  "value": ""
+                },
+                "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/knox/audit/db/spool",
+                "xasecure.audit.is.enabled": "true"
+              },
+              "ranger-knox-plugin-properties": {
+                "XAAUDIT.HDFS.DESTINTATION_FILE": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                }
+              },
+
+
+
+              "ranger-storm-policymgr-ssl": {
+                "xasecure.policymgr.clientssl.keystore.credential.file": {
+                  "value": "{RANGER_STORM_KEYSTORE_CREDENTIAL_FILE}",
+                  "template": "yes"
+                },
+                "xasecure.policymgr.clientssl.truststore.credential.file": {
+                  "value": "{RANGER_STORM_KEYSTORE_CREDENTIAL_FILE}",
+                  "template": "yes"
+                }
+              },
+              "ranger-storm-security": {
+                "ranger.plugin.storm.policy.source.impl": {
+                  "value": "org.apache.ranger.admin.client.RangerAdminRESTClient"
+                },
+                "ranger.plugin.storm.policy.rest.ssl.config.file": "/usr/hdp/current/storm-client/conf/ranger-policymgr-ssl.xml",
+                "ranger.plugin.storm.policy.pollIntervalMs": "30000",
+                "ranger.plugin.storm.policy.cache.dir": {
+                  "value": "{RANGER_PLUGIN_STORM_POLICY_CACHE_DIR}",
+                  "template": "yes"
+                }
+              },
+              "ranger-storm-audit": {
+                "xasecure.audit.credential.provider.file": {
+                  "value": "{RANGER_STORM_KEYSTORE_CREDENTIAL_FILE}",
+                  "template": "yes"
+                },
+                "xasecure.audit.destination.solr": {
+                  "value": "false"
+                },
+                "xasecure.audit.destination.solr.urls": {
+                  "value": "{{ranger_audit_solr_urls}}"
+                },
+                "xasecure.audit.destination.solr.zookeepers": {
+                  "value": "none"
+                },
+                "xasecure.audit.destination.solr.batch.filespool.dir": {
+                  "value": "/var/log/storm/audit/solr/spool"
+                },
+                "xasecure.audit.destination.db.jdbc.driver": {
+                  "value": "{STORM_JDBC_DRIVER}"
+                },
+                "xasecure.audit.destination.db.jdbc.url": {
+                  "value": "{STORM_AUDIT_JDBC_URL}"
+                },
+                "xasecure.audit.provider.summary.enabled": {
+                  "value": ""
+                },
+                "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/storm/audit/db/spool",
+                "xasecure.audit.is.enabled": "true"
+              },
+              "ranger-storm-plugin-properties": {
+                "XAAUDIT.HDFS.DESTINTATION_FILE": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                },
+                "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": {
+                  "remove": "yes"
+                }
+              },
                 "ranger-env": {
                     "oracle_home": {
                         "remove": "yes"
@@ -509,8 +1098,497 @@
                     "required-services": [
                         "RANGER"
                     ]
-                }
-            }
-        }
-    ]
+                },
+              "nimbus.host": "nimbus.seeds",
+              "mapreduce.job.speculative.speculativecap": {
+                "map-to": "mapreduce.job.speculative.speculative-cap-running-tasks",
+                "from-catalog": "mapred-site",
+                "to-catalog": "mapred-site",
+                "default": "0.1"
+              },
+              "SSL_KEYSTORE_FILE_PATH": {
+                "map-to": "xasecure.policymgr.clientssl.keystore",
+                "from-catalog": "ranger-hdfs-plugin-properties",
+                "to-catalog": "ranger-hdfs-policymgr-ssl",
+                "default": "/usr/hdp/current/hadoop-client/conf/ranger-plugin-keystore.jks",
+                "required-services": ["RANGER"]
+              },
+              "SSL_KEYSTORE_PASSWORD": {
+                "map-to": "xasecure.policymgr.clientssl.keystore.password",
+                "from-catalog": "ranger-hdfs-plugin-properties",
+                "to-catalog": "ranger-hdfs-policymgr-ssl",
+                "default": "myKeyFilePassword",
+                "required-services": ["RANGER"]
+              },
+              "SSL_TRUSTSTORE_FILE_PATH": {
+                "map-to": "xasecure.policymgr.clientssl.truststore",
+                "from-catalog": "ranger-hdfs-plugin-properties",
+                "to-catalog": "ranger-hdfs-policymgr-ssl",
+                "default": "/usr/hdp/current/hadoop-client/conf/ranger-plugin-keystore.jks",
+                "required-services": ["RANGER"]
+              },
+              "POLICY_MGR_URL": {
+                "map-to": "ranger.plugin.hdfs.policy.rest.url",
+                "from-catalog": "ranger-hdfs-plugin-properties",
+                "to-catalog": "ranger-hdfs-security",
+                "default": "{POLICYMGR_MGR_URL}",
+                "template": "yes",
+                "required-services": ["RANGER"]
+              },
+              "REPOSITORY_NAME": {
+                "map-to": "ranger.plugin.hdfs.service.name",
+                "from-catalog": "ranger-hdfs-plugin-properties",
+                "to-catalog": "ranger-hdfs-security",
+                "default": "{HDFS_RANGER_REPO_NAME}",
+                "template": "yes",
+                "required-services": ["RANGER"]
+              },
+              "XAAUDIT.DB.IS_ENABLED": {
+                "map-to": "xasecure.audit.destination.db",
+                "from-catalog": "ranger-hdfs-plugin-properties",
+                "to-catalog": "ranger-hdfs-audit",
+                "default": "FALSE",
+                "required-services": ["RANGER"]
+              },
+              "XAAUDIT.HDFS.DESTINATION_DIRECTORY": {
+                "map-to": "xasecure.audit.destination.hdfs.dir",
+                "from-catalog": "ranger-hdfs-plugin-properties",
+                "to-catalog": "ranger-hdfs-audit",
+                "default": "{XAAUDIT_HDFS_DESTINATION_DIRECTORY}",
+                "template": "yes",
+                "required-services": ["RANGER"]
+              },
+              "XAAUDIT.HDFS.IS_ENABLED": {
+                "map-to": "xasecure.audit.destination.hdfs",
+                "from-catalog": "ranger-hdfs-plugin-properties",
+                "to-catalog": "ranger-hdfs-audit",
+                "default": "TRUE",
+                "required-services": ["RANGER"]
+              },
+              "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": {
+                "map-to": "xasecure.audit.destination.hdfs.batch.filespool.dir",
+                "from-catalog": "ranger-hdfs-plugin-properties",
+                "to-catalog": "ranger-hdfs-audit",
+                "default": "/var/log/hadoop/hdfs/audit/hdfs/spool",
+                "required-services": ["RANGER"]
+              },
+              "XAAUDIT.DB.USER_NAME": {
+                "map-to": "xasecure.audit.destination.db.user",
+                "from-catalog": "ranger-hdfs-plugin-properties",
+                "to-catalog": "ranger-hdfs-audit",
+                "required-services": ["RANGER"]
+              },
+              "XAAUDIT.DB.PASSWORD": {
+                "map-to": "xasecure.audit.destination.db.password",
+                "from-catalog": "ranger-hdfs-plugin-properties",
+                "to-catalog": "ranger-hdfs-audit",
+                "required-services": ["RANGER"]
+              },
+              "hbase_SSL_KEYSTORE_FILE_PATH": {
+                "map-to": "xasecure.policymgr.clientssl.keystore",
+                "map-from": "SSL_KEYSTORE_FILE_PATH",
+                "from-catalog": "ranger-hbase-plugin-properties",
+                "to-catalog": "ranger-hbase-policymgr-ssl",
+                "default": "/usr/hdp/current/hbase-client/conf/ranger-plugin-keystore.jks",
+                "required-services": ["RANGER", "HBASE"]
+              },
+              "hbase_SSL_KEYSTORE_PASSWORD": {
+                "map-to": "xasecure.policymgr.clientssl.keystore.password",
+                "map-from": "SSL_KEYSTORE_PASSWORD",
+                "from-catalog": "ranger-hbase-plugin-properties",
+                "to-catalog": "ranger-hbase-policymgr-ssl",
+                "default": "myKeyFilePassword",
+                "required-services": ["RANGER", "HBASE"]
+              },
+              "hbase_SSL_TRUSTSTORE_FILE_PATH": {
+                "map-to": "xasecure.policymgr.clientssl.truststore",
+                "map-from": "SSL_TRUSTSTORE_FILE_PATH",
+                "from-catalog": "ranger-hbase-plugin-properties",
+                "to-catalog": "ranger-hbase-policymgr-ssl",
+                "default": "/usr/hdp/current/hbase-client/conf/ranger-plugin-truststore.jks",
+                "required-services": ["RANGER", "HBASE"]
+              },
+              "hbase_SSL_TRUSTSTORE_PASSWORD": {
+                "map-to": "xasecure.policymgr.clientssl.truststore.password",
+                "map-from": "SSL_TRUSTSTORE_PASSWORD",
+                "from-catalog": "ranger-hbase-plugin-properties",
+                "to-catalog": "ranger-hbase-policymgr-ssl",
+                "default": "changeit",
+                "required-services": ["RANGER", "HBASE"]
+              },
+              "UPDATE_XAPOLICIES_ON_GRANT_REVOKE": {
+                "map-to": "xasecure.hbase.update.xapolicies.on.grant.revoke",
+                "from-catalog": "ranger-hbase-plugin-properties",
+                "to-catalog": "ranger-hbase-security",
+                "default": "true",
+                "required-services": ["RANGER", "HBASE"]
+              },
+              "hbase_POLICY_MGR_URL": {
+                "map-from": "POLICY_MGR_URL",
+                "map-to": "ranger.plugin.hbase.policy.rest.url",
+                "from-catalog": "ranger-hbase-plugin-properties",
+                "to-catalog": "ranger-hbase-security",
+                "default": "{POLICYMGR_MGR_URL}",
+                "template": "yes",
+                "required-services": ["RANGER", "HBASE"]
+              },
+              "hbase_REPOSITORY_NAME": {
+                "map-from": "REPOSITORY_NAME",
+                "map-to": "ranger.plugin.hbase.service.name",
+                "from-catalog": "ranger-hbase-plugin-properties",
+                "to-catalog": "ranger-hbase-security",
+                "default": "{HBASE_RANGER_REPO_NAME}",
+                "template": "yes",
+                "required-services": ["RANGER", "HBASE"]
+              },
+              "hbase_XAAUDIT.DB.IS_ENABLED": {
+                "map-from": "XAAUDIT.DB.IS_ENABLED",
+                "map-to": "xasecure.audit.destination.db",
+                "from-catalog": "ranger-hbase-plugin-properties",
+                "to-catalog": "ranger-hbase-audit",
+                "default": "false",
+                "required-services": ["RANGER", "HBASE"]
+              },
+              "hbase_XAAUDIT.HDFS.DESTINATION_DIRECTORY": {
+                "map-from": "XAAUDIT.HDFS.DESTINATION_DIRECTORY",
+                "map-to": "xasecure.audit.destination.hdfs.dir",
+                "from-catalog": "ranger-hbase-plugin-properties",
+                "to-catalog": "ranger-hbase-audit",
+                "default": "{XAAUDIT_HDFS_DESTINATION_DIRECTORY}",
+                "template": "yes",
+                "required-services": ["RANGER", "HBASE"]
+              },
+              "hbase_XAAUDIT.HDFS.IS_ENABLED": {
+                "map-from": "XAAUDIT.HDFS.IS_ENABLED",
+                "map-to": "xasecure.audit.destination.hdfs",
+                "from-catalog": "ranger-hbase-plugin-properties",
+                "to-catalog": "ranger-hbase-audit",
+                "default": "true",
+                "required-services": ["RANGER", "HBASE"]
+              },
+              "hbase_XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": {
+                "map-from": "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY",
+                "map-to": "xasecure.audit.destination.hdfs.batch.filespool.dir",
+                "from-catalog": "ranger-hbase-plugin-properties",
+                "to-catalog": "ranger-hbase-audit",
+                "default": "/var/log/hbase/audit/hdfs/spool",
+                "required-services": ["RANGER", "HBASE"]
+              },
+              "hbase_XAAUDIT.DB.USER_NAME": {
+                "map-from": "XAAUDIT.DB.USER_NAME",
+                "map-to": "xasecure.audit.destination.db.user",
+                "from-catalog": "ranger-hbase-plugin-properties",
+                "to-catalog": "ranger-hbase-audit",
+                "required-services": ["RANGER", "HBASE"]
+              },
+              "hbase_XAAUDIT.DB.PASSWORD": {
+                "map-from": "XAAUDIT.DB.PASSWORD",
+                "map-to": "xasecure.audit.destination.db.password",
+                "from-catalog": "ranger-hbase-plugin-properties",
+                "to-catalog": "ranger-hbase-audit",
+                "required-services": ["RANGER", "HBASE"]
+              },
+              "hive_SSL_KEYSTORE_FILE_PATH": {
+                "map-to": "xasecure.policymgr.clientssl.keystore",
+                "map-from": "SSL_KEYSTORE_FILE_PATH",
+                "from-catalog": "ranger-hive-plugin-properties",
+                "to-catalog": "ranger-hive-policymgr-ssl",
+                "default": "/etc/hive/conf/conf.server/ranger-plugin-keystore.jks",
+                "required-services": ["RANGER", "HIVE"]
+              },
+              "hive_SSL_KEYSTORE_PASSWORD": {
+                "map-to": "xasecure.policymgr.clientssl.keystore.password",
+                "map-from": "SSL_KEYSTORE_PASSWORD",
+                "from-catalog": "ranger-hive-plugin-properties",
+                "to-catalog": "ranger-hive-policymgr-ssl",
+                "default": "myKeyFilePassword",
+                "required-services": ["RANGER", "HIVE"]
+              },
+              "hive_SSL_TRUSTSTORE_FILE_PATH": {
+                "map-to": "xasecure.policymgr.clientssl.truststore",
+                "map-from": "SSL_TRUSTSTORE_FILE_PATH",
+                "from-catalog": "ranger-hive-plugin-properties",
+                "to-catalog": "ranger-hive-policymgr-ssl",
+                "default": "/etc/hive/conf/conf.server/ranger-plugin-keystore.jks",
+                "required-services": ["RANGER", "HIVE"]
+              },
+              "hive_SSL_TRUSTSTORE_PASSWORD": {
+                "map-to": "xasecure.policymgr.clientssl.truststore.password",
+                "map-from": "SSL_TRUSTSTORE_PASSWORD",
+                "from-catalog": "ranger-hive-plugin-properties",
+                "to-catalog": "ranger-hive-policymgr-ssl",
+                "default": "changeit",
+                "required-services": ["RANGER", "HIVE"]
+              },
+              "hive_UPDATE_XAPOLICIES_ON_GRANT_REVOKE": {
+                "map-from": "UPDATE_XAPOLICIES_ON_GRANT_REVOKE",
+                "map-to": "xasecure.hbase.update.xapolicies.on.grant.revoke",
+                "from-catalog": "ranger-hive-plugin-properties",
+                "to-catalog": "ranger-hive-security",
+                "default": "true",
+                "required-services": ["RANGER", "HIVE"]
+              },
+              "hive_POLICY_MGR_URL": {
+                "map-from": "POLICY_MGR_URL",
+                "map-to": "ranger.plugin.hive.service.name",
+                "from-catalog": "ranger-hive-plugin-properties",
+                "to-catalog": "ranger-hive-security",
+                "default": "{HIVE_RANGER_REPO_NAME}",
+                "template": "yes",
+                "required-services": ["RANGER", "HIVE"]
+              },
+              "hive_REPOSITORY_NAME": {
+                "map-from": "REPOSITORY_NAME",
+                "map-to": "ranger.plugin.hive.policy.source.impl",
+                "from-catalog": "ranger-hive-plugin-properties",
+                "to-catalog": "ranger-hive-security",
+                "default": "org.apache.ranger.admin.client.RangerAdminRESTClient",
+                "required-services": ["RANGER", "HIVE"]
+              },
+              "hive_XAAUDIT.DB.IS_ENABLED": {
+                "map-from": "XAAUDIT.DB.IS_ENABLED",
+                "map-to": "xasecure.audit.destination.db",
+                "from-catalog": "ranger-hive-plugin-properties",
+                "to-catalog": "ranger-hive-audit",
+                "default": "false",
+                "required-services": ["RANGER", "HIVE"]
+              },
+              "hive_XAAUDIT.HDFS.DESTINATION_DIRECTORY": {
+                "map-from": "XAAUDIT.HDFS.DESTINATION_DIRECTORY",
+                "map-to": "xasecure.audit.destination.hdfs.dir",
+                "from-catalog": "ranger-hive-plugin-properties",
+                "to-catalog": "ranger-hive-audit",
+                "default": "{XAAUDIT_HDFS_DESTINATION_DIRECTORY}",
+                "template": "yes",
+                "required-services": ["RANGER", "HIVE"]
+              },
+              "hive_XAAUDIT.HDFS.IS_ENABLED": {
+                "map-from": "XAAUDIT.HDFS.IS_ENABLED",
+                "map-to": "xasecure.audit.destination.hdfs",
+                "from-catalog": "ranger-hive-plugin-properties",
+                "to-catalog": "ranger-hive-audit",
+                "default": "true",
+                "required-services": ["RANGER", "HIVE"]
+              },
+              "hive_XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": {
+                "map-from": "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY",
+                "map-to": "xasecure.audit.destination.hdfs.batch.filespool.dir",
+                "from-catalog": "ranger-hive-plugin-properties",
+                "to-catalog": "ranger-hive-audit",
+                "default": "/var/log/hive/audit/hdfs/spool",
+                "required-services": ["RANGER", "HIVE"]
+              },
+              "hive_XAAUDIT.DB.USER_NAME": {
+                "map-from": "XAAUDIT.DB.USER_NAME",
+                "map-to": "xasecure.audit.destination.db.user",
+                "from-catalog": "ranger-hive-plugin-properties",
+                "to-catalog": "ranger-hive-audit",
+                "required-services": ["RANGER", "HIVE"]
+              },
+              "hive_XAAUDIT.DB.PASSWORD": {
+                "map-from": "XAAUDIT.DB.PASSWORD",
+                "map-to": "xasecure.audit.destination.db.password",
+                "from-catalog": "ranger-hive-plugin-properties",
+                "to-catalog": "ranger-hive-audit",
+                "required-services": ["RANGER", "HIVE"]
+              },
+              "knox_SSL_KEYSTORE_FILE_PATH": {
+                "map-to": "xasecure.policymgr.clientssl.keystore",
+                "map-from": "SSL_KEYSTORE_FILE_PATH",
+                "from-catalog": "ranger-knox-plugin-properties",
+                "to-catalog": "ranger-knox-policymgr-ssl",
+                "default": "/usr/hdp/current/knox-server/conf/ranger-plugin-keystore.jks",
+                "required-services": ["RANGER", "KNOX"]
+              },
+              "knox_SSL_KEYSTORE_PASSWORD": {
+                "map-to": "xasecure.policymgr.clientssl.keystore.password",
+                "map-from": "SSL_KEYSTORE_PASSWORD",
+                "from-catalog": "ranger-knox-plugin-properties",
+                "to-catalog": "ranger-knox-policymgr-ssl",
+                "default": "myKeyFilePassword",
+                "required-services": ["RANGER", "KNOX"]
+              },
+              "knox_SSL_TRUSTSTORE_FILE_PATH": {
+                "map-to": "xasecure.policymgr.clientssl.truststore",
+                "map-from": "SSL_TRUSTSTORE_FILE_PATH",
+                "from-catalog": "ranger-knox-plugin-properties",
+                "to-catalog": "ranger-knox-policymgr-ssl",
+                "default": "/usr/hdp/current/knox-server/conf/ranger-plugin-keystore.jks",
+                "required-services": ["RANGER", "KNOX"]
+              },
+              "knox_SSL_TRUSTSTORE_PASSWORD": {
+                "map-to": "xasecure.policymgr.clientssl.truststore.password",
+                "map-from": "SSL_TRUSTSTORE_PASSWORD",
+                "from-catalog": "ranger-knox-plugin-properties",
+                "to-catalog": "ranger-knox-policymgr-ssl",
+                "default": "changeit",
+                "required-services": ["RANGER", "KNOX"]
+              },
+              "knox_POLICY_MGR_URL": {
+                "map-from": "POLICY_MGR_URL",
+                "map-to": "ranger.plugin.knox.policy.rest.url",
+                "from-catalog": "ranger-knox-plugin-properties",
+                "to-catalog": "ranger-knox-security",
+                "default": "{POLICYMGR_MGR_URL}",
+                "template": "yes",
+                "required-services": ["RANGER", "KNOX"]
+              },
+              "knox_REPOSITORY_NAME": {
+                "map-from": "REPOSITORY_NAME",
+                "map-to": "ranger.plugin.knox.service.name",
+                "from-catalog": "ranger-knox-plugin-properties",
+                "to-catalog": "ranger-knox-security",
+                "default": "{KNOX_RANGER_REPO_NAME}",
+                "template": "yes",
+                "required-services": ["RANGER", "KNOX"]
+              },
+              "knox_XAAUDIT.DB.IS_ENABLED": {
+                "map-from": "XAAUDIT.DB.IS_ENABLED",
+                "map-to": "xasecure.audit.destination.db",
+                "from-catalog": "ranger-knox-plugin-properties",
+                "to-catalog": "ranger-knox-audit",
+                "default": "false",
+                "required-services": ["RANGER", "KNOX"]
+              },
+              "knox_XAAUDIT.HDFS.DESTINATION_DIRECTORY": {
+                "map-from": "XAAUDIT.HDFS.DESTINATION_DIRECTORY",
+                "map-to": "xasecure.audit.destination.hdfs.dir",
+                "from-catalog": "ranger-knox-plugin-properties",
+                "to-catalog": "ranger-knox-audit",
+                "default": "{XAAUDIT_HDFS_DESTINATION_DIRECTORY}",
+                "template": "yes",
+                "required-services": ["RANGER", "KNOX"]
+              },
+              "knox_XAAUDIT.HDFS.IS_ENABLED": {
+                "map-from": "XAAUDIT.HDFS.IS_ENABLED",
+                "map-to": "xasecure.audit.destination.hdfs",
+                "from-catalog": "ranger-knox-plugin-properties",
+                "to-catalog": "ranger-knox-audit",
+                "default": "true",
+                "required-services": ["RANGER", "KNOX"]
+              },
+              "knox_XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": {
+                "map-from": "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY",
+                "map-to": "xasecure.audit.destination.hdfs.batch.filespool.dir",
+                "from-catalog": "ranger-knox-plugin-properties",
+                "to-catalog": "ranger-knox-audit",
+                "default": "/var/log/knox/audit/hdfs/spool",
+                "required-services": ["RANGER", "KNOX"]
+              },
+              "knox_XAAUDIT.DB.USER_NAME": {
+                "map-from": "XAAUDIT.DB.USER_NAME",
+                "map-to": "xasecure.audit.destination.db.user",
+                "from-catalog": "ranger-knox-plugin-properties",
+                "to-catalog": "ranger-knox-audit",
+                "required-services": ["RANGER", "KNOX"]
+              },
+              "knox_XAAUDIT.DB.PASSWORD": {
+                "map-from": "XAAUDIT.DB.PASSWORD",
+                "map-to": "xasecure.audit.destination.db.password",
+                "from-catalog": "ranger-knox-plugin-properties",
+                "to-catalog": "ranger-knox-audit",
+                "required-services": ["RANGER", "KNOX"]
+              },
+
+
+              "storm_SSL_KEYSTORE_FILE_PATH": {
+                "map-to": "xasecure.policymgr.clientssl.keystore",
+                "map-from": "SSL_KEYSTORE_FILE_PATH",
+                "from-catalog": "ranger-storm-plugin-properties",
+                "to-catalog": "ranger-storm-policymgr-ssl",
+                "default": "/usr/hdp/current/storm-client/conf/ranger-plugin-keystore.jks",
+                "required-services": ["RANGER", "STORM"]
+              },
+              "storm_SSL_KEYSTORE_PASSWORD": {
+                "map-to": "xasecure.policymgr.clientssl.keystore.password",
+                "map-from": "SSL_KEYSTORE_PASSWORD",
+                "from-catalog": "ranger-storm-plugin-properties",
+                "to-catalog": "ranger-storm-policymgr-ssl",
+                "default": "myKeyFilePassword",
+                "required-services": ["RANGER", "STORM"]
+              },
+              "storm_SSL_TRUSTSTORE_FILE_PATH": {
+                "map-to": "xasecure.policymgr.clientssl.truststore",
+                "map-from": "SSL_TRUSTSTORE_FILE_PATH",
+                "from-catalog": "ranger-storm-plugin-properties",
+                "to-catalog": "ranger-storm-policymgr-ssl",
+                "default": "/usr/hdp/current/storm-client/conf/ranger-plugin-keystore.jks",
+                "required-services": ["RANGER", "STORM"]
+              },
+              "storm_SSL_TRUSTSTORE_PASSWORD": {
+                "map-to": "xasecure.policymgr.clientssl.truststore.password",
+                "map-from": "SSL_TRUSTSTORE_PASSWORD",
+                "from-catalog": "ranger-storm-plugin-properties",
+                "to-catalog": "ranger-storm-policymgr-ssl",
+                "default": "changeit",
+                "required-services": ["RANGER", "STORM"]
+              },
+              "storm_POLICY_MGR_URL": {
+                "map-from": "POLICY_MGR_URL",
+                "map-to": "ranger.plugin.storm.policy.rest.url",
+                "from-catalog": "ranger-storm-plugin-properties",
+                "to-catalog": "ranger-storm-security",
+                "default": "{POLICYMGR_MGR_URL}",
+                "template": "yes",
+                "required-services": ["RANGER", "STORM"]
+              },
+              "storm_REPOSITORY_NAME": {
+                "map-from": "REPOSITORY_NAME",
+                "map-to": "ranger.plugin.storm.service.name",
+                "from-catalog": "ranger-storm-plugin-properties",
+                "to-catalog": "ranger-storm-security",
+                "default": "{STORM_RANGER_REPO_NAME}",
+                "template": "yes",
+                "required-services": ["RANGER", "STORM"]
+              },
+              "storm_XAAUDIT.DB.IS_ENABLED": {
+                "map-from": "XAAUDIT.DB.IS_ENABLED",
+                "map-to": "xasecure.audit.destination.db",
+                "from-catalog": "ranger-storm-plugin-properties",
+                "to-catalog": "ranger-storm-audit",
+                "default": "false",
+                "required-services": ["RANGER", "STORM"]
+              },
+              "storm_XAAUDIT.HDFS.DESTINATION_DIRECTORY": {
+                "map-from": "XAAUDIT.HDFS.DESTINATION_DIRECTORY",
+                "map-to": "xasecure.audit.destination.hdfs.dir",
+                "from-catalog": "ranger-storm-plugin-properties",
+                "to-catalog": "ranger-storm-audit",
+                "default": "{XAAUDIT_HDFS_DESTINATION_DIRECTORY}",
+                "template": "yes",
+                "required-services": ["RANGER", "STORM"]
+              },
+              "storm_XAAUDIT.HDFS.IS_ENABLED": {
+                "map-from": "XAAUDIT.HDFS.IS_ENABLED",
+                "map-to": "xasecure.audit.destination.hdfs",
+                "from-catalog": "ranger-storm-plugin-properties",
+                "to-catalog": "ranger-storm-audit",
+                "default": "true",
+                "required-services": ["RANGER", "STORM"]
+              },
+              "storm_XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": {
+                "map-from": "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY",
+                "map-to": "xasecure.audit.destination.hdfs.batch.filespool.dir",
+                "from-catalog": "ranger-storm-plugin-properties",
+                "to-catalog": "ranger-storm-audit",
+                "default": "/var/log/storm/audit/hdfs/spool",
+                "required-services": ["RANGER", "STORM"]
+              },
+              "storm_XAAUDIT.DB.USER_NAME": {
+                "map-from": "XAAUDIT.DB.USER_NAME",
+                "map-to": "xasecure.audit.destination.db.user",
+                "from-catalog": "ranger-storm-plugin-properties",
+                "to-catalog": "ranger-storm-audit",
+                "required-services": ["RANGER", "STORM"]
+              },
+              "storm_XAAUDIT.DB.PASSWORD": {
+                "map-from": "XAAUDIT.DB.PASSWORD",
+                "map-to": "xasecure.audit.destination.db.password",
+                "from-catalog": "ranger-storm-plugin-properties",
+                "to-catalog": "ranger-storm-audit",
+                "required-services": ["RANGER", "STORM"]
+              }
+     }
+    }
+  ]
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/73e86cb6/ambari-server/src/test/python/TestUpgradeHelper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestUpgradeHelper.py b/ambari-server/src/test/python/TestUpgradeHelper.py
index e51afe7..45b899e 100644
--- a/ambari-server/src/test/python/TestUpgradeHelper.py
+++ b/ambari-server/src/test/python/TestUpgradeHelper.py
@@ -520,6 +520,51 @@ class TestUpgradeHelper(TestCase):
     self.assertEqual(expected_result, actual_result)
 
   @patch.object(upgradeHelper, "curl")
+  def test_get_kafka_listeners(self, curl_mock):
+    return_curl_data = {
+      "host_components": [
+        {
+          "HostRoles": {
+            "host_name": "test.host.vm"
+          }
+        }
+      ]
+    }
+
+    curl_mock.return_value = copy.deepcopy(return_curl_data)
+
+    # build zookeeper quorum string from return_curl_data and remove trailing comas
+    expected_result = "PLAINTEXT://test.host.vm:6667"
+
+    # execute testing function
+    actual_result = upgradeHelper.get_kafka_listeners()
+
+    self.assertEqual(expected_result, actual_result)
+
+  @patch.object(upgradeHelper, "curl")
+  def test_get_ranger_xaaudit_hdfs_destination_directory(self, curl_mock):
+    return_curl_data = {
+      "host_components": [
+        {
+          "HostRoles": {
+            "host_name": "test.host.vm"
+          }
+        }
+      ]
+    }
+
+    curl_mock.return_value = copy.deepcopy(return_curl_data)
+
+    # build zookeeper quorum string from return_curl_data and remove trailing comas
+    expected_result = "hdfs://test.host.vm:8020/ranger/audit"
+
+    # execute testing function
+    actual_result = upgradeHelper.get_ranger_xaaudit_hdfs_destination_directory()
+
+    self.assertEqual(expected_result, actual_result)
+
+
+  @patch.object(upgradeHelper, "curl")
   def test_get_config_resp_all(self, curl_mock):
     cfg_type = "my type"
     cfg_tag = "my tag"


Mime
View raw message