ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alejan...@apache.org
Subject ambari git commit: AMBARI-17678. Misc Atlas fixes, remove conf dir from classpath of Falcon and Storm in HDP 2.5, add more security properties to Atlas Hooks, and delete deprecated configs (alejandro)
Date Wed, 13 Jul 2016 22:27:31 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 e7f225e54 -> b30d3c3a5


AMBARI-17678. Misc Atlas fixes, remove conf dir from classpath of Falcon and Storm in HDP
2.5, add more security properties to Atlas Hooks, and delete deprecated configs (alejandro)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b30d3c3a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b30d3c3a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b30d3c3a

Branch: refs/heads/branch-2.4
Commit: b30d3c3a5df54442638cbc804dd4d43d05f92469
Parents: e7f225e
Author: Alejandro Fernandez <afernandez@hortonworks.com>
Authored: Tue Jul 12 13:39:18 2016 -0700
Committer: Alejandro Fernandez <afernandez@hortonworks.com>
Committed: Wed Jul 13 15:30:14 2016 -0700

----------------------------------------------------------------------
 .../libraries/functions/constants.py            |  1 +
 .../libraries/functions/setup_atlas_hook.py     |  8 +++-
 .../0.1.0.2.3/configuration/atlas-log4j.xml     |  2 +-
 .../ATLAS/0.1.0.2.3/package/scripts/params.py   | 11 +++---
 .../configuration/application-properties.xml    | 40 ++++++++++++++------
 .../0.5.0.2.1/package/scripts/params_linux.py   |  3 +-
 .../STORM/0.9.1/package/scripts/params_linux.py |  8 ++--
 .../HDP/2.0.6/properties/stack_features.json    |  6 +++
 .../stacks/2.3/ATLAS/test_metadata_server.py    | 14 +++++--
 .../test/python/stacks/2.3/configs/default.json | 23 ++++++-----
 .../test/python/stacks/2.3/configs/secure.json  |  3 ++
 .../stacks/2.5/ATLAS/test_atlas_server.py       | 13 +++++--
 .../test/python/stacks/2.5/configs/default.json |  3 ++
 13 files changed, 95 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/b30d3c3a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index cd73049..5b9e900 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -90,6 +90,7 @@ class StackFeature:
   HBASE_HOME_DIRECTORY = "hbase_home_directory"
   ATLAS_RANGER_PLUGIN_SUPPORT = "atlas_ranger_plugin_support"
   ATLAS_UPGRADE_SUPPORT = "atlas_upgrade_support"
+  ATLAS_CONF_DIR_IN_PATH = "atlas_conf_dir_in_path"
   RANGER_PID_SUPPORT = "ranger_pid_support"
   RANGER_KMS_PID_SUPPORT = "ranger_kms_pid_support"
   RANGER_ADMIN_PASSWD_CHANGE = "ranger_admin_password_change"

http://git-wip-us.apache.org/repos/asf/ambari/blob/b30d3c3a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
index c7f6204..947d441 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
@@ -73,14 +73,18 @@ SHARED_ATLAS_HOOK_CONFIGS = set(
   "atlas.notification.topics",
   "atlas.notification.kafka.service.principal",
   "atlas.notification.kafka.keytab.location",
+  "atlas.cluster.name",
+
+  # Security properties
+  "atlas.kafka.sasl.kerberos.service.name",
+  "atlas.kafka.security.protocol",
   "atlas.jaas.KafkaClient.loginModuleName",
   "atlas.jaas.KafkaClient.loginModuleControlFlag",
   "atlas.jaas.KafkaClient.option.useKeyTab",
   "atlas.jaas.KafkaClient.option.storeKey",
   "atlas.jaas.KafkaClient.option.serviceName",
   "atlas.jaas.KafkaClient.option.keyTab",
-  "atlas.jaas.KafkaClient.option.principal",
-  "atlas.cluster.name"]
+  "atlas.jaas.KafkaClient.option.principal"]
 )
 
 def has_atlas_in_cluster():

http://git-wip-us.apache.org/repos/asf/ambari/blob/b30d3c3a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-log4j.xml
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-log4j.xml
index 372fd31..a071442 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-log4j.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-log4j.xml
@@ -28,7 +28,7 @@
   </property>
   <property>
     <name>audit_log_level</name>
-    <value>OFF</value>
+    <value>info</value>
     <description>Log level for audit logging</description>
     <on-ambari-upgrade add="true"/>
   </property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/b30d3c3a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index 8445214..0df8270 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -127,14 +127,18 @@ else:
 
 metadata_host = config['hostname']
 
+atlas_hosts = sorted(default('/clusterHostInfo/atlas_server_hosts', []))
+metadata_server_host = atlas_hosts[0] if len(atlas_hosts) > 0 else "UNKNOWN_HOST"
+metadata_server_url = format('{metadata_protocol}://{metadata_server_host}:{metadata_port}')
+
 # application properties
 application_properties = dict(config['configurations']['application-properties'])
-application_properties['atlas.server.bind.address'] = metadata_host
+application_properties["atlas.server.bind.address"] = metadata_host
+application_properties["atlas.rest.address"] = metadata_server_url
 
 # Atlas HA should populate
 # atlas.server.ids = id1,id2,...,idn
 # atlas.server.address.id# = host#:port
-atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', [])
 # User should not have to modify this property, but still allow overriding it to False if
multiple Atlas servers exist
 # This can be None, True, or False
 is_atlas_ha_enabled = default("/configurations/application-properties/atlas.server.ha.enabled",
None)
@@ -256,9 +260,6 @@ if has_ranger_admin and stack_supports_atlas_ranger_plugin:
   enable_ranger_atlas = config['configurations']['ranger-atlas-plugin-properties']['ranger-atlas-plugin-enabled']
   enable_ranger_atlas = not is_empty(enable_ranger_atlas) and enable_ranger_atlas.lower()
== 'yes'
   policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
-  atlas_hosts = sorted(default('/clusterHostInfo/atlas_server_hosts', []))
-  metadata_server_host = atlas_hosts[0]
-  metadata_server_url = format('{metadata_protocol}://{metadata_server_host}:{metadata_port}')
 
   downloaded_custom_connector = None
   driver_curl_source = None

http://git-wip-us.apache.org/repos/asf/ambari/blob/b30d3c3a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml
b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml
index aa9940e..c3bcbc3 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml
@@ -102,12 +102,6 @@
     <description>Path for the Atlas policy file.</description>
     <on-ambari-upgrade add="false"/>
   </property>
-  <property>
-    <name>atlas.kafka.data</name>
-    <value>{{metadata_home}}/data/kafka</value>
-    <description>The Kafka data directory.</description>
-    <on-ambari-upgrade add="false"/>
-  </property>
 
   <!-- Start: Shared Atlas Hooks that are also written out to configs for Falcon, Storm,
Hive, and Sqoop.
   There are several more properties for when Atlas is Kerberized.
@@ -150,12 +144,6 @@
     <on-ambari-upgrade add="false"/>
   </property>
   <property>
-    <name>atlas.kafka.entities.group.id</name>
-    <value>entities</value>
-    <description>Kafka group id for the entity topic.</description>
-    <on-ambari-upgrade add="false"/>
-  </property>
-  <property>
     <name>atlas.kafka.auto.commit.enable</name>
     <value>false</value>
     <description>Kafka auto commit setting for Atlas notifications.</description>
@@ -237,6 +225,34 @@
     <deleted>true</deleted>
     <on-ambari-upgrade add="false"/>
   </property>
+
+  <property>
+    <name>atlas.graph.index.search.directory</name>
+    <deleted>true</deleted>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>atlas.graph.index.search.elasticsearch.client-only</name>
+    <deleted>true</deleted>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>atlas.graph.index.search.elasticsearch.local-mode</name>
+    <deleted>true</deleted>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>atlas.graph.storage.directory</name>
+    <deleted>true</deleted>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>atlas.kafka.entities.group.id</name>
+    <description>Kafka group id for the entity topic.</description>
+    <deleted>true</deleted>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
   <property>
     <name>atlas.solr.kerberos.enable</name>
     <value>false</value>

http://git-wip-us.apache.org/repos/asf/ambari/blob/b30d3c3a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
index e58d205..ebe3634 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
@@ -154,7 +154,8 @@ atlas_hook_cp = ""
 if has_atlas_in_cluster():
   atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties')
 
-  if has_atlas_server_on_host:
+  # Only append /etc/atlas/conf to classpath if on HDP 2.4.* and atlas server is running
on this host.
+  if has_atlas_server_on_host and check_stack_feature(StackFeature.ATLAS_CONF_DIR_IN_PATH,
stack_version_formatted):
     atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else format('{stack_root}/current/atlas-server/conf')
     atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ
else format('{stack_root}/current/atlas-server')
     atlas_hook_cp = atlas_conf_dir + os.pathsep + os.path.join(atlas_home_dir, "hook", "falcon",
"*") + os.pathsep

http://git-wip-us.apache.org/repos/asf/ambari/blob/b30d3c3a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
index 2afb187..5941ca1 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
@@ -194,7 +194,6 @@ metric_collector_legacy_sink_jar = "/usr/lib/storm/lib/ambari-metrics-storm-sink
 
 jar_jvm_opts = ''
 
-
 ########################################################
 ############# Atlas related params #####################
 ########################################################
@@ -203,8 +202,11 @@ storm_atlas_application_properties = default('/configurations/storm-atlas-applic
 
 if has_atlas_in_cluster():
   atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties')
-  atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
-  jar_jvm_opts += '-Datlas.conf=' + atlas_conf_dir
+
+  # Only append /etc/atlas/conf to classpath if on HDP 2.4.*
+  if check_stack_feature(StackFeature.ATLAS_CONF_DIR_IN_PATH, stack_version_formatted):
+    atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
+    jar_jvm_opts += '-Datlas.conf=' + atlas_conf_dir
 #endregion
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/b30d3c3a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
index 7f85ad3..7f69fcd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -247,6 +247,12 @@
       "min_version": "2.5.0.0"
     },
     {
+      "name": "atlas_conf_dir_in_path",
+      "description": "Prepend the Atlas conf dir (/etc/atlas/conf) to the classpath of Storm
and Falcon",
+      "min_version": "2.3.0.0",
+      "max_version": "2.4.99.99"
+    },
+    {
       "name": "atlas_upgrade_support",
       "description": "Atlas supports express and rolling upgrades",
       "min_version": "2.5.0.0"

http://git-wip-us.apache.org/repos/asf/ambari/blob/b30d3c3a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
index 9e1c7f7..6285bb7 100644
--- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
+++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
@@ -83,8 +83,16 @@ class TestMetadataServer(RMFTestCase):
       self.assertResourceCalled('File', '/usr/hdp/current/atlas-server/server/webapp/atlas.war',
           content = StaticFile('/usr/hdp/current/atlas-server/server/webapp/atlas.war'),
       )
+      host_name = u"c6401.ambari.apache.org"
       app_props =  dict(self.getConfig()['configurations']['application-properties'])
-      app_props['atlas.server.bind.address'] = 'c6401.ambari.apache.org'
+      app_props['atlas.server.bind.address'] = host_name
+
+      metadata_protocol = "https" if app_props["atlas.enableTLS"] is True else "http"
+      metadata_port = app_props["atlas.server.https.port"] if metadata_protocol == "https"
else app_props["atlas.server.http.port"]
+      app_props["atlas.rest.address"] = u'%s://%s:%s' % (metadata_protocol, host_name, metadata_port)
+      app_props["atlas.server.ids"] = "id1"
+      app_props["atlas.server.address.id1"] = u"%s:%s" % (host_name, metadata_port)
+      app_props["atlas.server.ha.enabled"] = "false"
 
       self.assertResourceCalled('File', '/etc/atlas/conf/atlas-log4j.xml',
                           content=InlineTemplate(
@@ -114,8 +122,8 @@ class TestMetadataServer(RMFTestCase):
       self.assertResourceCalled('PropertiesFile',
                                 '/etc/atlas/conf/application.properties',
                                 properties=app_props,
-                                owner='atlas',
-                                group='hadoop',
+                                owner=u'atlas',
+                                group=u'hadoop',
                                 mode=0644,
       )
       self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr-client',

http://git-wip-us.apache.org/repos/asf/ambari/blob/b30d3c3a/ambari-server/src/test/python/stacks/2.3/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json
index 080274f..e85ca92 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json
@@ -336,17 +336,20 @@
             "c6401.ambari.apache.org",
             "c6402.ambari.apache.org"
         ],
-      "knox_gateway_hosts": [
-        "jaimin-knox-1.c.pramod-thangali.internal"
-      ],
-      "kafka_broker_hosts": [
-        "c6401.ambari.apache.org"
-      ],
-      "logsearch_solr_hosts": [
-        "c6401.ambari.apache.org"
-      ],
-       "zookeeper_hosts": [
+        "knox_gateway_hosts": [
+          "jaimin-knox-1.c.pramod-thangali.internal"
+        ],
+        "kafka_broker_hosts": [
+          "c6401.ambari.apache.org"
+        ],
+        "logsearch_solr_hosts": [
+          "c6401.ambari.apache.org"
+        ],
+        "zookeeper_hosts": [
          "c6401.ambari.apache.org"
+        ],
+        "atlas_server_hosts": [
+          "c6401.ambari.apache.org"
         ]
 
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/b30d3c3a/ambari-server/src/test/python/stacks/2.3/configs/secure.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/secure.json b/ambari-server/src/test/python/stacks/2.3/configs/secure.json
index a4e7096..2545505 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/secure.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/secure.json
@@ -338,6 +338,9 @@
     ],
     "zookeeper_hosts": [
       "c6401.ambari.apache.org"
+    ],
+    "atlas_server_hosts": [
+      "c6401.ambari.apache.org"
     ]
 
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/b30d3c3a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
index b3f2acc..3797cf3 100644
--- a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
+++ b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
@@ -79,10 +79,17 @@ class TestAtlasServer(RMFTestCase):
     self.assertResourceCalled('File', '/usr/hdp/current/atlas-server/server/webapp/atlas.war',
                               content = StaticFile('/usr/hdp/current/atlas-server/server/webapp/atlas.war'),
     )
+    host_name = u"c6401.ambari.apache.org"
     app_props =  dict(self.getConfig()['configurations'][
                        'application-properties'])
     app_props['atlas.http.authentication.kerberos.name.rules'] = ' \\ \n'.join(app_props['atlas.http.authentication.kerberos.name.rules'].splitlines())
-    app_props['atlas.server.bind.address'] = 'c6401.ambari.apache.org'
+    app_props['atlas.server.bind.address'] = host_name
+    metadata_protocol = "https" if app_props["atlas.enableTLS"] is True else "http"
+    metadata_port = app_props["atlas.server.https.port"] if metadata_protocol == "https"
else app_props["atlas.server.http.port"]
+    app_props["atlas.rest.address"] = u'%s://%s:%s' % (metadata_protocol, host_name, metadata_port)
+    app_props["atlas.server.ids"] = "id1"
+    app_props["atlas.server.address.id1"] = u"%s:%s" % (host_name, metadata_port)
+    app_props["atlas.server.ha.enabled"] = "false"
 
     self.assertResourceCalled('File', '/etc/atlas/conf/atlas-log4j.xml',
                           content=InlineTemplate(
@@ -112,8 +119,8 @@ class TestAtlasServer(RMFTestCase):
     self.assertResourceCalled('PropertiesFile',
                               '/etc/atlas/conf/atlas-application.properties',
                               properties=app_props,
-                              owner='atlas',
-                              group='hadoop',
+                              owner=u'atlas',
+                              group=u'hadoop',
                               mode=0644,
                               )
     self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr-client',

http://git-wip-us.apache.org/repos/asf/ambari/blob/b30d3c3a/ambari-server/src/test/python/stacks/2.5/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/default.json b/ambari-server/src/test/python/stacks/2.5/configs/default.json
index cf36728..d0680da 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/default.json
@@ -383,6 +383,9 @@
     ],
     "ranger_tagsync_hosts": [
       "c6401.ambari.apache.org"
+    ],
+    "atlas_server_hosts": [
+      "c6401.ambari.apache.org"
     ]
 
   }


Mime
View raw message