ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tbeerbo...@apache.org
Subject ambari git commit: AMBARI-16464 - Atlas fails to come up with solr as indexing search when the zookeeper quorum has more than one host
Date Thu, 12 May 2016 21:41:53 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 e02110f16 -> a9f512e6f


AMBARI-16464 - Atlas fails to come up with solr as indexing search when the zookeeper quorum
has more than one host


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a9f512e6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a9f512e6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a9f512e6

Branch: refs/heads/branch-2.4
Commit: a9f512e6f337c967809ba0c9107e50e0516e5b0a
Parents: e02110f
Author: tbeerbower <tbeerbower@hortonworks.com>
Authored: Thu May 12 13:58:19 2016 -0400
Committer: tbeerbower <tbeerbower@hortonworks.com>
Committed: Thu May 12 17:41:10 2016 -0400

----------------------------------------------------------------------
 .../0.1.0.2.3/package/files/policy-store.txt    | 26 ++++++++++++++++++++
 .../package/files/users-credentials.properties  |  7 +++---
 .../ATLAS/0.1.0.2.3/package/scripts/metadata.py |  7 ++++++
 .../ATLAS/0.1.0.2.3/package/scripts/params.py   | 21 ++++++++++++----
 .../configuration/application-properties.xml    |  7 ++++++
 .../stacks/HDP/2.5/services/stack_advisor.py    |  5 ++--
 .../stacks/2.3/ATLAS/test_metadata_server.py    |  6 +++++
 .../stacks/2.5/ATLAS/test_atlas_server.py       |  6 +++++
 .../stacks/2.5/common/test_stack_advisor.py     |  2 +-
 9 files changed, 75 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/a9f512e6/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/policy-store.txt
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/policy-store.txt
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/policy-store.txt
new file mode 100644
index 0000000..4b3b126
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/policy-store.txt
@@ -0,0 +1,26 @@
+##
+## Licensed to the Apache Software Foundation (ASF) under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  The ASF licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+##Policy Format
+##r-READ, w-WRITE, u-UPDATE, d-DELETE
+##Policy_Name;;User_Name1:Operations_Allowed,User_Name2:Operations_Allowed;;Group_Name1:Operations_Allowed,Group_Name2:Operations_Allowed;;Resource_Type1:Resource_Name,Resource_Type2:Resource_Name
+##
+adminPolicy;;admin:rwud;;ROLE_ADMIN:rwud;;type:*,entity:*,operation:*
+typeReadPolicy;;nixon:rw;;;;type:*,entity:*
+classReadPolicy;;saqeeb:r;;;;type:*,entity:*
+dataScientistPolicy;;;;DATA_SCIENTIST:r;;type:*,entity:*
+dataStewardPolicy;;;;DATA_STEWARD:rwu;;type:*,entity:*

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9f512e6/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties
index 4d264a0..33b326f 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties
@@ -16,6 +16,7 @@
 # limitations under the License.
 #
 
-#username=password
-admin=admin
-user=user123
\ No newline at end of file
+#username=group::sha256-password
+admin=ADMIN::8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
+michael=DATA_SCIENTIST::95bfb24de17d285d734b9eaa9109bfe922adc85f20d2e5e66a78bddb4a4ebddb
+paul=DATA_STEWARD::e7c0dcf5f8a93e93791e9bac1ae454a691c1d2a902fc4256d489e96c1b9ac68c

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9f512e6/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
index 8e2fcc2..3138161 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
@@ -114,6 +114,13 @@ def metadata():
          content=StaticFile('users-credentials.properties')
     )
 
+    File(format("{conf_dir}/policy-store.txt"),
+         mode=0644,
+         owner=params.metadata_user,
+         group=params.user_group,
+         content=StaticFile('policy-store.txt')
+    )
+
     if params.atlas_has_embedded_hbase:
       # hbase-site for embedded hbase used by Atlas
       XmlConfig( "hbase-site.xml",

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9f512e6/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index 51fa039..f9422ee 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -62,6 +62,7 @@ conf_dir = status_params.conf_dir # "/etc/metadata/conf"
 conf_file = status_params.conf_file
 
 atlas_login_credentials_file = os.path.join(conf_dir, "users-credentials.properties")
+atlas_policy_store_file = os.path.join(conf_dir, "policy-store.txt")
 
 atlas_hbase_conf_dir = os.path.join(metadata_home, "hbase", "conf")
 atlas_hbase_log_dir = os.path.join(metadata_home, "hbase", "logs")
@@ -161,13 +162,23 @@ for host in atlas_hosts:
   id += 1
   first_id = False
 
+zookeeper_hosts = config['clusterHostInfo']['zookeeper_hosts']
 zookeeper_port = default('/configurations/zoo.cfg/clientPort', None)
-# get comma separated list of zookeeper hosts from clusterHostInfo
+logsearch_solr_znode = default("/configurations/logsearch-solr-env/logsearch_solr_znode",
None)
+
+# get comma separated lists of zookeeper hosts from clusterHostInfo
 index = 0
 zookeeper_quorum = ""
-for host in config['clusterHostInfo']['zookeeper_hosts']:
-  zookeeper_quorum += host + ":" + str(zookeeper_port)
+solr_zookeeper_url = ""
+
+for host in zookeeper_hosts:
+  zookeeper_host = host + ":" + str(zookeeper_port)
+
+  if logsearch_solr_znode is not None:
+    solr_zookeeper_url += zookeeper_host + logsearch_solr_znode
+
+  zookeeper_quorum += zookeeper_host
   index += 1
-  if index < len(config['clusterHostInfo']['zookeeper_hosts']):
+  if index < len(zookeeper_hosts):
     zookeeper_quorum += ","
-
+    solr_zookeeper_url += ","
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9f512e6/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
index 78600ee..03f825b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
@@ -65,6 +65,8 @@
   <property>
     <name>atlas.graph.index.search.solr.zookeeper-url</name>
     <description>The ZooKeeper quorum setup for Solr as comma separated value.</description>
+  </property>
+  <property>
     <name>atlas.login.method</name>
     <value>file</value>
     <description>Atlas login method (e.g. file).</description>
@@ -74,4 +76,9 @@
     <value>{{atlas_login_credentials_file}}</value>
     <description>File path for file based login.</description>
   </property>
+  <property>
+    <name>atlas.auth.policy.file</name>
+    <value>{{atlas_policy_store_file}}</value>
+    <description>Path for the Atlas policy file.</description>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9f512e6/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 5ae3664..b954c5f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -134,8 +134,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
 
     include_logsearch = "LOGSEARCH" in servicesList
     if include_logsearch and "logsearch-solr-env" in services["configurations"]:
-      logsearch_solr_znode = services["configurations"]["logsearch-solr-env"]["properties"]['logsearch_solr_znode']
-      putAtlasApplicationProperty('atlas.graph.index.search.solr.zookeeper-url', '{{zookeeper_quorum}}'
+ logsearch_solr_znode)
+      putAtlasApplicationProperty('atlas.graph.index.search.solr.zookeeper-url', '{{solr_zookeeper_url}}')
 
   def recommendHIVEConfigurations(self, configurations, clusterData, services, hosts):
     super(HDP25StackAdvisor, self).recommendHIVEConfigurations(configurations, clusterData,
services, hosts)
@@ -1098,4 +1097,4 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     hosts_for_component = []
     if serviceName in servicesList:
       hosts_for_component = [component["hostnames"] for component in componentsList if component["component_name"]
== componentName][0]
-    return hosts_for_component
\ No newline at end of file
+    return hosts_for_component

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9f512e6/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
index 8ae2c11..80a345e 100644
--- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
+++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
@@ -116,6 +116,12 @@ class TestMetadataServer(RMFTestCase):
                                 group='hadoop',
                                 mode=0644,
                                 )
+      self.assertResourceCalled('File', '/etc/atlas/conf/policy-store.txt',
+                                content=StaticFile('policy-store.txt'),
+                                owner='atlas',
+                                group='hadoop',
+                                mode=0644,
+                                )
       self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
                             owner = 'atlas',
                             group = 'hadoop',

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9f512e6/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
index 422b1c9..daa40d6 100644
--- a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
+++ b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
@@ -114,6 +114,12 @@ class TestAtlasServer(RMFTestCase):
                               group='hadoop',
                               mode=0644,
                               )
+    self.assertResourceCalled('File', '/etc/atlas/conf/policy-store.txt',
+                              content=StaticFile('policy-store.txt'),
+                              owner='atlas',
+                              group='hadoop',
+                              mode=0644,
+                              )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
                               owner = 'atlas',
                               group = 'hadoop',

http://git-wip-us.apache.org/repos/asf/ambari/blob/a9f512e6/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index fc4ebc6..cf82a9c 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -3471,7 +3471,7 @@ class TestHDP25StackAdvisor(TestCase):
     expected = {
       'application-properties': {
         'properties': {
-          'atlas.graph.index.search.solr.zookeeper-url': '{{zookeeper_quorum}}/logsearch',
+          'atlas.graph.index.search.solr.zookeeper-url': '{{solr_zookeeper_url}}',
         }
       },
       "logsearch-solr-env": {


Mime
View raw message