ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tbeerbo...@apache.org
Subject ambari git commit: AMBARI-16853 - Atlas Integration : Atlas support for Logsearch
Date Thu, 26 May 2016 14:32:38 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 ce4a8c241 -> 92c89bbfb


AMBARI-16853 - Atlas Integration : Atlas support for Logsearch


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/92c89bbf
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/92c89bbf
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/92c89bbf

Branch: refs/heads/branch-2.4
Commit: 92c89bbfbc4c398b2daf6ea36604baa2c1e62a22
Parents: ce4a8c2
Author: tbeerbower <tbeerbower@hortonworks.com>
Authored: Thu May 26 10:24:59 2016 -0400
Committer: tbeerbower <tbeerbower@hortonworks.com>
Committed: Thu May 26 10:29:44 2016 -0400

----------------------------------------------------------------------
 .../configuration/application-properties.xml    | 30 ---------
 .../ATLAS/0.1.0.2.3/configuration/atlas-env.xml |  2 +-
 .../configuration/atlas-hbase-site.xml          | 58 -----------------
 .../ATLAS/0.1.0.2.3/metainfo.xml                | 15 -----
 .../0.1.0.2.3/package/scripts/atlas_client.py   |  2 +-
 .../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 65 ++++++++++++--------
 .../ATLAS/0.1.0.2.3/package/scripts/params.py   | 26 +++++---
 .../configuration/application-properties.xml    | 33 +++++++++-
 .../services/ATLAS/configuration/atlas-env.xml  | 19 +++---
 .../stacks/HDP/2.5/services/ATLAS/metainfo.xml  | 16 ++++-
 .../stacks/HDP/2.5/services/stack_advisor.py    | 58 ++++++++++++++++-
 .../stacks/2.3/ATLAS/test_metadata_server.py    | 28 +++------
 .../test/python/stacks/2.3/configs/default.json |  6 +-
 .../test/python/stacks/2.3/configs/secure.json  |  6 +-
 .../stacks/2.5/ATLAS/test_atlas_server.py       | 29 +++------
 .../stacks/2.5/common/test_stack_advisor.py     | 16 ++++-
 .../test/python/stacks/2.5/configs/default.json |  6 +-
 17 files changed, 211 insertions(+), 204 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
index bf0467e..ca8e4c5 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
@@ -136,31 +136,6 @@
     <description>Indicates whether or not the notification service should be embedded.</description>
   </property>
   <property>
-    <name>atlas.kafka.data</name>
-    <value>{{metadata_home}}/data/kafka</value>
-    <description>The Kafka data directory.</description>
-  </property>
-  <property>
-    <name>atlas.kafka.bootstrap.servers</name>
-    <value>{{kafka_bootstrap_servers}}</value>
-    <description>Comma separated list of Kafka broker endpoints in host:port form</description>
-  </property>
-  <property>
-    <name>atlas.kafka.zookeeper.connect</name>
-    <value>{{kafka_zookeeper_connect}}</value>
-    <description>Comma separated list of servers forming Zookeeper quorum used by Kafka.</description>
-  </property>
-  <property>
-    <name>atlas.kafka.hook.group.id</name>
-    <value>atlas</value>
-    <description>Kafka group id for the hook topic.</description>
-  </property>
-  <property>
-    <name>atlas.kafka.entities.group.id</name>
-    <value>entities</value>
-    <description>Kafka group id for the entity topic.</description>
-  </property>
-  <property>
     <name>atlas.cluster.name</name>
     <value>{{cluster_name}}</value>
     <description>The cluster name.</description>
@@ -220,9 +195,4 @@
     <value>1000</value>
     <description></description>
   </property>
-  <property>
-    <name>atlas.audit.hbase.zookeeper.quorum</name>
-    <value>localhost:{{atlas_hbase_zk_port}}</value>
-    <description></description>
-  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
index dd4b3e2..d52d1a3 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
@@ -103,7 +103,7 @@ export METADATA_DATA_DIR={{data_dir}}
 export METADATA_PID_DIR={{pid_dir}}
 
 # hbase conf dir
-export HBASE_CONF_DIR={{atlas_hbase_conf_dir}}
+export HBASE_CONF_DIR={{hbase_conf_dir}}
 
 # Where do you want to expand the war file. By Default it is in /server/webapp dir under
the base install dir.
 export METADATA_EXPANDED_WEBAPP_DIR={{expanded_war_dir}}

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml
deleted file mode 100644
index 3c4826d..0000000
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml
+++ /dev/null
@@ -1,58 +0,0 @@
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="true">
-  <property>
-    <name>hbase.rootdir</name>
-    <value>file://{{metadata_home}}/data/root</value>
-  </property>
-  <property>
-    <name>hbase.zookeeper.property.dataDir</name>
-    <value>{{metadata_home}}/data/zookeeper-data</value>
-  </property>
-  <property>
-    <name>hbase.master.info.port</name>
-    <value>61510</value>
-  </property>
-  <property>
-    <name>hbase.regionserver.info.port</name>
-    <value>61530</value>
-  </property>
-  <property>
-    <name>hbase.master.port</name>
-    <value>61500</value>
-  </property>
-  <property>
-    <name>hbase.regionserver.port</name>
-    <value>61520</value>
-  </property>
-  <property>
-    <name>hbase.zookeeper.property.clientPort</name>
-    <value>61581</value>
-  </property>
-  <property>
-    <name>hbase.zookeeper.property.peerport</name>
-    <value>61588</value>
-  </property>
-  <property>
-    <name>hbase.zookeeper.property.leaderport</name>
-    <value>61589</value>
-  </property>
-</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml
index f4115f7..d0d73ad 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml
@@ -59,11 +59,6 @@
             <fileName>atlas-log4j.xml</fileName>
             <dictionaryName>atlas-log4j</dictionaryName>
           </configFile>
-          <configFile>
-            <type>xml</type>
-            <fileName>hbase-site.xml</fileName>
-            <dictionaryName>atlas-hbase-site</dictionaryName>
-          </configFile>
         </component>
 
         <component>
@@ -93,11 +88,6 @@
             <fileName>atlas-log4j.xml</fileName>
             <dictionaryName>atlas-log4j</dictionaryName>
           </configFile>
-          <configFile>
-            <type>xml</type>
-            <fileName>atlas-hbase-site.xml</fileName>
-            <dictionaryName>atlas-hbase-site</dictionaryName>
-          </configFile>
         </component>
       </components>
 
@@ -118,15 +108,10 @@
         <timeout>300</timeout>
       </commandScript>
 
-      <requiredServices>
-        <service>KAFKA</service>
-      </requiredServices>
-
       <configuration-dependencies>
         <config-type>application-properties</config-type>
         <config-type>atlas-env</config-type>
         <config-type>atlas-log4j</config-type>
-        <config-type>atlas-hbase-site</config-type>
       </configuration-dependencies>
 
     </service>

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py
index 2bdb4c0..a642d14 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py
@@ -53,7 +53,7 @@ class AtlasClient(Script):
   def configure(self, env):
     import params
     env.set_params(params)
-    metadata()
+    metadata('client')
 
   def status(self, env):
     raise ClientComponentHasNoStatus()

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
index 3138161..f633618 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
@@ -17,13 +17,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
+import random
 from resource_management import Directory, Fail, Logger, File, \
     InlineTemplate, PropertiesFile, StaticFile, XmlConfig
 from resource_management.libraries.functions import format
 from resource_management.libraries.resources.template_config import TemplateConfig
+from resource_management.libraries.functions import solr_cloud_util
 
 
-def metadata():
+def metadata(type='server'):
     import params
 
     Directory([params.pid_dir],
@@ -50,22 +52,6 @@ def metadata():
               create_parents = True
     )
 
-    Directory(params.atlas_hbase_log_dir,
-              mode=0755,
-              cd_access='a',
-              owner=params.metadata_user,
-              group=params.user_group,
-              create_parents = True
-              )
-
-    Directory(params.atlas_hbase_data_dir,
-              mode=0755,
-              cd_access='a',
-              owner=params.metadata_user,
-              group=params.user_group,
-              create_parents = True
-              )
-
     Directory(params.data_dir,
               mode=0644,
               cd_access='a',
@@ -121,16 +107,43 @@ def metadata():
          content=StaticFile('policy-store.txt')
     )
 
-    if params.atlas_has_embedded_hbase:
-      # hbase-site for embedded hbase used by Atlas
-      XmlConfig( "hbase-site.xml",
-             conf_dir = params.atlas_hbase_conf_dir,
-             configurations = params.config['configurations']['atlas-hbase-site'],
-             configuration_attributes=params.config['configuration_attributes']['atlas-hbase-site'],
-             owner = params.metadata_user,
-             group = params.user_group
-             )
+    if type == 'server':
+      random_num = random.random()
+
+      upload_conf_set('basic_configs', random_num)
+
+      create_collection('vertex_index', 'basic_configs')
+      create_collection('edge_index', 'basic_configs')
+      create_collection('fulltext_index', 'basic_configs')
 
     if params.security_enabled:
         TemplateConfig(format(params.atlas_jaas_file),
                          owner=params.metadata_user)
+
+def upload_conf_set(config_set, random_num):
+  import params
+  tmp_config_set_folder = format('{tmp_dir}/solr_config_{config_set}_{random_num}')
+
+  solr_cloud_util.upload_configuration_to_zk(
+      zookeeper_quorum=params.zookeeper_quorum,
+      solr_znode=params.logsearch_solr_znode,
+      config_set_dir=format("{logsearch_solr_dir}/server/solr/configsets/{config_set}/conf"),
+      config_set=config_set,
+      tmp_config_set_dir=tmp_config_set_folder,
+      java64_home=params.java64_home,
+      user=params.metadata_user,
+      group=params.user_group)
+
+def create_collection(collection, config_set):
+  import params
+
+  solr_cloud_util.create_collection(
+      zookeeper_quorum=params.zookeeper_quorum,
+      solr_znode=params.logsearch_solr_znode,
+      collection = collection,
+      config_set=config_set,
+      java64_home=params.java64_home,
+      user=params.metadata_user,
+      group=params.user_group,
+      shards=params.atlas_solr_shards,
+      replication_factor = params.logsearch_solr_replication_factor)

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index 1ce9cda..edbb527 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -28,6 +28,7 @@ import status_params
 # server configurations
 config = Script.get_config()
 stack_root = Script.get_stack_root()
+tmp_dir = Script.get_tmp_dir()
 
 cluster_name = config['clusterName']
 
@@ -64,13 +65,6 @@ conf_file = status_params.conf_file
 atlas_login_credentials_file = os.path.join(conf_dir, "users-credentials.properties")
 atlas_policy_store_file = os.path.join(conf_dir, "policy-store.txt")
 
-atlas_hbase_conf_dir = os.path.join(metadata_home, "hbase", "conf")
-atlas_hbase_log_dir = os.path.join(metadata_home, "hbase", "logs")
-atlas_hbase_data_dir = os.path.join(metadata_home, "data")
-atlas_hbase_zk_port = default("/configurations/atlas-hbase-site/hbase.zookeeper.property.clientPort",
None)
-
-atlas_has_embedded_hbase = default("/configurations/atlas-env/has_embedded_hbase", False)
-
 # service locations
 hadoop_conf_dir = os.path.join(os.environ["HADOOP_HOME"], "conf") if 'HADOOP_HOME' in os.environ
else '/etc/hadoop/conf'
 
@@ -143,6 +137,10 @@ if not len(kafka_broker_hosts) == 0:
 
 kafka_zookeeper_connect = default("/configurations/kafka-broker/zookeeper.connect", None)
 
+# hbase
+hbase_zookeeper_quorum = default('/configurations/hbase-site/hbase.zookeeper.quorum', None)
+hbase_conf_dir = "/etc/hbase/conf"
+
 # atlas HA
 atlas_hosts = sorted(default('/clusterHostInfo/atlas_server_hosts', []))
 
@@ -163,9 +161,17 @@ for host in atlas_hosts:
   id += 1
   first_id = False
 
+# logsearch solr
+logsearch_solr_znode = default("/configurations/logsearch-solr-env/logsearch_solr_znode",
None)
+logsearch_solr_dir = '/usr/lib/ambari-logsearch-solr'
+logsearch_solr_hosts = default("/clusterHostInfo/logsearch_solr_hosts", [])
+logsearch_solr_replication_factor = 2 if len(logsearch_solr_hosts) > 1 else 1
+atlas_solr_shards = default("/configurations/atlas-env/atlas_solr-shards", 1)
+has_logsearch_solr = len(logsearch_solr_hosts) > 0
+
+# zookeeper
 zookeeper_hosts = config['clusterHostInfo']['zookeeper_hosts']
 zookeeper_port = default('/configurations/zoo.cfg/clientPort', None)
-logsearch_solr_znode = default("/configurations/logsearch-solr-env/logsearch_solr_znode",
None)
 
 # get comma separated lists of zookeeper hosts from clusterHostInfo
 index = 0
@@ -173,7 +179,9 @@ zookeeper_quorum = ""
 solr_zookeeper_url = ""
 
 for host in zookeeper_hosts:
-  zookeeper_host = host + ":" + str(zookeeper_port)
+  zookeeper_host = host
+  if zookeeper_port is not None:
+    zookeeper_host = host + ":" + str(zookeeper_port)
 
   if logsearch_solr_znode is not None:
     solr_zookeeper_url += zookeeper_host + logsearch_solr_znode

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
index 0631b7d..1b4d0ea 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml
@@ -49,7 +49,12 @@
   </property>
   <property>
     <name>atlas.graph.storage.hostname</name>
-    <value>localhost:{{atlas_hbase_zk_port}}</value>
+    <value>{{hbase_zookeeper_quorum}}</value>
+    <description></description>
+  </property>
+  <property>
+    <name>atlas.audit.hbase.zookeeper.quorum</name>
+    <value>{{hbase_zookeeper_quorum}}</value>
     <description></description>
   </property>
   <property>
@@ -64,6 +69,7 @@
   </property>
   <property>
     <name>atlas.graph.index.search.solr.zookeeper-url</name>
+    <value>{{solr_zookeeper_url}}</value>
     <description>The ZooKeeper quorum setup for Solr as comma separated value.</description>
   </property>
   <property>
@@ -82,6 +88,31 @@
     <description>Path for the Atlas policy file.</description>
   </property>
   <property>
+    <name>atlas.kafka.data</name>
+    <value>{{metadata_home}}/data/kafka</value>
+    <description>The Kafka data directory.</description>
+  </property>
+  <property>
+    <name>atlas.kafka.bootstrap.servers</name>
+    <value>{{kafka_bootstrap_servers}}</value>
+    <description>Comma separated list of Kafka broker endpoints in host:port form</description>
+  </property>
+  <property>
+    <name>atlas.kafka.zookeeper.connect</name>
+    <value>{{kafka_zookeeper_connect}}</value>
+    <description>Comma separated list of servers forming Zookeeper quorum used by Kafka.</description>
+  </property>
+  <property>
+    <name>atlas.kafka.hook.group.id</name>
+    <value>atlas</value>
+    <description>Kafka group id for the hook topic.</description>
+  </property>
+  <property>
+    <name>atlas.kafka.entities.group.id</name>
+    <value>entities</value>
+    <description>Kafka group id for the entity topic.</description>
+  </property>
+  <property>
     <name>atlas.kafka.auto.commit.enable</name>
     <value>false</value>
     <description>Kafka auto commit setting for Atlas notifications.</description>

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml
index 2a5f777..e44872d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml
@@ -80,16 +80,15 @@
     </value-attributes>
   </property>
   <property>
-    <name>has_embedded_hbase</name>
-    <value>true</value>
-    <description>Indicates whether or not Atlas can install embedded Hbase.</description>
-    <value-attributes>
-      <read-only>true</read-only>
-      <overridable>false</overridable>
-      <visible>false</visible>
-    </value-attributes>
+    <name>atlas_solr_shards</name>
+    <value>1</value>
+    <description>The number of shards set for LogSearch SOLR.</description>
+  </property>
+  <property>
+    <name>atlas_solr_shards</name>
+    <value>1</value>
+    <description>The number of shards set for collections created in LogSearch SOLR.</description>
   </property>
-
   <!-- metadata-env.sh -->
   <property>
     <name>content</name>
@@ -122,7 +121,7 @@
       export ATLAS_PID_DIR={{pid_dir}}
 
       # hbase conf dir
-      export HBASE_CONF_DIR={{atlas_hbase_conf_dir}}
+      export HBASE_CONF_DIR={{hbase_conf_dir}}
 
       # Where do you want to expand the war file. By Default it is in /server/webapp dir
under the base install dir.
       export ATLAS_EXPANDED_WEBAPP_DIR={{expanded_war_dir}}

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
index 15daeea..4890f00 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
@@ -25,12 +25,21 @@
         <component>
           <name>ATLAS_SERVER</name>
           <cardinality>1+</cardinality>
+          <dependencies>
+            <dependency>
+              <name>LOGSEARCH/LOGSEARCH_SOLR_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+          </dependencies>
           <configFile>
             <type>properties</type>
             <fileName>atlas-application.properties</fileName>
             <dictionaryName>application-properties</dictionaryName>
           </configFile>
-         </component>
+        </component>
 
         <component>
           <name>ATLAS_CLIENT</name>
@@ -41,6 +50,11 @@
           </configFile>
         </component>
       </components>
+
+      <requiredServices>
+        <service>KAFKA</service>
+      </requiredServices>
+
     </service>
   </services>
 </metainfo>

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 093c819..44407ca 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -51,6 +51,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
   def getServiceConfigurationValidators(self):
     parentValidators = super(HDP25StackAdvisor, self).getServiceConfigurationValidators()
     childValidators = {
+      "ATLAS": {"application-properties": self.validateAtlasConfigurations},
       "HIVE": {"hive-interactive-env": self.validateHiveInteractiveEnvConfigurations},
       "YARN": {"yarn-site": self.validateYarnConfigurations},
       "RANGER": {"ranger-tagsync-site": self.validateRangerTagsyncConfigurations}
@@ -58,6 +59,44 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     self.mergeValidators(parentValidators, childValidators)
     return parentValidators
 
+  def validateAtlasConfigurations(self, properties, recommendedDefaults, configurations,
services, hosts):
+    application_properties = getSiteProperties(configurations, "application-properties")
+    validationItems = []
+
+    if application_properties['atlas.graph.index.search.backend'] == 'solr5' and \
+            not application_properties['atlas.graph.index.search.solr.zookeeper-url']:
+      validationItems.append({"config-name": "atlas.graph.index.search.solr.zookeeper-url",
+                              "item": self.getErrorItem(
+                                  "If LOGSEARCH is not installed then the SOLR zookeeper
url configuration must be specified.")})
+
+    if not application_properties['atlas.kafka.bootstrap.servers']:
+      validationItems.append({"config-name": "atlas.kafka.bootstrap.servers",
+                              "item": self.getErrorItem(
+                                  "If KAFKA is not installed then the Kafka bootstrap servers
configuration must be specified.")})
+
+    if not application_properties['atlas.kafka.zookeeper.connect']:
+      validationItems.append({"config-name": "atlas.kafka.zookeeper.connect",
+                              "item": self.getErrorItem(
+                                  "If KAFKA is not installed then the Kafka zookeeper quorum
configuration must be specified.")})
+
+    if application_properties['atlas.graph.storage.backend'] == 'hbase':
+      if not application_properties['atlas.graph.storage.hostname']:
+        validationItems.append({"config-name": "atlas.graph.storage.hostname",
+                                "item": self.getErrorItem(
+                                    "If HBASE is not installed then the hbase zookeeper quorum
configuration must be specified.")})
+      elif application_properties['atlas.graph.storage.hostname'] == '{{hbase_zookeeper_quorum}}':
+        validationItems.append({"config-name": "atlas.graph.storage.hostname",
+                                "item": self.getWarnItem(
+                                    "Note that Atlas is configured to use the HBASE instance
being installed for this cluster.")})
+
+      if not application_properties['atlas.audit.hbase.zookeeper.quorum']:
+        validationItems.append({"config-name": "atlas.audit.hbase.zookeeper.quorum",
+                                "item": self.getErrorItem(
+                                    "If HBASE is not installed then the audit hbase zookeeper
quorum configuration must be specified.")})
+
+    validationProblems = self.toConfigurationValidationProblems(validationItems, "application-properties")
+    return validationProblems
+
   def validateYarnConfigurations(self, properties, recommendedDefaults, configurations, services,
hosts):
     parentValidationProblems = super(HDP25StackAdvisor, self).validateYARNConfigurations(properties,
recommendedDefaults, configurations, services, hosts)
     yarn_site_properties = getSiteProperties(configurations, "yarn-site")
@@ -134,9 +173,24 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
 
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
 
-    include_logsearch = "LOGSEARCH" in servicesList
-    if include_logsearch and "logsearch-solr-env" in services["configurations"]:
+    if "LOGSEARCH" in servicesList:
       putAtlasApplicationProperty('atlas.graph.index.search.solr.zookeeper-url', '{{solr_zookeeper_url}}')
+    else:
+      putAtlasApplicationProperty('atlas.graph.index.search.solr.zookeeper-url', "")
+
+    if "KAFKA" in servicesList:
+      putAtlasApplicationProperty('atlas.kafka.bootstrap.servers', '{{kafka_bootstrap_servers}}')
+      putAtlasApplicationProperty('atlas.kafka.zookeeper.connect', '{{kafka_zookeeper_connect}}')
+    else:
+      putAtlasApplicationProperty('atlas.kafka.bootstrap.servers', "")
+      putAtlasApplicationProperty('atlas.kafka.zookeeper.connect', "")
+
+    if "HBASE" in servicesList:
+      putAtlasApplicationProperty('atlas.graph.storage.hostname', '{{hbase_zookeeper_quorum}}')
+      putAtlasApplicationProperty('atlas.audit.hbase.zookeeper.quorum', '{{hbase_zookeeper_quorum}}')
+    else:
+      putAtlasApplicationProperty('atlas.graph.storage.hostname', "")
+      putAtlasApplicationProperty('atlas.audit.hbase.zookeeper.quorum', "")
 
   def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
     super(HDP25StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData,
services, hosts)

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
index 98fc678..faa7350 100644
--- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
+++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
@@ -52,20 +52,6 @@ class TestMetadataServer(RMFTestCase):
                                 cd_access='a',
                                 mode=0755
       )
-      self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/hbase/logs',
-                                owner='atlas',
-                                group='hadoop',
-                                create_parents = True,
-                                cd_access='a',
-                                mode=0755
-                                )
-      self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/data',
-                                owner='atlas',
-                                group='hadoop',
-                                create_parents = True,
-                                cd_access='a',
-                                mode=0755
-                                )
       self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/data',
                                 owner='atlas',
                                 group='hadoop',
@@ -122,13 +108,13 @@ class TestMetadataServer(RMFTestCase):
                                 group='hadoop',
                                 mode=0644,
                                 )
-      self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
-                            owner = 'atlas',
-                            group = 'hadoop',
-                            conf_dir = '/usr/hdp/current/atlas-server/hbase/conf',
-                            configurations = self.getConfig()['configurations']['atlas-hbase-site'],
-                            configuration_attributes = self.getConfig()['configuration_attributes']['atlas-hbase-site']
-                            )
+
+      self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45
; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch
--download-config -d /tmp/solr_config_basic_configs_0.[0-9]* -cs basic_configs -rt 5 -i 10')
+      self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45
; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch
--upload-config -d /usr/lib/ambari-logsearch-solr/server/solr/configsets/basic_configs/conf
-cs basic_configs -rt 5 -i 10')
+
+      self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45
; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch
--create-collection -c vertex_index -cs basic_configs -s 1 -r 1 -m 1 -rt 5 -i 10')
+      self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45
; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch
--create-collection -c edge_index -cs basic_configs -s 1 -r 1 -m 1 -rt 5 -i 10')
+      self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45
; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch
--create-collection -c fulltext_index -cs basic_configs -s 1 -r 1 -m 1 -rt 5 -i 10')
 
   def test_configure_default(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py",

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/test/python/stacks/2.3/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json
index c8b418b..ef04248 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json
@@ -171,6 +171,9 @@
         "log.index.interval.bytes": "4096",
         "log.retention.hours": "168"
       },
+      "logsearch-solr-env": {
+        "logsearch_solr_znode": "/logsearch"
+      },
       "application-properties": {
         "atlas.cluster.name" : "c2",
         "atlas.graph.storage.backend": "berkeleyje",
@@ -210,8 +213,7 @@
         "metadata_log_dir": "/var/log/atlas",
         "metadata_data_dir": "/var/lib/atlas/data",
         "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp",
-        "metadata_conf_file": "application.properties",
-        "has_embedded_hbase":"true"
+        "metadata_conf_file": "application.properties"
       },
       "atlas-log4j": {
         "content": "<property><name>content</name><description>Custom
log4j.properties</description><value></value></property>",

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/test/python/stacks/2.3/configs/secure.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/secure.json b/ambari-server/src/test/python/stacks/2.3/configs/secure.json
index 7ebcedf..166fd6f 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/secure.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/secure.json
@@ -162,6 +162,9 @@
       "log.index.interval.bytes": "4096",
       "log.retention.hours": "168"
     },
+    "logsearch-solr-env": {
+      "logsearch_solr_znode": "/logsearch"
+    },
     "application-properties": {
       "atlas.cluster.name" : "c2",
       "atlas.graph.storage.backend": "berkeleyje",
@@ -202,8 +205,7 @@
       "metadata_log_dir": "/var/log/atlas",
       "metadata_data_dir": "/var/lib/atlas/data",
       "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp",
-      "metadata_conf_file": "application.properties",
-      "has_embedded_hbase":"true"
+      "metadata_conf_file": "application.properties"
     },
     "atlas-log4j": {
       "content": "<property><name>content</name><description>Custom
log4j.properties</description><value></value></property>",

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
index 9d0f00c..6839dee 100644
--- a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
+++ b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
@@ -49,20 +49,6 @@ class TestAtlasServer(RMFTestCase):
                               cd_access='a',
                               mode=0755
                               )
-    self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/hbase/logs',
-                              owner='atlas',
-                              group='hadoop',
-                              create_parents = True,
-                              cd_access='a',
-                              mode=0755
-                              )
-    self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/data',
-                              owner='atlas',
-                              group='hadoop',
-                              create_parents = True,
-                              cd_access='a',
-                              mode=0755
-                              )
     self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/data',
                               owner='atlas',
                               group='hadoop',
@@ -120,13 +106,14 @@ class TestAtlasServer(RMFTestCase):
                               group='hadoop',
                               mode=0644,
                               )
-    self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
-                              owner = 'atlas',
-                              group = 'hadoop',
-                              conf_dir = '/usr/hdp/current/atlas-server/hbase/conf',
-                              configurations = self.getConfig()['configurations']['atlas-hbase-site'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['atlas-hbase-site']
-                              )
+
+    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45
; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch
--download-config -d /tmp/solr_config_basic_configs_0.[0-9]* -cs basic_configs -rt 5 -i 10')
+    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45
; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch
--upload-config -d /usr/lib/ambari-logsearch-solr/server/solr/configsets/basic_configs/conf
-cs basic_configs -rt 5 -i 10')
+
+    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45
; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch
--create-collection -c vertex_index -cs basic_configs -s 1 -r 1 -m 1 -rt 5 -i 10')
+    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45
; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch
--create-collection -c edge_index -cs basic_configs -s 1 -r 1 -m 1 -rt 5 -i 10')
+    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45
; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch
--create-collection -c fulltext_index -cs basic_configs -s 1 -r 1 -m 1 -rt 5 -i 10')
+
 
   def test_configure_default(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py",

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index a7e1293..01064a6 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -4772,7 +4772,11 @@ class TestHDP25StackAdvisor(TestCase):
     configurations = {
       "application-properties": {
         "properties": {
-          "atlas.graph.index.search.solr.zookeeper-url": ""
+          "atlas.graph.index.search.solr.zookeeper-url": "",
+          "atlas.audit.hbase.zookeeper.quorum": "",
+          "atlas.graph.storage.hostname": "",
+          "atlas.kafka.bootstrap.servers": "",
+          "atlas.kafka.zookeeper.connect": ""
         }
       },
       "logsearch-solr-env": {
@@ -4793,6 +4797,10 @@ class TestHDP25StackAdvisor(TestCase):
       'application-properties': {
         'properties': {
           'atlas.graph.index.search.solr.zookeeper-url': '{{solr_zookeeper_url}}',
+          "atlas.audit.hbase.zookeeper.quorum": "",
+          "atlas.graph.storage.hostname": "",
+          "atlas.kafka.bootstrap.servers": "",
+          "atlas.kafka.zookeeper.connect": ""
         }
       },
       "logsearch-solr-env": {
@@ -4831,7 +4839,11 @@ class TestHDP25StackAdvisor(TestCase):
       "configurations": {
         "application-properties": {
           "properties": {
-            "atlas.graph.index.search.solr.zookeeper-url": ""
+            "atlas.graph.index.search.solr.zookeeper-url": "",
+            "atlas.audit.hbase.zookeeper.quorum": "",
+            "atlas.graph.storage.hostname": "",
+            "atlas.kafka.bootstrap.servers": "",
+            "atlas.kafka.zookeeper.connect": ""
           }
         },
         "logsearch-solr-env": {

http://git-wip-us.apache.org/repos/asf/ambari/blob/92c89bbf/ambari-server/src/test/python/stacks/2.5/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/default.json b/ambari-server/src/test/python/stacks/2.5/configs/default.json
index e4a97f6..0077280 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/default.json
@@ -171,6 +171,9 @@
       "log.index.interval.bytes": "4096",
       "log.retention.hours": "168"
     },
+    "logsearch-solr-env": {
+      "logsearch_solr_znode": "/logsearch"
+    },
     "application-properties": {
       "atlas.cluster.name" : "c2",
       "atlas.graph.storage.backend": "berkeleyje",
@@ -210,8 +213,7 @@
       "metadata_log_dir": "/var/log/atlas",
       "metadata_data_dir": "/var/lib/atlas/data",
       "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp",
-      "metadata_conf_file": "atlas-application.properties",
-      "has_embedded_hbase":"true"
+      "metadata_conf_file": "atlas-application.properties"
     },
     "atlas-log4j": {
       "content": "<property><name>content</name><description>Custom
log4j.properties</description><value></value></property>",


Mime
View raw message