ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From oleew...@apache.org
Subject [59/61] [abbrv] ambari git commit: AMBARI-15679. Stack definition for Logsearch (oleewre)
Date Sat, 09 Apr 2016 10:15:55 GMT
AMBARI-15679. Stack definition for Logsearch (oleewre)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cb6a7b9b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cb6a7b9b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cb6a7b9b

Branch: refs/heads/branch-dev-logsearch
Commit: cb6a7b9bc145a5b01d51418c2fe335321a21d718
Parents: f729469
Author: oleewere <oleewere@gmail.com>
Authored: Sat Apr 9 11:59:06 2016 +0200
Committer: oleewere <oleewere@gmail.com>
Committed: Sat Apr 9 11:59:06 2016 +0200

----------------------------------------------------------------------
 .../libraries/functions/package_conditions.py   |    4 +
 .../common-services/LOGSEARCH/0.5.0/alerts.json |   63 +
 .../0.5.0/configuration/logfeeder-env.xml       |  118 ++
 .../0.5.0/configuration/logfeeder-log4j.xml     |   80 +
 .../0.5.0/configuration/logfeeder-site.xml      |   87 +
 .../logsearch-audit_logs-solrconfig.xml         | 1929 ++++++++++++++++++
 .../0.5.0/configuration/logsearch-env.xml       |  143 ++
 .../0.5.0/configuration/logsearch-log4j.xml     |  110 +
 .../logsearch-service_logs-solrconfig.xml       | 1928 +++++++++++++++++
 .../0.5.0/configuration/logsearch-site.xml      |   80 +
 .../0.5.0/configuration/logsearch-solr-env.xml  |  193 ++
 .../configuration/logsearch-solr-log4j.xml      |   57 +
 .../0.5.0/configuration/logsearch-solr-xml.xml  |   41 +
 .../LOGSEARCH/0.5.0/kerberos.json               |   17 +
 .../LOGSEARCH/0.5.0/metainfo.xml                |  119 ++
 .../0.5.0/package/scripts/logfeeder.py          |   67 +
 .../0.5.0/package/scripts/logsearch.py          |   65 +
 .../LOGSEARCH/0.5.0/package/scripts/params.py   |  231 +++
 .../0.5.0/package/scripts/setup_logfeeder.py    |   67 +
 .../0.5.0/package/scripts/setup_logsearch.py    |   93 +
 .../0.5.0/package/scripts/setup_solr.py         |   67 +
 .../LOGSEARCH/0.5.0/package/scripts/solr.py     |   66 +
 .../0.5.0/package/scripts/status_params.py      |   36 +
 .../package/templates/global.config.json.j2     |   28 +
 .../0.5.0/package/templates/grok-patterns.j2    |  144 ++
 .../templates/input.config-accumulo.json.j2     |  105 +
 .../templates/input.config-ambari.json.j2       |   93 +
 .../package/templates/input.config-ams.json.j2  |   92 +
 .../templates/input.config-atlas.json.j2        |   55 +
 .../templates/input.config-falcon.json.j2       |   55 +
 .../templates/input.config-hbase.json.j2        |   62 +
 .../package/templates/input.config-hdfs.json.j2 |  246 +++
 .../package/templates/input.config-hive.json.j2 |   62 +
 .../templates/input.config-kafka.json.j2        |  105 +
 .../package/templates/input.config-knox.json.j2 |   68 +
 .../templates/input.config-logsearch.json.j2    |   68 +
 .../templates/input.config-oozie.json.j2        |   56 +
 .../templates/input.config-ranger.json.j2       |  122 ++
 .../templates/input.config-storm.json.j2        |   86 +
 .../package/templates/input.config-yarn.json.j2 |   86 +
 .../templates/input.config-zookeeper.json.j2    |   56 +
 .../package/templates/logfeeder.properties.j2   |   18 +
 .../package/templates/logsearch.properties.j2   |   40 +
 .../package/templates/output.config.json.j2     |   97 +
 .../0.5.0/package/templates/zoo.cfg.j2          |   31 +
 .../LOGSEARCH/0.5.0/quicklinks/quicklinks.json  |   28 +
 .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml     |    3 +
 .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml     |    3 +
 .../stacks/HDP/2.3/role_command_order.json      |    5 +-
 .../HDP/2.3/services/LOGSEARCH/metainfo.xml     |   26 +
 .../stacks/HDP/2.3/upgrades/upgrade-2.3.xml     |    3 +
 .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml     |    3 +
 .../stacks/HDP/2.4/upgrades/upgrade-2.4.xml     |    3 +
 .../stacks/2.4/LOGSEARCH/test_logfeeder.py      |  152 ++
 .../stacks/2.4/LOGSEARCH/test_logsearch.py      |  159 ++
 .../python/stacks/2.4/LOGSEARCH/test_solr.py    |  155 ++
 .../test/python/stacks/2.4/configs/default.json |  398 ++++
 .../src/test/python/stacks/utils/RMFTestCase.py |   14 +-
 .../HDP/2.2.0/upgrades/upgrade_test_checks.xml  |    1 +
 .../app/mappers/service_metrics_mapper.js       |    3 +-
 ambari-web/app/models/quick_links.js            |   13 +
 .../app/views/common/quick_view_link_view.js    |    6 +-
 .../test/views/common/quick_link_view_test.js   |    5 +
 63 files changed, 8412 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
index df46ce0..e164f56 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
@@ -46,6 +46,10 @@ def should_install_ams_grafana():
   config = Script.get_config()
   return 'role' in config and config['role'] == "METRICS_GRAFANA"
 
+def should_install_logsearch_solr():
+  config = Script.get_config()
+  return 'role' in config and config['role'] != "LOGSEARCH_LOGFEEDER"
+
 def should_install_mysql():
   config = Script.get_config()
   hive_database = config['configurations']['hive-env']['hive_database']

http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json
new file mode 100644
index 0000000..32bbf67
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json
@@ -0,0 +1,63 @@
+{
+  "LOGSEARCH": {
+    "LOGSEARCH_SERVER": [
+      {
+        "name": "logsearch_ui",
+        "label": "LogSearch Web UI",
+        "description": "This host-level alert is triggered if the LogSearch UI is unreachable.",
+        "interval": 1,
+        "scope": "ANY",
+        "source": {
+          "type": "WEB",
+          "uri": {
+            "http": "{{logsearch-site/logsearch.ui.port}}",
+            "https": "{{logsearch-site/logsearch.ui.port}}",
+            "connection_timeout": 5.0,
+            "default_port": 61888
+          },
+          "reporting": {
+            "ok": {
+              "text": "HTTP {0} response in {2:.3f}s"
+            },
+            "warning":{
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
+            },
+            "critical": {
+              "text": "Connection failed to {1} ({3})"
+            }
+          }
+        }
+      }
+    ],
+    "LOGSEARCH_SOLR": [
+      {
+        "name": "logsearch_solr",
+        "label": "Solr Web UI",
+        "description": "This host-level alert is triggered if the Solr Cloud Instance is
unreachable.",
+        "interval": 1,
+        "scope": "ANY",
+        "source": {
+          "type": "WEB",
+          "uri": {
+            "http": "{{logsearch-solr-env/logsearch_solr_port}}",
+            "https": "{{logsearch-solr-env/logsearch_solr_port}}",
+            "connection_timeout": 5.0,
+            "default_port": 8886
+          },
+          "reporting": {
+            "ok": {
+              "text": "HTTP {0} response in {2:.3f}s"
+            },
+            "warning":{
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
+            },
+            "critical": {
+              "text": "Connection failed to {1} ({3})"
+            }
+          }
+        }
+      }
+    ]
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
new file mode 100644
index 0000000..2722be8
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
@@ -0,0 +1,118 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+
+<!-- This is a special config file for properties used to monitor status of the service
-->
+<configuration supports_adding_forbidden="true">
+
+  <property>
+    <name>logfeeder_pid_dir</name>
+    <value>/var/run/ambari-logsearch-logfeeder</value>
+    <description>logfeeder Process ID Directory</description>
+    <display-name>Logfeeder pid dir</display-name>
+  </property>
+
+  <property>
+    <name>logfeeder_log_dir</name>
+    <value>/var/log/ambari-logsearch-logfeeder</value>
+    <description>Log dir for logfeeder</description>
+    <display-name>Logfeeder log dir</display-name>
+  </property>
+
+  <property>
+    <name>logfeeder_checkpoint_folder</name>
+    <value>/etc/ambari-logsearch-logfeeder/conf/checkpoints</value>
+    <description>Checkpoint folder for logfeeder</description>
+    <display-name>Logfeeder checkpoint dir</display-name>
+  </property>
+
+  <property>
+    <name>logfeeder_config_files</name>
+    <value>{{logfeeder_config_files}}</value>
+    <description>Comma separated config files in grok format</description>
+    <display-name>Logfeeder config files</display-name>
+  </property>
+  
+  <property>
+    <name>logfeeder_user</name>
+    <value>logfeeder</value>
+    <property-type>USER</property-type>
+    <description>logfeeder user</description>
+    <display-name>Logsfeeder User</display-name>
+  </property>
+
+  <property>
+    <name>logfeeder_group</name>
+    <value>logfeeder</value>
+    <property-type>GROUP</property-type>
+    <description>logfeeder group</description>
+    <display-name>Logsfeeder Group</display-name>
+  </property> 
+
+  <property>
+    <name>logfeeder_max_mem</name>
+    <value>512m</value>
+    <description>Max memory for LogFeeder</description>
+    <display-name>Logsfeeder max memory</display-name>
+  </property>
+
+  <property>
+    <name>content</name>
+    <description>This is the jinja template for config.json file</description>
+    <value>#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+export LOGFEEDER_PATH={{logfeeder_dir}}
+
+export LOGFEEDER_CONF_DIR={{logsearch_logfeeder_conf}}
+
+#Logfile e.g. /var/log/logfeeder.log
+export LOGFILE={{logfeeder_log}}
+
+#pid file e.g. /var/run/logfeeder.pid
+export PID_FILE={{logfeeder_pid_file}}
+
+export JAVA_HOME={{java64_home}}
+
+if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then
+  export LOGFEEDER_JAVA_MEM=-Xmx{{logfeeder_max_mem}}
+fi
+    </value>
+  </property>
+
+  
+</configuration>  

http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml
new file mode 100644
index 0000000..5a3331e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<!-- This is a special config file for properties used to monitor status of the service
-->
+<configuration supports_adding_forbidden="true">
+
+  <!-- solrconfig.xml -->
+
+  <property>
+    <name>content</name>
+    <description>This is the jinja template for solrconfig.xml file for service logs</description>
+    <value>&lt;?xml version="1.0" encoding="UTF-8" ?&gt;
+&lt;!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+--&gt;
+&lt;!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"&gt;
+&lt;log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"&gt;
+  &lt;appender name="console" class="org.apache.log4j.ConsoleAppender"&gt;
+    &lt;param name="Target" value="System.out" /&gt;
+    &lt;layout class="org.apache.log4j.PatternLayout"&gt;
+      &lt;param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" /&gt;
+    &lt;/layout&gt;
+  &lt;/appender&gt;
+
+  &lt;appender name="rolling_file" class="org.apache.log4j.RollingFileAppender"&gt;

+    &lt;param name="file" value="{{logfeeder_log_dir}}/logfeeder.log" /&gt;
+    &lt;param name="append" value="true" /&gt; 
+    &lt;param name="maxFileSize" value="10MB" /&gt; 
+    &lt;param name="maxBackupIndex" value="10" /&gt; 
+    &lt;layout class="org.apache.log4j.PatternLayout"&gt; 
+      &lt;param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n"/&gt;

+    &lt;/layout&gt; 
+  &lt;/appender&gt; 
+
+  &lt;category name="org.apache.ambari.logfeeder" additivity="false"&gt;
+    &lt;priority value="info" /&gt;
+    &lt;appender-ref ref="rolling_file" /&gt;
+  &lt;/category&gt;
+
+  &lt;root&gt;
+    &lt;priority value="warn" /&gt;
+    &lt;appender-ref ref="rolling_file" /&gt;
+  &lt;/root&gt;
+&lt;/log4j:configuration&gt;  
+    </value>
+  </property>
+
+  
+</configuration>  

http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-site.xml
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-site.xml
new file mode 100644
index 0000000..40ee374
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-site.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+  <property>
+    <name>logfeeder.solr.service.logs.enable</name>
+    <value>true</value>
+    <display-name>Service Logs enabled</display-name>
+    <description>Enable service logs to Solr</description>
+  </property>
+
+  <property>
+    <name>logfeeder.solr.audit.logs.enable</name>
+    <value>true</value>
+    <display-name>Audit Logs enabled</display-name>
+    <description>Enable audit logs to Solr</description>
+  </property>
+
+  <property>
+    <name>logfeeder.kafka.service.logs.enable</name>
+    <value>false</value>
+    <display-name>Kafka Service Logs enabled</display-name>
+    <description>Enable service logs to Kafka</description>
+  </property>
+
+  <property>
+    <name>logfeeder.kafka.topic.service.logs</name>
+    <value>service_logs</value>
+    <display-name>Kafka Service Logs topic</display-name>
+    <description>Kafka topic for service logs</description>
+  </property>
+
+  <property>
+    <name>logfeeder.kafka.audit.logs.enable</name>
+    <value>false</value>
+    <display-name>Audit Logs enabled</display-name>
+    <description>Enable audit logs to Kafka</description>
+  </property>
+
+  <property>
+    <name>logfeeder.kafka.topic.audit.logs</name>
+    <value>audit_logs</value>
+    <display-name>Kafka Audit Logs topic</display-name>
+    <description>Kafka topic for audit logs</description>
+  </property>
+
+  <property>
+    <name>logfeeder.kafka.broker.list</name>
+    <value>NONE</value>
+    <display-name>Kafka broker list</display-name>
+    <description>List of brokers. e.g. broker1_host:port,broker2_host:port</description>
+  </property>
+
+  <property>
+    <name>logfeeder.kafka.security.protocol</name>
+    <value>NONE</value>
+    <display-name>Kafka security protocol</display-name>
+    <description>If security is enabled, then set this value. E.g. SASL_PLAINTEXT</description>
+  </property>
+
+  <property>
+    <name>logfeeder.kafka.kerberos.service.name</name>
+    <value>NONE</value>
+    <display-name>Kafka Kerberos service name</display-name>
+    <description>If security is enabled, then set this value. E.g. kafka</description>
+  </property>
+
+</configuration>


Mime
View raw message