metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject [13/51] [abbrv] metron git commit: METRON-1508 In Ubuntu14 Dev Indexing Fails to Write to Elasticsearch (nickwallen) closes apache/metron#1185
Date Wed, 24 Oct 2018 15:04:02 GMT
METRON-1508 In Ubuntu14 Dev Indexing Fails to Write to Elasticsearch (nickwallen) closes apache/metron#1185


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/08fe1cff
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/08fe1cff
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/08fe1cff

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 08fe1cff1430c71205d62e375cbacd6906e4fc4e
Parents: 0d34e77
Author: nickwallen <nick@nickallen.org>
Authored: Fri Sep 21 15:31:24 2018 -0400
Committer: nickallen <nickallen@apache.org>
Committed: Fri Sep 21 15:31:24 2018 -0400

----------------------------------------------------------------------
 .../ansible/roles/ambari_config/vars/single_node_vm.yml      | 4 ++--
 .../development/centos6/ansible/inventory/group_vars/all     | 1 +
 .../development/ubuntu14/ansible/inventory/group_vars/all    | 1 +
 .../ELASTICSEARCH/5.6.2/configuration/elastic-site.xml       | 8 ++++----
 4 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index 962726c..2ec956a 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -92,7 +92,7 @@ configurations:
       # Storm expects ambari metrics to be available in 2.6.  We do *not* install ambari
metrics in full-dev, so we need to revert to the old consumer
       storm.cluster.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer"}]'
       topology.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer",
"parallelism.hint": 1, "whitelist": ["kafkaOffset\\..+/", "__complete-latency", "__process-latency",
"__receive\\.population$", "__sendqueue\\.population$", "__execute-count", "__emit-count",
"__ack-count", "__fail-count", "memory/heap\\.usedBytes$", "memory/nonHeap\\.usedBytes$",
"GC/.+\\.count$", "GC/.+\\.timeMs$"]}]'
-      # Storm expects ambari metrics to be available in 2.6 and ambari metrics pulls data
via JMX, but since we don't use ambari metrics here, we don't have the javaagent around to
use and thus that must be removed from nimbus, supervisor and worker properties 
+      # Storm expects ambari metrics to be available in 2.6 and ambari metrics pulls data
via JMX, but since we don't use ambari metrics here, we don't have the javaagent around to
use and thus that must be removed from nimbus, supervisor and worker properties
       nimbus.childopts: '-Xmx1024m _JAAS_PLACEHOLDER'
       supervisor.childopts: '-Xmx256m _JAAS_PLACEHOLDER'
       worker.childopts: "-Xmx768m _JAAS_PLACEHOLDER"
@@ -117,7 +117,7 @@ configurations:
 required_configurations:
   - metron-env:
       storm_rest_addr: "http://{{ groups.ambari_slave[0] }}:8744"
-      es_hosts: "{{ groups.search | join(',') }}"
+      es_hosts: "{{ elasticsearch_hosts }}"
       zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995"
       solr_zookeeper_url: "{{ groups.search[0] }}:9983"
   - metron-rest-env:

http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/development/centos6/ansible/inventory/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/ansible/inventory/group_vars/all b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
index dcb7b42..550eb8f 100644
--- a/metron-deployment/development/centos6/ansible/inventory/group_vars/all
+++ b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
@@ -46,6 +46,7 @@ pycapa_home: "/opt/pycapa"
 snort_version: "2.9.8.0-1"
 snort_alert_csv_path: "/var/log/snort/alert.csv"
 threat_intel_bulk_load: False
+elasticsearch_hosts: "{{ groups.search | join(',') }}"
 
 # data directories - only required to override defaults
 zookeeper_data_dir: "/data1/hadoop/zookeeper"

http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
index ef9f705..1338d7c 100644
--- a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
+++ b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
@@ -46,6 +46,7 @@ pycapa_home: "/opt/pycapa"
 snort_version: "2.9.8.0-1"
 snort_alert_csv_path: "/var/log/snort/alert.csv"
 threat_intel_bulk_load: False
+elasticsearch_hosts: "localhost"
 
 # data directories - only required to override defaults
 zookeeper_data_dir: "/data1/hadoop/zookeeper"

http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
index 34df1e4..440c66d 100755
--- a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
@@ -111,7 +111,7 @@
         <value>0</value>
         <description>expected_data_nodes</description>
     </property>
-    <!--  Index -->  
+    <!--  Index -->
     <property>
         <name>index_merge_scheduler_max_thread_count</name>
         <value>5</value>
@@ -171,7 +171,7 @@
         <name>cluster_routing_allocation_disk_threshold_enabled</name>
         <value>true</value>
         <description>Property used when multiple drives are used to understand if thresholding
is active</description>
-    </property>   
+    </property>
    <property>
         <name>cluster_routing_allocation_disk_watermark_low</name>
         <value>.97</value>
@@ -185,7 +185,7 @@
     <property>
         <name>network_host</name>
         <value>[ _local_, _site_ ]</value>
-        <description>Network interface(s) ES will bind to within each node. "_site_"
or a more specific external address is required for all multi-node clusters, and also recommended
for single-node installs to allow access to ES reports from non-local hosts. Always include
the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html
for ES documentation.</description>
+        <description>Network interface(s) ES will bind to within each node. "_site_"
or a more specific external address is required for all multi-node clusters, and also recommended
for single-node installs to allow access to ES reports from non-local hosts. Always include
the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/5.6/modules-network.html
for ES documentation.</description>
     </property>
     <property>
         <name>network_publish_host</name>
@@ -193,6 +193,6 @@
         <value-attributes>
             <empty-value-valid>true</empty-value-valid>
         </value-attributes>
-        <description>Network address ES will publish for client and peer use. Empty
value causes it to pick from the values in network_host, which works in most simple environments.
MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html
for ES documentation.</description>
+        <description>Network address ES will publish for client and peer use. Empty
value causes it to pick from the values in network_host, which works in most simple environments.
MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/5.6/modules-network.html
for ES documentation.</description>
     </property>
 </configuration>


Mime
View raw message