From commits-return-3981-archive-asf-public=cust-asf.ponee.io@metron.apache.org Wed Oct 24 17:03:54 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id B407E1807B4 for ; Wed, 24 Oct 2018 17:03:52 +0200 (CEST) Received: (qmail 41441 invoked by uid 500); 24 Oct 2018 15:03:51 -0000 Mailing-List: contact commits-help@metron.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@metron.apache.org Delivered-To: mailing list commits@metron.apache.org Received: (qmail 41235 invoked by uid 99); 24 Oct 2018 15:03:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 24 Oct 2018 15:03:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id F07C2E11C8; Wed, 24 Oct 2018 15:03:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: otto@apache.org To: commits@metron.apache.org Date: Wed, 24 Oct 2018 15:04:02 -0000 Message-Id: In-Reply-To: <5246c22dfad54f259aa9e0292e5a0470@git.apache.org> References: <5246c22dfad54f259aa9e0292e5a0470@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [13/51] [abbrv] metron git commit: METRON-1508 In Ubuntu14 Dev Indexing Fails to Write to Elasticsearch (nickwallen) closes apache/metron#1185 METRON-1508 In Ubuntu14 Dev Indexing Fails to Write to Elasticsearch (nickwallen) closes apache/metron#1185 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/08fe1cff Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/08fe1cff Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/08fe1cff Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 08fe1cff1430c71205d62e375cbacd6906e4fc4e Parents: 0d34e77 Author: nickwallen Authored: Fri Sep 21 15:31:24 2018 -0400 Committer: nickallen Committed: Fri Sep 21 15:31:24 2018 -0400 ---------------------------------------------------------------------- .../ansible/roles/ambari_config/vars/single_node_vm.yml | 4 ++-- .../development/centos6/ansible/inventory/group_vars/all | 1 + .../development/ubuntu14/ansible/inventory/group_vars/all | 1 + .../ELASTICSEARCH/5.6.2/configuration/elastic-site.xml | 8 ++++---- 4 files changed, 8 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml index 962726c..2ec956a 100644 --- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml +++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml @@ -92,7 +92,7 @@ configurations: # Storm expects ambari metrics to be available in 2.6. We do *not* install ambari metrics in full-dev, so we need to revert to the old consumer storm.cluster.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer"}]' topology.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer", "parallelism.hint": 1, "whitelist": ["kafkaOffset\\..+/", "__complete-latency", "__process-latency", "__receive\\.population$", "__sendqueue\\.population$", "__execute-count", "__emit-count", "__ack-count", "__fail-count", "memory/heap\\.usedBytes$", "memory/nonHeap\\.usedBytes$", "GC/.+\\.count$", "GC/.+\\.timeMs$"]}]' - # Storm expects ambari metrics to be available in 2.6 and ambari metrics pulls data via JMX, but since we don't use ambari metrics here, we don't have the javaagent around to use and thus that must be removed from nimbus, supervisor and worker properties + # Storm expects ambari metrics to be available in 2.6 and ambari metrics pulls data via JMX, but since we don't use ambari metrics here, we don't have the javaagent around to use and thus that must be removed from nimbus, supervisor and worker properties nimbus.childopts: '-Xmx1024m _JAAS_PLACEHOLDER' supervisor.childopts: '-Xmx256m _JAAS_PLACEHOLDER' worker.childopts: "-Xmx768m _JAAS_PLACEHOLDER" @@ -117,7 +117,7 @@ configurations: required_configurations: - metron-env: storm_rest_addr: "http://{{ groups.ambari_slave[0] }}:8744" - es_hosts: "{{ groups.search | join(',') }}" + es_hosts: "{{ elasticsearch_hosts }}" zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995" solr_zookeeper_url: "{{ groups.search[0] }}:9983" - metron-rest-env: http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/development/centos6/ansible/inventory/group_vars/all ---------------------------------------------------------------------- diff --git a/metron-deployment/development/centos6/ansible/inventory/group_vars/all b/metron-deployment/development/centos6/ansible/inventory/group_vars/all index dcb7b42..550eb8f 100644 --- a/metron-deployment/development/centos6/ansible/inventory/group_vars/all +++ b/metron-deployment/development/centos6/ansible/inventory/group_vars/all @@ -46,6 +46,7 @@ pycapa_home: "/opt/pycapa" snort_version: "2.9.8.0-1" snort_alert_csv_path: "/var/log/snort/alert.csv" threat_intel_bulk_load: False +elasticsearch_hosts: "{{ groups.search | join(',') }}" # data directories - only required to override defaults zookeeper_data_dir: "/data1/hadoop/zookeeper" http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all ---------------------------------------------------------------------- diff --git a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all index ef9f705..1338d7c 100644 --- a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all +++ b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all @@ -46,6 +46,7 @@ pycapa_home: "/opt/pycapa" snort_version: "2.9.8.0-1" snort_alert_csv_path: "/var/log/snort/alert.csv" threat_intel_bulk_load: False +elasticsearch_hosts: "localhost" # data directories - only required to override defaults zookeeper_data_dir: "/data1/hadoop/zookeeper" http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml index 34df1e4..440c66d 100755 --- a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml @@ -111,7 +111,7 @@ 0 expected_data_nodes - + index_merge_scheduler_max_thread_count 5 @@ -171,7 +171,7 @@ cluster_routing_allocation_disk_threshold_enabled true Property used when multiple drives are used to understand if thresholding is active - + cluster_routing_allocation_disk_watermark_low .97 @@ -185,7 +185,7 @@ network_host [ _local_, _site_ ] - Network interface(s) ES will bind to within each node. "_site_" or a more specific external address is required for all multi-node clusters, and also recommended for single-node installs to allow access to ES reports from non-local hosts. Always include the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation. + Network interface(s) ES will bind to within each node. "_site_" or a more specific external address is required for all multi-node clusters, and also recommended for single-node installs to allow access to ES reports from non-local hosts. Always include the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/5.6/modules-network.html for ES documentation. network_publish_host @@ -193,6 +193,6 @@ true - Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation. + Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/5.6/modules-network.html for ES documentation.