Return-Path: X-Original-To: apmail-metron-commits-archive@minotaur.apache.org Delivered-To: apmail-metron-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 06E1719FA9 for ; Mon, 21 Mar 2016 17:06:20 +0000 (UTC) Received: (qmail 85232 invoked by uid 500); 21 Mar 2016 17:06:19 -0000 Delivered-To: apmail-metron-commits-archive@metron.apache.org Received: (qmail 85203 invoked by uid 500); 21 Mar 2016 17:06:19 -0000 Mailing-List: contact commits-help@metron.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@metron.incubator.apache.org Delivered-To: mailing list commits@metron.incubator.apache.org Received: (qmail 85194 invoked by uid 99); 21 Mar 2016 17:06:19 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd2-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 21 Mar 2016 17:06:19 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd2-us-west.apache.org (ASF Mail Server at spamd2-us-west.apache.org) with ESMTP id 2D3D01A1208 for ; Mon, 21 Mar 2016 17:06:19 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd2-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -3.221 X-Spam-Level: X-Spam-Status: No, score=-3.221 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-0.001] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd2-us-west.apache.org [10.40.0.9]) (amavisd-new, port 10024) with ESMTP id IGCTns975DA4 for ; Mon, 21 Mar 2016 17:05:54 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id D69D05FBDF for ; Mon, 21 Mar 2016 17:05:45 +0000 (UTC) Received: (qmail 82896 invoked by uid 99); 21 Mar 2016 17:05:45 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 21 Mar 2016 17:05:45 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D02F3DFF68; Mon, 21 Mar 2016 17:05:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: cestella@apache.org To: commits@metron.incubator.apache.org Date: Mon, 21 Mar 2016 17:06:07 -0000 Message-Id: In-Reply-To: <32393e648fb34ba89671c31238742381@git.apache.org> References: <32393e648fb34ba89671c31238742381@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [25/43] incubator-metron git commit: METRON-58 Remediate Deployment Integration Testing Issues (dlyle65535 via cestella) closes apache/incubator-metron#36 METRON-58 Remediate Deployment Integration Testing Issues (dlyle65535 via cestella) closes apache/incubator-metron#36 Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/2e9f2c6c Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/2e9f2c6c Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/2e9f2c6c Branch: refs/heads/Metron_0.1BETA Commit: 2e9f2c6ceac70fb19d38281a1abe1d3ee0d088bb Parents: 6638a71 Author: dlyle65535 Authored: Mon Mar 7 18:24:31 2016 -0500 Committer: cstella Committed: Mon Mar 7 18:24:31 2016 -0500 ---------------------------------------------------------------------- .../inventory/metron_example/group_vars/all | 6 - .../inventory/multinode-vagrant/group_vars/all | 6 - .../inventory/singlenode-vagrant/group_vars/all | 11 +- deployment/playbooks/ambari_install.yml | 14 +- deployment/playbooks/metron_full_install.yml | 4 + deployment/playbooks/metron_install.yml | 17 +- deployment/roles/ambari_common/tasks/main.yml | 2 +- .../roles/ambari_config/defaults/main.yml | 30 +++ .../roles/ambari_config/vars/single_node_vm.yml | 40 ++-- .../roles/ambari_master/defaults/main.yml | 19 ++ deployment/roles/ambari_master/tasks/main.yml | 7 + deployment/roles/bro/defaults/main.yml | 20 ++ deployment/roles/bro/tasks/main.yml | 8 + .../roles/elasticsearch/defaults/main.yml | 20 ++ deployment/roles/elasticsearch/tasks/main.yml | 4 +- deployment/roles/hadoop_setup/defaults/main.yml | 25 ++ deployment/roles/hadoop_setup/tasks/main.yml | 3 +- .../roles/metron_common/defaults/main.yml | 19 ++ deployment/roles/metron_common/vars/main.yml | 19 -- .../roles/metron_streaming/defaults/main.yml | 31 +++ .../roles/metron_streaming/files/extractor.json | 11 + .../files/source/bro-config.json | 14 ++ .../files/source/pcap-config.json | 14 ++ .../files/source/snort-config.json | 14 ++ .../files/source/yaf-config.json | 14 ++ .../roles/metron_streaming/handlers/main.yml | 4 +- .../metron_streaming/tasks/full_topology.yml | 26 +++ .../roles/metron_streaming/tasks/main.yml | 24 +- .../metron_streaming/tasks/small_topology.yml | 26 +++ .../metron_streaming/tasks/source_config.yml | 31 +++ .../metron_streaming/tasks/threat_intel.yml | 48 ++++ .../metron_streaming/templates/threat_ip.csv | 37 +++ deployment/roles/mysql/files/geoip_ddl.sql | 49 ---- deployment/roles/mysql/files/mylogin.cnf | 19 -- .../mysql57-community-release-el6-7.noarch.rpm | Bin 8848 -> 0 bytes deployment/roles/mysql/handlers/main.yml | 19 -- deployment/roles/mysql/tasks/main.yml | 85 ------- deployment/roles/mysql/templates/.my.cnf | 20 -- deployment/roles/mysql/vars/main.yml | 20 -- deployment/roles/mysql_client/tasks/main.yml | 34 +++ .../roles/mysql_client/templates/db_config.sql | 21 ++ .../roles/mysql_server/files/geoip_ddl.sql | 49 ++++ .../mysql57-community-release-el6-7.noarch.rpm | Bin 0 -> 8848 bytes deployment/roles/mysql_server/handlers/main.yml | 19 ++ deployment/roles/mysql_server/tasks/main.yml | 86 +++++++ deployment/roles/mysql_server/templates/.my.cnf | 20 ++ deployment/roles/mysql_server/vars/main.yml | 20 ++ deployment/roles/pcap_replay/defaults/main.yml | 21 ++ .../roles/pcap_replay/templates/pcap-replay | 2 +- deployment/roles/pcap_replay/vars/main.yml | 21 -- .../roles/tap_interface/defaults/main.yml | 19 ++ deployment/roles/tap_interface/tasks/main.yml | 30 +++ deployment/roles/yaf/defaults/main.yml | 29 +++ deployment/roles/yaf/tasks/main.yml | 9 +- deployment/roles/yaf/vars/main.yml | 22 -- metron-streaming/Metron-Common/pom.xml | 5 + .../metron/bolt/BulkMessageWriterBolt.java | 5 +- .../java/org/apache/metron/bolt/JoinBolt.java | 1 + .../org/apache/metron/domain/Enrichment.java | 11 + .../java/org/apache/metron/utils/JSONUtils.java | 70 ++++++ .../resources/config/source/bro-config.json | 9 +- .../resources/config/source/snort-config.json | 9 +- .../resources/config/source/yaf-config.json | 9 +- .../dataloads/bulk/ThreatIntelBulkLoader.java | 2 +- .../enrichment/bolt/EnrichmentJoinBolt.java | 19 +- .../enrichment/bolt/EnrichmentSplitterBolt.java | 2 + .../enrichment/bolt/GenericEnrichmentBolt.java | 6 +- .../enrichment/bolt/ThreatIntelJoinBolt.java | 13 +- .../metron/threatintel/ThreatIntelAdapter.java | 2 +- .../metron/indexing/TelemetryIndexingBolt.java | 11 + .../metron/writer/ElasticsearchWriter.java | 10 +- .../metron/parsing/parsers/BasicBroParser.java | 7 +- .../parsing/parsers/BasicSnortParser.java | 1 + .../metron/parsing/test/BasicBroParserTest.java | 22 ++ .../src/test/resources/BroParserTest.log | 3 +- .../util/integration/ComponentRunner.java | 33 ++- metron-streaming/Metron-Topologies/pom.xml | 17 +- .../apache/metron/utils/SourceConfigUtils.java | 37 ++- .../Metron_Configs/etc/env/config.properties | 3 +- .../Metron_Configs/topologies/bro/remote.yaml | 15 +- .../Metron_Configs/topologies/bro/test.yaml | 10 - .../topologies/enrichment/remote.yaml | 13 +- .../topologies/paloalto/test.yaml | 2 +- .../Metron_Configs/topologies/pcap/parse.yaml | 4 +- .../Metron_Configs/topologies/pcap/remote.yaml | 97 +++----- .../Metron_Configs/topologies/snort/remote.yaml | 10 - .../Metron_Configs/topologies/snort/test.yaml | 12 +- .../Metron_Configs/topologies/yaf/remote.yaml | 15 +- .../Metron_Configs/topologies/yaf/test.yaml | 10 - .../src/main/resources/SampleIndexed/YafIndexed | 20 +- .../src/main/resources/SampleParsed/SnortParsed | 6 +- .../integration/EnrichmentIntegrationTest.java | 226 +++++++++++++++++-- .../integration/ParserIntegrationTest.java | 15 +- .../metron/integration/util/TestUtils.java | 1 - .../integration/util/mock/MockGeoAdapter.java | 26 ++- pom.xml | 3 +- 96 files changed, 1382 insertions(+), 592 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/inventory/metron_example/group_vars/all ---------------------------------------------------------------------- diff --git a/deployment/inventory/metron_example/group_vars/all b/deployment/inventory/metron_example/group_vars/all index b8cf9dc..3a26769 100644 --- a/deployment/inventory/metron_example/group_vars/all +++ b/deployment/inventory/metron_example/group_vars/all @@ -31,12 +31,6 @@ pcap_hbase_table: pcap tracker_hbase_table: access_tracker threatintel_ip_hbase_table: malicious_ip -# kafka -pycapa_topic: pcap -bro_topic: bro -yaf_topic: ipfix -snort_topic: snort - #elasticsearch elasticsearch_transport_port: 9300 elasticsearch_network_interface: eth0 http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/inventory/multinode-vagrant/group_vars/all ---------------------------------------------------------------------- diff --git a/deployment/inventory/multinode-vagrant/group_vars/all b/deployment/inventory/multinode-vagrant/group_vars/all index bb41e89..fc3b56d 100644 --- a/deployment/inventory/multinode-vagrant/group_vars/all +++ b/deployment/inventory/multinode-vagrant/group_vars/all @@ -28,12 +28,6 @@ pcap_hbase_table: pcap tracker_hbase_table: access_tracker threatintel_ip_hbase_table: malicious_ip -# kafka -pycapa_topic: pcap -bro_topic: bro -yaf_topic: ipfix -snort_topic: snort - #elasticsearch elasticsearch_transport_port: 9300 elasticsearch_network_interface: eth1 http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/inventory/singlenode-vagrant/group_vars/all ---------------------------------------------------------------------- diff --git a/deployment/inventory/singlenode-vagrant/group_vars/all b/deployment/inventory/singlenode-vagrant/group_vars/all index 1e08a6a..6405eea 100644 --- a/deployment/inventory/singlenode-vagrant/group_vars/all +++ b/deployment/inventory/singlenode-vagrant/group_vars/all @@ -28,12 +28,6 @@ pcap_hbase_table: pcap tracker_hbase_table: access_tracker threatintel_ip_hbase_table: malicious_ip -# kafka -pycapa_topic: pcap -bro_topic: bro -yaf_topic: ipfix -snort_topic: snort - #elasticsearch elasticsearch_transport_port: 9300 elasticsearch_network_interface: eth1 @@ -55,7 +49,7 @@ snort_version: "2.9.8.0-1" snort_alert_csv_path: "/var/log/snort/alert.csv" #PCAP Replay -pcap_replay: True +pcap_replay: False pcap_replay_interface: eth1 #data directories - only required to override defaults @@ -73,3 +67,6 @@ storm_local_dir: "/data1/hadoop/storm" kafka_log_dirs: "/data1/kafka-log" elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch" +ambari_server_mem: 512 +topology_name: small_topology.yml +threat_intel_bulk_load: False http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/playbooks/ambari_install.yml ---------------------------------------------------------------------- diff --git a/deployment/playbooks/ambari_install.yml b/deployment/playbooks/ambari_install.yml index e1da427..c7f8249 100644 --- a/deployment/playbooks/ambari_install.yml +++ b/deployment/playbooks/ambari_install.yml @@ -19,17 +19,29 @@ sudo: yes roles: - role: ambari_common + tags: + - ambari-prereqs + - hdp-install - hosts: ambari_master sudo: yes roles: - - role: ambari_master + - role: ambari_master + tags: + - ambari-server + - hdp-install - hosts: ambari_slave sudo: yes roles: - role: ambari_slave + tags: + - ambari-agent + - hdp-install - hosts: ambari_master roles: - role: ambari_config + tags: + - hdp-install + - hdp-deploy http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/playbooks/metron_full_install.yml ---------------------------------------------------------------------- diff --git a/deployment/playbooks/metron_full_install.yml b/deployment/playbooks/metron_full_install.yml index 38203da..26ffd62 100644 --- a/deployment/playbooks/metron_full_install.yml +++ b/deployment/playbooks/metron_full_install.yml @@ -16,4 +16,8 @@ # --- - include: ambari_install.yml + tags: + - ambari - include: metron_install.yml + tags: + - metron http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/playbooks/metron_install.yml ---------------------------------------------------------------------- diff --git a/deployment/playbooks/metron_install.yml b/deployment/playbooks/metron_install.yml index ad070c9..b8646fc 100644 --- a/deployment/playbooks/metron_install.yml +++ b/deployment/playbooks/metron_install.yml @@ -19,12 +19,16 @@ sudo: yes roles: - role: metron_common + tags: + - metron-prereqs - hosts: hadoop_client sudo: yes roles: - role: ambari_gather_facts - role: hadoop_setup + tags: + - metron-prereqs - hosts: search sudo: yes @@ -38,13 +42,22 @@ - hosts: mysql sudo: yes roles: - - role: mysql + - role: mysql_server tags: - - mysql + - mysql-server + +- hosts: ambari_slave + sudo: yes + roles: + - role: mysql_client + tags: + - mysql-client + - hosts: sensors sudo: yes roles: + - { role: tap_interface, when: install_tap | default(False) == True } - role: ambari_gather_facts - role: flume - role: pycapa http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_common/tasks/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/ambari_common/tasks/main.yml b/deployment/roles/ambari_common/tasks/main.yml index 992468e..35f3fce 100644 --- a/deployment/roles/ambari_common/tasks/main.yml +++ b/deployment/roles/ambari_common/tasks/main.yml @@ -59,7 +59,7 @@ - name: install epel-repo rpm yum: pkg=/tmp/epel-release.rpm state=installed -- name: Download HDP repo +- name: Download Ambari repo get_url: url="{{ rhel_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo - name: Clean yum http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_config/defaults/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/ambari_config/defaults/main.yml b/deployment/roles/ambari_config/defaults/main.yml new file mode 100644 index 0000000..507b6e3 --- /dev/null +++ b/deployment/roles/ambari_config/defaults/main.yml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +zookeeper_data_dir: /hadoop/zookeeper +namenode_checkpoint_dir: /hadoop/hdfs/namesecondary +namenode_name_dir: /hadoop/hdfs/namenode +datanode_data_dir: /hadoop/hdfs/data +journalnode_edits_dir: /hadoop/hdfs/journalnode +jhs_recovery_store_ldb_path: /hadoop/mapreduce/jhs +nodemanager_local_dirs: /hadoop/yarn/local +timeline_ldb_store_path: /hadoop/yarn/timeline +timeline_ldb_state_path: /hadoop/yarn/timeline +nodemanager_log_dirs: /hadoop/yarn/log +storm_local_dir: /hadoop/storm +kafka_log_dirs: /kafka-log +cluster_type: small_cluster http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_config/vars/single_node_vm.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/ambari_config/vars/single_node_vm.yml b/deployment/roles/ambari_config/vars/single_node_vm.yml index d0d3b78..6b18825 100644 --- a/deployment/roles/ambari_config/vars/single_node_vm.yml +++ b/deployment/roles/ambari_config/vars/single_node_vm.yml @@ -29,47 +29,49 @@ zookeeper_slave: [ZOOKEEPER_CLIENT] hbase_master: [HBASE_MASTER, HBASE_CLIENT] hbase_slave: [HBASE_REGIONSERVER] -metron_components: "{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | union(storm_slave) | union(spark_slave) | union(kafka_broker) | union(hbase_slave) }}" +metron_components: "{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | union(storm_slave) | union(kafka_broker) | union(hbase_slave) }}" cluster_name: "metron_cluster" blueprint_name: "metron_blueprint" configurations: - zoo.cfg: - dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}' + dataDir: '{{ zookeeper_data_dir }}' - hadoop-env: - namenode_heapsize: 1024 - dtnode_heapsize: 1024 + hadoop_heapsize: 1024 + namenode_heapsize: 512 + dtnode_heapsize: 512 + namenode_opt_permsize: 128m - hbase-env: - hbase_regionserver_heapsize: 1024 - hbase_master_heapsize: 1024 + hbase_regionserver_heapsize: 512 + hbase_master_heapsize: 512 + hbase_regionserver_xmn_max: 512 - hdfs-site: - dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}' - dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}' - dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}' - dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}' + dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir }}' + dfs.namenode.name.dir: '{{ namenode_name_dir }}' + dfs.datanode.data.dir: '{{ datanode_data_dir }}' + dfs.journalnode.edits.dir: '{{ journalnode_edits_dir }}' - yarn-env: nodemanager_heapsize: 512 yarn_heapsize: 512 apptimelineserver_heapsize : 512 + resourcemanager_heapsize: 1024 - mapred-env: jobhistory_heapsize: 256 - mapred-site: - mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}' + mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path }}' - yarn-site: - yarn.nodemanager.resource.memory-mb: 1024 - yarn.scheduler.maximum-allocation-mb: 1024 - yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}' - yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}' - yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}' - yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}' + yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs }}' + yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path }}' + yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path }}' + yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs }}' - storm-site: supervisor.slots.ports: "[6700, 6701, 6702, 6703]" - storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}' + storm.local.dir: '{{ storm_local_dir }}' - kafka-env: content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}" - kafka-broker: - log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}' + log.dirs: '{{ kafka_log_dirs }}' blueprint: stack_name: HDP http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_master/defaults/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/ambari_master/defaults/main.yml b/deployment/roles/ambari_master/defaults/main.yml new file mode 100644 index 0000000..3b8cc73 --- /dev/null +++ b/deployment/roles/ambari_master/defaults/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +ambari_server_mem: 2048 + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_master/tasks/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/ambari_master/tasks/main.yml b/deployment/roles/ambari_master/tasks/main.yml index 8c78f06..daf4e41 100644 --- a/deployment/roles/ambari_master/tasks/main.yml +++ b/deployment/roles/ambari_master/tasks/main.yml @@ -24,6 +24,13 @@ register: ambari_server_setup failed_when: ambari_server_setup.stderr +- name: Set Ambari Server Max Memory + replace: + dest: /var/lib/ambari-server/ambari-env.sh + regexp: "\ -Xmx2048m\ " + replace: " -Xmx{{ ambari_server_mem }}m " + backup: no + - name: start ambari server service: name=ambari-server state=restarted http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/bro/defaults/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/bro/defaults/main.yml b/deployment/roles/bro/defaults/main.yml new file mode 100644 index 0000000..c7a2c1f --- /dev/null +++ b/deployment/roles/bro/defaults/main.yml @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +bro_crontab_minutes: 0-59/5 +bro_crontab_job: /usr/local/bro/bin/broctl cron + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/bro/tasks/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/bro/tasks/main.yml b/deployment/roles/bro/tasks/main.yml index 04dfe8f..0191052 100644 --- a/deployment/roles/bro/tasks/main.yml +++ b/deployment/roles/bro/tasks/main.yml @@ -31,6 +31,7 @@ - python-devel - swig - zlib-devel + - perl - include: librdkafka.yml @@ -46,3 +47,10 @@ - name: Start bro shell: /usr/local/bro/bin/broctl start + +- name: Bro Cronjob + cron: + name: Bro Cron + minute: "{{ bro_crontab_minutes }}" + job: "{{ bro_crontab_job }}" + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/elasticsearch/defaults/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/elasticsearch/defaults/main.yml b/deployment/roles/elasticsearch/defaults/main.yml new file mode 100644 index 0000000..d91fa1a --- /dev/null +++ b/deployment/roles/elasticsearch/defaults/main.yml @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +elasticsearch_data_dir: /var/lib/elasticsearch +elasticsearch_network_interface: eth0 + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/elasticsearch/tasks/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/elasticsearch/tasks/main.yml b/deployment/roles/elasticsearch/tasks/main.yml index 555666b..fa8d4f3 100644 --- a/deployment/roles/elasticsearch/tasks/main.yml +++ b/deployment/roles/elasticsearch/tasks/main.yml @@ -55,10 +55,10 @@ with_items: - { regexp: '#cluster\.name', line: 'cluster.name: metron' } - { regexp: '#network\.host:', line: 'network.host: _{{ - elasticsearch_network_interface | default("eth0") }}:ipv4_' } + elasticsearch_network_interface }}:ipv4_' } - { regexp: '#discovery\.zen\.ping\.unicast\.hosts', line: 'discovery.zen.ping.unicast.hosts: [ {{ es_hosts }} ]'} - - { regexp: '#path\.data', line: 'path.data: {{ elasticsearch_data_dir | default("/var/lib/elasticsearch")}}' } + - { regexp: '#path\.data', line: 'path.data: {{ elasticsearch_data_dir }}' } notify: restart elasticsearch - name: Start Elasticsearch. http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/hadoop_setup/defaults/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/hadoop_setup/defaults/main.yml b/deployment/roles/hadoop_setup/defaults/main.yml new file mode 100644 index 0000000..c783cea --- /dev/null +++ b/deployment/roles/hadoop_setup/defaults/main.yml @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +num_partitions: 1 +retention_in_gb: 10 +pycapa_topic: pcap +bro_topic: bro +yaf_topic: ipfix +snort_topic: snort +enrichments_topic: enrichments + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/hadoop_setup/tasks/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/hadoop_setup/tasks/main.yml b/deployment/roles/hadoop_setup/tasks/main.yml index 5e77b99..5b6c47c 100644 --- a/deployment/roles/hadoop_setup/tasks/main.yml +++ b/deployment/roles/hadoop_setup/tasks/main.yml @@ -26,10 +26,11 @@ #if kafka topic - name: Create Kafka topics - shell: "{{ kafka_home }}/bin/kafka-topics.sh --zookeeper {{ zookeeper_url }} --create --topic {{ item }} --partitions 1 --replication-factor 1" + shell: "{{ kafka_home }}/bin/kafka-topics.sh --zookeeper {{ zookeeper_url }} --create --topic {{ item }} --partitions {{ num_partitions }} --replication-factor 1 --config retention.bytes={{ retention_in_gb * 1024 * 1024 * 1024}}" ignore_errors: yes with_items: - "{{ pycapa_topic }}" - "{{ bro_topic }}" - "{{ yaf_topic }}" - "{{ snort_topic }}" + - "{{ enrichments_topic }}" http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_common/defaults/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_common/defaults/main.yml b/deployment/roles/metron_common/defaults/main.yml new file mode 100644 index 0000000..50aaefd --- /dev/null +++ b/deployment/roles/metron_common/defaults/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +metron_jar_name: Metron-Topologies-{{ metron_version }}.jar +metron_jar_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_jar_name }}" http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_common/vars/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_common/vars/main.yml b/deployment/roles/metron_common/vars/main.yml deleted file mode 100644 index 50aaefd..0000000 --- a/deployment/roles/metron_common/vars/main.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -metron_jar_name: Metron-Topologies-{{ metron_version }}.jar -metron_jar_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_jar_name }}" http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/defaults/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_streaming/defaults/main.yml b/deployment/roles/metron_streaming/defaults/main.yml new file mode 100644 index 0000000..cb425f9 --- /dev/null +++ b/deployment/roles/metron_streaming/defaults/main.yml @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +source_config_path: "{{ metron_directory }}/config/source" +threat_intel_bulk_load: True +threat_intel_bin: "{{ metron_directory }}/bin/threatintel_bulk_load.sh" +threat_intel_host: "{{ groups.ambari_master[0] }}" +threat_intel_work_dir: /tmp/ti_bulk +threat_intel_csv_filename: "threat_ip.csv" +threat_intel_csv_filepath: "../roles/metron_streaming/templates/{{ threat_intel_csv_filename }}" + +topology_name: full_topology.yml +pycapa_topic: pcap +bro_topic: bro +yaf_topic: ipfix +snort_topic: snort +enrichments_topic: enrichments http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/extractor.json ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_streaming/files/extractor.json b/deployment/roles/metron_streaming/files/extractor.json new file mode 100644 index 0000000..81429e8 --- /dev/null +++ b/deployment/roles/metron_streaming/files/extractor.json @@ -0,0 +1,11 @@ +{ + "config": { + "columns": { + "ip": 0 + }, + "indicator_column": "ip", + "separator": "," + }, + "extractor": "CSV" +} + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/source/bro-config.json ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_streaming/files/source/bro-config.json b/deployment/roles/metron_streaming/files/source/bro-config.json new file mode 100644 index 0000000..34109b8 --- /dev/null +++ b/deployment/roles/metron_streaming/files/source/bro-config.json @@ -0,0 +1,14 @@ +{ + "index": "bro", + "batchSize": 5, + "enrichmentFieldMap": + { + "geo": ["ip_dst_addr", "ip_src_addr"], + "host": ["host"] + }, + "threatIntelFieldMap": + { + "ip": ["ip_dst_addr", "ip_src_addr"] + } +} + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/source/pcap-config.json ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_streaming/files/source/pcap-config.json b/deployment/roles/metron_streaming/files/source/pcap-config.json new file mode 100644 index 0000000..4b9c639 --- /dev/null +++ b/deployment/roles/metron_streaming/files/source/pcap-config.json @@ -0,0 +1,14 @@ +{ + "index": "pcap", + "batchSize": 5, + "enrichmentFieldMap": + { + "geo": ["ip_src_addr", "ip_dst_addr"], + "host": ["ip_src_addr", "ip_dst_addr"] + }, + "threatIntelFieldMap": + { + "ip": ["ip_src_addr", "ip_dst_addr"] + } +} + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/source/snort-config.json ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_streaming/files/source/snort-config.json b/deployment/roles/metron_streaming/files/source/snort-config.json new file mode 100644 index 0000000..1208637 --- /dev/null +++ b/deployment/roles/metron_streaming/files/source/snort-config.json @@ -0,0 +1,14 @@ +{ + "index": "snort", + "batchSize": 1, + "enrichmentFieldMap": + { + "geo": ["ip_dst_addr", "ip_src_addr"], + "host": ["host"] + }, + "threatIntelFieldMap": + { + "ip": ["ip_dst_addr", "ip_src_addr"] + } +} + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/source/yaf-config.json ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_streaming/files/source/yaf-config.json b/deployment/roles/metron_streaming/files/source/yaf-config.json new file mode 100644 index 0000000..65de961 --- /dev/null +++ b/deployment/roles/metron_streaming/files/source/yaf-config.json @@ -0,0 +1,14 @@ +{ + "index": "yaf", + "batchSize": 5, + "enrichmentFieldMap": + { + "geo": ["ip_dst_addr", "ip_src_addr"], + "host": ["host"] + }, + "threatIntelFieldMap": + { + "ip": ["ip_dst_addr", "ip_src_addr"] + } +} + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/handlers/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_streaming/handlers/main.yml b/deployment/roles/metron_streaming/handlers/main.yml index 112c5ca..634d591 100644 --- a/deployment/roles/metron_streaming/handlers/main.yml +++ b/deployment/roles/metron_streaming/handlers/main.yml @@ -15,5 +15,5 @@ # limitations under the License. # --- -- name: restart elasticsearch - service: name=elasticsearch state=restarted +- name: Load Source Config + shell: java -cp {{ metron_directory }}/lib/{{ metron_jar_name }}::/usr/hdp/current/hadoop-client/lib/slf4j-api-1.7.10.jar org.apache.metron.utils.SourceConfigUtils -p {{ source_config_path }} -z {{ zookeeper_url }} && touch {{ source_config_path }}/configured http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/full_topology.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_streaming/tasks/full_topology.yml b/deployment/roles/metron_streaming/tasks/full_topology.yml new file mode 100644 index 0000000..060caf8 --- /dev/null +++ b/deployment/roles/metron_streaming/tasks/full_topology.yml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + +- name: Submit Metron topologies + command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ item }} --filter {{ metron_properties_config_path }} + with_items: + - "{{ metron_directory }}/config/topologies/bro/remote.yaml" + - "{{ metron_directory }}/config/topologies/snort/remote.yaml" + - "{{ metron_directory }}/config/topologies/yaf/remote.yaml" + - "{{ metron_directory }}/config/topologies/pcap/parse.yaml" + - "{{ metron_directory }}/config/topologies/enrichment/remote.yaml" http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_streaming/tasks/main.yml b/deployment/roles/metron_streaming/tasks/main.yml index 7d6fe9c..c1e1642 100644 --- a/deployment/roles/metron_streaming/tasks/main.yml +++ b/deployment/roles/metron_streaming/tasks/main.yml @@ -52,6 +52,10 @@ - "etc" - "topologies" +- name: Get Default mysql passowrd + include_vars: "../roles/mysql_server/vars/main.yml" + when: mysql_root_password is undefined + - name: Configure Metron topologies lineinfile: > dest={{ metron_properties_config_path }} @@ -59,6 +63,7 @@ line="{{ item.line }}" with_items: - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" } + - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" } - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" } - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" } - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" } @@ -70,7 +75,8 @@ - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" } - { regexp: "threat.intel.ip.table=", line: "threat.intel.ip.table={{ threatintel_ip_hbase_table }}" } - { regexp: "threat.intel.ip.cf=", line: "threat.intel.ip.cf=t" } - - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.search[0] }}" } + - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" } + - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" } - name: Add Elasticsearch templates for topologies uri: @@ -80,11 +86,11 @@ status_code: 200 body_format: json -- name: Submit Metron topologies - command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ item }} --filter {{ metron_properties_config_path }} - ignore_errors: yes - with_items: - - "{{ metron_directory }}/config/topologies/pcap/remote.yaml" - - "{{ metron_directory }}/config/topologies/bro/remote.yaml" - - "{{ metron_directory }}/config/topologies/snort/remote.yaml" - - "{{ metron_directory }}/config/topologies/yaf/remote.yaml" +- include: source_config.yml + run_once: true +- include: threat_intel.yml + run_once: true + when: threat_intel_bulk_load == True + +- include: "{{ topology_name }}" + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/small_topology.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_streaming/tasks/small_topology.yml b/deployment/roles/metron_streaming/tasks/small_topology.yml new file mode 100644 index 0000000..6707210 --- /dev/null +++ b/deployment/roles/metron_streaming/tasks/small_topology.yml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + +- name: Submit Metron topologies + command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ item }} --filter {{ metron_properties_config_path }} + with_items: + - "{{ metron_directory }}/config/topologies/bro/remote.yaml" + - "{{ metron_directory }}/config/topologies/pcap/parse.yaml" + - "{{ metron_directory }}/config/topologies/yaf/remote.yaml" + - "{{ metron_directory }}/config/topologies/enrichment/remote.yaml" + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/source_config.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_streaming/tasks/source_config.yml b/deployment/roles/metron_streaming/tasks/source_config.yml new file mode 100644 index 0000000..9233bac --- /dev/null +++ b/deployment/roles/metron_streaming/tasks/source_config.yml @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Create Source Config Directory + file: + path: "{{ source_config_path }}" + state: directory + +- name: Copy Source Config Files + copy: + src: "{{ item }}" + dest: "{{ source_config_path }}" + mode: 0644 + with_fileglob: + - ../roles/metron_streaming/files/source/*.json + notify: Load Source Config + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/threat_intel.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_streaming/tasks/threat_intel.yml b/deployment/roles/metron_streaming/tasks/threat_intel.yml new file mode 100644 index 0000000..0439e46 --- /dev/null +++ b/deployment/roles/metron_streaming/tasks/threat_intel.yml @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Create root user HDFS directory + command: su - hdfs -c "hdfs dfs -mkdir -p /user/root && hdfs dfs -chown root:root /user/root" + +- name: Create Bulk load working Directory + file: + path: "{{ threat_intel_work_dir }}" + state: directory + +- name: Copy extractor.json to {{ inventory_hostname }} + copy: + src: ../roles/metron_streaming/files/extractor.json + dest: "{{ threat_intel_work_dir }}" + mode: 0644 + +- name: Copy Bulk Load CSV File + template: + src: "{{ threat_intel_csv_filepath }}" + dest: "{{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }}" + mode: 0644 + +- name: Copy Bulk Load CSV File to HDFS + command: "hdfs dfs -put {{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }} ." + +- name: Run Threat Intel Bulk Load + shell: "{{ threat_intel_bin }} -f t --table malicious_ip -e {{ threat_intel_work_dir }}/extractor.json -i /user/root && touch {{ threat_intel_work_dir }}/loaded" + args: + creates: "{{ threat_intel_work_dir }}/loaded" + +- name: Clean up HDFS File + command: "hdfs dfs -rm {{ threat_intel_csv_filename }}" + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/templates/threat_ip.csv ---------------------------------------------------------------------- diff --git a/deployment/roles/metron_streaming/templates/threat_ip.csv b/deployment/roles/metron_streaming/templates/threat_ip.csv new file mode 100644 index 0000000..3ac38f3 --- /dev/null +++ b/deployment/roles/metron_streaming/templates/threat_ip.csv @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#Add single column of ip address to alert +#Public lists are available on the internet +# example: +23.113.113.105 +24.107.205.249 +24.108.62.255 +24.224.153.71 +27.4.1.212 +27.131.149.102 +31.24.30.31 +31.131.251.33 +31.186.99.250 +31.192.209.119 +31.192.209.150 +31.200.244.17 +37.34.52.185 +37.58.112.101 +37.99.146.27 +37.128.132.96 +37.140.195.177 +37.140.199.100 http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/files/geoip_ddl.sql ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql/files/geoip_ddl.sql b/deployment/roles/mysql/files/geoip_ddl.sql deleted file mode 100644 index 02616c6..0000000 --- a/deployment/roles/mysql/files/geoip_ddl.sql +++ /dev/null @@ -1,49 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -CREATE DATABASE IF NOT EXISTS GEO; - -USE GEO; - -DROP TABLE IF EXISTS `blocks`; -CREATE TABLE `blocks` ( `startIPNum` int(10) unsigned NOT NULL,`endIPNum` int(10) unsigned NOT NULL,`locID` -int(10) unsigned NOT NULL, PRIMARY KEY (`startIPNum`,`endIPNum`) ) -ENGINE=MyISAM DEFAULT CHARSET=latin1 PACK_KEYS=1 DELAY_KEY_WRITE=1; - -DROP TABLE IF EXISTS `location`; -CREATE TABLE `location` (`locID` int(10) unsigned NOT NULL,`country` char(2) default NULL,`region` char(2) - default NULL,`city` varchar(45) default NULL,`postalCode` char(7) default NULL,`latitude` double default -NULL,`longitude` double default NULL,`dmaCode` char(3) default NULL,`areaCode` char(3) default NULL,PRIMARY KEY - (`locID`),KEY `Index_Country` (`country`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ROW_FORMAT=FIXED; - -load data infile '/var/lib/mysql-files/GeoLiteCity-Blocks.csv' into table `blocks` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines; -load data infile '/var/lib/mysql-files/GeoLiteCity-Location.csv' into table `location` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines; - - -DELIMITER $$ -DROP FUNCTION IF EXISTS `IPTOLOCID` $$ -CREATE FUNCTION `IPTOLOCID`( ip VARCHAR(15)) RETURNS int(10) unsigned - BEGIN - DECLARE ipn INTEGER UNSIGNED; - DECLARE locID_var INTEGER; - IF ip LIKE '192.168.%' OR ip LIKE '10.%' THEN RETURN 0; - END IF; - SET ipn = INET_ATON(ip); - SELECT locID INTO locID_var FROM `blocks` INNER JOIN (SELECT MAX(startIPNum) AS start FROM `blocks` WHERE startIPNum <= ipn) AS s ON (startIPNum = s.start) WHERE endIPNum >= ipn; - RETURN locID_var; - END -$$ -DELIMITER ; http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/files/mylogin.cnf ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql/files/mylogin.cnf b/deployment/roles/mysql/files/mylogin.cnf deleted file mode 100644 index b8d5781..0000000 --- a/deployment/roles/mysql/files/mylogin.cnf +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[client] -user=root -password=P@ssw0rd \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm b/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm deleted file mode 100644 index 8603602..0000000 Binary files a/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/handlers/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql/handlers/main.yml b/deployment/roles/mysql/handlers/main.yml deleted file mode 100644 index 112c5ca..0000000 --- a/deployment/roles/mysql/handlers/main.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: restart elasticsearch - service: name=elasticsearch state=restarted http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/tasks/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql/tasks/main.yml b/deployment/roles/mysql/tasks/main.yml deleted file mode 100644 index 91db896..0000000 --- a/deployment/roles/mysql/tasks/main.yml +++ /dev/null @@ -1,85 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create temporary directories - file: - path: "/tmp/{{ item }}" - state: directory - mode: 0755 - with_items: - - "geoip" - -- name: Distribute Mysql - copy: - src: "{{ mysql_rpm_version }}.rpm" - dest: /tmp - -- name: Install Msyql Yum Repository - yum: - name: "/tmp/{{ mysql_rpm_version }}.rpm" - -- name: Install MySQL - yum: - name: "{{ item }}" - state: latest - with_items: - - "mysql-community-server" - - "MySQL-python" - -- name: Start MySQL - service: - name: mysqld - state: started - enabled: yes - -- name: Retrieve temporary root password - shell: "grep 'temporary password' /var/log/mysqld.log | sed 's/.*root@localhost: //'" - args: - creates: ~/.my.cnf - register: temp_root_password - -- name: Update mysql root password - command: "mysqladmin --user=root --password='{{ temp_root_password.stdout }}' password '{{ mysql_root_password }}'" - ignore_errors: yes - args: - creates: ~/.my.cnf - -- name: Copy mylogin.cnf - copy: - src: mylogin.cnf - dest: ~/.my.cnf - -- name: Download GeoIP databases - unarchive: - src: http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz - dest: /tmp/geoip - copy: no - creates: /tmp/geopip/*/GeoLiteCity-Blocks.csv - -- name: Copy to MySQL import directory - shell: "cp /tmp/geoip/*/*.csv /var/lib/mysql-files/" - -- name: Copy DDL - copy: - src: geoip_ddl.sql - dest: /tmp/geoip_ddl.sql - -- name: Import GeoIP DDL - mysql_db: - name: all - state: import - target: /tmp/geoip_ddl.sql http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/templates/.my.cnf ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql/templates/.my.cnf b/deployment/roles/mysql/templates/.my.cnf deleted file mode 100644 index d5c0825..0000000 --- a/deployment/roles/mysql/templates/.my.cnf +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -[client] -user=root -password={{ mysql_root_password }} -host=localhost \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/vars/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql/vars/main.yml b/deployment/roles/mysql/vars/main.yml deleted file mode 100644 index ccf2426..0000000 --- a/deployment/roles/mysql/vars/main.yml +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -mysql_rpm_version: mysql57-community-release-el6-7.noarch -mysql_root_password: P@ssw0rd - http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_client/tasks/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql_client/tasks/main.yml b/deployment/roles/mysql_client/tasks/main.yml new file mode 100644 index 0000000..5c98eb9 --- /dev/null +++ b/deployment/roles/mysql_client/tasks/main.yml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + +- name: Get Default mysql passowrd + include_vars: "../roles/mysql_server/vars/main.yml" + when: mysql_root_password is undefined + +- name: Allow remote login to mysql + template: + src: "../roles/mysql_client/templates/db_config.sql" + dest: "/tmp/{{ansible_fqdn}}.sql" + delegate_to: "{{ groups.mysql[0] }}" + +- name: Import DB_Config + mysql_db: + name: "all" + state: "import" + target: "/tmp/{{ansible_fqdn}}.sql" + delegate_to: "{{ groups.mysql[0] }}" http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_client/templates/db_config.sql ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql_client/templates/db_config.sql b/deployment/roles/mysql_client/templates/db_config.sql new file mode 100644 index 0000000..c407a13 --- /dev/null +++ b/deployment/roles/mysql_client/templates/db_config.sql @@ -0,0 +1,21 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +CREATE USER 'root'@'{{ ansible_fqdn }}' IDENTIFIED BY '{{ mysql_root_password }}'; +SET PASSWORD FOR 'root'@'{{ ansible_fqdn }}' = PASSWORD('{{ mysql_root_password }}'); +GRANT ALL PRIVILEGES ON *.* to 'root'@'{{ ansible_fqdn }}' WITH GRANT OPTION; +FLUSH PRIVILEGES; http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/files/geoip_ddl.sql ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql_server/files/geoip_ddl.sql b/deployment/roles/mysql_server/files/geoip_ddl.sql new file mode 100644 index 0000000..02616c6 --- /dev/null +++ b/deployment/roles/mysql_server/files/geoip_ddl.sql @@ -0,0 +1,49 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +CREATE DATABASE IF NOT EXISTS GEO; + +USE GEO; + +DROP TABLE IF EXISTS `blocks`; +CREATE TABLE `blocks` ( `startIPNum` int(10) unsigned NOT NULL,`endIPNum` int(10) unsigned NOT NULL,`locID` +int(10) unsigned NOT NULL, PRIMARY KEY (`startIPNum`,`endIPNum`) ) +ENGINE=MyISAM DEFAULT CHARSET=latin1 PACK_KEYS=1 DELAY_KEY_WRITE=1; + +DROP TABLE IF EXISTS `location`; +CREATE TABLE `location` (`locID` int(10) unsigned NOT NULL,`country` char(2) default NULL,`region` char(2) + default NULL,`city` varchar(45) default NULL,`postalCode` char(7) default NULL,`latitude` double default +NULL,`longitude` double default NULL,`dmaCode` char(3) default NULL,`areaCode` char(3) default NULL,PRIMARY KEY + (`locID`),KEY `Index_Country` (`country`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ROW_FORMAT=FIXED; + +load data infile '/var/lib/mysql-files/GeoLiteCity-Blocks.csv' into table `blocks` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines; +load data infile '/var/lib/mysql-files/GeoLiteCity-Location.csv' into table `location` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines; + + +DELIMITER $$ +DROP FUNCTION IF EXISTS `IPTOLOCID` $$ +CREATE FUNCTION `IPTOLOCID`( ip VARCHAR(15)) RETURNS int(10) unsigned + BEGIN + DECLARE ipn INTEGER UNSIGNED; + DECLARE locID_var INTEGER; + IF ip LIKE '192.168.%' OR ip LIKE '10.%' THEN RETURN 0; + END IF; + SET ipn = INET_ATON(ip); + SELECT locID INTO locID_var FROM `blocks` INNER JOIN (SELECT MAX(startIPNum) AS start FROM `blocks` WHERE startIPNum <= ipn) AS s ON (startIPNum = s.start) WHERE endIPNum >= ipn; + RETURN locID_var; + END +$$ +DELIMITER ; http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/files/mysql57-community-release-el6-7.noarch.rpm ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql_server/files/mysql57-community-release-el6-7.noarch.rpm b/deployment/roles/mysql_server/files/mysql57-community-release-el6-7.noarch.rpm new file mode 100644 index 0000000..8603602 Binary files /dev/null and b/deployment/roles/mysql_server/files/mysql57-community-release-el6-7.noarch.rpm differ http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/handlers/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql_server/handlers/main.yml b/deployment/roles/mysql_server/handlers/main.yml new file mode 100644 index 0000000..112c5ca --- /dev/null +++ b/deployment/roles/mysql_server/handlers/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: restart elasticsearch + service: name=elasticsearch state=restarted http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/tasks/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql_server/tasks/main.yml b/deployment/roles/mysql_server/tasks/main.yml new file mode 100644 index 0000000..987c160 --- /dev/null +++ b/deployment/roles/mysql_server/tasks/main.yml @@ -0,0 +1,86 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Create temporary directories + file: + path: "/tmp/{{ item }}" + state: directory + mode: 0755 + with_items: + - "geoip" + +- name: Distribute Mysql + copy: + src: "{{ mysql_rpm_version }}.rpm" + dest: /tmp + +- name: Install Msyql Yum Repository + yum: + name: "/tmp/{{ mysql_rpm_version }}.rpm" + +- name: Install MySQL + yum: + name: "{{ item }}" + state: latest + with_items: + - "mysql-community-server" + - "MySQL-python" + +- name: Start MySQL + service: + name: mysqld + state: started + enabled: yes + +- name: Retrieve temporary root password + shell: "grep 'temporary password' /var/log/mysqld.log | sed 's/.*root@localhost: //'" + args: + creates: ~/.my.cnf + register: temp_root_password + +- name: Update mysql root password + command: "mysqladmin --user=root --password='{{ temp_root_password.stdout }}' password '{{ mysql_root_password }}'" + ignore_errors: yes + args: + creates: ~/.my.cnf + +- name: Create .my.cnf + template: + src: "../roles/mysql_server/templates/.my.cnf" + dest: ~/.my.cnf + + +- name: Download GeoIP databases + unarchive: + src: http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz + dest: /tmp/geoip + copy: no + creates: /tmp/geopip/*/GeoLiteCity-Blocks.csv + +- name: Copy to MySQL import directory + shell: "cp /tmp/geoip/*/*.csv /var/lib/mysql-files/" + +- name: Copy DDL + copy: + src: geoip_ddl.sql + dest: /tmp/geoip_ddl.sql + +- name: Import GeoIP DDL + mysql_db: + name: all + state: import + target: /tmp/geoip_ddl.sql http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/templates/.my.cnf ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql_server/templates/.my.cnf b/deployment/roles/mysql_server/templates/.my.cnf new file mode 100644 index 0000000..d5c0825 --- /dev/null +++ b/deployment/roles/mysql_server/templates/.my.cnf @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[client] +user=root +password={{ mysql_root_password }} +host=localhost \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/vars/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/mysql_server/vars/main.yml b/deployment/roles/mysql_server/vars/main.yml new file mode 100644 index 0000000..ccf2426 --- /dev/null +++ b/deployment/roles/mysql_server/vars/main.yml @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +mysql_rpm_version: mysql57-community-release-el6-7.noarch +mysql_root_password: P@ssw0rd + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/pcap_replay/defaults/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/pcap_replay/defaults/main.yml b/deployment/roles/pcap_replay/defaults/main.yml new file mode 100644 index 0000000..b1fae1e --- /dev/null +++ b/deployment/roles/pcap_replay/defaults/main.yml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +pcap_replay_interface: eth0 +pcap_path: /opt/pcap-replay +tcpreplay_version: 4.1.1 +tcpreplay_prefix: /opt http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/pcap_replay/templates/pcap-replay ---------------------------------------------------------------------- diff --git a/deployment/roles/pcap_replay/templates/pcap-replay b/deployment/roles/pcap_replay/templates/pcap-replay index 56dc40c..b9ae0c3 100644 --- a/deployment/roles/pcap_replay/templates/pcap-replay +++ b/deployment/roles/pcap_replay/templates/pcap-replay @@ -24,7 +24,7 @@ DAEMON_PATH="{{ pcap_path }}" PCAPIN=`ls $DAEMON_PATH/*.pcap 2> /dev/null` -IFACE="{{ pcap_replay_interface | default("eth0") }}" +IFACE="{{ pcap_replay_interface }}" EXTRA_ARGS="${@:2}" DAEMON="{{ tcpreplay_prefix }}/bin/tcpreplay" DAEMONOPTS="--intf1=$IFACE --loop=0 $EXTRA_ARGS $PCAPIN" http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/pcap_replay/vars/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/pcap_replay/vars/main.yml b/deployment/roles/pcap_replay/vars/main.yml deleted file mode 100644 index b1fae1e..0000000 --- a/deployment/roles/pcap_replay/vars/main.yml +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -pcap_replay_interface: eth0 -pcap_path: /opt/pcap-replay -tcpreplay_version: 4.1.1 -tcpreplay_prefix: /opt http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/tap_interface/defaults/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/tap_interface/defaults/main.yml b/deployment/roles/tap_interface/defaults/main.yml new file mode 100644 index 0000000..ca752b4 --- /dev/null +++ b/deployment/roles/tap_interface/defaults/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +tap_if: tap0 +tap_ip: 10.0.0.1 http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/tap_interface/tasks/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/tap_interface/tasks/main.yml b/deployment/roles/tap_interface/tasks/main.yml new file mode 100644 index 0000000..d4590f7 --- /dev/null +++ b/deployment/roles/tap_interface/tasks/main.yml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install tunctl + yum: name=tunctl + +#TODO - only run when tap_if does not exist +- name: Create {{ tap_if }} + command: tunctl -p + +- name: Bring up {{ tap_if }} on {{ tap_ip }} + command: ifconfig {{ tap_if }} {{ tap_ip }} up + +- name: Put {{ tap_if }} in PROMISC + command: ip link set {{ tap_if }} promisc on + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/yaf/defaults/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/yaf/defaults/main.yml b/deployment/roles/yaf/defaults/main.yml new file mode 100644 index 0000000..f804cb5 --- /dev/null +++ b/deployment/roles/yaf/defaults/main.yml @@ -0,0 +1,29 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +fixbuf_version: 1.7.1 +yaf_version: 2.8.0 +yaf_home: /opt/yaf +yaf_topic: ipfix +hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo +yaf: /usr/local/bin/yaf +yaf_args: "" +yafscii: /usr/local/bin/yafscii +yaf_log: /var/log/yaf.log +yaf_lock: /var/lock/subsys/yaf +kafka_prod: /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/yaf/tasks/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/yaf/tasks/main.yml b/deployment/roles/yaf/tasks/main.yml index 1e1194d..468a4f9 100644 --- a/deployment/roles/yaf/tasks/main.yml +++ b/deployment/roles/yaf/tasks/main.yml @@ -43,13 +43,6 @@ - name: Install kafka yum: name=kafka -- set_fact: - yaf: /usr/local/bin/yaf - yafscii: /usr/local/bin/yafscii - yaf_log: /var/log/yaf.log - yaf_lock: /var/lock/subsys/yaf - kafka_prod: /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh - - name: Check for Java at "{{ java_home }}" stat: path="{{ java_home }}" register: jdk_dir @@ -69,6 +62,6 @@ when: not jdk_dir.stat.exists - name: Start yaf - shell: "daemonize -c {{ yaf_home }} -e {{ yaf_log }} -o {{ yaf_log }} -l {{ yaf_lock }} {{ yaf }} --in {{ sniff_interface }} --live pcap | {{ yafscii }} --tabular | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ yaf_topic }}" + shell: "daemonize -c {{ yaf_home }} -e {{ yaf_log }} -o {{ yaf_log }} -l {{ yaf_lock }} {{ yaf }} --in {{ sniff_interface }} --live pcap {{ yaf_args }} | {{ yafscii }} --tabular | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ yaf_topic }}" args: creates: "{{ yaf_lock }}" http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/yaf/vars/main.yml ---------------------------------------------------------------------- diff --git a/deployment/roles/yaf/vars/main.yml b/deployment/roles/yaf/vars/main.yml deleted file mode 100644 index 1d53958..0000000 --- a/deployment/roles/yaf/vars/main.yml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -fixbuf_version: 1.7.1 -yaf_version: 2.8.0 -yaf_home: /opt/yaf -yaf_topic: ipfix -hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Common/pom.xml ---------------------------------------------------------------------- diff --git a/metron-streaming/Metron-Common/pom.xml b/metron-streaming/Metron-Common/pom.xml index c4fc5aa..605c7ed 100644 --- a/metron-streaming/Metron-Common/pom.xml +++ b/metron-streaming/Metron-Common/pom.xml @@ -222,6 +222,11 @@ 1.4 true + + + *slf4j* + + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java ---------------------------------------------------------------------- diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java index 6d094ee..a8fda69 100644 --- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java +++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java @@ -66,6 +66,7 @@ public class BulkMessageWriterBolt extends ConfiguredBolt { @Override public void execute(Tuple tuple) { JSONObject message = (JSONObject) tuple.getValueByField("message"); + message.put("index." + bulkMessageWriter.getClass().getSimpleName().toLowerCase() + ".ts", "" + System.currentTimeMillis()); String sourceType = TopologyUtils.getSourceType(message); SourceConfig configuration = configurations.get(sourceType); int batchSize = configuration != null ? configuration.getBatchSize() : 1; @@ -80,7 +81,9 @@ public class BulkMessageWriterBolt extends ConfiguredBolt { sourceMessageMap.put(sourceType, messageList); } else { try { - bulkMessageWriter.write(sourceType, configuration, tupleList, messageList); + + String esType = sourceType + "_doc"; + bulkMessageWriter.write(esType, configuration, tupleList, messageList); for(Tuple t: tupleList) { collector.ack(t); } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java ---------------------------------------------------------------------- diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java index dac1c0a..653eade 100644 --- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java +++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java @@ -27,6 +27,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.Sets; +import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.HashMap; http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java ---------------------------------------------------------------------- diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java index 7079d5c..6f43739 100644 --- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java +++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java @@ -20,10 +20,12 @@ package org.apache.metron.domain; import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; import java.io.Serializable; +import java.util.List; public class Enrichment implements Serializable { private String type; + private List fields; private T adapter; public Enrichment() {} @@ -33,6 +35,15 @@ public class Enrichment implements Serializable { this.adapter = adapter; } + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } + public String getType() { return type; }