From commits-return-3886-archive-asf-public=cust-asf.ponee.io@metron.apache.org Tue Sep 18 16:54:51 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id A86781807A1 for ; Tue, 18 Sep 2018 16:54:49 +0200 (CEST) Received: (qmail 29011 invoked by uid 500); 18 Sep 2018 14:54:43 -0000 Mailing-List: contact commits-help@metron.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@metron.apache.org Delivered-To: mailing list commits@metron.apache.org Received: (qmail 28727 invoked by uid 99); 18 Sep 2018 14:54:43 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 18 Sep 2018 14:54:43 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 19B28E08DE; Tue, 18 Sep 2018 14:54:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: nickallen@apache.org To: commits@metron.apache.org Date: Tue, 18 Sep 2018 14:55:02 -0000 Message-Id: <751abb28b19f4666bc96318254229392@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [21/21] metron git commit: Merge remote-tracking branch 'apache/master' into feature/METRON-1699-create-batch-profiler Merge remote-tracking branch 'apache/master' into feature/METRON-1699-create-batch-profiler Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/cad2f408 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/cad2f408 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/cad2f408 Branch: refs/heads/feature/METRON-1699-create-batch-profiler Commit: cad2f4086a29f25a5ae2aeaf70d0933776df3681 Parents: fb623f6 02a1280 Author: Nick Allen Authored: Tue Sep 18 10:53:41 2018 -0400 Committer: Nick Allen Committed: Tue Sep 18 10:53:41 2018 -0400 ---------------------------------------------------------------------- .gitignore | 8 + metron-analytics/metron-maas-common/pom.xml | 2 +- metron-analytics/metron-maas-service/pom.xml | 2 +- metron-analytics/metron-profiler-client/pom.xml | 2 +- metron-analytics/metron-profiler-common/pom.xml | 2 +- .../profiler/DefaultMessageDistributor.java | 18 +- .../src/test/resources/log4j.properties | 3 + metron-analytics/metron-profiler-repl/pom.xml | 2 +- metron-analytics/metron-profiler-spark/pom.xml | 2 +- metron-analytics/metron-profiler-storm/pom.xml | 2 +- .../profiler/storm/ProfileBuilderBolt.java | 71 +- .../zookeeper/event-time-test/profiler.json | 19 +- .../profiler/storm/ProfileBuilderBoltTest.java | 22 + .../integration/ProfilerIntegrationTest.java | 351 +- .../src/test/resources/log4j.properties | 10 +- .../src/test/resources/telemetry.json | 100 + metron-analytics/metron-statistics/pom.xml | 2 +- metron-analytics/pom.xml | 2 +- metron-contrib/metron-docker/pom.xml | 2 +- metron-contrib/metron-performance/pom.xml | 2 +- metron-contrib/pom.xml | 2 +- metron-deployment/Kerberos-manual-setup.md | 4 +- metron-deployment/amazon-ec2/conf/defaults.yml | 2 +- .../ansible/playbooks/docker_probe_install.yml | 2 +- .../roles/ambari_master/defaults/main.yml | 2 +- .../centos6/ansible/inventory/group_vars/all | 2 +- .../ubuntu14/ansible/inventory/group_vars/all | 2 +- .../ambari/elasticsearch-mpack/pom.xml | 4 +- .../packaging/ambari/metron-mpack/pom.xml | 4 +- .../metron-mpack/src/main/resources/mpack.json | 2 +- .../packaging/docker/deb-docker/pom.xml | 4 +- .../docker/rpm-docker/SPECS/metron.spec | 6 +- .../packaging/docker/rpm-docker/pom.xml | 4 +- metron-deployment/pom.xml | 2 +- metron-interface/metron-alerts/.gitignore | 49 +- metron-interface/metron-alerts/.nvmrc | 1 + metron-interface/metron-alerts/README.md | 16 +- metron-interface/metron-alerts/angular-cli.json | 65 - metron-interface/metron-alerts/angular.json | 153 + .../metron-alerts/e2e/tsconfig.json | 17 +- metron-interface/metron-alerts/karma.conf.js | 18 +- .../metron-alerts/package-lock.json | 24240 ++++++++++++----- metron-interface/metron-alerts/package.json | 67 +- metron-interface/metron-alerts/pom.xml | 8 +- .../metron-alerts/protractor.conf.js | 12 +- .../scripts/prepend_license_header.sh | 2 +- .../metron-alerts/scripts/start-dev.sh | 2 +- .../scripts/start-server-for-e2e.sh | 2 +- .../alert-details/alert-details.component.ts | 2 +- .../alerts/alerts-list/alerts-list.component.ts | 6 +- .../table-view/table-view.component.spec.ts | 21 +- .../table-view/table-view.component.ts | 2 +- .../alerts-list/tree-view/tree-group-data.ts | 2 +- .../tree-view/tree-view.component.spec.ts | 6 +- .../tree-view/tree-view.component.ts | 2 +- .../configure-table.component.ts | 4 +- .../alerts/meta-alerts/meta-alerts.component.ts | 2 +- .../saved-searches.component.spec.ts | 9 +- .../saved-searches/saved-searches.component.ts | 5 +- .../metron-alerts/src/app/app-routing.module.ts | 2 +- .../metron-alerts/src/app/app.component.spec.ts | 4 +- .../metron-alerts/src/app/app.module.ts | 8 +- .../default-headers.interceptor.ts | 34 + .../src/app/login/login.component.spec.ts | 4 +- .../metron-alerts/src/app/model/rest-error.ts | 4 +- .../pcap/pcap-filters/pcap-filters.component.ts | 5 +- .../pcap-panel/pcap-panel.component.spec.ts | 9 +- .../app/pcap/pcap-panel/pcap-panel.component.ts | 4 +- .../metron-alerts/src/app/pcap/pcap.module.ts | 14 +- .../src/app/pcap/service/pcap.service.spec.ts | 236 +- .../src/app/pcap/service/pcap.service.ts | 115 +- .../src/app/service/alerts.service.ts | 17 +- .../src/app/service/authentication.service.ts | 44 +- .../src/app/service/cluster-metadata.service.ts | 9 +- .../src/app/service/column-names.service.ts | 6 +- .../src/app/service/configure-table.service.ts | 8 +- .../src/app/service/data-source.ts | 7 +- .../service/elasticsearch-localstorage-impl.ts | 36 +- .../src/app/service/global-config.service.ts | 21 +- .../src/app/service/meta-alert.service.ts | 43 +- .../src/app/service/save-search.service.ts | 8 +- .../src/app/service/search.service.ts | 54 +- .../src/app/service/update.service.ts | 41 +- .../metron-alerts/src/app/shared/auth-guard.ts | 2 +- .../metron-alerts/src/app/shared/index.ts | 18 - .../src/app/utils/elasticsearch-utils.ts | 4 +- .../metron-alerts/src/app/utils/httpUtil.ts | 21 +- .../metron-alerts/src/global-shim.ts | 18 + metron-interface/metron-alerts/src/main.ts | 5 +- metron-interface/metron-alerts/src/polyfills.ts | 2 + .../metron-alerts/src/tsconfig.app.json | 3 + .../metron-alerts/src/tsconfig.spec.json | 3 +- metron-interface/metron-config/README.md | 5 +- .../metron-config/package-lock.json | 2 +- metron-interface/metron-config/package.json | 2 +- metron-interface/metron-config/pom.xml | 2 +- .../metron-config/scripts/package.json | 2 +- metron-interface/metron-rest-client/pom.xml | 2 +- metron-interface/metron-rest/pom.xml | 2 +- .../src/main/resources/application.yml | 2 +- metron-interface/pom.xml | 2 +- metron-platform/Performance-tuning-guide.md | 4 +- metron-platform/README.md | 2 +- metron-platform/elasticsearch-shaded/pom.xml | 2 +- metron-platform/metron-common/pom.xml | 2 +- .../src/main/scripts/cluster_info.py | 2 +- metron-platform/metron-data-management/pom.xml | 2 +- metron-platform/metron-elasticsearch/pom.xml | 2 +- metron-platform/metron-enrichment/pom.xml | 2 +- metron-platform/metron-hbase-client/pom.xml | 2 +- metron-platform/metron-hbase/pom.xml | 2 +- metron-platform/metron-indexing/pom.xml | 2 +- metron-platform/metron-integration-test/pom.xml | 2 +- metron-platform/metron-job/pom.xml | 2 +- metron-platform/metron-management/pom.xml | 2 +- metron-platform/metron-parsers/pom.xml | 2 +- metron-platform/metron-pcap-backend/pom.xml | 2 +- .../src/main/scripts/pcap_zeppelin_run.sh | 2 +- metron-platform/metron-pcap/pom.xml | 2 +- metron-platform/metron-solr/pom.xml | 2 +- .../metron-storm-kafka-override/pom.xml | 2 +- metron-platform/metron-storm-kafka/pom.xml | 2 +- metron-platform/metron-test-utilities/pom.xml | 2 +- metron-platform/metron-writer/pom.xml | 2 +- metron-platform/metron-zookeeper/pom.xml | 2 +- metron-platform/pom.xml | 2 +- metron-stellar/pom.xml | 2 +- .../stellar-3rd-party-example/pom.xml | 2 +- metron-stellar/stellar-common/README.md | 4 +- metron-stellar/stellar-common/pom.xml | 2 +- metron-stellar/stellar-zeppelin/README.md | 6 +- metron-stellar/stellar-zeppelin/pom.xml | 2 +- pom.xml | 2 +- site-book/bin/generate-md.sh | 17 + site-book/pom.xml | 2 +- site/current-book/CONTRIBUTING.html | 8 +- site/current-book/Upgrading.html | 8 +- site/current-book/css/maven-base.css | 168 - site/current-book/css/maven-theme.css | 161 - site/current-book/css/print.css | 11 +- .../images/aggregated_parser_chaining_flow.svg | 14 + site/current-book/images/clustered.png | Bin 0 -> 218476 bytes site/current-book/images/collapsed.gif | Bin 53 -> 0 bytes site/current-book/images/drill_down.png | Bin 0 -> 246210 bytes site/current-book/images/expanded.gif | Bin 52 -> 0 bytes site/current-book/images/external.png | Bin 230 -> 0 bytes site/current-book/images/find_alerts.png | Bin 0 -> 581508 bytes site/current-book/images/icon_error_sml.gif | Bin 1010 -> 633 bytes site/current-book/images/icon_info_sml.gif | Bin 606 -> 638 bytes site/current-book/images/icon_success_sml.gif | Bin 990 -> 604 bytes site/current-book/images/icon_warning_sml.gif | Bin 576 -> 625 bytes .../images/message_routing_high_level.svg | 14 + .../metron-job_state_statechart_diagram.svg | 14 + site/current-book/images/newwindow.png | Bin 220 -> 0 bytes .../images/performance_measurement.png | Bin 0 -> 5790 bytes site/current-book/images/squid_search.png | Bin 0 -> 161855 bytes .../images/unified_enrichment_arch.svg | 14 + site/current-book/index.html | 8 +- site/current-book/metron-analytics/index.html | 8 +- .../metron-maas-service/index.html | 8 +- .../metron-profiler-client/index.html | 8 +- .../metron-analytics/metron-profiler/index.html | 53 +- .../metron-statistics/HLLP.html | 8 +- .../metron-statistics/index.html | 8 +- .../metron-contrib/metron-docker/index.html | 8 +- .../metron-performance/index.html | 10 +- .../Kerberos-ambari-setup.html | 8 +- .../Kerberos-manual-setup.html | 97 +- .../metron-deployment/amazon-ec2/index.html | 8 +- .../metron-deployment/ansible/index.html | 8 +- .../metron-deployment/ansible/roles/index.html | 8 +- .../ansible/roles/opentaxii/index.html | 8 +- .../ansible/roles/pcap_replay/index.html | 8 +- .../ansible/roles/sensor-stubs/index.html | 8 +- .../ansible/roles/sensor-test-mode/index.html | 8 +- .../development/centos6/index.html | 12 +- .../development/fastcapa/index.html | 8 +- .../metron-deployment/development/index.html | 8 +- .../development/ubuntu14/index.html | 12 +- site/current-book/metron-deployment/index.html | 13 +- .../metron-deployment/other-examples/index.html | 8 +- .../manual-install/Manual_Install_CentOS6.html | 8 +- .../ambari/elasticsearch-mpack/index.html | 8 +- .../packaging/ambari/index.html | 8 +- .../packaging/ambari/metron-mpack/index.html | 8 +- .../packaging/docker/ansible-docker/index.html | 8 +- .../packaging/docker/deb-docker/index.html | 8 +- .../packaging/docker/rpm-docker/index.html | 8 +- .../packaging/packer-build/index.html | 8 +- .../metron-interface/metron-alerts/index.html | 69 +- .../metron-interface/metron-config/index.html | 13 +- .../metron-interface/metron-rest/index.html | 204 +- .../Performance-tuning-guide.html | 15 +- site/current-book/metron-platform/index.html | 13 +- .../metron-platform/metron-api/index.html | 161 - .../metron-platform/metron-common/index.html | 43 +- .../metron-data-management/index.html | 11 +- .../metron-elasticsearch/index.html | 19 +- .../metron-enrichment/Performance.html | 11 +- .../metron-enrichment/index.html | 25 +- .../metron-platform/metron-indexing/index.html | 65 +- .../metron-platform/metron-job/index.html | 126 + .../metron-management/index.html | 11 +- .../metron-parsers/3rdPartyParser.html | 87 +- .../metron-parsers/ParserChaining.html | 259 + .../metron-platform/metron-parsers/index.html | 131 +- .../metron-parsers/parser-testing.html | 12 +- .../metron-pcap-backend/index.html | 15 +- .../metron-platform/metron-solr/index.html | 300 + .../metron-platform/metron-writer/index.html | 33 +- .../metron-sensors/fastcapa/index.html | 8 +- site/current-book/metron-sensors/index.html | 8 +- .../metron-sensors/pycapa/index.html | 181 +- .../stellar-3rd-party-example/index.html | 8 +- .../stellar-common/3rdPartyStellar.html | 8 +- .../metron-stellar/stellar-common/index.html | 27 +- .../metron-stellar/stellar-zeppelin/index.html | 14 +- .../use-cases/forensic_clustering/index.html | 245 +- .../geographic_login_outliers/index.html | 14 +- site/current-book/use-cases/index.html | 9 +- .../use-cases/parser_chaining/index.html | 412 + .../use-cases/typosquat_detection/index.html | 25 +- site/documentation/index.md | 15 +- 223 files changed, 20489 insertions(+), 9225 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-repl/pom.xml ---------------------------------------------------------------------- diff --cc metron-analytics/metron-profiler-repl/pom.xml index 2447b40,0000000..0876f70 mode 100644,000000..100644 --- a/metron-analytics/metron-profiler-repl/pom.xml +++ b/metron-analytics/metron-profiler-repl/pom.xml @@@ -1,150 -1,0 +1,150 @@@ + + + + 4.0.0 + + metron-analytics + org.apache.metron - 0.5.1 ++ 0.6.0 + + metron-profiler-repl + https://metron.apache.org/ + + UTF-8 + UTF-8 + + + + org.apache.metron + metron-profiler-common + ${project.parent.version} + + + org.apache.metron + metron-profiler-client + ${project.parent.version} + + + org.apache.metron + metron-common + ${project.parent.version} + + + com.google.guava + guava + + + org.slf4j + slf4j-log4j12 + + + + + + org.apache.metron + metron-statistics + ${project.parent.version} + + + kryo + com.esotericsoftware + + + + + org.apache.logging.log4j + log4j-api + ${global_log4j_core_version} + test + + + org.apache.logging.log4j + log4j-core + ${global_log4j_core_version} + test + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${global_shade_version} + + true + + + + package + + shade + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + .yaml + LICENSE.txt + ASL2.0 + NOTICE.txt + + + + + + + + + + + + + maven-assembly-plugin + + src/main/assembly/assembly.xml + + + + make-assembly + package + + single + + + + + + + http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-spark/pom.xml ---------------------------------------------------------------------- diff --cc metron-analytics/metron-profiler-spark/pom.xml index 387dce4,0000000..587b38c mode 100644,000000..100644 --- a/metron-analytics/metron-profiler-spark/pom.xml +++ b/metron-analytics/metron-profiler-spark/pom.xml @@@ -1,216 -1,0 +1,216 @@@ + + + + + 4.0.0 + + org.apache.metron + metron-analytics - 0.5.1 ++ 0.6.0 + + metron-profiler-spark + https://metron.apache.org/ + + UTF-8 + UTF-8 + + + + org.apache.spark + spark-core_2.11 + ${global_spark_version} + + + org.apache.spark + spark-sql_2.11 + ${global_spark_version} + + + org.antlr + antlr-runtime + + + + + org.apache.metron + metron-profiler-common + ${project.parent.version} + + + org.apache.metron + metron-profiler-client + ${project.parent.version} + test + + + org.apache.metron + metron-common + ${project.parent.version} + + + com.google.guava + guava + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.metron + metron-hbase + ${project.parent.version} + + + org.slf4j + slf4j-log4j12 + + + org.apache.storm + storm-hbase + + + + + org.apache.metron + metron-hbase + ${project.parent.version} + test + test-jar + + + org.apache.hbase + hbase-client + ${global_hbase_version} + + + org.apache.hadoop + hadoop-auth + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + + + + org.apache.metron + metron-statistics + ${project.parent.version} + + + kryo + com.esotericsoftware + + + + + org.apache.logging.log4j + log4j-api + ${global_log4j_core_version} + test + + + org.apache.logging.log4j + log4j-core + ${global_log4j_core_version} + test + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${global_shade_version} + + true + + + + package + + shade + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + com.tdunning + org.apache.metron.tdunning + + + + + storm:storm-core:* + storm:storm-lib:* + org.slf4j.impl* + org.slf4j:slf4j-log4j* + + + + + + .yaml + LICENSE.txt + ASL2.0 + NOTICE.txt + + + + + + + + + + + + + maven-assembly-plugin + + src/main/assembly/assembly.xml + + + + make-assembly + package + + single + + + + + + + http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-storm/pom.xml ---------------------------------------------------------------------- diff --cc metron-analytics/metron-profiler-storm/pom.xml index 22c6255,0000000..dd80467 mode 100644,000000..100644 --- a/metron-analytics/metron-profiler-storm/pom.xml +++ b/metron-analytics/metron-profiler-storm/pom.xml @@@ -1,407 -1,0 +1,407 @@@ + + + + + 4.0.0 + + org.apache.metron + metron-analytics - 0.5.1 ++ 0.6.0 + + metron-profiler-storm + https://metron.apache.org/ + + UTF-8 + UTF-8 + + + + org.apache.hadoop + hadoop-auth + ${global_hadoop_version} + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hadoop + hadoop-common + ${global_hadoop_version} + + + servlet-api + javax.servlet + + + commons-httpclient + commons-httpclient + + + org.slf4j + slf4j-log4j12 + + + asm + asm + + + + + org.apache.hadoop + hadoop-hdfs + ${global_hadoop_version} + + + servlet-api + javax.servlet + + + + + org.apache.metron + metron-writer + ${project.parent.version} + + + org.apache.metron + metron-storm-kafka + ${project.parent.version} + + + org.apache.metron + metron-statistics + ${project.parent.version} + + + kryo + com.esotericsoftware + + + + + org.apache.metron + metron-profiler-common + ${project.parent.version} + + + org.apache.metron + metron-profiler-client + ${project.parent.version} + + + org.apache.metron + metron-common + ${project.parent.version} + + + com.google.guava + guava + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.metron + metron-hbase + ${project.parent.version} + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.metron + metron-hbase + ${project.parent.version} + test + test-jar + + + org.slf4j + slf4j-log4j12 + + + + + com.esotericsoftware + kryo-shaded + ${global_kryo_version} + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${global_hadoop_version} + + + servlet-api + javax.servlet + + + commons-httpclient + commons-httpclient + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hbase + hbase-client + ${global_hbase_version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + com.google.guava + guava + + + org.apache.hadoop + hadoop-auth + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + + + org.apache.storm + storm-core + ${global_storm_version} + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + servlet-api + javax.servlet + + + log4j-over-slf4j + org.slf4j + + + log4j-slf4j-impl + org.apache.logging.log4j + + + kryo + com.esotericsoftware + + + provided + + + org.apache.logging.log4j + log4j-api + ${global_log4j_core_version} + test + + + org.apache.logging.log4j + log4j-core + ${global_log4j_core_version} + test + + + org.apache.kafka + kafka_2.10 + ${global_kafka_version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.kafka + kafka-clients + ${global_kafka_version} + test + test + + + log4j + log4j + + + + + org.mockito + mockito-all + ${global_mockito_version} + test + + + com.google.code.tempus-fugit + tempus-fugit + 1.2-20140129.191141-5 + test + + + org.apache.metron + metron-integration-test + ${project.parent.version} + test + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.metron + metron-test-utilities + ${project.parent.version} + test + + + org.slf4j + slf4j-log4j12 + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${global_shade_version} + + true + + + + package + + shade + + + true + uber + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + com.google.common + org.apache.metron.guava.metron-profiler + + + com.fasterxml.jackson + org.apache.metron.jackson + + + + + storm:storm-core:* + storm:storm-lib:* + org.slf4j.impl* + org.slf4j:slf4j-log4j* + + + + + + + .yaml + LICENSE.txt + ASL2.0 + NOTICE.txt + + + + + + + + + + + + + + + org.atteo.classindex + classindex-transformer + ${global_classindex_version} + + + + + maven-assembly-plugin + + src/main/assembly/assembly.xml + + + + make-assembly + package + + single + + + + + + + http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java ---------------------------------------------------------------------- diff --cc metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java index 205159e,0000000..a4cd4f8 mode 100644,000000..100644 --- a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java +++ b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java @@@ -1,509 -1,0 +1,552 @@@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.profiler.storm; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.curator.RetryPolicy; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.recipes.cache.TreeCacheEvent; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.ConfigurationType; +import org.apache.metron.common.configuration.ConfigurationsUtils; +import org.apache.metron.common.configuration.profiler.ProfileConfig; +import org.apache.metron.common.configuration.profiler.ProfilerConfigurations; +import org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater; +import org.apache.metron.common.zookeeper.configurations.ProfilerUpdater; +import org.apache.metron.common.zookeeper.configurations.Reloadable; +import org.apache.metron.profiler.DefaultMessageDistributor; +import org.apache.metron.profiler.MessageDistributor; +import org.apache.metron.profiler.MessageRoute; +import org.apache.metron.profiler.ProfileMeasurement; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.zookeeper.SimpleEventListener; +import org.apache.metron.zookeeper.ZKCache; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseWindowedBolt; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.windowing.TupleWindow; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; ++import java.util.LongSummaryStatistics; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; ++import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.apache.metron.profiler.storm.ProfileSplitterBolt.ENTITY_TUPLE_FIELD; +import static org.apache.metron.profiler.storm.ProfileSplitterBolt.MESSAGE_TUPLE_FIELD; +import static org.apache.metron.profiler.storm.ProfileSplitterBolt.PROFILE_TUPLE_FIELD; +import static org.apache.metron.profiler.storm.ProfileSplitterBolt.TIMESTAMP_TUPLE_FIELD; + +/** + * A Storm bolt that is responsible for building a profile. + * + *

This bolt maintains the state required to build a Profile. When the window + * period expires, the data is summarized as a {@link ProfileMeasurement}, all state is + * flushed, and the {@link ProfileMeasurement} is emitted. ++ * ++ *

There are two mechanisms that will cause a profile to flush. As new messages arrive, ++ * time is advanced. The splitter bolt attaches a timestamp to each message (which can be ++ * either event or system time.) This advances time and leads to profile measurements ++ * being flushed. Alternatively, if no messages arrive to advance time, then the "time-to-live" ++ * mechanism will flush a profile after no messages have been received for some period of time. + */ +public class ProfileBuilderBolt extends BaseWindowedBolt implements Reloadable { + + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private OutputCollector collector; + + /** + * The URL to connect to Zookeeper. + */ + private String zookeeperUrl; + + /** + * The Zookeeper client connection. + */ + protected CuratorFramework zookeeperClient; + + /** + * The Zookeeper cache. + */ + protected ZKCache zookeeperCache; + + /** + * Manages configuration for the Profiler. + */ + private ProfilerConfigurations configurations; + + /** + * The duration of each profile period in milliseconds. + */ + private long periodDurationMillis; + + /** + * The duration of Storm's event window. + */ + private long windowDurationMillis; + + /** + * If a message has not been applied to a Profile in this number of milliseconds, + * the Profile will be forgotten and its resources will be cleaned up. + * + *

WARNING: The TTL must be at least greater than the period duration. + */ + private long profileTimeToLiveMillis; + + /** + * The maximum number of {@link MessageRoute} routes that will be maintained by + * this bolt. After this value is exceeded, lesser used routes will be evicted + * from the internal cache. + */ + private long maxNumberOfRoutes; + + /** + * Distributes messages to the profile builders. + * + *

Since expired profiles are flushed on a separate thread, all access to this + * {@code MessageDistributor} needs to be protected. + */ + private MessageDistributor messageDistributor; + + /** + * Parses JSON messages. + */ + private transient JSONParser parser; + + /** + * Responsible for emitting {@link ProfileMeasurement} values. + * + *

The {@link ProfileMeasurement} values generated by a profile can be written to + * multiple endpoints like HBase or Kafka. Each endpoint is handled by a separate + * {@link ProfileMeasurementEmitter}. + */ + private List emitters; + + /** + * Signals when it is time to flush the active profiles. + */ + private FlushSignal activeFlushSignal; + + /** + * An executor that flushes expired profiles at a regular interval on a separate + * thread. + * + *

Flushing expired profiles ensures that any profiles that stop receiving messages + * for an extended period of time will continue to be flushed. + * + *

This introduces concurrency issues as the bolt is no longer single threaded. Due + * to this, all access to the {@code MessageDistributor} needs to be protected. + */ + private transient ScheduledExecutorService flushExpiredExecutor; + + public ProfileBuilderBolt() { + this.emitters = new ArrayList<>(); + } + + @Override + public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { + super.prepare(stormConf, context, collector); + + if(periodDurationMillis <= 0) { + throw new IllegalArgumentException("expect 'profiler.period.duration' >= 0"); + } + if(profileTimeToLiveMillis <= 0) { + throw new IllegalArgumentException("expect 'profiler.ttl' >= 0"); + } + if(profileTimeToLiveMillis < periodDurationMillis) { + throw new IllegalArgumentException("expect 'profiler.ttl' >= 'profiler.period.duration'"); + } + if(maxNumberOfRoutes <= 0) { + throw new IllegalArgumentException("expect 'profiler.max.routes.per.bolt' > 0"); + } + if(windowDurationMillis <= 0) { + throw new IllegalArgumentException("expect 'profiler.window.duration' > 0"); + } + if(windowDurationMillis > periodDurationMillis) { + throw new IllegalArgumentException("expect 'profiler.period.duration' >= 'profiler.window.duration'"); + } + if(periodDurationMillis % windowDurationMillis != 0) { + throw new IllegalArgumentException("expect 'profiler.period.duration' % 'profiler.window.duration' == 0"); + } + + this.collector = collector; + this.parser = new JSONParser(); + this.messageDistributor = new DefaultMessageDistributor(periodDurationMillis, profileTimeToLiveMillis, maxNumberOfRoutes); + this.configurations = new ProfilerConfigurations(); + this.activeFlushSignal = new FixedFrequencyFlushSignal(periodDurationMillis); + setupZookeeper(); + startFlushingExpiredProfiles(); + } + + @Override + public void cleanup() { + try { + zookeeperCache.close(); + zookeeperClient.close(); + flushExpiredExecutor.shutdown(); + + } catch(Throwable e) { + LOG.error("Exception when cleaning up", e); + } + } + + /** + * Setup connectivity to Zookeeper which provides the necessary configuration for the bolt. + */ + private void setupZookeeper() { + try { + if (zookeeperClient == null) { + RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); + zookeeperClient = CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy); + } + zookeeperClient.start(); + + // this is temporary to ensure that any validation passes. the individual bolt + // will reinitialize stellar to dynamically pull from zookeeper. + ConfigurationsUtils.setupStellarStatically(zookeeperClient); + if (zookeeperCache == null) { + ConfigurationsUpdater updater = createUpdater(); + SimpleEventListener listener = new SimpleEventListener.Builder() + .with( updater::update, TreeCacheEvent.Type.NODE_ADDED, TreeCacheEvent.Type.NODE_UPDATED) + .with( updater::delete, TreeCacheEvent.Type.NODE_REMOVED) + .build(); + zookeeperCache = new ZKCache.Builder() + .withClient(zookeeperClient) + .withListener(listener) + .withRoot(Constants.ZOOKEEPER_TOPOLOGY_ROOT) + .build(); + updater.forceUpdate(zookeeperClient); + zookeeperCache.start(); + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + protected ConfigurationsUpdater createUpdater() { + return new ProfilerUpdater(this, this::getConfigurations); + } + + public ProfilerConfigurations getConfigurations() { + return configurations; + } + + @Override + public void reloadCallback(String name, ConfigurationType type) { + // nothing to do + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + + if(emitters.size() == 0) { + throw new IllegalStateException("At least one destination handler must be defined."); + } + + // allow each emitter to define its own stream + emitters.forEach(emitter -> emitter.declareOutputFields(declarer)); + } + + private Context getStellarContext() { + + Map global = getConfigurations().getGlobalConfig(); + return new Context.Builder() + .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> zookeeperClient) + .with(Context.Capabilities.GLOBAL_CONFIG, () -> global) + .with(Context.Capabilities.STELLAR_CONFIG, () -> global) + .build(); + } + ++ /** ++ * Logs information about the {@link TupleWindow}. ++ * ++ * @param window The tuple window. ++ */ ++ private void log(TupleWindow window) { ++ // summarize the newly received tuples ++ LongSummaryStatistics received = window.get() ++ .stream() ++ .map(tuple -> getField(TIMESTAMP_TUPLE_FIELD, tuple, Long.class)) ++ .collect(Collectors.summarizingLong(Long::longValue)); ++ ++ LOG.debug("Tuple(s) received; count={}, min={}, max={}, range={} ms", ++ received.getCount(), ++ received.getMin(), ++ received.getMax(), ++ received.getMax() - received.getMin()); ++ ++ if (window.getExpired().size() > 0) { ++ // summarize the expired tuples ++ LongSummaryStatistics expired = window.getExpired() ++ .stream() ++ .map(tuple -> getField(TIMESTAMP_TUPLE_FIELD, tuple, Long.class)) ++ .collect(Collectors.summarizingLong(Long::longValue)); ++ ++ LOG.debug("Tuple(s) expired; count={}, min={}, max={}, range={} ms, lag={} ms", ++ expired.getCount(), ++ expired.getMin(), ++ expired.getMax(), ++ expired.getMax() - expired.getMin(), ++ received.getMin() - expired.getMin()); ++ } ++ } ++ + @Override + public void execute(TupleWindow window) { - - LOG.debug("Tuple window contains {} tuple(s), {} expired, {} new", - CollectionUtils.size(window.get()), - CollectionUtils.size(window.getExpired()), - CollectionUtils.size(window.getNew())); ++ if(LOG.isDebugEnabled()) { ++ log(window); ++ } + + try { - + // handle each tuple in the window + for(Tuple tuple : window.get()) { + handleMessage(tuple); + } + + // time to flush active profiles? + if(activeFlushSignal.isTimeToFlush()) { + flushActive(); + } + + } catch (Throwable e) { - + LOG.error("Unexpected error", e); + collector.reportError(e); + } + } + + /** + * Flush all active profiles. + */ + protected void flushActive() { + activeFlushSignal.reset(); + + // flush the active profiles + List measurements; + synchronized(messageDistributor) { + measurements = messageDistributor.flush(); + emitMeasurements(measurements); + } + + LOG.debug("Flushed active profiles and found {} measurement(s).", measurements.size()); - + } + + /** + * Flushes all expired profiles. + * + *

If a profile has not received a message for an extended period of time then it is + * marked as expired. Periodically we need to flush these expired profiles to ensure + * that their state is not lost. + */ + protected void flushExpired() { ++ List measurements = null; ++ try { ++ // flush the expired profiles ++ synchronized (messageDistributor) { ++ measurements = messageDistributor.flushExpired(); ++ emitMeasurements(measurements); ++ } + - // flush the expired profiles - List measurements; - synchronized (messageDistributor) { - measurements = messageDistributor.flushExpired(); - emitMeasurements(measurements); ++ } catch(Throwable t) { ++ // need to catch the exception, otherwise subsequent executions would be suppressed. ++ // see java.util.concurrent.ScheduledExecutorService#scheduleAtFixedRate ++ LOG.error("Failed to flush expired profiles", t); + } + - LOG.debug("Flushed expired profiles and found {} measurement(s).", measurements.size()); ++ LOG.debug("Flushed expired profiles and found {} measurement(s).", CollectionUtils.size(measurements)); + } + + /** + * Handles the processing of a single tuple. + * + * @param input The tuple containing a telemetry message. + */ + private void handleMessage(Tuple input) { + + // crack open the tuple + JSONObject message = getField(MESSAGE_TUPLE_FIELD, input, JSONObject.class); + ProfileConfig definition = getField(PROFILE_TUPLE_FIELD, input, ProfileConfig.class); + String entity = getField(ENTITY_TUPLE_FIELD, input, String.class); + Long timestamp = getField(TIMESTAMP_TUPLE_FIELD, input, Long.class); + + // keep track of time + activeFlushSignal.update(timestamp); + + // distribute the message + MessageRoute route = new MessageRoute(definition, entity, message, timestamp); + synchronized (messageDistributor) { + messageDistributor.distribute(route, getStellarContext()); + } + + LOG.debug("Message distributed: profile={}, entity={}, timestamp={}", definition.getProfile(), entity, timestamp); + } + + /** + * Handles the {@code ProfileMeasurement}s that are created when a profile is flushed. + * + * @param measurements The measurements to handle. + */ + private void emitMeasurements(List measurements) { + + // flush each profile + for(ProfileMeasurement measurement: measurements) { + + // allow each 'emitter' to emit the measurement + for (ProfileMeasurementEmitter emitter : emitters) { + emitter.emit(measurement, collector); + + LOG.debug("Measurement emitted; stream={}, profile={}, entity={}, value={}, start={}, end={}, duration={}, period={}", + emitter.getStreamId(), + measurement.getProfileName(), + measurement.getEntity(), + measurement.getProfileValue(), + measurement.getPeriod().getStartTimeMillis(), + measurement.getPeriod().getEndTimeMillis(), + measurement.getPeriod().getDurationMillis(), + measurement.getPeriod().getPeriod()); + } + } + + LOG.debug("Emitted {} measurement(s).", measurements.size()); + } + + /** + * Retrieves an expected field from a Tuple. If the field is missing an exception is thrown to + * indicate a fatal error. + * @param fieldName The name of the field. + * @param tuple The tuple from which to retrieve the field. + * @param clazz The type of the field value. + * @param The type of the field value. + */ + private T getField(String fieldName, Tuple tuple, Class clazz) { + + T value = ConversionUtils.convert(tuple.getValueByField(fieldName), clazz); + if(value == null) { + throw new IllegalStateException(format("Invalid tuple: missing or invalid field '%s'", fieldName)); + } + + return value; + } + + /** + * Creates a separate thread that regularly flushes expired profiles. + */ + private void startFlushingExpiredProfiles() { + + long initialDelay = profileTimeToLiveMillis; + long period = profileTimeToLiveMillis; + flushExpiredExecutor = Executors.newSingleThreadScheduledExecutor(); + flushExpiredExecutor.scheduleAtFixedRate(() -> flushExpired(), initialDelay, period, TimeUnit.MILLISECONDS); + } + + @Override + public BaseWindowedBolt withTumblingWindow(BaseWindowedBolt.Duration duration) { + + // need to capture the window duration to validate it along with other profiler settings + this.windowDurationMillis = duration.value; + return super.withTumblingWindow(duration); + } + + public long getPeriodDurationMillis() { + return periodDurationMillis; + } + + public ProfileBuilderBolt withPeriodDurationMillis(long periodDurationMillis) { + this.periodDurationMillis = periodDurationMillis; + return this; + } + + public ProfileBuilderBolt withPeriodDuration(int duration, TimeUnit units) { + return withPeriodDurationMillis(units.toMillis(duration)); + } + + public ProfileBuilderBolt withProfileTimeToLiveMillis(long timeToLiveMillis) { + this.profileTimeToLiveMillis = timeToLiveMillis; + return this; + } + + public long getWindowDurationMillis() { + return windowDurationMillis; + } + + public ProfileBuilderBolt withProfileTimeToLive(int duration, TimeUnit units) { + return withProfileTimeToLiveMillis(units.toMillis(duration)); + } + + public ProfileBuilderBolt withEmitter(ProfileMeasurementEmitter emitter) { + this.emitters.add(emitter); + return this; + } + + public MessageDistributor getMessageDistributor() { + return messageDistributor; + } + + public ProfileBuilderBolt withZookeeperUrl(String zookeeperUrl) { + this.zookeeperUrl = zookeeperUrl; + return this; + } + + public ProfileBuilderBolt withZookeeperClient(CuratorFramework zookeeperClient) { + this.zookeeperClient = zookeeperClient; + return this; + } + + public ProfileBuilderBolt withZookeeperCache(ZKCache zookeeperCache) { + this.zookeeperCache = zookeeperCache; + return this; + } + + public ProfileBuilderBolt withProfilerConfigurations(ProfilerConfigurations configurations) { + this.configurations = configurations; + return this; + } + + public ProfileBuilderBolt withMaxNumberOfRoutes(long maxNumberOfRoutes) { + this.maxNumberOfRoutes = maxNumberOfRoutes; + return this; + } + + public ProfileBuilderBolt withFlushSignal(FlushSignal flushSignal) { + this.activeFlushSignal = flushSignal; + return this; + } + + public ProfileBuilderBolt withMessageDistributor(MessageDistributor messageDistributor) { + this.messageDistributor = messageDistributor; + return this; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json ---------------------------------------------------------------------- diff --cc metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json index 9d727a3,0000000..534b7c6 mode 100644,000000..100644 --- a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json +++ b/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json @@@ -1,12 -1,0 +1,19 @@@ +{ ++ "timestampField": "timestamp", + "profiles": [ + { - "profile": "event-time-test", ++ "profile": "count-by-ip", + "foreach": "ip_src_addr", - "init": { "counter": "0" }, - "update": { "counter": "counter + 1" }, - "result": "counter" ++ "init": { "count": 0 }, ++ "update": { "count" : "count + 1" }, ++ "result": "count" ++ }, ++ { ++ "profile": "total-count", ++ "foreach": "'total'", ++ "init": { "count": 0 }, ++ "update": { "count": "count + 1" }, ++ "result": "count" + } - ], - "timestampField": "timestamp" ++ ] +}