metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject [49/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
Date Sun, 18 Nov 2018 18:16:02 GMT
METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8bf3b6ec
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8bf3b6ec
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8bf3b6ec

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 8bf3b6ec9c6871daadcaa2341d01082d2584e341
Parents: e7e19fb
Author: mmiklavc <michael.miklavcic@gmail.com>
Authored: Thu Nov 15 14:16:15 2018 -0700
Committer: Michael Miklavcic <michael.miklavcic@gmail.com>
Committed: Thu Nov 15 16:51:22 2018 -0700

----------------------------------------------------------------------
 .gitignore                                      |     1 +
 Upgrading.md                                    |    24 +
 dependencies_with_url.csv                       |     5 +-
 .../committer-utils/metron-committer-common     |   358 +
 dev-utilities/committer-utils/prepare-commit    |   241 +-
 dev-utilities/release-utils/README.md           |   129 +
 dev-utilities/release-utils/metron-rc-check     |     6 +-
 .../release-utils/prepare-release-candidate     |   334 +
 metron-analytics/metron-maas-common/pom.xml     |     2 +-
 .../metron-profiler-spark/README.md             |    53 +-
 .../metron/profiler/spark/BatchProfiler.java    |    21 +-
 .../profiler/spark/BatchProfilerConfig.java     |     9 +-
 .../reader/ColumnEncodedTelemetryReader.java    |    84 +
 .../profiler/spark/reader/TelemetryReader.java  |    43 +
 .../profiler/spark/reader/TelemetryReaders.java |   110 +
 .../reader/TextEncodedTelemetryReader.java      |    83 +
 .../spark/BatchProfilerIntegrationTest.java     |    36 +-
 .../ColumnEncodedTelemetryReaderTest.java       |   118 +
 .../spark/function/reader/IsValidJSON.java      |    38 +
 .../function/reader/TelemetryReadersTest.java   |    89 +
 .../reader/TextEncodedTelemetryReaderTest.java  |   114 +
 metron-deployment/Kerberos-manual-setup.md      |   154 +-
 .../roles/ambari_master/defaults/main.yml       |     1 +
 .../ambari_master/tasks/elasticsearch_mpack.yml |     4 +-
 .../ansible/roles/bro/tasks/bro.yml             |     4 +-
 .../ansible/roles/bro/tasks/dependencies.yml    |     4 +-
 .../roles/bro/tasks/metron-bro-plugin-kafka.yml |     4 +-
 .../ansible/roles/librdkafka/defaults/main.yml  |     4 +-
 metron-deployment/development/README.md         |    34 +
 metron-deployment/development/centos6/README.md |    26 +-
 .../development/knox-demo-ldap.ldif             |   101 +
 .../development/ubuntu14/README.md              |    26 +-
 .../configuration/metron-enrichment-env.xml     |     8 +-
 .../configuration/metron-indexing-env.xml       |     4 +-
 .../configuration/metron-profiler-env.xml       |     2 +-
 .../CURRENT/configuration/metron-rest-env.xml   |    10 +-
 .../configuration/metron-security-env.xml       |   186 +
 .../common-services/METRON/CURRENT/metainfo.xml |     3 +
 .../CURRENT/package/files/bro_index.template    |     3 +
 .../package/files/metaalert_index.template      |     4 +-
 .../CURRENT/package/files/snort_index.template  |     3 +
 .../CURRENT/package/files/yaf_index.template    |     3 +
 .../package/scripts/params/params_linux.py      |    27 +-
 .../CURRENT/package/scripts/rest_commands.py    |     4 +
 .../METRON/CURRENT/package/templates/metron.j2  |    16 +
 .../METRON/CURRENT/themes/metron_theme.json     |   171 +-
 .../docker/rpm-docker/SPECS/metron.spec         |     5 +-
 metron-interface/metron-alerts/cypress.json     |     7 +
 .../metron-alerts/cypress/fixtures/config.json  |    23 +
 .../cypress/fixtures/pcap.page-01.json          | 12383 +++++++++++
 .../cypress/fixtures/pcap.status-00.json        |     7 +
 .../cypress/fixtures/pcap.status-01.json        |     7 +
 .../cypress/fixtures/pcap.status-02.json        |     7 +
 .../metron-alerts/cypress/fixtures/search.json  |  5647 +++++
 .../cypress/integration/pcap/pcap.spec.js       |   228 +
 .../metron-alerts/package-lock.json             |  1134 +
 metron-interface/metron-alerts/package.json     |    11 +-
 .../alert-details/alert-details.component.ts    |    10 +-
 .../alerts/alerts-list/alerts-list.component.ts |     9 +-
 .../table-view/table-view.component.spec.ts     |     4 +-
 .../table-view/table-view.component.ts          |    49 +-
 .../tree-view/tree-view.component.spec.ts       |     4 +-
 .../tree-view/tree-view.component.ts            |    20 +-
 .../alerts/meta-alerts/meta-alerts.component.ts |     2 -
 .../save-search/save-search.component.spec.ts   |     4 +-
 .../alerts/save-search/save-search.component.ts |    12 +-
 .../saved-searches.component.spec.ts            |     5 +-
 .../saved-searches/saved-searches.component.ts  |    41 +-
 .../metron-alerts/src/app/app.component.html    |     1 +
 .../metron-alerts/src/app/app.component.spec.ts |     4 +
 .../metron-alerts/src/app/app.module.ts         |    12 +-
 .../src/app/model/confirmation-type.ts          |    21 +
 .../metron-alerts/src/app/model/dialog-type.ts  |    21 +
 .../app/pcap/pcap-panel/pcap-panel.component.ts |    68 +-
 .../src/app/service/dialog.service.spec.ts      |    78 +
 .../src/app/service/dialog.service.ts           |    55 +
 .../src/app/shared/metron-dialog-box.ts         |    91 -
 .../metron-dialog/metron-dialog.component.html  |    40 +
 .../metron-dialog/metron-dialog.component.scss  |    20 +
 .../metron-dialog.component.spec.ts             |   127 +
 .../metron-dialog/metron-dialog.component.ts    |    66 +
 metron-interface/metron-config/angular-cli.json |    51 -
 metron-interface/metron-config/angular.json     |   155 +
 metron-interface/metron-config/karma.conf.js    |    60 +-
 .../metron-config/package-lock.json             | 18987 ++++++-----------
 metron-interface/metron-config/package.json     |    70 +-
 metron-interface/metron-config/pom.xml          |    10 +-
 .../scripts/prepend_license_header.sh           |     2 +-
 .../metron-config/scripts/start_dev.sh          |     2 +-
 .../metron-config/src/app/_fonts.scss           |   101 +
 .../metron-config/src/app/_main.scss            |   112 -
 .../metron-config/src/app/app.component.spec.ts |    64 +-
 .../metron-config/src/app/app.component.ts      |     3 +-
 .../metron-config/src/app/app.config.ts         |     4 +-
 .../metron-config/src/app/app.module.ts         |    17 +-
 .../metron-config/src/app/app.routes.ts         |    10 +-
 .../general-settings.component.spec.ts          |    85 +-
 .../general-settings.component.ts               |    48 +-
 .../metron-config/src/app/global.scss           |     2 +-
 .../default-headers.interceptor.ts              |    32 +
 .../src/app/login/login.component.spec.ts       |    42 +-
 .../metron-config/src/app/model/rest-error.ts   |     4 +-
 .../src/app/navbar/navbar.component.spec.ts     |    19 +-
 .../src/app/navbar/navbar.component.ts          |    31 +-
 .../metron-config/src/app/navbar/navbar.html    |     2 +-
 .../metron-config/src/app/rxjs-operators.ts     |    32 -
 .../sensor-field-schema.component.html          |     2 +-
 .../sensor-field-schema.component.spec.ts       |  1088 +-
 .../sensor-grok/sensor-grok.component.spec.ts   |   137 +-
 .../sensor-grok/sensor-grok.component.ts        |     2 +-
 ...sor-parser-config-readonly.component.spec.ts |   430 +-
 .../sensor-parser-config-readonly.component.ts  |   389 +-
 .../sensor-parser-config.component.spec.ts      |   615 +-
 .../sensor-parser-config.component.ts           |   442 +-
 .../sensor-parser-list.component.spec.ts        |   514 +-
 .../sensor-parser-list.component.ts             |     2 +-
 .../sensor-raw-json.component.spec.ts           |   345 +-
 .../sensor-storm-settings.component.spec.ts     |   294 +-
 .../sensor-threat-triage.component.spec.ts      |   129 +-
 .../app/service/authentication.service.spec.ts  |   276 +-
 .../src/app/service/authentication.service.ts   |    73 +-
 .../app/service/global-config.service.spec.ts   |   101 +-
 .../src/app/service/global-config.service.ts    |    25 +-
 .../app/service/grok-validation.service.spec.ts |   122 +-
 .../src/app/service/grok-validation.service.ts  |    53 +-
 .../src/app/service/hdfs.service.spec.ts        |   129 +-
 .../src/app/service/hdfs.service.ts             |    63 +-
 .../src/app/service/kafka.service.spec.ts       |   123 +-
 .../src/app/service/kafka.service.ts            |    57 +-
 .../sensor-enrichment-config.service.spec.ts    |   240 +-
 .../service/sensor-enrichment-config.service.ts |    76 +-
 .../sensor-indexing-config.service.spec.ts      |   150 +-
 .../service/sensor-indexing-config.service.ts   |    62 +-
 ...sensor-parser-config-history.service.spec.ts |   107 +-
 .../sensor-parser-config-history.service.ts     |    55 +-
 .../sensor-parser-config.service.spec.ts        |   206 +-
 .../app/service/sensor-parser-config.service.ts |   130 +-
 .../src/app/service/stellar.service.spec.ts     |   187 +-
 .../src/app/service/stellar.service.ts          |    72 +-
 .../src/app/service/storm.service.spec.ts       |   402 +-
 .../src/app/service/storm.service.ts            |   156 +-
 .../shared/ace-editor/ace-editor.component.ts   |     9 +-
 .../metron-config/src/app/shared/auth-guard.ts  |     2 +-
 .../src/app/shared/metron-dialog-box.ts         |     6 +-
 .../sample-data/sample-data.component.html      |     2 +-
 .../sample-data/sample-data.component.spec.ts   |    41 +-
 .../metron-config/src/app/util/httpUtil.ts      |    24 +-
 .../src/app/util/httpUtils.spec.ts              |    57 +-
 .../verticalnavbar.component.spec.ts            |    30 +-
 .../verticalnavbar/verticalnavbar.component.ts  |     3 -
 .../src/app/verticalnavbar/verticalnavbar.html  |     4 +-
 metron-interface/metron-config/src/styles.scss  |     2 +-
 metron-interface/metron-config/src/test.ts      |    66 +-
 .../metron-config/src/tsconfig.app.json         |    23 +
 .../metron-config/src/tsconfig.json             |    24 -
 .../metron-config/src/tsconfig.spec.json        |    24 +
 metron-interface/metron-config/src/typings.d.ts |     2 +-
 metron-interface/metron-config/tsconfig.json    |    20 +
 metron-interface/metron-rest/README.md          |    38 +
 metron-interface/metron-rest/pom.xml            |    12 +
 .../src/main/config/rest_application.yml        |    32 +-
 .../apache/metron/rest/MetronRestConstants.java |     1 +
 .../metron/rest/config/WebSecurityConfig.java   |    73 +-
 .../metron/rest/controller/UserController.java  |    13 +
 .../impl/SensorParserConfigServiceImpl.java     |    24 +-
 .../src/main/resources/application-vagrant.yml  |    15 +-
 .../metron-rest/src/main/scripts/metron-rest.sh |    28 +-
 metron-platform/Performance-tuning-guide.md     |     6 +-
 metron-platform/elasticsearch-shaded/pom.xml    |    51 -
 metron-platform/metron-common/README.md         |    48 +-
 .../src/main/config/zookeeper/global.json       |     1 -
 .../common/bolt/ConfiguredEnrichmentBolt.java   |    14 +
 .../common/bolt/ConfiguredParserBolt.java       |    13 +
 .../common/configuration/ConfigOption.java      |     7 +
 .../metron-common/src/main/scripts/stellar      |    12 +-
 metron-platform/metron-data-management/pom.xml  |     5 +-
 metron-platform/metron-elasticsearch/README.md  |    63 +-
 metron-platform/metron-elasticsearch/pom.xml    |    35 +-
 .../client/ElasticsearchClient.java             |   245 +
 .../client/ElasticsearchClientFactory.java      |   189 +
 .../config/ElasticsearchClientConfig.java       |   187 +
 .../config/ElasticsearchClientOptions.java      |    60 +
 .../dao/ElasticsearchColumnMetadataDao.java     |    41 +-
 .../elasticsearch/dao/ElasticsearchDao.java     |    10 +-
 .../dao/ElasticsearchRequestSubmitter.java      |     6 +-
 .../dao/ElasticsearchRetrieveLatestDao.java     |     5 +-
 .../dao/ElasticsearchSearchDao.java             |    16 +-
 .../dao/ElasticsearchUpdateDao.java             |     5 +-
 .../utils/ElasticsearchClient.java              |   156 -
 .../elasticsearch/utils/ElasticsearchUtils.java |   203 +-
 .../elasticsearch/utils/FieldMapping.java       |     3 +
 .../elasticsearch/utils/FieldProperties.java    |     3 +
 .../writer/ElasticsearchWriter.java             |    10 +-
 .../dao/ElasticsearchColumnMetadataDaoTest.java |    13 +-
 .../elasticsearch/dao/ElasticsearchDaoTest.java |     3 +-
 .../dao/ElasticsearchRequestSubmitterTest.java  |     5 +-
 .../dao/ElasticsearchUpdateDaoTest.java         |    12 +-
 .../ElasticsearchSearchIntegrationTest.java     |   353 +-
 metron-platform/metron-enrichment/README.md     |    43 +-
 .../enrichment/bolt/GenericEnrichmentBolt.java  |     1 +
 .../enrichment/bolt/ThreatIntelJoinBolt.java    |     6 -
 .../main/scripts/start_enrichment_topology.sh   |     4 +-
 .../bolt/BulkMessageWriterBoltTest.java         |     2 +-
 .../indexing/dao/SearchIntegrationTest.java     |    34 +-
 .../src/main/sample/patterns/test               |     1 +
 metron-platform/metron-parsers/README.md        |    10 +-
 .../message-parser-implementation-notes.md      |    57 +
 .../parsers/DefaultMessageParserResult.java     |    76 +
 .../parsers/DefaultParserRunnerResults.java     |    71 +
 .../org/apache/metron/parsers/GrokParser.java   |   143 +-
 .../org/apache/metron/parsers/ParserRunner.java |    60 +
 .../apache/metron/parsers/ParserRunnerImpl.java |   322 +
 .../metron/parsers/ParserRunnerResults.java     |    33 +
 .../apache/metron/parsers/bolt/ParserBolt.java  |   315 +-
 .../parsers/filters/BroMessageFilter.java       |     2 +-
 .../metron/parsers/filters/StellarFilter.java   |     2 +-
 .../parsers/fireeye/BasicFireEyeParser.java     |   347 +-
 .../parsers/interfaces/MessageFilter.java       |     2 +-
 .../parsers/interfaces/MessageParser.java       |    37 +-
 .../parsers/interfaces/MessageParserResult.java |    48 +
 .../paloalto/BasicPaloAltoFirewallParser.java   |   413 +-
 .../apache/metron/parsers/paloalto/README.md    |    32 +
 .../metron/parsers/syslog/Syslog5424Parser.java |    44 +-
 .../parsers/topology/ParserComponent.java       |    56 +
 .../parsers/topology/ParserComponents.java      |    67 -
 .../parsers/topology/ParserTopologyBuilder.java |    39 +-
 .../parsers/websphere/GrokWebSphereParser.java  |   217 +-
 .../org/apache/metron/filters/FiltersTest.java  |     4 +-
 .../apache/metron/parsers/GrokParserTest.java   |     9 +-
 .../metron/parsers/MessageParserTest.java       |   108 +-
 .../metron/parsers/MultiLineGrokParserTest.java |   149 +
 .../MultiLineWithErrorsGrokParserTest.java      |   146 +
 .../metron/parsers/ParserRunnerImplTest.java    |   390 +
 .../metron/parsers/SampleGrokParserTest.java    |     3 +
 .../apache/metron/parsers/SquidParserTest.java  |     2 +
 .../apache/metron/parsers/YafParserTest.java    |     3 +
 .../metron/parsers/bolt/ParserBoltTest.java     |   873 +-
 .../parsers/integration/ParserDriver.java       |    62 +-
 .../BasicPaloAltoFirewallParserTest.java        |   244 +
 .../parsers/syslog/Syslog5424ParserTest.java    |    55 +-
 .../websphere/GrokWebSphereParserTest.java      |    69 +-
 .../test/resources/logData/multi_elb_log.txt    |    10 +
 .../logData/multi_elb_with_errors_log.txt       |    13 +
 .../src/main/config/schema/bro/schema.xml       |     1 +
 .../src/main/config/schema/snort/schema.xml     |     1 +
 .../src/main/config/schema/yaf/schema.xml       |     1 +
 .../integration/SolrSearchIntegrationTest.java  |     8 +-
 metron-platform/metron-writer/pom.xml           |     6 +
 .../metron/writer/BulkWriterComponent.java      |    33 +-
 .../writer/bolt/BulkMessageWriterBolt.java      |     2 +-
 .../metron/writer/BulkWriterComponentTest.java  |    61 +-
 metron-stellar/stellar-common/README.md         |   107 +-
 metron-stellar/stellar-common/pom.xml           |    40 +
 .../stellar/common/shell/cli/StellarShell.java  |     2 +
 .../org/apache/metron/stellar/dsl/Context.java  |     5 +
 .../metron/stellar/dsl/StellarFunction.java     |     9 +-
 .../metron/stellar/dsl/StellarFunctions.java    |     5 +
 .../stellar/dsl/functions/DateFunctions.java    |    42 +
 .../stellar/dsl/functions/RestConfig.java       |   166 +
 .../stellar/dsl/functions/RestFunctions.java    |   388 +
 .../resolver/BaseFunctionResolver.java          |    44 +
 .../functions/resolver/FunctionResolver.java    |    14 +-
 .../stellar/dsl/functions/BasicStellarTest.java |    20 +-
 .../dsl/functions/DateFunctionsTest.java        |    46 +
 .../dsl/functions/RestFunctionsTest.java        |   601 +
 .../resolver/BaseFunctionResolverTest.java      |   169 +
 pom.xml                                         |     5 +-
 site/develop/bylaws.md                          |   281 -
 site/develop/coding.md                          |    34 -
 site/develop/index.md                           |    61 -
 270 files changed, 39202 insertions(+), 20055 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index a0b9691..df3f030 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,7 @@ target/
 *.tmp
 *.bak
 *.class
+*.pyc
 
 tmp/**
 tmp/**/*

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/Upgrading.md
----------------------------------------------------------------------
diff --git a/Upgrading.md b/Upgrading.md
index 2124ac5..1da43fa 100644
--- a/Upgrading.md
+++ b/Upgrading.md
@@ -19,6 +19,30 @@ limitations under the License.
 This document constitutes a per-version listing of changes of
 configuration which are non-backwards compatible.
 
+## 0.6.0 to 0.6.1
+
+### [METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API](https://issues.apache.org/jira/browse/METRON-1834)
+The Elasticsearch Java client has now been migrated from TransportClient to the new Java REST client. The motivation for this change
+is that TransportClient will be deprecated in Elasticsearch 7.0 and removed entirely in 8.0. See [ES Java API ](https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.6/client.html) for more details.
+The primary client-facing change for upgrades will be the new properties for configuring the new client. An explanation of the new properties
+as well as a mapping from the old properties to the new can be found in [metron-elasticsearch](metron-platform/metron-elasticsearch/README.md#Properties) under `es.client.settings`.
+
+
+### [METRON-1855: Make unified enrichment topology the default and deprecate split-join](https://issues.apache.org/jira/browse/METRON-1855)
+The unified enrichment topology will be the new default in this release,
+and the split-join enrichment topology is now considered deprecated.
+If you wish to keep the deprecated split-join enrichment topology,
+you will need to make the following changes:
+
+* In Ambari > Metron > Config > Enrichment set the enrichment_topology setting to "Split-Join"
+* If running `start_enrichment_topology.sh` manually, pass in the parameters to start the Split-Join topology as follows
+
+    ```
+    $METRON_HOME/bin/start_enrichment_topology.sh --remote $METRON_HOME/flux/enrichment/remote-splitjoin.yaml --filter $METRON_HOME/config/enrichment-splitjoin.properties
+    ```
+
+* Restart the enrichment topology
+
 ## 0.4.2 to 0.5.0
 
 ### [METRON-941: native PaloAlto parser corrupts message when having a comma in the payload](https://issues.apache.org/jira/browse/METRON-941)

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 66497c3..bdc2508 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -337,6 +337,8 @@ org.springframework:spring-web:jar:5.0.5.RELEASE:compile,The Apache Software Lic
 org.springframework:spring-webmvc:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework
 org.springframework:spring-webmvc:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
 org.springframework:spring-webmvc:jar:5.0.5.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
+org.springframework.ldap:spring-ldap-core:jar:2.3.2.RELEASE:compile,ASLv2,https://spring.io/projects/spring-ldap
+org.springframework.security:spring-security-ldap:jar:5.1.1.RELEASE:compile,ASLv2,https://spring.io/projects/spring-security
 org.tukaani:xz:jar:1.0:compile,Public Domain,http://tukaani.org/xz/java.html
 org.xerial.snappy:snappy-java:jar:1.0.4.1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/snappy-java/
 org.xerial.snappy:snappy-java:jar:1.1.1.7:compile,The Apache Software License, Version 2.0,https://github.com/xerial/snappy-java
@@ -484,5 +486,6 @@ org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile
 org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
 com.zaxxer:HikariCP:jar:2.7.8:compile,ASLv2,https://github.com/brettwooldridge/HikariCP
 org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https://github.com/hibernate/hibernate-validator
-com.github.palindromicity:simple-syslog-5424:jar:0.0.8:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
+com.github.palindromicity:simple-syslog-5424:jar:0.0.9:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
 org.elasticsearch.client:elasticsearch-rest-high-level-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch.plugin:aggs-matrix-stats-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/dev-utilities/committer-utils/metron-committer-common
----------------------------------------------------------------------
diff --git a/dev-utilities/committer-utils/metron-committer-common b/dev-utilities/committer-utils/metron-committer-common
new file mode 100644
index 0000000..0f6c06a
--- /dev/null
+++ b/dev-utilities/committer-utils/metron-committer-common
@@ -0,0 +1,358 @@
+#!/usr/bin/env bash
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+#
+# common and unlikely to change vars
+#
+
+# the upstream apache git repo for apache metron
+METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron.git"
+# the upstream apache git repo for apache metron bro plugin kafka
+BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git"
+# the common configuration file with the committer info
+CONFIG_FILE=~/.metron-prepare-commit
+
+GITHUB_REMOTE="origin"
+BASE_BRANCH=master
+
+# other var setup by these functions
+PR=
+WORK=
+ORIGIN=
+UPSTREAM=
+PR_BRANCH=
+USER=
+EMAIL=
+JIRA=
+DESC=
+APACHE_NAME=
+APACHE_EMAIL=
+GITHUB_NAME=
+CHOSEN_REPO=
+
+#
+# Initialize the variables from the default configuration file, if it exists
+#
+function init_configuration {
+    # does a config file already exist?
+    echo "$CONFIG_FILE"
+    if [ -f ${CONFIG_FILE} ]; then
+        #shellcheck source=/dev/null
+        source ${CONFIG_FILE}
+        echo "  ...using settings from $CONFIG_FILE"
+    fi
+}
+
+
+#
+# Initialize the committer variables if they are not provided through the configuration file.
+# If it is not present, it will be written out for the next time
+#
+function init_committer_info {
+    # github account of committer (you)
+    if [ -z "$GITHUB_NAME" ]; then
+        read -p "  your github username [$GITHUB_NAME]: " INPUT
+        [ -n "$INPUT" ] && GITHUB_NAME=${INPUT}
+
+        # write setting to config file
+        echo "GITHUB_NAME=$GITHUB_NAME" >> ${CONFIG_FILE}
+    fi
+
+    # apache id of committer (you)
+    if [ -z "$APACHE_NAME" ]; then
+      read -p "  your apache userid [$APACHE_NAME]: " INPUT
+      [ -n "$INPUT" ] && APACHE_NAME=${INPUT}
+
+      # write setting to config file
+      echo "APACHE_NAME=$APACHE_NAME" >> ${CONFIG_FILE}
+    fi
+
+    # apache email addr of committer (you)
+    if [ -z "$APACHE_EMAIL" ]; then
+      APACHE_EMAIL=${APACHE_NAME}@apache.org
+      read -p "  your apache email [$APACHE_EMAIL]: " INPUT
+      [ -n "$INPUT" ] && APACHE_EMAIL=${INPUT}
+
+      # write setting to config file, so it is not needed next time
+      echo "APACHE_EMAIL=$APACHE_EMAIL" >> ${CONFIG_FILE}
+    fi
+}
+
+#
+# Provide the user with a choice of the metron or bro repositories
+#
+function choose_metron_or_bro_repo {
+    # which repo?  metron or metron-bro-plugin-kafka
+    echo "    [1] metron"
+    echo "    [2] metron-bro-plugin-kafka"
+    read -p "  which repo? [1]: " INPUT
+    case "${INPUT}" in
+      [Bb][Rr][Oo]|[Mm][Ee][Tt][Rr][Oo][Nn]-[Bb][Rr][Oo]-[Pp][Ll][Uu][Gg][Ii][Nn]-[Kk][Aa][Ff][Kk][Aa]|*metron-bro-plugin-kafka\.git|2)
+        INPUT="${BRO_PLUGIN_UPSTREAM}" ;;
+      [Mm][Ee][Tt][Rr][Oo][Nn]|*metron\.git|1|'')
+        INPUT="${METRON_UPSTREAM}" ;;
+      *)
+        echo "Invalid repo, provided \"${INPUT}\".  Please choose between metron or metron-bro-plugin-kafka"
+        exit 1
+        ;;
+    esac
+    [ -n "$INPUT" ] && UPSTREAM=${INPUT}
+
+    CHOSEN_REPO=$(basename ${UPSTREAM%%.git})
+}
+
+#
+# Ask the user for the PR number
+#
+function read_pull_request {
+    # retrieve the pull request identifier
+    read -p "  pull request: " PR
+    if [ -z "$PR" ]; then
+      echo "Error: missing pr"
+      exit 1
+    fi
+
+    # ensure that the pull request exists
+    PR_EXISTS=`curl -sI https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/${PR} | grep Status: | sed 's/[^0-9]//g'`
+    if [ "$PR_EXISTS" != "200" ]; then
+      echo "Error: pull request #$PR does not exist"
+      exit 1
+    fi
+}
+
+#
+# Setup the working directory.
+# It is possible to override the default directory name by passing
+# the desired directory name, using the available global variables
+#
+function setup_working_directory {
+    # working directory
+    if [ -z $1 ]; then
+        WORK=~/tmp/${CHOSEN_REPO}-pr${PR}
+    else
+        WORK=$1
+    fi
+
+    read -p "  local working directory [$WORK]: " INPUT
+    [ -n "$INPUT" ] && WORK=${INPUT}
+
+    # handle tilde expansion
+    WORK="${WORK/#\~/$HOME}"
+
+    # warn the user if the working directory exists
+    if [ -d "$WORK" ]; then
+      read -p "  directory exists [$WORK].  continue merge on existing repo? [yN] " -n 1 -r
+      echo
+      if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+        exit 1
+      fi
+    fi
+}
+
+#
+# If required will clone the desired repo.  The default repo is metron, but an override
+# may be passed as a parameter.
+# If the directory already exists, it will ask the user for the name of the remote to use, with the
+# default being "origin"
+#
+function setup_code {
+    # if working directory does not exist, checkout the base branch
+    if [ ! -d "$WORK" ]; then
+
+        REPO_NAME="metron"
+        if [ -n $1 ]; then
+            REPO_NAME=$1
+        fi
+
+        # origin repository
+        ORIGIN="https://github.com/apache/${REPO_NAME}"
+        read -p "  origin repo [$ORIGIN]: " INPUT
+        [ -n "$INPUT" ] && ORIGIN=${INPUT}
+
+        # what branch did the PR get submitted against?  could be a feature branch
+        BASE_BRANCH=`curl -s https://api.github.com/repos/apache/${REPO_NAME}/pulls/${PR} | python -c 'import sys, json; print json.load(sys.stdin)["base"]["ref"]'`
+        read -p "  base branch to merge into [$BASE_BRANCH]: " INPUT
+        [ -n "$INPUT" ] && BASE_BRANCH=${INPUT}
+
+        # clone the repository and fetch updates
+        mkdir -p ${WORK}
+        git clone ${ORIGIN} ${WORK}
+        cd ${WORK} || exit "Failed to cd to ${WORK}"
+
+        # setup the git user and email for your apache account
+        git config user.name "$APACHE_NAME"
+        git config user.email "$APACHE_EMAIL"
+
+        # fetch any changes from upstream
+        git remote add upstream ${UPSTREAM}
+        if git fetch upstream "$BASE_BRANCH"; then
+
+            if [ ${BASE_BRANCH} = "master" ]; then
+                # merge any changes from upstream
+                git checkout ${BASE_BRANCH}
+                git merge upstream/${BASE_BRANCH}
+
+            else
+                # create a local branch from the remote feature branch
+                git checkout -B ${BASE_BRANCH} upstream/${BASE_BRANCH}
+
+            fi
+
+          else
+            # unable to fetch the base branch
+            exit $?
+          fi
+
+    else
+
+      # if the repo already exists, allow the user to provide the name of the Github remote
+      # this is needed to checkout the code for the PR
+      read -p "  name of github remote [$GITHUB_REMOTE]: " INPUT
+      [ -n "$INPUT" ] && GITHUB_REMOTE=${INPUT}
+
+    fi
+
+    PR_BRANCH_REF="pull/$PR/head:pr-$PR"
+    PR_BRANCH="pr-$PR"
+    cd ${WORK} || exit "failed to move to ${WORK}"
+    git fetch ${GITHUB_REMOTE} ${PR_BRANCH_REF}
+    echo ""
+}
+
+#
+# Populates the contributor information from the PR information
+#
+function get_contributor_info {
+    # use github api to retrieve the contributor's login
+    USER=`curl -s https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/${PR} | grep login | head -1 | awk -F":" '{print $2}' | sed 's/[^a-zA-Z.@_-]//g'`
+    read -p "  github contributor's username [$USER]: " INPUT
+    [ -n "$INPUT" ] && USER=${INPUT}
+
+    # validate the github contributor
+    if [ -z "$USER" ]; then
+      echo "Error: missing username"
+      exit 1
+    fi
+
+    # retrieve the contributor's email from the git commit history
+    EMAIL=`git log ${PR_BRANCH} | grep Author | head -1 | awk -F"<" '{print $2}' | sed 's/[<>]//g'`
+    read -p "  github contributor's email [$EMAIL]: " INPUT
+    [ -n "$INPUT" ] && EMAIL=${INPUT}
+
+    # validate email
+    if [ -z "$EMAIL" ] || [ "$EMAIL" = "null" ]; then
+      echo "Error: missing email"
+      exit 1
+    fi
+}
+
+#
+# Populate the information for the JIRA associated with the PR
+#
+function get_jira_info {
+    # can we extract the JIRA from the PR title?
+    JIRA=`curl -s https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/${PR} | grep title | head -1 | egrep -o -i 'METRON-[0-9]+' | awk '{print toupper($0)}'`
+    read -p "  issue identifier in jira [$JIRA]: " INPUT
+    [ -n "$INPUT" ] && JIRA=${INPUT}
+
+    # validate the JIRA issue
+    if [ -z "$JIRA" ]; then
+      echo "Error: missing jira"
+      exit 1
+    fi
+
+    # attempt to use the jira api to get a description of the jira
+    DESC=`curl -s https://issues.apache.org/jira/si/jira.issueviews:issue-xml/${JIRA}/${JIRA}.xml | grep "<summary>" | sed 's/^.*<summary>//' | sed 's/<.summary>.*$//'`
+    read -p "  issue description [$DESC]: " INPUT
+    [ -n "$INPUT" ] && DESC=${INPUT}
+
+    # validate description
+    if [ -z "$DESC" ]; then
+      echo "Error: missing description"
+      exit 1
+    fi
+}
+
+#
+# Merge and Commit
+#
+function commit {
+    # commit message
+    AUTHOR="$USER <$EMAIL>"
+    if [ "$USER" == "$GITHUB_NAME" ]; then
+      MSG="$JIRA $DESC ($USER) closes apache/${CHOSEN_REPO}#$PR"
+    else
+      MSG="$JIRA $DESC ($USER via $GITHUB_NAME) closes apache/${CHOSEN_REPO}#$PR"
+    fi
+    read -p "  commit message [$MSG]: " INPUT
+    [ -n "$INPUT" ] && MSG=${INPUT}
+
+    # merge the contributor's branch and commit
+    echo ""
+    if git merge --squash "$PR_BRANCH"; then
+      git commit --author="$AUTHOR" -a -m "$MSG"
+    else
+      exit $?
+    fi
+}
+
+#
+# Displays the commit info ( the diff and log )
+#
+function review_commit_info {
+    # review the commit
+    echo ""
+    echo ""
+    git diff --stat --color "upstream/$BASE_BRANCH..$BASE_BRANCH"
+    echo ""
+    echo ""
+    git log --oneline "$BASE_BRANCH" "^upstream/$BASE_BRANCH"
+}
+
+#
+# Runs the metron unit, integration and metron-interface tests
+#
+function run_tests {
+    echo ""
+    echo ""
+    read -p "  run test suite? [yN] " -n 1 -r
+    echo
+    if [[ $REPLY =~ ^[Yy]$ ]]; then
+      if [ "${UPSTREAM}" == "${METRON_UPSTREAM}" ]; then
+          mvn -q -T 2C -DskipTests clean install &&
+              mvn -q -T 2C surefire:test@unit-tests &&
+              mvn -q surefire:test@integration-tests &&
+              mvn -q test --projects metron-interface/metron-config &&
+          dev-utilities/build-utils/verify_licenses.sh
+      elif [ "${UPSTREAM}" == "${BRO_PLUGIN_UPSTREAM}" ]; then
+        echo "We don't currently support running metron-bro-plugin-kafka tests in this script"
+      fi
+    fi
+
+}
+
+#
+# Gives the user instruction on next steps
+#
+function please_review_then {
+    echo ""
+    echo "Review commit carefully then run..."
+    echo "    cd $WORK"
+    echo "    git push upstream $BASE_BRANCH"
+    echo ""
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/dev-utilities/committer-utils/prepare-commit
----------------------------------------------------------------------
diff --git a/dev-utilities/committer-utils/prepare-commit b/dev-utilities/committer-utils/prepare-commit
index db15aaa..c5dd884 100755
--- a/dev-utilities/committer-utils/prepare-commit
+++ b/dev-utilities/committer-utils/prepare-commit
@@ -16,242 +16,31 @@
 #  limitations under the License.
 #
 
-# not likely to change
-METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron.git"
-BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git"
-CONFIG_FILE=~/.metron-prepare-commit
-GITHUB_REMOTE="origin"
-BASE_BRANCH=master
+LOCATION_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+#shellcheck source=./metron-committer-common
+source ${LOCATION_DIR}/metron-committer-common
 
-# does a config file already exist?
-if [ -f $CONFIG_FILE ]; then
-  . $CONFIG_FILE
-  echo "  ...using settings from $CONFIG_FILE"
-fi
+init_configuration
 
-# which repo?  metron or metron-bro-plugin-kafka
-echo "    [1] metron"
-echo "    [2] metron-bro-plugin-kafka"
-read -p "  which repo? [1]: " INPUT
-case "${INPUT}" in
-  [Bb][Rr][Oo]|[Mm][Ee][Tt][Rr][Oo][Nn]-[Bb][Rr][Oo]-[Pp][Ll][Uu][Gg][Ii][Nn]-[Kk][Aa][Ff][Kk][Aa]|*metron-bro-plugin-kafka\.git|2)
-    INPUT="${BRO_PLUGIN_UPSTREAM}" ;;
-  [Mm][Ee][Tt][Rr][Oo][Nn]|*metron\.git|1|'')
-    INPUT="${METRON_UPSTREAM}" ;;
-  *)
-    echo "Invalid repo, provided \"${INPUT}\".  Please choose between metron or metron-bro-plugin-kafka"
-    exit 1
-    ;;
-esac
-[ -n "$INPUT" ] && UPSTREAM=$INPUT
+choose_metron_or_bro_repo
 
-CHOSEN_REPO=$(basename ${UPSTREAM%%.git})
+init_committer_info
 
-# github account of committer (you)
-if [ -z "$GITHUB_NAME" ]; then
-  read -p "  your github username [$GITHUB_NAME]: " INPUT
-  [ -n "$INPUT" ] && GITHUB_NAME=$INPUT
+read_pull_request
 
-  # write setting to config file
-  echo "GITHUB_NAME=$GITHUB_NAME" >> $CONFIG_FILE
-fi
+setup_working_directory ~/tmp/${CHOSEN_REPO}-pr${PR}
 
-# apache id of committer (you)
-if [ -z "$APACHE_NAME" ]; then
-  read -p "  your apache userid [$APACHE_NAME]: " INPUT
-  [ -n "$INPUT" ] && APACHE_NAME=$INPUT
+setup_code ${CHOSEN_REPO}
 
-  # write setting to config file
-  echo "APACHE_NAME=$APACHE_NAME" >> $CONFIG_FILE
-fi
+get_contributor_info
 
-# apache email addr of committer (you)
-if [ -z "$APACHE_EMAIL" ]; then
-  APACHE_EMAIL=${APACHE_NAME}@apache.org
-  read -p "  your apache email [$APACHE_EMAIL]: " INPUT
-  [ -n "$INPUT" ] && APACHE_EMAIL=$INPUT
+get_jira_info
 
-  # write setting to config file, so it is not needed next time
-  echo "APACHE_EMAIL=$APACHE_EMAIL" >> $CONFIG_FILE
-fi
+commit
 
-# retrieve the pull request identifier
-read -p "  pull request: " PR
-if [ -z "$PR" ]; then
-  echo "Error: missing pr"
-  exit 1
-fi
+review_commit_info
 
-# ensure that the pull request exists
-PR_EXISTS=`curl -sI https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/$PR | grep Status: | sed 's/[^0-9]//g'`
-if [ "$PR_EXISTS" != "200" ]; then
-  echo "Error: pull request #$PR does not exist"
-  exit 1
-fi
+run_tests
 
-# working directory
-WORK=~/tmp/${CHOSEN_REPO}-pr$PR
-read -p "  local working directory [$WORK]: " INPUT
-[ -n "$INPUT" ] && WORK=$INPUT
+please_review_then
 
-# handle tilde expansion
-WORK="${WORK/#\~/$HOME}"
-
-# warn the user if the working directory exists
-if [ -d "$WORK" ]; then
-  read -p "  directory exists [$WORK].  continue merge on existing repo? [yN] " -n 1 -r
-  echo
-  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
-    exit 1
-  fi
-fi
-
-# if working directory does not exist, checkout the base branch
-if [ ! -d "$WORK" ]; then
-
-  # origin repository
-  ORIGIN="https://github.com/apache/${CHOSEN_REPO}"
-  read -p "  origin repo [$ORIGIN]: " INPUT
-  [ -n "$INPUT" ] && ORIGIN=$INPUT
-
-  # what branch did the PR get submitted against?  could be a feature branch
-  BASE_BRANCH=`curl -s https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/$PR | python -c 'import sys, json; print json.load(sys.stdin)["base"]["ref"]'`
-  read -p "  base branch to merge into [$BASE_BRANCH]: " INPUT
-  [ -n "$INPUT" ] && BASE_BRANCH=$INPUT
-
-  # clone the repository and fetch updates
-  mkdir -p $WORK
-  git clone $ORIGIN $WORK
-  cd $WORK
-
-  # setup the git user and email for your apache account
-  git config user.name "$APACHE_NAME"
-  git config user.email $APACHE_EMAIL
-
-  # fetch any changes from upstream
-  git remote add upstream $UPSTREAM
-  if git fetch upstream "$BASE_BRANCH"; then
-
-    if [ $BASE_BRANCH = "master" ]; then
-      # merge any changes from upstream
-      git checkout $BASE_BRANCH
-      git merge upstream/$BASE_BRANCH
-
-    else
-      # create a local branch from the remote feature branch
-      git checkout -B $BASE_BRANCH upstream/$BASE_BRANCH
-
-    fi
-
-  else
-    # unable to fetch the base branch
-    exit $?
-  fi
-
-else
-
-  # if the repo already exists, allow the user to provide the name of the Github remote
-  # this is needed to checkout the code for the PR
-  read -p "  name of github remote [$GITHUB_REMOTE]: " INPUT
-  [ -n "$INPUT" ] && GITHUB_REMOTE=$INPUT
-
-fi
-
-PR_BRANCH_REF="pull/$PR/head:pr-$PR"
-PR_BRANCH="pr-$PR"
-cd $WORK
-git fetch $GITHUB_REMOTE $PR_BRANCH_REF
-echo ""
-
-# use github api to retrieve the contributor's login
-USER=`curl -s https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/$PR | grep login | head -1 | awk -F":" '{print $2}' | sed 's/[^a-zA-Z.@_-]//g'`
-read -p "  github contributor's username [$USER]: " INPUT
-[ -n "$INPUT" ] && USER=$INPUT
-
-# validate the github contributor
-if [ -z "$USER" ]; then
-  echo "Error: missing username"
-  exit 1
-fi
-
-# retrieve the contributor's email from the git commit history
-EMAIL=`git log $PR_BRANCH | grep Author | head -1 | awk -F"<" '{print $2}' | sed 's/[<>]//g'`
-read -p "  github contributor's email [$EMAIL]: " INPUT
-[ -n "$INPUT" ] && EMAIL=$INPUT
-
-# validate email
-if [ -z "$EMAIL" ] || [ "$EMAIL" = "null" ]; then
-  echo "Error: missing email"
-  exit 1
-fi
-
-# can we extract the JIRA from the PR title?
-JIRA=`curl -s https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/$PR | grep title | head -1 | egrep -o -i 'METRON-[0-9]+' | awk '{print toupper($0)}'`
-read -p "  issue identifier in jira [$JIRA]: " INPUT
-[ -n "$INPUT" ] && JIRA=$INPUT
-
-# validate the JIRA issue
-if [ -z "$JIRA" ]; then
-  echo "Error: missing jira"
-  exit 1
-fi
-
-# attempt to use the jira api to get a description of the jira
-DESC=`curl -s https://issues.apache.org/jira/si/jira.issueviews:issue-xml/$JIRA/$JIRA.xml | grep "<summary>" | sed 's/^.*<summary>//' | sed 's/<.summary>.*$//'`
-read -p "  issue description [$DESC]: " INPUT
-[ -n "$INPUT" ] && DESC=$INPUT
-
-# validate description
-if [ -z "$DESC" ]; then
-  echo "Error: missing description"
-  exit 1
-fi
-
-# commit message
-AUTHOR="$USER <$EMAIL>"
-if [ "$USER" == "$GITHUB_NAME" ]; then
-  MSG="$JIRA $DESC ($USER) closes apache/${CHOSEN_REPO}#$PR"
-else
-  MSG="$JIRA $DESC ($USER via $GITHUB_NAME) closes apache/${CHOSEN_REPO}#$PR"
-fi
-read -p "  commit message [$MSG]: " INPUT
-[ -n "$INPUT" ] && MSG=$INPUT
-
-# merge the contributor's branch and commit
-echo ""
-if git merge --squash "$PR_BRANCH"; then
-  git commit --author="$AUTHOR" -a -m "$MSG"
-else
-  exit $?
-fi
-
-# review the commit
-echo ""
-echo ""
-git diff --stat --color "upstream/$BASE_BRANCH..$BASE_BRANCH"
-echo ""
-echo ""
-git log --oneline "$BASE_BRANCH" "^upstream/$BASE_BRANCH"
-
-# run tests?
-echo ""
-echo ""
-read -p "  run test suite? [yN] " -n 1 -r
-echo
-if [[ $REPLY =~ ^[Yy]$ ]]; then
-  if [ "${UPSTREAM}" == "${METRON_UPSTREAM}" ]; then
-      mvn -q -T 2C -DskipTests clean install &&
-          mvn -q -T 2C surefire:test@unit-tests &&
-          mvn -q surefire:test@integration-tests &&
-          mvn -q test --projects metron-interface/metron-config &&
-	  dev-utilities/build-utils/verify_licenses.sh
-  elif [ "${UPSTREAM}" == "${BRO_PLUGIN_UPSTREAM}" ]; then
-    echo "We don't currently support running metron-bro-plugin-kafka tests in this script"
-  fi
-fi
-
-echo ""
-echo "Review commit carefully then run..."
-echo "    cd $WORK"
-echo "    git push upstream $BASE_BRANCH"
-echo ""

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/dev-utilities/release-utils/README.md
----------------------------------------------------------------------
diff --git a/dev-utilities/release-utils/README.md b/dev-utilities/release-utils/README.md
new file mode 100644
index 0000000..d1fd816
--- /dev/null
+++ b/dev-utilities/release-utils/README.md
@@ -0,0 +1,129 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Release Tools
+
+This project contains tools to assist Apache Metron project committers.
+
+## Prepare Release Candidate
+
+This script automates the process of creating a release candidate from `apache/metron` or `apache/metron-bro-plugin-kafka`. The script will prompt for various information necessary.  Ensure your signing key is setup per [Release Signing](https://www.apache.org/dev/release-signing.html) and [Apache GnuPGP Instructions](https://www.apache.org/dev/openpgp.html#gnupg)
+
+When prompted the `[value in brackets]` is used by default.  To accept the default, simply press `enter`.  If you would like to change the default, type it in and hit `enter` when done.
+
+In the following example, enter the appropriate information
+
+1. Execute the script.  
+
+    The first time the script is run, you will be prompted for additional information including your Apache username and Apache email.  These values are persisted in `~/.metron-prepare-release-candidate`.  Subsequent executions of the script will retrieve these values, rather than prompting you again for them.
+
+    ```
+    $ ./prepare-release-candidate
+      your apache userid []: leet
+      your apache email [leet@apache.org]:
+    ```
+
+1. Select a repository we're creating an RC for.
+
+    ```
+        [1] metron
+        [2] metron-bro-plugin-kafka
+      which repo? [1]: 1
+    ```
+
+1. Enter the current version number.  This will be the base for the CHANGES file
+
+    ```
+      current version: 0.6.0
+    ```
+
+1. Enter the version being built.
+
+    ```
+      version being built: 0.6.1
+    ```
+
+1. Enter the current RC number
+
+    ```
+      release candidate number: 1
+    ```
+
+1. Enter the branch we're releasing from. In most cases, this will be master, but for maintenance releases it can be another branch.
+
+    ```
+      base revision branch or hash for release candidate [master]:
+    ```
+    
+1. Enter the signing key id.
+
+    ```
+      signing key id in 8-byte format (e.g. BADDCAFEDEADBEEF):
+    ```
+    
+1. Enter if this is a practice run. In a practice run, nothing is pushed to SVN, but everything is setup and built otherwise.
+
+    ```
+      do a live run (push to remote repositories?) [y/n]
+    ```
+
+1. Wait for all repos to be checked out to complete.  There will be some additional work done, e.g. along with branch and tag creation. In a live run, you may be prompted for Git credentials to push a branch.
+
+    ```
+      Checking out repo: https://dist.apache.org/repos/dist/dev/metron
+      Checking out repo: dev
+      Checking out repo:  https://dist.apache.org/repos/dist/release/metron
+      Checking out repo: release
+      Checking out git repo: https://git-wip-us.apache.org/repos/asf/metron.git
+      Cloning into '/Users/justinleet/tmp/metron-0.6.1/metron'...
+      remote: Counting objects: 46146, done.
+      remote: Compressing objects: 100% (15568/15568), done.
+      remote: Total 46146 (delta 21513), reused 43696 (delta 19489)
+      Receiving objects: 100% (46146/46146), 56.00 MiB | 1.04 MiB/s, done.
+      Resolving deltas: 100% (21513/21513), done.
+      Creating branch: Metron_0.6.1
+      Using git rev: master
+      Already on 'master'
+      Your branch is up to date with 'origin/master'.
+      Switched to a new branch 'Metron_0.6.1'
+      This is a practice run. Not running <git push --set-upstream origin Metron_0.6.1>
+      Creating tentative git tag <0.6.1-rc1>. Do not push this tag until RC is ready for community review.
+      Already on 'Metron_0.6.1'
+      Creating the RC tarball for tag apache-metron-0.6.1-rc1
+      Creating the SHA hash files
+    ```
+
+1. Provide the passphrase to `gpg` to sign the artifacts.
+
+   ```
+     Signing the release tarball
+     Copying release artifacts
+   ```
+
+1. Shortly afterwards the RC will be finalized. In a practice run, this will not be pushed back to SVN.
+   ```
+     Creating CHANGES file
+     Extracting LICENSE, NOTICE, and KEYS from tarball
+     x LICENSE
+     x NOTICE
+     This is a practice run. Not running the following commands:
+     <svn add 0.6.1-RC1>
+     <svn commit -m "Adding artifacts for metron 0.6.1-RC1">
+   ```
+
+At this point, all RC artifacts have been created.  In a live run, these will have been pushed to the appropriate repositories and are ready for community review.

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/dev-utilities/release-utils/metron-rc-check
----------------------------------------------------------------------
diff --git a/dev-utilities/release-utils/metron-rc-check b/dev-utilities/release-utils/metron-rc-check
index e3cc39d..143ba85 100755
--- a/dev-utilities/release-utils/metron-rc-check
+++ b/dev-utilities/release-utils/metron-rc-check
@@ -181,7 +181,7 @@ fi
 
 if [ -n "$BRO" ]; then
 
-  METRON_KAFKA_BRO_ASSEMBLY="$METRON_RC_DIST/apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz"
+  METRON_KAFKA_BRO_ASSEMBLY="$METRON_RC_DIST/apache-metron-bro-plugin-kafka_$BRO_VERSION-$RC.tar.gz"
   METRON_KAFKA_BRO_ASSEMBLY_ASC="$METRON_KAFKA_BRO_ASSEMBLY.asc"
 
   echo "Downloading $METRON_KAFKA_BRO_ASSEMBLY"
@@ -214,12 +214,12 @@ fi
 if [ -n "$BRO" ]; then
 
   echo "Verifying Bro Kafka Plugin Assembly"
-  if ! gpg --verify ./"apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz.asc" "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then
+  if ! gpg --verify ./"apache-metron-bro-plugin-kafka_$BRO_VERSION-$RC.tar.gz.asc" "apache-metron-bro-plugin-kafka_$BRO_VERSION-$RC.tar.gz" ; then
     echo "[ERROR] failed to verify Bro Kafka Plugin Assembly"
     exit 1
   fi
 
-  if ! tar -xzf "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then
+  if ! tar -xzf "apache-metron-bro-plugin-kafka_$BRO_VERSION-$RC.tar.gz" ; then
     echo "[ERROR] failed to unpack  Bro Kafka Plugin Assembly"
     exit 1
   fi

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/dev-utilities/release-utils/prepare-release-candidate
----------------------------------------------------------------------
diff --git a/dev-utilities/release-utils/prepare-release-candidate b/dev-utilities/release-utils/prepare-release-candidate
new file mode 100755
index 0000000..635285c
--- /dev/null
+++ b/dev-utilities/release-utils/prepare-release-candidate
@@ -0,0 +1,334 @@
+#!/bin/bash
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#
+# Creates and sets up the actual release artifacts into dev
+# See https://cwiki.apache.org/confluence/display/METRON/Release+Process
+
+
+set -eo pipefail
+
+# define constants
+# Git repos
+# To add a new submodule, add the repo name, upstream Git repo, and update the getrepo() selection function.
+# if versioning of the submodule isn't x.y.z format, retrieval of the git tag must also be adjusted.
+METRON_REPO_NAME="metron"
+BRO_PLUGIN_REPO_NAME="metron-bro-plugin-kafka"
+METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${METRON_REPO_NAME}.git"
+BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${BRO_PLUGIN_REPO_NAME}.git"
+
+DEV_REPO="https://dist.apache.org/repos/dist/dev/metron"
+RELEASE_REPO=" https://dist.apache.org/repos/dist/release/metron"
+PLUGIN_GIT_REPO="https://git-wip-us.apache.org/repos/asf/${BRO_PLUGIN_REPO_NAME}.git"
+
+RC_PREFIX=rc
+UPPER_RC_PREFIX="$(tr '[:lower:]' '[:upper:]' <<< ${RC_PREFIX})"
+TAG_POSTFIX="-release"
+
+CONFIG_FILE=~/.metron-prepare-release-candidate
+# does a config file already exist?
+if [ -f $CONFIG_FILE ]; then
+  . $CONFIG_FILE
+  echo "  ...using settings from $CONFIG_FILE"
+fi
+
+# apache id of committer (you)
+if [ -z "${APACHE_NAME}" ]; then
+  read -p "  your apache userid [${APACHE_NAME}]: " INPUT
+  [ -n "$INPUT" ] && APACHE_NAME=$INPUT
+
+  # write setting to config file
+  echo "APACHE_NAME=$APACHE_NAME" >> $CONFIG_FILE
+fi
+
+# apache email addr of committer (you)
+if [ -z "${APACHE_EMAIL}" ]; then
+  APACHE_EMAIL=${APACHE_NAME}@apache.org
+  read -p "  your apache email [${APACHE_EMAIL}]: " INPUT
+  [ -n "$INPUT" ] && APACHE_EMAIL=$INPUT
+
+  # write setting to config file, so it is not needed next time
+  echo "APACHE_EMAIL=$APACHE_EMAIL" >> $CONFIG_FILE
+fi
+
+# which repo?  metron or metron-bro-plugin-kafka
+getrepo() {
+  echo "    [1] ${METRON_REPO_NAME}"
+  echo "    [2] ${BRO_PLUGIN_REPO_NAME}"
+  read -p "  which repo? [1]: " INPUT
+  case "${INPUT}" in
+    [Bb][Rr][Oo]|[Mm][Ee][Tt][Rr][Oo][Nn]-[Bb][Rr][Oo]-[Pp][Ll][Uu][Gg][Ii][Nn]-[Kk][Aa][Ff][Kk][Aa]|*metron-bro-plugin-kafka\.git|2)
+      INPUT="${BRO_PLUGIN_UPSTREAM}" ;;
+    [Mm][Ee][Tt][Rr][Oo][Nn]|*metron\.git|1|'')
+      INPUT="${METRON_UPSTREAM}" ;;
+    *)
+      echo "Invalid repo, provided \"${INPUT}\".  Please choose between ${METRON_REPO_NAME} or ${BRO_PLUGIN_REPO_NAME}"
+      return 1
+      ;;
+  esac
+  [ -n "$INPUT" ] && UPSTREAM=$INPUT
+  return 0
+}
+until getrepo; do :; done
+
+CHOSEN_REPO=$(basename ${UPSTREAM%%.git})
+# Need the capitalized version of the repos some naming
+CAPITAL_REPO="$(tr '[:lower:]' '[:upper:]' <<< ${CHOSEN_REPO:0:1})${CHOSEN_REPO:1}"
+
+getcurrentversion() {
+  # currently released version. Used for CHANGES file
+  read -p "  current version: " CURRENT_VERSION
+  if ! [[ "${CURRENT_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    printf "  Please enter a valid x.y.z version\n"
+    return 1
+  fi
+  return 0
+}
+until getcurrentversion; do : ; done
+
+# version that we're building an RC for
+getversion() {
+  read -p "  version being built: " VERSION
+  if ! [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    printf "  Please enter a valid x.y.z version\n"
+    return 1
+  fi
+  return 0
+  }
+until getversion; do : ; done
+
+# RC number we're building
+getrcnum() {
+read -p "  release candidate number: " RC_NUM
+  if [[ "${RC_NUM}" =~ ^[0-9]+$ ]]; then
+    RC="${RC_PREFIX}${RC_NUM}"
+    return 0
+  else
+    printf "  Please enter an integer\n"
+    return 1
+  fi
+}
+until getrcnum; do : ; done
+
+# define default values
+TMPDIR="$HOME/tmp"
+WORKDIR="$TMPDIR/${CHOSEN_REPO}-${VERSION}"
+
+# warn the user if the working directory exists
+if [ -d "$WORKDIR" ]; then
+  read -p "  directory exists [$WORKDIR].  overwrite existing directory? [y/n] " -r
+  echo
+  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    exit 1
+  fi
+fi
+
+# Clear out the existing work directory
+rm -rf "$WORKDIR"
+mkdir "$WORKDIR"
+
+getbaserev() {
+  read -p "  base revision branch or hash for release candidate [master]: " GIT_REF
+  GIT_REF=${GIT_REF:-master}
+
+  # check to see if we were given branch
+  git rev-parse -q --verify ${GIT_REF} >> /dev/null
+  if [[ $? -ne 0 ]]; then
+    # check to see if we were given hash
+    git cat-file -e ${GIT_REF}
+    if [[ $? -ne 0 ]]; then
+      "Unable to find git revision"
+      return 1
+    fi
+  fi
+  return 0
+}
+until getbaserev; do : ; done
+
+# Signing key
+getkey() {
+  read -s -p "  signing key id in 8-byte format (e.g. BADDCAFEDEADBEEF): " SIGNING_KEY
+  printf "\n"
+  if ! [[ "${SIGNING_KEY}" =~ ^[A-F0-9]{16}$ ]]; then
+    printf "  Please enter a valid signing key\n"
+    return 1
+  fi
+  return 0
+}
+
+until getkey; do : ; done
+
+# Determine if this is a practice run or not.
+getpractice() {
+  read -p "  do a live run (push to remote repositories?) [y/n] " INPUT
+  case "${INPUT}" in
+    y)
+      PRACTICE_RUN=false
+      return 0 ;;
+    n)
+      PRACTICE_RUN=true
+      return 0 ;;
+    *)
+      printf "Please enter 'y' or 'n'\n"
+      return 1
+      ;;
+  esac
+}
+until getpractice; do : ; done
+
+## Prepare everything for building the release artifacts
+
+# Fetch the SVN repos. Always needed regardless of what's being released.
+fetch_svn_repo () { 
+  local SVN_REPO=$1
+  local SVN_DIR=$2
+  mkdir "$SVN_DIR"
+  cd "$SVN_DIR"
+  printf "Checking out repo: %s\n" "$SVN_REPO" "$(basename $SVN_DIR)"
+  svn co -q $SVN_REPO
+  cd "$(dirname $WORKDIR)"
+}
+
+fetch_svn_repo "$DEV_REPO" "$WORKDIR/dev"
+fetch_svn_repo "$RELEASE_REPO" "$WORKDIR/release"
+
+# Fetch the appropriate Git repo. Only need what we're releasing
+GIT_DIR="$WORKDIR/${CHOSEN_REPO}"
+printf "Checking out git repo: %s\n" "$UPSTREAM"
+git clone $UPSTREAM "${GIT_DIR}"
+cd "${GIT_DIR}"
+git fetch --tags
+
+# Create the release branch in the Git repo
+printf "Creating branch: %s_%s\n" "${CAPITAL_REPO}" "$VERSION"
+printf "Using git rev: %s\n" ${GIT_REF}
+cd "$GIT_DIR"
+git checkout ${GIT_REF}
+git checkout -b "${CAPITAL_REPO}_${VERSION}"
+
+if [ "${PRACTICE_RUN}" = true ]; then
+  printf "This is a practice run. Not running <git push --set-upstream origin %s_%s>\n" "${CAPITAL_REPO}" "$VERSION"
+else
+  printf "Pushing branch %s_%s\n" "${CAPITAL_REPO}" "$VERSION"
+  git push --set-upstream origin "${CAPITAL_REPO}_${VERSION}"
+fi
+
+# Create directory for release artifacts
+if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
+  ART_DIR="$WORKDIR/dev/metron/${VERSION}-${UPPER_RC_PREFIX}${RC_NUM}"
+else
+  # We're using a sub module, so put it in it's own directory.
+  ART_DIR="$WORKDIR/dev/metron/${CHOSEN_REPO}/${VERSION}-${UPPER_RC_PREFIX}${RC_NUM}"
+fi
+mkdir -p "$ART_DIR"
+
+# Setup various parameters we need for the release artifacts
+if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
+  CORE_PREFIX="apache-metron_"
+  ARTIFACT_PREFIX="${CORE_PREFIX}"
+  TAG_VERSION="${CURRENT_VERSION}"
+  TAG="${CORE_PREFIX}${TAG_VERSION}${TAG_POSTFIX}"
+elif [ "${CHOSEN_REPO}" = "${BRO_PLUGIN_REPO_NAME}" ]; then
+  BRO_PLUGIN_PREFIX="apache-metron-bro-plugin-kafka_"
+  ARTIFACT_PREFIX="${BRO_PLUGIN_PREFIX}"
+  TAG_VERSION="${CURRENT_VERSION}"
+  TAG="${BRO_PLUGIN_PREFIX}${TAG_VERSION}${TAG_POSTFIX}"
+else
+  # If we ever add new modules, add them as needed.
+  printf "Unrecognized module: %s\n" "${CHOSEN_REPO}"
+  exit 1
+fi
+ARTIFACT="${ARTIFACT_PREFIX}${VERSION}-${RC}"
+
+## Do the work of actually creating the release artifacts
+printf "Creating tentative git tag <%s%s-%s>. Do not push this tag until RC is ready for community review.\n" "${TAG_PREFIX}" "$VERSION" "$RC"
+cd "$GIT_DIR"
+git checkout "${CAPITAL_REPO}_${VERSION}"
+# The branch only exists if this is not a practice run
+if [ "${PRACTICE_RUN}" = false ]; then
+  printf "Pulling latest state of branch\n"
+  git pull
+fi
+git tag "${ARTIFACT}"
+
+# Create the rc tarball from the tag
+printf "Creating the RC tarball for tag %s\n" "$ARTIFACT"
+git archive "--prefix=${ARTIFACT}/" "${ARTIFACT}" | gzip > "${ARTIFACT}.tar.gz"
+
+# Create signing hash files
+printf "Creating the SHA hash files\n"
+gpg --print-md SHA512 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha512
+gpg --print-md SHA256 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha256
+
+# Sign the release tarball
+printf "Signing the release tarball\n"
+gpg -u ${SIGNING_KEY} --armor --output ${ARTIFACT}.tar.gz.asc --detach-sig ${ARTIFACT}.tar.gz
+if [[ $? -ne 0 ]]; then
+  # gpg will print out an error on its own
+  exit 1
+fi
+
+# Setup the release artifacts
+printf "Copying release artifacts\n"
+mv "${GIT_DIR}/${ARTIFACT}.tar.gz" "$ART_DIR"
+mv "${ARTIFACT}.tar.gz.sha512" "$ART_DIR"
+mv "${ARTIFACT}.tar.gz.sha256" "$ART_DIR"
+mv "${ARTIFACT}.tar.gz.asc" "$ART_DIR"
+
+# Create the CHANGES file
+# Do this by getting all commits in current branch that aren't in current release. Filter out any merges by making sure lines start with blankspace followed by "METRON"
+# i.e. make sure the lines starts with a ticket number to avoid merge commits into feature branches
+printf "Creating CHANGES file\n"
+git log "${CAPITAL_REPO}_${VERSION}" "^tags/${TAG}" --no-merges | grep -E "^[[:blank:]]+METRON" | sed 's/\[//g' | sed 's/\]//g' | grep -v "http" > "${ART_DIR}/CHANGES"
+if [[ $? -ne 0 ]]; then
+  "Error creating CHANGES file"
+  exit 1
+fi
+
+printf "Extracting LICENSE, NOTICE, and KEYS from tarball\n" # Only pull from core
+cd ${ART_DIR}
+
+if [ "${CHOSEN_REPO}" = "${BRO_PLUGIN_REPO_NAME}" ]; then
+  # Bro's convention for the LICENSE is different, so the file is called COPYING
+  tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/COPYING"
+else
+  tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/LICENSE"
+fi
+
+# TODO figure out what to do for bro repo here. The KEYS file only needs to live in the /dist root, rather than in each sub repo.
+# Should we have a separate process for adding to the KEYS file without doing a Metron release?
+#tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/KEYS"
+tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/NOTICE"
+
+# Add the directory and commit to subversion
+COMMIT_DIR=$(basename ${ART_DIR})
+if [ "${PRACTICE_RUN}" = true ]; then
+  printf "This is a practice run. Not running the following commands:\n"
+  printf "<svn add %s>\n" ${COMMIT_DIR}
+  printf "<svn commit -m \"Adding artifacts for %s %s\">\n" "${CHOSEN_REPO}" "${COMMIT_DIR}"
+else
+  printf "Adding artifacts for Metron ${VERSION}-${UPPER_RC_PREFIX}${RC_NUM} to dev SVN\n"
+  # Metron goes in the root of the dir, submodules go in folder
+  if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
+    cd "$WORKDIR/dev/metron/"
+  else
+    cd "$WORKDIR/dev/metron/${CHOSEN_REPO}"
+  fi
+  svn add ${COMMIT_DIR}
+  svn commit -m "Adding artifacts for ${CHOSEN_REPO} ${COMMIT_DIR}"
+fi
+

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-maas-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-common/pom.xml b/metron-analytics/metron-maas-common/pom.xml
index b5cf1c9..062e3db 100644
--- a/metron-analytics/metron-maas-common/pom.xml
+++ b/metron-analytics/metron-maas-common/pom.xml
@@ -62,7 +62,7 @@
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>
-      <version>4.3.2</version>
+      <version>${global_httpclient_version}</version>
     </dependency>
     <dependency>
       <groupId>org.codehaus.jackson</groupId>

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/README.md b/metron-analytics/metron-profiler-spark/README.md
index df143f1..5ed5f4b 100644
--- a/metron-analytics/metron-profiler-spark/README.md
+++ b/metron-analytics/metron-profiler-spark/README.md
@@ -162,13 +162,13 @@ ${SPARK_HOME}/bin/spark-submit \
 
 The Batch Profiler accepts the following arguments when run from the command line as shown above.  All arguments following the Profiler jar are passed to the Profiler.  All argument preceeding the Profiler jar are passed to Spark.
 
-| Argument         | Description
-|---               |---
-| -p, --profiles   | Path to the profile definitions.
-| -c, --config     | Path to the profiler properties file.
-| -g, --globals    | Path to the Stellar global config file.
-| -r, --reader     | Path to properties for the DataFrameReader.
-| -h, --help       | Print the help text.
+| Argument                              | Description
+|---                                    |---
+| [`-p`, `--profiles`](#--profiles)     | Path to the profile definitions.
+| [`-c`, `--config`](#--config)         | Path to the profiler properties file.
+| [`-g`, `--globals`](#--globals)       | Path to the Stellar global config file.
+| [`-r`, `--reader`](#--reader)         | Path to properties for the DataFrameReader.
+| `-h`, `--help`                        | Print the help text.
 
 #### `--profiles`
 
@@ -234,6 +234,28 @@ The Profiler can consume archived telemetry stored in a variety of input formats
 
 1. If additional options are required for your input format, then use the [`--reader`](#--reader) command-line argument when launching the Batch Profiler as [described here](#advanced-usage).
 
+#### Common Formats
+
+The following examples highlight the configuration values needed to read telemetry stored in common formats.  These values should be defined in the Profiler properties (see [`--config`](#--config)).
+
+##### JSON
+```
+profiler.batch.input.reader=json
+profiler.batch.input.path=/path/to/json/
+```
+
+##### [Apache ORC](https://orc.apache.org/)
+```
+profiler.batch.input.reader=orc
+profiler.batch.input.path=/path/to/orc/
+```
+
+##### [Apache Parquet](http://parquet.apache.org/)
+```
+profiler.batch.input.reader=parquet
+profiler.batch.input.path=/path/to/parquet/
+```
+
 
 ## Configuring the Profiler
 
@@ -244,6 +266,7 @@ You can store both settings for the Profiler along with settings for Spark in th
 | Setting                                                                       | Description
 |---                                                                            |---
 | [`profiler.batch.input.path`](#profilerbatchinputpath)                        | The path to the input data read by the Batch Profiler.
+| [`profiler.batch.input.reader`](#profilerbatchinputreader)                    | The telemetry reader used to read the input data.
 | [`profiler.batch.input.format`](#profilerbatchinputformat)                    | The format of the input data read by the Batch Profiler.
 | [`profiler.batch.input.begin`](#profilerbatchinputend)                        | Only messages with a timestamp after this will be profiled.
 | [`profiler.batch.input.end`](#profilerbatchinputbegin)                        | Only messages with a timestamp before this will be profiled.
@@ -259,11 +282,25 @@ You can store both settings for the Profiler along with settings for Spark in th
 
 The path to the input data read by the Batch Profiler.
 
+### `profiler.batch.input.reader`
+
+*Default*: json
+
+Defines how the input data is treated when read.  The value is not case sensitive so `JSON` and `json` are equivalent.
+
+ * `json`: Read text/json formatted telemetry
+ * `orc`: Read [Apache ORC](https://orc.apache.org/) formatted telemetry
+ * `parquet`: Read [Apache Parquet](http://parquet.apache.org/) formatted telemetry
+ * `text` Consumes input data stored as raw text. Should be defined along with [`profiler.batch.input.format`](#profilerbatchinputformat). Only use if the input format is not directly supported like `json`.
+ * `columnar` Consumes input data stored in columnar formats. Should be defined along with [`profiler.batch.input.format`](#profilerbatchinputformat).  Only use if the input format is not directly supported like `json`.
+
+See [Common Formats](#common-formats) for further information.
+
 ### `profiler.batch.input.format`
 
 *Default*: text
 
-The format of the input data read by the Batch Profiler.
+The format of the input data read by the Batch Profiler. This is optional and not required in most cases. For example, this property is not required when [`profiler.batch.input.reader`](#profilerbatchinputreader)  is `json`, `orc`, or `parquet`.
 
 ### `profiler.batch.input.begin`
 

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
index 39f8b3a..571545e 100644
--- a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
@@ -26,6 +26,8 @@ import org.apache.metron.profiler.spark.function.GroupByPeriodFunction;
 import org.apache.metron.profiler.spark.function.HBaseWriterFunction;
 import org.apache.metron.profiler.spark.function.MessageRouterFunction;
 import org.apache.metron.profiler.spark.function.ProfileBuilderFunction;
+import org.apache.metron.profiler.spark.reader.TelemetryReader;
+import org.apache.metron.profiler.spark.reader.TelemetryReaders;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.SparkSession;
@@ -40,8 +42,7 @@ import java.util.Properties;
 
 import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_BEGIN;
 import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_END;
-import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
-import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_READER;
 import static org.apache.spark.sql.functions.sum;
 
 /**
@@ -54,6 +55,7 @@ public class BatchProfiler implements Serializable {
   protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   private TimestampParser timestampParser;
+  private TelemetryReader reader;
 
   public BatchProfiler() {
     this.timestampParser = new TimestampParser();
@@ -77,17 +79,10 @@ public class BatchProfiler implements Serializable {
 
     LOG.debug("Building {} profile(s)", profiles.getProfiles().size());
     Map<String, String> globals = Maps.fromProperties(globalProperties);
-    String inputFormat = TELEMETRY_INPUT_FORMAT.get(profilerProps, String.class);
-    String inputPath = TELEMETRY_INPUT_PATH.get(profilerProps, String.class);
-    LOG.debug("Loading telemetry from '{}'", inputPath);
-
-    // fetch the archived telemetry
-    Dataset<String> telemetry = spark
-            .read()
-            .options(Maps.fromProperties(readerProps))
-            .format(inputFormat)
-            .load(inputPath)
-            .as(Encoders.STRING());
+    
+    // fetch the archived telemetry using the input reader
+    TelemetryReader reader = TelemetryReaders.create(TELEMETRY_INPUT_READER.get(profilerProps, String.class));
+    Dataset<String> telemetry = reader.read(spark, profilerProps, readerProps);
     LOG.debug("Found {} telemetry record(s)", telemetry.cache().count());
 
     // find all routes for each message

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
index e8cd160..148d970 100644
--- a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
@@ -25,6 +25,9 @@ import org.apache.metron.stellar.common.utils.ConversionUtils;
 import java.util.Map;
 import java.util.Properties;
 
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.JSON;
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.TEXT;
+
 /**
  * Defines the configuration values recognized by the Batch Profiler.
  */
@@ -44,9 +47,11 @@ public enum BatchProfilerConfig {
 
   HBASE_WRITE_DURABILITY("profiler.hbase.durability", Durability.USE_DEFAULT, Durability.class),
 
-  TELEMETRY_INPUT_FORMAT("profiler.batch.input.format", "text", String.class),
+  TELEMETRY_INPUT_READER("profiler.batch.input.reader", JSON.toString(), String.class),
+
+  TELEMETRY_INPUT_FORMAT("profiler.batch.input.format", "", String.class),
 
-  TELEMETRY_INPUT_PATH("profiler.batch.input.path", "hdfs://localhost:9000/apps/metron/indexing/indexed/*/*", String.class),
+  TELEMETRY_INPUT_PATH("profiler.batch.input.path", "hdfs://localhost:8020/apps/metron/indexing/indexed/*/*", String.class),
 
   TELEMETRY_INPUT_BEGIN("profiler.batch.input.begin", "", String.class),
 

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/ColumnEncodedTelemetryReader.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/ColumnEncodedTelemetryReader.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/ColumnEncodedTelemetryReader.java
new file mode 100644
index 0000000..41bf9b1
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/ColumnEncodedTelemetryReader.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.spark.reader;
+
+import com.google.common.collect.Maps;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Properties;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+
+/**
+ * Reads in a {@link Dataset} then converts all of the {@link Dataset}'s column
+ * into a single JSON-formatted string.
+ *
+ * <p>This {@link TelemetryReader} is useful for any column-oriented format that
+ * is supported by Spark.  For example, ORC and Parquet.
+ */
+public class ColumnEncodedTelemetryReader implements TelemetryReader {
+
+  protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  /**
+   * The input format to use when reading telemetry.
+   */
+  private String inputFormat;
+
+  /**
+   * Creates a {@link ColumnEncodedTelemetryReader}.
+   *
+   * <p>The input format used to read the telemetry is defined by the
+   * BatchProfilerConfig.TELEMETRY_INPUT_PATH property.
+   */
+  public ColumnEncodedTelemetryReader() {
+    this.inputFormat = null;
+  }
+
+  /**
+   * Creates a {@link ColumnEncodedTelemetryReader}.
+   *
+   * @param inputFormat The input format to use when reading telemetry.
+   */
+  public ColumnEncodedTelemetryReader(String inputFormat) {
+    this.inputFormat = inputFormat;
+  }
+
+  @Override
+  public Dataset<String> read(SparkSession spark, Properties profilerProps, Properties readerProps) {
+    String inputPath = TELEMETRY_INPUT_PATH.get(profilerProps, String.class);
+    if(inputFormat == null) {
+      inputFormat = TELEMETRY_INPUT_FORMAT.get(profilerProps, String.class);
+    }
+    LOG.debug("Loading telemetry; inputPath={}, inputFormat={}", inputPath, inputFormat);
+
+    return spark
+            .read()
+            .options(Maps.fromProperties(readerProps))
+            .format(inputFormat)
+            .load(inputPath)
+            .toJSON();
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReader.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReader.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReader.java
new file mode 100644
index 0000000..18861c0
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReader.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.spark.reader;
+
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.SparkSession;
+
+import java.io.Serializable;
+import java.util.Properties;
+
+/**
+ * A {@link TelemetryReader} is responsible for creating a {@link Dataset} containing
+ * telemetry that can be consumed by the {@link org.apache.metron.profiler.spark.BatchProfiler}.
+ */
+public interface TelemetryReader extends Serializable {
+
+  /**
+   * Read in the telemetry
+   *
+   * @param spark The spark session.
+   * @param profilerProps The profiler properties.
+   * @param readerProps The properties specific to reading input data.
+   * @return A {@link Dataset} containing archived telemetry.
+   */
+  Dataset<String> read(SparkSession spark, Properties profilerProps, Properties readerProps);
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReaders.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReaders.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReaders.java
new file mode 100644
index 0000000..0646155
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReaders.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.spark.reader;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Properties;
+import java.util.function.Supplier;
+
+/**
+ * Allows a user to easily define the value of the property
+ * {@link org.apache.metron.profiler.spark.BatchProfilerConfig#TELEMETRY_INPUT_READER}.
+ */
+public enum TelemetryReaders implements TelemetryReader {
+
+  /**
+   * A {@link TelemetryReader} that is able to consume text/json formatted data.
+   *
+   * <p>This serves as a configuration short-cut for users. The user only needs to define the
+   * {@link org.apache.metron.profiler.spark.BatchProfilerConfig#TELEMETRY_INPUT_READER} property
+   * with this value to consume text/json.
+   */
+  JSON(() -> new TextEncodedTelemetryReader("text")),
+
+  /**
+   * A {@link TelemetryReader} that is able to consume Apache ORC formatted data.
+   *
+   * <p>This serves as a configuration short-cut for users. The user only needs to define the
+   * {@link org.apache.metron.profiler.spark.BatchProfilerConfig#TELEMETRY_INPUT_READER} property
+   * with this value to consume Apache ORC formatted data.
+   */
+  ORC(() -> new ColumnEncodedTelemetryReader("org.apache.spark.sql.execution.datasources.orc")),
+
+  /**
+   * A {@link TelemetryReader} that is able to consume Apache Parquet formatted data.
+   *
+   * <p>This serves as a configuration short-cut for users. The user only needs to define the
+   * {@link org.apache.metron.profiler.spark.BatchProfilerConfig#TELEMETRY_INPUT_READER} property
+   * with this value to consume Apache Parquet formatted data.
+   */
+  PARQUET(() -> new ColumnEncodedTelemetryReader("parquet")),
+
+  /**
+   * Use a {@link TextEncodedTelemetryReader} by defining the property value as 'TEXT'.
+   */
+  TEXT(() -> new TextEncodedTelemetryReader()),
+
+  /**
+   * Use a {@link ColumnEncodedTelemetryReader} by defining the property value as 'COLUMNAR'.
+   */
+  COLUMNAR(() -> new ColumnEncodedTelemetryReader());
+
+  static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  private Supplier<TelemetryReader> supplier;
+
+  private TelemetryReaders(Supplier<TelemetryReader> supplier) {
+    this.supplier = supplier;
+  }
+
+  /**
+   * Returns a {@link TelemetryReader} based on a property value.
+   *
+   * @param propertyValue The property value.
+   * @return A {@link TelemetryReader}
+   * @throws IllegalArgumentException If the property value is invalid.
+   */
+  public static TelemetryReader create(String propertyValue) {
+    LOG.debug("Creating telemetry reader: telemetryReader={}", propertyValue);
+    TelemetryReader reader = null;
+    try {
+      String key = StringUtils.upperCase(propertyValue);
+      TelemetryReaders strategy = TelemetryReaders.valueOf(key);
+      reader = strategy.supplier.get();
+
+    } catch(IllegalArgumentException e) {
+      LOG.error("Unexpected telemetry reader: telemetryReader=" + propertyValue, e);
+      throw e;
+    }
+
+    return reader;
+  }
+
+  @Override
+  public Dataset<String> read(SparkSession spark, Properties profilerProps, Properties readerProps) {
+    return supplier.get().read(spark, profilerProps, readerProps);
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TextEncodedTelemetryReader.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TextEncodedTelemetryReader.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TextEncodedTelemetryReader.java
new file mode 100644
index 0000000..1f10b97
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TextEncodedTelemetryReader.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.spark.reader;
+
+import com.google.common.collect.Maps;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Properties;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+
+/**
+ * A {@link TelemetryReader} that consumes telemetry stored as raw text.
+ *
+ * <p>This {@link TelemetryReader} is useful for any text-encoded formats like JSON and CSV.
+ */
+public class TextEncodedTelemetryReader implements TelemetryReader {
+
+  protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  /**
+   * The input format to use when reading telemetry.
+   */
+  private String inputFormat;
+
+  /**
+   * Creates a {@link TextEncodedTelemetryReader}.
+   *
+   * <p>The input format used to read the telemetry is defined by the
+   * BatchProfilerConfig.TELEMETRY_INPUT_PATH property.
+   */
+  public TextEncodedTelemetryReader() {
+    this.inputFormat = null;
+  }
+
+  /**
+   * Creates a {@link TextEncodedTelemetryReader}.
+   *
+   * @param inputFormat The input format to use when reading telemetry.
+   */
+  public TextEncodedTelemetryReader(String inputFormat) {
+    this.inputFormat = inputFormat;
+  }
+
+  @Override
+  public Dataset<String> read(SparkSession spark, Properties profilerProps, Properties readerProps) {
+    String inputPath = TELEMETRY_INPUT_PATH.get(profilerProps, String.class);
+    if(inputFormat == null) {
+      inputFormat = TELEMETRY_INPUT_FORMAT.get(profilerProps, String.class);
+    }
+    LOG.debug("Loading telemetry; inputPath={}, inputFormat={}", inputPath, inputFormat);
+
+    return spark
+            .read()
+            .options(Maps.fromProperties(readerProps))
+            .format(inputFormat)
+            .load(inputPath)
+            .as(Encoders.STRING());
+  }
+}


Mime
View raw message