metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmiklav...@apache.org
Subject [20/50] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Date Wed, 11 Jul 2018 01:32:36 GMT
Merge branch 'master' into feature/METRON-1416-upgrade-solr


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/41708876
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/41708876
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/41708876

Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 41708876336d009b3a2ff2c533bdf0294af1477b
Parents: f241f87 32d132b
Author: cstella <cestella@gmail.com>
Authored: Mon Jun 11 17:50:48 2018 -0400
Committer: cstella <cestella@gmail.com>
Committed: Mon Jun 11 17:50:48 2018 -0400

----------------------------------------------------------------------
 KEYS                                            |    59 +
 Upgrading.md                                    |     2 +-
 dev-utilities/release-utils/metron-rc-check     |    77 +-
 .../release-utils/validate-jira-for-release     |    75 +-
 metron-analytics/metron-maas-common/pom.xml     |     2 +-
 metron-analytics/metron-maas-service/pom.xml    |     2 +-
 metron-analytics/metron-profiler-client/pom.xml |     2 +-
 .../profiler/client/stellar/GetProfileTest.java |     3 +-
 .../client/stellar/ProfilerFunctionsTest.java   |    15 +-
 metron-analytics/metron-profiler-common/pom.xml |     2 +-
 metron-analytics/metron-profiler/README.md      |    15 +
 metron-analytics/metron-profiler/pom.xml        |     2 +-
 .../src/main/flux/profiler/remote.yaml          |     3 +-
 metron-analytics/metron-statistics/pom.xml      |     2 +-
 .../StellarStatisticsFunctionsTest.java         |     5 +-
 .../sampling/SamplerFunctionsTest.java          |     3 +-
 metron-analytics/pom.xml                        |     2 +-
 metron-contrib/metron-docker/pom.xml            |     2 +-
 metron-contrib/metron-performance/pom.xml       |     2 +-
 metron-contrib/pom.xml                          |     2 +-
 metron-deployment/Kerberos-manual-setup.md      |     8 +-
 metron-deployment/README.md                     |    22 +-
 metron-deployment/amazon-ec2/conf/defaults.yml  |     2 +-
 .../ansible/playbooks/docker_probe_install.yml  |     2 +-
 .../roles/ambari_master/defaults/main.yml       |     2 +-
 .../roles/metron_pcapservice/defaults/main.yml  |     2 +-
 metron-deployment/development/centos6/README.md |     8 +-
 .../centos6/ansible/inventory/group_vars/all    |     2 +-
 .../development/ubuntu14/README.md              |     6 +-
 .../ubuntu14/ansible/inventory/group_vars/all   |     2 +-
 .../ambari/elasticsearch-mpack/pom.xml          |     4 +-
 .../packaging/ambari/metron-mpack/pom.xml       |     4 +-
 .../CURRENT/package/files/bro_index.template    |     2 +-
 .../CURRENT/package/files/error_index.template  |     2 +-
 .../package/files/metaalert_index.template      |     2 +-
 .../CURRENT/package/files/snort_index.template  |     2 +-
 .../CURRENT/package/files/yaf_index.template    |     2 +-
 .../CURRENT/package/scripts/metron_service.py   |     5 +
 .../metron-mpack/src/main/resources/mpack.json  |     2 +-
 .../packaging/docker/deb-docker/pom.xml         |     4 +-
 .../packaging/docker/rpm-docker/pom.xml         |     4 +-
 metron-deployment/pom.xml                       |     2 +-
 .../e2e/mock-data/alerts_ui_e2e_index.template  |     2 +-
 metron-interface/metron-alerts/package.json     |     2 +-
 metron-interface/metron-alerts/pom.xml          |     2 +-
 .../alert-details/alert-details.component.ts    |     2 +-
 .../alerts/alerts-list/alerts-list.component.ts |     2 +-
 .../table-view/table-view.component.html        |    10 +-
 .../table-view/table-view.component.ts          |     2 +-
 .../metron-alerts/src/app/model/alert-source.ts |     2 +-
 .../src/app/service/global-config.service.ts    |    10 +-
 metron-interface/metron-config/package.json     |     2 +-
 metron-interface/metron-config/pom.xml          |     2 +-
 .../metron-config/scripts/package.json          |     2 +-
 metron-interface/metron-rest-client/pom.xml     |     2 +-
 metron-interface/metron-rest/pom.xml            |     2 +-
 .../rest/service/impl/SearchServiceImpl.java    |    10 +-
 .../src/main/resources/application.yml          |     2 +-
 .../metron-rest/src/main/scripts/metron-rest.sh |     5 +-
 .../MetaAlertControllerIntegrationTest.java     |     4 +-
 .../service/impl/SearchServiceImplTest.java     |     3 +-
 metron-interface/pom.xml                        |     2 +-
 metron-platform/Performance-tuning-guide.md     |     4 +-
 metron-platform/README.md                       |     2 +-
 metron-platform/elasticsearch-shaded/pom.xml    |     2 +-
 metron-platform/metron-api/pom.xml              |     2 +-
 metron-platform/metron-common/README.md         |     5 +
 metron-platform/metron-common/pom.xml           |     2 +-
 .../org/apache/metron/common/Constants.java     |     1 +
 .../metron/common/bolt/ConfiguredBolt.java      |    14 +-
 .../common/bolt/ConfiguredEnrichmentBolt.java   |    11 +-
 .../common/bolt/ConfiguredIndexingBolt.java     |    13 +-
 .../common/bolt/ConfiguredParserBolt.java       |    13 +-
 .../common/bolt/ConfiguredProfilerBolt.java     |     9 +-
 .../common/configuration/Configurations.java    |     7 +-
 .../configuration/ConfigurationsUtils.java      |     7 +
 .../configuration/EnrichmentConfigurations.java |    26 +
 .../configuration/IndexingConfigurations.java   |    28 +-
 .../configuration/ParserConfigurations.java     |     1 +
 .../enrichment/handler/StellarConfig.java       |    11 +-
 .../profiler/ProfilerConfigurations.java        |    25 +
 .../writer/ConfigurationStrategy.java           |    44 +
 .../writer/ConfigurationsStrategies.java        |   144 +
 .../writer/EnrichmentWriterConfiguration.java   |   110 +
 .../writer/IndexingWriterConfiguration.java     |     5 +
 .../writer/ParserWriterConfiguration.java       |    12 +-
 .../writer/ProfilerWriterConfiguration.java     |   109 +
 .../writer/SingleBatchConfigurationFacade.java  |     6 +
 .../writer/WriterConfiguration.java             |    69 +
 .../common/field/DeDotFieldNameConverter.java   |    46 +
 .../metron/common/field/FieldNameConverter.java |    32 +
 .../common/field/FieldNameConverters.java       |   116 +
 .../common/field/NoopFieldNameConverter.java    |    32 +
 .../common/interfaces/FieldNameConverter.java   |    24 -
 .../src/main/scripts/cluster_info.py            |     2 +-
 .../configuration/ParserConfigurationsTest.java |   120 +
 .../profiler/ProfilerConfigTest.java            |    13 +-
 .../writer/ConfigurationsStrategiesTest.java    |    79 +
 .../EnrichmentWriterConfigurationTest.java      |    54 +
 .../writer/IndexingWriterConfigurationTest.java |    70 +
 .../writer/ParserWriterConfigurationTest.java   |    72 +
 .../writer/ProfilerWriterConfigurationTest.java |    54 +
 .../field/DeDotFieldNameConverterTest.java      |    38 +
 .../common/field/FieldNameConvertersTest.java   |   211 +
 .../StellarTransformationTest.java              |    30 +-
 .../writer/IndexingWriterConfigurationTest.java |    70 -
 .../writer/ParserWriterConfigurationTest.java   |    38 -
 metron-platform/metron-data-management/pom.xml  |     2 +-
 metron-platform/metron-elasticsearch/README.md  |     8 +-
 metron-platform/metron-elasticsearch/pom.xml    |     8 +-
 .../dao/ElasticsearchMetaAlertDao.java          |    33 +-
 .../writer/ElasticsearchFieldNameConverter.java |    32 -
 .../writer/ElasticsearchWriter.java             |    70 +-
 .../ElasticsearchIndexingIntegrationTest.java   |     7 +-
 .../ElasticsearchMetaAlertIntegrationTest.java  |    16 +-
 .../ElasticsearchFieldNameConverterTest.java    |    32 -
 .../metron-enrichment/Performance.md            |     2 +-
 metron-platform/metron-enrichment/README.md     |    15 +-
 metron-platform/metron-enrichment/pom.xml       |     2 +-
 .../main/flux/enrichment/remote-splitjoin.yaml  |     9 +-
 .../main/flux/enrichment/remote-unified.yaml    |     9 +-
 .../adapters/stellar/StellarAdapter.java        |    14 +-
 .../adapters/stellar/StellarAdapterTest.java    |    26 +
 .../bolt/BulkMessageWriterBoltTest.java         |   112 +-
 .../integration/EnrichmentIntegrationTest.java  |     2 +
 metron-platform/metron-hbase-client/pom.xml     |     2 +-
 metron-platform/metron-hbase/pom.xml            |     2 +-
 metron-platform/metron-indexing/README.md       |    20 +-
 metron-platform/metron-indexing/pom.xml         |     2 +-
 .../src/main/flux/indexing/batch/remote.yaml    |     4 +-
 .../flux/indexing/random_access/remote.yaml     |     4 +-
 .../apache/metron/indexing/dao/HBaseDao.java    |     2 +-
 .../dao/metaalert/MetaAlertConstants.java       |     2 +-
 .../indexing/dao/InMemoryMetaAlertDao.java      |     2 +-
 .../integration/HBaseDaoIntegrationTest.java    |    17 +
 .../HDFSIndexingIntegrationTest.java            |     4 +-
 .../integration/IndexingIntegrationTest.java    |     3 +-
 metron-platform/metron-integration-test/pom.xml |     2 +-
 .../main/config/zookeeper/enrichments/test.json |     1 +
 metron-platform/metron-management/pom.xml       |     2 +-
 .../metron/management/KafkaFunctions.java       |   573 +-
 .../management/IndexingConfigFunctionsTest.java |     7 +-
 .../KafkaFunctionsIntegrationTest.java          |   353 +-
 .../management/ThreatTriageFunctionsTest.java   |     7 +-
 .../metron-parsers/3rdPartyParser.md            |   439 +-
 metron-platform/metron-parsers/README.md        |     6 +-
 metron-platform/metron-parsers/pom.xml          |     2 +-
 .../apache/metron/parsers/bolt/ParserBolt.java  |   108 +-
 .../metron/parsers/bolt/WriterHandler.java      |    63 +-
 .../src/main/scripts/start_parser_topology.sh   |    27 +-
 .../metron/parsers/bolt/ParserBoltTest.java     |   176 +-
 .../SimpleHBaseEnrichmentWriterTest.java        |     6 +-
 .../integration/WriterBoltIntegrationTest.java  |     6 +-
 metron-platform/metron-pcap-backend/pom.xml     |     2 +-
 .../src/main/scripts/pcap_zeppelin_run.sh       |     2 +-
 metron-platform/metron-pcap/pom.xml             |     2 +-
 .../metron/pcap/filter/PcapFieldResolver.java   |     5 +
 metron-platform/metron-solr/pom.xml             |     2 +-
 .../SolrIndexingIntegrationTest.java            |     3 +-
 .../schema/SchemaValidationIntegrationTest.java |     5 +
 .../metron-storm-kafka-override/pom.xml         |     2 +-
 metron-platform/metron-storm-kafka/pom.xml      |     2 +-
 metron-platform/metron-test-utilities/pom.xml   |     2 +-
 metron-platform/metron-writer/pom.xml           |     2 +-
 .../metron/writer/BulkWriterComponent.java      |    36 +-
 .../metron/writer/bolt/BatchTimeoutHelper.java  |     6 +-
 .../writer/bolt/BulkMessageWriterBolt.java      |   138 +-
 .../apache/metron/writer/kafka/KafkaWriter.java |    92 +-
 metron-platform/metron-zookeeper/pom.xml        |     2 +-
 metron-platform/pom.xml                         |     2 +-
 metron-stellar/pom.xml                          |     2 +-
 .../stellar-3rd-party-example/pom.xml           |     2 +-
 metron-stellar/stellar-common/README.md         |     5 +-
 metron-stellar/stellar-common/pom.xml           |     2 +-
 .../stellar/common/BaseStellarProcessor.java    |    28 +-
 .../stellar/common/CachingStellarProcessor.java |   141 +-
 .../common/StellarPredicateProcessor.java       |     6 +
 .../metron/stellar/common/utils/ConcatMap.java  |   256 +
 .../common/utils/StellarProcessorUtils.java     |    51 +-
 .../metron/stellar/dsl/MapVariableResolver.java |     8 +
 .../metron/stellar/dsl/VariableResolver.java    |     1 +
 .../dsl/functions/DataStructureFunctions.java   |     4 +
 .../common/CachingStellarProcessorTest.java     |   195 +-
 .../shell/specials/AssignmentCommandTest.java   |     7 +-
 .../stellar/common/utils/ConcatMapTest.java     |    83 +
 .../stellar/dsl/functions/BasicStellarTest.java |    18 +
 .../dsl/functions/OrdinalFunctionsTest.java     |     9 +-
 metron-stellar/stellar-zeppelin/README.md       |     6 +-
 metron-stellar/stellar-zeppelin/pom.xml         |     2 +-
 pom.xml                                         |     5 +-
 site-book/pom.xml                               |     2 +-
 site/current-book/CONTRIBUTING.html             |   157 +
 site/current-book/Upgrading.html                |   354 +-
 .../css/apache-maven-fluido-1.3.0.min.css       |     9 -
 .../css/apache-maven-fluido-1.7.min.css         |    17 +
 site/current-book/css/maven-base.css            |   168 +
 site/current-book/css/maven-theme.css           |   161 +
 site/current-book/css/print.css                 |    11 +-
 .../fonts/glyphicons-halflings-regular.eot      |   Bin 0 -> 35283 bytes
 .../fonts/glyphicons-halflings-regular.svg      |   229 +
 .../fonts/glyphicons-halflings-regular.ttf      |   Bin 0 -> 55016 bytes
 .../fonts/glyphicons-halflings-regular.woff     |   Bin 0 -> 41793 bytes
 .../images/apache-maven-project-2.png           |   Bin 33442 -> 43073 bytes
 site/current-book/images/collapsed.gif          |   Bin 0 -> 53 bytes
 site/current-book/images/expanded.gif           |   Bin 0 -> 52 bytes
 site/current-book/images/external.png           |   Bin 0 -> 230 bytes
 site/current-book/images/icon_error_sml.gif     |   Bin 633 -> 1010 bytes
 site/current-book/images/icon_info_sml.gif      |   Bin 638 -> 606 bytes
 site/current-book/images/icon_success_sml.gif   |   Bin 604 -> 990 bytes
 site/current-book/images/icon_warning_sml.gif   |   Bin 625 -> 576 bytes
 site/current-book/images/newwindow.png          |   Bin 0 -> 220 bytes
 site/current-book/index.html                    |   347 +-
 .../js/apache-maven-fluido-1.3.0.min.js         |    21 -
 .../js/apache-maven-fluido-1.7.min.js           |    25 +
 site/current-book/metron-analytics/index.html   |   303 +-
 .../metron-maas-service/index.html              |   451 +-
 .../metron-profiler-client/index.html           |   888 +-
 .../metron-analytics/metron-profiler/index.html |  1338 +-
 .../metron-statistics/HLLP.html                 | 10962 ++++++-----------
 .../metron-statistics/index.html                |   870 +-
 .../metron-contrib/metron-docker/index.html     |   540 +-
 .../metron-performance/index.html               |   317 +
 .../Kerberos-ambari-setup.html                  |   372 +-
 .../Kerberos-manual-setup.html                  |  1218 +-
 .../metron-deployment/amazon-ec2/index.html     |   755 +-
 .../metron-deployment/ansible/index.html        |   128 +
 .../metron-deployment/ansible/roles/index.html  |   140 +
 .../ansible/roles/opentaxii/index.html          |   301 +
 .../ansible/roles/pcap_replay/index.html        |   172 +
 .../ansible/roles/sensor-stubs/index.html       |   221 +
 .../ansible/roles/sensor-test-mode/index.html   |   156 +
 .../development/centos6/index.html              |   233 +
 .../development/fastcapa/index.html             |   249 +
 .../metron-deployment/development/index.html    |   137 +
 .../development/ubuntu14/index.html             |   233 +
 site/current-book/metron-deployment/index.html  |   691 +-
 .../metron-deployment/other-examples/index.html |   363 +-
 .../manual-install/Manual_Install_CentOS6.html  |  1352 +-
 .../ambari/elasticsearch-mpack/index.html       |   190 +
 .../packaging/ambari/index.html                 |  1150 +-
 .../packaging/ambari/metron-mpack/index.html    |   337 +
 .../packaging/docker/ansible-docker/index.html  |   432 +-
 .../packaging/docker/deb-docker/index.html      |   194 +
 .../packaging/docker/rpm-docker/index.html      |   430 +-
 .../packaging/packer-build/index.html           |   382 +-
 .../metron-deployment/roles/index.html          |   332 -
 .../metron-deployment/roles/monit/index.html    |   388 -
 .../roles/opentaxii/index.html                  |   480 -
 .../roles/pcap_replay/index.html                |   355 -
 .../roles/sensor-stubs/index.html               |   401 -
 .../roles/sensor-test-mode/index.html           |   345 -
 .../vagrant/codelab-platform/index.html         |   383 -
 .../vagrant/fastcapa-test-platform/index.html   |   416 -
 .../vagrant/full-dev-platform/index.html        |   420 -
 .../metron-deployment/vagrant/index.html        |   305 -
 .../vagrant/quick-dev-platform/index.html       |   406 -
 .../metron-interface/metron-alerts/index.html   |   434 +-
 .../metron-interface/metron-config/index.html   |   415 +-
 .../metron-interface/metron-rest/index.html     |  2661 ++--
 .../Performance-tuning-guide.html               |  1123 +-
 site/current-book/metron-platform/index.html    |   346 +-
 .../metron-platform/metron-api/index.html       |   387 +-
 .../metron-platform/metron-common/index.html    |   877 +-
 .../metron-data-management/index.html           |  1348 +-
 .../metron-elasticsearch/index.html             |   706 +-
 .../metron-enrichment/Performance.html          |   802 ++
 .../metron-enrichment/index.html                |   760 +-
 .../metron-platform/metron-indexing/index.html  |   547 +-
 .../metron-management/index.html                |  1178 +-
 .../metron-parsers/3rdPartyParser.html          |   467 +
 .../metron-platform/metron-parsers/index.html   |   870 +-
 .../metron-parsers/parser-testing.html          |   434 +-
 .../metron-pcap-backend/index.html              |   503 +-
 .../metron-platform/metron-writer/index.html    |   377 +-
 .../metron-sensors/bro-plugin-kafka/index.html  |   519 -
 .../metron-sensors/fastcapa/index.html          |  1239 +-
 site/current-book/metron-sensors/index.html     |   299 +-
 .../metron-sensors/pycapa/index.html            |   605 +-
 .../stellar-3rd-party-example/index.html        |   263 +-
 .../stellar-common/3rdPartyStellar.html         |   330 +-
 .../metron-stellar/stellar-common/index.html    |  3603 ++----
 .../metron-stellar/stellar-zeppelin/index.html  |   239 +
 .../use-cases/forensic_clustering/index.html    |   517 +-
 .../geographic_login_outliers/index.html        |   442 +-
 site/current-book/use-cases/index.html          |   283 +-
 .../use-cases/typosquat_detection/index.html    |   545 +
 site/documentation/index.md                     |    14 +-
 287 files changed, 26742 insertions(+), 33226 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-deployment/packaging/ambari/metron-mpack/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
----------------------------------------------------------------------
diff --cc metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
index 82b9c11,1c92fcb..54759e4
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
@@@ -18,7 -18,8 +18,8 @@@
  package org.apache.metron.rest.service.impl;
  
  import static org.apache.metron.common.Constants.ERROR_TYPE;
 +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_TYPE;
+ import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY;
 -import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE;
  import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME;
  import static org.apache.metron.rest.MetronRestConstants.SEARCH_FACET_FIELDS_SPRING_PROPERTY;
  

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
----------------------------------------------------------------------
diff --cc metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index ab6c40c,ee3ca89..f73a640
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@@ -18,21 -18,17 +18,23 @@@
  
  package org.apache.metron.elasticsearch.dao;
  
 +import java.io.IOException;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Optional;
+ import org.apache.lucene.search.join.ScoreMode;
  import org.apache.metron.common.Constants;
+ import org.apache.metron.common.configuration.ConfigurationsUtils;
  import org.apache.metron.indexing.dao.AccessConfig;
  import org.apache.metron.indexing.dao.IndexDao;
 -import org.apache.metron.indexing.dao.MetaAlertDao;
  import org.apache.metron.indexing.dao.MultiIndexDao;
 +import org.apache.metron.indexing.dao.RetrieveLatestDao;
 +import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
 +import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
  import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
  import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
 +import org.apache.metron.indexing.dao.metaalert.MetaAlertDao;
  import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
 -import org.apache.metron.indexing.dao.metaalert.MetaScores;
  import org.apache.metron.indexing.dao.search.FieldType;
  import org.apache.metron.indexing.dao.search.GetRequest;
  import org.apache.metron.indexing.dao.search.GroupRequest;
@@@ -247,4 -714,12 +277,5 @@@ public class ElasticsearchMetaAlertDao 
    public void setPageSize(int pageSize) {
      this.pageSize = pageSize;
    }
+ 
 -  private String getFieldName(String globalConfigKey, String defaultFieldName) {
 -    if (this.elasticsearchDao == null || this.elasticsearchDao.getAccessConfig() == null)
{
 -      return defaultFieldName;
 -    }
 -    Map<String, Object> globalConfig = this.elasticsearchDao.getAccessConfig().getGlobalConfigSupplier().get();
 -    return ConfigurationsUtils.getFieldName(globalConfig, globalConfigKey, defaultFieldName);
 -  }
  }

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --cc metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
index 6fa6956,adc1a27..99f0490
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
@@@ -18,14 -18,18 +18,13 @@@
  
  package org.apache.metron.elasticsearch.integration;
  
 -import static org.apache.metron.common.Constants.SENSOR_TYPE;
 -import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY;
 -import static org.apache.metron.indexing.dao.MetaAlertDao.ALERT_FIELD;
 -import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERTS_INDEX;
 -import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_FIELD;
 -import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE;
 -import static org.apache.metron.indexing.dao.MetaAlertDao.STATUS_FIELD;
 -import static org.apache.metron.indexing.dao.MetaAlertDao.THREAT_FIELD_PROPERTY;
 +import static org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao.METAALERTS_INDEX;
- import static org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD;
 +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.ALERT_FIELD;
 +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_DOC;
 +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_FIELD;
 +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_TYPE;
  
  import com.fasterxml.jackson.core.JsonProcessingException;
 -import com.google.common.base.Joiner;
 -import com.google.common.collect.Iterables;
  import java.io.File;
  import java.io.IOException;
  import java.text.SimpleDateFormat;
@@@ -46,12 -52,23 +45,13 @@@ import org.apache.metron.elasticsearch.
  import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
  import org.apache.metron.indexing.dao.AccessConfig;
  import org.apache.metron.indexing.dao.IndexDao;
 -import org.apache.metron.indexing.dao.MetaAlertDao;
 -import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
 -import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
++import org.apache.metron.indexing.dao.metaalert.MetaAlertDao;
 +import org.apache.metron.indexing.dao.metaalert.MetaAlertIntegrationTest;
  import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
  import org.apache.metron.indexing.dao.search.GetRequest;
 -import org.apache.metron.indexing.dao.search.Group;
 -import org.apache.metron.indexing.dao.search.GroupRequest;
 -import org.apache.metron.indexing.dao.search.GroupResponse;
 -import org.apache.metron.indexing.dao.search.GroupResult;
 -import org.apache.metron.indexing.dao.search.InvalidSearchException;
  import org.apache.metron.indexing.dao.search.SearchRequest;
  import org.apache.metron.indexing.dao.search.SearchResponse;
 -import org.apache.metron.indexing.dao.search.SearchResult;
  import org.apache.metron.indexing.dao.search.SortField;
 -import org.apache.metron.indexing.dao.update.Document;
 -import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
 -import org.apache.metron.indexing.dao.update.PatchRequest;
  import org.junit.After;
  import org.junit.AfterClass;
  import org.junit.Assert;
@@@ -119,6 -196,12 +119,12 @@@ public class ElasticsearchMetaAlertInte
          .withIndexDir(new File(INDEX_DIR))
          .build();
      es.start();
+   }
+ 
+   @Before
+   public void setup() throws IOException {
 -    es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, template.replace("%MAPPING_NAME%",
"metaalert"));
++    es.createIndexWithMapping(METAALERTS_INDEX, METAALERT_DOC, template.replace("%MAPPING_NAME%",
"metaalert"));
+     es.createIndexWithMapping(INDEX, "index_doc", template.replace("%MAPPING_NAME%", "index"));
  
      AccessConfig accessConfig = new AccessConfig();
      Map<String, Object> globalConfig = new HashMap<String, Object>() {
@@@ -135,19 -218,9 +141,11 @@@
  
      esDao = new ElasticsearchDao();
      esDao.init(accessConfig);
 -    metaDao = new ElasticsearchMetaAlertDao(esDao);
 +    ElasticsearchMetaAlertDao elasticsearchMetaDao = new ElasticsearchMetaAlertDao(esDao);
 +    elasticsearchMetaDao.setPageSize(5);
 +    metaDao = elasticsearchMetaDao;
    }
  
-   @Before
-   public void setup() throws IOException {
-     es.createIndexWithMapping(METAALERTS_INDEX, METAALERT_DOC,
-         template.replace("%MAPPING_NAME%", "metaalert"));
-     es.createIndexWithMapping(
-         INDEX_WITH_SEPARATOR, "index_doc", template.replace("%MAPPING_NAME%", "index"));
-   }
- 
    @AfterClass
    public static void teardown() {
      if (es != null) {

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/README.md
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java
----------------------------------------------------------------------
diff --cc metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java
index a055db5,0000000..daa5424
mode 100644,000000..100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java
@@@ -1,30 -1,0 +1,30 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +
 +package org.apache.metron.indexing.dao.metaalert;
 +
 +public class MetaAlertConstants {
 +  public static String METAALERT_TYPE = "metaalert";
 +  public static String METAALERT_FIELD = "metaalerts";
 +  public static String METAALERT_DOC = METAALERT_TYPE + "_doc";
 +  public static String THREAT_FIELD_DEFAULT = "threat:triage:score";
 +  public static String THREAT_SORT_DEFAULT = "sum";
-   public static String ALERT_FIELD = "alert";
++  public static String ALERT_FIELD = "metron_alert";
 +  public static String STATUS_FIELD = "status";
 +  public static String GROUPS_FIELD = "groups";
 +}

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-solr/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
----------------------------------------------------------------------
diff --cc metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
index 918a4ae,256f23b..2f9b285
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
@@@ -17,16 -17,10 +17,16 @@@
   */
  package org.apache.metron.solr.integration;
  
 +import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
 +
  import com.google.common.base.Function;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Properties;
 +import javax.annotation.Nullable;
  import org.apache.metron.common.configuration.Configurations;
  import org.apache.metron.common.configuration.ConfigurationsUtils;
- import org.apache.metron.common.interfaces.FieldNameConverter;
+ import org.apache.metron.common.field.FieldNameConverter;
  import org.apache.metron.common.utils.JSONUtils;
  import org.apache.metron.enrichment.integration.utils.SampleUtil;
  import org.apache.metron.indexing.integration.IndexingIntegrationTest;
@@@ -39,10 -33,14 +39,11 @@@ import org.apache.metron.integration.co
  import org.apache.metron.integration.components.ZKServerComponent;
  import org.apache.metron.solr.integration.components.SolrComponent;
  
 -import javax.annotation.Nullable;
 -import java.util.List;
 -import java.util.Map;
 -import java.util.Properties;
+ 
  public class SolrIndexingIntegrationTest extends IndexingIntegrationTest {
  
 -  private String collection = "metron";
 +  private String collection = "yaf";
 +
    private FieldNameConverter fieldNameConverter = fieldName -> fieldName;
    @Override
    public FieldNameConverter getFieldNameConverter() {

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
----------------------------------------------------------------------
diff --cc metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
index 2b523d8,0000000..1a8e290
mode 100644,000000..100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
@@@ -1,196 -1,0 +1,201 @@@
 +/**
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.metron.solr.integration.schema;
 +
 +import com.google.common.collect.Iterables;
 +import com.google.common.io.Files;
 +import org.apache.metron.common.configuration.writer.WriterConfiguration;
 +import org.apache.metron.common.utils.JSONUtils;
 +import org.apache.metron.common.writer.BulkWriterResponse;
 +import org.apache.metron.solr.integration.components.SolrComponent;
 +import org.apache.metron.solr.writer.SolrWriter;
 +import org.apache.metron.stellar.common.utils.ConversionUtils;
 +import org.apache.solr.client.solrj.SolrServerException;
 +import org.apache.storm.tuple.Tuple;
 +import org.apache.zookeeper.KeeperException;
 +import org.json.simple.JSONObject;
 +import org.junit.Assert;
 +import org.junit.Before;
 +import org.junit.Test;
 +import org.mockito.Mock;
 +import java.io.File;
 +import java.io.IOException;
 +import java.nio.charset.Charset;
 +import java.util.*;
 +
 +import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
 +import static org.mockito.Mockito.mock;
 +
 +public class SchemaValidationIntegrationTest {
 +  public static Iterable<String> getData(String sensor) throws IOException {
 +    return Iterables.filter(
 +            Files.readLines(new File("src/test/resources/example_data/" + sensor), Charset.defaultCharset()),
 +            s -> !s.startsWith("#") && s.length() > 0
 +    );
 +  }
 +
 +  public static Map<String, Object> getGlobalConfig(String sensorType, SolrComponent
component) {
 +    Map<String, Object> globalConfig = new HashMap<>();
 +    globalConfig.put(SOLR_ZOOKEEPER, component.getZookeeperUrl());
 +    return globalConfig;
 +  }
 +
 +  public static SolrComponent createSolrComponent(String sensor) throws Exception {
 +    return new SolrComponent.Builder().build();
 +  }
 +
 +  @Test
 +  public void testError() throws Exception {
 +    test("error");
 +  }
 +
 +  @Test
 +  public void testBro() throws Exception {
 +    test("bro");
 +  }
 +
 +  @Test
 +  public void testSnort() throws Exception {
 +    test("snort");
 +  }
 +
 +  @Test
 +  public void testYaf() throws Exception {
 +    test("yaf");
 +  }
 +
 +  public String getGuid(Map<String, Object> m) {
 +    if(m.containsKey("guid")) {
 +      return (String)m.get("guid");
 +    }
 +    else {
 +      return (String) m.get("original_string");
 +    }
 +  }
 +
 +  public void test(String sensorType) throws Exception {
 +    SolrComponent component = null;
 +    try {
 +      component = createSolrComponent(sensorType);
 +      component.start();
 +      component.addCollection(String.format("%s", sensorType), String.format("src/main/config/schema/%s",
sensorType));
 +      Map<String, Object> globalConfig = getGlobalConfig(sensorType, component);
 +
 +      List<JSONObject> inputs = new ArrayList<>();
 +      List<Tuple> tuples = new ArrayList<>();
 +      Map<String, Map<String, Object>> index = new HashMap<>();
 +      for (String message : getData(sensorType)) {
 +        if (message.trim().length() > 0) {
 +          Tuple t = mock(Tuple.class);
 +          tuples.add(t);
 +          Map<String, Object> m = JSONUtils.INSTANCE.load(message.trim(), JSONUtils.MAP_SUPPLIER);
 +          String guid = getGuid(m);
 +          index.put(guid, m);
 +          inputs.add(new JSONObject(m));
 +        }
 +      }
 +      Assert.assertTrue(inputs.size() > 0);
 +
 +      SolrWriter solrWriter = new SolrWriter();
 +
 +      WriterConfiguration writerConfig = new WriterConfiguration() {
 +        @Override
 +        public int getBatchSize(String sensorName) {
 +          return inputs.size();
 +        }
 +
 +        @Override
 +        public int getBatchTimeout(String sensorName) {
 +          return 0;
 +        }
 +
 +        @Override
 +        public List<Integer> getAllConfiguredTimeouts() {
 +          return new ArrayList<>();
 +        }
 +
 +        @Override
 +        public String getIndex(String sensorName) {
 +          return sensorType;
 +        }
 +
 +        @Override
 +        public boolean isEnabled(String sensorName) {
 +          return true;
 +        }
 +
 +        @Override
 +        public Map<String, Object> getSensorConfig(String sensorName) {
 +          return new HashMap<String, Object>() {{
 +            put("index", sensorType);
 +            put("batchSize", inputs.size());
 +            put("enabled", true);
 +          }};
 +        }
 +
 +        @Override
 +        public Map<String, Object> getGlobalConfig() {
 +          return globalConfig;
 +        }
 +
 +        @Override
 +        public boolean isDefault(String sensorName) {
 +          return false;
 +        }
++
++        @Override
++        public String getFieldNameConverter(String sensorName) {
++          return null;
++        }
 +      };
 +
 +      solrWriter.init(null, null, writerConfig);
 +
 +      BulkWriterResponse response = solrWriter.write(sensorType, writerConfig, tuples, inputs);
 +      Assert.assertTrue(response.getErrors().isEmpty());
 +      for (Map<String, Object> m : component.getAllIndexedDocs(sensorType)) {
 +        Map<String, Object> expected = index.get(getGuid(m));
 +        for (Map.Entry<String, Object> field : expected.entrySet()) {
 +          if (field.getValue() instanceof Collection && ((Collection) field.getValue()).size()
== 0) {
 +            continue;
 +          }
 +          if(m.get(field.getKey()) instanceof Number) {
 +            Number n1 = ConversionUtils.convert(field.getValue(), Double.class);
 +            Number n2 = (Number)m.get(field.getKey());
 +            boolean isSame = Math.abs(n1.doubleValue() - n2.doubleValue()) < 1e-3;
 +            if(!isSame) {
 +              String s1 = "" + n1.doubleValue();
 +              String s2 = "" + n2.doubleValue();
 +              isSame = s1.startsWith(s2) || s2.startsWith(s1);
 +            }
 +            Assert.assertTrue("Unable to validate " + field.getKey() + ": " + n1 + " !=
" + n2, isSame);
 +          }
 +          else {
 +            Assert.assertEquals("Unable to find " + field.getKey(), "" + field.getValue(),
"" + m.get(field.getKey()));
 +          }
 +        }
 +      }
 +    }
 +    finally {
 +      if(component != null) {
 +        component.stop();
 +      }
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/pom.xml
----------------------------------------------------------------------


Mime
View raw message