metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [2/5] incubator-metron git commit: METRON-141: The ability to do threat triage closes apache/incubator-metron#108
Date Tue, 10 May 2016 16:53:54 GMT
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/Whois_CSV_to_JSON.py
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/Whois_CSV_to_JSON.py b/metron-platform/metron-data-management/src/main/bash/Whois_CSV_to_JSON.py
deleted file mode 100755
index 2091418..0000000
--- a/metron-platform/metron-data-management/src/main/bash/Whois_CSV_to_JSON.py
+++ /dev/null
@@ -1,208 +0,0 @@
-#!/usr/bin/python
-
-"""
-Copyright 2014 Cisco Systems, Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import sys
-import os
-import csv
-import json
-import multiprocessing
-import logging
-logging.basicConfig(level=logging.DEBUG)
-
-
-def is_field_excluded(fieldname=None):
-    """
-    Checks to see if a field name is a member of a list of names to exclude. Modify to suit your own list.
-
-    :param fieldname: A string representing a field name
-    :return: True or False
-    """
-    import re
-
-    # List of fields names to exclude
-    excluded_fields = [
-        'Audit_auditUpdatedDate',
-        #'domainName'
-    ]
-
-    if fieldname in excluded_fields:
-        return True
-
-    # Regexes to match for exclusion
-    excluded_regexes = [
-        ['_rawText$', re.IGNORECASE],
-    ]
-
-    for regex in excluded_regexes:
-        if re.search(regex[0], fieldname, regex[1]):
-            return True
-
-    return False
-
-
-def process_csv(in_filename, out_filename):
-    """
-    Processes a CSV file of WHOIS data and converts each line to a JSON element, skipping specific fields that
-    are not deemed necessary (domainName, *_rawText, Audit_auditUpdatedDate)
-
-    :param in_filename: Input CSV filename with full path
-    :param out_filename: Output JSON filename with full path
-    :return: None
-    """
-    if out_filename:
-        out_fh = open(out_filename, 'wb')
-        logging.debug('%s: Converting %s to %s' % (multiprocessing.current_process().name, in_filename, out_filename))
-    else:
-        logging.debug('%s: Analyzing %s' % (multiprocessing.current_process().name, in_filename))
-
-    with open(in_filename, 'rb') as f:
-        reader = csv.DictReader(f, delimiter=',', quotechar='"')
-        line_num = 0
-        try:
-            for row in reader:
-                line_num += 1
-                try:
-                    if out_filename:
-                        # json conversion and output
-                        new_row = {}
-                        for field in reader.fieldnames:
-                            # fields we don't want include these + anything with rawText
-                            #if field not in ['Audit_auditUpdatedDate', 'domainName'] and not field.endswith('_rawText'):
-                            if not is_field_excluded(field):
-                                new_row[field] = row.get(field)
-                        json.dump(new_row, out_fh)
-                        out_fh.write('\n')
-                    else:
-                        # analysis .. check to be sure fileheader and csv row counts match
-                        if len(row) != len(reader.fieldnames):
-                            raise Exception('Field count mismatch: row: %s / fields: %s' % (len(row), len(reader.fieldnames)))
-                except Exception, e:
-                    logging.warn("Error with file %s, line %s: %s" % (in_filename, line_num, e))
-
-            if not out_filename:
-                logging.info('Analyzed %s: OK' % in_filename)
-        except Exception, e:
-            logging.warn(e)
-
-        out_fh.close()
-
-
-##-------------------------------------------------------------------------
-
-def process_files(source_dir, output_dir, max_processes=10, overwrite=False):
-    """
-    Generates a multiprocessing.Pool() queue with a list of input and output files to be processed with processCSV.
-    Files are added by walking the source_dir and adding any file with a CSV extension. Output is placed into a single
-    directory for processing. Output filenames are generated using the first part of the directory name so a file
-    named source_dir/com/1.csv would become outputDir/com_1.json
-
-    :param source_dir: Source directory of CSV files
-    :param output_dir: Output directory for resultant JSON files
-    :param max_processes: Maximum number of processes run
-    :return:
-    """
-    logging.info("Processing Whois files from %s" % source_dir)
-
-    if output_dir and not os.path.exists(output_dir):
-        logging.debug("Creating output directory %s" % output_dir)
-        os.makedirs(output_dir)
-
-    logging.info("Starting %s pool workers" % max_processes)
-
-    if sys.version.startswith('2.6'):
-        # no maxtaskperchild in 2.6
-        pool = multiprocessing.Pool(processes=max_processes)
-    else:
-        pool = multiprocessing.Pool(processes=max_processes, maxtasksperchild=4)
-
-    filecount = 0
-    for dirname, dirnames, filenames in os.walk(source_dir):
-        for filename in filenames:
-            if filename.endswith('.csv'):
-                # output files go to outputDir and are named using the last subdirectory from the dirname
-                if output_dir:
-                    out_filename = filename.replace('csv', 'json')
-                    out_filename = os.path.join(output_dir, '%s_%s' % (os.path.split(dirname)[-1], out_filename))
-
-                    # if file does not exist or if overwrite is true, add file process to the pool
-                    if not os.path.isfile(out_filename) or overwrite:
-                        pool.apply_async(process_csv, args=(os.path.join(dirname, filename), out_filename))
-                        filecount += 1
-                    else:
-                        logging.info("Skipping %s, %s exists and overwrite is false" % (filename, out_filename))
-                else:
-                    # no outputdir so we just analyze the files
-                    pool.apply_async(process_csv, args=(os.path.join(dirname, filename), None))
-                    filecount += 1
-
-    try:
-        pool.close()
-        logging.info("Starting activities on %s CSV files" % filecount)
-        pool.join()
-    except KeyboardInterrupt:
-        logging.info("Aborting")
-        pool.terminate()
-
-    logging.info("Completed")
-
-
-##-------------------------------------------------------------------------
-
-if __name__ == "__main__":
-
-    max_cpu = multiprocessing.cpu_count()
-
-    from optparse import OptionParser
-    parser = OptionParser()
-    parser.add_option('-s', '--source', dest='source_dir', action='store',
-                      help='Source directory to walk for CSV files')
-    parser.add_option('-o', '--output', dest='out_dir', action='store',
-                      help='Output directory for JSON files')
-    parser.add_option('-O', '--overwrite', dest='overwrite', action='store_true',
-                      help='Overwrite existing files in output directory')
-    parser.add_option('-p', '--processes', dest='max_processes', action='store', default=max_cpu, type='int',
-                      help='Max number of processes to spawn')
-    parser.add_option('-a', '--analyze', dest='analyze', action='store_true',
-                      help='Analyze CSV files for validity, no file output')
-    parser.add_option('-d', '--debug', dest='debug', action='store_true',
-                      help='Enable debug messages')
-
-    (options, args) = parser.parse_args()
-
-    if not options.source_dir:
-        logging.error("Source directory required")
-        sys.exit(-1)
-
-    if not options.out_dir or options.analyze:
-        out_dir = None
-    elif not options.out_dir:
-        logging.error("Ouput directory or analysis option required")
-        sys.exit(-1)
-    else:
-        out_dir = options.out_dir
-
-    if options.max_processes > max_cpu:
-        logging.warn('Max Processes (%s) is greater than available Processors (%s)' % (options.max_processes, max_cpu))
-
-    if options.debug:
-        # enable debug level and multiprocessing debugging
-        logging.basicConfig(level=logging.DEBUG)
-        multiprocessing.log_to_stderr(logging.DEBUG)
-
-    process_files(options.source_dir, options.out_dir, options.max_processes, options.overwrite)
-

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/flatfile_loader.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/flatfile_loader.sh b/metron-platform/metron-data-management/src/main/bash/flatfile_loader.sh
deleted file mode 100755
index 03fd641..0000000
--- a/metron-platform/metron-data-management/src/main/bash/flatfile_loader.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-# 
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#     http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# 
-
-BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
-[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
-
-# Autodetect JAVA_HOME if not defined
-if [ -e /usr/libexec/bigtop-detect-javahome ]; then
-  . /usr/libexec/bigtop-detect-javahome
-elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
-  . /usr/lib/bigtop-utils/bigtop-detect-javahome
-fi
-
-export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
-CP=/usr/metron/0.1BETA/lib/metron-data-management-0.1BETA.jar:/usr/metron/0.1BETA/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
-HADOOP_CLASSPATH=$(echo $CP )
-for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
-  if [ -f $jar ];then
-    LIBJARS="$jar,$LIBJARS"
-  fi
-done
-export HADOOP_CLASSPATH
-export METRON_VERSION=${project.version}
-export METRON_HOME=/usr/metron/$METRON_VERSION
-export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
-hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.nonbulk.flatfile.SimpleEnrichmentFlatFileLoader "$@"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/prune_elasticsearch_indices.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/prune_elasticsearch_indices.sh b/metron-platform/metron-data-management/src/main/bash/prune_elasticsearch_indices.sh
deleted file mode 100644
index c3f1d05..0000000
--- a/metron-platform/metron-data-management/src/main/bash/prune_elasticsearch_indices.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}.jar org.apache.metron.dataloads.bulk.ElasticsearchDataPrunerRunner "$@"
-

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/prune_hdfs_files.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/prune_hdfs_files.sh b/metron-platform/metron-data-management/src/main/bash/prune_hdfs_files.sh
deleted file mode 100644
index b8d9372..0000000
--- a/metron-platform/metron-data-management/src/main/bash/prune_hdfs_files.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}.jar org.apache.metron.dataloads.bulk.HDFSDataPruner "$@"
-

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_load.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_load.sh b/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_load.sh
deleted file mode 100755
index 865d0ad..0000000
--- a/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_load.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-# 
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#     http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# 
-
-BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
-[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
-
-# Autodetect JAVA_HOME if not defined
-if [ -e /usr/libexec/bigtop-detect-javahome ]; then
-  . /usr/libexec/bigtop-detect-javahome
-elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
-  . /usr/lib/bigtop-utils/bigtop-detect-javahome
-fi
-
-export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
-HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
-for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
-  if [ -f $jar ];then
-    LIBJARS="$jar,$LIBJARS"
-  fi
-done
-export HADOOP_CLASSPATH
-export METRON_VERSION=${project.version}
-export METRON_HOME=/usr/metron/$METRON_VERSION
-export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
-hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.bulk.ThreatIntelBulkLoader -libjars ${LIBJARS} "$@"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_prune.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_prune.sh b/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_prune.sh
deleted file mode 100755
index 6156027..0000000
--- a/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_prune.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#     http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# 
-BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
-[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
-
-# Autodetect JAVA_HOME if not defined
-if [ -e /usr/libexec/bigtop-detect-javahome ]; then
-  . /usr/libexec/bigtop-detect-javahome
-elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
-  . /usr/lib/bigtop-utils/bigtop-detect-javahome
-fi
-
-export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
-HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
-for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
-  if [ -f $jar ];then
-    LIBJARS="$jar,$LIBJARS"
-  fi
-done
-export HADOOP_CLASSPATH
-export METRON_VERSION=${project.version}
-export METRON_HOME=/usr/metron/$METRON_VERSION
-export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
-hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.bulk.LeastRecentlyUsedPruner -libjars ${LIBJARS} "$@"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/threatintel_taxii_load.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/threatintel_taxii_load.sh b/metron-platform/metron-data-management/src/main/bash/threatintel_taxii_load.sh
deleted file mode 100755
index 23d09ba..0000000
--- a/metron-platform/metron-data-management/src/main/bash/threatintel_taxii_load.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-# 
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#     http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# 
-
-BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
-[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
-
-# Autodetect JAVA_HOME if not defined
-if [ -e /usr/libexec/bigtop-detect-javahome ]; then
-  . /usr/libexec/bigtop-detect-javahome
-elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
-  . /usr/lib/bigtop-utils/bigtop-detect-javahome
-fi
-
-export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
-CP=/usr/metron/0.1BETA/lib/metron-data-management-0.1BETA.jar:/usr/metron/0.1BETA/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
-HADOOP_CLASSPATH=$(echo $CP )
-for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
-  if [ -f $jar ];then
-    LIBJARS="$jar,$LIBJARS"
-  fi
-done
-export HADOOP_CLASSPATH
-export METRON_VERSION=${project.version}
-export METRON_HOME=/usr/metron/$METRON_VERSION
-export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
-hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.nonbulk.taxii.TaxiiLoader "$@"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
index 2e63ef2..4e2da61 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.metron.dataloads.extractor.ExtractorHandler;
 import org.apache.metron.dataloads.hbase.mr.BulkLoadMapper;
-import org.apache.metron.common.configuration.EnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig;
 import org.apache.metron.enrichment.converter.HbaseConverter;
 import org.apache.metron.enrichment.converter.EnrichmentConverter;
 import org.apache.metron.common.utils.JSONUtils;
@@ -238,10 +238,10 @@ public class ThreatIntelBulkLoader  {
     if(BulkLoadOptions.CONVERTER.has(cli)) {
       converterClass = BulkLoadOptions.CONVERTER.get(cli);
     }
-    EnrichmentConfig enrichmentConfig = null;
+    SensorEnrichmentUpdateConfig sensorEnrichmentUpdateConfig = null;
     if(BulkLoadOptions.ENRICHMENT_CONFIG.has(cli)) {
-      enrichmentConfig = JSONUtils.INSTANCE.load( new File(BulkLoadOptions.ENRICHMENT_CONFIG.get(cli))
-              , EnrichmentConfig.class
+      sensorEnrichmentUpdateConfig = JSONUtils.INSTANCE.load( new File(BulkLoadOptions.ENRICHMENT_CONFIG.get(cli))
+              , SensorEnrichmentUpdateConfig.class
       );
     }
 
@@ -252,8 +252,8 @@ public class ThreatIntelBulkLoader  {
     if(!jobRet) {
       System.exit(1);
     }
-    if(enrichmentConfig != null) {
-        enrichmentConfig.updateSensorConfigs();
+    if(sensorEnrichmentUpdateConfig != null) {
+        sensorEnrichmentUpdateConfig.updateSensorConfigs();
     }
     System.exit(0);
   }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoader.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoader.java
index cbd3beb..0c7501a 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoader.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoader.java
@@ -31,7 +31,7 @@ import org.apache.log4j.PropertyConfigurator;
 import org.apache.metron.dataloads.extractor.Extractor;
 import org.apache.metron.dataloads.extractor.ExtractorHandler;
 import org.apache.metron.dataloads.extractor.inputformat.WholeFileFormat;
-import org.apache.metron.common.configuration.EnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig;
 import org.apache.metron.hbase.HTableProvider;
 import org.apache.metron.enrichment.converter.HbaseConverter;
 import org.apache.metron.enrichment.converter.EnrichmentConverter;
@@ -239,10 +239,10 @@ public class SimpleEnrichmentFlatFileLoader {
     );
     boolean lineByLine = !handler.getInputFormatHandler().getClass().equals(WholeFileFormat.class);
     Extractor e = handler.getExtractor();
-    EnrichmentConfig enrichmentConfig = null;
+    SensorEnrichmentUpdateConfig sensorEnrichmentUpdateConfig = null;
     if(LoadOptions.ENRICHMENT_CONFIG.has(cli)) {
-      enrichmentConfig = JSONUtils.INSTANCE.load( new File(LoadOptions.ENRICHMENT_CONFIG.get(cli))
-              , EnrichmentConfig.class
+      sensorEnrichmentUpdateConfig = JSONUtils.INSTANCE.load( new File(LoadOptions.ENRICHMENT_CONFIG.get(cli))
+              , SensorEnrichmentUpdateConfig.class
       );
     }
     HbaseConverter converter = new EnrichmentConverter();
@@ -254,8 +254,8 @@ public class SimpleEnrichmentFlatFileLoader {
     for (File f : inputFiles) {
       loader.loadFile(f, e, table, LoadOptions.HBASE_CF.get(cli), converter, lineByLine);
     }
-    if(enrichmentConfig != null) {
-      enrichmentConfig.updateSensorConfigs();
+    if(sensorEnrichmentUpdateConfig != null) {
+      sensorEnrichmentUpdateConfig.updateSensorConfigs();
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
index df803c1..689a08f 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
@@ -31,7 +31,7 @@ import org.apache.log4j.PropertyConfigurator;
 import org.apache.metron.dataloads.extractor.Extractor;
 import org.apache.metron.dataloads.extractor.ExtractorHandler;
 import org.apache.metron.dataloads.extractor.stix.StixExtractor;
-import org.apache.metron.common.configuration.EnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig;
 import org.apache.metron.common.utils.JSONUtils;
 
 import javax.annotation.Nullable;
@@ -176,12 +176,12 @@ public class TaxiiLoader {
     }
     ExtractorHandler handler = ExtractorHandler.load(FileUtils.readFileToString(new File(TaxiiOptions.EXTRACTOR_CONFIG.get(cli))));
     Extractor e = handler.getExtractor();
-    EnrichmentConfig enrichmentConfig = null;
+    SensorEnrichmentUpdateConfig sensorEnrichmentUpdateConfig = null;
     if(TaxiiOptions.ENRICHMENT_CONFIG.has(cli)) {
-      enrichmentConfig = JSONUtils.INSTANCE.load( new File(TaxiiOptions.ENRICHMENT_CONFIG.get(cli))
-              , EnrichmentConfig.class
+      sensorEnrichmentUpdateConfig = JSONUtils.INSTANCE.load( new File(TaxiiOptions.ENRICHMENT_CONFIG.get(cli))
+              , SensorEnrichmentUpdateConfig.class
       );
-      enrichmentConfig.updateSensorConfigs();
+      sensorEnrichmentUpdateConfig.updateSensorConfigs();
     }
 
     Timer timer = new Timer();

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/Whois_CSV_to_JSON.py
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/Whois_CSV_to_JSON.py b/metron-platform/metron-data-management/src/main/scripts/Whois_CSV_to_JSON.py
new file mode 100755
index 0000000..2091418
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/Whois_CSV_to_JSON.py
@@ -0,0 +1,208 @@
+#!/usr/bin/python
+
+"""
+Copyright 2014 Cisco Systems, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import sys
+import os
+import csv
+import json
+import multiprocessing
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+
+def is_field_excluded(fieldname=None):
+    """
+    Checks to see if a field name is a member of a list of names to exclude. Modify to suit your own list.
+
+    :param fieldname: A string representing a field name
+    :return: True or False
+    """
+    import re
+
+    # List of fields names to exclude
+    excluded_fields = [
+        'Audit_auditUpdatedDate',
+        #'domainName'
+    ]
+
+    if fieldname in excluded_fields:
+        return True
+
+    # Regexes to match for exclusion
+    excluded_regexes = [
+        ['_rawText$', re.IGNORECASE],
+    ]
+
+    for regex in excluded_regexes:
+        if re.search(regex[0], fieldname, regex[1]):
+            return True
+
+    return False
+
+
+def process_csv(in_filename, out_filename):
+    """
+    Processes a CSV file of WHOIS data and converts each line to a JSON element, skipping specific fields that
+    are not deemed necessary (domainName, *_rawText, Audit_auditUpdatedDate)
+
+    :param in_filename: Input CSV filename with full path
+    :param out_filename: Output JSON filename with full path
+    :return: None
+    """
+    if out_filename:
+        out_fh = open(out_filename, 'wb')
+        logging.debug('%s: Converting %s to %s' % (multiprocessing.current_process().name, in_filename, out_filename))
+    else:
+        logging.debug('%s: Analyzing %s' % (multiprocessing.current_process().name, in_filename))
+
+    with open(in_filename, 'rb') as f:
+        reader = csv.DictReader(f, delimiter=',', quotechar='"')
+        line_num = 0
+        try:
+            for row in reader:
+                line_num += 1
+                try:
+                    if out_filename:
+                        # json conversion and output
+                        new_row = {}
+                        for field in reader.fieldnames:
+                            # fields we don't want include these + anything with rawText
+                            #if field not in ['Audit_auditUpdatedDate', 'domainName'] and not field.endswith('_rawText'):
+                            if not is_field_excluded(field):
+                                new_row[field] = row.get(field)
+                        json.dump(new_row, out_fh)
+                        out_fh.write('\n')
+                    else:
+                        # analysis .. check to be sure fileheader and csv row counts match
+                        if len(row) != len(reader.fieldnames):
+                            raise Exception('Field count mismatch: row: %s / fields: %s' % (len(row), len(reader.fieldnames)))
+                except Exception, e:
+                    logging.warn("Error with file %s, line %s: %s" % (in_filename, line_num, e))
+
+            if not out_filename:
+                logging.info('Analyzed %s: OK' % in_filename)
+        except Exception, e:
+            logging.warn(e)
+
+        out_fh.close()
+
+
+##-------------------------------------------------------------------------
+
+def process_files(source_dir, output_dir, max_processes=10, overwrite=False):
+    """
+    Generates a multiprocessing.Pool() queue with a list of input and output files to be processed with processCSV.
+    Files are added by walking the source_dir and adding any file with a CSV extension. Output is placed into a single
+    directory for processing. Output filenames are generated using the first part of the directory name so a file
+    named source_dir/com/1.csv would become outputDir/com_1.json
+
+    :param source_dir: Source directory of CSV files
+    :param output_dir: Output directory for resultant JSON files
+    :param max_processes: Maximum number of processes run
+    :return:
+    """
+    logging.info("Processing Whois files from %s" % source_dir)
+
+    if output_dir and not os.path.exists(output_dir):
+        logging.debug("Creating output directory %s" % output_dir)
+        os.makedirs(output_dir)
+
+    logging.info("Starting %s pool workers" % max_processes)
+
+    if sys.version.startswith('2.6'):
+        # no maxtaskperchild in 2.6
+        pool = multiprocessing.Pool(processes=max_processes)
+    else:
+        pool = multiprocessing.Pool(processes=max_processes, maxtasksperchild=4)
+
+    filecount = 0
+    for dirname, dirnames, filenames in os.walk(source_dir):
+        for filename in filenames:
+            if filename.endswith('.csv'):
+                # output files go to outputDir and are named using the last subdirectory from the dirname
+                if output_dir:
+                    out_filename = filename.replace('csv', 'json')
+                    out_filename = os.path.join(output_dir, '%s_%s' % (os.path.split(dirname)[-1], out_filename))
+
+                    # if file does not exist or if overwrite is true, add file process to the pool
+                    if not os.path.isfile(out_filename) or overwrite:
+                        pool.apply_async(process_csv, args=(os.path.join(dirname, filename), out_filename))
+                        filecount += 1
+                    else:
+                        logging.info("Skipping %s, %s exists and overwrite is false" % (filename, out_filename))
+                else:
+                    # no outputdir so we just analyze the files
+                    pool.apply_async(process_csv, args=(os.path.join(dirname, filename), None))
+                    filecount += 1
+
+    try:
+        pool.close()
+        logging.info("Starting activities on %s CSV files" % filecount)
+        pool.join()
+    except KeyboardInterrupt:
+        logging.info("Aborting")
+        pool.terminate()
+
+    logging.info("Completed")
+
+
+##-------------------------------------------------------------------------
+
+if __name__ == "__main__":
+
+    max_cpu = multiprocessing.cpu_count()
+
+    from optparse import OptionParser
+    parser = OptionParser()
+    parser.add_option('-s', '--source', dest='source_dir', action='store',
+                      help='Source directory to walk for CSV files')
+    parser.add_option('-o', '--output', dest='out_dir', action='store',
+                      help='Output directory for JSON files')
+    parser.add_option('-O', '--overwrite', dest='overwrite', action='store_true',
+                      help='Overwrite existing files in output directory')
+    parser.add_option('-p', '--processes', dest='max_processes', action='store', default=max_cpu, type='int',
+                      help='Max number of processes to spawn')
+    parser.add_option('-a', '--analyze', dest='analyze', action='store_true',
+                      help='Analyze CSV files for validity, no file output')
+    parser.add_option('-d', '--debug', dest='debug', action='store_true',
+                      help='Enable debug messages')
+
+    (options, args) = parser.parse_args()
+
+    if not options.source_dir:
+        logging.error("Source directory required")
+        sys.exit(-1)
+
+    if not options.out_dir or options.analyze:
+        out_dir = None
+    elif not options.out_dir:
+        logging.error("Ouput directory or analysis option required")
+        sys.exit(-1)
+    else:
+        out_dir = options.out_dir
+
+    if options.max_processes > max_cpu:
+        logging.warn('Max Processes (%s) is greater than available Processors (%s)' % (options.max_processes, max_cpu))
+
+    if options.debug:
+        # enable debug level and multiprocessing debugging
+        logging.basicConfig(level=logging.DEBUG)
+        multiprocessing.log_to_stderr(logging.DEBUG)
+
+    process_files(options.source_dir, options.out_dir, options.max_processes, options.overwrite)
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
new file mode 100755
index 0000000..03fd641
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+# 
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+
+BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
+[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
+
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+  . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+  . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
+CP=/usr/metron/0.1BETA/lib/metron-data-management-0.1BETA.jar:/usr/metron/0.1BETA/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
+HADOOP_CLASSPATH=$(echo $CP )
+for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
+  if [ -f $jar ];then
+    LIBJARS="$jar,$LIBJARS"
+  fi
+done
+export HADOOP_CLASSPATH
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.nonbulk.flatfile.SimpleEnrichmentFlatFileLoader "$@"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/prune_elasticsearch_indices.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/prune_elasticsearch_indices.sh b/metron-platform/metron-data-management/src/main/scripts/prune_elasticsearch_indices.sh
new file mode 100644
index 0000000..c3f1d05
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/prune_elasticsearch_indices.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}.jar org.apache.metron.dataloads.bulk.ElasticsearchDataPrunerRunner "$@"
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh b/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh
new file mode 100644
index 0000000..b8d9372
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}.jar org.apache.metron.dataloads.bulk.HDFSDataPruner "$@"
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_load.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_load.sh b/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_load.sh
new file mode 100755
index 0000000..865d0ad
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_load.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+# 
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+
+BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
+[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
+
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+  . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+  . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
+HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
+for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
+  if [ -f $jar ];then
+    LIBJARS="$jar,$LIBJARS"
+  fi
+done
+export HADOOP_CLASSPATH
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.bulk.ThreatIntelBulkLoader -libjars ${LIBJARS} "$@"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh b/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh
new file mode 100755
index 0000000..6156027
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
+[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
+
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+  . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+  . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
+HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
+for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
+  if [ -f $jar ];then
+    LIBJARS="$jar,$LIBJARS"
+  fi
+done
+export HADOOP_CLASSPATH
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.bulk.LeastRecentlyUsedPruner -libjars ${LIBJARS} "$@"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh b/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh
new file mode 100755
index 0000000..23d09ba
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+# 
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+
+BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
+[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
+
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+  . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+  . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
+CP=/usr/metron/0.1BETA/lib/metron-data-management-0.1BETA.jar:/usr/metron/0.1BETA/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
+HADOOP_CLASSPATH=$(echo $CP )
+for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
+  if [ -f $jar ];then
+    LIBJARS="$jar,$LIBJARS"
+  fi
+done
+export HADOOP_CLASSPATH
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.nonbulk.taxii.TaxiiLoader "$@"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
index 6b54fec..989abfb 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
@@ -19,7 +19,7 @@ package org.apache.metron.elasticsearch.writer;
 
 import backtype.storm.tuple.Tuple;
 import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.common.interfaces.BulkMessageWriter;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml
index 756a0b4..87e5096 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/pom.xml
@@ -159,6 +159,7 @@
                     </systemProperties>
                 </configuration>
             </plugin>
+
             <!-- Normally, dependency report takes time, skip it -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
index f2e0113..22629a4 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
@@ -64,7 +64,7 @@ public class SimpleHBaseAdapter implements EnrichmentAdapter<CacheKey>,Serializa
   public JSONObject enrich(CacheKey value) {
     JSONObject enriched = new JSONObject();
     List<String> enrichmentTypes = value.getConfig()
-                                        .getFieldToEnrichmentTypeMap()
+                                        .getEnrichment().getFieldToTypeMap()
                                         .get(EnrichmentUtils.toTopLevelField(value.getField()));
     if(enrichmentTypes != null && value.getValue() != null) {
       try {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
index dbdf6ec..ee5636b 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
@@ -54,7 +54,7 @@ public class ThreatIntelAdapter implements EnrichmentAdapter<CacheKey>,Serializa
 
   @Override
   public void logAccess(CacheKey value) {
-    List<String> enrichmentTypes = value.getConfig().getFieldToThreatIntelTypeMap().get(value.getField());
+    List<String> enrichmentTypes = value.getConfig().getThreatIntel().getFieldToTypeMap().get(value.getField());
     if(enrichmentTypes != null) {
       for(String enrichmentType : enrichmentTypes) {
         lookup.getAccessTracker().logAccess(new EnrichmentKey(enrichmentType, value.getValue()));
@@ -67,7 +67,7 @@ public class ThreatIntelAdapter implements EnrichmentAdapter<CacheKey>,Serializa
   public JSONObject enrich(CacheKey value) {
     JSONObject enriched = new JSONObject();
     List<String> enrichmentTypes = value.getConfig()
-                                        .getFieldToThreatIntelTypeMap()
+                                        .getThreatIntel().getFieldToTypeMap()
                                         .get(EnrichmentUtils.toTopLevelField(value.getField()));
     if(enrichmentTypes != null) {
       int i = 0;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
index f3e742d..7cfa34d 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
@@ -24,7 +24,7 @@ import backtype.storm.tuple.Fields;
 import backtype.storm.tuple.Tuple;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.bolt.ConfiguredBolt;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.common.utils.ErrorUtils;
 import org.apache.metron.common.utils.MessageUtils;
 import org.apache.metron.common.interfaces.BulkMessageWriter;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/CacheKey.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/CacheKey.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/CacheKey.java
index 1338b44..4dbd33e 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/CacheKey.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/CacheKey.java
@@ -17,7 +17,7 @@
  */
 package org.apache.metron.enrichment.bolt;
 
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 
 public class CacheKey {
   private String field;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
index 8ef44d0..48e09f8 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
@@ -18,7 +18,7 @@
 package org.apache.metron.enrichment.bolt;
 
 import backtype.storm.task.TopologyContext;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.common.utils.MessageUtils;
 import org.json.simple.JSONObject;
 import org.slf4j.Logger;
@@ -48,6 +48,10 @@ public class EnrichmentJoinBolt extends JoinBolt<JSONObject> {
   public Set<String> getStreamIds(JSONObject message) {
     Set<String> streamIds = new HashSet<>();
     String sourceType = MessageUtils.getSensorType(message);
+    if(sourceType == null) {
+      String errorMessage = "Unable to find source type for message: " + message;
+      throw new IllegalStateException(errorMessage);
+    }
     Map<String, List<String>>  fieldMap = getFieldMap(sourceType);
     if(fieldMap != null) {
       for (String enrichmentType : getFieldMap(sourceType).keySet()) {
@@ -83,8 +87,8 @@ public class EnrichmentJoinBolt extends JoinBolt<JSONObject> {
   public Map<String, List<String>> getFieldMap(String sourceType) {
     if(sourceType != null) {
       SensorEnrichmentConfig config = configurations.getSensorEnrichmentConfig(sourceType);
-      if (config != null) {
-        return config.getEnrichmentFieldMap();
+      if (config != null && config.getEnrichment() != null) {
+        return config.getEnrichment().getFieldMap();
       }
       else {
         LOG.error("Unable to retrieve a sensor enrichment config of " + sourceType);

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
index 6b49edb..c367173 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
@@ -21,7 +21,7 @@ import backtype.storm.task.TopologyContext;
 import backtype.storm.topology.OutputFieldsDeclarer;
 import backtype.storm.tuple.Tuple;
 import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.enrichment.configuration.Enrichment;
 import org.apache.metron.enrichment.utils.EnrichmentUtils;
 import org.apache.metron.common.utils.MessageUtils;
@@ -127,7 +127,7 @@ public class EnrichmentSplitterBolt extends SplitBolt<JSONObject> {
         if(sensorType != null) {
             SensorEnrichmentConfig config = configurations.getSensorEnrichmentConfig(sensorType);
             if (config != null) {
-                return config.getEnrichmentFieldMap();
+                return config.getEnrichment().getFieldMap();
             } else {
                 LOG.error("Unable to retrieve a sensor enrichment config of " + sensorType);
             }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
index e5b8ca6..b79d6c7 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
@@ -31,7 +31,7 @@ import org.apache.metron.common.Constants;
 import org.apache.metron.common.bolt.ConfiguredBolt;
 import org.apache.metron.common.configuration.Configurations;
 import org.apache.metron.enrichment.configuration.Enrichment;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
 import org.apache.metron.common.utils.ErrorUtils;
 import org.json.simple.JSONObject;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
index 68e56ed..1964961 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
@@ -29,6 +29,8 @@ import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.Sets;
 import org.apache.metron.common.bolt.ConfiguredBolt;
+import org.apache.metron.common.utils.ErrorUtils;
+import org.json.simple.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import java.util.HashMap;
@@ -66,10 +68,12 @@ public abstract class JoinBolt<V> extends ConfiguredBolt {
   public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
     super.prepare(map, topologyContext, outputCollector);
     this.collector = outputCollector;
-    if (this.maxCacheSize == null)
+    if (this.maxCacheSize == null) {
       throw new IllegalStateException("maxCacheSize must be specified");
-    if (this.maxTimeRetain == null)
+    }
+    if (this.maxTimeRetain == null) {
       throw new IllegalStateException("maxTimeRetain must be specified");
+    }
     loader = new CacheLoader<String, Map<String, V>>() {
       public Map<String, V> load(String key) throws Exception {
         return new HashMap<>();
@@ -96,13 +100,18 @@ public abstract class JoinBolt<V> extends ConfiguredBolt {
       streamMessageMap.put(streamId, message);
       Set<String> streamIds = getStreamIds(message);
       Set<String> streamMessageKeys = streamMessageMap.keySet();
-      if (streamMessageKeys.size() == streamIds.size() && Sets.symmetricDifference
-              (streamMessageKeys, streamIds)
-              .isEmpty()) {
-        collector.emit("message", tuple, new Values(key, joinMessages
-                (streamMessageMap)));
-        collector.ack(tuple);
+      if ( streamMessageKeys.size() == streamIds.size()
+        && Sets.symmetricDifference(streamMessageKeys, streamIds)
+               .isEmpty()
+         ) {
+        collector.emit( "message"
+                      , tuple
+                      , new Values( key
+                                  , joinMessages(streamMessageMap)
+                                  )
+                      );
         cache.invalidate(key);
+        collector.ack(tuple);
       } else {
         cache.put(key, streamMessageMap);
         if(LOG.isDebugEnabled()) {
@@ -111,15 +120,19 @@ public abstract class JoinBolt<V> extends ConfiguredBolt {
                    );
         }
       }
-    } catch (ExecutionException e) {
+    } catch (Exception e) {
+      LOG.error("[Metron] Unable to join messages: " + message, e);
+      JSONObject error = ErrorUtils.generateErrorMessage("Joining problem: " + message, e);
+      collector.ack(tuple);
+      collector.emit("error", new Values(error));
       collector.reportError(e);
-      LOG.error(e.getMessage(), e);
     }
   }
 
   @Override
   public void declareOutputFields(OutputFieldsDeclarer declarer) {
     declarer.declareStream("message", new Fields("key", "message"));
+    declarer.declareStream("error", new Fields("message"));
   }
 
   public abstract void prepare(Map map, TopologyContext topologyContext);

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
index a2b0e78..c08bd0d 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
@@ -17,7 +17,12 @@
  */
 package org.apache.metron.enrichment.bolt;
 
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import com.google.common.base.Joiner;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig;
+import org.apache.metron.common.utils.MessageUtils;
+import org.apache.metron.threatintel.triage.ThreatTriageProcessor;
 import org.json.simple.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -38,7 +43,7 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt {
   public Map<String, List<String>> getFieldMap(String sourceType) {
     SensorEnrichmentConfig config = configurations.getSensorEnrichmentConfig(sourceType);
     if(config != null) {
-      return config.getThreatIntelFieldMap();
+      return config.getThreatIntel().getFieldMap();
     }
     else {
       LOG.error("Unable to retrieve sensor config: " + sourceType);
@@ -49,12 +54,54 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt {
   @Override
   public JSONObject joinMessages(Map<String, JSONObject> streamMessageMap) {
     JSONObject ret = super.joinMessages(streamMessageMap);
-    for(Object key : ret.keySet()) {
-      if(key.toString().startsWith("threatintels") && !key.toString().endsWith(".ts")) {
-        ret.put("is_alert" , "true");
-        break;
+    boolean isAlert = ret.containsKey("is_alert");
+    if(!isAlert) {
+      for (Object key : ret.keySet()) {
+        if (key.toString().startsWith("threatintels") && !key.toString().endsWith(".ts")) {
+          isAlert = true;
+          break;
+        }
       }
     }
+    if(isAlert) {
+      ret.put("is_alert" , "true");
+      String sourceType = MessageUtils.getSensorType(ret);
+      SensorEnrichmentConfig config = configurations.getSensorEnrichmentConfig(sourceType);
+      ThreatTriageConfig triageConfig = null;
+      if(config != null) {
+        triageConfig = config.getThreatIntel().getTriageConfig();
+        if(LOG.isDebugEnabled()) {
+          LOG.debug(sourceType + ": Found sensor enrichment config.");
+        }
+      }
+      else {
+        LOG.debug(sourceType + ": Unable to find threat config.");
+      }
+      if(triageConfig != null) {
+        if(LOG.isDebugEnabled()) {
+          LOG.debug(sourceType + ": Found threat triage config: " + triageConfig);
+        }
+
+        if(LOG.isDebugEnabled() && (triageConfig.getRiskLevelRules() == null || triageConfig.getRiskLevelRules().isEmpty())) {
+          LOG.debug(sourceType + ": Empty rules!");
+        }
+
+        ThreatTriageProcessor threatTriageProcessor = new ThreatTriageProcessor(triageConfig);
+        Double triageLevel = threatTriageProcessor.apply(ret);
+        if(LOG.isDebugEnabled()) {
+          String rules = Joiner.on('\n').join(triageConfig.getRiskLevelRules().entrySet());
+          LOG.debug("Marked " + sourceType + " as triage level " + triageLevel + " with rules " + rules);
+        }
+        if(triageLevel != null && triageLevel > 0) {
+          ret.put("threat.triage.level", triageLevel);
+        }
+      }
+      else {
+        LOG.debug(sourceType + ": Unable to find threat triage config!");
+      }
+
+    }
+
     return ret;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
index 1429b2c..3cd1780 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
@@ -17,7 +17,7 @@
  */
 package org.apache.metron.enrichment.bolt;
 
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.enrichment.utils.ThreatIntelUtils;
 
 import java.util.HashMap;
@@ -35,7 +35,7 @@ public class ThreatIntelSplitterBolt extends EnrichmentSplitterBolt {
     if (sensorType != null) {
       SensorEnrichmentConfig config = configurations.getSensorEnrichmentConfig(sensorType);
       if (config != null) {
-        return config.getThreatIntelFieldMap();
+        return config.getThreatIntel().getFieldMap();
       } else {
         LOG.error("Unable to retrieve sensor config: " + sensorType);
       }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
new file mode 100644
index 0000000..bb4d387
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.threatintel.triage;
+
+import com.google.common.base.Function;
+import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig;
+import org.apache.metron.common.query.MapVariableResolver;
+import org.apache.metron.common.query.PredicateProcessor;
+import org.apache.metron.common.query.VariableResolver;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ThreatTriageProcessor implements Function<Map, Double> {
+  private ThreatTriageConfig config;
+  public ThreatTriageProcessor(ThreatTriageConfig config) {
+    this.config = config;
+  }
+
+  @Nullable
+  @Override
+  public Double apply(@Nullable Map input) {
+    List<Number> scores = new ArrayList<>();
+    PredicateProcessor predicateProcessor = new PredicateProcessor();
+    VariableResolver resolver = new MapVariableResolver(input);
+    for(Map.Entry<String, Number> kv : config.getRiskLevelRules().entrySet()) {
+      if(predicateProcessor.parse(kv.getKey(), resolver)) {
+        scores.add(kv.getValue());
+      }
+    }
+    return config.getAggregator().aggregate(scores, config.getAggregationConfig());
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
index 1c79f12..e2c6309 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
@@ -19,7 +19,7 @@ package org.apache.metron.enrichment.adapters.simplehbase;
 
 
 import org.adrianwalker.multilinestring.Multiline;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.enrichment.bolt.CacheKey;
 import org.apache.metron.enrichment.converter.EnrichmentKey;
 import org.apache.metron.enrichment.converter.EnrichmentValue;
@@ -52,26 +52,28 @@ public class SimpleHBaseAdapterTest {
   }};
 
   /**
-   * {
-   * "10.0.2.3.orientation":"north"
-   * }
+    {
+    "10.0.2.3.orientation":"north"
+    }
    */
   @Multiline
   private String expectedMessageString;
 
   /**
-   * {
-   * "index": "bro",
-   * "batchSize": 5,
-   * "enrichmentFieldMap": {
-   * "geo": ["ip_dst_addr", "ip_src_addr"],
-   * "host": ["host"]
-   * },
-   * "fieldToEnrichmentTypeMap": {
-   * "ip_dst_addr" : [ "10.0.2.3" ],
-   * "ip_src_addr" : [ "10.3.30.120" ]
-   * }
-   * }
+    {
+      "index": "bro",
+      "batchSize": 5,
+      "enrichment": {
+        "fieldMap": {
+          "geo": ["ip_dst_addr", "ip_src_addr"],
+          "host": ["host"]
+        },
+      "fieldToTypeMap": {
+        "ip_dst_addr" : [ "10.0.2.3" ],
+        "ip_src_addr" : [ "10.3.30.120" ]
+        }
+      }
+   }
    */
   @Multiline
   private String sourceConfigStr;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
index 62c8b43..2afeb5b 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
@@ -20,7 +20,7 @@ package org.apache.metron.enrichment.adapters.threatintel;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.enrichment.bolt.CacheKey;
 import org.apache.metron.hbase.TableProvider;
 import org.apache.metron.enrichment.converter.EnrichmentKey;
@@ -62,29 +62,33 @@ public class ThreatIntelAdapterTest {
   private EnrichmentLookup lookup;
 
   /**
-   * {
-   * "10.0.2.3":"alert"
-   * }
+    {
+    "10.0.2.3":"alert"
+    }
    */
   @Multiline
   private String expectedMessageString;
 
   /**
-   * {
-   * "index": "bro",
-   * "batchSize": 5,
-   * "enrichmentFieldMap": {
-   * "geo": ["ip_dst_addr", "ip_src_addr"],
-   * "host": ["host"]
-   * },
-   * "threatIntelFieldMap": {
-   * "hbaseThreatIntel": ["ip_dst_addr", "ip_src_addr"]
-   * },
-   * "fieldToThreatIntelTypeMap": {
-   * "ip_dst_addr" : [ "10.0.2.3" ],
-   * "ip_src_addr" : [ "malicious_ip" ]
-   * }
-   * }
+    {
+      "index": "bro",
+      "batchSize": 5,
+      "enrichment": {
+        "fieldMap": {
+          "geo": ["ip_dst_addr", "ip_src_addr"],
+          "host": ["host"]
+        }
+      },
+      "threatIntel" : {
+        "fieldMap": {
+          "hbaseThreatIntel": ["ip_dst_addr", "ip_src_addr"]
+        },
+        "fieldToTypeMap": {
+          "ip_dst_addr" : [ "10.0.2.3" ],
+          "ip_src_addr" : [ "malicious_ip" ]
+        }
+      }
+    }
    */
   @Multiline
   private static String sourceConfigStr;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
index d5a90fb..5a09f79 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
@@ -22,9 +22,9 @@ import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.TestConstants;
 import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
 import org.apache.metron.enrichment.configuration.Enrichment;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-import org.apache.metron.common.cli.ConfigurationsUtils;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
 import org.hamcrest.Description;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
index 306c3e1..d3a5414 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
@@ -17,8 +17,12 @@
  */
 package org.apache.metron.enrichment.bolt;
 
+import com.fasterxml.jackson.databind.JsonMappingException;
 import junit.framework.Assert;
 import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig;
+import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
@@ -78,12 +82,37 @@ public class ThreatIntelJoinBoltTest extends BaseEnrichmentBoltTest {
     alertMessage = (JSONObject) parser.parse(alertMessageString);
   }
 
-  @Test
-  public void test() throws IOException {
+  /**
+    {
+    "riskLevelRules" : {
+        "enrichedField1 == 'enrichedValue1'" : 10
+                          }
+   ,"aggregator" : "MAX"
+   }
+   */
+  @Multiline
+  private static String threatTriageConfigStr;
+
+  public void test(String threatTriageConfig, boolean badConfig) throws IOException {
     ThreatIntelJoinBolt threatIntelJoinBolt = new ThreatIntelJoinBolt("zookeeperUrl");
     threatIntelJoinBolt.setCuratorFramework(client);
     threatIntelJoinBolt.setTreeCache(cache);
-    threatIntelJoinBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(sampleSensorEnrichmentConfigPath));
+    SensorEnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(new FileInputStream(sampleSensorEnrichmentConfigPath), SensorEnrichmentConfig.class);
+    boolean withThreatTriage = threatTriageConfig != null;
+    if(withThreatTriage) {
+      try {
+        enrichmentConfig.getThreatIntel().setTriageConfig(JSONUtils.INSTANCE.load(threatTriageConfig, ThreatTriageConfig.class));
+        if(badConfig) {
+          Assert.fail(threatTriageConfig + "\nThis should not parse!");
+        }
+      }
+      catch(JsonMappingException pe) {
+        if(!badConfig) {
+          throw pe;
+        }
+      }
+    }
+    threatIntelJoinBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, enrichmentConfig);
     threatIntelJoinBolt.withMaxCacheSize(100);
     threatIntelJoinBolt.withMaxTimeRetain(10000);
     threatIntelJoinBolt.prepare(new HashMap<>(), topologyContext, outputCollector);
@@ -101,5 +130,35 @@ public class ThreatIntelJoinBoltTest extends BaseEnrichmentBoltTest {
     streamMessageMap.put("message", alertMessage);
     joinedMessage = threatIntelJoinBolt.joinMessages(streamMessageMap);
     Assert.assertTrue(joinedMessage.containsKey("is_alert") && "true".equals(joinedMessage.get("is_alert")));
+    if(withThreatTriage && !badConfig) {
+      Assert.assertTrue(joinedMessage.containsKey("threat.triage.level") && Math.abs(10d - (Double) joinedMessage.get("threat.triage.level")) < 1e-10);
+    }
+    else {
+      Assert.assertFalse(joinedMessage.containsKey("threat.triage.level"));
+    }
+  }
+  /**
+    {
+    "riskLevelRules" : {
+        "enrichedField1 == 'enrichedValue1" : 10
+                          }
+   ,"aggregator" : "MAX"
+   }
+   */
+  @Multiline
+  private static String badRuleThreatTriageConfigStr;
+
+
+  @Test
+  public void testWithTriage() throws IOException {
+    test(threatTriageConfigStr, false);
+  }
+  @Test
+  public void testWithBadTriageRule() throws IOException {
+    test(badRuleThreatTriageConfigStr, true);
+  }
+  @Test
+  public void testWithoutTriage() throws IOException {
+    test(null, false);
   }
 }


Mime
View raw message