metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject metron git commit: METRON-1624 Set Profiler and Enrichment batch parameters in Ambari (nickwallen) closes apache/metron#1069
Date Fri, 22 Jun 2018 20:20:01 GMT
Repository: metron
Updated Branches:
  refs/heads/master b575da15c -> b06bb5f48


METRON-1624 Set Profiler and Enrichment batch parameters in Ambari (nickwallen) closes apache/metron#1069


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b06bb5f4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b06bb5f4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b06bb5f4

Branch: refs/heads/master
Commit: b06bb5f4860c656023b9c357dfa83c2f0244b9ef
Parents: b575da1
Author: nickwallen <nick@nickallen.org>
Authored: Fri Jun 22 16:19:37 2018 -0400
Committer: nickallen <nickallen@apache.org>
Committed: Fri Jun 22 16:19:37 2018 -0400

----------------------------------------------------------------------
 .../configuration/metron-enrichment-env.xml     | 12 ++++++
 .../configuration/metron-profiler-env.xml       | 12 ++++++
 .../CURRENT/package/scripts/metron_service.py   | 20 ++++++++++
 .../package/scripts/params/params_linux.py      |  4 ++
 .../METRON/CURRENT/themes/metron_theme.json     | 41 +++++++++++++++++++-
 5 files changed, 88 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/b06bb5f4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
index 81b135c..b41c455 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
@@ -89,6 +89,18 @@
     <value>indexing</value>
     <display-name>Threat Intel Error Topic</display-name>
   </property>
+  <property>
+    <name>enrichment_kafka_writer_batch_size</name>
+    <value>15</value>
+    <description>The number of records to batch when writing to Kakfa.</description>
+    <display-name>Kafka Writer Batch Size</display-name>
+  </property>
+  <property>
+    <name>enrichment_kafka_writer_batch_timeout</name>
+    <value>0</value>
+    <description>The timeout in milliseconds after which a batch will be written to
Kafka, even if the batch size has not been met.  If unspecified, or set to `0`, this defaults
to a system-determined duration.</description>
+    <display-name>Kafka Writer Batch Timeout</display-name>
+  </property>
 
   <!--
     hbase parameters

http://git-wip-us.apache.org/repos/asf/metron/blob/b06bb5f4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml
index 8546b56..07b8d11 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml
@@ -221,4 +221,16 @@
     <description>The profiler storm topology acker executors</description>
     <display-name>Number of Acker Executors</display-name>
   </property>
+  <property>
+    <name>profiler_kafka_writer_batch_size</name>
+    <value>15</value>
+    <description>The number of records to batch when writing to Kakfa.</description>
+    <display-name>Kafka Writer Batch Size</display-name>
+  </property>
+  <property>
+    <name>profiler_kafka_writer_batch_timeout</name>
+    <value>0</value>
+    <description>The timeout in milliseconds after which a batch will be written to
Kafka, even if the batch size has not been met.  If unspecified, or set to `0`, this defaults
to a system-determined duration.</description>
+    <display-name>Kafka Writer Batch Timeout</display-name>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/metron/blob/b06bb5f4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
index 1999992..e3e48ab 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@ -119,6 +119,26 @@ def build_global_config_patch(params, patch_file):
         "op": "add",
         "path": "/threat.triage.score.field",
         "value": "{{threat_triage_score_field}}"
+    },
+    {
+        "op": "add",
+        "path": "/enrichment.writer.batchSize",
+        "value": "{{enrichment_kafka_writer_batch_size}}"
+    },
+    {
+        "op": "add",
+        "path": "/enrichment.writer.batchTimeout",
+        "value": "{{enrichment_kafka_writer_batch_timeout}}"
+    },
+    {
+        "op": "add",
+        "path": "/profiler.writer.batchSize",
+        "value": "{{profiler_kafka_writer_batch_size}}"
+    },
+    {
+        "op": "add",
+        "path": "/profiler.writer.batchTimeout",
+        "value": "{{profiler_kafka_writer_batch_timeout}}"
     }
   ]
   """

http://git-wip-us.apache.org/repos/asf/metron/blob/b06bb5f4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index f054c00..ec01292 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -268,6 +268,8 @@ enrichment_input_topic = status_params.enrichment_input_topic
 enrichment_output_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic']
 enrichment_error_topic = config['configurations']['metron-enrichment-env']['enrichment_error_topic']
 threatintel_error_topic = config['configurations']['metron-enrichment-env']['threatintel_error_topic']
+enrichment_kafka_writer_batch_size = config['configurations']['metron-enrichment-env']['enrichment_kafka_writer_batch_size']
+enrichment_kafka_writer_batch_timeout = config['configurations']['metron-enrichment-env']['enrichment_kafka_writer_batch_timeout']
 
 # Enrichment - Storm common parameters
 enrichment_workers = config['configurations']['metron-enrichment-env']['enrichment_workers']
@@ -328,6 +330,8 @@ profiler_window_lag=config['configurations']['metron-profiler-env']['profiler_wi
 profiler_window_lag_units=config['configurations']['metron-profiler-env']['profiler_window_lag_units']
 profiler_topology_message_timeout_secs=config['configurations']['metron-profiler-env']['profiler_topology_message_timeout_secs']
 profiler_topology_max_spout_pending=config['configurations']['metron-profiler-env']['profiler_topology_max_spout_pending']
+profiler_kafka_writer_batch_size = config['configurations']['metron-profiler-env']['profiler_kafka_writer_batch_size']
+profiler_kafka_writer_batch_timeout = config['configurations']['metron-profiler-env']['profiler_kafka_writer_batch_timeout']
 
 # Indexing
 ra_indexing_kafka_start = config['configurations']['metron-indexing-env']['ra_indexing_kafka_start']

http://git-wip-us.apache.org/repos/asf/metron/blob/b06bb5f4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index d09f270..d757ec6 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -484,6 +484,14 @@
           "subsection-name": "subsection-enrichment-kafka"
         },
         {
+          "config": "metron-enrichment-env/enrichment_kafka_writer_batch_size",
+          "subsection-name": "subsection-enrichment-kafka"
+        },
+        {
+          "config": "metron-enrichment-env/enrichment_kafka_writer_batch_timeout",
+          "subsection-name": "subsection-enrichment-kafka"
+        },
+        {
           "config": "metron-enrichment-env/enrichment_workers",
           "subsection-name": "subsection-enrichment-storm"
         },
@@ -668,6 +676,14 @@
           "subsection-name": "subsection-profiler-kafka"
         },
         {
+          "config": "metron-profiler-env/profiler_kafka_writer_batch_size",
+          "subsection-name": "subsection-profiler-kafka"
+        },
+        {
+          "config": "metron-profiler-env/profiler_kafka_writer_batch_timeout",
+          "subsection-name": "subsection-profiler-kafka"
+        },
+        {
           "config": "metron-profiler-env/profiler_period_duration",
           "subsection-name": "subsection-profiler-setup"
         },
@@ -864,7 +880,6 @@
           "type": "text-field"
         }
       },
-
       {
         "config": "metron-enrichment-env/geoip_url",
         "widget": {
@@ -908,6 +923,18 @@
         }
       },
       {
+        "config": "metron-enrichment-env/enrichment_kafka_writer_batch_size",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-enrichment-env/enrichment_kafka_writer_batch_timeout",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
         "config": "metron-enrichment-env/enrichment_workers",
         "widget": {
           "type": "text-field"
@@ -1186,6 +1213,18 @@
         }
       },
       {
+        "config": "metron-profiler-env/profiler_kafka_writer_batch_size",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-profiler-env/profiler_kafka_writer_batch_timeout",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
         "config": "metron-profiler-env/profiler_period_duration",
         "widget": {
           "type": "text-field"


Mime
View raw message