metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject [07/52] [abbrv] metron git commit: METRON-1471: Migrate shuffle connections to local or shuffle closes apache/metron#949
Date Wed, 18 Apr 2018 14:59:37 GMT
METRON-1471: Migrate shuffle connections to local or shuffle closes apache/metron#949


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8bc7732e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8bc7732e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8bc7732e

Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual
Commit: 8bc7732e4c904085e4f78469abded84635971632
Parents: abb152b
Author: cstella <cestella@gmail.com>
Authored: Thu Mar 8 17:17:57 2018 -0500
Committer: cstella <cestella@gmail.com>
Committed: Thu Mar 8 17:17:57 2018 -0500

----------------------------------------------------------------------
 .../metron-profiler/src/main/flux/profiler/remote.yaml         | 6 +++---
 .../metron-indexing/src/main/flux/indexing/batch/remote.yaml   | 4 ++--
 .../src/main/flux/indexing/random_access/remote.yaml           | 4 ++--
 .../apache/metron/parsers/topology/ParserTopologyBuilder.java  | 4 ++--
 4 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
index 752640a..9ec5ba4 100644
--- a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
+++ b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
@@ -168,7 +168,7 @@ streams:
         from: "kafkaSpout"
         to: "splitterBolt"
         grouping:
-            type: SHUFFLE
+            type: LOCAL_OR_SHUFFLE
 
     -   name: "splitter -> builder"
         from: "splitterBolt"
@@ -182,11 +182,11 @@ streams:
         to: "hbaseBolt"
         grouping:
             streamId: "hbase"
-            type: SHUFFLE
+            type: LOCAL_OR_SHUFFLE
 
     -   name: "builder -> kafka"
         from: "builderBolt"
         to: "kafkaBolt"
         grouping:
             streamId: "kafka"
-            type: SHUFFLE
+            type: LOCAL_OR_SHUFFLE

http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml b/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
index 85e3baa..3087dd9 100644
--- a/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
+++ b/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
@@ -158,7 +158,7 @@ streams:
         from: "kafkaSpout"
         to: "hdfsIndexingBolt"
         grouping:
-            type: SHUFFLE
+            type: LOCAL_OR_SHUFFLE
 
 
     -   name: "hdfsBolt -> errorIndexingBolt"
@@ -166,4 +166,4 @@ streams:
         to: "indexingErrorBolt"
         grouping:
             streamId: "error"
-            type: SHUFFLE
+            type: LOCAL_OR_SHUFFLE

http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
b/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
index cadc1ec..429ba45 100644
--- a/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
+++ b/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
@@ -130,11 +130,11 @@ streams:
         from: "kafkaSpout"
         to: "indexingBolt"
         grouping:
-            type: SHUFFLE
+            type: LOCAL_OR_SHUFFLE
 
     -   name: "indexingBolt -> errorIndexingBolt"
         from: "indexingBolt"
         to: "indexingErrorBolt"
         grouping:
             streamId: "error"
-            type: SHUFFLE
+            type: LOCAL_OR_SHUFFLE

http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
index 1039e56..2865dd6 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
@@ -126,14 +126,14 @@ public class ParserTopologyBuilder {
     ParserBolt parserBolt = createParserBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol,
configs, parserConfig, outputTopic);
     builder.setBolt("parserBolt", parserBolt, parserParallelism)
             .setNumTasks(parserNumTasks)
-            .shuffleGrouping("kafkaSpout");
+            .localOrShuffleGrouping("kafkaSpout");
 
     // create the error bolt, if needed
     if (errorWriterNumTasks > 0) {
       WriterBolt errorBolt = createErrorBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol,
configs, parserConfig);
       builder.setBolt("errorMessageWriter", errorBolt, errorWriterParallelism)
               .setNumTasks(errorWriterNumTasks)
-              .shuffleGrouping("parserBolt", Constants.ERROR_STREAM);
+              .localOrShuffleGrouping("parserBolt", Constants.ERROR_STREAM);
     }
 
     return new ParserTopology(builder, stormConfigSupplier.get(parserConfig, Config.class));


Mime
View raw message