metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject incubator-metron git commit: METRON-668: Remove the "tickUpdate" profile config and make the "init" phase not reset variables closes apache/incubator-metron#420
Date Thu, 26 Jan 2017 13:52:02 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master 0219e56a9 -> c74c7de27


METRON-668: Remove the "tickUpdate" profile config and make the "init" phase not reset variables
closes apache/incubator-metron#420


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/c74c7de2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/c74c7de2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/c74c7de2

Branch: refs/heads/master
Commit: c74c7de27007d192e55c62f84e9b1b9089fd9cdf
Parents: 0219e56
Author: cstella <cestella@gmail.com>
Authored: Thu Jan 26 08:51:45 2017 -0500
Committer: cstella <cestella@gmail.com>
Committed: Thu Jan 26 08:51:45 2017 -0500

----------------------------------------------------------------------
 .../apache/metron/profiler/ProfileBuilder.java  | 13 ----
 .../stellar/DefaultStellarExecutor.java         |  6 ++
 .../metron/profiler/ProfileBuilderTest.java     | 81 ++++++++++----------
 .../profiler/bolt/ProfileBuilderBolt.java       |  6 +-
 metron-analytics/metron-statistics/README.md    |  4 -
 .../configuration/profiler/ProfileConfig.java   | 18 -----
 .../SolrIndexingIntegrationTest.java            | 10 +--
 7 files changed, 54 insertions(+), 84 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c74c7de2/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java
index f9beaeb..4c38fac 100644
--- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java
+++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java
@@ -152,19 +152,6 @@ public class ProfileBuilder implements Serializable {
     // execute the 'groupBy' expression(s) - can refer to value of 'result' expression
     List<Object> groups = execute(definition.getGroupBy(), ImmutableMap.of("result",
value), "groupBy");
 
-    // execute the 'tickUpdate' expression(s) - can refer to value of 'result' expression
-    assign(definition.getTickUpdate(), ImmutableMap.of("result", value),"tickUpdate");
-
-    // save a copy of current state then clear it to prepare for the next window
-    Map<String, Object> state = executor.getState();
-    executor.clearState();
-
-    // the 'tickUpdate' state is not flushed - make sure to bring that state along to the
next period
-    definition.getTickUpdate().forEach((var, expr) -> {
-      Object val = state.get(var);
-      executor.assign(var, val);
-    });
-
     isInitialized = false;
 
     return new ProfileMeasurement()

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c74c7de2/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java
index 27d23e2..62bab85 100644
--- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java
+++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java
@@ -94,11 +94,17 @@ public class DefaultStellarExecutor implements StellarExecutor, Serializable
{
   @Override
   public void assign(String variable, String expression, Map<String, Object> transientState)
{
     Object result = execute(expression, transientState);
+    if(result == null || variable == null) {
+      return;
+    }
     state.put(variable, result);
   }
 
   @Override
   public void assign(String variable, Object value) {
+    if(value == null || variable == null) {
+      return;
+    }
     state.put(variable, value);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c74c7de2/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java
b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java
index c9873af..1434353 100644
--- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java
+++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java
@@ -269,8 +269,8 @@ public class ProfileBuilderTest {
    *   "profile": "test",
    *   "foreach": "ip_src_addr",
    *   "init": {
-   *     "x": "0",
-   *     "y": "0"
+   *     "x": "if exists(x) then x else 0",
+   *     "y": "if exists(y) then y else 0"
    *   },
    *   "update": {
    *     "x": "x + 1",
@@ -283,7 +283,7 @@ public class ProfileBuilderTest {
   private String testFlushProfile;
 
   @Test
-  public void testFlushClearsState() throws Exception {
+  public void testFlushDoesNotClearsState() throws Exception {
     // setup
     definition = JSONUtils.INSTANCE.load(testFlushProfile, ProfileConfig.class);
     builder = new ProfileBuilder.Builder()
@@ -304,80 +304,81 @@ public class ProfileBuilderTest {
     ProfileMeasurement m = builder.flush();
 
     // validate
-    assertEquals(3, (int) convert(m.getValue(), Integer.class));
+    assertEquals(33, (int) convert(m.getValue(), Integer.class));
   }
 
   /**
    * {
    *   "profile": "test",
    *   "foreach": "ip_src_addr",
-   *   "result": "100"
+   *   "init": {
+   *     "x": "0",
+   *     "y": "0"
+   *   },
+   *   "update": {
+   *     "x": "x + 1",
+   *     "y": "y + 2"
+   *   },
+   *   "result": "x + y"
    * }
    */
   @Multiline
-  private String testEntityProfile;
+  private String testFlushProfileWithNaiveInit;
 
-  /**
-   * Ensure that the entity is correctly set on the resulting profile measurements.
-   */
   @Test
-  public void testEntity() throws Exception {
+  public void testFlushDoesNotClearsStateButInitDoes() throws Exception {
     // setup
-    final String entity = "10.0.0.1";
-    definition = JSONUtils.INSTANCE.load(testFlushProfile, ProfileConfig.class);
+    definition = JSONUtils.INSTANCE.load(testFlushProfileWithNaiveInit, ProfileConfig.class);
     builder = new ProfileBuilder.Builder()
             .withDefinition(definition)
-            .withEntity(entity)
+            .withEntity("10.0.0.1")
             .withPeriodDuration(10, TimeUnit.MINUTES)
             .build();
 
-    // execute
+    // execute - accumulate some state then flush it
+    int count = 10;
+    for(int i=0; i<count; i++) {
+      builder.apply(message);
+    }
+    builder.flush();
+
+    // apply another message to accumulate new state, then flush again to validate original
state was cleared
     builder.apply(message);
     ProfileMeasurement m = builder.flush();
 
     // validate
-    assertEquals(entity, m.getEntity());
+    assertEquals(3, (int) convert(m.getValue(), Integer.class));
   }
-
   /**
    * {
    *   "profile": "test",
    *   "foreach": "ip_src_addr",
-   *   "tickUpdate": {
-   *     "ticks": "ticks + 1"
-   *   },
-   *   "result": "if exists(ticks) then ticks else 0"
+   *   "result": "100"
    * }
    */
   @Multiline
-  private String testTickUpdateProfile;
+  private String testEntityProfile;
 
+  /**
+   * Ensure that the entity is correctly set on the resulting profile measurements.
+   */
   @Test
-  public void testTickUpdate() throws Exception {
+  public void testEntity() throws Exception {
     // setup
-    definition = JSONUtils.INSTANCE.load(testTickUpdateProfile, ProfileConfig.class);
+    final String entity = "10.0.0.1";
+    definition = JSONUtils.INSTANCE.load(testFlushProfile, ProfileConfig.class);
     builder = new ProfileBuilder.Builder()
             .withDefinition(definition)
-            .withEntity("10.0.0.1")
+            .withEntity(entity)
             .withPeriodDuration(10, TimeUnit.MINUTES)
             .build();
 
-    // 'tickUpdate' only executed when flushed - 'result' only has access to the 'old' tick
value, not latest
-    {
-      ProfileMeasurement m = builder.flush();
-      assertEquals(0, (int) convert(m.getValue(), Integer.class));
-    }
-
-    // execute many flushes
-    int count = 10;
-    for(int i=0; i<count; i++) {
-      builder.flush();
-    }
+    // execute
+    builder.apply(message);
+    ProfileMeasurement m = builder.flush();
 
-    {
-      // validate - the tickUpdate state should not be cleared between periods and is only
run once per period
-      ProfileMeasurement m = builder.flush();
-      assertEquals(11, (int) convert(m.getValue(), Integer.class));
-    }
+    // validate
+    assertEquals(entity, m.getEntity());
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c74c7de2/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
index 1fcba30..a22361d 100644
--- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
+++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
@@ -157,8 +157,10 @@ public class ProfileBuilderBolt extends ConfiguredProfilerBolt {
 
       // when a 'tick' is received, flush the profile and emit the completed profile measurement
       profileCache.asMap().forEach((key, profileBuilder) -> {
-        ProfileMeasurement measurement = profileBuilder.flush();
-        collector.emit(new Values(measurement, profileBuilder.getDefinition()));
+        if(profileBuilder.isInitialized()) {
+          ProfileMeasurement measurement = profileBuilder.flush();
+          collector.emit(new Values(measurement, profileBuilder.getDefinition()));
+        }
       });
 
       // cache maintenance

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c74c7de2/metron-analytics/metron-statistics/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-statistics/README.md b/metron-analytics/metron-statistics/README.md
index 346179b..e654097 100644
--- a/metron-analytics/metron-statistics/README.md
+++ b/metron-analytics/metron-statistics/README.md
@@ -355,10 +355,6 @@ Create the following file at
         "s": "OUTLIER_MAD_STATE_MERGE(PROFILE_GET('sketchy_mad',
 'global', 5, 'MINUTES'))"
                },
-      "tickUpdate": {
-        "s": "OUTLIER_MAD_STATE_MERGE(PROFILE_GET('sketchy_mad',
-'global', 5, 'MINUTES'), s)"
-                },
       "update": {
         "s": "OUTLIER_MAD_ADD(s, value)"
                 },

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c74c7de2/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
index d6f1a37..d8f8acf 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
@@ -64,13 +64,6 @@ public class ProfileConfig implements Serializable {
    */
   private Map<String, String> update = new HashMap<>();
 
-   /**
-   * A set of expressions that is executed when a tick happens.
-   * A map is expected where the key is the variable name and the value is a Stellar
-   * expression.  The map can include 0 or more variables/expressions.
-   */
-  private Map<String, String> tickUpdate = new LinkedHashMap<>();
-
   /**
    * A list of Stellar expressions that is executed in order and used to group the
    * resulting profile data.
@@ -131,14 +124,6 @@ public class ProfileConfig implements Serializable {
     this.update = update;
   }
 
-  public Map<String, String> getTickUpdate() {
-    return tickUpdate;
-  }
-
-  public void setTickUpdate(Map<String, String> tickUpdate) {
-    this.tickUpdate = tickUpdate;
-  }
-
   public List<String> getGroupBy() {
     return groupBy;
   }
@@ -171,7 +156,6 @@ public class ProfileConfig implements Serializable {
             ", onlyif='" + onlyif + '\'' +
             ", init=" + init +
             ", update=" + update +
-            ", tickUpdate=" + tickUpdate +
             ", groupBy=" + groupBy +
             ", result='" + result + '\'' +
             ", expires=" + expires +
@@ -190,7 +174,6 @@ public class ProfileConfig implements Serializable {
     if (onlyif != null ? !onlyif.equals(that.onlyif) : that.onlyif != null) return false;
     if (init != null ? !init.equals(that.init) : that.init != null) return false;
     if (update != null ? !update.equals(that.update) : that.update != null) return false;
-    if (tickUpdate != null ? !tickUpdate.equals(that.tickUpdate) : that.tickUpdate != null)
return false;
     if (groupBy != null ? !groupBy.equals(that.groupBy) : that.groupBy != null) return false;
     if (result != null ? !result.equals(that.result) : that.result != null) return false;
     return expires != null ? expires.equals(that.expires) : that.expires == null;
@@ -204,7 +187,6 @@ public class ProfileConfig implements Serializable {
     result1 = 31 * result1 + (onlyif != null ? onlyif.hashCode() : 0);
     result1 = 31 * result1 + (init != null ? init.hashCode() : 0);
     result1 = 31 * result1 + (update != null ? update.hashCode() : 0);
-    result1 = 31 * result1 + (tickUpdate != null ? tickUpdate.hashCode() : 0);
     result1 = 31 * result1 + (groupBy != null ? groupBy.hashCode() : 0);
     result1 = 31 * result1 + (result != null ? result.hashCode() : 0);
     result1 = 31 * result1 + (expires != null ? expires.hashCode() : 0);

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c74c7de2/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
index a504e52..abf3a8a 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
@@ -20,19 +20,15 @@ package org.apache.metron.solr.integration;
 import com.google.common.base.Function;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
 import org.apache.metron.common.interfaces.FieldNameConverter;
+import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.enrichment.integration.utils.SampleUtil;
 import org.apache.metron.indexing.integration.IndexingIntegrationTest;
-import org.apache.metron.integration.ComponentRunner;
-import org.apache.metron.integration.InMemoryComponent;
-import org.apache.metron.integration.Processor;
-import org.apache.metron.integration.ProcessorResult;
-import org.apache.metron.integration.ReadinessState;
+import org.apache.metron.integration.*;
 import org.apache.metron.integration.components.KafkaComponent;
 import org.apache.metron.integration.components.ZKServerComponent;
 import org.apache.metron.solr.integration.components.SolrComponent;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.utils.JSONUtils;
 
 import javax.annotation.Nullable;
 import java.io.IOException;


Mime
View raw message