metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject [metron] branch master updated: METRON-1986 Batch Profiler Fails to Resolve Stats Stellar Functions (nickwallen) closes apache/metron#1328
Date Thu, 14 Feb 2019 22:33:44 GMT
This is an automated email from the ASF dual-hosted git repository.

nickallen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git


The following commit(s) were added to refs/heads/master by this push:
     new 2283487  METRON-1986 Batch Profiler Fails to Resolve Stats Stellar Functions (nickwallen)
closes apache/metron#1328
2283487 is described below

commit 22834879017355fa456f42a6e71a3ed44d0b43fe
Author: nickwallen <nick@nickallen.org>
AuthorDate: Thu Feb 14 17:33:22 2019 -0500

    METRON-1986 Batch Profiler Fails to Resolve Stats Stellar Functions (nickwallen) closes
apache/metron#1328
---
 metron-analytics/metron-profiler-spark/pom.xml     | 37 ++++++++--
 .../spark/BatchProfilerIntegrationTest.java        | 83 ++++++++++++++++------
 2 files changed, 90 insertions(+), 30 deletions(-)

diff --git a/metron-analytics/metron-profiler-spark/pom.xml b/metron-analytics/metron-profiler-spark/pom.xml
index 000d884..40bd551 100644
--- a/metron-analytics/metron-profiler-spark/pom.xml
+++ b/metron-analytics/metron-profiler-spark/pom.xml
@@ -52,7 +52,6 @@
             <groupId>org.apache.metron</groupId>
             <artifactId>metron-profiler-client</artifactId>
             <version>${project.parent.version}</version>
-            <scope>test</scope>
             <exclusions>
                 <exclusion>
                     <groupId>org.antlr</groupId>
@@ -62,6 +61,11 @@
         </dependency>
         <dependency>
             <groupId>org.apache.metron</groupId>
+            <artifactId>stellar-common</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
             <artifactId>metron-common</artifactId>
             <version>${project.parent.version}</version>
             <exclusions>
@@ -182,8 +186,7 @@
                                 </excludes>
                             </artifactSet>
                             <transformers>
-                                <transformer
-                                        implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
                                     <resources>
                                         <resource>.yaml</resource>
                                         <resource>LICENSE.txt</resource>
@@ -191,16 +194,36 @@
                                         <resource>NOTICE.txt</resource>
                                     </resources>
                                 </transformer>
-                                <transformer
-                                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
-                                <transformer
-                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                     <mainClass></mainClass>
                                 </transformer>
+                                <!--
+                                    The ClassIndex library is used to create an index of
all Stellar functions at
+                                    compile-time. This index is packaged in the JAR as
+                                    `META-INF/annotations/org.apache.metron.stellar.dsl.Stellar`.
+
+                                    When creating a shaded jar using the Maven Shade Plugin,
the index files created by
+                                    ClassIndex are overwritten by default. ClassIndex provides
a special transformer for
+                                    Maven which merges the index files instead of overwriting
them.
+
+                                    Without using this transformer, the index files are overwritten
and Stellar function
+                                    resolution will not work.
+
+                                    https://github.com/atteo/classindex#making-shaded-jar
+                                    -->
+                                <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
                             </transformers>
                         </configuration>
                     </execution>
                 </executions>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.atteo.classindex</groupId>
+                        <artifactId>classindex-transformer</artifactId>
+                        <version>${global_classindex_version}</version>
+                    </dependency>
+                </dependencies>
             </plugin>
             <plugin>
                 <artifactId>maven-assembly-plugin</artifactId>
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
index 9ea151a..b36cf8c 100644
--- a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
@@ -70,29 +70,6 @@ import static org.junit.Assert.assertTrue;
 public class BatchProfilerIntegrationTest {
 
   private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-  /**
-   * {
-   *   "timestampField": "timestamp",
-   *   "profiles": [
-   *      {
-   *        "profile": "count-by-ip",
-   *        "foreach": "ip_src_addr",
-   *        "init": { "count": 0 },
-   *        "update": { "count" : "count + 1" },
-   *        "result": "count"
-   *      },
-   *      {
-   *        "profile": "total-count",
-   *        "foreach": "'total'",
-   *        "init": { "count": 0 },
-   *        "update": { "count": "count + 1" },
-   *        "result": "count"
-   *      }
-   *   ]
-   * }
-   */
-  @Multiline
-  private static String profileJson;
   private static SparkSession spark;
   private Properties profilerProperties;
   private Properties readerProperties;
@@ -152,6 +129,30 @@ public class BatchProfilerIntegrationTest {
   }
 
   /**
+   * {
+   *   "timestampField": "timestamp",
+   *   "profiles": [
+   *      {
+   *        "profile": "count-by-ip",
+   *        "foreach": "ip_src_addr",
+   *        "init": { "count": 0 },
+   *        "update": { "count" : "count + 1" },
+   *        "result": "count"
+   *      },
+   *      {
+   *        "profile": "total-count",
+   *        "foreach": "'total'",
+   *        "init": { "count": 0 },
+   *        "update": { "count": "count + 1" },
+   *        "result": "count"
+   *      }
+   *   ]
+   * }
+   */
+  @Multiline
+  private static String profileJson;
+
+  /**
    * This test uses the Batch Profiler to seed two profiles using archived telemetry.
    *
    * The first profile counts the number of messages by 'ip_src_addr'.  The second profile
counts the total number
@@ -328,6 +329,42 @@ public class BatchProfilerIntegrationTest {
   }
 
   /**
+    * {
+    *   "timestampField": "timestamp",
+    *   "profiles": [
+    *      {
+    *        "profile": "count-by-ip",
+    *        "foreach": "ip_src_addr",
+    *        "init": { "count": "STATS_INIT()" },
+    *        "update": { "count" : "STATS_ADD(count, 1)" },
+    *        "result": "TO_INTEGER(STATS_COUNT(count))"
+    *      },
+    *      {
+    *        "profile": "total-count",
+    *        "foreach": "'total'",
+    *        "init": { "count": "STATS_INIT()" },
+    *        "update": { "count": "STATS_ADD(count, 1)" },
+    *        "result": "TO_INTEGER(STATS_COUNT(count))"
+    *      }
+    *   ]
+    * }
+    */
+  @Multiline
+  private static String statsProfileJson;
+
+  @Test
+  public void testBatchProfilerWithStatsFunctions() throws Exception {
+    profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), JSON.toString());
+    profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), "src/test/resources/telemetry.json");
+
+    BatchProfiler profiler = new BatchProfiler();
+    profiler.run(spark, profilerProperties, getGlobals(), readerProperties, fromJSON(statsProfileJson));
+
+    // the profiles do the exact same counting, but using the STATS functions
+    validateProfiles();
+  }
+
+  /**
    * Validates the profiles that were built.
    *
    * These tests use the Batch Profiler to seed two profiles with archived telemetry.  The
first profile


Mime
View raw message