metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject [1/2] metron git commit: METRON-1741 Move REPL Port of Profiler to Separate Project (nickwallen) closes apache/metron#1170
Date Tue, 04 Sep 2018 21:17:14 GMT
Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1699-create-batch-profiler f5eaef3c2 -> 9455c4ee3


http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-analytics/metron-profiler-repl/src/test/java/org/apache/metron/profiler/repl/StandAloneProfilerTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-repl/src/test/java/org/apache/metron/profiler/repl/StandAloneProfilerTest.java
b/metron-analytics/metron-profiler-repl/src/test/java/org/apache/metron/profiler/repl/StandAloneProfilerTest.java
new file mode 100644
index 0000000..94c8fc8
--- /dev/null
+++ b/metron-analytics/metron-profiler-repl/src/test/java/org/apache/metron/profiler/repl/StandAloneProfilerTest.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.repl;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.profiler.ProfileMeasurement;
+import org.apache.metron.stellar.dsl.Context;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests the StandAloneProfiler class.
+ */
+public class StandAloneProfilerTest {
+
+  /**
+   * {
+   *   "profiles": [
+   *   ]
+   * }
+   */
+  @Multiline
+  private String noProfiles;
+
+  /**
+   * {
+   *   "profiles": [
+   *      {
+   *        "profile": "profile1",
+   *        "foreach": "'global'",
+   *        "init": { "count": 0 },
+   *        "update": { "count": "count + 1" },
+   *        "result": "count"
+   *      }
+   *   ]
+   * }
+   */
+  @Multiline
+  private String oneProfile;
+
+  /**
+   * {
+   *   "profiles": [
+   *      {
+   *        "profile": "profile1",
+   *        "foreach": "'global1'",
+   *        "result": "'result'"
+   *      },
+   *      {
+   *        "profile": "profile2",
+   *        "foreach": "'global2'",
+   *        "result": "'result'"
+   *      }
+   *   ]
+   * }
+   */
+  @Multiline
+  private String twoProfiles;
+
+  /**
+   * {
+   *   "ip_src_addr": "10.0.0.1",
+   *   "ip_dst_addr": "10.0.0.20",
+   *   "protocol": "HTTP",
+   *   "timestamp": 2222222222222,
+   * }
+   */
+  @Multiline
+  private String messageJson;
+
+  private JSONObject message;
+
+  private long periodDurationMillis = TimeUnit.MINUTES.toMillis(15);
+
+  private Context context = Context.EMPTY_CONTEXT();
+
+  @Before
+  public void setup() throws Exception {
+
+    // parse the input message
+    JSONParser parser = new JSONParser();
+    message = (JSONObject) parser.parse(messageJson);
+  }
+
+  @Test
+  public void testWithOneProfile() throws Exception {
+
+    StandAloneProfiler profiler = createProfiler(oneProfile);
+    profiler.apply(message);
+    profiler.apply(message);
+    profiler.apply(message);
+
+    List<ProfileMeasurement> measurements = profiler.flush();
+    assertEquals(1, measurements.size());
+
+    // expect 1 measurement for the 1 profile that has been defined
+    ProfileMeasurement m = measurements.get(0);
+    assertEquals("profile1", m.getProfileName());
+    assertEquals(3, m.getProfileValue());
+  }
+
+
+  @Test
+  public void testWithTwoProfiles() throws Exception {
+
+    StandAloneProfiler profiler = createProfiler(twoProfiles);
+    profiler.apply(message);
+    profiler.apply(message);
+    profiler.apply(message);
+
+    List<ProfileMeasurement> measurements = profiler.flush();
+    assertEquals(2, measurements.size());
+
+    // expect 2 measurements, 1 for each profile
+    List<String> expected = Arrays.asList(new String[] { "profile1", "profile2" });
+    {
+      ProfileMeasurement m = measurements.get(0);
+      assertTrue(expected.contains(m.getProfileName()));
+      assertEquals("result", m.getProfileValue());
+    }
+    {
+      ProfileMeasurement m = measurements.get(1);
+      assertTrue(expected.contains(m.getProfileName()));
+      assertEquals("result", m.getProfileValue());
+    }
+  }
+
+  /**
+   * The message count and route count will always be equal, if there is only one
+   * profile defined.  The message count and route count can be different when there
+   * are multiple profiles defined that each use the same message.
+   */
+  @Test
+  public void testRouteAndMessageCounters() throws Exception {
+    {
+      StandAloneProfiler profiler = createProfiler(noProfiles);
+
+      profiler.apply(message);
+      assertEquals(1, profiler.getMessageCount());
+      assertEquals(0, profiler.getRouteCount());
+
+      profiler.apply(message);
+      assertEquals(2, profiler.getMessageCount());
+      assertEquals(0, profiler.getRouteCount());
+
+      profiler.apply(message);
+      assertEquals(3, profiler.getMessageCount());
+      assertEquals(0, profiler.getRouteCount());
+    }
+    {
+      StandAloneProfiler profiler = createProfiler(oneProfile);
+
+      profiler.apply(message);
+      assertEquals(1, profiler.getMessageCount());
+      assertEquals(1, profiler.getRouteCount());
+
+      profiler.apply(message);
+      assertEquals(2, profiler.getMessageCount());
+      assertEquals(2, profiler.getRouteCount());
+
+      profiler.apply(message);
+      assertEquals(3, profiler.getMessageCount());
+      assertEquals(3, profiler.getRouteCount());
+    }
+    {
+      StandAloneProfiler profiler = createProfiler(twoProfiles);
+
+      profiler.apply(message);
+      assertEquals(1, profiler.getMessageCount());
+      assertEquals(2, profiler.getRouteCount());
+
+      profiler.apply(message);
+      assertEquals(2, profiler.getMessageCount());
+      assertEquals(4, profiler.getRouteCount());
+
+      profiler.apply(message);
+      assertEquals(3, profiler.getMessageCount());
+      assertEquals(6, profiler.getRouteCount());
+    }
+  }
+
+  @Test
+  public void testProfileCount() throws Exception {
+    {
+      StandAloneProfiler profiler = createProfiler(noProfiles);
+      assertEquals(0, profiler.getProfileCount());
+    }
+    {
+      StandAloneProfiler profiler = createProfiler(oneProfile);
+      assertEquals(1, profiler.getProfileCount());
+    }
+    {
+      StandAloneProfiler profiler = createProfiler(twoProfiles);
+      assertEquals(2, profiler.getProfileCount());
+    }
+  }
+
+  /**
+   * Creates a ProfilerConfig based on a string containing JSON.
+   *
+   * @param configAsJSON The config as JSON.
+   * @return The ProfilerConfig.
+   * @throws Exception
+   */
+  private ProfilerConfig toProfilerConfig(String configAsJSON) throws Exception {
+
+    InputStream in = new ByteArrayInputStream(configAsJSON.getBytes("UTF-8"));
+    return JSONUtils.INSTANCE.load(in, ProfilerConfig.class);
+  }
+
+  /**
+   * Creates the StandAloneProfiler
+   *
+   * @param profileJson The Profiler configuration to use as a String containing JSON.
+   * @throws Exception
+   */
+  private StandAloneProfiler createProfiler(String profileJson) throws Exception {
+
+    // the TTL and max routes need not be bounded
+    long profileTimeToLiveMillis = Long.MAX_VALUE;
+    long maxNumberOfRoutes = Long.MAX_VALUE;
+
+    ProfilerConfig config = toProfilerConfig(profileJson);
+    return new StandAloneProfiler(config, periodDurationMillis, profileTimeToLiveMillis,
maxNumberOfRoutes, context);
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-analytics/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/pom.xml b/metron-analytics/pom.xml
index dc72f3b..37ee2b0 100644
--- a/metron-analytics/pom.xml
+++ b/metron-analytics/pom.xml
@@ -47,6 +47,7 @@
 		<module>metron-profiler-client</module>
 		<module>metron-profiler-common</module>
 		<module>metron-profiler-spark</module>
+		<module>metron-profiler-repl</module>
     </modules>
 	<dependencies>
 		<dependency>

http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index eae756a..a5a3a26 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -410,6 +410,9 @@
               <name>metron-profiler-spark</name>
             </package>
             <package>
+              <name>metron-profiler-repl</name>
+            </package>
+            <package>
               <name>metron-indexing</name>
             </package>
             <package>

http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-deployment/packaging/docker/deb-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/pom.xml b/metron-deployment/packaging/docker/deb-docker/pom.xml
index 90f7810..a0df09a 100644
--- a/metron-deployment/packaging/docker/deb-docker/pom.xml
+++ b/metron-deployment/packaging/docker/deb-docker/pom.xml
@@ -138,6 +138,12 @@
                                     </includes>
                                 </resource>
                                 <resource>
+                                    <directory>${metron_dir}/metron-analytics/metron-profiler-repl/target/</directory>
+                                    <includes>
+                                        <include>*.tar.gz</include>
+                                    </includes>
+                                </resource>
+                                <resource>
                                     <directory>${metron_dir}/metron-interface/metron-rest/target/</directory>
                                     <includes>
                                         <include>*.tar.gz</include>

http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 47bcad3..cee9cd9 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -59,6 +59,7 @@ Source12:       metron-maas-service-%{full_version}-archive.tar.gz
 Source13:       metron-alerts-%{full_version}-archive.tar.gz
 Source14:       metron-performance-%{full_version}-archive.tar.gz
 Source15:       metron-profiler-spark-%{full_version}-archive.tar.gz
+Source16:       metron-profiler-repl-%{full_version}-archive.tar.gz
 
 %description
 Apache Metron provides a scalable advanced security analytics framework
@@ -97,6 +98,7 @@ tar -xzf %{SOURCE12} -C %{buildroot}%{metron_home}
 tar -xzf %{SOURCE13} -C %{buildroot}%{metron_home}
 tar -xzf %{SOURCE14} -C %{buildroot}%{metron_home}
 tar -xzf %{SOURCE15} -C %{buildroot}%{metron_home}
+tar -xzf %{SOURCE16} -C %{buildroot}%{metron_home}
 
 install %{buildroot}%{metron_home}/bin/metron-management-ui %{buildroot}/etc/init.d/
 install %{buildroot}%{metron_home}/bin/metron-alerts-ui %{buildroot}/etc/init.d/
@@ -560,6 +562,23 @@ This package installs the Metron Profiler for Spark %{metron_home}
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+%package        profiler-repl
+Summary:        Metron Profiler for the Stellar REPL
+Group:          Applications/Internet
+Provides:       profiler-repl = %{version}
+
+%description    profiler-repl
+This package installs the Metron Profiler for the Stellar REPL %{metron_home}
+
+%files          profiler-repl
+%defattr(-,root,root,755)
+%dir %{metron_root}
+%dir %{metron_home}
+%dir %{metron_home}/lib
+%attr(0644,root,root) %{metron_home}/lib/metron-profiler-repl-%{full_version}.jar
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 %post config
 chkconfig --add metron-management-ui
 chkconfig --add metron-alerts-ui
@@ -569,6 +588,8 @@ chkconfig --del metron-management-ui
 chkconfig --del metron-alerts-ui
 
 %changelog
+* Tue Aug 21 2018 Apache Metron <dev@metron.apache.org> - 0.5.1
+- Add Profiler for REPL
 * Tue Aug 14 2018 Apache Metron <dev@metron.apache.org> - 0.5.1
 - Add Profiler for Spark
 * Thu Feb 1 2018 Apache Metron <dev@metron.apache.org> - 0.4.3

http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-deployment/packaging/docker/rpm-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index 1ea8d46..0d9c4d1 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -174,6 +174,12 @@
                                     </includes>
                                 </resource>
                                 <resource>
+                                    <directory>${metron_dir}/metron-analytics/metron-profiler-repl/target/</directory>
+                                    <includes>
+                                        <include>*.tar.gz</include>
+                                    </includes>
+                                </resource>
+                                <resource>
                                     <directory>${metron_dir}/metron-interface/metron-rest/target/</directory>
                                     <includes>
                                         <include>*.tar.gz</include>

http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-platform/metron-common/src/main/scripts/stellar
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/scripts/stellar b/metron-platform/metron-common/src/main/scripts/stellar
index 431ed7b..c831c62 100644
--- a/metron-platform/metron-common/src/main/scripts/stellar
+++ b/metron-platform/metron-common/src/main/scripts/stellar
@@ -33,4 +33,5 @@ export METRON_VERSION=${project.version}
 export METRON_HOME=/usr/metron/$METRON_VERSION
 export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar)
 export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar)
-java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS"
org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
+export PROFILER_LIB=$(find $METRON_HOME/lib/ -name metron-profiler-repl*.jar)
+java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$PROFILER_LIB:$HBASE_CONFIGS"
org.apache.metron.stellar.common.shell.cli.StellarShell "$@"


Mime
View raw message