metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject [09/30] metron git commit: METRON-1708 Run the Batch Profiler in Spark (nickwallen) closes apache/metron#1161
Date Fri, 28 Sep 2018 13:09:05 GMT
METRON-1708 Run the Batch Profiler in Spark (nickwallen) closes apache/metron#1161


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c6d0721b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c6d0721b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c6d0721b

Branch: refs/heads/master
Commit: c6d0721b8f02752cea9f91acb29042ca9ddf0bc9
Parents: 3bfbf01
Author: nickwallen <nick@nickallen.org>
Authored: Mon Aug 27 13:47:28 2018 -0400
Committer: nickallen <nickallen@apache.org>
Committed: Mon Aug 27 13:47:28 2018 -0400

----------------------------------------------------------------------
 metron-analytics/metron-profiler-spark/pom.xml  |  15 ++
 .../src/main/assembly/assembly.xml              |  60 +++++++
 .../src/main/config/batch-profiler.properties   |  20 +++
 .../profiler/spark/cli/BatchProfilerCLI.java    | 174 +++++++++++++++++++
 .../spark/cli/BatchProfilerCLIOptions.java      | 144 +++++++++++++++
 .../src/main/scripts/start_batch_profiler.sh    |  32 ++++
 .../spark/cli/BatchProfilerCLITest.java         |  61 +++++++
 .../src/test/resources/globals.json             |   3 +
 .../src/test/resources/profiles-empty.json      |   5 +
 .../resources/profiles-no-timestamp-field.json  |  12 ++
 .../src/test/resources/profiles.json            |  13 ++
 11 files changed, 539 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/pom.xml b/metron-analytics/metron-profiler-spark/pom.xml
index 93ce08a..2d5ec98 100644
--- a/metron-analytics/metron-profiler-spark/pom.xml
+++ b/metron-analytics/metron-profiler-spark/pom.xml
@@ -190,6 +190,21 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptor>src/main/assembly/assembly.xml</descriptor>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id> <!-- this is used for inheritance
merges -->
+                        <phase>package</phase> <!-- bind to the packaging
phase -->
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 </project>

http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml b/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml
new file mode 100644
index 0000000..02f97eb
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml
@@ -0,0 +1,60 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~      http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+<assembly>
+    <id>archive</id>
+    <formats>
+        <format>tar.gz</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <fileSets>
+        <fileSet>
+            <directory>${project.basedir}/src/main/config</directory>
+            <outputDirectory>config</outputDirectory>
+            <useDefaultExcludes>true</useDefaultExcludes>
+            <excludes>
+                <exclude>**/*.formatted</exclude>
+                <exclude>**/*.filtered</exclude>
+            </excludes>
+            <fileMode>0644</fileMode>
+            <lineEnding>unix</lineEnding>
+            <filtered>true</filtered>
+        </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/src/main/scripts</directory>
+            <outputDirectory>bin</outputDirectory>
+            <useDefaultExcludes>true</useDefaultExcludes>
+            <excludes>
+                <exclude>**/*.formatted</exclude>
+                <exclude>**/*.filtered</exclude>
+            </excludes>
+            <fileMode>0755</fileMode>
+            <lineEnding>unix</lineEnding>
+            <filtered>true</filtered>
+        </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/target</directory>
+            <includes>
+                <include>${project.artifactId}-${project.version}.jar</include>
+            </includes>
+            <outputDirectory>lib</outputDirectory>
+            <useDefaultExcludes>true</useDefaultExcludes>
+        </fileSet>
+    </fileSets>
+</assembly>

http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties
b/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties
new file mode 100644
index 0000000..c651791
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+spark.master=local
+spark.app.name=Batch Profiler

http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java
b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java
new file mode 100644
index 0000000..bdcf231
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java
@@ -0,0 +1,174 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.cli;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.io.IOUtils;
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
+import org.apache.metron.profiler.spark.BatchProfiler;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.lang.invoke.MethodHandles;
+import java.util.Properties;
+
+import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.CONFIGURATION_FILE;
+import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.GLOBALS_FILE;
+import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.PROFILE_DEFN_FILE;
+import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.parse;
+
+/**
+ * The main entry point which launches the Batch Profiler in Spark.
+ *
+ * With this class the Batch Profiler can be submitted using the following command.
+ *
+ * <pre>{@code
+ *  $SPARK_HOME/bin/spark-submit \
+ *    --class org.apache.metron.profiler.spark.cli.BatchProfilerCLI \
+ *     --properties-file spark.properties \
+ *     metron-profiler-spark-<version>.jar \
+ *     --config profiler.properties \
+ *     --globals global.properties \
+ *     --profiles profiles.json
+ * }</pre>
+ */
+public class BatchProfilerCLI implements Serializable {
+
+  protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  public static Properties globals;
+  public static Properties profilerProps;
+  public static ProfilerConfig profiles;
+
+  public static void main(String[] args) throws IOException, org.apache.commons.cli.ParseException
{
+    // parse the command line
+    CommandLine commandLine = parseCommandLine(args);
+    profilerProps = handleProfilerProperties(commandLine);
+    globals = handleGlobals(commandLine);
+    profiles = handleProfileDefinitions(commandLine);
+
+    // the batch profiler must use 'event time'
+    if(!profiles.getTimestampField().isPresent()) {
+      throw new IllegalArgumentException("The Batch Profiler must use event time. The 'timestampField'
must be defined.");
+    }
+
+    // one or more profiles must be defined
+    if(profiles.getProfiles().size() == 0) {
+      throw new IllegalArgumentException("No profile definitions found.");
+    }
+
+    SparkSession spark = SparkSession
+            .builder()
+            .config(new SparkConf())
+            .getOrCreate();
+
+    BatchProfiler profiler = new BatchProfiler();
+    long count = profiler.run(spark, profilerProps, globals, profiles);
+    LOG.info("Profiler produced {} profile measurement(s)", count);
+  }
+
+  /**
+   * Load the Stellar globals from a file.
+   *
+   * @param commandLine The command line.
+   */
+  private static Properties handleGlobals(CommandLine commandLine) throws IOException {
+    Properties globals = new Properties();
+    if(GLOBALS_FILE.has(commandLine)) {
+      String globalsPath = GLOBALS_FILE.get(commandLine);
+
+      LOG.info("Loading global properties from '{}'", globalsPath);
+      globals.load(new FileInputStream(globalsPath));
+
+      LOG.info("Globals = {}", globals);
+    }
+    return globals;
+  }
+
+  /**
+   * Load the Profiler configuration from a file.
+   *
+   * @param commandLine The command line.
+   */
+  private static Properties handleProfilerProperties(CommandLine commandLine) throws IOException
{
+    Properties config = new Properties();
+    if(CONFIGURATION_FILE.has(commandLine)) {
+      String propertiesPath = CONFIGURATION_FILE.get(commandLine);
+
+      LOG.info("Loading profiler properties from '{}'", propertiesPath);
+      config.load(new FileInputStream(propertiesPath));
+
+      LOG.info("Properties = {}", config.toString());
+    }
+    return config;
+  }
+
+  /**
+   * Load the profile definitions from a file.
+   *
+   * @param commandLine The command line.
+   */
+  private static ProfilerConfig handleProfileDefinitions(CommandLine commandLine) throws
IOException {
+    ProfilerConfig profiles;
+    if(PROFILE_DEFN_FILE.has(commandLine)) {
+      String profilePath = PROFILE_DEFN_FILE.get(commandLine);
+
+      LOG.info("Loading profiles from '{}'", profilePath);
+      String contents = IOUtils.toString(new FileInputStream(profilePath));
+
+      profiles = ProfilerConfig.fromJSON(contents);
+      LOG.info("Loaded {} profile(s)", profiles.getProfiles().size());
+
+    } else {
+      throw new IllegalArgumentException("No profile(s) defined");
+    }
+    return profiles;
+  }
+
+  /**
+   * Parse the command line arguments submitted by the user.
+   * @param args The command line arguments to parse.
+   * @throws org.apache.commons.cli.ParseException
+   */
+  private static CommandLine parseCommandLine(String[] args) throws ParseException {
+    CommandLineParser parser = new PosixParser();
+    return parse(parser, args);
+  }
+
+  public static Properties getGlobals() {
+    return globals;
+  }
+
+  public static Properties getProfilerProps() {
+    return profilerProps;
+  }
+
+  public static ProfilerConfig getProfiles() {
+    return profiles;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java
b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java
new file mode 100644
index 0000000..f5dfe12
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java
@@ -0,0 +1,144 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.cli;
+
+import com.google.common.base.Joiner;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import java.util.function.Supplier;
+
+/**
+ * Defines the command line interface (CLI) options accepted by the Batch
+ * Profiler.
+ */
+public enum BatchProfilerCLIOptions {
+
+  PROFILE_DEFN_FILE(() -> {
+    Option o = new Option("p", "profiles", true, "Path to a file containing profile definitions.");
+    o.setRequired(true);
+    return o;
+  }),
+
+  CONFIGURATION_FILE(() -> {
+    Option o = new Option("c", "config", true, "Path to the profiler properties file.");
+    o.setRequired(false);
+    return o;
+  }),
+
+  GLOBALS_FILE(() -> {
+    Option o = new Option("g", "globals", true, "Path to the Stellar global config file.");
+    o.setRequired(false);
+    return o;
+  }),
+
+  HELP(() -> {
+    Option o = new Option("h", "help", false, "Usage instructions.");
+    o.setRequired(false);
+    return o;
+  });
+
+  private Option option;
+
+  BatchProfilerCLIOptions(Supplier<Option> optionSupplier) {
+    this.option = optionSupplier.get();
+  }
+
+  /**
+   * Returns true, if the command line contains this option.
+   *
+   * @param cli The command line to parse.
+   */
+  public boolean has(CommandLine cli) {
+    return cli.hasOption(option.getOpt());
+  }
+
+  /**
+   * Returns the option value from the command line.
+   *
+   * @param cli The command line to parse.
+   */
+  public String get(CommandLine cli) {
+    return cli.getOptionValue(option.getOpt());
+  }
+
+  /**
+   * Returns the option value from the command line.  The default value is
+   * returned if the value is not provided on the command line.
+   *
+   * @param cli The command line to parse.
+   * @param defaultValue The default value.
+   */
+  public String get(CommandLine cli, String defaultValue) {
+    return has(cli) ? cli.getOptionValue(option.getOpt()) : defaultValue;
+  }
+
+  /**
+   * Extracts the valid command line options from a given command line.
+   *
+   * @param parser The command line parser.
+   * @param args The command line to parse.
+   * @return The command line options.
+   * @throws ParseException If the command line cannot be parsed successfully.
+   */
+  public static CommandLine parse(CommandLineParser parser, String[] args) throws ParseException
{
+    try {
+      CommandLine cli = parser.parse(getOptions(), args);
+      if(HELP.has(cli)) {
+        printHelp();
+        System.exit(0);
+      }
+      return cli;
+
+    } catch (ParseException e) {
+      System.err.println("invalid arguments: " + Joiner.on(' ').join(args));
+      e.printStackTrace(System.err);
+      printHelp();
+      throw e;
+    }
+  }
+
+  /**
+   * Print a help screen.
+   */
+  public static void printHelp() {
+    HelpFormatter formatter = new HelpFormatter();
+    String header = "options:";
+    String footer = "";
+    String cmd = String.format("spark-submit --class %s --properties-file [spark.properties]
[jar] [options]",
+            BatchProfilerCLI.class.getCanonicalName());
+    formatter.printHelp(cmd, header, getOptions(), footer);
+  }
+
+  /**
+   * Returns all valid CLI options.
+   */
+  public static Options getOptions() {
+    Options allOptions = new Options();
+    for(BatchProfilerCLIOptions o : BatchProfilerCLIOptions.values()) {
+      allOptions.addOption(o.option);
+    }
+    return allOptions;
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh
b/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh
new file mode 100644
index 0000000..c489af7
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+METRON_VERSION=${project.version}
+METRON_HOME=/usr/metron/${METRON_VERSION}
+PROFILER_JAR=${METRON_HOME}/lib/${project.artifactId}-${METRON_VERSION}.jar
+MAIN_CLASS=org.apache.metron.profiler.spark.cli.BatchProfilerCLI
+PROFILER_PROPS=${PROFILER_PROPS:-"${METRON_HOME}/config/batch-profiler.properties"}
+PROFILES_FILE=${PROFILES:-"${METRON_HOME}/config/zookeeper/profiler.json"}
+SPARK_HOME=${SPARK_HOME:-"/usr/hdp/current/spark2-client"}
+
+${SPARK_HOME}/bin/spark-submit \
+    --class ${MAIN_CLASS} \
+    --properties-file ${PROFILER_PROPS} \
+    ${PROFILER_JAR} \
+    --config ${PROFILER_PROPS} \
+    --profiles ${PROFILES_FILE}

http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLITest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLITest.java
b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLITest.java
new file mode 100644
index 0000000..c27495e
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLITest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.spark.cli;
+
+import org.apache.commons.cli.MissingOptionException;
+import org.junit.Test;
+
+/**
+ * Tests the {@link BatchProfilerCLI} class.
+ */
+public class BatchProfilerCLITest {
+
+  /**
+   * The user must provide a Profiler configuration that defines the 'timestampField'.  The
+   * Batch Profiler only operates using event time, not processing time.
+   */
+  @Test(expected = IllegalArgumentException.class)
+  public void mustDefineTimestampField() throws Exception {
+    String[] args = new String[] {
+      "--profiles", "src/test/resources/profiles-no-timestamp-field.json"
+    };
+    BatchProfilerCLI.main(args);
+  }
+
+  /**
+   * The user must define the -p, --profiles option.  The Profiler cannot work without profiles.
+   */
+  @Test(expected = MissingOptionException.class)
+  public void mustDefineProfilesOption() throws Exception {
+    String[] args = new String[] {};
+    BatchProfilerCLI.main(args);
+  }
+
+  /**
+   * If the profile definition contains no valid profiles, we have a problem.
+   */
+  @Test(expected = IllegalArgumentException.class)
+  public void mustDefineProfiles() throws Exception {
+    String[] args = new String[] {
+            "--profiles", "src/test/resources/profiles-empty.json"
+    };
+    BatchProfilerCLI.main(args);
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/resources/globals.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/globals.json b/metron-analytics/metron-profiler-spark/src/test/resources/globals.json
new file mode 100644
index 0000000..2e295f1
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/resources/globals.json
@@ -0,0 +1,3 @@
+{
+  "global-key": "global-value"
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-empty.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/profiles-empty.json
b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-empty.json
new file mode 100644
index 0000000..ce57978
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-empty.json
@@ -0,0 +1,5 @@
+{
+  "profiles": [
+  ],
+  "timestampField":"timestamp"
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-no-timestamp-field.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/profiles-no-timestamp-field.json
b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-no-timestamp-field.json
new file mode 100644
index 0000000..e47a6b5
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-no-timestamp-field.json
@@ -0,0 +1,12 @@
+{
+  "profiles": [
+    {
+      "profile": "hello-world",
+      "onlyif":  "exists(ip_src_addr)",
+      "foreach": "ip_src_addr",
+      "init":    { "count": "0" },
+      "update":  { "count": "count + 1" },
+      "result":  "count"
+    }
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/resources/profiles.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/profiles.json b/metron-analytics/metron-profiler-spark/src/test/resources/profiles.json
new file mode 100644
index 0000000..ac77f4b
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/resources/profiles.json
@@ -0,0 +1,13 @@
+{
+  "profiles": [
+    {
+      "profile": "hello-world",
+      "onlyif":  "exists(ip_src_addr)",
+      "foreach": "ip_src_addr",
+      "init":    { "count": "0" },
+      "update":  { "count": "count + 1" },
+      "result":  "count"
+    }
+  ],
+  "timestampField":"timestamp"
+}
\ No newline at end of file


Mime
View raw message