metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject [33/52] [abbrv] metron git commit: METRON-1449 Set Zookeeper URL for Stellar Running in Zeppelin Notebook (nickwallen) closes apache/metron#931
Date Wed, 18 Apr 2018 15:00:03 GMT
METRON-1449 Set Zookeeper URL for Stellar Running in Zeppelin Notebook (nickwallen) closes
apache/metron#931


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ab4f8e65
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ab4f8e65
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ab4f8e65

Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual
Commit: ab4f8e65e13aaea3a10491290a7411b3b6dc955e
Parents: 0ab39a3
Author: nickwallen <nick@nickallen.org>
Authored: Fri Apr 6 16:40:01 2018 -0400
Committer: nickallen <nickallen@apache.org>
Committed: Fri Apr 6 16:40:01 2018 -0400

----------------------------------------------------------------------
 metron-platform/metron-management/pom.xml       |   1 -
 .../shell/DefaultStellarShellExecutor.java      |  32 +++---
 .../shell/DefaultStellarShellExecutorTest.java  |  11 ++
 metron-stellar/stellar-zeppelin/README.md       |  80 ++++----------
 metron-stellar/stellar-zeppelin/pom.xml         |  12 +++
 .../stellar/zeppelin/StellarInterpreter.java    |  95 ++++++++++-------
 .../zeppelin/StellarInterpreterProperty.java    |  79 ++++++++++++++
 .../StellarInterpreterPropertyTest.java         |  62 +++++++++++
 .../zeppelin/StellarInterpreterTest.java        |  60 ++++++++++-
 .../integration/ConfigUploadComponent.java      |  82 +++++++++++++++
 .../StellarInterpreterIntegrationTest.java      | 104 +++++++++++++++++++
 11 files changed, 501 insertions(+), 117 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-platform/metron-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml
index c185662..962fd46 100644
--- a/metron-platform/metron-management/pom.xml
+++ b/metron-platform/metron-management/pom.xml
@@ -183,7 +183,6 @@
             </exclusions>
         </dependency>
     </dependencies>
-
     <build>
         <plugins>
             <plugin>

http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
index 3f2c495..781a0cf 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
@@ -52,6 +52,7 @@ import java.io.ByteArrayInputStream;
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -271,10 +272,7 @@ public class DefaultStellarShellExecutor implements StellarShellExecutor
{
       globals = (Map<String, Object>) capability.get();
 
     } else {
-      // if it does not exist, create it.  this creates the global config for the current
stellar executor
-      // session only.  this does not change the global config maintained externally in zookeeper
-      globals = new HashMap<>();
-      getContext().addCapability(GLOBAL_CONFIG, () -> globals);
+      throw new IllegalStateException("'GLOBAL_CONFIG' is missing");
     }
 
     return globals;
@@ -344,21 +342,25 @@ public class DefaultStellarShellExecutor implements StellarShellExecutor
{
    * @param zkClient An optional Zookeeper client.
    */
   private Context createContext(Properties properties, Optional<CuratorFramework> zkClient)
throws Exception {
+    Context.Builder contextBuilder = new Context.Builder();
+    Map<String, Object> globals;
+    if (zkClient.isPresent()) {
 
-    Context.Builder contextBuilder = new Context.Builder()
-            .with(SHELL_VARIABLES, () -> variables)
-            .with(STELLAR_CONFIG, () -> properties);
+      // fetch globals from zookeeper
+      globals = fetchGlobalConfig(zkClient.get());
+      contextBuilder.with(ZOOKEEPER_CLIENT, () -> zkClient.get());
 
-    // load global configuration from zookeeper
-    if (zkClient.isPresent()) {
-      Map<String, Object> global = fetchGlobalConfig(zkClient.get());
-      contextBuilder
-              .with(GLOBAL_CONFIG, () -> global)
-              .with(ZOOKEEPER_CLIENT, () -> zkClient.get())
-              .with(STELLAR_CONFIG, () -> getStellarConfig(global, properties));
+    } else {
+
+      // use empty globals to allow a user to '%define' their own
+      globals = new HashMap<>();
     }
 
-    return contextBuilder.build();
+    return contextBuilder
+            .with(SHELL_VARIABLES, () -> variables)
+            .with(GLOBAL_CONFIG, () -> globals)
+            .with(STELLAR_CONFIG, () -> getStellarConfig(globals, properties))
+            .build();
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutorTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutorTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutorTest.java
index ebba84f..23b0204 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutorTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutorTest.java
@@ -295,4 +295,15 @@ public class DefaultStellarShellExecutorTest {
     assertTrue(result.getValue().isPresent());
     assertEquals("", result.getValue().get());
   }
+
+  /**
+   * If the executor is initialized without a connection to Zookeeper, the globals should
be
+   * defined, but empty.  This allows a user to '%define' their own with magic commands even
+   * without Zookeeper.
+   */
+  @Test
+  public void testEmptyGlobalsWithNoZookeeper() {
+    assertNotNull(executor.getGlobalConfig());
+    assertEquals(0, executor.getGlobalConfig().size());
+  }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-stellar/stellar-zeppelin/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-zeppelin/README.md b/metron-stellar/stellar-zeppelin/README.md
index 1190658..60dcb3a 100644
--- a/metron-stellar/stellar-zeppelin/README.md
+++ b/metron-stellar/stellar-zeppelin/README.md
@@ -48,77 +48,27 @@ To install the Stellar Interpreter in your Apache Zeppelin installation,
follow
     mvn clean install -DskipTests
     ```
 
-1. If you do not already have Zeppelin installed, [download and unpack Apache Zeppelin](https://zeppelin.apache.org/download.html).
 The directory in which you unpack Zeppelin will be referred to as `$ZEPPELIN_HOME`.
-
-1. If Zeppelin was already installed, make sure that it is not running.
-
-1. Create a settings directory for the Stellar interpreter.
+1. If you do not already have Zeppelin installed, [download and unpack Apache Zeppelin](https://zeppelin.apache.org/download.html).
 Then change directories to the root of your Zeppelin download.
 
     ```
-    mkdir $ZEPPELIN_HOME/interpreter/stellar
-    cat <<EOF > $ZEPPELIN_HOME/interpreter/stellar/interpreter-setting.json
-    [
-      {
-        "group": "stellar",
-        "name": "stellar",
-        "className": "org.apache.metron.stellar.zeppelin.StellarInterpreter",
-        "properties": {
-        }
-      }
-    ]
-    EOF
+    cd $ZEPPELIN_HOME
     ```
 
-1. Create a Zeppelin Site file (`$ZEPPELIN_HOME/conf/zeppelin-site.xml`).
-
-    ```
-    cp $ZEPPELIN_HOME/conf/zeppelin-site.xml.template $ZEPPELIN_HOME/conf/zeppelin-site.xml
-    ```
+1. Use Zeppelin's installation utility to install the Stellar Interpreter.
 
-1. In the Zeppelin site file, add `org.apache.metron.stellar.zeppelin.StellarInterpreter`
to the comma-separated list of Zeppelin interpreters under the `zeppelin.interpreters` property.
+    If Zeppelin was already installed, make sure that it is stopped before running this command.
 Update the version, '0.4.3' in the example below, to whatever is appropriate for your environment.
 
-    The property will likely look-like the following.
     ```
-    <property>
-      <name>zeppelin.interpreters</name>
-      <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreS
 qlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.metron.stellar.zeppelin.StellarInterpreter</value>
-      <description>Comma separated interpreter configurations. First interpreter become
a default</description>
-    </property>
+    bin/install-interpreter.sh --name stellar --artifact org.apache.metron:stellar-zeppelin:0.4.3
     ```
 
 1. Start Zeppelin.  
 
     ```
-    $ZEPPELIN_HOME/bin/zeppelin-daemon.sh start
+    bin/zeppelin-daemon.sh start
     ```
 
-1. Navigate to Zeppelin running at [http://localhost:8080/](http://localhost:8080/).
-
-1. Register the Stellar interpreter in Zeppelin.
-
-    1. Click on the top-right menu item labelled "Anonymous" then choose "Interpreter" in
the drop-down that opens.    
-
-1. Configure the Stellar interpreter.
-
-    1. Click on '**+ Create**' near the top-right.
-
-    1. Define the following values.
-        * **Interpreter Name** = `stellar`
-        * **Interpreter Group** = `stellar`
-
-    1. Under **Options**, set the following values.
-        * The interpreter will be instantiated **Per Note**  in **isolated** process.
-
-    1. Under **Dependencies**, define the following fields, then click the "+" icon.  Replace
the Metron version as required.
-        * **Artifact** = `org.apache.metron:stellar-zeppelin:0.4.3`
-
-    1. Click "Save"
-
-1. Wait for the intrepreter to start.
-
-    1. Near the title '**stellar**', will be a status icon.  This will indicate that it is
downloading the dependencies.  
-
-    1. Once the icon is shown as green, the interpreter is ready to work.
+1. Navigate to Zeppelin running at [http://localhost:8080/](http://localhost:8080/).  The
Stellar Interpreter should be ready for use with a basic set of functions.
 
 Usage
 -----
@@ -141,19 +91,25 @@ Usage
 
 1. In the next block, check which functions are available to you.
 
+    When executing Stellar's magic functions, you must explicitly define which interpreter
should be used in the code block.  If you define 'stellar' as the default interpreter when
creating a notebook, then this is only required when using Stellar's magic functions.
+
     ```
+    %stellar
+
     %functions
     ```
 
     You will **only** 'see' the functions defined within `stellar-common` since that is the
only library that we added to the interpreter.  
 
-1. To see how additional functions can be added, go back to the Stellar interpreter configuration
and add another dependency as follows.
+1. Add additional Stellar functions to your session.
 
-    ```
-    org.apache.metron:metron-statistics:0.4.3
-    ```
+    1. Go back to the Stellar interpreter configuration and add another dependency as follows.
+
+        ```
+        org.apache.metron:metron-statistics:0.4.3
+        ```
 
-    Reload the Stellar interpreter and run `%functions` again.  You will see the additional
functions defined within the `metron-statistics` project.
+    1. Go back to your notebook and run `%functions` again.  You will now see the additional
functions defined within the `metron-statistics` project.
 
 1. Auto-completion is also available for Stellar expressions.  
 

http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-stellar/stellar-zeppelin/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-zeppelin/pom.xml b/metron-stellar/stellar-zeppelin/pom.xml
index 7809342..732c62b 100644
--- a/metron-stellar/stellar-zeppelin/pom.xml
+++ b/metron-stellar/stellar-zeppelin/pom.xml
@@ -35,6 +35,18 @@
             <version>${project.parent.version}</version>
         </dependency>
         <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-integration-test</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-common</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-all</artifactId>
             <version>${global_mockito_version}</version>

http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreter.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreter.java
b/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreter.java
index 58287dc..5a7a175 100644
--- a/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreter.java
+++ b/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreter.java
@@ -18,16 +18,6 @@
 
 package org.apache.metron.stellar.zeppelin;
 
-import static org.apache.zeppelin.interpreter.InterpreterResult.Code.ERROR;
-import static org.apache.zeppelin.interpreter.InterpreterResult.Code.SUCCESS;
-import static org.apache.zeppelin.interpreter.InterpreterResult.Type.TEXT;
-
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.Properties;
-
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.metron.stellar.common.shell.DefaultStellarAutoCompleter;
 import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor;
@@ -41,6 +31,16 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Properties;
+
+import static org.apache.zeppelin.interpreter.InterpreterResult.Code.ERROR;
+import static org.apache.zeppelin.interpreter.InterpreterResult.Code.SUCCESS;
+import static org.apache.zeppelin.interpreter.InterpreterResult.Type.TEXT;
+
 /**
  * A Zeppelin Interpreter for Stellar.
  */
@@ -65,16 +65,21 @@ public class StellarInterpreter extends Interpreter {
 
   public StellarInterpreter(Properties properties) {
     super(properties);
-    this.autoCompleter = new DefaultStellarAutoCompleter();
   }
 
   @Override
   public void open() {
     try {
-      executor = createExecutor();
+      // create the auto-completer
+      this.autoCompleter = new DefaultStellarAutoCompleter();
+
+      // create the stellar executor
+      Properties props = getProperty();
+      this.executor = createExecutor(props);
 
     } catch (Exception e) {
       LOG.error("Unable to create a StellarShellExecutor", e);
+      throw new RuntimeException(e);
     }
   }
 
@@ -85,36 +90,55 @@ public class StellarInterpreter extends Interpreter {
 
   @Override
   public InterpreterResult interpret(final String input, InterpreterContext context) {
-    InterpreterResult result;
+    InterpreterResult result = new InterpreterResult(SUCCESS, TEXT, "");
+
     try {
 
-      // execute the input
-      StellarResult stellarResult = executor.execute(input);
-      if(stellarResult.isSuccess()) {
+      // allow separate expressions on each line
+      String[] expressions = input.split(System.lineSeparator());
+      for (String expression : expressions) {
+        result = execute(expression);
+      }
+
+    } catch(Throwable t){
 
-        // on success - if no result, use a blank value
-        Object value = stellarResult.getValue().orElse("");
-        String text = value.toString();
-        result = new InterpreterResult(SUCCESS, TEXT, text);
+      // unexpected exception
+      String message = getErrorMessage(Optional.of(t), input);
+      result = new InterpreterResult(ERROR, TEXT, message);
+    }
 
-      } else if(stellarResult.isError()) {
+    // result is from the last expression that was executed
+    return result;
+  }
 
-        // an error occurred
-        Optional<Throwable> e = stellarResult.getException();
-        String message = getErrorMessage(e, input);
-        result = new InterpreterResult(ERROR, TEXT, message);
+  /**
+   * Execute a single Stellar expression.
+   * @param expression The Stellar expression to execute.
+   * @return The result of execution.
+   */
+  private InterpreterResult execute(final String expression) {
+    InterpreterResult result;
 
-      } else {
+    // execute the expression
+    StellarResult stellarResult = executor.execute(expression);
+    if (stellarResult.isSuccess()) {
 
-        // should never happen
-        throw new IllegalStateException("Unexpected error. result=" + stellarResult);
-      }
+      // on success - if no result, use a blank value
+      Object value = stellarResult.getValue().orElse("");
+      String text = value.toString();
+      result = new InterpreterResult(SUCCESS, TEXT, text);
 
-    } catch(Throwable t) {
+    } else if (stellarResult.isError()) {
 
-      // unexpected exception
-      String message = getErrorMessage(Optional.of(t), input);
+      // an error occurred
+      Optional<Throwable> e = stellarResult.getException();
+      String message = getErrorMessage(e, expression);
       result = new InterpreterResult(ERROR, TEXT, message);
+
+    } else {
+
+      // should never happen
+      throw new IllegalStateException("Unexpected error. result=" + stellarResult);
     }
 
     return result;
@@ -176,10 +200,11 @@ public class StellarInterpreter extends Interpreter {
    * Create an executor that will run the Stellar code for the Zeppelin Notebook.
    * @return The stellar executor.
    */
-  private StellarShellExecutor createExecutor() throws Exception {
+  private StellarShellExecutor createExecutor(Properties properties) throws Exception {
 
-    Properties props = getProperty();
-    StellarShellExecutor executor = new DefaultStellarShellExecutor(props, Optional.empty());
+    // a zookeeper URL may be defined
+    String zookeeperURL = StellarInterpreterProperty.ZOOKEEPER_URL.get(properties, String.class);
+    StellarShellExecutor executor = new DefaultStellarShellExecutor(properties, Optional.ofNullable(zookeeperURL));
 
     // register the auto-completer to be notified
     executor.addSpecialListener((magic) -> autoCompleter.addCandidateFunction(magic.getCommand()));

http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreterProperty.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreterProperty.java
b/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreterProperty.java
new file mode 100644
index 0000000..7392219
--- /dev/null
+++ b/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreterProperty.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.zeppelin;
+
+import org.apache.metron.stellar.common.utils.ConversionUtils;
+
+import java.util.Map;
+
+/**
+ * Defines the properties that a user can define when configuring
+ * the Stellar Interpreter in Zeppelin.
+ */
+public enum StellarInterpreterProperty {
+
+  /**
+   * A property that defines the URL for connecting to Zookeeper.  By default this is empty.
+   */
+  ZOOKEEPER_URL("zookeeper.url", null);
+
+  /**
+   * The key or property name.
+   */
+  String key;
+
+  /**
+   * The default value of the property.
+   */
+  Object defaultValue;
+
+  StellarInterpreterProperty(String key, Object defaultValue) {
+    this.key = key;
+    this.defaultValue = defaultValue;
+  }
+
+  /**
+   * @return The key or name of the property.
+   */
+  public String getKey() {
+    return key;
+  }
+
+  /**
+   * @return The default value of the property.
+   */
+  public <T> T getDefault(Class<T> clazz) {
+    return ConversionUtils.convert(defaultValue, clazz);
+  }
+
+  /**
+   * Retrieves the property value from a map of properties.
+   * @param properties A map of properties.
+   * @return The value of the property within the map.
+   */
+  public <T> T get(Map<Object, Object> properties, Class<T> clazz) {
+    Object o = properties.getOrDefault(key, defaultValue);
+    return o == null ? null : ConversionUtils.convert(o, clazz);
+  }
+
+  @Override
+  public String toString() {
+    return key;
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterPropertyTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterPropertyTest.java
b/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterPropertyTest.java
new file mode 100644
index 0000000..d474eaf
--- /dev/null
+++ b/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterPropertyTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.zeppelin;
+
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.Map;
+
+import static org.apache.metron.stellar.zeppelin.StellarInterpreterProperty.ZOOKEEPER_URL;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests the StellarInterpreterProperty class.
+ */
+public class StellarInterpreterPropertyTest {
+
+  /**
+   * By defining the 'zookeeper.url' property a user is able to 'set' the Zookeeper URL.
+   */
+  @Test
+  public void testGet() {
+
+    // define the zookeeper URL property
+    final String expected = "zookeeper:2181";
+    Map<Object, Object> props = Collections.singletonMap("zookeeper.url", expected);
+
+    // should be able to get the zookeeper URL property from the properties
+    String actual = ZOOKEEPER_URL.get(props, String.class);
+    assertEquals(expected, actual);
+  }
+
+  /**
+   * The default value should be returned when the user does not defined a 'zookeeper.url'.
+   */
+  @Test
+  public void testGetWhenPropertyNotDefined() {
+
+    // the property is not defined
+    Map<Object, Object> props = Collections.singletonMap("foo", "bar");
+    String actual = ZOOKEEPER_URL.get(props, String.class);
+
+    // expect to get the default value since its not defined
+    String expected = ZOOKEEPER_URL.getDefault(String.class);
+    assertEquals(expected, actual);
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterTest.java
b/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterTest.java
index 363938e..c817747 100644
--- a/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterTest.java
+++ b/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterTest.java
@@ -17,22 +17,29 @@
  */
 package org.apache.metron.stellar.zeppelin;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-
 import com.google.common.collect.Iterables;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.stellar.common.shell.VariableResult;
+import org.apache.metron.stellar.dsl.Context;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResultMessage;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.Properties;
 
+import static org.apache.metron.stellar.zeppelin.StellarInterpreterProperty.ZOOKEEPER_URL;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
 /**
  * Tests the StellarInterpreter.
  */
@@ -189,4 +196,49 @@ public class StellarInterpreterTest {
     // expect no completions
     assertEquals(0, completions.size());
   }
+
+  /**
+   * No Zookeeper client connection should be made if the Zookeeper URL is not defined
+   */
+  @Test
+  public void testOpenWithNoZookeeperURL() {
+
+    // no zookeeper URL defined
+    Properties props = new Properties();
+
+    // open the interpreter
+    interpreter = new StellarInterpreter(props);
+    interpreter.open();
+
+    // no zookeeper client should be defined
+    Optional<Object> zk = interpreter.getExecutor().getContext().getCapability(Context.Capabilities.ZOOKEEPER_CLIENT,
false);
+    assertFalse(zk.isPresent());
+  }
+
+  /**
+   * Ensure that we can run Stellar code in the interpreter.
+   */
+  @Test
+  public void testExecuteStellarMultipleLines() {
+
+    // multi-line input
+    String input =
+            "x := 2 + 2" + System.lineSeparator() +
+            "y := 4 + 4";
+    InterpreterResult result = interpreter.interpret(input, context);
+
+    // expect x == 4 and y == 8
+    Map<String, VariableResult> vars = interpreter.getExecutor().getState();
+    assertEquals(4, vars.get("x").getResult());
+    assertEquals(8, vars.get("y").getResult());
+
+    // validate the result
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    assertEquals(1, result.message().size());
+
+    // the output is the result of only the 'last' expression
+    InterpreterResultMessage message = result.message().get(0);
+    assertEquals("8", message.getData());
+    assertEquals(InterpreterResult.Type.TEXT, message.getType());
+  }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/integration/ConfigUploadComponent.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/integration/ConfigUploadComponent.java
b/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/integration/ConfigUploadComponent.java
new file mode 100644
index 0000000..9257e62
--- /dev/null
+++ b/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/integration/ConfigUploadComponent.java
@@ -0,0 +1,82 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.stellar.zeppelin.integration;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.metron.integration.InMemoryComponent;
+import org.apache.metron.integration.UnableToStartException;
+
+import java.util.Map;
+
+import static org.apache.metron.common.configuration.ConfigurationsUtils.getClient;
+import static org.apache.metron.common.configuration.ConfigurationsUtils.writeGlobalConfigToZookeeper;
+
+/**
+ * Uploads configuration to Zookeeper.
+ */
+public class ConfigUploadComponent implements InMemoryComponent {
+
+  private String zookeeperURL;
+  private Map<String, Object> globals;
+
+  @Override
+  public void start() throws UnableToStartException {
+    try {
+      upload();
+
+    } catch (Exception e) {
+      throw new UnableToStartException(e.getMessage(), e);
+    }
+  }
+
+  @Override
+  public void stop() {
+    // nothing to do
+  }
+
+  /**
+   * Uploads configuration to Zookeeper.
+   * @throws Exception
+   */
+  private void upload() throws Exception {
+    assert zookeeperURL != null;
+    try(CuratorFramework client = getClient(zookeeperURL)) {
+      if(client.getState() != CuratorFrameworkState.STARTED) {
+        client.start();
+      }
+
+      if (globals != null) {
+        writeGlobalConfigToZookeeper(globals, client);
+      }
+    }
+  }
+
+
+  public ConfigUploadComponent withZookeeperURL(String zookeeperURL) {
+    this.zookeeperURL = zookeeperURL;
+    return this;
+  }
+
+  public ConfigUploadComponent withGlobals(Map<String, Object> globals) {
+    this.globals = globals;
+    return this;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/integration/StellarInterpreterIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/integration/StellarInterpreterIntegrationTest.java
b/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/integration/StellarInterpreterIntegrationTest.java
new file mode 100644
index 0000000..b6395eb
--- /dev/null
+++ b/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/integration/StellarInterpreterIntegrationTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.zeppelin.integration;
+
+import org.apache.metron.integration.BaseIntegrationTest;
+import org.apache.metron.integration.ComponentRunner;
+import org.apache.metron.integration.components.ZKServerComponent;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.zeppelin.StellarInterpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+
+import static org.apache.metron.stellar.zeppelin.StellarInterpreterProperty.ZOOKEEPER_URL;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+/**
+ * An integration test for the StellarInterpreter.
+ */
+public class StellarInterpreterIntegrationTest extends BaseIntegrationTest {
+
+  private StellarInterpreter interpreter;
+  private InterpreterContext context;
+  private Properties properties;
+  private String zookeeperURL;
+  private ZKServerComponent zkServer;
+  private ComponentRunner runner;
+
+  @Before
+  public void setup() throws Exception {
+
+    // a component that uploads the global configuration
+    Map<String, Object> globals = new HashMap<>();
+    ConfigUploadComponent configUploader = new ConfigUploadComponent()
+            .withGlobals(globals);
+
+    // create zookeeper component
+    properties = new Properties();
+    zkServer = getZKServerComponent(properties);
+
+    // can only get the zookeeperUrl AFTER it has started
+    zkServer.withPostStartCallback((zk) -> {
+      zookeeperURL = zk.getConnectionString();
+      configUploader.withZookeeperURL(zookeeperURL);
+    });
+
+    // start the integration test components
+    runner = new ComponentRunner.Builder()
+            .withComponent("zk", zkServer)
+            .withComponent("config", configUploader)
+            .build();
+    runner.start();
+
+    context = mock(InterpreterContext.class);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    runner.stop();
+  }
+
+  /**
+   * A user should be able to define a Zookeeper URL as a property.  When this property
+   * is defined, a connection to Zookeeper is created and available in the Stellar session.
+   */
+  @Test
+  public void testOpenWithZookeeperURL() {
+
+    // define a zookeeper URL
+    Properties props = new Properties();
+    props.put(ZOOKEEPER_URL.toString(), zookeeperURL);
+
+    // open the interpreter
+    interpreter = new StellarInterpreter(props);
+    interpreter.open();
+
+    // a zookeeper client should be defined
+    Optional<Object> zk = interpreter.getExecutor().getContext().getCapability(Context.Capabilities.ZOOKEEPER_CLIENT,
false);
+    assertTrue(zk.isPresent());
+  }
+
+}


Mime
View raw message