metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject [4/4] metron git commit: METRON-1382 Run Stellar in a Zeppelin Notebook (nickwallen) closes apache/metron#884
Date Fri, 12 Jan 2018 18:59:48 GMT
METRON-1382 Run Stellar in a Zeppelin Notebook (nickwallen) closes apache/metron#884


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2dd01b17
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2dd01b17
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2dd01b17

Branch: refs/heads/master
Commit: 2dd01b17782af7756776b8e07afe3661ac19bb0a
Parents: 55a7379
Author: nickwallen <nick@nickallen.org>
Authored: Fri Jan 12 13:08:32 2018 -0500
Committer: nickallen <nickallen@apache.org>
Committed: Fri Jan 12 13:08:32 2018 -0500

----------------------------------------------------------------------
 dependencies_with_url.csv                       |  16 +
 .../metron-common/src/main/scripts/stellar      |   2 +-
 .../metron/management/ShellFunctions.java       |  57 +-
 .../management/ConfigurationFunctionsTest.java  |   1 -
 .../EnrichmentConfigFunctionsTest.java          |  24 +-
 .../management/IndexingConfigFunctionsTest.java |  10 +-
 .../management/ParserConfigFunctionsTest.java   |  10 +-
 .../metron/management/ShellFunctionsTest.java   |  22 +-
 .../management/ThreatTriageFunctionsTest.java   |  26 +-
 metron-stellar/pom.xml                          |   1 +
 metron-stellar/stellar-common/README.md         |  35 +-
 metron-stellar/stellar-common/pom.xml           |   2 -
 .../stellar/common/StellarAssignment.java       |   4 +-
 .../shell/DefaultStellarAutoCompleter.java      | 220 ++++++++
 .../shell/DefaultStellarShellExecutor.java      | 413 ++++++++++++++
 .../stellar/common/shell/PausableInput.java     | 372 -------------
 .../common/shell/StellarAutoCompleter.java      |  45 ++
 .../common/shell/StellarExecutionListeners.java |  51 ++
 .../common/shell/StellarExecutionNotifier.java  |  44 ++
 .../stellar/common/shell/StellarExecutor.java   | 327 -----------
 .../stellar/common/shell/StellarResult.java     | 195 +++++++
 .../stellar/common/shell/StellarShell.java      | 557 -------------------
 .../common/shell/StellarShellExecutor.java      |  79 +++
 .../shell/StellarShellOptionsValidator.java     | 127 -----
 .../stellar/common/shell/VariableResult.java    | 101 ++++
 .../stellar/common/shell/cli/PausableInput.java | 372 +++++++++++++
 .../stellar/common/shell/cli/StellarShell.java  | 427 ++++++++++++++
 .../shell/cli/StellarShellOptionsValidator.java | 127 +++++
 .../shell/specials/AssignmentCommand.java       |  85 +++
 .../stellar/common/shell/specials/Comment.java  |  53 ++
 .../common/shell/specials/DocCommand.java       | 105 ++++
 .../shell/specials/MagicDefineGlobal.java       |  86 +++
 .../shell/specials/MagicListFunctions.java      |  72 +++
 .../common/shell/specials/MagicListGlobals.java |  54 ++
 .../shell/specials/MagicListVariables.java      |  88 +++
 .../shell/specials/MagicUndefineGlobal.java     |  70 +++
 .../common/shell/specials/QuitCommand.java      |  51 ++
 .../common/shell/specials/SpecialCommand.java   |  60 ++
 .../org/apache/metron/stellar/dsl/Context.java  |  12 +-
 .../stellar-common/src/main/scripts/stellar     |   2 +-
 .../stellar/common/StellarCompilerTest.java     | 169 ++++++
 .../stellar/common/StellarInterpreterTest.java  | 169 ------
 .../shell/DefaultStellarAutoCompleterTest.java  | 190 +++++++
 .../shell/DefaultStellarShellExecutorTest.java  | 298 ++++++++++
 .../stellar/common/shell/StellarResultTest.java | 165 ++++++
 .../shell/StellarShellOptionsValidatorTest.java | 187 -------
 .../cli/StellarShellOptionsValidatorTest.java   | 184 ++++++
 .../common/shell/cli/StellarShellTest.java      | 199 +++++++
 .../shell/specials/AssignmentCommandTest.java   | 212 +++++++
 .../common/shell/specials/CommentTest.java      |  91 +++
 .../common/shell/specials/DocCommandTest.java   |  86 +++
 .../shell/specials/MagicDefineGlobalTest.java   | 140 +++++
 .../shell/specials/MagicListFunctionsTest.java  | 136 +++++
 .../shell/specials/MagicListGlobalsTest.java    | 113 ++++
 .../shell/specials/MagicListVariablesTest.java  | 113 ++++
 .../shell/specials/MagicUndefineGlobalTest.java | 117 ++++
 metron-stellar/stellar-zeppelin/README.md       | 160 ++++++
 metron-stellar/stellar-zeppelin/pom.xml         |  95 ++++
 .../src/main/assembly/assembly.xml              |  30 +
 .../stellar/zeppelin/StellarInterpreter.java    | 200 +++++++
 .../src/main/resources/interpreter-setting.json |  12 +
 .../zeppelin/StellarInterpreterTest.java        | 192 +++++++
 62 files changed, 5835 insertions(+), 1828 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index b94a984..a1f431b 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -341,3 +341,19 @@ org.eclipse.persistence:org.eclipse.persistence.asm:jar:2.6.4:compile,EPL 1.0,ht
 org.eclipse.persistence:org.eclipse.persistence.core:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink
 org.eclipse.persistence:org.eclipse.persistence.jpa.jpql:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink
 org.eclipse.persistence:org.eclipse.persistence.jpa:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink
+
+com.google.code.gson:gson:jar:2.2:compile
+  org.codehaus.plexus:plexus-classworlds:jar:2.4:compile
+  org.codehaus.plexus:plexus-component-annotations:jar:1.5.5:compile
+  org.codehaus.plexus:plexus-interpolation:jar:1.14:compile
+  org.codehaus.plexus:plexus-utils:jar:2.0.7:compile
+  org.jsoup:jsoup:jar:1.6.1:compile
+  org.sonatype.aether:aether-api:jar:1.12:compile
+  org.sonatype.aether:aether-connector-file:jar:1.12:compile
+  org.sonatype.aether:aether-connector-wagon:jar:1.12:compile
+  org.sonatype.aether:aether-impl:jar:1.12:compile
+  org.sonatype.aether:aether-spi:jar:1.12:compile
+  org.sonatype.aether:aether-util:jar:1.12:compile
+  org.sonatype.sisu:sisu-guice:jar:no_aop:3.0.2:compile
+  org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile
+  org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-common/src/main/scripts/stellar
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/scripts/stellar b/metron-platform/metron-common/src/main/scripts/stellar
index 3f53c49..431ed7b 100644
--- a/metron-platform/metron-common/src/main/scripts/stellar
+++ b/metron-platform/metron-common/src/main/scripts/stellar
@@ -33,4 +33,4 @@ export METRON_VERSION=${project.version}
 export METRON_HOME=/usr/metron/$METRON_VERSION
 export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar)
 export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar)
-java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.StellarShell "$@"
+java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@"

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
index 52ff157..afac7f0 100644
--- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
+++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
@@ -17,9 +17,20 @@
  */
 package org.apache.metron.management;
 
-import static org.apache.metron.stellar.common.shell.StellarExecutor.CONSOLE;
-
 import com.jakewharton.fliptables.FlipTable;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.text.WordUtils;
+import org.apache.metron.stellar.common.shell.VariableResult;
+import org.apache.metron.stellar.common.shell.cli.PausableInput;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.BaseStellarFunction;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+import org.jboss.aesh.console.Console;
+import org.slf4j.LoggerFactory;
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
@@ -31,22 +42,16 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.text.WordUtils;
-import org.apache.metron.stellar.common.shell.PausableInput;
-import org.apache.metron.stellar.common.shell.StellarExecutor;
-import org.apache.metron.stellar.common.utils.ConversionUtils;
-import org.apache.metron.stellar.dsl.BaseStellarFunction;
-import org.apache.metron.stellar.dsl.Context;
-import org.apache.metron.stellar.dsl.ParseException;
-import org.apache.metron.stellar.dsl.Stellar;
-import org.apache.metron.stellar.dsl.StellarFunction;
-import org.jboss.aesh.console.Console;
-import org.slf4j.LoggerFactory;
+
+import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE;
 
 public class ShellFunctions {
   private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
+  private static Map<String, VariableResult> getVariables(Context context) {
+    return (Map<String, VariableResult>) context.getCapability(Context.Capabilities.SHELL_VARIABLES).get();
+  }
+
   @Stellar(
            namespace = "SHELL"
           ,name = "MAP2TABLE"
@@ -90,7 +95,7 @@ public class ShellFunctions {
     @Override
     public Object apply(List<Object> args, Context context) throws ParseException {
 
-      Map<String, StellarExecutor.VariableResult> variables = (Map<String, StellarExecutor.VariableResult>) context.getCapability(StellarExecutor.SHELL_VARIABLES).get();
+      Map<String, VariableResult> variables = getVariables(context);
       String[] headers = {"VARIABLE", "VALUE", "EXPRESSION"};
       String[][] data = new String[variables.size()][3];
       int wordWrap = -1;
@@ -98,11 +103,11 @@ public class ShellFunctions {
         wordWrap = ConversionUtils.convert(args.get(0), Integer.class);
       }
       int i = 0;
-      for(Map.Entry<String, StellarExecutor.VariableResult> kv : variables.entrySet()) {
-        StellarExecutor.VariableResult result = kv.getValue();
+      for(Map.Entry<String, VariableResult> kv : variables.entrySet()) {
+        VariableResult result = kv.getValue();
         data[i++] = new String[] { toWrappedString(kv.getKey().toString(), wordWrap)
                                  , toWrappedString(result.getResult(), wordWrap)
-                                 , toWrappedString(result.getExpression(), wordWrap)
+                                 , toWrappedString(result.getExpression().get(), wordWrap)
                                  };
       }
       return FlipTable.of(headers, data);
@@ -139,16 +144,16 @@ public class ShellFunctions {
 
     @Override
     public Object apply(List<Object> args, Context context) throws ParseException {
-      Map<String, StellarExecutor.VariableResult> variables = (Map<String, StellarExecutor.VariableResult>) context.getCapability(StellarExecutor.SHELL_VARIABLES).get();
+      Map<String, VariableResult> variables = getVariables(context);
       LinkedHashMap<String, String> ret = new LinkedHashMap<>();
       for(Object arg : args) {
         if(arg == null) {
           continue;
         }
         String variable = (String)arg;
-        StellarExecutor.VariableResult result = variables.get(variable);
-        if(result != null && result.getExpression() != null) {
-          ret.put(variable, result.getExpression());
+        VariableResult result = variables.get(variable);
+        if(result != null && result.getExpression().isPresent()) {
+          ret.put(variable, result.getExpression().orElseGet(() -> ""));
         }
       }
       return ret;
@@ -177,7 +182,7 @@ public class ShellFunctions {
 
     @Override
     public Object apply(List<Object> args, Context context) throws ParseException {
-      Map<String, StellarExecutor.VariableResult> variables = (Map<String, StellarExecutor.VariableResult>) context.getCapability(StellarExecutor.SHELL_VARIABLES).get();
+      Map<String, VariableResult> variables = getVariables(context);
       if(args.size() == 0) {
         return null;
       }
@@ -185,9 +190,9 @@ public class ShellFunctions {
       if(variable == null) {
         return null;
       }
-      StellarExecutor.VariableResult result = variables.get(variable);
-      if(result != null && result.getExpression() != null) {
-        return result.getExpression();
+      VariableResult result = variables.get(variable);
+      if(result != null && result.getExpression().isPresent()) {
+        return result.getExpression().get();
       }
       return null;
     }

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
index 31eeafe..1920031 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
@@ -31,7 +31,6 @@ import org.apache.metron.test.utils.UnitTestHelper;
 import org.json.simple.parser.JSONParser;
 import org.json.simple.JSONObject;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java
index 51a24b3..d552aa1 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java
@@ -23,7 +23,7 @@ import org.apache.metron.common.configuration.enrichment.EnrichmentConfig;
 import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.stellar.common.StellarProcessor;
-import org.apache.metron.stellar.common.shell.StellarExecutor;
+import org.apache.metron.stellar.common.shell.VariableResult;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.DefaultVariableResolver;
 import org.apache.metron.stellar.dsl.StellarFunctions;
@@ -44,7 +44,7 @@ import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMEN
 public class EnrichmentConfigFunctionsTest {
 
   String configStr = emptyTransformationsConfig();
-  Map<String, StellarExecutor.VariableResult> variables;
+  Map<String, VariableResult> variables;
   Context context = null;
   String enrichmentType = null;
   String group = null;
@@ -73,17 +73,15 @@ public class EnrichmentConfigFunctionsTest {
     });
   }
 
-
-
   @Before
   public void setup() {
     variables = ImmutableMap.of(
-            "upper", new StellarExecutor.VariableResult("TO_UPPER('foo')", "FOO"),
-            "lower", new StellarExecutor.VariableResult("TO_LOWER('FOO')", "foo")
+            "upper", VariableResult.withExpression("FOO", "TO_UPPER('foo')"),
+            "lower", VariableResult.withExpression("foo", "TO_LOWER('FOO')")
     );
 
     context = new Context.Builder()
-            .with(StellarExecutor.SHELL_VARIABLES, () -> variables)
+            .with(Context.Capabilities.SHELL_VARIABLES, () -> variables)
             .build();
   }
 
@@ -151,7 +149,7 @@ public class EnrichmentConfigFunctionsTest {
     );
     Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig));
     Assert.assertEquals(1, size(stellarFunctions));
-    Assert.assertEquals(variables.get("upper").getExpression(), get(stellarFunctions,"upper"));
+    Assert.assertEquals(variables.get("upper").getExpression().get(), get(stellarFunctions,"upper"));
   }
 
   @Test
@@ -173,8 +171,8 @@ public class EnrichmentConfigFunctionsTest {
     );
     Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig));
     Assert.assertEquals(2, size(stellarFunctions));
-    Assert.assertEquals(variables.get("upper").getExpression(), get(stellarFunctions,"upper"));
-    Assert.assertEquals(variables.get("lower").getExpression(), get(stellarFunctions,"lower"));
+    Assert.assertEquals(variables.get("upper").getExpression().get(), get(stellarFunctions,"upper"));
+    Assert.assertEquals(variables.get("lower").getExpression().get(), get(stellarFunctions,"lower"));
   }
 
   @Test
@@ -208,7 +206,7 @@ public class EnrichmentConfigFunctionsTest {
     );
     Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig));
     Assert.assertEquals(1, size(stellarFunctions));
-    Assert.assertEquals(variables.get("upper").getExpression(), get(stellarFunctions,"upper"));
+    Assert.assertEquals(variables.get("upper").getExpression().get(), get(stellarFunctions,"upper"));
   }
 
   @Test
@@ -229,7 +227,7 @@ public class EnrichmentConfigFunctionsTest {
     );
     Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig));
     Assert.assertEquals(1, size(stellarFunctions));
-    Assert.assertEquals(variables.get("lower").getExpression(), get(stellarFunctions,"lower"));
+    Assert.assertEquals(variables.get("lower").getExpression().get(), get(stellarFunctions,"lower"));
   }
 
   @Test
@@ -270,7 +268,7 @@ public class EnrichmentConfigFunctionsTest {
     );
     Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig));
     Assert.assertEquals(1, size(stellarFunctions));
-    Assert.assertEquals(variables.get("lower").getExpression(), get(stellarFunctions,"lower"));
+    Assert.assertEquals(variables.get("lower").getExpression().get(), get(stellarFunctions,"lower"));
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java
index 29c80b8..c931df5 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java
@@ -19,11 +19,11 @@ package org.apache.metron.management;
 
 import com.google.common.collect.ImmutableMap;
 import org.apache.metron.common.configuration.IndexingConfigurations;
+import org.apache.metron.stellar.common.shell.VariableResult;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.DefaultVariableResolver;
 import org.apache.metron.stellar.dsl.StellarFunctions;
 import org.apache.metron.stellar.common.StellarProcessor;
-import org.apache.metron.stellar.common.shell.StellarExecutor;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -36,7 +36,7 @@ import static org.apache.metron.management.EnrichmentConfigFunctionsTest.toMap;
 
 public class IndexingConfigFunctionsTest {
 
-  Map<String, StellarExecutor.VariableResult> variables;
+  Map<String, VariableResult> variables;
   Context context = null;
 
   private Object run(String rule, Map<String, Object> variables) {
@@ -47,12 +47,12 @@ public class IndexingConfigFunctionsTest {
   @Before
   public void setup() {
     variables = ImmutableMap.of(
-            "upper", new StellarExecutor.VariableResult("TO_UPPER('foo')", "FOO"),
-            "lower", new StellarExecutor.VariableResult("TO_LOWER('FOO')", "foo")
+            "upper", VariableResult.withExpression("FOO", "TO_UPPER('foo')"),
+            "lower", VariableResult.withExpression("foo", "TO_LOWER('FOO')")
     );
 
     context = new Context.Builder()
-            .with(StellarExecutor.SHELL_VARIABLES, () -> variables)
+            .with(Context.Capabilities.SHELL_VARIABLES, () -> variables)
             .build();
   }
 

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java
index f830e62..e27a7ad 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java
@@ -21,8 +21,8 @@ import com.google.common.collect.ImmutableMap;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.common.configuration.FieldTransformer;
 import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.stellar.common.shell.VariableResult;
 import org.apache.metron.stellar.dsl.Context;
-import org.apache.metron.stellar.common.shell.StellarExecutor;
 import org.json.simple.JSONObject;
 import org.junit.Assert;
 import org.junit.Before;
@@ -40,17 +40,17 @@ public class ParserConfigFunctionsTest {
 
   String emptyTransformationsConfig = slurp(PARSER_CONFIGS_PATH + "/parsers/bro.json");
   String existingTransformationsConfig = slurp(PARSER_CONFIGS_PATH + "/parsers/squid.json");
-  Map<String, StellarExecutor.VariableResult> variables ;
+  Map<String, VariableResult> variables ;
   Context context = null;
   @Before
   public void setup() {
     variables = ImmutableMap.of(
-            "upper" , new StellarExecutor.VariableResult("TO_UPPER('foo')", "FOO"),
-            "lower" , new StellarExecutor.VariableResult("TO_LOWER('FOO')", "foo")
+            "upper" , VariableResult.withExpression("FOO", "TO_UPPER('foo')"),
+            "lower" , VariableResult.withExpression("foo", "TO_LOWER('FOO'")
     );
 
     context = new Context.Builder()
-            .with(StellarExecutor.SHELL_VARIABLES , () -> variables)
+            .with(Context.Capabilities.SHELL_VARIABLES, () -> variables)
             .build();
   }
 

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
index 401454e..83c2bce 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
@@ -19,27 +19,28 @@ package org.apache.metron.management;
 
 import com.google.common.collect.ImmutableMap;
 import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.stellar.common.shell.VariableResult;
 import org.apache.metron.stellar.dsl.Context;
-import org.apache.metron.stellar.common.shell.StellarExecutor;
 import org.junit.Assert;
 import org.junit.Test;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
 
 public class ShellFunctionsTest {
 
-  Map<String, StellarExecutor.VariableResult> variables = ImmutableMap.of(
-          "var1" , new StellarExecutor.VariableResult("TO_UPPER('casey')", "CASEY"),
-          "var2" , new StellarExecutor.VariableResult(null, "foo"),
-          "var3" , new StellarExecutor.VariableResult(null, null),
-          "var4" , new StellarExecutor.VariableResult("blah", null)
+  Map<String, VariableResult> variables = ImmutableMap.of(
+          "var1" , VariableResult.withExpression("CASEY", "TO_UPPER('casey')"),
+          "var2" , VariableResult.withValue("foo"),
+          "var3" , VariableResult.withValue(null),
+          "var4" , VariableResult.withExpression(null, "blah")
   );
 
   Context context = new Context.Builder()
-            .with(StellarExecutor.SHELL_VARIABLES , () -> variables)
+            .with(Context.Capabilities.SHELL_VARIABLES , () -> variables)
             .build();
 /**
 ╔══════════╤═══════╤════════════╗
@@ -53,10 +54,11 @@ public class ShellFunctionsTest {
 
   @Test
   public void testListVarsWithVars() {
-    Map<String, StellarExecutor.VariableResult> variables = ImmutableMap.of("foo", new StellarExecutor.VariableResult("1 + 1", 2.0));
+    Map<String, VariableResult> variables = ImmutableMap.of(
+            "foo", VariableResult.withExpression(2.0, "1 + 1"));
 
     Context context = new Context.Builder()
-            .with(StellarExecutor.SHELL_VARIABLES , () -> variables)
+            .with(Context.Capabilities.SHELL_VARIABLES , () -> variables)
             .build();
     Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
     Assert.assertEquals(expectedListWithFoo, out);
@@ -75,7 +77,7 @@ public class ShellFunctionsTest {
   @Test
   public void testListVarsWithoutVars() {
     Context context = new Context.Builder()
-            .with(StellarExecutor.SHELL_VARIABLES , () -> new HashMap<>())
+            .with(Context.Capabilities.SHELL_VARIABLES, () -> new HashMap<>())
             .build();
     Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
     Assert.assertEquals(expectedEmptyList, out);

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java
index 2b154d8..e281038 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java
@@ -22,7 +22,7 @@ import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.common.configuration.enrichment.threatintel.RiskLevelRule;
 import org.apache.metron.stellar.common.StellarProcessor;
-import org.apache.metron.stellar.common.shell.StellarExecutor;
+import org.apache.metron.stellar.common.shell.VariableResult;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.DefaultVariableResolver;
 import org.apache.metron.stellar.dsl.MapVariableResolver;
@@ -44,18 +44,18 @@ import static org.apache.metron.management.EnrichmentConfigFunctionsTest.toMap;
 public class ThreatTriageFunctionsTest {
 
   String configStr = emptyTransformationsConfig();
-  Map<String, StellarExecutor.VariableResult> variables;
+  Map<String, VariableResult> variables;
   Context context = null;
 
  @Before
   public void setup() {
     variables = ImmutableMap.of(
-            "less", new StellarExecutor.VariableResult("1 < 2", true),
-            "greater", new StellarExecutor.VariableResult("1 > 2", false)
+            "less", VariableResult.withExpression(true, "1 < 2"),
+            "greater", VariableResult.withExpression(false, "1 > 2")
     );
 
     context = new Context.Builder()
-            .with(StellarExecutor.SHELL_VARIABLES, () -> variables)
+            .with(Context.Capabilities.SHELL_VARIABLES, () -> variables)
             .build();
   }
 
@@ -127,7 +127,7 @@ public class ThreatTriageFunctionsTest {
     List<RiskLevelRule> triageRules = getTriageRules(newConfig);
     Assert.assertEquals(1, triageRules.size());
     RiskLevelRule rule = triageRules.get(0);
-    Assert.assertEquals(variables.get("less").getExpression(), rule.getRule() );
+    Assert.assertEquals(variables.get("less").getExpression().get(), rule.getRule() );
     Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 );
   }
 
@@ -165,11 +165,11 @@ public class ThreatTriageFunctionsTest {
     List<RiskLevelRule> triageRules = getTriageRules(newConfig);
     Assert.assertEquals(2, triageRules.size());
     RiskLevelRule less = triageRules.get(0);
-    Assert.assertEquals(variables.get("less").getExpression(), less.getRule() );
+    Assert.assertEquals(variables.get("less").getExpression().get(), less.getRule() );
     Assert.assertEquals(10.0, less.getScore().doubleValue(), 1e-6 );
 
     RiskLevelRule greater = triageRules.get(1);
-    Assert.assertEquals(variables.get("greater").getExpression(), greater.getRule() );
+    Assert.assertEquals(variables.get("greater").getExpression().get(), greater.getRule() );
     Assert.assertEquals(20.0, greater.getScore().doubleValue(), 1e-6 );
   }
 
@@ -200,7 +200,7 @@ public class ThreatTriageFunctionsTest {
     List<RiskLevelRule> triageRules = getTriageRules(newConfig);
     Assert.assertEquals(1, triageRules.size());
     RiskLevelRule rule = triageRules.get(0);
-    Assert.assertEquals(variables.get("less").getExpression(), rule.getRule() );
+    Assert.assertEquals(variables.get("less").getExpression().get(), rule.getRule() );
     Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 );
   }
 
@@ -256,7 +256,7 @@ public class ThreatTriageFunctionsTest {
     List<RiskLevelRule> triageRules = getTriageRules(newConfig);
     Assert.assertEquals(1, triageRules.size());
     RiskLevelRule rule = triageRules.get(0);
-    Assert.assertEquals(variables.get("less").getExpression(), rule.getRule() );
+    Assert.assertEquals(variables.get("less").getExpression().get(), rule.getRule() );
     Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 );
   }
 
@@ -281,7 +281,7 @@ public class ThreatTriageFunctionsTest {
     List<RiskLevelRule> triageRules = engine.getRiskLevelRules();
     Assert.assertEquals(1, triageRules.size());
     RiskLevelRule rule = triageRules.get(0);
-    Assert.assertEquals(variables.get("less").getExpression(), rule.getRule() );
+    Assert.assertEquals(variables.get("less").getExpression().get(), rule.getRule() );
     Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 );
   }
 
@@ -320,11 +320,11 @@ public class ThreatTriageFunctionsTest {
     List<RiskLevelRule> triageRules = getTriageRules(newConfig);
     Assert.assertEquals(2, triageRules.size());
     RiskLevelRule less = triageRules.get(0);
-    Assert.assertEquals(variables.get("less").getExpression(), less.getRule() );
+    Assert.assertEquals(variables.get("less").getExpression().get(), less.getRule() );
     Assert.assertEquals(10.0, less.getScore().doubleValue(), 1e-6 );
 
     RiskLevelRule greater = triageRules.get(1);
-    Assert.assertEquals(variables.get("greater").getExpression(), greater.getRule() );
+    Assert.assertEquals(variables.get("greater").getExpression().get(), greater.getRule() );
     Assert.assertEquals(20.0, greater.getScore().doubleValue(), 1e-6 );
   }
 

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/pom.xml b/metron-stellar/pom.xml
index c2181fd..6b5cb67 100644
--- a/metron-stellar/pom.xml
+++ b/metron-stellar/pom.xml
@@ -43,6 +43,7 @@
     <modules>
         <module>stellar-3rd-party-example</module>
         <module>stellar-common</module>
+        <module>stellar-zeppelin</module>
     </modules>
     <dependencies>
         <dependency>

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md
index 4796889..076f250 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -29,6 +29,7 @@ For a variety of components (threat intelligence triage and field transformation
     * [Variable Assignment](#variable-assignment)
     * [Magic Commands](#magic-commands)
     * [Advanced Usage](#advanced-usage)
+    * [Implementation](#implementation)
 * [Stellar Configuration](#stellar-configuration)
 
 
@@ -1387,7 +1388,7 @@ IS_EMAIL
 To run the Stellar Shell directly from the Metron source code, run a command like the following.  Ensure that Metron has already been built and installed with `mvn clean install -DskipTests`.
 ```
 $ mvn exec:java \
-   -Dexec.mainClass="org.apache.metron.stellar.common.shell.StellarShell" \
+   -Dexec.mainClass="org.apache.metron.stellar.common.shell.cli.StellarShell" \
    -pl metron-platform/metron-enrichment
 ...
 Stellar, Go!
@@ -1403,7 +1404,7 @@ This can be useful for troubleshooting function resolution problems.  The previo
 
 ```
  $ mvn exec:java \
-   -Dexec.mainClass="org.apache.metron.stellar.common.shell.StellarShell" \
+   -Dexec.mainClass="org.apache.metron.stellar.common.shell.cli.StellarShell" \
    -pl metron-analytics/metron-profiler
 ...
 Stellar, Go!
@@ -1413,6 +1414,36 @@ Please note that functions are loading lazily in the background and will be unav
 ABS, APPEND_IF_MISSING, BIN, BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, CHOMP, CHOP, COUNT_MATCHES, DAY_OF_MONTH, DAY_OF_WEEK, DAY_OF_YEAR, DOMAIN_REMOVE_SUBDOMAINS, DOMAIN_REMOVE_TLD, DOMAIN_TO_TLD, ENDS_WITH, FILL_LEFT, FILL_RIGHT, FILTER, FORMAT, GET, GET_FIRST, GET_LAST, HLLP_ADD, HLLP_CARDINALITY, HLLP_INIT, HLLP_MERGE, IN_SUBNET, IS_DATE, IS_DOMAIN, IS_EMAIL, IS_EMPTY, IS_INTEGER, IS_IP, IS_URL, JOIN, LENGTH, LIST_ADD, MAAS_GET_ENDPOINT, MAAS_MODEL_APPLY, MAP, MAP_EXISTS, MAP_GET, MONTH, OUTLIER_MAD_ADD, OUTLIER_MAD_SCORE, OUTLIER_MAD_STATE_MERGE, PREPEND_IF_MISSING, PROFILE_FIXED, PROFILE_GET, PROFILE_WINDOW, PROTOCOL_TO_NAME, REDUCE, REGEXP_MATCH, SPLIT, STARTS_WITH, STATS_ADD, STATS_BIN, STATS_COUNT, STATS_GEOMETRIC_MEAN, STATS_INIT, STATS_KURTOSIS, STATS_MAX, STATS_MEAN, STATS_MERGE, STATS_MIN, STATS_PERCENTILE, STATS_POPULATION_VARIANCE, STATS_QUADRATIC_MEAN, STATS_SD, STATS_SKEWNESS, STATS_SUM, STATS_SUM_LOGS, STATS_SUM_SQUARES, STATS_VARIANCE, STRING_ENTROPY, SYS
 TEM_ENV_GET, SYSTEM_PROPERTY_GET, TO_DOUBLE, TO_EPOCH_TIMESTAMP, TO_FLOAT, TO_INTEGER, TO_LONG, TO_LOWER, TO_STRING, TO_UPPER, TRIM, URL_TO_HOST, URL_TO_PATH, URL_TO_PORT, URL_TO_PROTOCOL, WEEK_OF_MONTH, WEEK_OF_YEAR, YEAR 
 ```
 
+### Implementation
+
+The Stellar Shell can be executed both from the command line and from within a Stellar Notebook.  The behavior and underlying implementation of the behavior is exactly the same across these two environments.
+
+#### `org.apache.metron.stellar.common.shell`  
+
+This package contains classes that are reused across both the CLI and Zeppelin shell environments.
+
+* `StellarShellExecutor` Executes Stellar in a shell-like environment.  Provides the Stellar language extensions like variable assignment, comments, magics, and doc strings that are only accessible in the shell.
+
+* `StellarAutoCompleter` Handles auto-completion for Stellar.
+
+* `StellarExecutorListeners` An event listener that can be notified when variables, functions, and specials are defined.  This is how a `StellarAutoCompleter` is notified throughout the life of a shell session.
+
+#### `org.apache.metron.stellar.common.shell.specials`
+
+All Stellar language extensions are contained within this package.
+
+* `SpecialCommand` The interface for all special commands.  A 'special command' is anything that is not directly provided by the Stellar language itself.  That includes variable assignment, comments, doc strings, magics, and quit. 
+
+#### `org.apache.metron.stellar.common.shell.cli`
+
+This package contains classes that are specific to the CLI-driven REPL.
+
+* `StellarShell`  This is the main class that drives the CLI REPL.  All functionality not related to the command line interface is performed by the shared logic in `org.apache.metron.stellar.common.shell`.
+ 
+#### `org.apache.metron.stellar.zeppelin`
+
+This package is contained within the `stellar-zeppelin` project and performs all logic for interfacing with Zeppelin.  Again, all functionality not related to Zeppelin is performed by the shared logic in `org.apache.metron.stellar.common.shell`.
+
 ## Stellar Configuration
 
 Stellar can be configured in a variety of ways from the [Global Configuration](../../metron-platform/metron-common/README.md#global-configuration).

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml
index fbb1a05..a5dd20b 100644
--- a/metron-stellar/stellar-common/pom.xml
+++ b/metron-stellar/stellar-common/pom.xml
@@ -29,7 +29,6 @@
         <commons.config.version>1.10</commons.config.version>
     </properties>
     <dependencies>
-
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-auth</artifactId>
@@ -240,7 +239,6 @@
                     </execution>
                 </executions>
             </plugin>
-
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java
index e77211a..587b558 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java
@@ -38,7 +38,9 @@ public class StellarAssignment implements Map.Entry<String, Object>{
   }
 
   public static boolean isAssignment(String statement) {
-    return statement != null && statement.contains(":=");
+    return statement != null &&
+            statement.contains(":=") && // has the assignment operator
+            !statement.trim().startsWith("%"); // not a magic like %define x := 2
   }
 
   public static StellarAssignment from(String statement) {

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java
new file mode 100644
index 0000000..cf52955
--- /dev/null
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java
@@ -0,0 +1,220 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.stellar.common.shell;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+import org.apache.commons.collections4.IterableUtils;
+import org.apache.commons.collections4.trie.PatriciaTrie;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+
+/**
+ * Provides auto-completion for Stellar.
+ */
+public class DefaultStellarAutoCompleter implements StellarAutoCompleter {
+
+  enum OperationType {
+    DOC,
+    MAGIC,
+    NORMAL
+  }
+
+  enum AutoCompleteType implements AutoCompleteTransformation {
+    FUNCTION((type, key) -> {
+      if(OperationType.DOC == type) {
+        return "?" + key;
+
+      } else if(OperationType.NORMAL == type) {
+        return key + "(";
+      }
+
+      return key;
+    }),
+    VARIABLE((type, key) -> key),
+    TOKEN((type, key) -> key);
+
+    AutoCompleteTransformation transform;
+
+    AutoCompleteType(AutoCompleteTransformation transform) {
+      this.transform = transform;
+    }
+
+    @Override
+    public String transform(OperationType type, String key) {
+      return transform.transform(type, key);
+    }
+  }
+
+  /**
+   * Prefix tree index of auto-completes.
+   */
+  private PatriciaTrie<AutoCompleteType> autocompleteIndex;
+
+  private ReadWriteLock indexLock = new ReentrantReadWriteLock();
+
+  public interface AutoCompleteTransformation {
+    String transform(OperationType type, String key);
+  }
+
+  public DefaultStellarAutoCompleter() {
+    this.autocompleteIndex = initializeIndex();
+  }
+
+  @Override
+  public Iterable<String> autoComplete(String buffer) {
+    Iterable<String> candidates = IterableUtils.emptyIterable();
+
+    final String lastToken = getLastToken(buffer);
+    if(StringUtils.isNotEmpty(lastToken)) {
+
+      if (isDoc(lastToken)) {
+        candidates = autoCompleteDoc(lastToken.substring(1));
+
+      } else if (isMagic(lastToken)) {
+        candidates = autoCompleteMagic(lastToken);
+
+      } else {
+        candidates = autoCompleteNormal(lastToken);
+      }
+    }
+
+    return candidates;
+  }
+
+  /**
+   * Is a given expression a built-in magic?
+   * @param expression The expression.
+   */
+  private boolean isMagic(String expression) {
+    return StringUtils.startsWith(expression, "%");
+  }
+
+  /**
+   * Is a given expression asking for function documentation?
+   * @param expression The expression.
+   */
+  private boolean isDoc(String expression) {
+    return StringUtils.startsWith(expression, "?");
+  }
+
+  /**
+   * Auto-completes a partial Stellar expression
+   * @param buffer The partial buffer that needs auto-completed.
+   * @return Viable candidates for auto-completion.
+   */
+  private Iterable<String> autoCompleteNormal(String buffer) {
+    return autoComplete(buffer, OperationType.NORMAL);
+  }
+
+  /**
+   * Auto-completes a partial doc command.
+   * @param buffer The partial buffer that needs auto-completed.
+   * @return Viable candidates for auto-completion.
+   */
+  private Iterable<String> autoCompleteDoc(String buffer) {
+    return autoComplete(buffer, OperationType.DOC);
+  }
+
+  /**
+   * Auto-completes a partial magic commands.
+   * @param buffer The partial buffer that needs auto-completed.
+   * @return Viable candidates for auto-completion.
+   */
+  private Iterable<String> autoCompleteMagic(String buffer) {
+    return autoComplete(buffer, OperationType.MAGIC);
+  }
+
+  /**
+   * Returns a list of viable candidates for auto-completion.
+   * @param buffer The current buffer.
+   * @param opType The type of operation needing auto-completion.
+   * @return Viable candidates for auto-completion.
+   */
+  private Iterable<String> autoComplete(String buffer, final OperationType opType) {
+    indexLock.readLock().lock();
+    try {
+      SortedMap<String, AutoCompleteType> ret = autocompleteIndex.prefixMap(buffer);
+      if (ret.isEmpty()) {
+        return new ArrayList<>();
+      }
+      return Iterables.transform(ret.entrySet(), kv -> kv.getValue().transform(opType, kv.getKey()));
+    }
+    finally {
+      indexLock.readLock().unlock();
+    }
+  }
+
+  /**
+   * Adds a candidate for auto-completing function names.
+   * @param name The name of the function candidate.
+   */
+  @Override
+  public void addCandidateFunction(String name) {
+    add(name, AutoCompleteType.FUNCTION);
+  }
+
+  /**
+   * Adds a candidate for auto-completing variable names.
+   * @param name The name of the function candidate.
+   */
+  @Override
+  public void addCandidateVariable(String name) {
+    add(name, AutoCompleteType.VARIABLE);
+  }
+
+  /**
+   * Add a candidate for auto-completion.
+   * @param name The name of the candidate.
+   * @param type The type of candidate.
+   */
+  private void add(String name, AutoCompleteType type) {
+    if(StringUtils.isNotBlank(name)) {
+      // add the candidate to the auto-complete index
+      indexLock.writeLock().lock();
+      try {
+        this.autocompleteIndex.put(name, type);
+      } finally {
+        indexLock.writeLock().unlock();
+      }
+    }
+  }
+
+  private PatriciaTrie<AutoCompleteType> initializeIndex() {
+    Map<String, AutoCompleteType> index = new HashMap<>();
+    index.put("==", AutoCompleteType.TOKEN);
+    index.put(">=", AutoCompleteType.TOKEN);
+    index.put("<=", AutoCompleteType.TOKEN);
+
+    return new PatriciaTrie<>(index);
+  }
+
+  private static String getLastToken(String buffer) {
+    String lastToken = Iterables.getLast(Splitter.on(" ").split(buffer), null);
+    return lastToken.trim();
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
new file mode 100644
index 0000000..f83bb9e
--- /dev/null
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
@@ -0,0 +1,413 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.stellar.common.shell;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.google.common.collect.Maps;
+import org.apache.commons.collections.map.UnmodifiableMap;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.metron.stellar.common.StellarProcessor;
+import org.apache.metron.stellar.common.configuration.ConfigurationsUtils;
+import org.apache.metron.stellar.common.shell.StellarExecutionListeners.FunctionDefinedListener;
+import org.apache.metron.stellar.common.shell.StellarExecutionListeners.SpecialDefinedListener;
+import org.apache.metron.stellar.common.shell.StellarExecutionListeners.VariableDefinedListener;
+import org.apache.metron.stellar.common.shell.specials.AssignmentCommand;
+import org.apache.metron.stellar.common.shell.specials.Comment;
+import org.apache.metron.stellar.common.shell.specials.DocCommand;
+import org.apache.metron.stellar.common.shell.specials.MagicDefineGlobal;
+import org.apache.metron.stellar.common.shell.specials.MagicListFunctions;
+import org.apache.metron.stellar.common.shell.specials.MagicListGlobals;
+import org.apache.metron.stellar.common.shell.specials.MagicListVariables;
+import org.apache.metron.stellar.common.shell.specials.MagicUndefineGlobal;
+import org.apache.metron.stellar.common.shell.specials.QuitCommand;
+import org.apache.metron.stellar.common.shell.specials.SpecialCommand;
+import org.apache.metron.stellar.common.utils.JSONUtils;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.MapVariableResolver;
+import org.apache.metron.stellar.dsl.StellarFunctionInfo;
+import org.apache.metron.stellar.dsl.StellarFunctions;
+import org.apache.metron.stellar.dsl.VariableResolver;
+import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+
+import static org.apache.metron.stellar.common.configuration.ConfigurationsUtils.readGlobalConfigBytesFromZookeeper;
+import static org.apache.metron.stellar.common.shell.StellarResult.noop;
+import static org.apache.metron.stellar.common.shell.StellarResult.error;
+import static org.apache.metron.stellar.common.shell.StellarResult.success;
+import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG;
+import static org.apache.metron.stellar.dsl.Context.Capabilities.STELLAR_CONFIG;
+import static org.apache.metron.stellar.dsl.Context.Capabilities.ZOOKEEPER_CLIENT;
+
+/**
+ * Default implementation of a StellarShellExecutor.
+ */
+public class DefaultStellarShellExecutor implements StellarShellExecutor {
+
+  private static final Logger LOG =  LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  public static final String SHELL_VARIABLES = "shellVariables";
+
+  /**
+   * The variables known by Stellar.
+   */
+  private Map<String, VariableResult> variables;
+
+  /**
+   * The function resolver.
+   */
+  private FunctionResolver functionResolver;
+
+  /**
+   * A Zookeeper client. Only defined if given a valid Zookeeper URL.
+   */
+  private Optional<CuratorFramework> zkClient;
+
+  /**
+   * A registry of all special commands; like %magic, ?doc, and quit.
+   */
+  private List<SpecialCommand> specials;
+
+  /**
+   * The Stellar execution context.
+   */
+  private Context context;
+
+  /**
+   * Listeners that are notified when a function is defined.
+   */
+  private List<FunctionDefinedListener> functionListeners;
+
+  /**
+   * Listeners that are notified when a variable is defined.
+   */
+  private List<VariableDefinedListener> variableListeners;
+
+  /**
+   * Listeners that are notified when a special command is defined.
+   */
+  private List<SpecialDefinedListener> specialListeners;
+
+
+  public DefaultStellarShellExecutor(
+          FunctionResolver functionResolver,
+          Properties properties,
+          Optional<String> zookeeperUrl,
+          List<SpecialCommand> specials) throws Exception {
+
+    this.functionListeners = new ArrayList<>();
+    this.variableListeners = new ArrayList<>();
+    this.specialListeners = new ArrayList<>();
+    this.variables = new HashMap<>();
+    this.zkClient = createZookeeperClient(zookeeperUrl);
+    this.context = createContext(properties, this.zkClient);
+    this.functionResolver = functionResolver;
+    this.specials = specials;
+  }
+
+  public DefaultStellarShellExecutor(
+          FunctionResolver functionResolver,
+          Properties properties,
+          Optional<String> zookeeperUrl) throws Exception {
+
+    this(functionResolver, properties, zookeeperUrl, defaultSpecials());
+  }
+
+  public DefaultStellarShellExecutor(
+          Properties properties,
+          Optional<String> zookeeperUrl) throws Exception {
+
+    this(StellarFunctions.FUNCTION_RESOLVER(), properties, zookeeperUrl);
+  }
+
+  /**
+   * The default specials that will be made available, if none are specified otherwise.
+   * @return The default special commands.
+   */
+  public static List<SpecialCommand> defaultSpecials() {
+    return Arrays.asList(
+            new AssignmentCommand(),
+            new DocCommand(),
+            new QuitCommand(),
+            new Comment(),
+            new MagicListFunctions(),
+            new MagicListVariables(),
+            new MagicDefineGlobal(),
+            new MagicUndefineGlobal(),
+            new MagicListGlobals()
+    );
+  }
+
+  @Override
+  public void init() {
+    StellarFunctions.initialize(this.context);
+
+    // notify listeners about the available specials
+    for(SpecialCommand command : specials) {
+      notifySpecialListeners(command);
+    }
+
+    // notify listeners about the available functions
+    for(StellarFunctionInfo fn : functionResolver.getFunctionInfo()) {
+      notifyFunctionListeners(fn);
+    }
+  }
+
+  /**
+   * Add a listener that will be notified when a function is defined.
+   * @param listener The listener to notify.
+   */
+  @Override
+  public void addFunctionListener(FunctionDefinedListener listener) {
+    this.functionListeners.add(listener);
+  }
+
+  /**
+   * Notify function listeners that a function has been defined.
+   * @param functionInfo The function that was defined.
+   */
+  private void notifyFunctionListeners(StellarFunctionInfo functionInfo) {
+    for(FunctionDefinedListener listener : functionListeners) {
+      listener.whenFunctionDefined(functionInfo);
+    }
+  }
+
+  /**
+   * Add a listener that will be notified when a variable is defined.
+   * @param listener The listener to notify.
+   */
+  @Override
+  public void addVariableListener(VariableDefinedListener listener) {
+    this.variableListeners.add(listener);
+  }
+
+  /**
+   * Notify variable listeners that a variable has been (re)defined.
+   * @param variableName The variable name.
+   * @param result The variable result.
+   */
+  private void notifyVariableListeners(String variableName, VariableResult result) {
+    for(VariableDefinedListener listener : variableListeners) {
+      listener.whenVariableDefined(variableName, result);
+    }
+  }
+
+  /**
+   * Add a listener that will be notified when a magic command is defined.
+   * @param listener The listener to notify.
+   */
+  @Override
+  public void addSpecialListener(SpecialDefinedListener listener) {
+    this.specialListeners.add(listener);
+  }
+
+  /**
+   * Notify listeners that a magic command has been defined.
+   * @param specialCommand The magic command.
+   */
+  private void notifySpecialListeners(SpecialCommand specialCommand) {
+    for(SpecialDefinedListener listener : specialListeners) {
+      listener.whenSpecialDefined(specialCommand);
+    }
+  }
+
+  @Override
+  public StellarResult execute(String expression) {
+
+    // if only whitespace, there is nothing to do
+    expression = StringUtils.trimToEmpty(expression);
+    if(StringUtils.isBlank(expression)) {
+      return noop();
+    }
+
+    // is this a special command?
+    for(SpecialCommand command : specials) {
+      if(command.getMatcher().apply(expression)) {
+        return command.execute(expression, this);
+      }
+    }
+
+    // otherwise, this must be a stellar expression
+    return executeStellar(expression);
+  }
+
+  /**
+   * Retrieves the GLOBAL_CONFIG, if it exists.  If it does not, it creates the GLOBAL_CONFIG
+   * and adds it to the Stellar execution context.
+   *
+   * @return The global configuration.
+   */
+  @Override
+  public Map<String, Object> getGlobalConfig() {
+    Map<String, Object> globals;
+    Optional<Object> capability = getContext().getCapability(GLOBAL_CONFIG, false);
+    if (capability.isPresent()) {
+      globals = (Map<String, Object>) capability.get();
+
+    } else {
+      // if it does not exist, create it.  this creates the global config for the current stellar executor
+      // session only.  this does not change the global config maintained externally in zookeeper
+      globals = new HashMap<>();
+      getContext().addCapability(GLOBAL_CONFIG, () -> globals);
+    }
+
+    return globals;
+  }
+
+  @Override
+  public void assign(String variableName, Object value, Optional<String> expression) {
+
+    // perform the variable assignment
+    VariableResult varResult = VariableResult.withExpression(value, expression);
+    this.variables.put(variableName, varResult);
+
+    // notify any listeners
+    notifyVariableListeners(variableName, varResult);
+  }
+
+  @Override
+  public FunctionResolver getFunctionResolver() {
+    return functionResolver;
+  }
+
+  @Override
+  public Map<String, VariableResult> getState() {
+    return UnmodifiableMap.decorate(variables);
+  }
+
+  /**
+   * Returns all variables that have been defined.  Unlike 'getState' this unwraps
+   * the VariableResult so that we have the actual value.
+   *
+   * @return All variables that have been defined.
+   */
+  public Map<String, Object> getVariables() {
+    return Maps.transformValues(variables, (v) -> v.getResult());
+  }
+
+  @Override
+  public Context getContext() {
+    return context;
+  }
+
+  /**
+   * Creates a Zookeeper client.
+   * @param zookeeperUrl The Zookeeper URL.
+   */
+  private Optional<CuratorFramework> createZookeeperClient(Optional<String> zookeeperUrl) {
+    Optional<CuratorFramework> client = Optional.empty();
+
+    // can only create client, if have valid zookeeper URL
+    if(zookeeperUrl.isPresent()) {
+      String url = zookeeperUrl.get();
+      if (StringUtils.isNotBlank(url)) {
+
+        LOG.debug(String.format("Connecting to Zookeeper; url=%s", url));
+        CuratorFramework c = ConfigurationsUtils.getClient(url);
+        c.start();
+        client = Optional.of(c);
+      }
+    }
+
+    return client;
+  }
+
+  /**
+   * Creates a Context initialized with configuration stored in Zookeeper.
+   * @param properties Properties to configure the context.
+   * @param zkClient An optional Zookeeper client.
+   */
+  private Context createContext(Properties properties, Optional<CuratorFramework> zkClient) throws Exception {
+
+    Context.Builder contextBuilder = new Context.Builder()
+            .with(SHELL_VARIABLES, () -> variables)
+            .with(STELLAR_CONFIG, () -> properties);
+
+    // load global configuration from zookeeper
+    if (zkClient.isPresent()) {
+      Map<String, Object> global = fetchGlobalConfig(zkClient.get());
+      contextBuilder
+              .with(GLOBAL_CONFIG, () -> global)
+              .with(ZOOKEEPER_CLIENT, () -> zkClient.get())
+              .with(STELLAR_CONFIG, () -> getStellarConfig(global, properties));
+    }
+
+    return contextBuilder.build();
+  }
+
+  /**
+   * Fetches the global configuration from Zookeeper.
+   * @param zkClient The Zookeeper client.
+   * @return The global configuration retrieved from Zookeeper.
+   * @throws Exception
+   */
+  private Map<String, Object> fetchGlobalConfig(CuratorFramework zkClient) throws Exception {
+    byte[] raw = readGlobalConfigBytesFromZookeeper(zkClient);
+    return JSONUtils.INSTANCE.load(
+            new ByteArrayInputStream(raw),
+            new TypeReference<Map<String, Object>>() {});
+  }
+
+  /**
+   * @param globalConfig The global configuration.
+   * @param props Property values
+   * @return The Stellar configuration.
+   */
+  private Map<String, Object> getStellarConfig(Map<String, Object> globalConfig, Properties props) {
+    Map<String, Object> stellarConfig = new HashMap<>();
+    stellarConfig.putAll(globalConfig);
+    if(props != null) {
+      for (Map.Entry<Object, Object> kv : props.entrySet()) {
+        stellarConfig.put(kv.getKey().toString(), kv.getValue());
+      }
+    }
+    return stellarConfig;
+  }
+
+  /**
+   * Executes Stellar expressions.
+   * @param expression The expression to execute.
+   */
+  private StellarResult executeStellar(String expression) {
+    StellarResult result;
+
+    try {
+      // execute the stellar expression
+      VariableResolver variableResolver = new MapVariableResolver(getVariables());
+      Object exprResult = new StellarProcessor().parse(expression, variableResolver, functionResolver, context);
+      result = success(exprResult);
+
+    } catch (Throwable t) {
+      result = error(t);
+    }
+
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/PausableInput.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/PausableInput.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/PausableInput.java
deleted file mode 100644
index 2acd058..0000000
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/PausableInput.java
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- *
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- */
-package org.apache.metron.stellar.common.shell;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * An input stream which mirrors System.in, but allows you to 'pause' and 'unpause' it.
- * The Aeshell has an external thread which is constantly polling System.in.  If you
- * need to spawn a program externally (i.e. an editor) which shares stdin, this thread
- * and the spawned program both share a buffer.  This causes contention and unpredictable
- * results (e.g. an input may be consumed by either the aeshell thread or the spawned program)
- *
- * Because you can inject an input stream into the console, we create this which can act as a
- * facade to System.in under normal 'unpaused' circumstances, and when paused, turn off the
- * access to System.in.  This allows us to turn off access to aeshell while maintaining access
- * to the external program.
- *
- */
-public class PausableInput extends InputStream {
-  InputStream in = System.in;
-  boolean paused = false;
-  private PausableInput() {
-    super();
-  }
-
-  /**
-   * Stop mirroring stdin
-   */
-  public void pause() {
-    paused = true;
-  }
-
-  /**
-   * Resume mirroring stdin.
-   * @throws IOException
-   */
-  public void unpause() throws IOException {
-    in.read(new byte[in.available()]);
-    paused = false;
-  }
-
-  public final static PausableInput INSTANCE = new PausableInput();
-
-  /**
-   * Reads the next byte of data from the input stream. The value byte is
-   * returned as an <code>int</code> in the range <code>0</code> to
-   * <code>255</code>. If no byte is available because the end of the stream
-   * has been reached, the value <code>-1</code> is returned. This method
-   * blocks until input data is available, the end of the stream is detected,
-   * or an exception is thrown.
-   * <p>
-   * <p> A subclass must provide an implementation of this method.
-   *
-   * @return the next byte of data, or <code>-1</code> if the end of the
-   * stream is reached.
-   * @throws IOException if an I/O error occurs.
-   */
-  @Override
-  public int read() throws IOException {
-
-    return in.read();
-  }
-
-  /**
-   * Reads some number of bytes from the input stream and stores them into
-   * the buffer array <code>b</code>. The number of bytes actually read is
-   * returned as an integer.  This method blocks until input data is
-   * available, end of file is detected, or an exception is thrown.
-   * <p>
-   * <p> If the length of <code>b</code> is zero, then no bytes are read and
-   * <code>0</code> is returned; otherwise, there is an attempt to read at
-   * least one byte. If no byte is available because the stream is at the
-   * end of the file, the value <code>-1</code> is returned; otherwise, at
-   * least one byte is read and stored into <code>b</code>.
-   * <p>
-   * <p> The first byte read is stored into element <code>b[0]</code>, the
-   * next one into <code>b[1]</code>, and so on. The number of bytes read is,
-   * at most, equal to the length of <code>b</code>. Let <i>k</i> be the
-   * number of bytes actually read; these bytes will be stored in elements
-   * <code>b[0]</code> through <code>b[</code><i>k</i><code>-1]</code>,
-   * leaving elements <code>b[</code><i>k</i><code>]</code> through
-   * <code>b[b.length-1]</code> unaffected.
-   * <p>
-   * <p> The <code>read(b)</code> method for class <code>InputStream</code>
-   * has the same effect as: <pre><code> read(b, 0, b.length) </code></pre>
-   *
-   * @param b the buffer into which the data is read.
-   * @return the total number of bytes read into the buffer, or
-   * <code>-1</code> if there is no more data because the end of
-   * the stream has been reached.
-   * @throws IOException          If the first byte cannot be read for any reason
-   *                              other than the end of the file, if the input stream has been closed, or
-   *                              if some other I/O error occurs.
-   * @throws NullPointerException if <code>b</code> is <code>null</code>.
-   * @see InputStream#read(byte[], int, int)
-   */
-  @Override
-  public int read(byte[] b) throws IOException {
-
-    if(paused) {
-      try {
-        Thread.sleep(1000);
-      } catch (InterruptedException e) {
-        e.printStackTrace();
-      }
-      return 0;
-    }
-    int ret = in.read(b);
-    return ret;
-  }
-
-  /**
-   * Reads up to <code>len</code> bytes of data from the input stream into
-   * an array of bytes.  An attempt is made to read as many as
-   * <code>len</code> bytes, but a smaller number may be read.
-   * The number of bytes actually read is returned as an integer.
-   * <p>
-   * <p> This method blocks until input data is available, end of file is
-   * detected, or an exception is thrown.
-   * <p>
-   * <p> If <code>len</code> is zero, then no bytes are read and
-   * <code>0</code> is returned; otherwise, there is an attempt to read at
-   * least one byte. If no byte is available because the stream is at end of
-   * file, the value <code>-1</code> is returned; otherwise, at least one
-   * byte is read and stored into <code>b</code>.
-   * <p>
-   * <p> The first byte read is stored into element <code>b[off]</code>, the
-   * next one into <code>b[off+1]</code>, and so on. The number of bytes read
-   * is, at most, equal to <code>len</code>. Let <i>k</i> be the number of
-   * bytes actually read; these bytes will be stored in elements
-   * <code>b[off]</code> through <code>b[off+</code><i>k</i><code>-1]</code>,
-   * leaving elements <code>b[off+</code><i>k</i><code>]</code> through
-   * <code>b[off+len-1]</code> unaffected.
-   * <p>
-   * <p> In every case, elements <code>b[0]</code> through
-   * <code>b[off]</code> and elements <code>b[off+len]</code> through
-   * <code>b[b.length-1]</code> are unaffected.
-   * <p>
-   * <p> The <code>read(b,</code> <code>off,</code> <code>len)</code> method
-   * for class <code>InputStream</code> simply calls the method
-   * <code>read()</code> repeatedly. If the first such call results in an
-   * <code>IOException</code>, that exception is returned from the call to
-   * the <code>read(b,</code> <code>off,</code> <code>len)</code> method.  If
-   * any subsequent call to <code>read()</code> results in a
-   * <code>IOException</code>, the exception is caught and treated as if it
-   * were end of file; the bytes read up to that point are stored into
-   * <code>b</code> and the number of bytes read before the exception
-   * occurred is returned. The default implementation of this method blocks
-   * until the requested amount of input data <code>len</code> has been read,
-   * end of file is detected, or an exception is thrown. Subclasses are encouraged
-   * to provide a more efficient implementation of this method.
-   *
-   * @param b   the buffer into which the data is read.
-   * @param off the start offset in array <code>b</code>
-   *            at which the data is written.
-   * @param len the maximum number of bytes to read.
-   * @return the total number of bytes read into the buffer, or
-   * <code>-1</code> if there is no more data because the end of
-   * the stream has been reached.
-   * @throws IOException               If the first byte cannot be read for any reason
-   *                                   other than end of file, or if the input stream has been closed, or if
-   *                                   some other I/O error occurs.
-   * @throws NullPointerException      If <code>b</code> is <code>null</code>.
-   * @throws IndexOutOfBoundsException If <code>off</code> is negative,
-   *                                   <code>len</code> is negative, or <code>len</code> is greater than
-   *                                   <code>b.length - off</code>
-   * @see InputStream#read()
-   */
-  @Override
-  public int read(byte[] b, int off, int len) throws IOException {
-    if(paused) {
-      try {
-        Thread.sleep(1000);
-      } catch (InterruptedException e) {
-        e.printStackTrace();
-      }
-      return 0;
-    }
-    int ret = in.read(b, off, len);
-    return ret;
-  }
-
-  /**
-   * Skips over and discards <code>n</code> bytes of data from this input
-   * stream. The <code>skip</code> method may, for a variety of reasons, end
-   * up skipping over some smaller number of bytes, possibly <code>0</code>.
-   * This may result from any of a number of conditions; reaching end of file
-   * before <code>n</code> bytes have been skipped is only one possibility.
-   * The actual number of bytes skipped is returned. If {@code n} is
-   * negative, the {@code skip} method for class {@code InputStream} always
-   * returns 0, and no bytes are skipped. Subclasses may handle the negative
-   * value differently.
-   * <p>
-   * <p> The <code>skip</code> method of this class creates a
-   * byte array and then repeatedly reads into it until <code>n</code> bytes
-   * have been read or the end of the stream has been reached. Subclasses are
-   * encouraged to provide a more efficient implementation of this method.
-   * For instance, the implementation may depend on the ability to seek.
-   *
-   * @param n the number of bytes to be skipped.
-   * @return the actual number of bytes skipped.
-   * @throws IOException if the stream does not support seek,
-   *                     or if some other I/O error occurs.
-   */
-  @Override
-  public long skip(long n) throws IOException {
-
-    return in.skip(n);
-  }
-
-  /**
-   * Returns an estimate of the number of bytes that can be read (or
-   * skipped over) from this input stream without blocking by the next
-   * invocation of a method for this input stream. The next invocation
-   * might be the same thread or another thread.  A single read or skip of this
-   * many bytes will not block, but may read or skip fewer bytes.
-   * <p>
-   * <p> Note that while some implementations of {@code InputStream} will return
-   * the total number of bytes in the stream, many will not.  It is
-   * never correct to use the return value of this method to allocate
-   * a buffer intended to hold all data in this stream.
-   * <p>
-   * <p> A subclass' implementation of this method may choose to throw an
-   * {@link IOException} if this input stream has been closed by
-   * invoking the {@link #close()} method.
-   * <p>
-   * <p> The {@code available} method for class {@code InputStream} always
-   * returns {@code 0}.
-   * <p>
-   * <p> This method should be overridden by subclasses.
-   *
-   * @return an estimate of the number of bytes that can be read (or skipped
-   * over) from this input stream without blocking or {@code 0} when
-   * it reaches the end of the input stream.
-   * @throws IOException if an I/O error occurs.
-   */
-  @Override
-  public int available() throws IOException {
-
-    return in.available();
-  }
-
-  /**
-   * Closes this input stream and releases any system resources associated
-   * with the stream.
-   * <p>
-   * <p> The <code>close</code> method of <code>InputStream</code> does
-   * nothing.
-   *
-   * @throws IOException if an I/O error occurs.
-   */
-  @Override
-  public void close() throws IOException {
-    in.close();
-  }
-
-  /**
-   * Marks the current position in this input stream. A subsequent call to
-   * the <code>reset</code> method repositions this stream at the last marked
-   * position so that subsequent reads re-read the same bytes.
-   * <p>
-   * <p> The <code>readlimit</code> arguments tells this input stream to
-   * allow that many bytes to be read before the mark position gets
-   * invalidated.
-   * <p>
-   * <p> The general contract of <code>mark</code> is that, if the method
-   * <code>markSupported</code> returns <code>true</code>, the stream somehow
-   * remembers all the bytes read after the call to <code>mark</code> and
-   * stands ready to supply those same bytes again if and whenever the method
-   * <code>reset</code> is called.  However, the stream is not required to
-   * remember any data at all if more than <code>readlimit</code> bytes are
-   * read from the stream before <code>reset</code> is called.
-   * <p>
-   * <p> Marking a closed stream should not have any effect on the stream.
-   * <p>
-   * <p> The <code>mark</code> method of <code>InputStream</code> does
-   * nothing.
-   *
-   * @param readlimit the maximum limit of bytes that can be read before
-   *                  the mark position becomes invalid.
-   * @see InputStream#reset()
-   */
-  @Override
-  public synchronized void mark(int readlimit) {
-    in.mark(readlimit);
-  }
-
-  /**
-   * Repositions this stream to the position at the time the
-   * <code>mark</code> method was last called on this input stream.
-   * <p>
-   * <p> The general contract of <code>reset</code> is:
-   * <p>
-   * <ul>
-   * <li> If the method <code>markSupported</code> returns
-   * <code>true</code>, then:
-   * <p>
-   * <ul><li> If the method <code>mark</code> has not been called since
-   * the stream was created, or the number of bytes read from the stream
-   * since <code>mark</code> was last called is larger than the argument
-   * to <code>mark</code> at that last call, then an
-   * <code>IOException</code> might be thrown.
-   * <p>
-   * <li> If such an <code>IOException</code> is not thrown, then the
-   * stream is reset to a state such that all the bytes read since the
-   * most recent call to <code>mark</code> (or since the start of the
-   * file, if <code>mark</code> has not been called) will be resupplied
-   * to subsequent callers of the <code>read</code> method, followed by
-   * any bytes that otherwise would have been the next input data as of
-   * the time of the call to <code>reset</code>. </ul>
-   * <p>
-   * <li> If the method <code>markSupported</code> returns
-   * <code>false</code>, then:
-   * <p>
-   * <ul><li> The call to <code>reset</code> may throw an
-   * <code>IOException</code>.
-   * <p>
-   * <li> If an <code>IOException</code> is not thrown, then the stream
-   * is reset to a fixed state that depends on the particular type of the
-   * input stream and how it was created. The bytes that will be supplied
-   * to subsequent callers of the <code>read</code> method depend on the
-   * particular type of the input stream. </ul></ul>
-   * <p>
-   * <p>The method <code>reset</code> for class <code>InputStream</code>
-   * does nothing except throw an <code>IOException</code>.
-   *
-   * @throws IOException if this stream has not been marked or if the
-   *                     mark has been invalidated.
-   * @see InputStream#mark(int)
-   * @see IOException
-   */
-  @Override
-  public synchronized void reset() throws IOException {
-    in.reset();
-  }
-
-  /**
-   * Tests if this input stream supports the <code>mark</code> and
-   * <code>reset</code> methods. Whether or not <code>mark</code> and
-   * <code>reset</code> are supported is an invariant property of a
-   * particular input stream instance. The <code>markSupported</code> method
-   * of <code>InputStream</code> returns <code>false</code>.
-   *
-   * @return <code>true</code> if this stream instance supports the mark
-   * and reset methods; <code>false</code> otherwise.
-   * @see InputStream#mark(int)
-   * @see InputStream#reset()
-   */
-  @Override
-  public boolean markSupported() {
-    return in.markSupported();
-  }
-}

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java
new file mode 100644
index 0000000..35ab5ff
--- /dev/null
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java
@@ -0,0 +1,45 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.stellar.common.shell;
+
+/**
+ * Provides auto-complete functionality for Stellar.
+ */
+public interface StellarAutoCompleter {
+
+  /**
+   * Auto-completes based on the given buffer.
+   * @param buffer The partial buffer that needs auto-completed.
+   * @return Viable candidates for auto-completion.
+   */
+  Iterable<String> autoComplete(String buffer);
+
+  /**
+   * Adds a candidate for auto-completing function names.
+   * @param name The name of the function candidate.
+   */
+  void addCandidateFunction(String name);
+
+  /**
+   * Adds a candidate for auto-completing variable names.
+   * @param name The name of the function candidate.
+   */
+  void addCandidateVariable(String name);
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java
new file mode 100644
index 0000000..9734241
--- /dev/null
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java
@@ -0,0 +1,51 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.stellar.common.shell;
+
+import org.apache.metron.stellar.common.shell.specials.SpecialCommand;
+import org.apache.metron.stellar.dsl.StellarFunctionInfo;
+
+/**
+ * A listener will be notified about events that occur during the
+ * execution of Stellar expressions.
+ */
+public class StellarExecutionListeners {
+
+  /**
+   * A listener that is notified when a function is defined.
+   */
+  public interface FunctionDefinedListener {
+    void whenFunctionDefined(StellarFunctionInfo functionInfo);
+  }
+
+  /**
+   * A listener that is notified when a variable is defined or redefined.
+   */
+  public interface VariableDefinedListener {
+    void whenVariableDefined(String variableName, VariableResult result);
+  }
+
+  /**
+   * A listener that is notified when a special command is defined.
+   */
+  public interface SpecialDefinedListener {
+    void whenSpecialDefined(SpecialCommand magic);
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java
new file mode 100644
index 0000000..f4ebf2e
--- /dev/null
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java
@@ -0,0 +1,44 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.stellar.common.shell;
+
+/**
+ * Notifies listeners when events occur during the execution of Stellar expressions.
+ */
+public interface StellarExecutionNotifier {
+
+  /**
+   * Add a listener that will be notified when a magic command is defined.
+   * @param listener The listener to notify.
+   */
+  void addSpecialListener(StellarExecutionListeners.SpecialDefinedListener listener);
+
+  /**
+   * Add a listener that will be notified when a function is defined.
+   * @param listener The listener to notify.
+   */
+  void addFunctionListener(StellarExecutionListeners.FunctionDefinedListener listener);
+
+  /**
+   * Add a listener that will be notified when a variable is defined.
+   * @param listener The listener to notify.
+   */
+  void addVariableListener(StellarExecutionListeners.VariableDefinedListener listener);
+}


Mime
View raw message