metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [10/10] incubator-metron git commit: METRON-503: Metron REST API this closes apache/incubator-metron#316
Date Thu, 02 Mar 2017 20:51:56 GMT
METRON-503: Metron REST API this closes apache/incubator-metron#316


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/e6628499
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/e6628499
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/e6628499

Branch: refs/heads/master
Commit: e662849932a74187708b0037042c6b06d4750116
Parents: a61dbcf
Author: merrimanr <merrimanr@gmail.com>
Authored: Thu Mar 2 11:04:06 2017 -0500
Committer: cstella <cestella@gmail.com>
Committed: Thu Mar 2 15:24:11 2017 -0500

----------------------------------------------------------------------
 dependencies_with_url.csv                       |   3 +
 metron-interface/metron-rest-client/pom.xml     |   2 +-
 .../metron/rest/model/GrokValidation.java       |  13 +-
 .../metron/rest/model/ParseMessageRequest.java  |   9 +
 .../metron/rest/model/SensorParserContext.java  |  51 +++
 .../metron/rest/model/TopologyResponse.java     |  18 +
 .../metron/rest/model/TopologyStatus.java       |  28 +-
 .../metron/rest/model/TopologySummary.java      |  17 +
 .../rest/model/TransformationValidation.java    |  51 ---
 metron-interface/metron-rest/.gitignore         |   1 +
 metron-interface/metron-rest/README.md          | 222 ++++++++----
 metron-interface/metron-rest/pom.xml            |  39 +--
 .../metron/rest/controller/HdfsController.java  |  89 +++++
 .../rest/controller/StellarController.java      |  80 +++++
 .../controller/TransformationController.java    |  80 -----
 .../apache/metron/rest/service/GrokService.java |   3 +
 .../apache/metron/rest/service/HdfsService.java |  13 +-
 .../rest/service/SensorParserConfigService.java |   3 +
 .../metron/rest/service/StellarService.java     |  39 +++
 .../rest/service/TransformationService.java     |  39 ---
 .../rest/service/impl/GrokServiceImpl.java      |  59 +++-
 .../rest/service/impl/HdfsServiceImpl.java      |  55 ++-
 .../impl/SensorEnrichmentConfigServiceImpl.java |   8 +-
 .../impl/SensorIndexingConfigServiceImpl.java   |   8 +-
 .../impl/SensorParserConfigServiceImpl.java     | 141 +-------
 .../rest/service/impl/StellarServiceImpl.java   |  92 +++++
 .../service/impl/StormAdminServiceImpl.java     |  13 +-
 .../rest/service/impl/StormCLIWrapper.java      |   6 +-
 .../service/impl/StormStatusServiceImpl.java    |   8 +-
 .../service/impl/TransformationServiceImpl.java |  92 -----
 .../src/main/resources/application-docker.yml   |   6 +-
 .../src/main/resources/application-vagrant.yml  |  51 +++
 .../src/main/resources/application.yml          |   2 +-
 .../metron-rest/src/main/scripts/start.sh       |  33 --
 .../src/main/scripts/start_metron_rest.sh       |  25 ++
 .../GlobalConfigControllerIntegrationTest.java  |   8 +
 .../GrokControllerIntegrationTest.java          |  24 +-
 .../HdfsControllerIntegrationTest.java          | 101 ++++++
 ...richmentConfigControllerIntegrationTest.java |  23 +-
 ...IndexingConfigControllerIntegrationTest.java |   2 +
 ...orParserConfigControllerIntegrationTest.java |  36 +-
 .../StellarControllerIntegrationTest.java       | 122 +++++++
 .../StormControllerIntegrationTest.java         |   3 +
 ...TransformationControllerIntegrationTest.java | 122 -------
 .../rest/service/SensorParserConfigTest.java    | 116 -------
 .../service/impl/DockerStormCLIWrapperTest.java |  14 +-
 .../rest/service/impl/GrokServiceImplTest.java  | 136 ++++++--
 .../impl/HdfsServiceImplExceptionTest.java      | 101 ++++++
 .../rest/service/impl/HdfsServiceImplTest.java  | 143 ++++----
 .../rest/service/impl/KafkaServiceImplTest.java |   4 +-
 .../SensorEnrichmentConfigServiceImplTest.java  | 256 ++++++++++++++
 .../SensorIndexingConfigServiceImplTest.java    | 234 +++++++++++++
 .../impl/SensorParserConfigServiceImplTest.java | 344 +++++++++++++++++++
 .../service/impl/StellarServiceImplTest.java    |  92 +++++
 .../service/impl/StormAdminServiceImplTest.java | 156 +++++++++
 .../rest/service/impl/StormCLIWrapperTest.java  | 216 ++++++++++++
 .../impl/StormStatusServiceImplTest.java        | 221 ++++++++++++
 .../metron-rest/src/test/resources/README.vm    | 121 +++++--
 metron-interface/pom.xml                        |   2 +-
 59 files changed, 3013 insertions(+), 983 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 35acced..1d428c3 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -283,3 +283,6 @@ org.springframework.security:spring-security-config:jar:4.1.3.RELEASE:compile,AS
 org.springframework.security:spring-security-core:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security
 org.springframework.security:spring-security-web:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security
 antlr:antlr:jar:2.7.7:compile,BSD 3-Clause License,http://www.antlr2.org
+com.h2database:h2:jar:1.4.192:compile,EPL 1.0,http://www.h2database.com/html/license.html
+de.jollyday:jollyday:jar:0.5.2:compile,ASLv2,http://jollyday.sourceforge.net/license.html
+org.threeten:threeten-extra:jar:1.0:compile,BSD,http://www.threeten.org/threeten-extra/license.html

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/pom.xml b/metron-interface/metron-rest-client/pom.xml
index 66ccf52..0a49986 100644
--- a/metron-interface/metron-rest-client/pom.xml
+++ b/metron-interface/metron-rest-client/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <groupId>org.apache.metron</groupId>
         <artifactId>metron-interface</artifactId>
-        <version>0.3.0</version>
+        <version>0.3.1</version>
     </parent>
     <artifactId>metron-rest-client</artifactId>
     <properties>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
index ccd2c5c..2795320 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
@@ -22,10 +22,19 @@ import java.util.Map;
 
 public class GrokValidation {
 
+    private String patternLabel;
     private String statement;
     private String sampleData;
     private Map<String, Object> results;
 
+    public String getPatternLabel() {
+        return patternLabel;
+    }
+
+    public void setPatternLabel(String patternLabel) {
+        this.patternLabel = patternLabel;
+    }
+
     public String getStatement() {
         return statement;
     }
@@ -60,6 +69,7 @@ public class GrokValidation {
 
         GrokValidation that = (GrokValidation) o;
 
+        if (patternLabel != null ? !patternLabel.equals(that.patternLabel) : that.patternLabel != null) return false;
         if (statement != null ? !statement.equals(that.statement) : that.statement != null) return false;
         if (sampleData != null ? !sampleData.equals(that.sampleData) : that.sampleData != null) return false;
         return results != null ? results.equals(that.results) : that.results == null;
@@ -67,7 +77,8 @@ public class GrokValidation {
 
     @Override
     public int hashCode() {
-        int result = statement != null ? statement.hashCode() : 0;
+        int result = patternLabel != null ? patternLabel.hashCode() : 0;
+        result = 31 * result + (statement != null ? statement.hashCode() : 0);
         result = 31 * result + (sampleData != null ? sampleData.hashCode() : 0);
         result = 31 * result + (results != null ? results.hashCode() : 0);
         return result;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
index 8dfe17b..50eb88d 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
@@ -22,6 +22,7 @@ import org.apache.metron.common.configuration.SensorParserConfig;
 public class ParseMessageRequest {
 
     private SensorParserConfig sensorParserConfig;
+    private String grokStatement;
     private String sampleData;
 
     public SensorParserConfig getSensorParserConfig() {
@@ -32,6 +33,14 @@ public class ParseMessageRequest {
         this.sensorParserConfig = sensorParserConfig;
     }
 
+    public String getGrokStatement() {
+        return grokStatement;
+    }
+
+    public void setGrokStatement(String grokStatement) {
+        this.grokStatement = grokStatement;
+    }
+
     public String getSampleData() {
         return sampleData;
     }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java
new file mode 100644
index 0000000..cb10cfe
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+import org.apache.metron.common.configuration.SensorParserConfig;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class SensorParserContext {
+
+    private Map<String, Object> sampleData;
+    private SensorParserConfig sensorParserConfig;
+
+    public Map<String, Object> getSampleData() {
+        if (sampleData == null) {
+            return new HashMap<>();
+        }
+        return sampleData;
+    }
+
+    public void setSampleData(Map<String, Object> sampleData) {
+        this.sampleData = sampleData;
+    }
+
+    public SensorParserConfig getSensorParserConfig() {
+        if (sensorParserConfig == null) {
+            return new SensorParserConfig();
+        }
+        return sensorParserConfig;
+    }
+
+    public void setSensorParserConfig(SensorParserConfig sensorParserConfig) {
+        this.sensorParserConfig = sensorParserConfig;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
index 6894e89..0ad2e9f 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
@@ -39,4 +39,22 @@ public class TopologyResponse {
     this.status = TopologyResponseCode.ERROR;
     this.message = message;
   }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    TopologyResponse that = (TopologyResponse) o;
+
+    if (status != null ? !status.equals(that.status) : that.status != null) return false;
+    return message != null ? message.equals(that.message) : that.message == null;
+  }
+
+  @Override
+  public int hashCode() {
+    int result = status != null ? status.hashCode() : 0;
+    result = 31 * result + (message != null ? message.hashCode() : 0);
+    return result;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
index 0e2d28a..6fc1c01 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
@@ -26,8 +26,8 @@ public class TopologyStatus {
   private String name;
   private TopologyStatusCode status;
   private Map<String, Object>[] topologyStats;
-  private double latency = 0;
-  private double throughput = 0;
+  private Double latency = 0.0;
+  private Double throughput = 0.0;
 
   public String getId() {
     return id;
@@ -73,4 +73,28 @@ public class TopologyStatus {
       }
     }
   }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    TopologyStatus that = (TopologyStatus) o;
+
+    if (id != null ? !id.equals(that.id) : that.id != null) return false;
+    if (name != null ? !name.equals(that.name) : that.name != null) return false;
+    if (status != null ? !status.equals(that.status) : that.status != null) return false;
+    if (!latency.equals(that.latency)) return false;
+    return throughput.equals(that.throughput);
+  }
+
+  @Override
+  public int hashCode() {
+    int result = id != null ? id.hashCode() : 0;
+    result = 31 * result + (name != null ? name.hashCode() : 0);
+    result = 31 * result + (status != null ? status.hashCode() : 0);
+    result = 31 * result + (latency != null ? latency.hashCode() : 0);
+    result = 31 * result + (throughput != null ? throughput.hashCode() : 0);
+    return result;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
index b9d39a4..8621daf 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
@@ -17,6 +17,8 @@
  */
 package org.apache.metron.rest.model;
 
+import java.util.Arrays;
+
 public class TopologySummary {
 
     private TopologyStatus[] topologies;
@@ -31,4 +33,19 @@ public class TopologySummary {
     public void setTopologies(TopologyStatus[] topologies) {
         this.topologies = topologies;
     }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) return true;
+      if (o == null || getClass() != o.getClass()) return false;
+
+      TopologySummary that = (TopologySummary) o;
+
+      return topologies != null ? Arrays.equals(topologies, that.topologies) : that.topologies != null;
+    }
+
+    @Override
+    public int hashCode() {
+      return topologies != null ? Arrays.hashCode(topologies) : 0;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java
deleted file mode 100644
index c2e39e4..0000000
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.model;
-
-import org.apache.metron.common.configuration.SensorParserConfig;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class TransformationValidation {
-
-    private Map<String, Object> sampleData;
-    private SensorParserConfig sensorParserConfig;
-
-    public Map<String, Object> getSampleData() {
-        if (sampleData == null) {
-            return new HashMap<>();
-        }
-        return sampleData;
-    }
-
-    public void setSampleData(Map<String, Object> sampleData) {
-        this.sampleData = sampleData;
-    }
-
-    public SensorParserConfig getSensorParserConfig() {
-        if (sensorParserConfig == null) {
-            return new SensorParserConfig();
-        }
-        return sensorParserConfig;
-    }
-
-    public void setSensorParserConfig(SensorParserConfig sensorParserConfig) {
-        this.sensorParserConfig = sensorParserConfig;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/.gitignore
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/.gitignore b/metron-interface/metron-rest/.gitignore
new file mode 100644
index 0000000..26b3af1
--- /dev/null
+++ b/metron-interface/metron-rest/.gitignore
@@ -0,0 +1 @@
+metrondb.*.db

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 63a2d34..cf41bf3 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -1,66 +1,83 @@
-# Metron REST and Configuration UI
+# Metron REST
 
-This UI exposes and aids in sensor configuration.
+This module provides a RESTful API for interacting with Metron.
 
 ## Prerequisites
 
 * A running Metron cluster
-* A running instance of MySQL
 * Java 8 installed
 * Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed
 
 ## Installation
-1. Package the Application with Maven:
-    ```
-    mvn clean package
-    ```
+1. Package the application with Maven:
+```
+mvn clean package
+```
 
 1. Untar the archive in the target directory.  The directory structure will look like:
-    ```
-    bin
-      start.sh
-    lib
-      metron-rest-version.jar
-    ```
+```
+bin
+  start_metron_rest.sh
+lib
+  metron-rest-$METRON_VERSION.jar
+```
+
+1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml).  Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.*` properties as needed (see the [Security](#security) section for more details).
 
-1. Install Hibernate by downloading version 5.0.11.Final from (http://hibernate.org/orm/downloads/).  Unpack the archive and set the HIBERNATE_HOME environment variable to the absolute path of the top level directory.
-    ```
-    export HIBERNATE_HOME=/path/to/hibernate-release-5.0.11.Final
-    ```
+1. Start the application with this command:
+```
+./bin/start_metron_rest.sh /path/to/application.yml
+```
 
-1. Install the MySQL client by downloading version 5.1.40 from (https://dev.mysql.com/downloads/connector/j/).  Unpack the archive and set the MYSQL_CLIENT_HOME environment variable to the absolute path of the top level directory.
-    ```
-    export MYSQL_CLIENT_HOME=/path/to/mysql-connector-java-5.1.40
-    ```
+## Usage
 
-1. Create a MySQL user for the Config UI (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html).
+The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/.  The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port.
 
-1. Create a Config UI database in MySQL with this command:
-    ```
-    CREATE DATABASE IF NOT EXISTS metronrest
-    ```
+## Security
 
-1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml).  Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.username` and `spring.datasource.password` properties using the MySQL credentials from step 4.
+The metron-rest module uses [Spring Security](http://projects.spring.io/spring-security/) for authentication and stores user credentials in a relational database.  The H2 database is configured by default and is intended only for development purposes.  The "dev" profile can be used to automatically load test users:
+```
+./bin/start_metron_rest.sh /path/to/application.yml --spring.profiles.active=dev
+```
 
-1. Start the UI with this command:
-    ```
-    ./bin/start.sh /path/to/application.yml
-    ```
+For [production use](http://docs.spring.io/spring-boot/docs/1.4.1.RELEASE/reference/htmlsingle/#boot-features-connect-to-production-database), a relational database should be configured.  For example, configuring MySQL would be done as follows:
 
-## Usage
+1. Create a MySQL user for the Metron REST application (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html).
 
-The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/.  The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port.  Users can be added with this SQL statement:
+1. Connect to MySQL and create a Metron REST database:
+```
+CREATE DATABASE IF NOT EXISTS metronrest
+```
+
+1. Add users:
 ```
 use metronrest;
 insert into users (username, password, enabled) values ('your_username','your_password',1);
 insert into authorities (username, authority) values ('your_username', 'ROLE_USER');
 ```
-Users can be added to additional groups with this SQL statement:
+
+1. Replace the H2 connection information in the application.yml file with MySQL connection information:
 ```
-use metronrest;
-insert into authorities (username, authority) values ('your_username', 'your_group');
+spring:
+  datasource:
+        driverClassName: com.mysql.jdbc.Driver
+        url: jdbc:mysql://mysql_host:3306/metronrest
+        username: metron_rest_user
+        password: metron_rest_password
+        platform: mysql
 ```
 
+1. Add a dependency for the MySQL JDBC connector in the metron-rest pom.xml:
+```
+<dependency>
+  <groupId>mysql</groupId>
+  <artifactId>mysql-connector-java</artifactId>
+  <version>${mysql.client.version}</version>
+</dependency>
+```
+
+1. Follow the steps in the [Installation](#installation) section
+
 ## API
 
 Request and Response objects are JSON formatted.  The JSON schemas are available in the Swagger UI.
@@ -72,6 +89,10 @@ Request and Response objects are JSON formatted.  The JSON schemas are available
 | [ `POST /api/v1/global/config`](#post-apiv1globalconfig)|
 | [ `GET /api/v1/grok/list`](#get-apiv1groklist)|
 | [ `POST /api/v1/grok/validate`](#post-apiv1grokvalidate)|
+| [ `POST /api/v1/hdfs`](#post-apiv1hdfs)|
+| [ `GET /api/v1/hdfs`](#get-apiv1hdfs)|
+| [ `DELETE /api/v1/hdfs`](#delete-apiv1hdfs)|
+| [ `GET /api/v1/hdfs/list`](#get-apiv1hdfslist)|
 | [ `GET /api/v1/kafka/topic`](#get-apiv1kafkatopic)|
 | [ `POST /api/v1/kafka/topic`](#post-apiv1kafkatopic)|
 | [ `GET /api/v1/kafka/topic/{name}`](#get-apiv1kafkatopic{name})|
@@ -93,6 +114,11 @@ Request and Response objects are JSON formatted.  The JSON schemas are available
 | [ `GET /api/v1/sensor/parser/config/reload/available`](#get-apiv1sensorparserconfigreloadavailable)|
 | [ `DELETE /api/v1/sensor/parser/config/{name}`](#delete-apiv1sensorparserconfig{name})|
 | [ `GET /api/v1/sensor/parser/config/{name}`](#get-apiv1sensorparserconfig{name})|
+| [ `POST /api/v1/stellar/apply/transformations`](#post-apiv1stellarapplytransformations)|
+| [ `GET /api/v1/stellar/list`](#get-apiv1stellarlist)|
+| [ `GET /api/v1/stellar/list/functions`](#get-apiv1stellarlistfunctions)|
+| [ `GET /api/v1/stellar/list/simple/functions`](#get-apiv1stellarlistsimplefunctions)|
+| [ `POST /api/v1/stellar/validate/rules`](#post-apiv1stellarvalidaterules)|
 | [ `GET /api/v1/storm`](#get-apiv1storm)|
 | [ `GET /api/v1/storm/client/status`](#get-apiv1stormclientstatus)|
 | [ `GET /api/v1/storm/enrichment`](#get-apiv1stormenrichment)|
@@ -110,11 +136,6 @@ Request and Response objects are JSON formatted.  The JSON schemas are available
 | [ `GET /api/v1/storm/parser/start/{name}`](#get-apiv1stormparserstart{name})|
 | [ `GET /api/v1/storm/parser/stop/{name}`](#get-apiv1stormparserstop{name})|
 | [ `GET /api/v1/storm/{name}`](#get-apiv1storm{name})|
-| [ `GET /api/v1/transformation/list`](#get-apiv1transformationlist)|
-| [ `GET /api/v1/transformation/list/functions`](#get-apiv1transformationlistfunctions)|
-| [ `GET /api/v1/transformation/list/simple/functions`](#get-apiv1transformationlistsimplefunctions)|
-| [ `POST /api/v1/transformation/validate`](#post-apiv1transformationvalidate)|
-| [ `POST /api/v1/transformation/validate/rules`](#post-apiv1transformationvalidaterules)|
 | [ `GET /api/v1/user`](#get-apiv1user)|
 
 ### `GET /api/v1/global/config`
@@ -145,10 +166,41 @@ Request and Response objects are JSON formatted.  The JSON schemas are available
 ### `POST /api/v1/grok/validate`
   * Description: Applies a Grok statement to a sample message
   * Input:
-    * grokValidation - Object containing Grok statment and sample message
+    * grokValidation - Object containing Grok statement and sample message
   * Returns:
     * 200 - JSON results
 
+### `POST /api/v1/hdfs`
+  * Description: Writes contents to an HDFS file.  Warning: this will overwite the contents of a file if it already exists.
+  * Input:
+    * path - Path to HDFS file
+    * contents - File contents
+  * Returns:
+    * 200 - Contents were written
+
+### `GET /api/v1/hdfs`
+  * Description: Reads a file from HDFS and returns the contents
+  * Input:
+    * path - Path to HDFS file
+  * Returns:
+    * 200 - Returns file contents
+
+### `DELETE /api/v1/hdfs`
+  * Description: Deletes a file from HDFS
+  * Input:
+    * path - Path to HDFS file
+    * recursive - Delete files recursively
+  * Returns:
+    * 200 - File was deleted
+    * 404 - File was not found in HDFS
+
+### `GET /api/v1/hdfs/list`
+  * Description: Reads a file from HDFS and returns the contents
+  * Input:
+    * path - Path to HDFS directory
+  * Returns:
+    * 200 - Returns file contents
+
 ### `GET /api/v1/kafka/topic`
   * Description: Retrieves all Kafka topics
   * Returns:
@@ -296,6 +348,35 @@ Request and Response objects are JSON formatted.  The JSON schemas are available
     * 200 - Returns SensorParserConfig
     * 404 - SensorParserConfig is missing
 
+### `POST /api/v1/stellar/apply/transformations`
+  * Description: Executes transformations against a sample message
+  * Input:
+    * transformationValidation - Object containing SensorParserConfig and sample message
+  * Returns:
+    * 200 - Returns transformation results
+
+### `GET /api/v1/stellar/list`
+  * Description: Retrieves field transformations
+  * Returns:
+    * 200 - Returns a list field transformations
+
+### `GET /api/v1/stellar/list/functions`
+  * Description: Lists the Stellar functions that can be found on the classpath
+  * Returns:
+    * 200 - Returns a list of Stellar functions
+
+### `GET /api/v1/stellar/list/simple/functions`
+  * Description: Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath
+  * Returns:
+    * 200 - Returns a list of simple Stellar functions
+
+### `POST /api/v1/stellar/validate/rules`
+  * Description: Tests Stellar statements to ensure they are well-formed
+  * Input:
+    * statements - List of statements to validate
+  * Returns:
+    * 200 - Returns validation results
+
 ### `GET /api/v1/storm`
   * Description: Retrieves the status of all Storm topologies
   * Returns:
@@ -399,40 +480,43 @@ Request and Response objects are JSON formatted.  The JSON schemas are available
     * 200 - Returns topology status information
     * 404 - Topology is missing
 
-### `GET /api/v1/transformation/list`
-  * Description: Retrieves field transformations
+### `GET /api/v1/user`
+  * Description: Retrieves the current user
   * Returns:
-    * 200 - Returns a list field transformations
+    * 200 - Current user
 
-### `GET /api/v1/transformation/list/functions`
-  * Description: Lists the Stellar functions that can be found on the classpath
-  * Returns:
-    * 200 - Returns a list of Stellar functions
+## Testing
 
-### `GET /api/v1/transformation/list/simple/functions`
-  * Description: Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath
-  * Returns:
-    * 200 - Returns a list of simple Stellar functions
+Profiles are includes for both the metron-docker and Quick Dev environments.
 
-### `POST /api/v1/transformation/validate`
-  * Description: Executes transformations against a sample message
-  * Input:
-    * transformationValidation - Object containing SensorParserConfig and sample message
-  * Returns:
-    * 200 - Returns transformation results
+### metron-docker
 
-### `POST /api/v1/transformation/validate/rules`
-  * Description: Tests Stellar statements to ensure they are well-formed
-  * Input:
-    * statements - List of statements to validate
-  * Returns:
-    * 200 - Returns validation results
+Start the [metron-docker](../../metron-docker) environment.  Build the metron-rest module and start it with the Spring Boot Maven plugin:
+```
+mvn clean package
+mvn spring-boot:run -Drun.profiles=docker,dev
+```
+The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/.
 
-### `GET /api/v1/user`
-  * Description: Retrieves the current user
-  * Returns:
-    * 200 - Current user
+### Quick Dev
 
+Start the [Quick Dev](../../metron-deployment/vagrant/quick-dev-platform) environment.  Build the metron-rest module and start it with the Spring Boot Maven plugin:
+```
+mvn clean package
+mvn spring-boot:run -Drun.profiles=vagrant,dev
+```
+The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/.
+
+To run the application locally on the Quick Dev host, package the application and scp the archive to node1:
+```
+mvn clean package
+scp ./target/metron-rest-$METRON_VERSION-archive.tar.gz root@node1:~/
+```
+Login to node1 and unarchive the metron-rest application.  Start the application on a different port to avoid conflicting with Ambari:
+```
+java -jar ./lib/metron-rest-$METRON_VERSION.jar --spring.profiles.active=vagrant,dev --server.port=8082
+```
+The metron-rest application will be available at http://node1:8082/swagger-ui.html#/.
 
 ## License
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml
index c2beb2d..849455e 100644
--- a/metron-interface/metron-rest/pom.xml
+++ b/metron-interface/metron-rest/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <groupId>org.apache.metron</groupId>
         <artifactId>metron-interface</artifactId>
-        <version>0.3.0</version>
+        <version>0.3.1</version>
     </parent>
     <artifactId>metron-rest</artifactId>
     <properties>
@@ -61,15 +61,15 @@
             <artifactId>spring-boot-starter-jdbc</artifactId>
         </dependency>
         <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
-            <version>${mysql.client.version}</version>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-recipes</artifactId>
             <version>${curator.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+              </exclusions>
         </dependency>
         <dependency>
             <groupId>com.googlecode.json-simple</groupId>
@@ -105,6 +105,10 @@
                     <groupId>com.fasterxml.jackson.core</groupId>
                     <artifactId>jackson-databind</artifactId>
                 </exclusion>
+                <exclusion>
+                  <groupId>org.reflections</groupId>
+                  <artifactId>reflections</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -122,8 +126,16 @@
                 </exclusion>
                 <exclusion>
                     <groupId>org.apache.metron</groupId>
+                    <artifactId>metron-statistics</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.metron</groupId>
                     <artifactId>metron-writer</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -147,18 +159,6 @@
             <version>0.1.0</version>
         </dependency>
         <dependency>
-            <groupId>org.reflections</groupId>
-            <artifactId>reflections</artifactId>
-            <version>0.9.10</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.google.code.findbugs</groupId>
-                    <artifactId>annotations</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-
-        <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>
             <scope>test</scope>
@@ -166,7 +166,6 @@
         <dependency>
             <groupId>com.h2database</groupId>
             <artifactId>h2</artifactId>
-            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.springframework.security</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
new file mode 100644
index 0000000..bf7ae84
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.HdfsService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+@RestController
+@RequestMapping("/api/v1/hdfs")
+public class HdfsController {
+
+  @Autowired
+  private HdfsService hdfsService;
+
+  @ApiOperation(value = "Reads a file from HDFS and returns the contents")
+  @ApiResponse(message = "Returns file contents", code = 200)
+  @RequestMapping(value = "/list", method = RequestMethod.GET)
+  ResponseEntity<List<String>> list(@ApiParam(name = "path", value = "Path to HDFS directory", required = true) @RequestParam String path) throws RestException {
+    return new ResponseEntity<>(hdfsService.list(new Path(path)), HttpStatus.OK);
+  }
+
+  @ApiOperation(value = "Reads a file from HDFS and returns the contents")
+  @ApiResponse(message = "Returns file contents", code = 200)
+  @RequestMapping(method = RequestMethod.GET)
+  ResponseEntity<String> read(@ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam String path) throws RestException {
+    String contents = hdfsService.read(new Path(path));
+    if (contents != null) {
+      return new ResponseEntity<>(hdfsService.read(new Path(path)), HttpStatus.OK);
+    } else {
+      return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+    }
+
+  }
+
+  @ApiOperation(value = "Writes contents to an HDFS file.  Warning: this will overwite the contents of a file if it already exists.")
+  @ApiResponse(message = "Contents were written", code = 200)
+  @RequestMapping(method = RequestMethod.POST)
+  ResponseEntity<Void> write(@ApiParam(name="path", value="Path to HDFS file", required=true) @RequestParam String path,
+                                              @ApiParam(name="contents", value="File contents", required=true) @RequestBody String contents) throws RestException {
+    hdfsService.write(new Path(path), contents.getBytes(UTF_8));
+    return new ResponseEntity<>(HttpStatus.OK);
+
+  }
+
+  @ApiOperation(value = "Deletes a file from HDFS")
+  @ApiResponses(value = { @ApiResponse(message = "File was deleted", code = 200),
+          @ApiResponse(message = "File was not found in HDFS", code = 404) })
+  @RequestMapping(method = RequestMethod.DELETE)
+  ResponseEntity<Boolean> delete(@ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam String path,
+                                 @ApiParam(name = "recursive", value = "Delete files recursively") @RequestParam(required = false, defaultValue = "false") boolean recursive) throws RestException {
+    if (hdfsService.delete(new Path(path), recursive)) {
+      return new ResponseEntity<>(HttpStatus.OK);
+    } else {
+      return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java
new file mode 100644
index 0000000..13327f9
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import org.apache.metron.common.field.transformation.FieldTransformations;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.StellarFunctionDescription;
+import org.apache.metron.rest.model.SensorParserContext;
+import org.apache.metron.rest.service.StellarService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/api/v1/stellar")
+public class StellarController {
+
+    @Autowired
+    private StellarService stellarService;
+
+  @ApiOperation(value = "Tests Stellar statements to ensure they are well-formed")
+  @ApiResponse(message = "Returns validation results", code = 200)
+    @RequestMapping(value = "/validate/rules", method = RequestMethod.POST)
+    ResponseEntity<Map<String, Boolean>> validateRules(@ApiParam(name="statements", value="List of statements to validate", required=true)@RequestBody List<String> statements) throws RestException {
+        return new ResponseEntity<>(stellarService.validateRules(statements), HttpStatus.OK);
+    }
+
+  @ApiOperation(value = "Executes transformations against a sample message")
+  @ApiResponse(message = "Returns transformation results", code = 200)
+    @RequestMapping(value = "/apply/transformations", method = RequestMethod.POST)
+    ResponseEntity<Map<String, Object>> applyTransformations(@ApiParam(name="transformationValidation", value="Object containing SensorParserConfig and sample message", required=true)@RequestBody SensorParserContext sensorParserContext) throws RestException {
+        return new ResponseEntity<>(stellarService.applyTransformations(sensorParserContext), HttpStatus.OK);
+    }
+
+  @ApiOperation(value = "Retrieves field transformations")
+  @ApiResponse(message = "Returns a list field transformations", code = 200)
+    @RequestMapping(value = "/list", method = RequestMethod.GET)
+    ResponseEntity<FieldTransformations[]> list() throws RestException {
+        return new ResponseEntity<>(stellarService.getTransformations(), HttpStatus.OK);
+    }
+
+  @ApiOperation(value = "Lists the Stellar functions that can be found on the classpath")
+  @ApiResponse(message = "Returns a list of Stellar functions", code = 200)
+    @RequestMapping(value = "/list/functions", method = RequestMethod.GET)
+    ResponseEntity<List<StellarFunctionDescription>> listFunctions() throws RestException {
+        return new ResponseEntity<>(stellarService.getStellarFunctions(), HttpStatus.OK);
+    }
+
+  @ApiOperation(value = "Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath")
+  @ApiResponse(message = "Returns a list of simple Stellar functions", code = 200)
+    @RequestMapping(value = "/list/simple/functions", method = RequestMethod.GET)
+    ResponseEntity<List<StellarFunctionDescription>> listSimpleFunctions() throws RestException {
+        return new ResponseEntity<>(stellarService.getSimpleStellarFunctions(), HttpStatus.OK);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java
deleted file mode 100644
index bc8b201..0000000
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.controller;
-
-import io.swagger.annotations.ApiOperation;
-import io.swagger.annotations.ApiParam;
-import io.swagger.annotations.ApiResponse;
-import org.apache.metron.common.field.transformation.FieldTransformations;
-import org.apache.metron.rest.RestException;
-import org.apache.metron.rest.model.StellarFunctionDescription;
-import org.apache.metron.rest.model.TransformationValidation;
-import org.apache.metron.rest.service.TransformationService;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.RestController;
-
-import java.util.List;
-import java.util.Map;
-
-@RestController
-@RequestMapping("/api/v1/transformation")
-public class TransformationController {
-
-    @Autowired
-    private TransformationService transformationService;
-
-  @ApiOperation(value = "Tests Stellar statements to ensure they are well-formed")
-  @ApiResponse(message = "Returns validation results", code = 200)
-    @RequestMapping(value = "/validate/rules", method = RequestMethod.POST)
-    ResponseEntity<Map<String, Boolean>> validateRule(@ApiParam(name="statements", value="List of statements to validate", required=true)@RequestBody List<String> statements) throws RestException {
-        return new ResponseEntity<>(transformationService.validateRules(statements), HttpStatus.OK);
-    }
-
-  @ApiOperation(value = "Executes transformations against a sample message")
-  @ApiResponse(message = "Returns transformation results", code = 200)
-    @RequestMapping(value = "/validate", method = RequestMethod.POST)
-    ResponseEntity<Map<String, Object>> validateTransformation(@ApiParam(name="transformationValidation", value="Object containing SensorParserConfig and sample message", required=true)@RequestBody TransformationValidation transformationValidation) throws RestException {
-        return new ResponseEntity<>(transformationService.validateTransformation(transformationValidation), HttpStatus.OK);
-    }
-
-  @ApiOperation(value = "Retrieves field transformations")
-  @ApiResponse(message = "Returns a list field transformations", code = 200)
-    @RequestMapping(value = "/list", method = RequestMethod.GET)
-    ResponseEntity<FieldTransformations[]> list() throws RestException {
-        return new ResponseEntity<>(transformationService.getTransformations(), HttpStatus.OK);
-    }
-
-  @ApiOperation(value = "Lists the Stellar functions that can be found on the classpath")
-  @ApiResponse(message = "Returns a list of Stellar functions", code = 200)
-    @RequestMapping(value = "/list/functions", method = RequestMethod.GET)
-    ResponseEntity<List<StellarFunctionDescription>> listFunctions() throws RestException {
-        return new ResponseEntity<>(transformationService.getStellarFunctions(), HttpStatus.OK);
-    }
-
-  @ApiOperation(value = "Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath")
-  @ApiResponse(message = "Returns a list of simple Stellar functions", code = 200)
-    @RequestMapping(value = "/list/simple/functions", method = RequestMethod.GET)
-    ResponseEntity<List<StellarFunctionDescription>> listSimpleFunctions() throws RestException {
-        return new ResponseEntity<>(transformationService.getSimpleStellarFunctions(), HttpStatus.OK);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
index 95ce1ba..268d396 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
@@ -20,6 +20,7 @@ package org.apache.metron.rest.service;
 import org.apache.metron.rest.RestException;
 import org.apache.metron.rest.model.GrokValidation;
 
+import java.io.File;
 import java.util.Map;
 
 public interface GrokService {
@@ -28,4 +29,6 @@ public interface GrokService {
 
     GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException;
 
+    File saveTemporary(String statement, String name) throws RestException;
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
index 97888ff..d5932c7 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
@@ -17,19 +17,18 @@
  */
 package org.apache.metron.rest.service;
 
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
-import org.springframework.stereotype.Service;
+import org.apache.metron.rest.RestException;
 
-import java.io.IOException;
+import java.util.List;
 
 public interface HdfsService {
 
-    byte[] read(Path path) throws IOException;
+    String read(Path path) throws RestException;
 
-    void write(Path path, byte[] contents) throws IOException;
+    void write(Path path, byte[] contents) throws RestException;
 
-    FileStatus[] list(Path path) throws IOException;
+    List<String> list(Path path) throws RestException;
 
-    boolean delete(Path path, boolean recursive) throws IOException;
+    boolean delete(Path path, boolean recursive) throws RestException;
  }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
index efda639..9b863b8 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
@@ -22,6 +22,7 @@ import org.apache.metron.rest.RestException;
 import org.apache.metron.rest.model.ParseMessageRequest;
 import org.json.simple.JSONObject;
 
+import java.util.List;
 import java.util.Map;
 
 public interface SensorParserConfigService {
@@ -32,6 +33,8 @@ public interface SensorParserConfigService {
 
   Iterable<SensorParserConfig> getAll() throws RestException;
 
+  List<String> getAllTypes() throws RestException;
+
   boolean delete(String name) throws RestException;
 
   Map<String, String> getAvailableParsers();

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java
new file mode 100644
index 0000000..14cd31f
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.metron.common.field.transformation.FieldTransformations;
+import org.apache.metron.rest.model.StellarFunctionDescription;
+import org.apache.metron.rest.model.SensorParserContext;
+
+import java.util.List;
+import java.util.Map;
+
+public interface StellarService {
+
+    Map<String, Boolean> validateRules(List<String> rules);
+
+    Map<String, Object> applyTransformations(SensorParserContext sensorParserContext);
+
+    FieldTransformations[] getTransformations();
+
+    List<StellarFunctionDescription> getStellarFunctions();
+
+    List<StellarFunctionDescription> getSimpleStellarFunctions();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java
deleted file mode 100644
index d1400c6..0000000
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.service;
-
-import org.apache.metron.common.field.transformation.FieldTransformations;
-import org.apache.metron.rest.model.StellarFunctionDescription;
-import org.apache.metron.rest.model.TransformationValidation;
-
-import java.util.List;
-import java.util.Map;
-
-public interface TransformationService {
-
-    Map<String, Boolean> validateRules(List<String> rules);
-
-    Map<String, Object> validateTransformation(TransformationValidation transformationValidation);
-
-    FieldTransformations[] getTransformations();
-
-    List<StellarFunctionDescription> getStellarFunctions();
-
-    List<StellarFunctionDescription> getSimpleStellarFunctions();
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
index 323ca78..3f2de2f 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
@@ -20,22 +20,35 @@ package org.apache.metron.rest.service.impl;
 import oi.thekraken.grok.api.Grok;
 import oi.thekraken.grok.api.Match;
 import org.apache.directory.api.util.Strings;
+import org.apache.hadoop.fs.Path;
 import org.apache.metron.rest.RestException;
 import org.apache.metron.rest.model.GrokValidation;
 import org.apache.metron.rest.service.GrokService;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Service;
 
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.StringReader;
 import java.util.Map;
 
+import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
+
 @Service
 public class GrokServiceImpl implements GrokService {
+
+    private Environment environment;
+
     private Grok commonGrok;
 
     @Autowired
-    public GrokServiceImpl(Grok commonGrok) {
+    public GrokServiceImpl(Environment environment, Grok commonGrok) {
+        this.environment = environment;
         this.commonGrok = commonGrok;
     }
 
@@ -48,20 +61,21 @@ public class GrokServiceImpl implements GrokService {
     public GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException {
         Map<String, Object> results;
         try {
-            String statement = Strings.isEmpty(grokValidation.getStatement()) ? "" : grokValidation.getStatement();
-
+            if (grokValidation.getPatternLabel() == null) {
+              throw new RestException("Pattern label is required");
+            }
+            if (Strings.isEmpty(grokValidation.getStatement())) {
+              throw new RestException("Grok statement is required");
+            }
             Grok grok = new Grok();
             grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream("/patterns/common")));
-            grok.addPatternFromReader(new StringReader(statement));
-            String patternLabel = statement.substring(0, statement.indexOf(" "));
-            String grokPattern = "%{" + patternLabel + "}";
+            grok.addPatternFromReader(new StringReader(grokValidation.getStatement()));
+            String grokPattern = "%{" + grokValidation.getPatternLabel() + "}";
             grok.compile(grokPattern);
             Match gm = grok.match(grokValidation.getSampleData());
             gm.captures();
             results = gm.toMap();
-            results.remove(patternLabel);
-        } catch (StringIndexOutOfBoundsException e) {
-            throw new RestException("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)", e.getCause());
+            results.remove(grokValidation.getPatternLabel());
         } catch (Exception e) {
             throw new RestException(e);
         }
@@ -69,4 +83,31 @@ public class GrokServiceImpl implements GrokService {
         return grokValidation;
     }
 
+    @Override
+    public File saveTemporary(String statement, String name) throws RestException {
+        if (statement != null) {
+            try {
+                File grokDirectory = new File(getTemporaryGrokRootPath());
+                if (!grokDirectory.exists()) {
+                  grokDirectory.mkdirs();
+                }
+                File path = new File(grokDirectory, name);
+                FileWriter fileWriter = new FileWriter(new File(grokDirectory, name));
+                fileWriter.write(statement);
+                fileWriter.close();
+                return path;
+            } catch (IOException e) {
+                throw new RestException(e);
+            }
+        } else {
+            throw new RestException("A grokStatement must be provided");
+        }
+    }
+
+    private String getTemporaryGrokRootPath() {
+      String grokTempPath = environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY);
+      Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
+      return new Path(grokTempPath, authentication.getName()).toString();
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
index c14ec0c..789c421 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
@@ -19,44 +19,73 @@ package org.apache.metron.rest.service.impl;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.metron.rest.RestException;
 import org.apache.metron.rest.service.HdfsService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 @Service
 public class HdfsServiceImpl implements HdfsService {
 
-    @Autowired
     private Configuration configuration;
 
+    @Autowired
+    public HdfsServiceImpl(Configuration configuration) {
+        this.configuration = configuration;
+    }
+
     @Override
-    public byte[] read(Path path) throws IOException {
-        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
-        IOUtils.copyBytes(FileSystem.get(configuration).open(path), byteArrayOutputStream, configuration);
-        return byteArrayOutputStream.toByteArray();
+    public List<String> list(Path path) throws RestException {
+      try {
+          return Arrays.asList(FileSystem.get(configuration).listStatus(path)).stream().map(fileStatus -> fileStatus.getPath().getName()).collect(Collectors.toList());
+      } catch (IOException e) {
+          throw new RestException(e);
+      }
     }
 
     @Override
-    public void write(Path path, byte[] contents) throws IOException {
-        FSDataOutputStream fsDataOutputStream = FileSystem.get(configuration).create(path, true);
-        fsDataOutputStream.write(contents);
-        fsDataOutputStream.close();
+    public String read(Path path) throws RestException {
+      ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+      try {
+        IOUtils.copyBytes(FileSystem.get(configuration).open(path), byteArrayOutputStream, configuration);
+      } catch (FileNotFoundException e) {
+        return null;
+      } catch (IOException e) {
+        throw new RestException(e);
+      }
+      return new String(byteArrayOutputStream.toByteArray(), UTF_8);
     }
 
     @Override
-    public FileStatus[] list(Path path) throws IOException {
-        return FileSystem.get(configuration).listStatus(path);
+    public void write(Path path, byte[] contents) throws RestException {
+      FSDataOutputStream fsDataOutputStream;
+      try {
+        fsDataOutputStream = FileSystem.get(configuration).create(path, true);
+        fsDataOutputStream.write(contents);
+        fsDataOutputStream.close();
+      } catch (IOException e) {
+        throw new RestException(e);
+      }
     }
 
     @Override
-    public boolean delete(Path path, boolean recursive) throws IOException {
+    public boolean delete(Path path, boolean recursive) throws RestException {
+      try {
         return FileSystem.get(configuration).delete(path, recursive);
+      } catch (IOException e) {
+        throw new RestException(e);
+      }
     }
  }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
index 8b3dbb7..2bfef89 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
@@ -36,12 +36,16 @@ import java.util.Map;
 @Service
 public class SensorEnrichmentConfigServiceImpl implements SensorEnrichmentConfigService {
 
-    @Autowired
     private ObjectMapper objectMapper;
 
-    @Autowired
     private CuratorFramework client;
 
+    @Autowired
+    public SensorEnrichmentConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client) {
+      this.objectMapper = objectMapper;
+      this.client = client;
+    }
+
     @Override
     public SensorEnrichmentConfig save(String name, SensorEnrichmentConfig sensorEnrichmentConfig) throws RestException {
       try {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
index ab46418..9f984e0 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
@@ -38,12 +38,16 @@ import java.util.Map;
 @Service
 public class SensorIndexingConfigServiceImpl implements SensorIndexingConfigService {
 
-  @Autowired
   private ObjectMapper objectMapper;
 
-  @Autowired
   private CuratorFramework client;
 
+  @Autowired
+  public SensorIndexingConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client) {
+    this.objectMapper = objectMapper;
+    this.client = client;
+  }
+
   @Override
   public Map<String, Object> save(String name, Map<String, Object> sensorIndexingConfig) throws RestException {
     try {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index cb88708..eddfc8d 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -17,10 +17,8 @@
  */
 package org.apache.metron.rest.service.impl;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.curator.framework.CuratorFramework;
-import org.apache.hadoop.fs.Path;
 import org.apache.metron.common.configuration.ConfigurationType;
 import org.apache.metron.common.configuration.ConfigurationsUtils;
 import org.apache.metron.common.configuration.SensorParserConfig;
@@ -28,20 +26,15 @@ import org.apache.metron.parsers.interfaces.MessageParser;
 import org.apache.metron.rest.MetronRestConstants;
 import org.apache.metron.rest.RestException;
 import org.apache.metron.rest.model.ParseMessageRequest;
-import org.apache.metron.rest.service.HdfsService;
+import org.apache.metron.rest.service.GrokService;
 import org.apache.metron.rest.service.SensorParserConfigService;
 import org.apache.zookeeper.KeeperException;
 import org.json.simple.JSONObject;
 import org.reflections.Reflections;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.env.Environment;
-import org.springframework.security.core.Authentication;
-import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Service;
 
 import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -49,72 +42,40 @@ import java.util.Map;
 import java.util.Set;
 
 import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
-import static org.apache.metron.rest.MetronRestConstants.GROK_DEFAULT_PATH_SPRING_PROPERTY;
-import static org.apache.metron.rest.MetronRestConstants.GROK_PATH_KEY;
-import static org.apache.metron.rest.MetronRestConstants.GROK_PATTERN_LABEL_KEY;
-import static org.apache.metron.rest.MetronRestConstants.GROK_STATEMENT_KEY;
-import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
 
 @Service
 public class SensorParserConfigServiceImpl implements SensorParserConfigService {
 
-  @Autowired
-  private Environment environment;
-
-  @Autowired
   private ObjectMapper objectMapper;
 
   private CuratorFramework client;
 
+  private GrokService grokService;
+
   @Autowired
-  public void setClient(CuratorFramework client) {
+  public SensorParserConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client, GrokService grokService) {
+    this.objectMapper = objectMapper;
     this.client = client;
+    this.grokService = grokService;
   }
 
-  @Autowired
-  private HdfsService hdfsService;
-
   private Map<String, String> availableParsers;
 
   @Override
   public SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException {
-    String serializedConfig;
-    if (isGrokConfig(sensorParserConfig)) {
-      addGrokPathToConfig(sensorParserConfig);
-      sensorParserConfig.getParserConfig().putIfAbsent(MetronRestConstants.GROK_PATTERN_LABEL_KEY, sensorParserConfig.getSensorTopic().toUpperCase());
-      String statement = (String) sensorParserConfig.getParserConfig().remove(MetronRestConstants.GROK_STATEMENT_KEY);
-      serializedConfig = serialize(sensorParserConfig);
-      sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_STATEMENT_KEY, statement);
-      saveGrokStatement(sensorParserConfig);
-    } else {
-      serializedConfig = serialize(sensorParserConfig);
-    }
     try {
-      ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), serializedConfig.getBytes(), client);
+      ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client);
     } catch (Exception e) {
       throw new RestException(e);
     }
     return sensorParserConfig;
   }
 
-  private String serialize(SensorParserConfig sensorParserConfig) throws RestException {
-    String serializedConfig;
-    try {
-      serializedConfig = objectMapper.writeValueAsString(sensorParserConfig);
-    } catch (JsonProcessingException e) {
-      throw new RestException("Could not serialize SensorParserConfig", "Could not serialize " + sensorParserConfig.toString(), e.getCause());
-    }
-    return serializedConfig;
-  }
-
   @Override
   public SensorParserConfig findOne(String name) throws RestException {
     SensorParserConfig sensorParserConfig;
     try {
       sensorParserConfig = ConfigurationsUtils.readSensorParserConfigFromZookeeper(name, client);
-      if (isGrokConfig(sensorParserConfig)) {
-        addGrokStatementToConfig(sensorParserConfig);
-      }
     } catch (KeeperException.NoNodeException e) {
       return null;
     } catch (Exception e) {
@@ -145,7 +106,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
     return true;
   }
 
-  private List<String> getAllTypes() throws RestException {
+  @Override
+  public List<String> getAllTypes() throws RestException {
     List<String> types;
     try {
       types = client.getChildren().forPath(ConfigurationType.PARSER.getZookeeperRoot());
@@ -190,20 +152,21 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
     } else if (sensorParserConfig.getParserClassName() == null) {
       throw new RestException("SensorParserConfig must have a parserClassName");
     } else {
-      MessageParser<JSONObject> parser = null;
+      MessageParser<JSONObject> parser;
       try {
         parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName()).newInstance();
       } catch (Exception e) {
         throw new RestException(e.toString(), e.getCause());
       }
+      File temporaryGrokFile = null;
       if (isGrokConfig(sensorParserConfig)) {
-        saveTemporaryGrokStatement(sensorParserConfig);
-        sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_PATH_KEY, new File(getTemporaryGrokRootPath(), sensorParserConfig.getSensorTopic()).toString());
+        temporaryGrokFile = grokService.saveTemporary(parseMessageRequest.getGrokStatement(), parseMessageRequest.getSensorParserConfig().getSensorTopic());
+        sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_PATH_KEY, temporaryGrokFile.toString());
       }
       parser.configure(sensorParserConfig.getParserConfig());
       JSONObject results = parser.parse(parseMessageRequest.getSampleData().getBytes()).get(0);
-      if (isGrokConfig(sensorParserConfig)) {
-        deleteTemporaryGrokStatement(sensorParserConfig);
+      if (isGrokConfig(sensorParserConfig) && temporaryGrokFile != null) {
+        temporaryGrokFile.delete();
       }
       return results;
     }
@@ -212,78 +175,4 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
   private boolean isGrokConfig(SensorParserConfig sensorParserConfig) {
     return GROK_CLASS_NAME.equals(sensorParserConfig.getParserClassName());
   }
-
-  private void addGrokStatementToConfig(SensorParserConfig sensorParserConfig) throws RestException {
-    String grokStatement = "";
-    String grokPath = (String) sensorParserConfig.getParserConfig().get(GROK_PATH_KEY);
-    if (grokPath != null) {
-      String fullGrokStatement = getGrokStatement(grokPath);
-      String patternLabel = (String) sensorParserConfig.getParserConfig().get(GROK_PATTERN_LABEL_KEY);
-      grokStatement = fullGrokStatement.replaceFirst(patternLabel + " ", "");
-    }
-    sensorParserConfig.getParserConfig().put(GROK_STATEMENT_KEY, grokStatement);
-  }
-
-  private void addGrokPathToConfig(SensorParserConfig sensorParserConfig) {
-    if (sensorParserConfig.getParserConfig().get(GROK_PATH_KEY) == null) {
-      String grokStatement = (String) sensorParserConfig.getParserConfig().get(GROK_STATEMENT_KEY);
-      if (grokStatement != null) {
-        sensorParserConfig.getParserConfig().put(GROK_PATH_KEY,
-                new Path(environment.getProperty(GROK_DEFAULT_PATH_SPRING_PROPERTY), sensorParserConfig.getSensorTopic()).toString());
-      }
-    }
-  }
-
-  private String getGrokStatement(String path) throws RestException {
-    try {
-      return new String(hdfsService.read(new Path(path)));
-    } catch (IOException e) {
-      throw new RestException(e);
-    }
-  }
-
-  private void saveGrokStatement(SensorParserConfig sensorParserConfig) throws RestException {
-    saveGrokStatement(sensorParserConfig, false);
-  }
-
-  private void saveTemporaryGrokStatement(SensorParserConfig sensorParserConfig) throws RestException {
-    saveGrokStatement(sensorParserConfig, true);
-  }
-
-  private void saveGrokStatement(SensorParserConfig sensorParserConfig, boolean isTemporary) throws RestException {
-    String patternLabel = (String) sensorParserConfig.getParserConfig().get(GROK_PATTERN_LABEL_KEY);
-    String grokPath = (String) sensorParserConfig.getParserConfig().get(GROK_PATH_KEY);
-    String grokStatement = (String) sensorParserConfig.getParserConfig().get(GROK_STATEMENT_KEY);
-    if (grokStatement != null) {
-      String fullGrokStatement = patternLabel + " " + grokStatement;
-      try {
-        if (!isTemporary) {
-          hdfsService.write(new Path(grokPath), fullGrokStatement.getBytes());
-        } else {
-          File grokDirectory = new File(getTemporaryGrokRootPath());
-          if (!grokDirectory.exists()) {
-            grokDirectory.mkdirs();
-          }
-          FileWriter fileWriter = new FileWriter(new File(grokDirectory, sensorParserConfig.getSensorTopic()));
-          fileWriter.write(fullGrokStatement);
-          fileWriter.close();
-        }
-      } catch (IOException e) {
-        throw new RestException(e);
-      }
-    } else {
-      throw new RestException("A grokStatement must be provided");
-    }
-  }
-
-  private void deleteTemporaryGrokStatement(SensorParserConfig sensorParserConfig) {
-    File file = new File(getTemporaryGrokRootPath(), sensorParserConfig.getSensorTopic());
-    file.delete();
-  }
-
-  private String getTemporaryGrokRootPath() {
-    String grokTempPath = environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY);
-    Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
-    return new Path(grokTempPath, authentication.getName()).toString();
-  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java
new file mode 100644
index 0000000..f5392be
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.common.dsl.StellarFunctionInfo;
+import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver;
+import org.apache.metron.common.field.transformation.FieldTransformations;
+import org.apache.metron.common.stellar.StellarProcessor;
+import org.apache.metron.rest.model.StellarFunctionDescription;
+import org.apache.metron.rest.model.SensorParserContext;
+import org.apache.metron.rest.service.StellarService;
+import org.json.simple.JSONObject;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Service
+public class StellarServiceImpl implements StellarService {
+
+    @Override
+    public Map<String, Boolean> validateRules(List<String> rules) {
+        Map<String, Boolean> results = new HashMap<>();
+        StellarProcessor stellarProcessor = new StellarProcessor();
+        for(String rule: rules) {
+            try {
+                boolean result = stellarProcessor.validate(rule, Context.EMPTY_CONTEXT());
+                results.put(rule, result);
+            } catch (ParseException e) {
+                results.put(rule, false);
+            }
+        }
+        return results;
+    }
+
+    @Override
+    public Map<String, Object> applyTransformations(SensorParserContext sensorParserContext) {
+        JSONObject sampleJson = new JSONObject(sensorParserContext.getSampleData());
+        sensorParserContext.getSensorParserConfig().getFieldTransformations().forEach(fieldTransformer -> {
+                    fieldTransformer.transformAndUpdate(sampleJson, sensorParserContext.getSensorParserConfig().getParserConfig(), Context.EMPTY_CONTEXT());
+                }
+        );
+        return sampleJson;
+    }
+
+    @Override
+    public FieldTransformations[] getTransformations() {
+        return FieldTransformations.values();
+    }
+
+    @Override
+    public List<StellarFunctionDescription> getStellarFunctions() {
+        List<StellarFunctionDescription> stellarFunctionDescriptions = new ArrayList<>();
+        Iterable<StellarFunctionInfo> stellarFunctionsInfo = SingletonFunctionResolver.getInstance().getFunctionInfo();
+        stellarFunctionsInfo.forEach(stellarFunctionInfo -> {
+            stellarFunctionDescriptions.add(new StellarFunctionDescription(
+                    stellarFunctionInfo.getName(),
+                    stellarFunctionInfo.getDescription(),
+                    stellarFunctionInfo.getParams(),
+                    stellarFunctionInfo.getReturns()));
+        });
+        return stellarFunctionDescriptions;
+    }
+
+    @Override
+    public List<StellarFunctionDescription> getSimpleStellarFunctions() {
+      List<StellarFunctionDescription> stellarFunctionDescriptions = getStellarFunctions();
+      return stellarFunctionDescriptions.stream().filter(stellarFunctionDescription ->
+              stellarFunctionDescription.getParams().length == 1).sorted((o1, o2) -> o1.getName().compareTo(o2.getName())).collect(Collectors.toList());
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
index cb7c449..9bd368f 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
@@ -33,16 +33,17 @@ public class StormAdminServiceImpl implements StormAdminService {
 
     private StormCLIWrapper stormCLIClientWrapper;
 
+    private GlobalConfigService globalConfigService;
+
+    private SensorParserConfigService sensorParserConfigService;
+
     @Autowired
-    public void setStormCLIClientWrapper(StormCLIWrapper stormCLIClientWrapper) {
+    public StormAdminServiceImpl(StormCLIWrapper stormCLIClientWrapper, GlobalConfigService globalConfigService, SensorParserConfigService sensorParserConfigService) {
         this.stormCLIClientWrapper = stormCLIClientWrapper;
+        this.globalConfigService = globalConfigService;
+        this.sensorParserConfigService = sensorParserConfigService;
     }
 
-    @Autowired
-    private GlobalConfigService globalConfigService;
-
-    @Autowired
-    private SensorParserConfigService sensorParserConfigService;
 
     @Override
     public TopologyResponse startParserTopology(String name) throws RestException {



Mime
View raw message