metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [09/10] incubator-metron git commit: METRON-503: Metron REST API this closes apache/incubator-metron#316
Date Thu, 02 Mar 2017 20:51:55 GMT
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
index a472515..c5569c5 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
@@ -35,9 +35,13 @@ import static org.apache.metron.rest.MetronRestConstants.INDEXING_TOPOLOGY_NAME;
 
 public class StormCLIWrapper {
 
-  @Autowired
   private Environment environment;
 
+  @Autowired
+  public void setEnvironment(final Environment environment) {
+    this.environment = environment;
+  }
+
   public int startParserTopology(String name) throws RestException {
     return runCommand(getParserStartCommand(name));
   }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java
index 280d72a..a02ee75 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java
@@ -38,12 +38,16 @@ import static org.apache.metron.rest.MetronRestConstants.TOPOLOGY_URL;
 @Service
 public class StormStatusServiceImpl implements StormStatusService {
 
-  @Autowired
   private Environment environment;
 
-  @Autowired
   private RestTemplate restTemplate;
 
+  @Autowired
+  public StormStatusServiceImpl(Environment environment, RestTemplate restTemplate) {
+    this.environment = environment;
+    this.restTemplate = restTemplate;
+  }
+
   @Override
   public TopologySummary getTopologySummary() {
     return restTemplate.getForObject("http://" + environment.getProperty(STORM_UI_SPRING_PROPERTY) + TOPOLOGY_SUMMARY_URL, TopologySummary.class);

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/TransformationServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/TransformationServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/TransformationServiceImpl.java
deleted file mode 100644
index b26dc82..0000000
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/TransformationServiceImpl.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.service.impl;
-
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.StellarFunctionInfo;
-import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver;
-import org.apache.metron.common.field.transformation.FieldTransformations;
-import org.apache.metron.common.stellar.StellarProcessor;
-import org.apache.metron.rest.model.StellarFunctionDescription;
-import org.apache.metron.rest.model.TransformationValidation;
-import org.apache.metron.rest.service.TransformationService;
-import org.json.simple.JSONObject;
-import org.springframework.stereotype.Service;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-@Service
-public class TransformationServiceImpl implements TransformationService {
-
-    @Override
-    public Map<String, Boolean> validateRules(List<String> rules) {
-        Map<String, Boolean> results = new HashMap<>();
-        StellarProcessor stellarProcessor = new StellarProcessor();
-        for(String rule: rules) {
-            try {
-                boolean result = stellarProcessor.validate(rule, Context.EMPTY_CONTEXT());
-                results.put(rule, result);
-            } catch (ParseException e) {
-                results.put(rule, false);
-            }
-        }
-        return results;
-    }
-
-    @Override
-    public Map<String, Object> validateTransformation(TransformationValidation transformationValidation) {
-        JSONObject sampleJson = new JSONObject(transformationValidation.getSampleData());
-        transformationValidation.getSensorParserConfig().getFieldTransformations().forEach(fieldTransformer -> {
-                    fieldTransformer.transformAndUpdate(sampleJson, transformationValidation.getSensorParserConfig().getParserConfig(), Context.EMPTY_CONTEXT());
-                }
-        );
-        return sampleJson;
-    }
-
-    @Override
-    public FieldTransformations[] getTransformations() {
-        return FieldTransformations.values();
-    }
-
-    @Override
-    public List<StellarFunctionDescription> getStellarFunctions() {
-        List<StellarFunctionDescription> stellarFunctionDescriptions = new ArrayList<>();
-        Iterable<StellarFunctionInfo> stellarFunctionsInfo = SingletonFunctionResolver.getInstance().getFunctionInfo();
-        stellarFunctionsInfo.forEach(stellarFunctionInfo -> {
-            stellarFunctionDescriptions.add(new StellarFunctionDescription(
-                    stellarFunctionInfo.getName(),
-                    stellarFunctionInfo.getDescription(),
-                    stellarFunctionInfo.getParams(),
-                    stellarFunctionInfo.getReturns()));
-        });
-        return stellarFunctionDescriptions;
-    }
-
-    @Override
-    public List<StellarFunctionDescription> getSimpleStellarFunctions() {
-      List<StellarFunctionDescription> stellarFunctionDescriptions = getStellarFunctions();
-      return stellarFunctionDescriptions.stream().filter(stellarFunctionDescription ->
-              stellarFunctionDescription.getParams().length == 1).sorted((o1, o2) -> o1.getName().compareTo(o2.getName())).collect(Collectors.toList());
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/resources/application-docker.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-docker.yml b/metron-interface/metron-rest/src/main/resources/application-docker.yml
index 859ca5e..5891417 100644
--- a/metron-interface/metron-rest/src/main/resources/application-docker.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-docker.yml
@@ -21,11 +21,11 @@ docker:
 
 spring:
   datasource:
-      driverClassName: com.mysql.jdbc.Driver
-      url: jdbc:mysql://${docker.host.address}:3306/metronrest
+      driverClassName: org.h2.Driver
+      url: jdbc:h2:file:./metrondb
       username: root
       password: root
-      platform: mysql
+      platform: h2
   jpa:
     hibernate:
       ddl-auto: update

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
new file mode 100644
index 0000000..76bef56
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+spring:
+  datasource:
+      driverClassName: org.h2.Driver
+      url: jdbc:h2:file:./metrondb
+      username: root
+      password: root
+      platform: h2
+  jpa:
+    hibernate:
+      ddl-auto: update
+
+zookeeper:
+  url:  node1:2181
+
+kafka:
+  broker:
+    url: node1:6667
+
+hdfs:
+  namenode:
+    url: hdfs://node1:8020
+
+grok:
+  path:
+    temp: ./patterns/temp
+    default: /apps/metron/patterns
+
+storm:
+  ui:
+    url:   node1:8744
+  parser:
+    script.path: /usr/metron/${metron.version}/bin/start_parser_topology.sh
+  enrichment:
+    script.path: /usr/metron/${metron.version}/bin/start_enrichment_topology.sh
+  indexing:
+    script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/resources/application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml
index 7e45ed9..2bb1462 100644
--- a/metron-interface/metron-rest/src/main/resources/application.yml
+++ b/metron-interface/metron-rest/src/main/resources/application.yml
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 metron:
-  version: 0.3.0
+  version: 0.3.1
 
 logging:
   level:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/scripts/start.sh
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/scripts/start.sh b/metron-interface/metron-rest/src/main/scripts/start.sh
deleted file mode 100755
index d83a2fd..0000000
--- a/metron-interface/metron-rest/src/main/scripts/start.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-METRON_VERSION=0.3.0
-SCRIPTS_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-if [ "$#" -ne 1 ]; then
-  echo "Usage: start.sh <path to application.yml file>"
-  echo "Path can be absolute or relative to $SCRIPTS_ROOT"
-else
-  if [ -z "$MYSQL_CLIENT_HOME" ]; then
-    echo MYSQL_CLIENT_HOME is not set
-  else
-    if [ -z "$HIBERNATE_HOME" ]; then
-      echo HIBERNATE_HOME is not set
-    else
-      java -Dloader.path=$MYSQL_CLIENT_HOME/*,$HIBERNATE_HOME/lib/envers/*,$HIBERNATE_HOME/lib/jpa/*,$HIBERNATE_HOME/lib/required/* -jar $SCRIPTS_ROOT/../lib/metron-rest-$METRON_VERSION.jar --spring.config.location=$1
-    fi
-  fi
-fi
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh b/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh
new file mode 100755
index 0000000..ea5d6c9
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+METRON_VERSION=${project.version}
+SCRIPTS_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+if [ "$#" -lt 1 ]; then
+  echo "Usage: start.sh <path to application.yml file> <spring options>"
+  echo "Path can be absolute or relative to $SCRIPTS_ROOT"
+else
+  java -jar $SCRIPTS_ROOT/../lib/metron-rest-$METRON_VERSION.jar --spring.config.location=$@
+fi
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java
index 7796c8b..f4e18ea 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java
@@ -18,6 +18,7 @@
 package org.apache.metron.rest.controller;
 
 import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.rest.service.GlobalConfigService;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -57,6 +58,9 @@ public class GlobalConfigControllerIntegrationTest {
     public static String globalJson;
 
     @Autowired
+    private GlobalConfigService globalConfigService;
+
+    @Autowired
     private WebApplicationContext wac;
 
     private MockMvc mockMvc;
@@ -84,6 +88,8 @@ public class GlobalConfigControllerIntegrationTest {
 
     @Test
     public void test() throws Exception {
+        this.globalConfigService.delete();
+
         this.mockMvc.perform(get(globalConfigUrl).with(httpBasic(user,password)))
                 .andExpect(status().isNotFound());
 
@@ -103,5 +109,7 @@ public class GlobalConfigControllerIntegrationTest {
 
         this.mockMvc.perform(delete(globalConfigUrl).with(httpBasic(user,password)).with(csrf()))
                 .andExpect(status().isNotFound());
+
+        this.globalConfigService.delete();
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
index 4532616..e0a9c5b 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
@@ -48,6 +48,7 @@ public class GrokControllerIntegrationTest {
     /**
      {
      "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html",
+     "patternLabel":"SQUID",
      "statement":"SQUID %{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}"
      }
      */
@@ -57,11 +58,20 @@ public class GrokControllerIntegrationTest {
     /**
      {
      "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html",
-     "statement":""
+     "statement":"SQUID %{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}"
+     }
+     */
+    @Multiline
+    public static String missingPatternLabelGrokValidationJson;
+
+    /**
+     {
+     "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html",
+     "patternLabel":"SQUID"
      }
      */
     @Multiline
-    public static String badGrokValidationJson;
+    public static String missingStatementGrokValidationJson;
 
     @Autowired
     private WebApplicationContext wac;
@@ -101,11 +111,17 @@ public class GrokControllerIntegrationTest {
                 .andExpect(jsonPath("$.results.timestamp").value("1467011157.401"))
                 .andExpect(jsonPath("$.results.url").value("http://www.aliexpress.com/af/shoes.html?"));
 
-        this.mockMvc.perform(post(grokUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(badGrokValidationJson))
+        this.mockMvc.perform(post(grokUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(missingPatternLabelGrokValidationJson))
+                .andExpect(status().isInternalServerError())
+                .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+                .andExpect(jsonPath("$.responseCode").value(500))
+                .andExpect(jsonPath("$.message").value("Pattern label is required"));
+
+        this.mockMvc.perform(post(grokUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(missingStatementGrokValidationJson))
                 .andExpect(status().isInternalServerError())
                 .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
                 .andExpect(jsonPath("$.responseCode").value(500))
-                .andExpect(jsonPath("$.message").value("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)"));
+                .andExpect(jsonPath("$.message").value("Grok statement is required"));
 
         this.mockMvc.perform(get(grokUrl + "/list").with(httpBasic(user,password)))
                 .andExpect(status().isOk())

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java
new file mode 100644
index 0000000..a6c3469
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.rest.service.HdfsService;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class HdfsControllerIntegrationTest {
+
+    @Autowired
+    private HdfsService hdfsService;
+
+    @Autowired
+    private WebApplicationContext wac;
+
+    private MockMvc mockMvc;
+
+    private String hdfsUrl = "/api/v1/hdfs";
+    private String user = "user";
+    private String password = "password";
+    private String path = "./target/hdfsTest.txt";
+    private String fileContents = "file contents";
+
+    @Before
+    public void setup() throws Exception {
+        this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+    }
+
+    @Test
+    public void testSecurity() throws Exception {
+        this.mockMvc.perform(post(hdfsUrl).with(csrf()).contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")).content(fileContents))
+                .andExpect(status().isUnauthorized());
+
+        this.mockMvc.perform(get(hdfsUrl))
+                .andExpect(status().isUnauthorized());
+
+        this.mockMvc.perform(delete(hdfsUrl).with(csrf()))
+                .andExpect(status().isUnauthorized());
+    }
+
+    @Test
+    public void test() throws Exception {
+        this.hdfsService.delete(new Path(path), false);
+
+        this.mockMvc.perform(get(hdfsUrl + "?path=" + path).with(httpBasic(user,password)))
+                .andExpect(status().isNotFound());
+
+        this.mockMvc.perform(post(hdfsUrl + "?path=" + path).with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")).content(fileContents))
+                .andExpect(status().isOk());
+
+        this.mockMvc.perform(get(hdfsUrl + "?path=" + path).with(httpBasic(user,password)))
+                .andExpect(status().isOk())
+                .andExpect(content().contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")))
+                .andExpect(content().bytes(fileContents.getBytes()));
+
+        this.mockMvc.perform(delete(hdfsUrl + "?path=" + path).with(httpBasic(user,password)).with(csrf()))
+                .andExpect(status().isOk());
+
+        this.mockMvc.perform(delete(hdfsUrl + "?path=" + path).with(httpBasic(user,password)).with(csrf()))
+                .andExpect(status().isNotFound());
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
index a659772..6a67473 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
@@ -94,9 +94,12 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
    ]
    },
    "triageConfig": {
-   "riskLevelRules": {
-   "ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'": 10
-   },
+   "riskLevelRules": [
+   {
+   "rule": "ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'",
+   "score": 10
+   }
+   ],
    "aggregator": "MAX"
    }
    }
@@ -160,7 +163,8 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
             .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[1]").value("ip_dst_addr"))
             .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_src_addr[0]").value("malicious_ip"))
             .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_dst_addr[0]").value("malicious_ip"))
-            .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[\"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\"]").value(10))
+            .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].rule").value("ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'"))
+            .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].score").value(10))
             .andExpect(jsonPath("$.threatIntel.triageConfig.aggregator").value("MAX"));
 
     this.mockMvc.perform(post(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
@@ -177,7 +181,8 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
             .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[1]").value("ip_dst_addr"))
             .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_src_addr[0]").value("malicious_ip"))
             .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_dst_addr[0]").value("malicious_ip"))
-            .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[\"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\"]").value(10))
+            .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].rule").value("ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'"))
+            .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].score").value(10))
             .andExpect(jsonPath("$.threatIntel.triageConfig.aggregator").value("MAX"));
 
     this.mockMvc.perform(get(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user,password)))
@@ -194,7 +199,8 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
             .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[1]").value("ip_dst_addr"))
             .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_src_addr[0]").value("malicious_ip"))
             .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_dst_addr[0]").value("malicious_ip"))
-            .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[\"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\"]").value(10))
+            .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].rule").value("ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'"))
+            .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].score").value(10))
             .andExpect(jsonPath("$.threatIntel.triageConfig.aggregator").value("MAX"));
 
     this.mockMvc.perform(get(sensorEnrichmentConfigUrl).with(httpBasic(user,password)))
@@ -211,7 +217,8 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
                     "@.broTest.threatIntel.fieldMap.hbaseThreatIntel[1] == 'ip_dst_addr' &&" +
                     "@.broTest.threatIntel.fieldToTypeMap.ip_src_addr[0] == 'malicious_ip' &&" +
                     "@.broTest.threatIntel.fieldToTypeMap.ip_dst_addr[0] == 'malicious_ip' &&" +
-                    "@.broTest.threatIntel.triageConfig.riskLevelRules[\"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\"] == 10 &&" +
+                    "@.broTest.threatIntel.triageConfig.riskLevelRules[0].rule == \"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\" &&" +
+                    "@.broTest.threatIntel.triageConfig.riskLevelRules[0].score == 10 &&" +
                     "@.broTest.threatIntel.triageConfig.aggregator == 'MAX'" +
                     ")]").exists());
 
@@ -235,6 +242,8 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
             .andExpect(jsonPath("$[0]").value("geo"))
             .andExpect(jsonPath("$[1]").value("host"))
             .andExpect(jsonPath("$[2]").value("whois"));
+
+    sensorEnrichmentConfigService.delete("broTest");
   }
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
index 10edadc..cebcde6 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
@@ -135,6 +135,8 @@ public class SensorIndexingConfigControllerIntegrationTest {
             .andExpect(status().isOk())
             .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
             .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").doesNotExist());
+
+    sensorIndexingConfigService.delete("broTest");
   }
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
index 09b1b0a..45ee1f5 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
@@ -20,6 +20,8 @@ package org.apache.metron.rest.controller;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.commons.io.FileUtils;
 import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.rest.service.GrokService;
+import org.apache.metron.rest.service.SensorParserConfigService;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -58,7 +60,7 @@ public class SensorParserConfigControllerIntegrationTest {
    "sensorTopic": "squidTest",
    "parserConfig": {
    "patternLabel": "SQUIDTEST",
-   "grokStatement": "%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}",
+   "grokPath": "target/patterns/squidTest",
    "timestampField": "timestamp"
    },
    "fieldTransformations" : [
@@ -78,19 +80,6 @@ public class SensorParserConfigControllerIntegrationTest {
 
   /**
    {
-   "parserClassName": "org.apache.metron.parsers.GrokParser",
-   "sensorTopic": "squidTest",
-   "parserConfig": {
-   "patternLabel": "SQUIDTEST",
-   "timestampField": "timestamp"
-   }
-   }
-   */
-  @Multiline
-  public static String missingGrokJson;
-
-  /**
-   {
    "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
    "sensorTopic":"broTest",
    "parserConfig": {}
@@ -106,12 +95,12 @@ public class SensorParserConfigControllerIntegrationTest {
    "parserClassName": "org.apache.metron.parsers.GrokParser",
    "sensorTopic": "squidTest",
    "parserConfig": {
-   "grokStatement": "%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}",
-   "patternLabel": "SQUIDTEST",
+   "patternLabel": "SQUID_DELIMITED",
    "grokPath":"./squidTest",
    "timestampField": "timestamp"
    }
    },
+   "grokStatement":"SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})?",
    "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html"
    }
    */
@@ -168,6 +157,9 @@ public class SensorParserConfigControllerIntegrationTest {
   private Environment environment;
 
   @Autowired
+  private SensorParserConfigService sensorParserConfigService;
+
+  @Autowired
   private WebApplicationContext wac;
 
   private MockMvc mockMvc;
@@ -199,6 +191,8 @@ public class SensorParserConfigControllerIntegrationTest {
   @Test
   public void test() throws Exception {
     cleanFileSystem();
+    this.sensorParserConfigService.delete("broTest");
+    this.sensorParserConfigService.delete("squidTest");
 
     this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(squidJson))
             .andExpect(status().isCreated())
@@ -207,7 +201,6 @@ public class SensorParserConfigControllerIntegrationTest {
             .andExpect(jsonPath("$.sensorTopic").value("squidTest"))
             .andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest"))
             .andExpect(jsonPath("$.parserConfig.patternLabel").value("SQUIDTEST"))
-            .andExpect(jsonPath("$.parserConfig.grokStatement").value("%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}/%{WORD:UNWANTED}"))
             .andExpect(jsonPath("$.parserConfig.timestampField").value("timestamp"))
             .andExpect(jsonPath("$.fieldTransformations[0].transformation").value("STELLAR"))
             .andExpect(jsonPath("$.fieldTransformations[0].output[0]").value("full_hostname"))
@@ -215,12 +208,6 @@ public class SensorParserConfigControllerIntegrationTest {
             .andExpect(jsonPath("$.fieldTransformations[0].config.full_hostname").value("URL_TO_HOST(url)"))
             .andExpect(jsonPath("$.fieldTransformations[0].config.domain_without_subdomains").value("DOMAIN_REMOVE_SUBDOMAINS(full_hostname)"));
 
-    this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(missingGrokJson))
-            .andExpect(status().isInternalServerError())
-            .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
-            .andExpect(jsonPath("$.responseCode").value(500))
-            .andExpect(jsonPath("$.message").value("A grokStatement must be provided"));
-
     this.mockMvc.perform(get(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password)))
             .andExpect(status().isOk())
             .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
@@ -228,7 +215,6 @@ public class SensorParserConfigControllerIntegrationTest {
             .andExpect(jsonPath("$.sensorTopic").value("squidTest"))
             .andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest"))
             .andExpect(jsonPath("$.parserConfig.patternLabel").value("SQUIDTEST"))
-            .andExpect(jsonPath("$.parserConfig.grokStatement").value("%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}/%{WORD:UNWANTED}"))
             .andExpect(jsonPath("$.parserConfig.timestampField").value("timestamp"))
             .andExpect(jsonPath("$.fieldTransformations[0].transformation").value("STELLAR"))
             .andExpect(jsonPath("$.fieldTransformations[0].output[0]").value("full_hostname"))
@@ -350,6 +336,8 @@ public class SensorParserConfigControllerIntegrationTest {
             .andExpect(jsonPath("$.responseCode").value(500))
             .andExpect(jsonPath("$.message").value("java.lang.ClassNotFoundException: badClass"));
 
+    this.sensorParserConfigService.delete("broTest");
+    this.sensorParserConfigService.delete("squidTest");
   }
 
   private void cleanFileSystem() throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StellarControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StellarControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StellarControllerIntegrationTest.java
new file mode 100644
index 0000000..7a57173
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StellarControllerIntegrationTest.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class StellarControllerIntegrationTest {
+
+    private String valid = "TO_LOWER(test)";
+    private String invalid = "BAD_FUNCTION(test)";
+    private String rulesJson = "[\"" + valid + "\",\"" + invalid + "\"]";
+
+    /**
+     {
+     "sensorParserConfig": { "fieldTransformations" : [{"transformation" : "STELLAR","output" : ["url_host"],"config" : {"url_host" : "TO_LOWER(URL_TO_HOST(url))"}}]},
+     "sampleData": {"url": "https://caseystella.com/blog"}
+     }
+     */
+    @Multiline
+    public static String sensorParseContext;
+
+
+    @Autowired
+    private WebApplicationContext wac;
+
+    private MockMvc mockMvc;
+
+    private String stellarUrl = "/api/v1/stellar";
+    private String user = "user";
+    private String password = "password";
+
+    @Before
+    public void setup() throws Exception {
+        this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+    }
+
+    @Test
+    public void testSecurity() throws Exception {
+        this.mockMvc.perform(post(stellarUrl + "/validate/rules").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(rulesJson))
+                .andExpect(status().isUnauthorized());
+
+        this.mockMvc.perform(post(stellarUrl + "/apply/transformations").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(sensorParseContext))
+                .andExpect(status().isUnauthorized());
+
+        this.mockMvc.perform(get(stellarUrl + "/list"))
+                .andExpect(status().isUnauthorized());
+
+        this.mockMvc.perform(get(stellarUrl + "/list/functions"))
+                .andExpect(status().isUnauthorized());
+    }
+
+    @Test
+    public void test() throws Exception {
+        this.mockMvc.perform(post(stellarUrl + "/validate/rules").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(rulesJson))
+                .andExpect(status().isOk())
+                .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+                .andExpect(jsonPath("$.['" + valid + "']").value(Boolean.TRUE))
+                .andExpect(jsonPath("$.['" + invalid + "']").value(Boolean.FALSE));
+
+        this.mockMvc.perform(post(stellarUrl + "/apply/transformations").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(sensorParseContext))
+                .andExpect(status().isOk())
+                .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+                .andExpect(jsonPath("$.url").value("https://caseystella.com/blog"))
+                .andExpect(jsonPath("$.url_host").value("caseystella.com"));
+
+        this.mockMvc.perform(get(stellarUrl + "/list").with(httpBasic(user,password)))
+                .andExpect(status().isOk())
+                .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+                .andExpect(jsonPath("$", hasSize(greaterThan(0))));
+
+        this.mockMvc.perform(get(stellarUrl + "/list/functions").with(httpBasic(user,password)))
+                .andExpect(status().isOk())
+                .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+                .andExpect(jsonPath("$", hasSize(greaterThan(0))));
+
+        this.mockMvc.perform(get(stellarUrl + "/list/simple/functions").with(httpBasic(user,password)))
+                .andExpect(status().isOk())
+                .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+                .andExpect(jsonPath("$", hasSize(greaterThan(0))));
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
index c592159..69d8da6 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
@@ -316,5 +316,8 @@ public class StormControllerIntegrationTest {
             .andExpect(jsonPath("$.parserScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_parser_topology.sh"))
             .andExpect(jsonPath("$.enrichmentScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_enrichment_topology.sh"))
             .andExpect(jsonPath("$.indexingScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_elasticsearch_topology.sh"));
+
+    globalConfigService.delete();
+    sensorParserConfigService.delete("broTest");
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/TransformationControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/TransformationControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/TransformationControllerIntegrationTest.java
deleted file mode 100644
index ec987e0..0000000
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/TransformationControllerIntegrationTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.controller;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.http.MediaType;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.junit4.SpringRunner;
-import org.springframework.test.web.servlet.MockMvc;
-import org.springframework.test.web.servlet.setup.MockMvcBuilders;
-import org.springframework.web.context.WebApplicationContext;
-
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.hasSize;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
-import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
-
-@RunWith(SpringRunner.class)
-@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
-@ActiveProfiles(TEST_PROFILE)
-public class TransformationControllerIntegrationTest {
-
-    private String valid = "TO_LOWER(test)";
-    private String invalid = "BAD_FUNCTION(test)";
-    private String rulesJson = "[\"" + valid + "\",\"" + invalid + "\"]";
-
-    /**
-     {
-     "sensorParserConfig": { "fieldTransformations" : [{"transformation" : "STELLAR","output" : ["url_host"],"config" : {"url_host" : "TO_LOWER(URL_TO_HOST(url))"}}]},
-     "sampleData": {"url": "https://caseystella.com/blog"}
-     }
-     */
-    @Multiline
-    public static String sensorParseConfigJson;
-
-
-    @Autowired
-    private WebApplicationContext wac;
-
-    private MockMvc mockMvc;
-
-    private String transformationUrl = "/api/v1/transformation";
-    private String user = "user";
-    private String password = "password";
-
-    @Before
-    public void setup() throws Exception {
-        this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
-    }
-
-    @Test
-    public void testSecurity() throws Exception {
-        this.mockMvc.perform(post(transformationUrl + "/validate/rules").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(rulesJson))
-                .andExpect(status().isUnauthorized());
-
-        this.mockMvc.perform(post(transformationUrl + "/validate").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(sensorParseConfigJson))
-                .andExpect(status().isUnauthorized());
-
-        this.mockMvc.perform(get(transformationUrl + "/list"))
-                .andExpect(status().isUnauthorized());
-
-        this.mockMvc.perform(get(transformationUrl + "/list/functions"))
-                .andExpect(status().isUnauthorized());
-    }
-
-    @Test
-    public void test() throws Exception {
-        this.mockMvc.perform(post(transformationUrl + "/validate/rules").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(rulesJson))
-                .andExpect(status().isOk())
-                .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
-                .andExpect(jsonPath("$.['" + valid + "']").value(Boolean.TRUE))
-                .andExpect(jsonPath("$.['" + invalid + "']").value(Boolean.FALSE));
-
-        this.mockMvc.perform(post(transformationUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(sensorParseConfigJson))
-                .andExpect(status().isOk())
-                .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
-                .andExpect(jsonPath("$.url").value("https://caseystella.com/blog"))
-                .andExpect(jsonPath("$.url_host").value("caseystella.com"));
-
-        this.mockMvc.perform(get(transformationUrl + "/list").with(httpBasic(user,password)))
-                .andExpect(status().isOk())
-                .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
-                .andExpect(jsonPath("$", hasSize(greaterThan(0))));
-
-        this.mockMvc.perform(get(transformationUrl + "/list/functions").with(httpBasic(user,password)))
-                .andExpect(status().isOk())
-                .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
-                .andExpect(jsonPath("$", hasSize(greaterThan(0))));
-
-        this.mockMvc.perform(get(transformationUrl + "/list/simple/functions").with(httpBasic(user,password)))
-                .andExpect(status().isOk())
-                .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
-                .andExpect(jsonPath("$", hasSize(greaterThan(0))));
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java
deleted file mode 100644
index 561548e..0000000
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.service;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.api.DeleteBuilder;
-import org.apache.curator.framework.api.GetChildrenBuilder;
-import org.apache.metron.common.configuration.ConfigurationType;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.configuration.SensorParserConfig;
-import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.rest.service.impl.SensorParserConfigServiceImpl;
-import org.apache.zookeeper.KeeperException;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.MockitoAnnotations;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-import static org.powermock.api.mockito.PowerMockito.verifyStatic;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({ConfigurationsUtils.class})
-public class SensorParserConfigTest {
-
-  @Mock
-  private GetChildrenBuilder getChildrenBuilder;
-
-  @Mock
-  private DeleteBuilder deleteBuilder;
-
-  @Mock
-  private ObjectMapper objectMapper;
-
-  @Mock
-  private CuratorFramework client;
-
-  @Mock
-  private GrokService grokService;
-
-  @InjectMocks
-  private SensorParserConfigServiceImpl sensorParserConfigService;
-
-  @Before
-  public void setUp() throws Exception {
-    MockitoAnnotations.initMocks(this);
-    Mockito.when(client.getChildren()).thenReturn(getChildrenBuilder);
-    Mockito.when(client.delete()).thenReturn(deleteBuilder);
-
-  }
-
-  @Test
-  public void test() throws Exception {
-    mockStatic(ConfigurationsUtils.class);
-    SensorParserConfig broParserConfig = new SensorParserConfig();
-    broParserConfig.setParserClassName("org.apache.metron.parsers.bro.BasicBroParser");
-    broParserConfig.setSensorTopic("broTest");
-    Mockito.when(objectMapper.writeValueAsString(broParserConfig)).thenReturn(new String(JSONUtils.INSTANCE.toJSON(broParserConfig)));
-    sensorParserConfigService.save(broParserConfig);
-    verifyStatic(times(1));
-    ConfigurationsUtils.writeSensorParserConfigToZookeeper("broTest", JSONUtils.INSTANCE.toJSON(broParserConfig), client);
-
-    PowerMockito.when(ConfigurationsUtils.readSensorParserConfigFromZookeeper("broTest", client)).thenReturn(broParserConfig);
-    assertEquals(broParserConfig, sensorParserConfigService.findOne("broTest"));
-
-    SensorParserConfig squidParserConfig = new SensorParserConfig();
-    squidParserConfig.setParserClassName("org.apache.metron.parsers.GrokParser");
-    squidParserConfig.setSensorTopic("squid");
-    PowerMockito.when(ConfigurationsUtils.readSensorParserConfigFromZookeeper("squidTest", client)).thenReturn(squidParserConfig);
-
-    List<String> allTypes = new ArrayList<String>() {{
-      add("broTest");
-      add("squidTest");
-    }};
-    Mockito.when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())).thenReturn(allTypes);
-    assertEquals(new ArrayList<SensorParserConfig>() {{ add(broParserConfig); add(squidParserConfig); }}, sensorParserConfigService.getAll());
-
-    Mockito.when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())).thenThrow(new KeeperException.NoNodeException());
-    assertEquals(new ArrayList<>(), sensorParserConfigService.getAll());
-
-    assertTrue(sensorParserConfigService.delete("broTest"));
-    verify(deleteBuilder, times(1)).forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/broTest");
-    Mockito.when(deleteBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/broTest")).thenThrow(new KeeperException.NoNodeException());
-    assertFalse(sensorParserConfigService.delete("broTest"));
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
index 1217bcb..bc91b91 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
@@ -24,7 +24,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 import org.springframework.core.env.Environment;
 
-import java.io.IOException;
+import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -60,12 +60,7 @@ public class DockerStormCLIWrapperTest {
     when(processBuilder.command()).thenReturn(new ArrayList<>());
 
     Process process = mock(Process.class);
-    InputStream inputStream = new InputStream() {
-      @Override
-      public int read() throws IOException {
-        return -1;
-      }
-    };
+    InputStream inputStream = new ByteArrayInputStream("export DOCKER_HOST=\"tcp://192.168.99.100:2376\"".getBytes());
 
     when(processBuilder.start()).thenReturn(process);
     when(process.getInputStream()).thenReturn(inputStream);
@@ -75,7 +70,10 @@ public class DockerStormCLIWrapperTest {
 
     ProcessBuilder actualBuilder = dockerStormCLIWrapper.getProcessBuilder("oo", "ooo");
 
-    assertEquals(new HashMap<String, String>() {{ put("METRON_VERSION", "1"); }}, actualBuilder.environment());
+    assertEquals(new HashMap<String, String>() {{
+      put("METRON_VERSION", "1");
+      put("DOCKER_HOST", "tcp://192.168.99.100:2376");
+    }}, actualBuilder.environment());
     assertEquals(new ArrayList<>(), actualBuilder.command());
 
     verify(process).waitFor();

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
index 7fc8748..6dd95c5 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
@@ -18,6 +18,7 @@
 package org.apache.metron.rest.service.impl;
 
 import oi.thekraken.grok.api.Grok;
+import org.apache.commons.io.FileUtils;
 import org.apache.metron.rest.RestException;
 import org.apache.metron.rest.model.GrokValidation;
 import org.apache.metron.rest.service.GrokService;
@@ -25,26 +26,42 @@ import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
-
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.springframework.core.env.Environment;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
+import static org.powermock.api.mockito.PowerMockito.when;
+import static org.powermock.api.mockito.PowerMockito.whenNew;
 
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({GrokServiceImpl.class, FileWriter.class})
 public class GrokServiceImplTest {
   @Rule
   public final ExpectedException exception = ExpectedException.none();
 
+  private Environment environment;
   private Grok grok;
   private GrokService grokService;
 
   @Before
   public void setUp() throws Exception {
+    environment = mock(Environment.class);
     grok = mock(Grok.class);
-    grokService = new GrokServiceImpl(grok);
+    grokService = new GrokServiceImpl(environment, grok);
   }
 
   @Test
@@ -56,24 +73,42 @@ public class GrokServiceImplTest {
 
   @Test
   public void getCommonGrokPattersShouldCallGrokToGetPatternsAndNotAlterValue() throws Exception {
-    Map<String, String> patterns = new HashMap<String, String>() {{
+    final Map<String, String> actual = new HashMap<String, String>() {{
       put("k", "v");
       put("k1", "v1");
     }};
 
-    when(grok.getPatterns()).thenReturn(patterns);
+    when(grok.getPatterns()).thenReturn(actual);
+
+    Map<String, String> expected = new HashMap<String, String>() {{
+      put("k", "v");
+      put("k1", "v1");
+    }};
+    assertEquals(expected, grokService.getCommonGrokPatterns());
+  }
+
+  @Test
+  public void validateGrokStatementShouldThrowExceptionWithNullStringAsPatternLabel() throws Exception {
+    exception.expect(RestException.class);
+    exception.expectMessage("Pattern label is required");
+
+    GrokValidation grokValidation = new GrokValidation();
+    grokValidation.setResults(new HashMap<>());
+    grokValidation.setSampleData("asdf asdf");
+    grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
 
-    assertEquals(patterns, grokService.getCommonGrokPatterns());
+    grokService.validateGrokStatement(grokValidation);
   }
 
   @Test
   public void validateGrokStatementShouldThrowExceptionWithEmptyStringAsStatement() throws Exception {
     exception.expect(RestException.class);
-    exception.expectMessage("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)");
+    exception.expectMessage("Grok statement is required");
 
     GrokValidation grokValidation = new GrokValidation();
     grokValidation.setResults(new HashMap<>());
     grokValidation.setSampleData("asdf asdf");
+    grokValidation.setPatternLabel("LABEL");
     grokValidation.setStatement("");
 
     grokService.validateGrokStatement(grokValidation);
@@ -82,11 +117,12 @@ public class GrokServiceImplTest {
   @Test
   public void validateGrokStatementShouldThrowExceptionWithNullStringAsStatement() throws Exception {
     exception.expect(RestException.class);
-    exception.expectMessage("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)");
+    exception.expectMessage("Grok statement is required");
 
     GrokValidation grokValidation = new GrokValidation();
     grokValidation.setResults(new HashMap<>());
     grokValidation.setSampleData("asdf asdf");
+    grokValidation.setPatternLabel("LABEL");
     grokValidation.setStatement(null);
 
     grokService.validateGrokStatement(grokValidation);
@@ -94,47 +130,55 @@ public class GrokServiceImplTest {
 
   @Test
   public void validateGrokStatementShouldProperlyMatchSampleDataAgainstGivenStatement() throws Exception {
-    GrokValidation grokValidation = new GrokValidation();
+    final GrokValidation grokValidation = new GrokValidation();
     grokValidation.setResults(new HashMap<>());
     grokValidation.setSampleData("asdf asdf");
     grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+    grokValidation.setPatternLabel("LABEL");
 
-    GrokValidation expectedGrokValidation = new GrokValidation();
-    expectedGrokValidation.setResults(new HashMap<String, Object>() {{ put("word1", "asdf"); put("word2", "asdf"); }});
-    expectedGrokValidation.setSampleData("asdf asdf");
-    expectedGrokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+    GrokValidation expected = new GrokValidation();
+    expected.setResults(new HashMap<String, Object>() {{ put("word1", "asdf"); put("word2", "asdf"); }});
+    expected.setSampleData("asdf asdf");
+    expected.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+    expected.setPatternLabel("LABEL");
 
-    assertEquals(expectedGrokValidation, grokService.validateGrokStatement(grokValidation));
+    GrokValidation actual = grokService.validateGrokStatement(grokValidation);
+    assertEquals(expected, actual);
+    assertEquals(expected.hashCode(), actual.hashCode());
   }
 
   @Test
   public void validateGrokStatementShouldProperlyMatchNothingAgainstEmptyString() throws Exception {
-    GrokValidation grokValidation = new GrokValidation();
+    final GrokValidation grokValidation = new GrokValidation();
     grokValidation.setResults(new HashMap<>());
     grokValidation.setSampleData("");
+    grokValidation.setPatternLabel("LABEL");
     grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
 
-    GrokValidation expectedGrokValidation = new GrokValidation();
-    expectedGrokValidation.setResults(new HashMap<>());
-    expectedGrokValidation.setSampleData("");
-    expectedGrokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+    GrokValidation expected = new GrokValidation();
+    expected.setResults(new HashMap<>());
+    expected.setSampleData("");
+    expected.setPatternLabel("LABEL");
+    expected.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
 
-    assertEquals(expectedGrokValidation, grokService.validateGrokStatement(grokValidation));
+    assertEquals(expected, grokService.validateGrokStatement(grokValidation));
   }
 
   @Test
   public void validateGrokStatementShouldProperlyMatchNothingAgainstNullString() throws Exception {
-    GrokValidation grokValidation = new GrokValidation();
+    final GrokValidation grokValidation = new GrokValidation();
     grokValidation.setResults(new HashMap<>());
     grokValidation.setSampleData(null);
     grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+    grokValidation.setPatternLabel("LABEL");
 
-    GrokValidation expectedGrokValidation = new GrokValidation();
-    expectedGrokValidation.setResults(new HashMap<>());
-    expectedGrokValidation.setSampleData(null);
-    expectedGrokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+    GrokValidation expected = new GrokValidation();
+    expected.setResults(new HashMap<>());
+    expected.setSampleData(null);
+    expected.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+    expected.setPatternLabel("LABEL");
 
-    assertEquals(expectedGrokValidation, grokService.validateGrokStatement(grokValidation));
+    assertEquals(expected, grokService.validateGrokStatement(grokValidation));
   }
 
   @Test
@@ -148,4 +192,44 @@ public class GrokServiceImplTest {
 
     grokService.validateGrokStatement(grokValidation);
   }
+
+  @Test
+  public void saveTemporaryShouldProperlySaveFile() throws Exception {
+    new File("./target/user1").delete();
+    String statement = "grok statement";
+
+    Authentication authentication = mock(Authentication.class);
+    when(authentication.getName()).thenReturn("user1");
+    SecurityContextHolder.getContext().setAuthentication(authentication);
+    when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target");
+
+    grokService.saveTemporary(statement, "squid");
+
+    File testFile = new File("./target/user1/squid");
+    assertEquals(statement, FileUtils.readFileToString(testFile));
+    testFile.delete();
+  }
+
+  @Test
+  public void saveTemporaryShouldWrapExceptionInRestException() throws Exception {
+    exception.expect(RestException.class);
+
+    String statement = "grok statement";
+
+    Authentication authentication = mock(Authentication.class);
+    when(authentication.getName()).thenReturn("user1");
+    SecurityContextHolder.getContext().setAuthentication(authentication);
+    when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target");
+    whenNew(FileWriter.class).withParameterTypes(File.class).withArguments(any()).thenThrow(new IOException());
+
+    grokService.saveTemporary(statement, "squid");
+  }
+
+  @Test
+  public void missingGrokStatementShouldThrowRestException() throws Exception {
+    exception.expect(RestException.class);
+    exception.expectMessage("A grokStatement must be provided");
+
+    grokService.saveTemporary(null, "squid");
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
new file mode 100644
index 0000000..d11f5a4
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.HdfsService;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.IOException;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({HdfsServiceImpl.class, FileSystem.class})
+public class HdfsServiceImplExceptionTest {
+    @Rule
+    public final ExpectedException exception = ExpectedException.none();
+
+    private Configuration configuration;
+    private HdfsService hdfsService;
+    private String testDir = "./target/hdfsUnitTest";
+
+    @Before
+    public void setup() throws IOException {
+        configuration = new Configuration();
+        hdfsService = new HdfsServiceImpl(configuration);
+
+        mockStatic(FileSystem.class);
+    }
+
+    @Test
+    public void listShouldWrapExceptionInRestException() throws Exception {
+      exception.expect(RestException.class);
+
+      FileSystem fileSystem = mock(FileSystem.class);
+      when(FileSystem.get(configuration)).thenReturn(fileSystem);
+      when(fileSystem.listStatus(new Path(testDir))).thenThrow(new IOException());
+
+      hdfsService.list(new Path(testDir));
+    }
+
+    @Test
+    public void readShouldWrapExceptionInRestException() throws Exception {
+        exception.expect(RestException.class);
+
+        FileSystem fileSystem = mock(FileSystem.class);
+        when(FileSystem.get(configuration)).thenReturn(fileSystem);
+        when(fileSystem.open(new Path(testDir))).thenThrow(new IOException());
+
+        hdfsService.read(new Path(testDir));
+    }
+
+    @Test
+    public void writeShouldWrapExceptionInRestException() throws Exception {
+        exception.expect(RestException.class);
+
+        FileSystem fileSystem = mock(FileSystem.class);
+        when(FileSystem.get(configuration)).thenReturn(fileSystem);
+        when(fileSystem.create(new Path(testDir), true)).thenThrow(new IOException());
+
+        hdfsService.write(new Path(testDir), "contents".getBytes(UTF_8));
+    }
+
+    @Test
+    public void deleteShouldWrapExceptionInRestException() throws Exception {
+        exception.expect(RestException.class);
+
+        FileSystem fileSystem = mock(FileSystem.class);
+        when(FileSystem.get(configuration)).thenReturn(fileSystem);
+        when(fileSystem.delete(new Path(testDir), false)).thenThrow(new IOException());
+
+        hdfsService.delete(new Path(testDir), false);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
index d67892e..9c18dc6 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
@@ -18,100 +18,87 @@
 package org.apache.metron.rest.service.impl;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
-import org.apache.metron.rest.config.HadoopConfig;
 import org.apache.metron.rest.service.HdfsService;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Profile;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.junit.rules.ExpectedException;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.List;
 
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
 
-@RunWith(SpringJUnit4ClassRunner.class)
-@ContextConfiguration(classes={HadoopConfig.class, HdfsServiceImplTest.HdfsServiceTestContextConfiguration.class})
-@ActiveProfiles(TEST_PROFILE)
 public class HdfsServiceImplTest {
+    @Rule
+    public final ExpectedException exception = ExpectedException.none();
 
-    @Configuration
-    @Profile("test")
-    static class HdfsServiceTestContextConfiguration {
-
-        @Bean
-        public HdfsService hdfsService() {
-            return new HdfsServiceImpl();
+    private Configuration configuration;
+    private HdfsService hdfsService;
+    private String testDir = "./target/hdfsUnitTest";
+
+    @Before
+    public void setup() throws IOException {
+        configuration = new Configuration();
+        hdfsService = new HdfsServiceImpl(configuration);
+        File file = new File(testDir);
+        if (!file.exists()) {
+          file.mkdirs();
         }
+        FileUtils.cleanDirectory(file);
     }
 
-    @Autowired
-    private HdfsService hdfsService;
+    @After
+    public void teardown() throws IOException {
+        File file = new File(testDir);
+        FileUtils.cleanDirectory(file);
+    }
 
     @Test
-    public void test() throws IOException {
-        String rootDir = "./src/test/tmp";
-        File rootFile = new File(rootDir);
-        Path rootPath = new Path(rootDir);
-        if (rootFile.exists()) {
-            FileUtils.cleanDirectory(rootFile);
-            FileUtils.deleteDirectory(rootFile);
-        }
-        assertEquals(true, rootFile.mkdir());
-        String fileName1 = "fileName1";
-        String fileName2 = "fileName2";
-        Path path1 = new Path(rootDir, fileName1);
-        String value1 = "value1";
-        String value2 = "value2";
-        Path path2 = new Path(rootDir, fileName2);
-        String invalidFile = "invalidFile";
-        Path pathInvalidFile = new Path(rootDir, invalidFile);
-
-        FileStatus[] fileStatuses = hdfsService.list(new Path(rootDir));
-        assertEquals(0, fileStatuses.length);
-
-
-        hdfsService.write(path1, value1.getBytes());
-        assertEquals(value1, FileUtils.readFileToString(new File(rootDir, fileName1)));
-        assertEquals(value1, new String(hdfsService.read(path1)));
-
-        fileStatuses = hdfsService.list(rootPath);
-        assertEquals(1, fileStatuses.length);
-        assertEquals(fileName1, fileStatuses[0].getPath().getName());
-
-        hdfsService.write(path2, value2.getBytes());
-        assertEquals(value2, FileUtils.readFileToString(new File(rootDir, fileName2)));
-        assertEquals(value2, new String(hdfsService.read(path2)));
-
-        fileStatuses = hdfsService.list(rootPath);
-        assertEquals(2, fileStatuses.length);
-        assertEquals(fileName1, fileStatuses[0].getPath().getName());
-        assertEquals(fileName1, fileStatuses[0].getPath().getName());
-
-        assertEquals(true, hdfsService.delete(path1, false));
-        fileStatuses = hdfsService.list(rootPath);
-        assertEquals(1, fileStatuses.length);
-        assertEquals(fileName2, fileStatuses[0].getPath().getName());
-        assertEquals(true, hdfsService.delete(path2, false));
-        fileStatuses = hdfsService.list(rootPath);
-        assertEquals(0, fileStatuses.length);
-
-        try {
-            hdfsService.read(pathInvalidFile);
-            fail("Exception should be thrown when reading invalid file name");
-        } catch(IOException e) {
-        }
-        assertEquals(false, hdfsService.delete(pathInvalidFile, false));
+    public void listShouldListFiles() throws Exception {
+        FileUtils.writeStringToFile(new File(testDir, "file1.txt"), "value1");
+        FileUtils.writeStringToFile(new File(testDir, "file2.txt"), "value2");
+
+        List<String> paths = hdfsService.list(new Path(testDir));
+        assertEquals(2, paths.size());
+        assertEquals("file1.txt", paths.get(0));
+        assertEquals("file2.txt", paths.get(1));
+    }
+
+
+    @Test
+    public void readShouldProperlyReadContents() throws Exception {
+        String contents = "contents";
+        FileUtils.writeStringToFile(new File(testDir, "readTest.txt"), contents);
+
+        assertEquals("contents", hdfsService.read(new Path(testDir, "readTest.txt")));
+    }
+
+    @Test
+    public void writeShouldProperlyWriteContents() throws Exception {
+        String contents = "contents";
+        hdfsService.write(new Path(testDir, "writeTest.txt"), contents.getBytes(UTF_8));
+
+        assertEquals("contents", FileUtils.readFileToString(new File(testDir, "writeTest.txt")));
+    }
+
+    @Test
+    public void deleteShouldProperlyDeleteFile() throws Exception {
+        String contents = "contents";
+        FileUtils.writeStringToFile(new File(testDir, "deleteTest.txt"), contents);
+
+        List<String> paths = hdfsService.list(new Path(testDir));
+        assertEquals(1, paths.size());
+        assertEquals("deleteTest.txt", paths.get(0));
+
+        hdfsService.delete(new Path(testDir, "deleteTest.txt"), false);
 
-        FileUtils.deleteDirectory(new File(rootDir));
+        paths = hdfsService.list(new Path(testDir));
+        assertEquals(0, paths.size());
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
index b211ee6..c7d42b3 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
@@ -213,7 +213,9 @@ public class KafkaServiceImplTest {
     when(kafkaConsumer.listTopics()).thenReturn(topics);
     when(kafkaConsumer.partitionsFor("t")).thenReturn(Lists.newArrayList(partitionInfo));
 
-    assertEquals(expected, kafkaService.getTopic("t"));
+    KafkaTopic actual = kafkaService.getTopic("t");
+    assertEquals(expected, actual);
+    assertEquals(expected.hashCode(), actual.hashCode());
   }
 
   @Test


Mime
View raw message