metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject [48/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
Date Sun, 18 Nov 2018 18:16:01 GMT
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
index c33644f..83800af 100644
--- a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
@@ -58,8 +58,11 @@ import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INP
 import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_END;
 import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
 import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_READER;
 import static org.junit.Assert.assertTrue;
 
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.*;
+
 /**
  * An integration test for the {@link BatchProfiler}.
  */
@@ -159,8 +162,8 @@ public class BatchProfilerIntegrationTest {
   @Test
   public void testBatchProfilerWithJSON() throws Exception {
     // the input telemetry is text/json stored in the local filesystem
+    profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), JSON.toString());
     profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), "src/test/resources/telemetry.json");
-    profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "text");
 
     BatchProfiler profiler = new BatchProfiler();
     profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile());
@@ -170,20 +173,41 @@ public class BatchProfilerIntegrationTest {
 
   @Test
   public void testBatchProfilerWithORC() throws Exception {
-    // re-write the test data as ORC
+    // re-write the test data as column-oriented ORC
     String pathToORC = tempFolder.getRoot().getAbsolutePath();
     spark.read()
-            .format("text")
+            .format("json")
             .load("src/test/resources/telemetry.json")
-            .as(Encoders.STRING())
             .write()
             .mode("overwrite")
             .format("org.apache.spark.sql.execution.datasources.orc")
             .save(pathToORC);
 
     // tell the profiler to use the ORC input data
+    profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), ORC.toString());
     profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToORC);
-    profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "org.apache.spark.sql.execution.datasources.orc");
+
+    BatchProfiler profiler = new BatchProfiler();
+    profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile());
+
+    validateProfiles();
+  }
+
+  @Test
+  public void testBatchProfilerWithParquet() throws Exception {
+    // re-write the test data as column-oriented ORC
+    String inputPath = tempFolder.getRoot().getAbsolutePath();
+    spark.read()
+            .format("json")
+            .load("src/test/resources/telemetry.json")
+            .write()
+            .mode("overwrite")
+            .format("parquet")
+            .save(inputPath);
+
+    // tell the profiler to use the ORC input data
+    profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), PARQUET.toString());
+    profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), inputPath);
 
     BatchProfiler profiler = new BatchProfiler();
     profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile());
@@ -206,7 +230,9 @@ public class BatchProfilerIntegrationTest {
             .save(pathToCSV);
 
     // tell the profiler to use the CSV input data
+    // CSV is an example of needing to define both the reader and the input format
     profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToCSV);
+    profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), "text");
     profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "csv");
 
     // set a reader property; tell the reader to expect a header

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java
new file mode 100644
index 0000000..0bcfb3f
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function.reader;
+
+import org.apache.metron.profiler.spark.reader.TelemetryReaders;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.SparkSession;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.util.Properties;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+
+/**
+ * Tests the {@link org.apache.metron.profiler.spark.reader.ColumnEncodedTelemetryReader} class.
+ */
+public class ColumnEncodedTelemetryReaderTest {
+
+  @Rule
+  public TemporaryFolder tempFolder = new TemporaryFolder();
+  private static SparkSession spark;
+  private Properties profilerProperties;
+  private Properties readerProperties;
+
+  @BeforeClass
+  public static void setupSpark() {
+    SparkConf conf = new SparkConf()
+            .setMaster("local")
+            .setAppName("BatchProfilerIntegrationTest")
+            .set("spark.sql.shuffle.partitions", "8");
+    spark = SparkSession
+            .builder()
+            .config(conf)
+            .getOrCreate();
+  }
+
+  @AfterClass
+  public static void tearDownSpark() {
+    if(spark != null) {
+      spark.close();
+    }
+  }
+
+  @Before
+  public void setup() {
+    readerProperties = new Properties();
+    profilerProperties = new Properties();
+  }
+
+  @Test
+  public void testParquet() {
+    // re-write the test data as column-oriented ORC
+    String inputPath = tempFolder.getRoot().getAbsolutePath();
+    spark.read()
+            .format("json")
+            .load("src/test/resources/telemetry.json")
+            .write()
+            .mode("overwrite")
+            .format("parquet")
+            .save(inputPath);
+
+    // tell the profiler to use the CSV input data
+    profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), inputPath);
+    profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "parquet");
+
+    // set a reader property; tell the reader to expect a header
+    readerProperties.put("header", "true");
+
+    // there should be 100 valid JSON records
+    Dataset<String> telemetry = TelemetryReaders.COLUMNAR.read(spark, profilerProperties, readerProperties);
+    Assert.assertEquals(100, telemetry.filter(new IsValidJSON()).count());
+  }
+
+  @Test
+  public void testORC() {
+    // re-write the test data as column-oriented ORC
+    String pathToORC = tempFolder.getRoot().getAbsolutePath();
+    spark.read()
+            .format("json")
+            .load("src/test/resources/telemetry.json")
+            .write()
+            .mode("overwrite")
+            .format("org.apache.spark.sql.execution.datasources.orc")
+            .save(pathToORC);
+
+    // tell the profiler to use the CSV input data
+    profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToORC);
+    profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "org.apache.spark.sql.execution.datasources.orc");
+
+    // there should be 100 valid JSON records
+    Dataset<String> telemetry = TelemetryReaders.COLUMNAR.read(spark, profilerProperties, readerProperties);
+    Assert.assertEquals(100, telemetry.filter(new IsValidJSON()).count());
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/IsValidJSON.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/IsValidJSON.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/IsValidJSON.java
new file mode 100644
index 0000000..50144f9
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/IsValidJSON.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function.reader;
+
+import org.apache.spark.api.java.function.FilterFunction;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+
+/**
+ * A filter function that filters out invalid JSON records.
+ */
+public class IsValidJSON implements FilterFunction<String> {
+
+  @Override
+  public boolean call(String text) throws Exception {
+    JSONParser parser = new JSONParser();
+    JSONObject json = (JSONObject) parser.parse(text);
+
+    // all of the test data has at least 32 fields in each JSON record
+    return json.keySet().size() >= 32;
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TelemetryReadersTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TelemetryReadersTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TelemetryReadersTest.java
new file mode 100644
index 0000000..e525ae0
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TelemetryReadersTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function.reader;
+
+import org.apache.metron.profiler.spark.reader.ColumnEncodedTelemetryReader;
+import org.apache.metron.profiler.spark.reader.TelemetryReaders;
+import org.apache.metron.profiler.spark.reader.TextEncodedTelemetryReader;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.COLUMNAR;
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.JSON;
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.ORC;
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.PARQUET;
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.TEXT;
+
+public class TelemetryReadersTest {
+
+  @Test
+  public void testJsonReader() {
+    String key = JSON.toString();
+    Assert.assertTrue(TelemetryReaders.create(key) instanceof TextEncodedTelemetryReader);
+  }
+
+  @Test
+  public void testJsonReaderLowerCase() {
+    String key = JSON.toString().toLowerCase();
+    Assert.assertTrue(TelemetryReaders.create(key) instanceof TextEncodedTelemetryReader);
+  }
+
+  @Test
+  public void testOrcReader() {
+    String key = ORC.toString();
+    Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader);
+  }
+
+
+  @Test
+  public void testOrcReaderLowerCase() {
+    String key = ORC.toString().toLowerCase();
+    Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader);
+  }
+
+  @Test
+  public void testParquetReader() {
+    String key = PARQUET.toString();
+    Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader);
+  }
+
+  @Test
+  public void testParquetReaderLowerCase() {
+    String key = PARQUET.toString().toLowerCase();
+    Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader);
+  }
+
+  @Test
+  public void testTextReader() {
+    String key = TEXT.toString();
+    Assert.assertTrue(TelemetryReaders.create(key) instanceof TextEncodedTelemetryReader);
+  }
+
+  @Test
+  public void testColumnReader() {
+    String key = COLUMNAR.toString();
+    Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testInvalidReader() {
+    TelemetryReaders.create("invalid");
+    Assert.fail("exception expected");
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TextEncodedTelemetryReaderTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TextEncodedTelemetryReaderTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TextEncodedTelemetryReaderTest.java
new file mode 100644
index 0000000..3b26bb9
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TextEncodedTelemetryReaderTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function.reader;
+
+import org.apache.metron.profiler.spark.reader.TelemetryReaders;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SparkSession;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.util.Properties;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+
+/**
+ * Tests the {@link org.apache.metron.profiler.spark.reader.TextEncodedTelemetryReader} class.
+ */
+public class TextEncodedTelemetryReaderTest {
+
+  @Rule
+  public TemporaryFolder tempFolder = new TemporaryFolder();
+  private static SparkSession spark;
+  private Properties profilerProperties;
+  private Properties readerProperties;
+
+  @BeforeClass
+  public static void setupSpark() {
+    SparkConf conf = new SparkConf()
+            .setMaster("local")
+            .setAppName("BatchProfilerIntegrationTest")
+            .set("spark.sql.shuffle.partitions", "8");
+    spark = SparkSession
+            .builder()
+            .config(conf)
+            .getOrCreate();
+  }
+
+  @AfterClass
+  public static void tearDownSpark() {
+    if(spark != null) {
+      spark.close();
+    }
+  }
+
+  @Before
+  public void setup() {
+    readerProperties = new Properties();
+    profilerProperties = new Properties();
+  }
+
+  @Test
+  public void testCSV() {
+    // re-write the test data as a CSV with a header record
+    String pathToCSV = tempFolder.getRoot().getAbsolutePath();
+    spark.read()
+            .format("text")
+            .load("src/test/resources/telemetry.json")
+            .as(Encoders.STRING())
+            .write()
+            .mode("overwrite")
+            .option("header", "true")
+            .format("csv")
+            .save(pathToCSV);
+
+    // tell the profiler to use the CSV input data
+    profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToCSV);
+    profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "csv");
+
+    // set a reader property; tell the reader to expect a header
+    readerProperties.put("header", "true");
+
+    // there should be 100 valid JSON records
+    Dataset<String> telemetry = TelemetryReaders.TEXT.read(spark, profilerProperties, readerProperties);
+    Assert.assertEquals(100, telemetry.filter(new IsValidJSON()).count());
+  }
+
+  @Test
+  public void testJSON() {
+    // use the test telemetry that is stored as raw json
+    profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), "src/test/resources/telemetry.json");
+    profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "text");
+
+    // set a reader property; tell the reader to expect a header
+    readerProperties.put("header", "true");
+
+    // there should be 100 valid JSON records
+    Dataset<String> telemetry = TelemetryReaders.TEXT.read(spark, profilerProperties, readerProperties);
+    Assert.assertEquals(100, telemetry.filter(new IsValidJSON()).count());
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/Kerberos-manual-setup.md
----------------------------------------------------------------------
diff --git a/metron-deployment/Kerberos-manual-setup.md b/metron-deployment/Kerberos-manual-setup.md
index 4bd4516..6d40552 100644
--- a/metron-deployment/Kerberos-manual-setup.md
+++ b/metron-deployment/Kerberos-manual-setup.md
@@ -563,17 +563,16 @@ X-Pack
     sudo -u hdfs hdfs dfs -chown metron:metron /apps/metron/elasticsearch/xpack-password
     ```
 
-1. New settings have been added to configure the Elasticsearch client. By default the client will run as the normal ES prebuilt transport client. If you enable X-Pack you should set the es.client.class as shown below.
+1. New settings have been added to configure the Elasticsearch client.
 
-    Add the `es.client.settings` to global.json
+    Modify the `es.client.settings` key in global.json
 
     ```
     $METRON_HOME/config/zookeeper/global.json ->
 
       "es.client.settings" : {
-          "es.client.class" : "org.elasticsearch.xpack.client.PreBuiltXPackTransportClient",
-          "es.xpack.username" : "transport_client_user",
-          "es.xpack.password.file" : "/apps/metron/elasticsearch/xpack-password"
+          "xpack.username" : "transport_client_user",
+          "xpack.password.file" : "/apps/metron/elasticsearch/xpack-password"
       }
     ```
 
@@ -583,151 +582,6 @@ X-Pack
     $METRON_HOME/bin/zk_load_configs.sh -m PUSH -i $METRON_HOME/config/zookeeper/ -z $ZOOKEEPER
     ```
 
-1. The last step before restarting the topology is to create a custom X-Pack shaded and relocated jar. This is up to you because of licensing restrictions, but here is a sample Maven pom file that should help.
-
-    ```
-    <?xml version="1.0" encoding="UTF-8"?>
-    <!--
-      Licensed to the Apache Software
-    	Foundation (ASF) under one or more contributor license agreements. See the
-    	NOTICE file distributed with this work for additional information regarding
-    	copyright ownership. The ASF licenses this file to You under the Apache License,
-    	Version 2.0 (the "License"); you may not use this file except in compliance
-    	with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-    	Unless required by applicable law or agreed to in writing, software distributed
-    	under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
-    	OR CONDITIONS OF ANY KIND, either express or implied. See the License for
-      the specific language governing permissions and limitations under the License.
-      -->
-    <project xmlns="http://maven.apache.org/POM/4.0.0"
-             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-             xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-        <modelVersion>4.0.0</modelVersion>
-        <groupId>org.elasticsearch</groupId>
-        <artifactId>elasticsearch-xpack-shaded</artifactId>
-        <name>elasticsearch-xpack-shaded</name>
-        <packaging>jar</packaging>
-        <version>5.6.2</version>
-        <repositories>
-            <repository>
-                <id>elasticsearch-releases</id>
-                <url>https://artifacts.elastic.co/maven</url>
-                <releases>
-                    <enabled>true</enabled>
-                </releases>
-                <snapshots>
-                    <enabled>false</enabled>
-                </snapshots>
-            </repository>
-        </repositories>
-        <dependencies>
-            <dependency>
-                <groupId>org.elasticsearch.client</groupId>
-                <artifactId>x-pack-transport</artifactId>
-                <version>5.6.2</version>
-                <exclusions>
-                  <exclusion>
-                    <groupId>com.fasterxml.jackson.dataformat</groupId>
-                    <artifactId>jackson-dataformat-yaml</artifactId>
-                  </exclusion>
-                  <exclusion>
-                    <groupId>com.fasterxml.jackson.dataformat</groupId>
-                    <artifactId>jackson-dataformat-cbor</artifactId>
-                  </exclusion>
-                  <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-core</artifactId>
-                  </exclusion>
-                  <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-api</artifactId>
-                  </exclusion>
-                  <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-log4j12</artifactId>
-                  </exclusion>
-                  <exclusion>
-                    <groupId>log4j</groupId>
-                    <artifactId>log4j</artifactId>
-                  </exclusion>
-                </exclusions>
-              </dependency>
-        </dependencies>
-        <build>
-            <plugins>
-                <plugin>
-                    <groupId>org.apache.maven.plugins</groupId>
-                    <artifactId>maven-shade-plugin</artifactId>
-                    <version>3.2.0</version>
-                    <configuration>
-                        <createDependencyReducedPom>true</createDependencyReducedPom>
-                    </configuration>
-                    <executions>
-                        <execution>
-                            <phase>package</phase>
-                            <goals>
-                                <goal>shade</goal>
-                            </goals>
-                            <configuration>
-                              <filters>
-                                <filter>
-                                  <artifact>*:*</artifact>
-                                  <excludes>
-                                    <exclude>META-INF/*.SF</exclude>
-                                    <exclude>META-INF/*.DSA</exclude>
-                                    <exclude>META-INF/*.RSA</exclude>
-                                  </excludes>
-                                </filter>
-                              </filters>
-                              <relocations>
-    				<relocation>
-                                        <pattern>io.netty</pattern>
-                                        <shadedPattern>org.apache.metron.io.netty</shadedPattern>
-                                    </relocation>
-                                    <relocation>
-                                        <pattern>org.apache.logging.log4j</pattern>
-                                        <shadedPattern>org.apache.metron.logging.log4j</shadedPattern>
-                                    </relocation>
-                                </relocations>
-                                <artifactSet>
-                                    <excludes>
-                                        <exclude>org.slf4j.impl*</exclude>
-                                        <exclude>org.slf4j:slf4j-log4j*</exclude>
-                                    </excludes>
-                                </artifactSet>
-                                <transformers>
-                                    <transformer
-                                      implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
-                                         <resources>
-                                            <resource>.yaml</resource>
-                                            <resource>LICENSE.txt</resource>
-                                            <resource>ASL2.0</resource>
-                                            <resource>NOTICE.txt</resource>
-                                          </resources>
-                                    </transformer>
-                                    <transformer
-                                            implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
-                                    <transformer
-                                            implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                        <mainClass></mainClass>
-                                    </transformer>
-                                </transformers>
-                            </configuration>
-                        </execution>
-                    </executions>
-                </plugin>
-            </plugins>
-        </build>
-    </project>
-    ```
-
-1. Once you've built the `elasticsearch-xpack-shaded-5.6.2.jar`, it needs to be made available to Storm when you submit the topology. Create a contrib directory for indexing and put the jar file in this directory.
-
-    ```
-    mkdir $METRON_HOME/indexing_contrib
-    cp elasticsearch-xpack-shaded-5.6.2.jar $METRON_HOME/indexing_contrib/elasticsearch-xpack-shaded-5.6.2.jar
-    ```
-
 1. Now you can restart the Elasticsearch topology. Note, you should perform this step manually, as follows.
 
     ```

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
index 8bae8ce..f6d76e5 100644
--- a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
+++ b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
@@ -19,5 +19,6 @@ ambari_server_mem: 2048
 ambari_mpack_version: 0.6.1.0
 metron_mpack_name: metron_mpack-{{ ambari_mpack_version }}.tar.gz
 metron_mpack_path: "{{ playbook_dir }}/../../packaging/ambari/metron-mpack/target/{{ metron_mpack_name }}"
+elasticsearch_mpack_version: 5.6.2
 elasticsearch_mpack_name: elasticsearch_mpack-{{ ambari_mpack_version }}.tar.gz
 elasticsearch_mpack_path: "{{ playbook_dir }}/../../packaging/ambari/elasticsearch-mpack/target/{{ elasticsearch_mpack_name }}"

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml b/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
index 7ce1a13..ea5b0f4 100644
--- a/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
+++ b/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
@@ -21,6 +21,6 @@
     dest: /tmp
 
 - name: Install Elasticsearch MPack on Ambari Host
-  shell: ambari-server install-mpack --mpack=/tmp/elasticsearch_mpack-{{ ambari_mpack_version }}.tar.gz
+  shell: "ambari-server install-mpack --mpack=/tmp/{{ elasticsearch_mpack_name }}"
   args:
-    creates: /var/lib/ambari-server/resources/mpacks/elasticsearch-ambari.mpack-{{ ambari_mpack_version }}/addon-services
+    creates: /var/lib/ambari-server/resources/mpacks/elasticsearch-ambari.mpack-{{ elasticsearch_mpack_version }}/addon-services

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/ansible/roles/bro/tasks/bro.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/tasks/bro.yml b/metron-deployment/ansible/roles/bro/tasks/bro.yml
index 222ef0e..9ba3ffa 100644
--- a/metron-deployment/ansible/roles/bro/tasks/bro.yml
+++ b/metron-deployment/ansible/roles/bro/tasks/bro.yml
@@ -30,8 +30,8 @@
 - name: Compile and Install bro
   shell: "{{ item }}"
   environment:
-    CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
-    CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
+    CXX: /opt/rh/devtoolset-6/root/usr/bin/g++
+    CC: /opt/rh/devtoolset-6/root/usr/bin/gcc
   args:
     chdir: "/tmp/bro-{{ bro_version }}"
     creates: "{{ bro_home }}/bin/bro"

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
index 3cd3bae..f8b1e28 100644
--- a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
+++ b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
@@ -38,8 +38,8 @@
     - perl
     - crontabs
     - net-tools
-    - devtoolset-4-gcc
-    - devtoolset-4-gcc-c++
+    - devtoolset-6-gcc
+    - devtoolset-6-gcc-c++
     - python27
     - rh-git29
   register: result

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
index 7043387..ec7af97 100644
--- a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
+++ b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
@@ -30,8 +30,8 @@
   environment:
     PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}"
     LD_LIBRARY_PATH: "{{ httpd24_lib }}:{{ python27_lib }}"
-    CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
-    CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
+    CXX: /opt/rh/devtoolset-6/root/usr/bin/g++
+    CC: /opt/rh/devtoolset-6/root/usr/bin/gcc
   command: "{{ python27_bin }}/bro-pkg install apache/metron-bro-plugin-kafka --force --version {{ metron_bro_plugin_kafka_version }}"
 
 - name: Configure bro-kafka plugin

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
index 063c22f..6e25cb3 100644
--- a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
+++ b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
@@ -15,6 +15,6 @@
 #  limitations under the License.
 #
 ---
-librdkafka_version: 0.9.4
-librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz
+librdkafka_version: 0.11.5
+librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz
 librdkafka_home: /usr/local

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/development/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/development/README.md b/metron-deployment/development/README.md
index 2a04e5f..b86a5c4 100644
--- a/metron-deployment/development/README.md
+++ b/metron-deployment/development/README.md
@@ -27,3 +27,37 @@ This directory contains environments useful for Metron developers.  These enviro
 ## Vagrant Cachier recommendations
 
 The development boxes are designed to be spun up and destroyed on a regular basis as part of the development cycle. In order to avoid the overhead of re-downloading many of the heavy platform dependencies, Vagrant can use the [vagrant-cachier](http://fgrehm.viewdocs.io/vagrant-cachier/) plugin to store package caches between builds. If the plugin has been installed to your vagrant it will be used, and packages will be cached in ~/.vagrant/cache.
+
+## Knox Demo LDAP
+
+The development environment can be set up to authenticate against Knox's demo LDAP.
+
+A couple notes
+* A custom LDIF file is used to setup users. This is to get the roles and passwords setup correctly.
+* The demo LDAP uses plaintext passwords with no encryption prefix (e.g. {SSHA}).
+* You may need or want to shut down any or all of the topologies. This is optional, but clears some room
+
+To setup this up, start full dev.
+* In Ambari, add the Knox service (Actions -> +Add Service).  Accept all defaults and let it install. The configs that will be set how we need by default are:
+  * LDAP URL = ldap://localhost:33389
+  * User dn pattern = uid={0},ou=people,dc=hadoop,dc=apache,dc=org
+  * LDAP user searchbase = ou=people,dc=hadoop,dc=apache,dc=org
+  * Group Search Base = ou=groups,dc=hadoop,dc=apache,dc=org
+  * Group Search Filter = member={0}
+  * User Base DN = uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
+  * User Search Filter is empty
+  * User password attribute = userPassword
+  * LDAP group role attribute = cn
+  * Bind User = uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
+  * LDAP Truststore is empty
+  * LDAP Truststore Password is empty
+  
+* In the Knox configuration, go to "Advanced users-ldif". We have a custom ldif file "knox-demo-ldap.ldif" in "metron-deployment/development" that contains a customized variant of the users and groups defined here. Replace the default ldif configuration with the contents of "knox-demo-ldap.ldif"
+* Start the Demo LDAP (In Knox, "Service Actions -> Start Demo LDAP)
+* In Metron's configs, we're going to make two changes
+  * Set "LDAP Enabled" to "On"
+  * In Security, set "Bind user password" to match the admin user's password from the ldif file (admin-password).
+* Restart the REST application
+
+Now, when you go to Swagger or the UIs, you should be able to give a user and password.
+"admin" will have the roles ROLE_ADMIN and ROLE_USER, which can be verified via the "/whoami/roles" endpoint in Swagger. Similarly, there is a user "sam" that only has ROLE_USER. A third user, "tom" has neither role.

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/development/centos6/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/README.md b/metron-deployment/development/centos6/README.md
index e873452..99ec967 100644
--- a/metron-deployment/development/centos6/README.md
+++ b/metron-deployment/development/centos6/README.md
@@ -97,8 +97,30 @@ In addition to re-running the entire provisioning play book, you may now re-run
 vagrant --ansible-tags="sensor-stubs" provision
 ```
 
-Tags are listed in the playbooks, some frequently used tags:
+Tags are listed in the playbooks.  Here are some frequently used tags:
 + `hdp-install` - Install HDP
 + `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services)
-+ `sensors` - Deploy and start the sensors.
++ `sensors` - Deploy the sensors (see [Sensors](#sensors) for more details regarding this tag)
 + `sensor-stubs` - Deploy and start the sensor stubs.
+
+#### Sensors
+
+By default, the Metron development environment uses sensor stubs to mimic the behavior of the full sensors.  This is done because the full sensors take a significant amount of time and CPU to build, install, and run.
+
+From time to time you may want to install the full sensors for testing (see the specifics of what that means [here](../../ansible/playbooks/sensor_install.yml)).  This can be done by running the following command:
+
+```
+vagrant --ansible-skip-tags="sensor-stubs" up
+```
+
+This will skip only the `sensor-stubs` tag, allowing the ansible roles with the `sensors` tag to be run.  This provisions the full sensors in a 'testing mode' so that they are more active, and thus more useful for testing (more details on that [here](../../ansible/roles/sensor-test-mode/)).  **However**, when vagrant completes the sensors will NOT be running.  In order to start the sensors and simulate traffic through them (which will create a fair amount of load on your test system), complete the below steps:
+
+```
+vagrant ssh
+sudo su -
+service pcap-replay restart
+service yaf restart
+service snortd restart
+service snort-producer restart
+```
+

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/development/knox-demo-ldap.ldif
----------------------------------------------------------------------
diff --git a/metron-deployment/development/knox-demo-ldap.ldif b/metron-deployment/development/knox-demo-ldap.ldif
new file mode 100644
index 0000000..3097a64
--- /dev/null
+++ b/metron-deployment/development/knox-demo-ldap.ldif
@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: 1
+
+# Please replace with site specific values
+dn: dc=hadoop,dc=apache,dc=org
+objectclass: organization
+objectclass: dcObject
+o: Hadoop
+dc: hadoop
+
+# Entry for a sample people container
+# Please replace with site specific values
+dn: ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:organizationalUnit
+ou: people
+
+# Entry for a sample end user
+# Please replace with site specific values
+dn: uid=guest,ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:person
+objectclass:organizationalPerson
+objectclass:inetOrgPerson
+cn: Guest
+sn: User
+uid: guest
+userPassword:guest-password
+
+
+# entry for sample user admin
+dn: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:person
+objectclass:organizationalPerson
+objectclass:inetOrgPerson
+cn: Admin
+sn: Admin
+uid: admin
+userPassword:admin-password
+
+# entry for sample user sam
+dn: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:person
+objectclass:organizationalPerson
+objectclass:inetOrgPerson
+cn: sam
+sn: sam
+uid: sam
+userPassword:sam-password
+
+# entry for sample user tom
+dn: uid=tom,ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:person
+objectclass:organizationalPerson
+objectclass:inetOrgPerson
+cn: tom
+sn: tom
+uid: tom
+userPassword:tom-password
+
+# create FIRST Level groups branch
+dn: ou=groups,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:organizationalUnit
+ou: groups
+description: generic groups branch
+
+# create the admin group under groups
+dn: cn=admin,ou=groups,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass: groupofnames
+cn: admin
+description:admin group
+member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
+
+# create the user group under groups
+dn: cn=user,ou=groups,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass: groupofnames
+cn: user
+description: user group
+member: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org
+member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/development/ubuntu14/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/README.md b/metron-deployment/development/ubuntu14/README.md
index 494937e..53bea67 100644
--- a/metron-deployment/development/ubuntu14/README.md
+++ b/metron-deployment/development/ubuntu14/README.md
@@ -98,8 +98,30 @@ In addition to re-running the entire provisioning play book, you may now re-run
 vagrant --ansible-tags="sensor-stubs" provision
 ```
 
-Tags are listed in the playbooks, some frequently used tags:
+Tags are listed in the playbooks.  Here are some frequently used tags:
 + `hdp-install` - Install HDP
 + `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services)
-+ `sensors` - Deploy and start the sensors.
++ `sensors` - Deploy the sensors (see [Sensors](#sensors) for more details regarding this tag)
 + `sensor-stubs` - Deploy and start the sensor stubs.
+
+#### Sensors
+
+By default, the Metron development environment uses sensor stubs to mimic the behavior of the full sensors.  This is done because the full sensors take a significant amount of time and CPU to build, install, and run.
+
+From time to time you may want to install the full sensors for testing (see the specifics of what that means [here](../../ansible/playbooks/sensor_install.yml)).  This can be done by running the following command:
+
+```
+vagrant --ansible-skip-tags="sensor-stubs" up
+```
+
+This will skip only the `sensor-stubs` tag, allowing the ansible roles with the `sensors` tag to be run.  This provisions the full sensors in a 'testing mode' so that they are more active, and thus more useful for testing (more details on that [here](../../ansible/roles/sensor-test-mode/)).  **However**, when vagrant completes the sensors will NOT be running.  In order to start the sensors and simulate traffic through them (which will create a fair amount of load on your test system), complete the below steps:
+
+```
+vagrant ssh
+sudo su -
+service pcap-replay restart
+service yaf restart
+service snortd restart
+service snort-producer restart
+```
+

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
index b41c455..69dce3f 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
@@ -165,17 +165,17 @@
   </property>
   <property>
     <name>enrichment_topology</name>
-    <description>Which Enrichment topology to execute</description>
-    <value>Split-Join</value>
+    <description>Which Enrichment topology to execute. Note: Split-Join is deprecated in favor of the Unified topology.</description>
+    <value>Unified</value>
     <display-name>Enrichment Topology</display-name>
     <value-attributes>
       <type>value-list</type>
       <entries>
         <entry>
-          <value>Split-Join</value>
+          <value>Unified</value>
         </entry>
         <entry>
-          <value>Unified</value>
+          <value>Split-Join</value>
         </entry>
       </entries>
       <selection-cardinality>1</selection-cardinality>

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
index 3d268b7..ebc9759 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
@@ -146,7 +146,7 @@
     <property>
         <name>ra_indexing_topology_max_spout_pending</name>
         <description>Indexing Topology Spout Max Pending Tuples</description>
-        <value/>
+        <value>300</value>
         <display-name>Indexing Max Pending for Random Access</display-name>
         <value-attributes>
             <empty-value-valid>true</empty-value-valid>
@@ -155,7 +155,7 @@
      <property>
         <name>batch_indexing_topology_max_spout_pending</name>
         <description>Indexing Topology Spout Max Pending Tuples</description>
-        <value/>
+        <value>300</value>
         <display-name>Indexing Max Pending for HDFS</display-name>
         <value-attributes>
             <empty-value-valid>true</empty-value-valid>

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml
index 07b8d11..c45d799 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml
@@ -165,7 +165,7 @@
     <name>profiler_topology_max_spout_pending</name>
     <description>Profiler Topology Spout Max Pending Tuples</description>
     <display-name>Spout Max Pending Tuples</display-name>
-    <value/>
+    <value>300</value>
     <value-attributes>
         <empty-value-valid>true</empty-value-valid>
     </value-attributes>

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index f4b2327..55b880f 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -35,32 +35,32 @@
     </property>
     <property>
         <name>metron_spring_profiles_active</name>
-        <description>Active Spring profiles</description>
+        <description>Active Spring profiles. 'ldap' is used to enable authentication via LDAP.</description>
         <display-name>Active Spring profiles</display-name>
         <value/>
         <value-attributes>
             <empty-value-valid>true</empty-value-valid>
         </value-attributes>
     </property>
-    <property require-input="true">
+    <property>
         <name>metron_jdbc_driver</name>
         <value></value>
         <description>Class name of the JDBC Driver used by Metron</description>
         <display-name>Metron JDBC Driver</display-name>
     </property>
-    <property require-input="true">
+    <property>
         <name>metron_jdbc_url</name>
         <value></value>
         <description>JDBC Connection URL used by Metron</description>
         <display-name>Metron JDBC URL</display-name>
     </property>
-    <property require-input="true">
+    <property>
         <name>metron_jdbc_username</name>
         <value></value>
         <description>Metron JDBC Username</description>
         <display-name>Metron JDBC username</display-name>
     </property>
-    <property require-input="true">
+    <property>
         <name>metron_jdbc_password</name>
         <value></value>
         <property-type>PASSWORD</property-type>

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml
new file mode 100644
index 0000000..ab1fe6c
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml
@@ -0,0 +1,186 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<configuration supports_final="true">
+  <property>
+    <name>metron.ldap.enabled</name>
+    <display-name>LDAP Enabled</display-name>
+    <value>false</value>
+    <description>Enable LDAP for Authentication</description>
+    <value-attributes>
+      <type>value-list</type>
+      <entries>
+        <entry>
+          <value>true</value>
+          <label>On</label>
+        </entry>
+        <entry>
+          <value>false</value>
+          <label>Off</label>
+        </entry>
+      </entries>
+      <selection-cardinality>1</selection-cardinality>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.url</name>
+    <display-name>LDAP URL</display-name>
+    <value>ldap://localhost:33389</value>
+    <description>LDAP Server URL</description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.user.dnpattern</name>
+    <value>uid={0},ou=people,dc=hadoop,dc=apache,dc=org</value>
+    <display-name>User dn pattern</display-name>
+    <description>LDAP user DN</description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.user.searchbase</name>
+    <display-name>User Search Base</display-name>
+    <value>ou=people,dc=hadoop,dc=apache,dc=org</value>
+    <description>LDAP user searchbase</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.group.searchbase</name>
+    <display-name>Group Search Base</display-name>
+    <value>ou=groups,dc=hadoop,dc=apache,dc=org</value>
+    <description>LDAP group searchbase</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.group.searchfilter</name>
+    <display-name>Group Search Filter</display-name>
+    <value>member={0}</value>
+    <description>LDAP group search filter</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.user.basedn</name>
+    <display-name>User Base DN</display-name>
+    <value>uid=admin,ou=people,dc=hadoop,dc=apache,dc=org</value>
+    <description>LDAP User Base DN</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.user.searchfilter</name>
+    <display-name>User Search Filter</display-name>
+    <value></value>
+    <description>Search filter used for Bind Authentication</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.user.password</name>
+    <value>userPassword</value>
+    <display-name>User password attribute</display-name>
+    <description>LDAP attribute for the user password</description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.group.roleattribute</name>
+    <display-name>LDAP group role attribute</display-name>
+    <value>cn</value>
+    <description>The LDAP group attribute to be used for determining roles</description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.bind.dn</name>
+    <display-name>Bind User</display-name>
+    <value>uid=admin,ou=people,dc=hadoop,dc=apache,dc=org</value>
+    <description>Full distinguished name (DN), of an LDAP user account that has privileges to search for users. </description>
+    <on-ambari-upgrade add="true"/>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+  <property>
+    <name>metron.ldap.bind.password</name>
+    <display-name>Bind User Password</display-name>
+    <value></value>
+    <property-type>PASSWORD</property-type>
+    <description>Password for the account that can search for users</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
+  <property>
+    <name>metron.ldap.ssl.truststore</name>
+    <display-name>LDAP Truststore</display-name>
+    <value></value>
+    <description>Path of truststore with SSL certs for LDAP</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.ssl.truststore.password</name>
+    <display-name>LDAP Truststore Password</display-name>
+    <value></value>
+    <property-type>PASSWORD</property-type>
+    <description>Password for the truststore with SSL certs for LDAP</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
+</configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index ad1f7a9..97b5749 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -322,6 +322,7 @@
             <scriptType>PYTHON</scriptType>
           </commandScript>
           <configuration-dependencies>
+            <config-type>metron-security-env</config-type>
             <config-type>metron-indexing-env</config-type>
             <config-type>metron-rest-env</config-type>
             <config-type>metron-pcap-env</config-type>
@@ -371,6 +372,7 @@
             <scriptType>PYTHON</scriptType>
           </commandScript>
             <configuration-dependencies>
+                <config-type>metron-security-env</config-type>
                 <config-type>metron-rest-env</config-type>
                 <config-type>metron-management-ui-env</config-type>
             </configuration-dependencies>
@@ -397,6 +399,7 @@
             <scriptType>PYTHON</scriptType>
           </commandScript>
           <configuration-dependencies>
+            <config-type>metron-security-env</config-type>
             <config-type>metron-rest-env</config-type>
             <config-type>metron-alerts-ui-env</config-type>
           </configuration-dependencies>

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
index 17ad4d2..48e76c7 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
@@ -133,6 +133,9 @@
         "source:type": {
           "type": "keyword"
         },
+        "alert_status": {
+          "type": "keyword"
+        },
         /*
          * Widely-used Bro fields (potentially renamed during Metron ingest)
          */

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
index 05d5e32..040c411 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
@@ -5,8 +5,8 @@
       "dynamic_templates": [
         {
           "alert_template": {
-          "path_match": "alert.*",
-          "match_mapping_type": "string",
+          "path_match": "metron_alert.*",
+          "match_mapping_type": "*",
           "mapping": {
             "type": "keyword"
           }

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
index f7c6e59..139dbc9 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
@@ -123,6 +123,9 @@
         "source:type": {
           "type": "keyword"
         },
+        "alert_status": {
+          "type": "keyword"
+        },
         "ip_dst_addr": {
           "type": "ip"
         },

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
index f4093ba..4a3e5f9 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
@@ -123,6 +123,9 @@
         "source:type": {
           "type": "keyword"
         },
+        "alert_status": {
+          "type": "keyword"
+        },
         "ip_dst_addr": {
           "type": "ip"
         },

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index dd00e9c..5635330 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -58,7 +58,17 @@ metron_management_ui_port = status_params.metron_management_ui_port
 metron_alerts_ui_host = status_params.metron_alerts_ui_host
 metron_alerts_ui_port = status_params.metron_alerts_ui_port
 metron_jvm_flags = config['configurations']['metron-rest-env']['metron_jvm_flags']
-metron_spring_profiles_active = config['configurations']['metron-rest-env']['metron_spring_profiles_active']
+
+# Construct the profiles as a temp variable first. Only the first time it's set will carry through
+metron_spring_profiles_temp = config['configurations']['metron-rest-env']['metron_spring_profiles_active']
+if config['configurations']['metron-security-env']['metron.ldap.enabled']:
+    if metron_spring_profiles_temp:
+        metron_spring_profiles_active = metron_spring_profiles_temp + ',ldap'
+    else:
+        metron_spring_profiles_active = 'ldap'
+else:
+    metron_spring_profiles_active = metron_spring_profiles_temp
+
 metron_jdbc_driver = config['configurations']['metron-rest-env']['metron_jdbc_driver']
 metron_jdbc_url = config['configurations']['metron-rest-env']['metron_jdbc_url']
 metron_jdbc_username = config['configurations']['metron-rest-env']['metron_jdbc_username']
@@ -266,6 +276,21 @@ if security_enabled:
     if 'solr-config-env' in config['configurations']:
         solr_principal_name = solr_principal_name.replace('_HOST', hostname_lowercase)
 
+# LDAP
+metron_ldap_url = config['configurations']['metron-security-env']['metron.ldap.url']
+metron_ldap_userdn = config['configurations']['metron-security-env']['metron.ldap.bind.dn']
+metron_ldap_password = config['configurations']['metron-security-env']['metron.ldap.bind.password']
+metron_ldap_user_pattern = config['configurations']['metron-security-env']['metron.ldap.user.dnpattern']
+metron_ldap_user_password = config['configurations']['metron-security-env']['metron.ldap.user.password']
+metron_ldap_user_dnbase = config['configurations']['metron-security-env']['metron.ldap.user.basedn']
+metron_ldap_user_searchbase = config['configurations']['metron-security-env']['metron.ldap.user.searchbase']
+metron_ldap_user_searchfilter = config['configurations']['metron-security-env']['metron.ldap.user.searchfilter']
+metron_ldap_group_searchbase = config['configurations']['metron-security-env']['metron.ldap.group.searchbase']
+metron_ldap_group_searchfilter = config['configurations']['metron-security-env']['metron.ldap.group.searchfilter']
+metron_ldap_group_role = config['configurations']['metron-security-env']['metron.ldap.group.roleattribute']
+metron_ldap_ssl_truststore = config['configurations']['metron-security-env']['metron.ldap.ssl.truststore']
+metron_ldap_ssl_truststore_password = config['configurations']['metron-security-env']['metron.ldap.ssl.truststore.password']
+
 # Management UI
 metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", [hostname])[0]
 

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
index c410b94..bab9129 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -145,8 +145,12 @@ class RestCommands:
           "export METRON_PID_FILE={pid_file};"
           "export HDP_VERSION={hdp_version};"
           "export METRON_RA_INDEXING_WRITER={ra_indexing_writer};"
+          "export METRON_LDAP_PASSWORD={metron_ldap_password!p};"
+          "export METRON_LDAP_SSL_TRUSTSTORE_PASSWORD={metron_ldap_ssl_truststore_password!p};"
           "{metron_home}/bin/metron-rest.sh;"
           "unset METRON_JDBC_PASSWORD;"
+          "unset METRON_LDAP_PASSWORD;"
+          "unset METRON_LDAP_SSL_TRUSTSTORE_PASSWORD;"
         ))
 
         Execute(cmd,

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
index a7d01e5..08d4281 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
@@ -21,6 +21,8 @@ METRON_PID_DIR="{{metron_pid_dir}}"
 METRON_REST_PORT={{metron_rest_port}}
 METRON_JVMFLAGS="{{metron_jvm_flags}}"
 METRON_SPRING_PROFILES_ACTIVE="{{metron_spring_profiles_active}}"
+
+#JDBC
 METRON_JDBC_DRIVER="{{metron_jdbc_driver}}"
 METRON_JDBC_URL="{{metron_jdbc_url}}"
 METRON_JDBC_USERNAME="{{metron_jdbc_username}}"
@@ -28,6 +30,20 @@ METRON_JDBC_PLATFORM="{{metron_jdbc_platform}}"
 METRON_JDBC_CLIENT_PATH="{{metron_jdbc_client_path}}"
 METRON_TEMP_GROK_PATH="{{metron_temp_grok_path}}"
 METRON_SPRING_OPTIONS="{{metron_spring_options}}"
+
+#LDAP
+METRON_LDAP_URL="{{metron_ldap_url}}"
+METRON_LDAP_USERDN="{{metron_ldap_userdn}}"
+METRON_LDAP_USER_PATTERN="{{metron_ldap_user_pattern}}"
+METRON_LDAP_USER_DNBASE="{{metron_ldap_user_dnbase}}"
+METRON_LDAP_USER_SEARCHBASE="{{metron_ldap_user_searchbase}}"
+METRON_LDAP_USER_PASSWORD="{{metron_ldap_user_password}}"
+METRON_LDAP_USER_SEARCHFILTER="{{metron_ldap_user_searchfilter}}"
+METRON_LDAP_GROUP_SEARCHBASE="{{metron_ldap_group_searchbase}}"
+METRON_LDAP_GROUP_SEARCHFILTER="{{metron_ldap_group_searchfilter}}"
+METRON_LDAP_GROUP_ROLE="{{metron_ldap_group_role}}"
+METRON_LDAP_SSL_TRUSTSTORE="{{metron_ldap_ssl_truststore}}"
+
 ZOOKEEPER="{{zookeeper_quorum}}"
 BROKERLIST="{{kafka_brokers}}"
 HADOOP_CONF_DIR="/etc/hadoop/conf/"

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 26c7f4e..7f84f1d 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -125,7 +125,7 @@
                   ]
                 },
                 {
-                  "name": "section-enrichment-splitjoin",
+                  "name": "section-enrichment-unified",
                   "row-index": "3",
                   "column-index": "0",
                   "row-span": "1",
@@ -134,8 +134,8 @@
                   "section-rows": "1",
                   "subsections": [
                     {
-                      "name": "subsection-enrichment-splitjoin",
-                      "display-name": "Split Join Topology",
+                      "name": "subsection-enrichment-unified",
+                      "display-name": "Unified Topology",
                       "row-index": "0",
                       "column-index": "0",
                       "row-span": "1",
@@ -144,7 +144,7 @@
                   ]
                 },
                 {
-                  "name": "section-enrichment-unified",
+                  "name": "section-enrichment-splitjoin",
                   "row-index": "4",
                   "column-index": "0",
                   "row-span": "1",
@@ -153,8 +153,8 @@
                   "section-rows": "1",
                   "subsections": [
                     {
-                      "name": "subsection-enrichment-unified",
-                      "display-name": "Unified Topology",
+                      "name": "subsection-enrichment-splitjoin",
+                      "display-name": "Split Join Topology",
                       "row-index": "0",
                       "column-index": "0",
                       "row-span": "1",
@@ -422,6 +422,35 @@
             }
           },
           {
+            "name": "security",
+            "display-name": "Security",
+            "layout": {
+              "tab-columns": "2",
+              "tab-rows": "1",
+              "sections": [
+                {
+                  "name": "section-security-ldap",
+                  "row-index": "0",
+                  "column-index": "0",
+                  "row-span": "1",
+                  "column-span": "1",
+                  "section-columns": "1",
+                  "section-rows": "1",
+                  "subsections": [
+                    {
+                      "name": "subsection-security-ldap",
+                      "display-name": "LDAP",
+                      "row-index": "0",
+                      "column-index": "0",
+                      "row-span": "1",
+                      "column-span": "1"
+                    }
+                  ]
+                }
+              ]
+            }
+          },
+          {
             "name": "metron-pcap",
             "display-name": "PCAP",
             "layout": {
@@ -852,6 +881,58 @@
           "subsection-name": "subsection-alerts-ui"
         },
         {
+          "config": "metron-security-env/metron.ldap.enabled",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
+          "config": "metron-security-env/metron.ldap.url",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
+          "config": "metron-security-env/metron.ldap.bind.dn",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
+          "config": "metron-security-env/metron.ldap.bind.password",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
+          "config": "metron-security-env/metron.ldap.user.dnpattern",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
+          "config": "metron-security-env/metron.ldap.user.password",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
+          "config": "metron-security-env/metron.ldap.user.searchbase",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
+          "config": "metron-security-env/metron.ldap.user.searchfilter",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
+          "config": "metron-security-env/metron.ldap.group.searchbase",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
+          "config": "metron-security-env/metron.ldap.group.searchfilter",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
+          "config": "metron-security-env/metron.ldap.group.roleattribute",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
+          "config": "metron-security-env/metron.ldap.ssl.truststore",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
+          "config": "metron-security-env/metron.ldap.ssl.truststore.password",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
           "config": "metron-pcap-env/pcap_topology_workers",
           "subsection-name": "subsection-pcap"
         },
@@ -1529,6 +1610,84 @@
         }
       },
       {
+        "config": "metron-security-env/metron.ldap.enabled",
+        "widget": {
+          "type": "toggle"
+        }
+      },
+      {
+        "config": "metron-security-env/metron.ldap.url",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-security-env/metron.ldap.bind.dn",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-security-env/metron.ldap.bind.password",
+        "widget": {
+          "type": "password"
+        }
+      },
+      {
+        "config": "metron-security-env/metron.ldap.ssl.truststore",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-security-env/metron.ldap.ssl.truststore.password",
+        "widget": {
+          "type": "password"
+        }
+      },
+      {
+        "config": "metron-security-env/metron.ldap.user.dnpattern",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-security-env/metron.ldap.user.password",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-security-env/metron.ldap.user.searchbase",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-security-env/metron.ldap.user.searchfilter",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-security-env/metron.ldap.group.searchbase",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-security-env/metron.ldap.group.searchfilter",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-security-env/metron.ldap.group.roleattribute",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
         "config": "metron-pcap-env/pcap_topology_workers",
         "widget": {
           "type": "text-field"

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index ed22a28..14c08a1 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -461,15 +461,16 @@ This package installs the Metron Management UI %{metron_home}
 %attr(0755,root,root) %{metron_home}/web/expressjs/node_modules/.bin/*
 %attr(0755,root,root) %{metron_home}/web/expressjs/server.js
 %attr(0644,root,root) %{metron_home}/web/expressjs/package.json
+%attr(0644,root,root) %{metron_home}/web/management-ui/styles.*.css
 %attr(0644,root,root) %{metron_home}/web/management-ui/favicon.ico
 %attr(0644,root,root) %{metron_home}/web/management-ui/index.html
 %attr(0644,root,root) %{metron_home}/web/management-ui/*.js
-%attr(0644,root,root) %{metron_home}/web/management-ui/*.js.gz
 %attr(0644,root,root) %{metron_home}/web/management-ui/*.ttf
 %attr(0644,root,root) %{metron_home}/web/management-ui/*.svg
 %attr(0644,root,root) %{metron_home}/web/management-ui/*.eot
 %attr(0644,root,root) %{metron_home}/web/management-ui/*.woff
 %attr(0644,root,root) %{metron_home}/web/management-ui/*.woff2
+%attr(0644,root,root) %{metron_home}/web/management-ui/3rdpartylicenses.txt
 %attr(0644,root,root) %{metron_home}/web/management-ui/assets/ace/*.js
 %attr(0644,root,root) %{metron_home}/web/management-ui/assets/ace/LICENSE
 %attr(0644,root,root) %{metron_home}/web/management-ui/assets/ace/snippets/*.js
@@ -589,6 +590,8 @@ chkconfig --del metron-management-ui
 chkconfig --del metron-alerts-ui
 
 %changelog
+* Wed Oct 31 2018 Apache Metron <dev@metron.apache.org> - 0.6.1
+- Update files in Management UI from Angular upgrade
 * Thu Aug 30 2018 Apache Metron <dev@metron.apache.org> - 0.6.1
 - Update compiled css file name for Alerts UI
 * Fri Aug 24 2018 Apache Metron <dev@metron.apache.org> - 0.6.1

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-alerts/cypress.json
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/cypress.json b/metron-interface/metron-alerts/cypress.json
new file mode 100644
index 0000000..7c0410d
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress.json
@@ -0,0 +1,7 @@
+{
+  "viewportWidth": 1435,
+  "viewportHeight": 850,
+  "video": false,
+  "supportFile": false,
+  "pluginsFile": false
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-alerts/cypress/fixtures/config.json
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/cypress/fixtures/config.json b/metron-interface/metron-alerts/cypress/fixtures/config.json
new file mode 100644
index 0000000..190e514
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress/fixtures/config.json
@@ -0,0 +1,23 @@
+{
+  "es.clustername":"metron",
+  "es.ip":"node1:9300",
+  "es.date.format":"yyyy.MM.dd.HH",
+  "parser.error.topic":"indexing",
+  "update.hbase.table":"metron_update",
+  "update.hbase.cf":"t",
+  "es.client.settings":{
+     "client.transport.ping_timeout":"500s"
+  },
+  "profiler.client.period.duration":"15",
+  "profiler.client.period.duration.units":"MINUTES",
+  "user.settings.hbase.table":"user_settings",
+  "user.settings.hbase.cf":"cf",
+  "bootstrap.servers":"node1:6667",
+  "source.type.field":"source:type",
+  "threat.triage.score.field":"threat:triage:score",
+  "enrichment.writer.batchSize":"15",
+  "enrichment.writer.batchTimeout":"0",
+  "profiler.writer.batchSize":"15",
+  "profiler.writer.batchTimeout":"0",
+  "geo.hdfs.file":"/apps/metron/geo/default/GeoLite2-City.mmdb.gz"
+}
\ No newline at end of file


Mime
View raw message