ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From oleew...@apache.org
Subject [46/61] [abbrv] [partial] ambari git commit: AMBARI-15679. Initial commit for LogSearch service definition (oleewre)
Date Sat, 09 Apr 2016 10:15:42 GMT
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/timeline/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/timeline/configuration/Configuration.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/timeline/configuration/Configuration.java
new file mode 100644
index 0000000..a0380e1
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/timeline/configuration/Configuration.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics2.sink.timeline.configuration;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class Configuration {
+  public final Log LOG = LogFactory.getLog(this.getClass());
+  private final Properties properties;
+
+  public Configuration(String configFile) {
+    properties = new Properties();
+
+    //Get property file stream from classpath
+    InputStream inputStream = Configuration.class.getResourceAsStream(configFile);
+
+    if (inputStream == null) {
+      throw new IllegalArgumentException(configFile + " not found in classpath");
+    }
+
+    // load the properties
+    try {
+      properties.load(inputStream);
+      inputStream.close();
+    } catch (FileNotFoundException fnf) {
+      LOG.info("No configuration file " + configFile + " found in classpath.", fnf);
+    } catch (IOException ie) {
+      throw new IllegalArgumentException("Can't read configuration file " +
+          configFile, ie);
+    }
+  }
+
+  public String getProperty(String key) {
+    return properties.getProperty(key);
+  }
+
+  public String getProperty(String key, String defaultValue) {
+    return properties.getProperty(key, defaultValue);
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/util/Servers.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/util/Servers.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/util/Servers.java
new file mode 100644
index 0000000..b3dc46f
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/util/Servers.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metrics2.sink.util;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Helpers to handle server addresses
+ */
+public class Servers {
+  /**
+   * This class is not intended to be instantiated
+   */
+  private Servers() {}
+
+  /**
+   * Parses a space and/or comma separated sequence of server specifications
+   * of the form <i>hostname</i> or <i>hostname:port</i>.  If
+   * the specs string is null, defaults to localhost:defaultPort.
+   *
+   * @param specs   server specs (see description)
+   * @param defaultPort the default port if not specified
+   * @return a list of InetSocketAddress objects.
+   */
+  public static List<InetSocketAddress> parse(String specs, int defaultPort) {
+    List<InetSocketAddress> result = new ArrayList<InetSocketAddress>();
+    if (specs == null) {
+      result.add(new InetSocketAddress("localhost", defaultPort));
+    } else {
+      String[] specStrings = specs.split("[ ,]+");
+      for (String specString : specStrings) {
+        result.add(createSocketAddr(specString, defaultPort));
+      }
+    }
+    return result;
+  }
+
+  /**
+   * @param host
+   * @param port
+   * @return a InetSocketAddress created with the specified host and port
+   */
+  private static InetSocketAddress createSocketAddr(String target, int defaultPort) {
+    String helpText = "";
+    if (target == null) {
+      throw new IllegalArgumentException("Target address cannot be null." + helpText);
+    }
+    boolean hasScheme = target.contains("://");
+    URI uri = null;
+    try {
+      uri = hasScheme ? URI.create(target) : URI.create("dummyscheme://" + target);
+    } catch (IllegalArgumentException e) {
+      throw new IllegalArgumentException("Does not contain a valid host:port authority: " + target + helpText);
+    }
+
+    String host = uri.getHost();
+    int port = uri.getPort();
+    if (port == -1) {
+      port = defaultPort;
+    }
+    String path = uri.getPath();
+
+    if ((host == null) || (port < 0) || (!hasScheme && path != null && !path.isEmpty())) {
+      throw new IllegalArgumentException("Does not contain a valid host:port authority: " + target + helpText);
+    }
+    return createSocketAddrForHost(host, port);
+  }
+
+  /**
+   * @param host
+   * @param port
+   * @return a InetSocketAddress created with the specified host and port
+   */
+  private static InetSocketAddress createSocketAddrForHost(String host, int port) {
+    InetSocketAddress addr;
+    try {
+      InetAddress iaddr = InetAddress.getByName(host);
+      iaddr = InetAddress.getByAddress(host, iaddr.getAddress());
+      addr = new InetSocketAddress(iaddr, port);
+    } catch (UnknownHostException e) {
+      addr = InetSocketAddress.createUnresolved(host, port);
+    }
+    return addr;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control
new file mode 100644
index 0000000..40cd855
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+Package: [[artifactId]]
+Version: [[package-version]]-[[package-release]]
+Section: [[deb.section]]
+Priority: [[deb.priority]]
+Depends: [[deb.dependency.list]]
+Architecture: [[deb.architecture]]
+Description: [[description]]
+Maintainer: [[deb.publisher]]

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst
new file mode 100644
index 0000000..21a01fa
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm
new file mode 100644
index 0000000..21a01fa
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst
new file mode 100644
index 0000000..21a01fa
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm
new file mode 100644
index 0000000..21a01fa
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json
new file mode 100644
index 0000000..d8a239a
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json
@@ -0,0 +1,42 @@
+{
+	"input": {
+		"file": {
+			"klass": "org.apache.ambari.logfeeder.input.InputFile"
+		}
+
+	},
+	"filter": {
+		"json": {
+			"klass": "org.apache.ambari.logfeeder.filter.JSONFilterCode"
+		},
+		"keyvalue": {
+			"klass": "org.apache.ambari.logfeeder.filter.FilterKeyValue"
+		},
+		"grok": {
+			"klass": "org.apache.ambari.logfeeder.filter.FilterGrok"
+		}
+	},
+	  
+	 "mapper": {
+		"map_date": {
+			"klass": "org.apache.ambari.logfeeder.mapper.MapperDate"
+		},
+		"map_fieldname": {
+			"klass": "org.apache.ambari.logfeeder.mapper.MapperFieldName"
+		},
+		"map_fieldvalue": {
+			"klass": "org.apache.ambari.logfeeder.mapper.MapperFieldValue"
+		}
+	},
+	  "output": {
+		"solr": {
+			"klass": "org.apache.ambari.logfeeder.output.OutputSolr"
+		},
+		"file": {
+			"klass": "org.apache.ambari.logfeeder.output.OutputFile"
+		},
+		"kafka": {
+			"klass": "org.apache.ambari.logfeeder.output.OutputKafka"
+		}
+	}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2
new file mode 100644
index 0000000..163ee2b
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2
@@ -0,0 +1,995 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements.  See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership.  The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License.  You may obtain a copy of the License at
+ #
+ #   http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+	"global":{
+		"add_fields":{
+			"cluster":"{{cluster_name}}"
+		},
+		"source":"file",
+		"tail":"true",
+		"gen_event_md5":"true",
+		"start_position":"beginning"
+	},
+	"input":[
+		{
+			"type":"accumulo_gc",
+			"rowtype":"service",
+			"path":"{{accumulo_log_dir}}/gc_*.log"
+		},
+		{
+			"type":"accumulo_master",
+			"rowtype":"service",
+			"path":"{{accumulo_log_dir}}/master_*.log"
+		},
+		{
+			"type":"accumulo_monitor",
+			"rowtype":"service",
+			"path":"{{accumulo_log_dir}}/monitor_*.log"
+		},
+		{
+			"type":"accumulo_tracer",
+			"rowtype":"service",
+			"path":"{{accumulo_log_dir}}/tracer_*.log"
+		},
+		{
+			"type":"accumulo_tserver",
+			"rowtype":"service",
+			"path":"{{accumulo_log_dir}}/tserver_*.log"
+		},
+		{
+			"type":"atlas_app",
+			"rowtype":"service",
+			"path":"{{atlas_log_dir}}/application.log"
+		},
+		{
+			"type":"ambari_agent",
+			"rowtype":"service",
+			"path":"{{ambari_agent_log_dir}}/ambari-agent.log"
+		},
+		{
+			"type":"ambari_server",
+			"rowtype":"service",
+			"path":"{{ambari_server_log_dir}}/ambari-server.log"
+		},
+		{
+			"type":"ams_hbase_master",
+			"rowtype":"service",
+			"path":"{{metrics_collector_log_dir}}/hbase-ams-master-*.log"
+		},
+		{
+			"type":"ams_hbase_regionserver",
+			"rowtype":"service",
+			"path":"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log"
+		},
+		{
+			"type":"ams_collector",
+			"rowtype":"service",
+			"path":"{{metrics_collector_log_dir}}/ambari-metrics-collector.log"
+		},
+		{
+			"type":"falcon_app",
+			"rowtype":"service",
+			"path":"{{falcon_log_dir}}/falcon.application.log"
+		},
+		{
+			"type":"hbase_master",
+			"rowtype":"service",
+			"path":"{{hbase_log_dir}}/hbase-hbase-master-*.log"
+		},
+		{
+			"type":"hbase_regionserver",
+			"rowtype":"service",
+			"path":"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log"
+		},
+		{
+			"type":"hdfs_datanode",
+			"rowtype":"service",
+			"path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log"
+		},
+		{
+			"type":"hdfs_namenode",
+			"rowtype":"service",
+			"path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log"
+		},
+		{
+			"type":"hdfs_journalnode",
+			"rowtype":"service",
+			"path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log"
+		},
+		{
+			"type":"hdfs_secondarynamenode",
+			"rowtype":"service",
+			"path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log"
+		},
+		{
+			"type":"hdfs_zkfc",
+			"rowtype":"service",
+			"path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log"
+		},
+		{
+			"type":"hive_hiveserver2",
+			"rowtype":"service",
+			"path":"{{hive_log_dir}}/hiveserver2.log"
+		},
+		{
+			"type":"hive_metastore",
+			"rowtype":"service",
+			"path":"{{hive_log_dir}}/hivemetastore.log"
+		},
+		{
+			"type":"kafka_controller",
+			"rowtype":"service",
+			"path":"{{kafka_log_dir}}/controller.log"
+		},
+		{
+			"type":"kafka_request",
+			"rowtype":"service",
+			"path":"{{kafka_log_dir}}/kafka-request.log"
+		},
+		{
+			"type":"kafka_logcleaner",
+			"rowtype":"service",
+			"path":"{{kafka_log_dir}}/log-cleaner.log"
+		},
+		{
+			"type":"kafka_server",
+			"rowtype":"service",
+			"path":"{{kafka_log_dir}}/server.log"
+		},
+		{
+			"type":"kafka_statechange",
+			"rowtype":"service",
+			"path":"{{kafka_log_dir}}/state-change.log"
+		},
+		{
+			"type":"knox_gateway",
+			"rowtype":"service",
+			"path":"{{knox_log_dir}}/gateway.log"
+		},
+		{
+			"type":"knox_cli",
+			"rowtype":"service",
+			"path":"{{knox_log_dir}}/knoxcli.log"
+		},
+		{
+			"type":"knox_ldap",
+			"rowtype":"service",
+			"path":"{{knox_log_dir}}/ldap.log"
+		},
+		{
+			"type":"mapred_historyserver",
+			"rowtype":"service",
+			"path":"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log"
+		},
+		{
+			"type":"logsearch_app",
+			"rowtype":"service",
+			"path":"{{logsearch_log_dir}}/logsearch.log"
+		},
+		{
+			"type":"logsearch_feeder",
+			"rowtype":"service",
+			"path":"{{logfeeder_log_dir}}/logfeeder.log"
+		},
+		{
+			"type":"logsearch_perf",
+			"rowtype":"service",
+			"path":"{{logsearch_log_dir}}/logsearch-performance.log"
+		},
+		{
+			"type":"ranger_admin",
+			"rowtype":"service",
+			"path":"{{ranger_admin_log_dir}}/xa_portal.log"
+		},
+		{
+			"type":"ranger_dbpatch",
+			"is_enabled":"true",
+			"path":"{{ranger_admin_log_dir}}/ranger_db_patch.log"
+		},
+		{
+			"type":"ranger_kms",
+			"rowtype":"service",
+			"path":"{{ranger_kms_log_dir}}/kms.log"
+		},
+		{
+			"type":"ranger_usersync",
+			"rowtype":"service",
+			"path":"{{ranger_usersync_log_dir}}/usersync.log"
+		},
+		{
+			"type":"oozie_app",
+			"rowtype":"service",
+			"path":"{{oozie_log_dir}}/oozie.log"
+		},
+		{
+			"type":"yarn_nodemanager",
+			"rowtype":"service",
+			"path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log"
+		},
+		{
+			"type":"yarn_resourcemanager",
+			"rowtype":"service",
+			"path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log"
+		},
+		{
+			"type":"yarn_timelineserver",
+			"rowtype":"service",
+			"path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log"
+		},
+		{
+			"type":"yarn_historyserver",
+			"rowtype":"service",
+			"path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log"
+		},
+		{
+			"type":"yarn_jobsummary",
+			"rowtype":"service",
+			"path":"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log"
+		},
+		{
+			"type":"storm_drpc",
+			"rowtype":"service",
+			"path":"{{storm_log_dir}}/drpc.log"
+		},
+		{
+			"type":"storm_logviewer",
+			"rowtype":"service",
+			"path":"{{storm_log_dir}}/logviewer.log"
+		},
+		{
+			"type":"storm_nimbus",
+			"rowtype":"service",
+			"path":"{{storm_log_dir}}/nimbus.log"
+		},
+		{
+			"type":"storm_supervisor",
+			"rowtype":"service",
+			"path":"{{storm_log_dir}}/supervisor.log"
+		},
+		{
+			"type":"storm_ui",
+			"rowtype":"service",
+			"path":"{{storm_log_dir}}/ui.log"
+		},
+		{
+			"type":"storm_worker",
+			"rowtype":"service",
+			"path":"{{storm_log_dir}}/*worker*.log"
+		},
+		{
+			"type":"zookeeper",
+			"rowtype":"service",
+			"path":"{{zk_log_dir}}/zookeeper/zookeeper*.out"
+		},
+		{
+			"type":"hdfs_audit",
+			"rowtype":"audit",
+			"is_enabled":"true",
+			"add_fields":{
+				"logType":"HDFSAudit",
+				"enforcer":"hadoop-acl",
+				"repoType":"1",
+				"repo":"hdfs"
+			},
+			"path":"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log"
+		}
+		
+	],
+	"filter":[
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"accumulo_master"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"comment":"This one has one extra space after LEVEL",
+			"conditions":{
+				"fields":{
+					"type":[
+						"accumulo_gc",
+						"accumulo_monitor",
+						"accumulo_tracer",
+						"accumulo_tserver"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"atlas_app",
+						"falcon_app"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"ams_collector"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %p %c: %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"ams_hbase_master",
+						"ams_hbase_regionserver",
+						"hbase_master",
+						"hbase_regionserver"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"ambari_agent"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"",
+			"multiline_pattern":"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				},
+				"level":{
+					"map_fieldvalue":{
+						"pre_value":"WARNING",
+						"post_value":"WARN"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"ambari_server"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n",
+			"multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+			"message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"dd MMM yyyy HH:mm:ss"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"hdfs_datanode",
+						"hdfs_journalnode",
+						"hdfs_secondarynamenode",
+						"hdfs_namenode",
+						"hdfs_zkfc",
+						"knox_gateway",
+						"knox_cli",
+						"knox_ldap",
+						"mapred_historyserver",
+						"yarn_historyserver",
+						"yarn_jobsummary",
+						"yarn_nodemanager",
+						"yarn_resourcemanager",
+						"yarn_timelineserver"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"hive_hiveserver2",
+						"hive_metastore"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"kafka_controller",
+						"kafka_request",
+						"kafka_logcleaner"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"[%d] %p %m (%c)%n",
+			"multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+			"message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed",
+			"conditions":{
+				"fields":{
+					"type":[
+						"kafka_server",
+						"kafka_statechange"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"[%d] %p %m (%c)%n",
+			"multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+			"message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"oozie_app"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"logsearch_app",
+						"logsearch_feeder",
+						"logsearch_perf",
+						"ranger_admin",
+						"ranger_dbpatch"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"ranger_kms"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"ranger_usersync"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n",
+			"multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+			"message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"dd MMM yyyy HH:mm:ss"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"storm_drpc",
+						"storm_logviewer",
+						"storm_nimbus",
+						"storm_supervisor",
+						"storm_ui",
+						"storm_worker"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"zookeeper"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"hdfs_audit"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"evtTime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"keyvalue",
+			"sort_order":1,
+			"conditions":{
+				"fields":{
+					"type":[
+						"hdfs_audit"
+					]
+					
+				}
+				
+			},
+			"source_field":"log_message",
+			"value_split":"=",
+			"field_split":"\t",
+			"post_map_values":{
+				"src":{
+					"map_fieldname":{
+						"new_fieldname":"resource"
+					}
+					
+				},
+				"ip":{
+					"map_fieldname":{
+						"new_fieldname":"cliIP"
+					}
+					
+				},
+				"allowed":[
+					{
+						"map_fieldvalue":{
+							"pre_value":"true",
+							"post_value":"1"
+						}
+						
+					},
+					{
+						"map_fieldvalue":{
+							"pre_value":"false",
+							"post_value":"0"
+						}
+						
+					},
+					{
+						"map_fieldname":{
+							"new_fieldname":"result"
+						}
+						
+					}
+					
+				],
+				"cmd":{
+					"map_fieldname":{
+						"new_fieldname":"action"
+					}
+					
+				},
+				"proto":{
+					"map_fieldname":{
+						"new_fieldname":"cliType"
+					}
+					
+				},
+				"callerContext":{
+					"map_fieldname":{
+						"new_fieldname":"req_caller_id"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"sort_order":2,
+			"source_field":"ugi",
+			"remove_source_field":"false",
+			"conditions":{
+				"fields":{
+					"type":[
+						"hdfs_audit"
+					]
+					
+				}
+				
+			},
+			"message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
+			"post_map_values":{
+				"user":{
+					"map_fieldname":{
+						"new_fieldname":"reqUser"
+					}
+					
+				},
+				"x_user":{
+					"map_fieldname":{
+						"new_fieldname":"reqUser"
+					}
+					
+				},
+				"p_user":{
+					"map_fieldname":{
+						"new_fieldname":"reqUser"
+					}
+					
+				},
+				"k_user":{
+					"map_fieldname":{
+						"new_fieldname":"proxyUsers"
+					}
+					
+				},
+				"p_authType":{
+					"map_fieldname":{
+						"new_fieldname":"authType"
+					}
+					
+				},
+				"k_authType":{
+					"map_fieldname":{
+						"new_fieldname":"proxyAuthType"
+					}
+					
+				}
+				
+			}
+			
+		}
+		
+	],
+	"output":[
+		{
+			"is_enabled":"{{solr_service_logs_enable}}",
+			"comment":"Output to solr for service logs",
+			"destination":"solr",
+			"zk_hosts":"{{zookeeper_quorum}}{{solr_znode}}",
+			"collection":"{{solr_collection_service_logs}}",
+			"number_of_shards": "{{logsearch_numshards}}",
+			"splits_interval_mins": "{{service_logs_collection_splits_interval_mins}}",
+			"conditions":{
+				"fields":{
+					"rowtype":[
+						"service"
+					]
+					
+				}
+				
+			}
+			
+		},
+		{
+			"comment":"Output to solr for audit records",
+			"is_enabled":"{{solr_audit_logs_enable}}",
+			"destination":"solr",
+			"zk_hosts":"{{zookeeper_quorum}}{{solr_znode}}",
+			"collection":"{{solr_collection_audit_logs}}",
+			"number_of_shards": "{{logsearch_numshards}}",
+			"splits_interval_mins": "{{audit_logs_collection_splits_interval_mins}}",
+			"conditions":{
+				"fields":{
+					"rowtype":[
+						"audit"
+					]
+					
+				}
+				
+			}
+			
+		},
+		{
+			"is_enabled":"{{kafka_service_logs_enable}}",
+			"destination":"kafka",
+			"broker_list":"{{kafka_broker_list}}",
+			"topic":"{{kafka_topic_service_logs}}",
+			"kafka.security.protocol":"{{kafka_security_protocol}}",
+			"kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}",
+			"conditions":{
+				"fields":{
+					"rowtype":[
+						"service"
+					]
+					
+				}
+				
+			}
+			
+		},
+		{
+			"is_enabled":"{{kafka_topic_service_logs}}",
+			"destination":"kafka",
+			"broker_list":"{{kafka_broker_list}}",
+			"topic":"{{kafka_topic_audit_logs}}",
+			"kafka.security.protocol":"{{kafka_security_protocol}}",
+			"kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}",
+			"conditions":{
+				"fields":{
+					"rowtype":[
+						"audit"
+					]
+					
+				}
+				
+			}
+			
+		}
+		
+	]
+	
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/filters.config.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/filters.config.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/filters.config.json
new file mode 100644
index 0000000..9493c6c
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/filters.config.json
@@ -0,0 +1,626 @@
+{
+	"filter":[
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"accumulo_master"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"comment":"This one has one extra space after LEVEL",
+			"conditions":{
+				"fields":{
+					"type":[
+						"accumulo_gc",
+						"accumulo_monitor",
+						"accumulo_tracer",
+						"accumulo_tserver"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"atlas_app",
+						"falcon_app"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"ams_collector"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %p %c: %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"ams_hbase_master",
+						"ams_hbase_regionserver",
+						"hbase_master",
+						"hbase_regionserver"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"ambari_agent"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"",
+			"multiline_pattern":"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				},
+				"level":{
+					"map_fieldvalue":{
+						"pre_value":"WARNING",
+						"post_value":"WARN"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"ambari_server"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n",
+			"multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+			"message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"dd MMM yyyy HH:mm:ss"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"hdfs_datanode",
+						"hdfs_journalnode",
+						"hdfs_secondarynamenode",
+						"hdfs_namenode",
+						"hdfs_zkfc",
+						"knox_gateway",
+						"knox_cli",
+						"knox_ldap",
+						"mapred_historyserver",
+						"yarn_historyserver",
+						"yarn_jobsummary",
+						"yarn_nodemanager",
+						"yarn_resourcemanager",
+						"yarn_timelineserver"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"hive_hiveserver2",
+						"hive_metastore"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"kafka_controller",
+						"kafka_request",
+						"kafka_logcleaner"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"[%d] %p %m (%c)%n",
+			"multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+			"message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed",
+			"conditions":{
+				"fields":{
+					"type":[
+						"kafka_server",
+						"kafka_statechange"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"[%d] %p %m (%c)%n",
+			"multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+			"message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"oozie_app"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"logsearch_app",
+						"logsearch_feeder",
+					    	"logsearch_perf",
+						"ranger_admin",
+						"ranger_dbpatch"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"ranger_kms"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"ranger_usersync"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n",
+			"multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+			"message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"dd MMM yyyy HH:mm:ss"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"storm_drpc",
+						"storm_logviewer",
+						"storm_nimbus",
+						"storm_supervisor",
+						"storm_ui",
+						"storm_worker"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"zookeeper"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"logtime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"conditions":{
+				"fields":{
+					"type":[
+						"hdfs_audit"
+					]
+					
+				}
+				
+			},
+			"log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+			"multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})",
+			"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+			"post_map_values":{
+				"evtTime":{
+					"map_date":{
+						"date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"keyvalue",
+			"sort_order":1,
+			"conditions":{
+				"fields":{
+					"type":[
+						"hdfs_audit"
+					]
+					
+				}
+				
+			},
+			"source_field":"log_message",
+			"value_split":"=",
+			"field_split":"\t",
+			"post_map_values":{
+				"src":{
+					"map_fieldname":{
+						"new_fieldname":"resource"
+					}
+					
+				},
+				"ip":{
+					"map_fieldname":{
+						"new_fieldname":"cliIP"
+					}
+					
+				},
+				"allowed":[
+					{
+						"map_fieldvalue":{
+							"pre_value":"true",
+							"post_value":"1"
+						}
+						
+					},
+					{
+						"map_fieldvalue":{
+							"pre_value":"false",
+							"post_value":"0"
+						}
+						
+					},
+					{
+						"map_fieldname":{
+							"new_fieldname":"result"
+						}
+						
+					}
+					
+				],
+				"cmd":{
+					"map_fieldname":{
+						"new_fieldname":"action"
+					}
+					
+				},
+				"proto":{
+					"map_fieldname":{
+						"new_fieldname":"cliType"
+					}
+					
+				},
+				"callerContext":{
+					"map_fieldname":{
+						"new_fieldname":"req_caller_id"
+					}
+					
+				}
+				
+			}
+			
+		},
+		{
+			"filter":"grok",
+			"sort_order":2,
+			"source_field":"ugi",
+			"remove_source_field":"false",
+			"conditions":{
+				"fields":{
+					"type":[
+						"hdfs_audit"
+					]
+					
+				}
+				
+			},
+			"message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
+			"post_map_values":{
+				"user":{
+					"map_fieldname":{
+						"new_fieldname":"reqUser"
+					}
+					
+				},
+				"x_user":{
+					"map_fieldname":{
+						"new_fieldname":"reqUser"
+					}
+					
+				},
+				"p_user":{
+					"map_fieldname":{
+						"new_fieldname":"reqUser"
+					}
+					
+				},
+				"k_user":{
+					"map_fieldname":{
+						"new_fieldname":"proxyUsers"
+					}
+					
+				},
+				"p_authType":{
+					"map_fieldname":{
+						"new_fieldname":"authType"
+					}
+					
+				},
+				"k_authType":{
+					"map_fieldname":{
+						"new_fieldname":"proxyAuthType"
+					}
+					
+				}
+				
+			}
+			
+		}
+		
+	]
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/global.config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/global.config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/global.config.json.j2
new file mode 100644
index 0000000..cd51118
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/global.config.json.j2
@@ -0,0 +1,28 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements.  See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership.  The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License.  You may obtain a copy of the License at
+ #
+ #   http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+	"global":{
+		"add_fields":{
+			"cluster":"{{cluster_name}}"
+		},
+		"source":"file",
+		"tail":"true",
+		"gen_event_md5":"true",
+		"start_position":"beginning"
+	}	
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/grok-patterns
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/grok-patterns b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/grok-patterns
new file mode 100644
index 0000000..d25a78b
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/grok-patterns
@@ -0,0 +1,145 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#Updated JAVACLASS to be same as JAVAFILE. Because if class doesn't have package, then it doesn't work.
+JAVACLASS (?:[A-Za-z$0-9_. -]+)
+#JAVACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]*
+#JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+
+
+#Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source'
+JAVAFILE (?:[A-Za-z0-9_. -]+)
+#Allow special <init> or <clinit> method
+JAVAMETHOD (?:(<init>)|(<clinit>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
+#Line number is optional in special cases 'Native method' or 'Unknown source'
+JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\)
+# Java Logs
+JAVATHREAD (?:[A-Z]{2}-Processor[\d]+)
+
+JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\)
+JAVALOGMESSAGE (.*)
+# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM
+CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)
+# yyyy-MM-dd HH:mm:ss,SSS ZZZ eg: 2014-01-09 17:32:25,527 -0800
+TOMCAT_DATESTAMP 20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE}
+CATALINALOG %{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage}
+# 2014-01-09 20:03:28,269 -0800 | ERROR | com.example.service.ExampleService - something compeletely unexpected happened...
+TOMCATLOG %{TOMCAT_DATESTAMP:timestamp} \| %{LOGLEVEL:level} \| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage}
+
+USERNAME [a-zA-Z0-9._-]+
+USER %{USERNAME}
+EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+
+EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME}
+HTTPDUSER %{EMAILADDRESS}|%{USER}
+INT (?:[+-]?(?:[0-9]+))
+BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
+NUMBER (?:%{BASE10NUM})
+BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
+BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
+
+POSINT \b(?:[1-9][0-9]*)\b
+NONNEGINT \b(?:[0-9]+)\b
+WORD \b\w+\b
+NOTSPACE \S+
+SPACE \s*
+DATA .*?
+GREEDYDATA .*
+QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
+UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
+
+# Networking
+MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
+CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
+WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
+COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
+IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5
 ]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
+IPV4 (?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])
+IP (?:%{IPV6}|%{IPV4})
+HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
+IPORHOST (?:%{IP}|%{HOSTNAME})
+HOSTPORT %{IPORHOST}:%{POSINT}
+
+# paths
+PATH (?:%{UNIXPATH}|%{WINPATH})
+UNIXPATH (/([\w_%!$@:.,~-]+|\\.)*)+
+TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
+WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
+URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
+URIHOST %{IPORHOST}(?::%{POSINT:port})?
+# uripath comes loosely from RFC1738, but mostly from what Firefox
+# doesn't turn into %XX
+URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
+#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
+URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*
+URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
+URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
+
+# Months: January, Feb, 3, 03, 12, December
+MONTH \b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|รค)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\b
+MONTHNUM (?:0?[1-9]|1[0-2])
+MONTHNUM2 (?:0[1-9]|1[0-2])
+MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
+
+# Days: Monday, Tue, Thu, etc...
+DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
+
+# Years?
+YEAR (?>\d\d){1,2}
+HOUR (?:2[0123]|[01]?[0-9])
+MINUTE (?:[0-5][0-9])
+# '60' is a leap second in most time standards and thus is valid.
+SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
+TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
+# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
+DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
+DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
+ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
+ISO8601_SECOND (?:%{SECOND}|60)
+TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
+DATE %{DATE_US}|%{DATE_EU}
+DATESTAMP %{DATE}[- ]%{TIME}
+TZ (?:[PMCE][SD]T|UTC)
+DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
+DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
+DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
+DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
+HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
+
+# Syslog Dates: Month Day HH:MM:SS
+SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
+PROG [\x21-\x5a\x5c\x5e-\x7e]+
+SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
+SYSLOGHOST %{IPORHOST}
+SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
+HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
+
+# Shortcuts
+QS %{QUOTEDSTRING}
+
+# Log formats
+SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
+COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
+COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
+HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}
+HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
+HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
+
+
+# Log Levels
+LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
+
+
+# Custom
+USER_SYNC_DATE %{MONTHDAY} %{MONTH} %{YEAR} %{TIME}

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/input.config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/input.config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/input.config.json.j2
new file mode 100644
index 0000000..bc48503
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/input.config.json.j2
@@ -0,0 +1,284 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements.  See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership.  The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License.  You may obtain a copy of the License at
+ #
+ #   http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+	"input":[
+		{
+			"type":"accumulo_gc",
+			"rowtype":"service",
+			"path":"{{accumulo_log_dir}}/gc_*.log"
+		},
+		{
+			"type":"accumulo_master",
+			"rowtype":"service",
+			"path":"{{accumulo_log_dir}}/master_*.log"
+		},
+		{
+			"type":"accumulo_monitor",
+			"rowtype":"service",
+			"path":"{{accumulo_log_dir}}/monitor_*.log"
+		},
+		{
+			"type":"accumulo_tracer",
+			"rowtype":"service",
+			"path":"{{accumulo_log_dir}}/tracer_*.log"
+		},
+		{
+			"type":"accumulo_tserver",
+			"rowtype":"service",
+			"path":"{{accumulo_log_dir}}/tserver_*.log"
+		},
+		{
+			"type":"atlas_app",
+			"rowtype":"service",
+			"path":"{{atlas_log_dir}}/application.log"
+		},
+		{
+			"type":"ambari_agent",
+			"rowtype":"service",
+			"path":"{{ambari_agent_log_dir}}/ambari-agent.log"
+		},
+		{
+			"type":"ambari_server",
+			"rowtype":"service",
+			"path":"{{ambari_server_log_dir}}/ambari-server.log"
+		},
+		{
+			"type":"ams_hbase_master",
+			"rowtype":"service",
+			"path":"{{metrics_collector_log_dir}}/hbase-ams-master-*.log"
+		},
+		{
+			"type":"ams_hbase_regionserver",
+			"rowtype":"service",
+			"path":"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log"
+		},
+		{
+			"type":"ams_collector",
+			"rowtype":"service",
+			"path":"{{metrics_collector_log_dir}}/ambari-metrics-collector.log"
+		},
+		{
+			"type":"falcon_app",
+			"rowtype":"service",
+			"path":"{{falcon_log_dir}}/falcon.application.log"
+		},
+		{
+			"type":"hbase_master",
+			"rowtype":"service",
+			"path":"{{hbase_log_dir}}/hbase-hbase-master-*.log"
+		},
+		{
+			"type":"hbase_regionserver",
+			"rowtype":"service",
+			"path":"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log"
+		},
+		{
+			"type":"hdfs_datanode",
+			"rowtype":"service",
+			"path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log"
+		},
+		{
+			"type":"hdfs_namenode",
+			"rowtype":"service",
+			"path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log"
+		},
+		{
+			"type":"hdfs_journalnode",
+			"rowtype":"service",
+			"path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log"
+		},
+		{
+			"type":"hdfs_secondarynamenode",
+			"rowtype":"service",
+			"path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log"
+		},
+		{
+			"type":"hdfs_zkfc",
+			"rowtype":"service",
+			"path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log"
+		},
+		{
+			"type":"hive_hiveserver2",
+			"rowtype":"service",
+			"path":"{{hive_log_dir}}/hiveserver2.log"
+		},
+		{
+			"type":"hive_metastore",
+			"rowtype":"service",
+			"path":"{{hive_log_dir}}/hivemetastore.log"
+		},
+		{
+			"type":"kafka_controller",
+			"rowtype":"service",
+			"path":"{{kafka_log_dir}}/controller.log"
+		},
+		{
+			"type":"kafka_request",
+			"rowtype":"service",
+			"path":"{{kafka_log_dir}}/kafka-request.log"
+		},
+		{
+			"type":"kafka_logcleaner",
+			"rowtype":"service",
+			"path":"{{kafka_log_dir}}/log-cleaner.log"
+		},
+		{
+			"type":"kafka_server",
+			"rowtype":"service",
+			"path":"{{kafka_log_dir}}/server.log"
+		},
+		{
+			"type":"kafka_statechange",
+			"rowtype":"service",
+			"path":"{{kafka_log_dir}}/state-change.log"
+		},
+		{
+			"type":"knox_gateway",
+			"rowtype":"service",
+			"path":"{{knox_log_dir}}/gateway.log"
+		},
+		{
+			"type":"knox_cli",
+			"rowtype":"service",
+			"path":"{{knox_log_dir}}/knoxcli.log"
+		},
+		{
+			"type":"knox_ldap",
+			"rowtype":"service",
+			"path":"{{knox_log_dir}}/ldap.log"
+		},
+		{
+			"type":"mapred_historyserver",
+			"rowtype":"service",
+			"path":"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log"
+		},
+		{
+			"type":"logsearch_app",
+			"rowtype":"service",
+			"path":"{{logsearch_log_dir}}/logsearch.log"
+		},
+		{
+			"type":"logsearch_feeder",
+			"rowtype":"service",
+			"path":"{{logfeeder_log_dir}}/logfeeder.log"
+		},
+		{
+			"type":"logsearch_perf",
+			"rowtype":"service",
+			"path":"{{logsearch_log_dir}}/logsearch-performance.log"
+		},
+		{
+			"type":"ranger_admin",
+			"rowtype":"service",
+			"path":"{{ranger_admin_log_dir}}/xa_portal.log"
+		},
+		{
+			"type":"ranger_dbpatch",
+			"is_enabled":"true",
+			"path":"{{ranger_admin_log_dir}}/ranger_db_patch.log"
+		},
+		{
+			"type":"ranger_kms",
+			"rowtype":"service",
+			"path":"{{ranger_kms_log_dir}}/kms.log"
+		},
+		{
+			"type":"ranger_usersync",
+			"rowtype":"service",
+			"path":"{{ranger_usersync_log_dir}}/usersync.log"
+		},
+		{
+			"type":"oozie_app",
+			"rowtype":"service",
+			"path":"{{oozie_log_dir}}/oozie.log"
+		},
+		{
+			"type":"yarn_nodemanager",
+			"rowtype":"service",
+			"path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log"
+		},
+		{
+			"type":"yarn_resourcemanager",
+			"rowtype":"service",
+			"path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log"
+		},
+		{
+			"type":"yarn_timelineserver",
+			"rowtype":"service",
+			"path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log"
+		},
+		{
+			"type":"yarn_historyserver",
+			"rowtype":"service",
+			"path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log"
+		},
+		{
+			"type":"yarn_jobsummary",
+			"rowtype":"service",
+			"path":"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log"
+		},
+		{
+			"type":"storm_drpc",
+			"rowtype":"service",
+			"path":"{{storm_log_dir}}/drpc.log"
+		},
+		{
+			"type":"storm_logviewer",
+			"rowtype":"service",
+			"path":"{{storm_log_dir}}/logviewer.log"
+		},
+		{
+			"type":"storm_nimbus",
+			"rowtype":"service",
+			"path":"{{storm_log_dir}}/nimbus.log"
+		},
+		{
+			"type":"storm_supervisor",
+			"rowtype":"service",
+			"path":"{{storm_log_dir}}/supervisor.log"
+		},
+		{
+			"type":"storm_ui",
+			"rowtype":"service",
+			"path":"{{storm_log_dir}}/ui.log"
+		},
+		{
+			"type":"storm_worker",
+			"rowtype":"service",
+			"path":"{{storm_log_dir}}/*worker*.log"
+		},
+		{
+			"type":"zookeeper",
+			"rowtype":"service",
+			"path":"{{zk_log_dir}}/zookeeper/zookeeper*.out"
+		},
+		{
+			"type":"hdfs_audit",
+			"rowtype":"audit",
+			"is_enabled":"true",
+			"add_fields":{
+				"logType":"HDFSAudit",
+				"enforcer":"hadoop-acl",
+				"repoType":"1",
+				"repo":"hdfs"
+			},
+			"path":"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log"
+		}
+		
+	]	
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
new file mode 100644
index 0000000..0717477
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+  <appender name="console" class="org.apache.log4j.ConsoleAppender">
+    <param name="Target" value="System.out" />
+    <layout class="org.apache.log4j.PatternLayout">
+      <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" />
+      <!-- <param name="ConversionPattern" value="%d [%t] %-5p %c %x - %m%n"/> -->
+    </layout>
+  </appender>
+
+  <appender name="daily_rolling_file" class="org.apache.log4j.DailyRollingFileAppender"> 
+    <param name="file" value="logs/logsearch-logfeeder.log" /> 
+    <param name="datePattern"  value="'.'yyyy-MM-dd" /> 
+    <param name="append" value="true" /> 
+    <layout class="org.apache.log4j.PatternLayout"> 
+      <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n"/> 
+    </layout> 
+  </appender> 
+
+  <!-- Logs to suppress BEGIN -->
+  <category name="org.apache.solr.common.cloud.ZkStateReader" additivity="false">
+    <priority value="error" />
+    <appender-ref ref="rolling_file" />
+  </category>
+
+  <category name="apache.solr.client.solrj.impl.CloudSolrClient" additivity="false">
+    <priority value="fatal" />
+    <appender-ref ref="rolling_file" />
+  </category>
+  <!-- Logs to suppress END -->
+
+  <category name="org.apache.ambari.logfeeder" additivity="false">
+    <priority value="info" />
+    <appender-ref ref="console" /> 
+    <!-- <appender-ref ref="daily_rolling_file" /> -->
+  </category>
+
+  <root>
+    <priority value="warn" />
+    <!-- <appender-ref ref="console" /> -->
+    <!-- <appender-ref ref="daily_rolling_file" /> -->
+  </root>
+ 
+</log4j:configuration>  

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml.j2
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml.j2
new file mode 100644
index 0000000..4338ee3
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml.j2
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+  <appender name="console" class="org.apache.log4j.ConsoleAppender">
+    <param name="Target" value="System.out" />
+    <layout class="org.apache.log4j.PatternLayout">
+      <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" />
+      <!-- <param name="ConversionPattern" value="%d [%t] %-5p %c %x - %m%n"/> -->
+    </layout>
+  </appender>
+
+  <appender name="rolling_file" class="org.apache.log4j.RollingFileAppender">
+    <param name="file" value="{{logfeeder_log_dir}}/logfeeder.log" />
+    <param name="append" value="true" />
+    <layout class="org.apache.log4j.PatternLayout">
+      <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n"/>
+    </layout>
+  </appender>
+
+  <!-- Logs to suppress BEGIN -->
+  <category name="org.apache.solr.common.cloud.ZkStateReader" additivity="false">
+    <priority value="error" />
+    <appender-ref ref="rolling_file" />
+  </category>
+
+  <category name="apache.solr.client.solrj.impl.CloudSolrClient" additivity="false">
+    <priority value="fatal" />
+    <appender-ref ref="rolling_file" />
+  </category>
+
+  <!-- Logs to suppress END -->
+
+  <category name="org.apache.ambari.logfeeder" additivity="false">
+    <priority value="{{logfeeder_log_level}}" />
+    <appender-ref ref="rolling_file" />
+  </category>
+
+  <root>
+    <level value="warn" />
+    <!-- <appender-ref ref="console" /> -->
+    <appender-ref ref="rolling_file" />
+  </root>
+</log4j:configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
new file mode 100644
index 0000000..22f3b78
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+logfeeder.checkpoint.folder=
+metrics.collector.hosts=
+
+#filter config
+logfeeder.solr.url=
+logfeeder.solr.zkhosts=
+logfeeder.solr.core.history=history
+logfeeder.log.filter.enable=true
+#Internal to fetch filter config from solr in sec
+logfeeder.solr.config.internal=5
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/output.config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/output.config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/output.config.json.j2
new file mode 100644
index 0000000..d0aea47
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/output.config.json.j2
@@ -0,0 +1,97 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements.  See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership.  The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License.  You may obtain a copy of the License at
+ #
+ #   http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+	"output":[
+		{
+			"is_enabled":"{{solr_service_logs_enable}}",
+			"comment":"Output to solr for service logs",
+			"destination":"solr",
+			"zk_hosts":"{{zookeeper_quorum}}{{solr_znode}}",
+			"collection":"{{solr_collection_service_logs}}",
+			"number_of_shards": "{{logsearch_numshards}}",
+			"splits_interval_mins": "{{service_logs_collection_splits_interval_mins}}",
+			"conditions":{
+				"fields":{
+					"rowtype":[
+						"service"
+					]
+					
+				}
+				
+			}
+			
+		},
+		{
+			"comment":"Output to solr for audit records",
+			"is_enabled":"{{solr_audit_logs_enable}}",
+			"destination":"solr",
+			"zk_hosts":"{{zookeeper_quorum}}{{solr_znode}}",
+			"collection":"{{solr_collection_audit_logs}}",
+			"number_of_shards": "{{logsearch_numshards}}",
+			"splits_interval_mins": "{{audit_logs_collection_splits_interval_mins}}",
+			"conditions":{
+				"fields":{
+					"rowtype":[
+						"audit"
+					]
+					
+				}
+				
+			}
+			
+		},
+		{
+			"is_enabled":"{{kafka_service_logs_enable}}",
+			"destination":"kafka",
+			"broker_list":"{{kafka_broker_list}}",
+			"topic":"{{kafka_topic_service_logs}}",
+			"kafka.security.protocol":"{{kafka_security_protocol}}",
+			"kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}",
+			"conditions":{
+				"fields":{
+					"rowtype":[
+						"service"
+					]
+					
+				}
+				
+			}
+			
+		},
+		{
+			"is_enabled":"{{kafka_topic_service_logs}}",
+			"destination":"kafka",
+			"broker_list":"{{kafka_broker_list}}",
+			"topic":"{{kafka_topic_audit_logs}}",
+			"kafka.security.protocol":"{{kafka_security_protocol}}",
+			"kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}",
+			"conditions":{
+				"fields":{
+					"rowtype":[
+						"audit"
+					]
+					
+				}
+				
+			}
+			
+		}
+		
+	]
+	
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f7294694/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh
new file mode 100644
index 0000000..1e7185a
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+curr_dir=`pwd`
+cd `dirname $0`; script_dir=`pwd`; cd $curr_dir
+
+foreground=0
+if [ "$1" = "-foreground" ]; then
+    foreground=1
+    shift
+fi
+
+if [ ! -z "$LOGFEEDER_INCLUDE" ]; then
+   source $LOGFEEDER_INCLUDE
+fi
+
+JAVA=java
+if [ -x $JAVA_HOME/bin/java ]; then
+    JAVA=$JAVA_HOME/bin/java
+fi
+
+if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then
+    LOGFEEDER_JAVA_MEM="-Xmx512m"
+fi
+
+if [ "$LOGFILE" = "" ]; then
+    LOGFILE="/var/log/logfeeder/logfeeder.out"
+fi
+
+if [ "$PID_FILE" = "" ]; then
+    LOGFEEDER_PID_DIR=$HOME
+    PID_FILE=$LOGFEEDER_PID_DIR/logsearch-logfeeder-$USER.pid
+fi
+
+if [ "$LOGFEEDER_CONF_DIR" = "" ]; then
+    LOGFEEDER_CONF_DIR="/etc/logfeeder/conf"
+fi
+
+LOGFEEDER_GC_LOGFILE=`dirname $LOGFILE`/logfeeder_gc.log
+LOGFEEDER_GC_OPTS="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$LOGFEEDER_GC_LOGFILE"
+
+#LOGFEEDER_JAVA_OPTS=
+#JMX="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=2098"
+
+if [ $foreground -eq 0 ]; then
+    if [ -f ${PID_FILE} ]; then
+	PID=`cat ${PID_FILE}`
+	if kill -0 $PID 2>/dev/null; then
+	    echo "logfeeder already running (${PID}) killing..."
+	    kill $PID 2>/dev/null
+	    sleep 5
+	    if kill -0 $PID 2>/dev/null; then
+		echo "logfeeder still running. Will kill process forcefully in another 10 seconds..."
+		sleep 10
+		kill -9 $PID 2>/dev/null
+		sleep 2
+	    fi
+	fi
+
+	if kill -0 $PID 2>/dev/null; then
+	    echo "ERROR: Even after all efforts to stop logfeeder, it is still running. pid=$PID. Please manually kill the service and try again."
+	    exit 1
+	fi
+    fi
+
+    echo "Starting logfeeder. Output file=$LOGFILE pid_file=$PID_FILE"
+    #LOGFEEDER_CLI_CLASSPATH=
+    #set -x
+    nohup $JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes:$script_dir/LogProcessor.jar" $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder $* > $LOGFILE 2>&1 &
+    echo $! > $PID_FILE
+else
+    $JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes:$script_dir/LogProcessor.jar" $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder $*
+fi
+


Mime
View raw message